diff --git a/.gitignore b/.gitignore index acf349fb..b9c05822 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ nosetests.xml .mr.developer.cfg .project .pydevproject +venv \ No newline at end of file diff --git a/CHANGES.txt b/CHANGES.txt index adf71b69..3f1ecd07 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,3 +6,11 @@ 0.0.5 - Fixed seamless configuration api for Article and Source objects. Enabled multi language support in 10+ languages including non-western languages like Arabic, Korean, Chinese. Fixed bug where we made a wrong assumption of calling .text from the requests module. + +0.0.6 - Fixed a bunch of small bugs in the source.py file (still need to update readme). Batch + downloading articles was not setting the article is_downloaded boolean. I was also using + the del keyword very irresponsibly... Made many modifications where the source object + had to filter out urls. Mostly replaced with list comprehensions. + Added a pull request from Alex K. where he added an option for just the article html + extraction. Feel free to toggle this option in the configs. I have yet to add this to the + docs once again. diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index b0a38955..bd3d0106 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -6,6 +6,7 @@ Thanks to the following contributors: ------------------------------------- - Michael Hood - https://github.com/michaelhood - Juliano Fischer - https://github.com/julianofischer +- Alex Kessinger - https://github.com/voidfiles Newspaper relied on some code of a few other open source projects: ------------------------------------------------------------------ diff --git a/HISTORY.md b/HISTORY.md index adf71b69..3f1ecd07 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -6,3 +6,11 @@ 0.0.5 - Fixed seamless configuration api for Article and Source objects. Enabled multi language support in 10+ languages including non-western languages like Arabic, Korean, Chinese. Fixed bug where we made a wrong assumption of calling .text from the requests module. + +0.0.6 - Fixed a bunch of small bugs in the source.py file (still need to update readme). Batch + downloading articles was not setting the article is_downloaded boolean. I was also using + the del keyword very irresponsibly... Made many modifications where the source object + had to filter out urls. Mostly replaced with list comprehensions. + Added a pull request from Alex K. where he added an option for just the article html + extraction. Feel free to toggle this option in the configs. I have yet to add this to the + docs once again. diff --git a/README.rst b/README.rst index be1fdf12..0b2bdc2b 100644 --- a/README.rst +++ b/README.rst @@ -29,6 +29,16 @@ Newspaper: Article scraping & curation sv Swedish zh Chinese +Praise: +------- + + Newspaper is an amazing python library for extracting & curating articles. + -- `tweeted by`_ Kenneth Reitz, Author of `requests`_ + + Newspaper delivers Instapaper style article extraction. -- `The Changelog`_ + +.. _`tweeted by`: https://twitter.com/kennethreitz/status/419520678862548992 +.. _`The Changelog`: http://thechangelog.com/ A Glance: --------- @@ -105,9 +115,12 @@ If no language is specified, Newspaper will attempt to auto detect a language. >>> a.parse() >>> print a.text[:150] - 香港行政长官梁振英在各方压力下就其大宅的违章建筑（僭建）问题到立法会接受质询，并向香港民众道歉。 - 梁振英在星期二（12月10日）的答问大会开始之际在其演说中道歉，但强调他在违章建筑问题上没有隐瞒的意图和动机。 - 一些亲北京阵营议员欢迎梁振英道歉，且认为应能获得香港民众接受，但这些议员也质问梁振英有 + 香港行政长官梁振英在各方压力下就其大宅的违章建 + 筑（僭建）问题到立法会接受质询，并向香港民众道歉。 + 梁振英在星期二（12月10日）的答问大会开始之际 + 在其演说中道歉，但强调他在违章建筑问题上没有隐瞒的 + 意图和动机。 一些亲北京阵营议员欢迎梁振英道歉， + 且认为应能获得香港民众接受，但这些议员也质问梁振英有 >>> print a.title 港特首梁振英就住宅违建事件道歉 @@ -132,12 +145,16 @@ If you are certain that an *entire* news source is in one language, **go ahead a >>> article.parse() >>> print article.text - 新浪武汉汽车综合 随着汽车市场的日趋成熟，传统的“集全家之力抱得爱车归”的全额购车模式已然过时，另一种轻松的新兴 - 车模式――金融购车正逐步成为时下消费者购买爱车最为时尚的消费理念，他们认为，这种新颖的购车模式既能在短期内 + 新浪武汉汽车综合 随着汽车市场的日趋成熟， + 传统的“集全家之力抱得爱车归”的全额购车模式已然过时， + 另一种轻松的新兴 车模式――金融购车正逐步成为时下消费者购 + 买爱车最为时尚的消费理念，他们认为，这种新颖的购车 + 模式既能在短期内 ... >>> print article.title - 两年双免0手续0利率 科鲁兹掀背金融轻松购_武汉车市_武汉汽车网_新浪汽车_新浪网 + 两年双免0手续0利率 科鲁兹掀背金融轻松购_武汉车市_武汉汽 + 车网_新浪汽车_新浪网 Documentation @@ -208,5 +225,5 @@ Todo List .. _`Quickstart guide`: https://newspaper.readthedocs.org/en/latest/ .. _`The Documentation`: http://newspaper.readthedocs.org .. _`lxml`: http://lxml.de/ -.. _`requests`: http://docs.python-requests.org/en/latest/ +.. _`requests`: https://github.com/kennethreitz/requests diff --git a/docs/index.rst b/docs/index.rst index 74dbef02..8c5a9941 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ Newspaper: Article scraping & curation ====================================== -Release v0.0.4. :ref:`(Installation) `. +Release v0.0.5. :ref:`(Installation) `. *Newspaper* is a Python 2 library for extracting & curating articles from the web. It is inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed. @@ -103,9 +103,12 @@ If no language is specified, Newspaper will attempt to auto detect a language. >>> a.parse() >>> print a.text[:150] - 香港行政长官梁振英在各方压力下就其大宅的违章建筑（僭建）问题到立法会接受质询，并向香港民众道歉。 - 梁振英在星期二（12月10日）的答问大会开始之际在其演说中道歉，但强调他在违章建筑问题上没有隐瞒的意图和动机。 - 一些亲北京阵营议员欢迎梁振英道歉，且认为应能获得香港民众接受，但这些议员也质问梁振英有 + 香港行政长官梁振英在各方压力下就其大宅的违章建 + 筑（僭建）问题到立法会接受质询，并向香港民众道歉。 + 梁振英在星期二（12月10日）的答问大会开始之际在其 + 演说中道歉，但强调他在违章建筑问题上没有隐瞒的意 + 图和动机。 一些亲北京阵营议员欢迎梁振英道歉， + 且认为应能获得香港民众接受，但这些议员也质问梁振英有 >>> print a.title 港特首梁振英就住宅违建事件道歉 @@ -130,12 +133,16 @@ If you are certain that an *entire* news source is in one language, **go ahead a >>> article.parse() >>> print article.text - 新浪武汉汽车综合 随着汽车市场的日趋成熟，传统的“集全家之力抱得爱车归”的全额购车模式已然过时，另一种轻松的新兴 - 车模式――金融购车正逐步成为时下消费者购买爱车最为时尚的消费理念，他们认为，这种新颖的购车模式既能在短期内 + 新浪武汉汽车综合 随着汽车市场的日趋成熟，传统的“集 + 全家之力抱得爱车归”的全额购车模式已然过时，另一种轻 + 松的新兴 车模式――金融购车正逐步成为时下消费者购买 + 爱车最为时尚的消费理 念，他们认为，这种新颖的购车模 + 式既能在短期内 ... >>> print article.title - 两年双免0手续0利率 科鲁兹掀背金融轻松购_武汉车市_武汉汽车网_新浪汽车_新浪网 + 两年双免0手续0利率 科鲁兹掀背金融轻松购_武汉车市_武汉 + 汽车网_新浪汽车_新浪网 Features diff --git a/newspaper/article.py b/newspaper/article.py index fcc75fa2..abe9e292 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -85,6 +85,9 @@ def __init__(self, url, title=u'', source_url=u'', config=None, **kwargs): # the article's unchanged and raw html self.html = u'' + # The html of the main article node + self.article_html = u'' + # flags warning users in-case they forget to download() or parse() self.is_parsed = False self.is_downloaded = False @@ -130,8 +133,7 @@ def download(self): downloading batch articles. """ html = network.get_html(self.url, self.config) - self.html = encodeValue(html) - self.is_downloaded = True + self.set_html(html) def parse(self): """ @@ -190,7 +192,8 @@ def parse(self): self.set_movies(video_extractor.get_videos()) self.top_node = self.extractor.post_cleanup(self.top_node) - text = output_formatter.get_formatted_text(self) + text, article_html = output_formatter.get_formatted(self) + self.set_article_html(article_html) self.set_text(text) if self.raw_doc is not None: @@ -350,7 +353,13 @@ def set_reddit_top_img(self): def set_title(self, title): """ + The prechecked_title boolean is important for cases where our + educated guess of an article's title works and is actually + better than the actual title being extracted. """ + prechecked_title = (self.title and not title) + if prechecked_title: + return title = title[:self.config.MAX_TITLE] title = encodeValue(title) if title: @@ -364,6 +373,22 @@ def set_text(self, text): if text: self.text = text + def set_html(self, html): + """ + This method is quite important because many other objects + besides this one will be modifying and setting the html. + """ + self.is_downloaded = True + if html: + self.html = encodeValue(html) + + def set_article_html(self, article_html): + """ + Sets the html of just our article body, the "top node". + """ + if article_html: + self.article_html = encodeValue(article_html) + def set_top_img(self, src_url): """ We want to provide 2 api's for images. One at @@ -447,3 +472,4 @@ def set_movies(self, movie_objects): """ movie_urls = [o.src for o in movie_objects if o and o.src] self.movies = movie_urls + diff --git a/newspaper/configuration.py b/newspaper/configuration.py index 1fb1db04..36a9d065 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -45,6 +45,9 @@ def __init__(self): # don't toggle this variable self.use_meta_language = True + # you may keep the html of just the main article body + self.keep_article_html = False + # english is our fallback self._language = 'en' diff --git a/newspaper/outputformatters.py b/newspaper/outputformatters.py index 12e58293..366526b0 100644 --- a/newspaper/outputformatters.py +++ b/newspaper/outputformatters.py @@ -5,6 +5,8 @@ from HTMLParser import HTMLParser from .text import innerTrim +import lxml + class OutputFormatter(object): def __init__(self, config): @@ -26,14 +28,26 @@ def get_language(self, article): def get_top_node(self): return self.top_node - def get_formatted_text(self, article): + def get_formatted(self, article): + """ + Returns the body text of an article, and also the body article + html if specified. Returns in (text, html) form. + """ self.top_node = article.top_node + html, text = u'', u'' + self.remove_negativescores_nodes() + + if article.config.keep_article_html: + html = self.convert_to_html() + self.links_to_text() self.add_newline_to_br() self.replace_with_text() self.remove_fewwords_paragraphs(article) - return self.convert_to_text() + + text = self.convert_to_text() + return (text, html) def convert_to_text(self): txts = [] @@ -45,6 +59,10 @@ def convert_to_text(self): txts.extend(txt_lis) return '\n\n'.join(txts) + def convert_to_html(self): + cleaned_node = self.parser.clean_article_html(self.get_top_node()) + return self.parser.node_to_string(cleaned_node) + def add_newline_to_br(self): for e in self.parser.getElementsByTag(self.top_node, tag='br'): e.text = r'\n' diff --git a/newspaper/parsers.py b/newspaper/parsers.py index ce7f120f..be10f37c 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -8,9 +8,10 @@ import lxml.html from lxml.html import soupparser +from lxml.html.clean import Cleaner from lxml import etree -from copy import deepcopy +from copy import deepcopy from .text import innerTrim from .utils import encodeValue @@ -133,21 +134,6 @@ def get_meta_type(cls, doc): print str(e) log.critical(e) return u'' - @classmethod - def get_description(cls, doc): - """ - Returns meta description. - """ - try: - _list = doc.xpath('//meta[@name="description"]') - if len(_list) > 0: - content_list = _list[0].xpath('@content') - if len(content_list) > 0: - return content_list[0] - except Exception, e: - print str(e) - log.critical(e) - return u'' @classmethod def css_select(cls, node, selector): @@ -168,6 +154,19 @@ def fromstring(cls, html): # def set_doc(cls, html): # cls.doc = cls.fromstring(html) + @classmethod + def node_to_string(cls, node): + return lxml.html.tostring(node) + + @classmethod + def clean_article_html(cls, node): + article_cleaner = Cleaner() + article_cleaner.javascript = True + article_cleaner.style = True + article_cleaner.allow_tags = ['a', 'span', 'p', 'br', 'strong', 'b', 'em'] + article_cleaner.remove_unknown_tags = False + return article_cleaner.clean_html(node) + @classmethod def nodeToString(cls, node): return etree.tostring(node) diff --git a/newspaper/source.py b/newspaper/source.py index 36b2666f..0d0d73c5 100644 --- a/newspaper/source.py +++ b/newspaper/source.py @@ -38,7 +38,7 @@ class Feed(object): def __init__(self, url): self.url = encodeValue(url) self.rss = None - # TODO self.dom = None ;; speed up Feedparser + # TODO self.dom = None, speed up Feedparser class Source(object): @@ -107,35 +107,21 @@ def build(self): self.generate_articles() - def purge_articles(self, reason, in_articles=None): + def purge_articles(self, reason, articles): """ Delete rejected articles, if there is an articles param, we purge from there, otherwise purge from our source instance. + + Reference this excellent StackOverflow post for some of the wonky + syntax below: + http://stackoverflow.com/questions/1207406/remove-items-from-a- + list-while-iterating-in-python """ - # TODO Figure out why using the 'del' command on input list reference - # isn't actually filtering the list?! - # cur_articles = self.articles if in_articles is None else in_articles - new_articles = [] - - for index, article in enumerate(in_articles): - if reason == 'url' and not article.is_valid_url(): - #print 'deleting article', cur_articles[index].url - #del cur_articles[index] - #del in_articles[index] - pass - elif reason == 'url': - new_articles.append(in_articles[index]) - - if reason == 'body' and not article.is_valid_body(): - #del cur_articles[index] - pass - elif reason == 'body': - new_articles.append(in_articles[index]) - - if in_articles is not None: # if they give an input, output filtered - return new_articles - #else: # no input, we are playing with self.articles - # self.articles = new_articles + if reason == 'url': + articles[:] = [a for a in articles if a.is_valid_url()] + elif reason == 'body': + articles[:] = [a for a in articles if a.is_valid_body()] + return articles @cache_disk(seconds=(86400*1), cache_folder=ANCHOR_DIRECTORY) def _get_category_urls(self, domain): @@ -166,7 +152,6 @@ def set_description(self): query the desc html attribute. """ desc = self.extractor.get_meta_description(self) - # desc = self.parser.get_description(self.doc) self.description = encodeValue(desc) def download(self): @@ -175,7 +160,6 @@ def download(self): """ self.html = network.get_html(self.url, config=self.config) - # @print_duration def download_categories(self): """ Download all category html, can use mthreading. @@ -183,8 +167,6 @@ def download_categories(self): category_urls = [c.url for c in self.categories] requests = network.multithread_request(category_urls, self.config) - # the weird for loop is like this because the del keyword auto adjusts - # the list index after deletion only if the list being iterated contains elem deleted for index, _ in enumerate(self.categories): req = requests[index] if req.resp is not None: @@ -192,9 +174,8 @@ def download_categories(self): else: if self.config.verbose: print 'deleting category', self.categories[index].url, 'due to download err' - del self.categories[index] # TODO + self.categories = [c for c in self.categories if c.html] - # @print_duration def download_feeds(self): """ Download all feed html, can use mthreading. @@ -202,8 +183,6 @@ def download_feeds(self): feed_urls = [f.url for f in self.feeds] requests = network.multithread_request(feed_urls, self.config) - # the weird for loop is like this because the del keyword auto adjusts - # the list index after deletion only if the list being iterated contains elem deleted for index, _ in enumerate(self.feeds): req = requests[index] if req.resp is not None: @@ -211,7 +190,7 @@ def download_feeds(self): else: if self.config.verbose: print 'deleting feed', self.categories[index].url, 'due to download err' - del self.feeds[index] # TODO + self.feeds = [f for f in self.feeds if f.rss] def parse(self): """ @@ -238,9 +217,9 @@ def parse_categories(self): self.categories = [c for c in self.categories if c.doc is not None] - # TODO Use this method after we figure out how to make it fast def parse_feeds(self): """ + **THIS METHOD IS CURRENTLY RETIRED** Due to the slow speed of feedparser, we won't be dom parsing our .rss feeds, but rather regex searching for urls in the .rss text and then relying on our article logic to detect false urls. @@ -270,7 +249,7 @@ def feeds_to_articles(self): url=url, source_url=self.url, config=self.config - # title=? # TODO: It **must** be fast + # (pre) title=? # TODO Fast title regexing? ) cur_articles.append(article) @@ -343,16 +322,13 @@ def generate_articles(self, limit=5000): """ articles = self._generate_articles() self.articles = articles[:limit] + log.debug(len(articles), 'articles generated and cutoff at', limit) - # for a in self.articles: - # print 'test examine url:', a.url - # log.critical('total', len(articles), 'articles and cutoff was at', limit) - - # @print_duration def download_articles(self, threads=1): """ Downloads all articles attached to self. """ + # TODO fix how the article's is_downloaded is not set! urls = [a.url for a in self.articles] failed_articles = [] @@ -360,23 +336,21 @@ def download_articles(self, threads=1): for index, article in enumerate(self.articles): url = urls[index] html = network.get_html(url, config=self.config) - if html: - self.articles[index].html = html - else: + self.articles[index].set_html(html) + if not html: failed_articles.append(self.articles[index]) - del self.articles[index] # TODO iffy using del here + self.articles = [a for a in self.articles if a.html] else: - print ('Alert! We recommend you not multithread individual sources as ' - 'you will probably get rate limited. Instead, use newspapers custom ' - 'multithread framework') + if threads > 5: + print 'Using 5+ threads on a single source may get you rate limited!' filled_requests = network.multithread_request(urls, self.config) # Note that the responses are returned in original order for index, req in enumerate(filled_requests): - if req.resp is not None: - self.articles[index].html = network.get_html(req.url, response=req.resp) - else: + html = network.get_html(req.url, response=req.resp) + self.articles[index].set_html(html) + if not req.resp: failed_articles.append(self.articles[index]) - del self.articles[index] # TODO iffy using del here + self.articles = [a for a in self.articles if a.html] self.is_downloaded = True if len(failed_articles) > 0: @@ -386,7 +360,7 @@ def download_articles(self, threads=1): def parse_articles(self): """ - Sync parse all articles, delete if too small. + Parse all articles, delete if too small. """ for index, article in enumerate(self.articles): article.parse() @@ -465,3 +439,4 @@ def print_summary(self): print 'feed_urls:', self.feed_urls() print '\r\n' print 'category_urls:', self.category_urls() + diff --git a/newspaper/utils/__init__.py b/newspaper/utils/__init__.py index 81519e92..5fef23bd 100644 --- a/newspaper/utils/__init__.py +++ b/newspaper/utils/__init__.py @@ -268,7 +268,7 @@ def encodeValue(value): value = smart_str(value) except: value = string_org - return value + return value.strip() def memoize_articles(source, articles): """ diff --git a/newspaper/version.py b/newspaper/version.py index 19fa12e5..b882b738 100644 --- a/newspaper/version.py +++ b/newspaper/version.py @@ -3,5 +3,5 @@ To change the version of entire package, just edit this one location. """ -version_info = (0, 0, 5) +version_info = (0, 0, 6) __version__ = ".".join(map(str, version_info)) diff --git a/setup.py b/setup.py index b94ac7a0..10d36727 100644 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ setup( name='newspaper', - version='0.0.5', + version='0.0.6', description='Simplified python article discovery & extraction.', # long_description=readme+'\r\n'+history, author='Lucas Ou-Yang',