From 92bda7fddc3f6e4f6856b0b96948e9c1648b78b7 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Wed, 17 Dec 2014 11:37:09 -0800 Subject: [PATCH 01/13] Remove CONTRIBUTORS.md, just look at github contribs --- CONTRIBUTORS.md | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 CONTRIBUTORS.md diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md deleted file mode 100644 index f493311c..00000000 --- a/CONTRIBUTORS.md +++ /dev/null @@ -1,25 +0,0 @@ -Maintained and authored by: ---------------------------- -Lucas Ou-Yang -- lucasyangpersonal@gmail.com - -Thanks to the following contributors: -------------------------------------- -https://github.com/codelucas/newspaper/graphs/contributors - -Newspaper relied on some code of a few other open source projects: ------------------------------------------------------------------- -Thanks to all who have contributed to python-goose. -You can find the contributors list here: -https://github.com/grangier/python-goose/graphs/contributors - -Thanks to all who have contributed to PyTeaser. -You can find the contributors list here: -https://github.com/xiaoxu193/PyTeaser/graphs/contributors - -Thanks to all who have contributed to gravity-goose. -You can find the contributors list here: -https://github.com/GravityLabs/goose/graphs/contributors - -Thanks to all who have contributed to python-jieba. -You can find the contributors list here: -https://github.com/fxsjy/jieba/graphs/contributors From b55b7dd72046da23ee3e91f1ba21fc709fe93e95 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Wed, 17 Dec 2014 11:40:08 -0800 Subject: [PATCH 02/13] Remove `related projects` section --- README.rst | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/README.rst b/README.rst index d7b1516e..f9ea87c0 100644 --- a/README.rst +++ b/README.rst @@ -257,22 +257,6 @@ Feel free to give our testing suite a shot:: $ python tests/unit_tests.py -Related Projects ----------------- - -- `ruby-readability`_ is a port of arc90's readability project to Ruby. -- `python-goose`_ is a port of Gravity's goose project to Python. -- `java-boilerpipe`_ is an article extraction library in Java. - -.. _`python-goose`: https://github.com/grangier/python-goose -.. _`ruby-readability`: https://github.com/cantino/ruby-readability -.. _`java-boilerpipe`: http://boilerpipe-web.appspot.com/ - -.. _`Quickstart guide`: https://newspaper.readthedocs.org/en/latest/ -.. _`The Documentation`: http://newspaper.readthedocs.org -.. _`lxml`: http://lxml.de/ -.. _`requests`: https://github.com/kennethreitz/requests - LICENSE ------- From e26f1def50ff0daeddb7bd9ecdfbc798ccf3af43 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Wed, 17 Dec 2014 11:44:27 -0800 Subject: [PATCH 03/13] Update jieba requirement to be an exact version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a00c64b9..869456c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ responses==0.3.0 feedparser==5.1.3 tldextract==1.5.1 feedfinder2==0.0.1 -jieba3k +jieba3k==0.35.1 From dddbbba3fd49a96c850c580e4e8e8765ff869d5d Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Wed, 17 Dec 2014 12:12:24 -0800 Subject: [PATCH 04/13] Modify download_corpora.py and README.rst to show python3 updates --- README.rst | 60 +++++++++++++++++++++++++++------------------ download_corpora.py | 3 +-- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/README.rst b/README.rst index f9ea87c0..4de62448 100644 --- a/README.rst +++ b/README.rst @@ -15,6 +15,9 @@ Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed: .. _`tweeted by`: https://twitter.com/kennethreitz/status/419520678862548992 .. _`The Changelog`: http://thechangelog.com/newspaper-delivers-instapaper-style-article-extraction/ + +Newspaper is a Python3 library! Alternatively, view the `Python2 branch`_ + Basic Demo: http://newspaper-demo.herokuapp.com **We support 10+ languages and everything is in unicode!** @@ -57,16 +60,16 @@ A Glance: >>> for article in cnn_paper.articles: >>> print(article.url) - u'http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/' - u'http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html' + 'http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/' + 'http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html' ... >>> for category in cnn_paper.category_urls(): >>> print(category) - u'http://lifestyle.cnn.com' - u'http://cnn.com/world' - u'http://tech.cnn.com' + 'http://lifestyle.cnn.com' + 'http://cnn.com/world' + 'http://tech.cnn.com' ... .. code-block:: pycon @@ -78,23 +81,23 @@ A Glance: >>> article.download() >>> article.html - u'>> article.parse() >>> article.authors - [u'Leigh Ann Caldwell', 'John Honway'] + ['Leigh Ann Caldwell', 'John Honway'] >>> article.text - u'Washington (CNN) -- Not everyone subscribes to a New Year's resolution...' + 'Washington (CNN) -- Not everyone subscribes to a New Year's resolution...' >>> article.top_image - u'http://someCDN.com/blah/blah/blah/file.png' + 'http://someCDN.com/blah/blah/blah/file.png' >>> article.movies - [u'http://youtube.com/path/to/link.com', ...] + ['http://youtube.com/path/to/link.com', ...] .. code-block:: pycon @@ -104,7 +107,7 @@ A Glance: ['New Years', 'resolution', ...] >>> article.summary - u'The study shows that 93% of people ...' + 'The study shows that 93% of people ...' Newspaper has *seamless* language extraction and detection. @@ -141,9 +144,9 @@ If you are certain that an *entire* news source is in one language, **go ahead a >>> for category in sina_paper.category_urls(): >>> print(category) - u'http://health.sina.com.cn' - u'http://eladies.sina.com.cn' - u'http://english.sina.com' + 'http://health.sina.com.cn' + 'http://eladies.sina.com.cn' + 'http://english.sina.com' ... >>> article = sina_paper.articles[0] @@ -172,6 +175,7 @@ Interested in adding a new language for us? Refer to: `Docs - Adding new languag Features -------- +- Full Python3 and Python2 support - Works in 10+ languages (English, Chinese, German, Arabic, ...) - Multi-threaded article download framework - News url identification @@ -189,6 +193,9 @@ Get it now Installing newspaper is simple with `pip `_. However, you will run into fixable issues if you are trying to install on ubuntu. +Note that our Python3 package name is ``newspaper3k`` while our Python2 +package name is ``newspaper`` + **If you are on Debian / Ubuntu**, install using the following: - Python development version, needed for Python.h:: @@ -205,11 +212,11 @@ However, you will run into fixable issues if you are trying to install on ubuntu - Install the distribution via pip:: - $ pip install newspaper + $ pip3 install newspaper3k - Download NLP related corpora:: - $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python2.7 + $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3 **If you are on OSX**, install using the following, you may use both homebrew or macports: @@ -220,9 +227,9 @@ However, you will run into fixable issues if you are trying to install on ubuntu $ brew install libtiff libjpeg webp little-cms2 - $ pip install newspaper + $ pip3 install newspaper3k - $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python2.7 + $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3 **Otherwise**, install with the following: @@ -235,13 +242,16 @@ NOTE: You will still most likely need to install the following libraries via you :: - $ pip install newspaper + $ pip3 install newspaper3k - $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python + $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3 Development ----------- +Newspaper has two branches up for development. *This* branch, the master, is our Python3 +codebase while our Python2 branch is located on *python-2-head*. + If you'd like to contribute and hack on the newspaper project, feel free to clone a development version of this repository locally:: @@ -250,12 +260,12 @@ a development version of this repository locally:: Once you have a copy of the source, you can embed it in your Python package, or install it into your site-packages easily:: - $ pip install -r requirements.txt - $ python setup.py install + $ pip3 install -r requirements.txt + $ python3 setup.py install -Feel free to give our testing suite a shot:: +Feel free to give our testing suite a shot, everything is mocked!:: - $ python tests/unit_tests.py + $ python3 tests/unit_tests.py LICENSE ------- @@ -271,3 +281,5 @@ to talk about the future of this library and news extraction in general! .. _`email & contact me`: mailto:lucasyangpersonal@gmail.com .. _`python-goose's`: https://github.com/grangier/python-goose .. _`here`: https://github.com/codelucas/newspaper/blob/master/GOOSE-LICENSE.txt + +.. _`Python2 branch`: https://github.com/codelucas/newspaper/tree/python-2-head diff --git a/download_corpora.py b/download_corpora.py index d2e0e684..94abfc70 100644 --- a/download_corpora.py +++ b/download_corpora.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python2.7 # -*- coding: utf-8 -*- """ Downloads the necessary NLTK models and corpora required to support @@ -11,7 +10,7 @@ 'punkt', # Required for WordTokenizer 'maxent_treebank_pos_tagger', # Required for NLTKTagger 'movie_reviews', # Required for NaiveBayesAnalyzer - 'wordnet', # Required for lemmatization and Wordnet + 'wordnet', # Required for lemmatization and Wordnet 'stopwords' ] From 76956d28dcc005c0a9e5bef4be1a2a55e2159698 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Wed, 17 Dec 2014 12:12:24 -0800 Subject: [PATCH 05/13] Modify download_corpora.py and README.rst to show python3 updates --- README.rst | 70 ++++++++++++++++++++++++++++----------------- download_corpora.py | 3 +- 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/README.rst b/README.rst index f9ea87c0..b846b5b5 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -Newspaper: Article scraping & curation -======================================= +Newspaper3k: Article scraping & curation +======================================== .. image:: https://badge.fury.io/py/newspaper.png :target: http://badge.fury.io/py/newspaper @@ -15,7 +15,8 @@ Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed: .. _`tweeted by`: https://twitter.com/kennethreitz/status/419520678862548992 .. _`The Changelog`: http://thechangelog.com/newspaper-delivers-instapaper-style-article-extraction/ -Basic Demo: http://newspaper-demo.herokuapp.com + +Newspaper is a **Python3** library! Alternatively, view the `Python2 branch`_ **We support 10+ languages and everything is in unicode!** @@ -57,16 +58,16 @@ A Glance: >>> for article in cnn_paper.articles: >>> print(article.url) - u'http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/' - u'http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html' + 'http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/' + 'http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html' ... >>> for category in cnn_paper.category_urls(): >>> print(category) - u'http://lifestyle.cnn.com' - u'http://cnn.com/world' - u'http://tech.cnn.com' + 'http://lifestyle.cnn.com' + 'http://cnn.com/world' + 'http://tech.cnn.com' ... .. code-block:: pycon @@ -78,23 +79,23 @@ A Glance: >>> article.download() >>> article.html - u'>> article.parse() >>> article.authors - [u'Leigh Ann Caldwell', 'John Honway'] + ['Leigh Ann Caldwell', 'John Honway'] >>> article.text - u'Washington (CNN) -- Not everyone subscribes to a New Year's resolution...' + 'Washington (CNN) -- Not everyone subscribes to a New Year's resolution...' >>> article.top_image - u'http://someCDN.com/blah/blah/blah/file.png' + 'http://someCDN.com/blah/blah/blah/file.png' >>> article.movies - [u'http://youtube.com/path/to/link.com', ...] + ['http://youtube.com/path/to/link.com', ...] .. code-block:: pycon @@ -104,7 +105,7 @@ A Glance: ['New Years', 'resolution', ...] >>> article.summary - u'The study shows that 93% of people ...' + 'The study shows that 93% of people ...' Newspaper has *seamless* language extraction and detection. @@ -141,9 +142,9 @@ If you are certain that an *entire* news source is in one language, **go ahead a >>> for category in sina_paper.category_urls(): >>> print(category) - u'http://health.sina.com.cn' - u'http://eladies.sina.com.cn' - u'http://english.sina.com' + 'http://health.sina.com.cn' + 'http://eladies.sina.com.cn' + 'http://english.sina.com' ... >>> article = sina_paper.articles[0] @@ -172,6 +173,7 @@ Interested in adding a new language for us? Refer to: `Docs - Adding new languag Features -------- +- Full Python3 and Python2 support - Works in 10+ languages (English, Chinese, German, Arabic, ...) - Multi-threaded article download framework - News url identification @@ -189,6 +191,9 @@ Get it now Installing newspaper is simple with `pip `_. However, you will run into fixable issues if you are trying to install on ubuntu. +Note that our Python3 package name is ``newspaper3k`` while our Python2 +package name is ``newspaper`` + **If you are on Debian / Ubuntu**, install using the following: - Python development version, needed for Python.h:: @@ -205,11 +210,11 @@ However, you will run into fixable issues if you are trying to install on ubuntu - Install the distribution via pip:: - $ pip install newspaper + $ pip3 install newspaper3k - Download NLP related corpora:: - $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python2.7 + $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3 **If you are on OSX**, install using the following, you may use both homebrew or macports: @@ -220,9 +225,9 @@ However, you will run into fixable issues if you are trying to install on ubuntu $ brew install libtiff libjpeg webp little-cms2 - $ pip install newspaper + $ pip3 install newspaper3k - $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python2.7 + $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3 **Otherwise**, install with the following: @@ -235,13 +240,16 @@ NOTE: You will still most likely need to install the following libraries via you :: - $ pip install newspaper + $ pip3 install newspaper3k - $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python + $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3 Development ----------- +Newspaper has two branches up for development. *This* branch, the master, is our Python3 +codebase while our Python2 branch is located on *python-2-head*. + If you'd like to contribute and hack on the newspaper project, feel free to clone a development version of this repository locally:: @@ -250,12 +258,18 @@ a development version of this repository locally:: Once you have a copy of the source, you can embed it in your Python package, or install it into your site-packages easily:: - $ pip install -r requirements.txt - $ python setup.py install + $ pip3 install -r requirements.txt + $ python3 setup.py install + +Feel free to give our testing suite a shot, everything is mocked!:: -Feel free to give our testing suite a shot:: + $ python3 tests/unit_tests.py - $ python tests/unit_tests.py + +Demo +---- + +View a working online demo here: http://newspaper-demo.herokuapp.com LICENSE ------- @@ -271,3 +285,5 @@ to talk about the future of this library and news extraction in general! .. _`email & contact me`: mailto:lucasyangpersonal@gmail.com .. _`python-goose's`: https://github.com/grangier/python-goose .. _`here`: https://github.com/codelucas/newspaper/blob/master/GOOSE-LICENSE.txt + +.. _`Python2 branch`: https://github.com/codelucas/newspaper/tree/python-2-head diff --git a/download_corpora.py b/download_corpora.py index d2e0e684..94abfc70 100644 --- a/download_corpora.py +++ b/download_corpora.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python2.7 # -*- coding: utf-8 -*- """ Downloads the necessary NLTK models and corpora required to support @@ -11,7 +10,7 @@ 'punkt', # Required for WordTokenizer 'maxent_treebank_pos_tagger', # Required for NLTKTagger 'movie_reviews', # Required for NaiveBayesAnalyzer - 'wordnet', # Required for lemmatization and Wordnet + 'wordnet', # Required for lemmatization and Wordnet 'stopwords' ] From b4a88c088b27b9031d413388400329fe4b3f11d9 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Wed, 17 Dec 2014 13:48:40 -0800 Subject: [PATCH 06/13] Classify newspaper3k as a python 3 package, fix bugs in README.rst --- README.rst | 9 +++++++-- setup.py | 10 ++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 41af7ed9..1c5ed672 100644 --- a/README.rst +++ b/README.rst @@ -18,6 +18,8 @@ Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed: **Newspaper is a Python3 library**! Or, view the `Python2 branch`_ +.. _`Python2 branch`: https://github.com/codelucas/newspaper/tree/python-2-head + **We support 10+ languages and everything is in unicode!** .. code-block:: pycon @@ -192,7 +194,7 @@ Installing newspaper is simple with `pip `_. However, you will run into fixable issues if you are trying to install on ubuntu. Note that our Python3 package name is ``newspaper3k`` while our Python2 -package name is ``newspaper`` +package name is ``newspaper``. **If you are on Debian / Ubuntu**, install using the following: @@ -286,4 +288,7 @@ to talk about the future of this library and news extraction in general! .. _`python-goose's`: https://github.com/grangier/python-goose .. _`here`: https://github.com/codelucas/newspaper/blob/master/GOOSE-LICENSE.txt -.. _`Python2 branch`: https://github.com/codelucas/newspaper/tree/python-2-head +.. _`Quickstart guide`: https://newspaper.readthedocs.org/en/latest/ +.. _`The Documentation`: http://newspaper.readthedocs.org +.. _`lxml`: http://lxml.de/ +.. _`requests`: https://github.com/kennethreitz/requests diff --git a/setup.py b/setup.py index 2f614443..09d9cd61 100755 --- a/setup.py +++ b/setup.py @@ -2,9 +2,6 @@ # -*- coding: utf-8 -*- """ Lucas Ou 2014 -- http://lucasou.com - -Setup guide: http://guide.python-distribute.org/creation.html -python setup.py sdist bdist_wininst upload """ import sys @@ -23,7 +20,7 @@ if sys.argv[-1] == 'publish': - os.system('python setup.py sdist bdist_wininst upload') + os.system('python3 setup.py sdist upload') # bdist_wininst sys.exit() @@ -48,4 +45,9 @@ install_requires=required, license='MIT', zip_safe=False, + classifiers=[ + 'Programming Language :: Python :: 3', + 'Natural Language :: English', + 'Intended Audience :: Developers', + ], ) From af80d65e28d1e9d5185efbe8546eaac70f6913bf Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Wed, 17 Dec 2014 13:53:53 -0800 Subject: [PATCH 07/13] Bugfix in MANIFEST.in, forgot "include" keyword --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index a3b291e1..d2fff404 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -requirements.txt README.rst LICENSE \ No newline at end of file +include requirements.txt README.rst LICENSE \ No newline at end of file From a638bdf9859de78cce0cf75394c2d3024b5a5376 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Wed, 17 Dec 2014 14:32:59 -0800 Subject: [PATCH 08/13] Bugfix in MANIFEST.in, forgot to include module recursively --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index d2fff404..7de2595d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include requirements.txt README.rst LICENSE \ No newline at end of file +include requirements.txt README.rst LICENSE recursive-include newspaper * \ No newline at end of file From fd4f1acae5304c7df5bc715356a2841924ca0a46 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Fri, 26 Dec 2014 23:51:50 -0800 Subject: [PATCH 09/13] Add top image extraction unit test --- tests/unit_tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 02f57275..50288066 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -119,11 +119,11 @@ def test_parse_html(self): with open(os.path.join(TEXT_FN, 'cnn.txt'), 'r') as f: assert self.article.text == f.read() - # TOP_IMG = ('http://i2.cdn.turner.com/cnn/dam/assets/131129200805-' - # '01-weather-1128-story-top.jpg') - # `top_img` isn't tested because it is extracted for this particular - # article with the "reddit method", it requires internet connection - # TODO: assert self.article.top_img == TOP_IMG + # NOTE: top_img extraction requires an internet connection + # unlike the rest of this test file + TOP_IMG = ('http://i2.cdn.turner.com/cnn/dam/assets/131129200805-' + '01-weather-1128-story-top.jpg') + assert self.article.top_img == TOP_IMG assert sorted(self.article.authors) == AUTHORS assert self.article.title == TITLE From f2d0681c3b55c8314b087129454b39e215a192d0 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Sat, 27 Dec 2014 00:04:42 -0800 Subject: [PATCH 10/13] [bugfix] decode bytestring returned from lxml's `toString` before sending it out to outer code --- newspaper/cleaners.py | 2 +- newspaper/parsers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/newspaper/cleaners.py b/newspaper/cleaners.py index d9b84096..5de87b06 100644 --- a/newspaper/cleaners.py +++ b/newspaper/cleaners.py @@ -170,7 +170,7 @@ def replace_walk_left_right(self, kid, kid_text, and self.parser.getTag(next_node) == "a" \ and self.parser.getAttribute( next_node, 'grv-usedalready') != 'yes': - outer = " " + self.parser.outerHtml(next_node).decode('utf-8') + " " + outer = " " + self.parser.outerHtml(next_node) + " " replacement_text.append(outer) nodes_to_remove.append(next_node) self.parser.setAttribute(next_node, attr='grv-usedalready', diff --git a/newspaper/parsers.py b/newspaper/parsers.py index 06d7879d..10bbd540 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -72,7 +72,7 @@ def clean_article_html(cls, node): @classmethod def nodeToString(cls, node): - return lxml.etree.tostring(node) + return lxml.etree.tostring(node).decode() @classmethod def replaceTag(cls, node, tag): From f2de9788989ec3c9c664c2c23b2d19ecc5567bf7 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Sat, 27 Dec 2014 01:35:42 -0800 Subject: [PATCH 11/13] Handle exception logging using tracebacks, removed custom exception alert stdout print statements --- newspaper/article.py | 2 +- newspaper/images.py | 18 ++++++------------ newspaper/mthreading.py | 7 ++++--- newspaper/parsers.py | 10 ++++++---- newspaper/source.py | 2 -- 5 files changed, 17 insertions(+), 22 deletions(-) diff --git a/newspaper/article.py b/newspaper/article.py index a33b770d..4c7a8290 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -160,7 +160,7 @@ def parse(self): self.clean_doc = copy.deepcopy(self.doc) if self.doc is None: - print('[Article parse ERR] %s' % self.url) + # `parse` call failed, return nothing return # TODO: Fix this, sync in our fix_url() method diff --git a/newspaper/images.py b/newspaper/images.py index 09115084..1ae5a6ab 100644 --- a/newspaper/images.py +++ b/newspaper/images.py @@ -11,6 +11,7 @@ import logging import math import io +import traceback import urllib.request, urllib.parse, urllib.error import urllib.request, urllib.error, urllib.parse @@ -116,28 +117,21 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False): while not p.image and new_data: try: p.feed(new_data) - except IOError as e: - # pil failed to install, jpeg codec broken - # **should work if you install via pillow - print(('***jpeg misconfiguration! check pillow or pil' - 'installation this machine: %s' % str(e))) + except IOError: + traceback.print_exc() p = None break - except ValueError as ve: - log.debug('cant read image format: %s' % url) + except ValueError: + traceback.print_exc() p = None break except Exception as e: # For some favicon.ico images, the image is so small # that our PIL feed() method fails a length test. - # We add a check below for this. is_favicon = (urls.url_to_filetype(url) == 'ico') if is_favicon: - print('we caught a favicon!: %s' % url) + pass else: - # import traceback - # print(traceback.format_exc()) - print('PIL feed() failure for image:', url, str(e)) raise e p = None break diff --git a/newspaper/mthreading.py b/newspaper/mthreading.py index 8113d2ff..c1cb355c 100644 --- a/newspaper/mthreading.py +++ b/newspaper/mthreading.py @@ -10,6 +10,7 @@ __copyright__ = 'Copyright 2014, Lucas Ou-Yang' import queue +import traceback from threading import Thread @@ -28,12 +29,12 @@ def run(self): try: func, args, kargs = self.tasks.get() except queue.Empty: - print('thread breaking b/c queue is empty') + traceback.print_exc() break try: func(*args, **kargs) - except Exception as e: - print('critical multi-thread err %s' % e) + except Exception: + traceback.print_exc() self.tasks.task_done() diff --git a/newspaper/parsers.py b/newspaper/parsers.py index 10bbd540..95326af4 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -10,6 +10,7 @@ import lxml.etree import lxml.html import lxml.html.clean +import traceback from copy import deepcopy @@ -42,14 +43,15 @@ def css_select(cls, node, selector): @classmethod def fromstring(cls, html): html = utils.encodeValue(html) + # don't bring the entire library down because one article + # or article failed to parse try: cls.doc = lxml.html.fromstring(html) - except Exception as e: - print('[Parse lxml ERR]', str(e)) + return cls.doc + except Exception: + traceback.print_exc() return None - return cls.doc - # @classmethod # def set_doc(cls, html): # cls.doc = cls.fromstring(html) diff --git a/newspaper/source.py b/newspaper/source.py index 74338755..0a0200ff 100644 --- a/newspaper/source.py +++ b/newspaper/source.py @@ -201,8 +201,6 @@ def parse_categories(self): for category in self.categories: doc = self.config.get_parser().fromstring(category.html) category.doc = doc - if category.doc is None: - print('[Category parse ERR]', category.url) self.categories = [c for c in self.categories if c.doc is not None] From 800651c08fdf5d1832b1a8ff56ed22896599865c Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Sat, 27 Dec 2014 02:34:51 -0800 Subject: [PATCH 12/13] Remove encoding tag in HTML before having lxml parse a DOM out This was a very annoying bug in the past, which caused DOM parse failures: Reference https://github.com/codelucas/newspaper/pull/97 https://github.com/codelucas/newspaper/issues/78 --- newspaper/parsers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/newspaper/parsers.py b/newspaper/parsers.py index 95326af4..cbc86216 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -10,6 +10,7 @@ import lxml.etree import lxml.html import lxml.html.clean +import re import traceback from copy import deepcopy @@ -46,6 +47,14 @@ def fromstring(cls, html): # don't bring the entire library down because one article # or article failed to parse try: + # remove encoding tag because lxml won't accept it for + # unicode objects (Issue #78) + if isinstance(html, bytes): + if html.startswith(b'', b'', html, flags=re.DOTALL) + else: + if html.startswith('', '', html, flags=re.DOTALL) cls.doc = lxml.html.fromstring(html) return cls.doc except Exception: From 0289aa7aa8b48d16c2a95ce5315101d816c44de0 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Sat, 27 Dec 2014 02:43:51 -0800 Subject: [PATCH 13/13] Update version for new python3 release --- newspaper/version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/newspaper/version.py b/newspaper/version.py index 75c7440b..558d6c6c 100644 --- a/newspaper/version.py +++ b/newspaper/version.py @@ -7,5 +7,5 @@ __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' -version_info = (0, 1, 0) +version_info = (0, 1, 1) __version__ = ".".join(map(str, version_info)) diff --git a/setup.py b/setup.py index 09d9cd61..eb18e300 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( name='newspaper3k', - version='0.1.0', + version='0.1.1', description='Simplified python article discovery & extraction.', long_description=readme, author='Lucas Ou-Yang',