diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md deleted file mode 100644 index f493311c..00000000 --- a/CONTRIBUTORS.md +++ /dev/null @@ -1,25 +0,0 @@ -Maintained and authored by: ---------------------------- -Lucas Ou-Yang -- lucasyangpersonal@gmail.com - -Thanks to the following contributors: -------------------------------------- -https://github.com/codelucas/newspaper/graphs/contributors - -Newspaper relied on some code of a few other open source projects: ------------------------------------------------------------------- -Thanks to all who have contributed to python-goose. -You can find the contributors list here: -https://github.com/grangier/python-goose/graphs/contributors - -Thanks to all who have contributed to PyTeaser. -You can find the contributors list here: -https://github.com/xiaoxu193/PyTeaser/graphs/contributors - -Thanks to all who have contributed to gravity-goose. -You can find the contributors list here: -https://github.com/GravityLabs/goose/graphs/contributors - -Thanks to all who have contributed to python-jieba. -You can find the contributors list here: -https://github.com/fxsjy/jieba/graphs/contributors diff --git a/MANIFEST.in b/MANIFEST.in index a3b291e1..7de2595d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -requirements.txt README.rst LICENSE \ No newline at end of file +include requirements.txt README.rst LICENSE recursive-include newspaper * \ No newline at end of file diff --git a/README.rst b/README.rst index d7b1516e..1c5ed672 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -Newspaper: Article scraping & curation -======================================= +Newspaper3k: Article scraping & curation +======================================== .. image:: https://badge.fury.io/py/newspaper.png :target: http://badge.fury.io/py/newspaper @@ -15,7 +15,10 @@ Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed: .. _`tweeted by`: https://twitter.com/kennethreitz/status/419520678862548992 .. _`The Changelog`: http://thechangelog.com/newspaper-delivers-instapaper-style-article-extraction/ -Basic Demo: http://newspaper-demo.herokuapp.com + +**Newspaper is a Python3 library**! Or, view the `Python2 branch`_ + +.. _`Python2 branch`: https://github.com/codelucas/newspaper/tree/python-2-head **We support 10+ languages and everything is in unicode!** @@ -57,16 +60,16 @@ A Glance: >>> for article in cnn_paper.articles: >>> print(article.url) - u'http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/' - u'http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html' + 'http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/' + 'http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html' ... >>> for category in cnn_paper.category_urls(): >>> print(category) - u'http://lifestyle.cnn.com' - u'http://cnn.com/world' - u'http://tech.cnn.com' + 'http://lifestyle.cnn.com' + 'http://cnn.com/world' + 'http://tech.cnn.com' ... .. code-block:: pycon @@ -78,23 +81,23 @@ A Glance: >>> article.download() >>> article.html - u'>> article.parse() >>> article.authors - [u'Leigh Ann Caldwell', 'John Honway'] + ['Leigh Ann Caldwell', 'John Honway'] >>> article.text - u'Washington (CNN) -- Not everyone subscribes to a New Year's resolution...' + 'Washington (CNN) -- Not everyone subscribes to a New Year's resolution...' >>> article.top_image - u'http://someCDN.com/blah/blah/blah/file.png' + 'http://someCDN.com/blah/blah/blah/file.png' >>> article.movies - [u'http://youtube.com/path/to/link.com', ...] + ['http://youtube.com/path/to/link.com', ...] .. code-block:: pycon @@ -104,7 +107,7 @@ A Glance: ['New Years', 'resolution', ...] >>> article.summary - u'The study shows that 93% of people ...' + 'The study shows that 93% of people ...' Newspaper has *seamless* language extraction and detection. @@ -141,9 +144,9 @@ If you are certain that an *entire* news source is in one language, **go ahead a >>> for category in sina_paper.category_urls(): >>> print(category) - u'http://health.sina.com.cn' - u'http://eladies.sina.com.cn' - u'http://english.sina.com' + 'http://health.sina.com.cn' + 'http://eladies.sina.com.cn' + 'http://english.sina.com' ... >>> article = sina_paper.articles[0] @@ -172,6 +175,7 @@ Interested in adding a new language for us? Refer to: `Docs - Adding new languag Features -------- +- Full Python3 and Python2 support - Works in 10+ languages (English, Chinese, German, Arabic, ...) - Multi-threaded article download framework - News url identification @@ -189,6 +193,9 @@ Get it now Installing newspaper is simple with `pip `_. However, you will run into fixable issues if you are trying to install on ubuntu. +Note that our Python3 package name is ``newspaper3k`` while our Python2 +package name is ``newspaper``. + **If you are on Debian / Ubuntu**, install using the following: - Python development version, needed for Python.h:: @@ -205,11 +212,11 @@ However, you will run into fixable issues if you are trying to install on ubuntu - Install the distribution via pip:: - $ pip install newspaper + $ pip3 install newspaper3k - Download NLP related corpora:: - $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python2.7 + $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3 **If you are on OSX**, install using the following, you may use both homebrew or macports: @@ -220,9 +227,9 @@ However, you will run into fixable issues if you are trying to install on ubuntu $ brew install libtiff libjpeg webp little-cms2 - $ pip install newspaper + $ pip3 install newspaper3k - $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python2.7 + $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3 **Otherwise**, install with the following: @@ -235,13 +242,16 @@ NOTE: You will still most likely need to install the following libraries via you :: - $ pip install newspaper + $ pip3 install newspaper3k - $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python + $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3 Development ----------- +Newspaper has two branches up for development. *This* branch, the master, is our Python3 +codebase while our Python2 branch is located on *python-2-head*. + If you'd like to contribute and hack on the newspaper project, feel free to clone a development version of this repository locally:: @@ -250,28 +260,18 @@ a development version of this repository locally:: Once you have a copy of the source, you can embed it in your Python package, or install it into your site-packages easily:: - $ pip install -r requirements.txt - $ python setup.py install + $ pip3 install -r requirements.txt + $ python3 setup.py install -Feel free to give our testing suite a shot:: +Feel free to give our testing suite a shot, everything is mocked!:: - $ python tests/unit_tests.py + $ python3 tests/unit_tests.py -Related Projects ----------------- -- `ruby-readability`_ is a port of arc90's readability project to Ruby. -- `python-goose`_ is a port of Gravity's goose project to Python. -- `java-boilerpipe`_ is an article extraction library in Java. +Demo +---- -.. _`python-goose`: https://github.com/grangier/python-goose -.. _`ruby-readability`: https://github.com/cantino/ruby-readability -.. _`java-boilerpipe`: http://boilerpipe-web.appspot.com/ - -.. _`Quickstart guide`: https://newspaper.readthedocs.org/en/latest/ -.. _`The Documentation`: http://newspaper.readthedocs.org -.. _`lxml`: http://lxml.de/ -.. _`requests`: https://github.com/kennethreitz/requests +View a working online demo here: http://newspaper-demo.herokuapp.com LICENSE ------- @@ -287,3 +287,8 @@ to talk about the future of this library and news extraction in general! .. _`email & contact me`: mailto:lucasyangpersonal@gmail.com .. _`python-goose's`: https://github.com/grangier/python-goose .. _`here`: https://github.com/codelucas/newspaper/blob/master/GOOSE-LICENSE.txt + +.. _`Quickstart guide`: https://newspaper.readthedocs.org/en/latest/ +.. _`The Documentation`: http://newspaper.readthedocs.org +.. _`lxml`: http://lxml.de/ +.. _`requests`: https://github.com/kennethreitz/requests diff --git a/download_corpora.py b/download_corpora.py index d2e0e684..94abfc70 100644 --- a/download_corpora.py +++ b/download_corpora.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python2.7 # -*- coding: utf-8 -*- """ Downloads the necessary NLTK models and corpora required to support @@ -11,7 +10,7 @@ 'punkt', # Required for WordTokenizer 'maxent_treebank_pos_tagger', # Required for NLTKTagger 'movie_reviews', # Required for NaiveBayesAnalyzer - 'wordnet', # Required for lemmatization and Wordnet + 'wordnet', # Required for lemmatization and Wordnet 'stopwords' ] diff --git a/newspaper/article.py b/newspaper/article.py index a33b770d..4c7a8290 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -160,7 +160,7 @@ def parse(self): self.clean_doc = copy.deepcopy(self.doc) if self.doc is None: - print('[Article parse ERR] %s' % self.url) + # `parse` call failed, return nothing return # TODO: Fix this, sync in our fix_url() method diff --git a/newspaper/cleaners.py b/newspaper/cleaners.py index d9b84096..5de87b06 100644 --- a/newspaper/cleaners.py +++ b/newspaper/cleaners.py @@ -170,7 +170,7 @@ def replace_walk_left_right(self, kid, kid_text, and self.parser.getTag(next_node) == "a" \ and self.parser.getAttribute( next_node, 'grv-usedalready') != 'yes': - outer = " " + self.parser.outerHtml(next_node).decode('utf-8') + " " + outer = " " + self.parser.outerHtml(next_node) + " " replacement_text.append(outer) nodes_to_remove.append(next_node) self.parser.setAttribute(next_node, attr='grv-usedalready', diff --git a/newspaper/images.py b/newspaper/images.py index 09115084..1ae5a6ab 100644 --- a/newspaper/images.py +++ b/newspaper/images.py @@ -11,6 +11,7 @@ import logging import math import io +import traceback import urllib.request, urllib.parse, urllib.error import urllib.request, urllib.error, urllib.parse @@ -116,28 +117,21 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False): while not p.image and new_data: try: p.feed(new_data) - except IOError as e: - # pil failed to install, jpeg codec broken - # **should work if you install via pillow - print(('***jpeg misconfiguration! check pillow or pil' - 'installation this machine: %s' % str(e))) + except IOError: + traceback.print_exc() p = None break - except ValueError as ve: - log.debug('cant read image format: %s' % url) + except ValueError: + traceback.print_exc() p = None break except Exception as e: # For some favicon.ico images, the image is so small # that our PIL feed() method fails a length test. - # We add a check below for this. is_favicon = (urls.url_to_filetype(url) == 'ico') if is_favicon: - print('we caught a favicon!: %s' % url) + pass else: - # import traceback - # print(traceback.format_exc()) - print('PIL feed() failure for image:', url, str(e)) raise e p = None break diff --git a/newspaper/mthreading.py b/newspaper/mthreading.py index 8113d2ff..c1cb355c 100644 --- a/newspaper/mthreading.py +++ b/newspaper/mthreading.py @@ -10,6 +10,7 @@ __copyright__ = 'Copyright 2014, Lucas Ou-Yang' import queue +import traceback from threading import Thread @@ -28,12 +29,12 @@ def run(self): try: func, args, kargs = self.tasks.get() except queue.Empty: - print('thread breaking b/c queue is empty') + traceback.print_exc() break try: func(*args, **kargs) - except Exception as e: - print('critical multi-thread err %s' % e) + except Exception: + traceback.print_exc() self.tasks.task_done() diff --git a/newspaper/parsers.py b/newspaper/parsers.py index 06d7879d..cbc86216 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -10,6 +10,8 @@ import lxml.etree import lxml.html import lxml.html.clean +import re +import traceback from copy import deepcopy @@ -42,14 +44,23 @@ def css_select(cls, node, selector): @classmethod def fromstring(cls, html): html = utils.encodeValue(html) + # don't bring the entire library down because one article + # or article failed to parse try: + # remove encoding tag because lxml won't accept it for + # unicode objects (Issue #78) + if isinstance(html, bytes): + if html.startswith(b'', b'', html, flags=re.DOTALL) + else: + if html.startswith('', '', html, flags=re.DOTALL) cls.doc = lxml.html.fromstring(html) - except Exception as e: - print('[Parse lxml ERR]', str(e)) + return cls.doc + except Exception: + traceback.print_exc() return None - return cls.doc - # @classmethod # def set_doc(cls, html): # cls.doc = cls.fromstring(html) @@ -72,7 +83,7 @@ def clean_article_html(cls, node): @classmethod def nodeToString(cls, node): - return lxml.etree.tostring(node) + return lxml.etree.tostring(node).decode() @classmethod def replaceTag(cls, node, tag): diff --git a/newspaper/source.py b/newspaper/source.py index 74338755..0a0200ff 100644 --- a/newspaper/source.py +++ b/newspaper/source.py @@ -201,8 +201,6 @@ def parse_categories(self): for category in self.categories: doc = self.config.get_parser().fromstring(category.html) category.doc = doc - if category.doc is None: - print('[Category parse ERR]', category.url) self.categories = [c for c in self.categories if c.doc is not None] diff --git a/newspaper/version.py b/newspaper/version.py index 75c7440b..558d6c6c 100644 --- a/newspaper/version.py +++ b/newspaper/version.py @@ -7,5 +7,5 @@ __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' -version_info = (0, 1, 0) +version_info = (0, 1, 1) __version__ = ".".join(map(str, version_info)) diff --git a/requirements.txt b/requirements.txt index a00c64b9..869456c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,4 @@ responses==0.3.0 feedparser==5.1.3 tldextract==1.5.1 feedfinder2==0.0.1 -jieba3k +jieba3k==0.35.1 diff --git a/setup.py b/setup.py index 2f614443..eb18e300 100755 --- a/setup.py +++ b/setup.py @@ -2,9 +2,6 @@ # -*- coding: utf-8 -*- """ Lucas Ou 2014 -- http://lucasou.com - -Setup guide: http://guide.python-distribute.org/creation.html -python setup.py sdist bdist_wininst upload """ import sys @@ -23,7 +20,7 @@ if sys.argv[-1] == 'publish': - os.system('python setup.py sdist bdist_wininst upload') + os.system('python3 setup.py sdist upload') # bdist_wininst sys.exit() @@ -37,7 +34,7 @@ setup( name='newspaper3k', - version='0.1.0', + version='0.1.1', description='Simplified python article discovery & extraction.', long_description=readme, author='Lucas Ou-Yang', @@ -48,4 +45,9 @@ install_requires=required, license='MIT', zip_safe=False, + classifiers=[ + 'Programming Language :: Python :: 3', + 'Natural Language :: English', + 'Intended Audience :: Developers', + ], ) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 02f57275..50288066 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -119,11 +119,11 @@ def test_parse_html(self): with open(os.path.join(TEXT_FN, 'cnn.txt'), 'r') as f: assert self.article.text == f.read() - # TOP_IMG = ('http://i2.cdn.turner.com/cnn/dam/assets/131129200805-' - # '01-weather-1128-story-top.jpg') - # `top_img` isn't tested because it is extracted for this particular - # article with the "reddit method", it requires internet connection - # TODO: assert self.article.top_img == TOP_IMG + # NOTE: top_img extraction requires an internet connection + # unlike the rest of this test file + TOP_IMG = ('http://i2.cdn.turner.com/cnn/dam/assets/131129200805-' + '01-weather-1128-story-top.jpg') + assert self.article.top_img == TOP_IMG assert sorted(self.article.authors) == AUTHORS assert self.article.title == TITLE