From e809be8c16558b7fe967bc83dd67e3fcd8050f72 Mon Sep 17 00:00:00 2001 From: Dave Crumbacher Date: Thu, 26 Mar 2015 23:31:34 -0400 Subject: [PATCH 01/28] Correct parsing of author with period in name --- newspaper/extractors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index ec25b7d3..5272f179 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -95,9 +95,9 @@ def parse_byline(search_str): search_str = search_str.strip() # Chunk the line by non alphanumeric tokens (few name exceptions) - # >>> re.split("[^\w\'\-]", "Lucas Ou, Dean O'Brian and Ronald") - # ['Lucas Ou', '', 'Dean O'Brian', 'and', 'Ronald'] - name_tokens = re.split("[^\w\'\-]", search_str) + # >>> re.split("[^\w\'\-\.]", "Tyler G. Jones, Lucas Ou, Dean O'Brian and Ronald") + # ['Tyler', 'G.', 'Jones', '', 'Lucas', 'Ou', '', 'Dean', "O'Brian", 'and', 'Ronald'] + name_tokens = re.split("[^\w\'\-\.]", search_str) name_tokens = [s.strip() for s in name_tokens] _authors = [] From e5bea031515eafd24a8b86abda0a03bace55f78c Mon Sep 17 00:00:00 2001 From: Dave Crumbacher Date: Thu, 26 Mar 2015 23:36:40 -0400 Subject: [PATCH 02/28] Use title() instead of capitalize() to properly capitalize hyphenated names --- newspaper/extractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 5272f179..d00f039a 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -149,7 +149,7 @@ def parse_byline(search_str): uniq = list(set([s.lower() for s in _authors])) for name in uniq: - names = [w.capitalize() for w in name.split(' ')] + names = [w.title() for w in name.split(' ')] authors.append(' '.join(names)) return authors or [] From 20aeb460f5e39f93fa258f53260d28b52d51026d Mon Sep 17 00:00:00 2001 From: Dave Crumbacher Date: Thu, 26 Mar 2015 23:43:38 -0400 Subject: [PATCH 03/28] Correct parsing of author with period in name --- newspaper/extractors.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index ec25b7d3..5272f179 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -95,9 +95,9 @@ def parse_byline(search_str): search_str = search_str.strip() # Chunk the line by non alphanumeric tokens (few name exceptions) - # >>> re.split("[^\w\'\-]", "Lucas Ou, Dean O'Brian and Ronald") - # ['Lucas Ou', '', 'Dean O'Brian', 'and', 'Ronald'] - name_tokens = re.split("[^\w\'\-]", search_str) + # >>> re.split("[^\w\'\-\.]", "Tyler G. Jones, Lucas Ou, Dean O'Brian and Ronald") + # ['Tyler', 'G.', 'Jones', '', 'Lucas', 'Ou', '', 'Dean', "O'Brian", 'and', 'Ronald'] + name_tokens = re.split("[^\w\'\-\.]", search_str) name_tokens = [s.strip() for s in name_tokens] _authors = [] From 0efc4971b6c3fab5954b755f4ad8ffd2f3eeb9f9 Mon Sep 17 00:00:00 2001 From: Dave Crumbacher Date: Thu, 26 Mar 2015 23:43:53 -0400 Subject: [PATCH 04/28] Use title() instead of capitalize() to properly capitalize hyphenated names --- newspaper/extractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 5272f179..d00f039a 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -149,7 +149,7 @@ def parse_byline(search_str): uniq = list(set([s.lower() for s in _authors])) for name in uniq: - names = [w.capitalize() for w in name.split(' ')] + names = [w.title() for w in name.split(' ')] authors.append(' '.join(names)) return authors or [] From ab721393ac094cfb4cb0ea79c15e9c523c04af43 Mon Sep 17 00:00:00 2001 From: Dave Crumbacher Date: Fri, 27 Mar 2015 17:36:11 -0400 Subject: [PATCH 05/28] Better handle more complex author names; simplify process to make author list unique --- newspaper/extractors.py | 28 +++++++++++++++++----------- tests/data/html/cnn_article.html | 12 ++++++------ tests/unit_tests.py | 8 ++++---- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index d00f039a..1f62a1c8 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -79,6 +79,18 @@ def get_authors(self, doc): def contains_digits(d): return bool(_digits.search(d)) + def uniqify_list(l): + """Remove duplicates from provided list but maintain original order. + Derived from http://www.peterbe.com/plog/uniqifiers-benchmark + """ + seen = {} + result = [] + for item in l: + if item in seen: continue + seen[item] = 1 + result.append(item) + return result + def parse_byline(search_str): """Takes a candidate line of html or text and extracts out the name(s) in list form @@ -103,13 +115,11 @@ def parse_byline(search_str): _authors = [] # List of first, last name tokens curname = [] - DELIM = ['and', ''] + DELIM = ['and', ',', ''] for token in name_tokens: if token in DELIM: - # should we allow middle names? - valid_name = (len(curname) == 2) - if valid_name: + if len(curname) > 0: _authors.append(' '.join(curname)) curname = [] @@ -128,7 +138,7 @@ def parse_byline(search_str): ATTRS = ['name', 'rel', 'itemprop', 'class', 'id'] VALS = ['author', 'byline'] matches = [] - _authors, authors = [], [] + authors = [] for attr in ATTRS: for val in VALS: @@ -145,13 +155,9 @@ def parse_byline(search_str): else: content = match.text or '' if len(content) > 0: - _authors.extend(parse_byline(content)) + authors.extend(parse_byline(content)) - uniq = list(set([s.lower() for s in _authors])) - for name in uniq: - names = [w.title() for w in name.split(' ')] - authors.append(' '.join(names)) - return authors or [] + return uniqify_list(authors) # TODO Method 2: Search raw html for a by-line # match = re.search('By[\: ].*\\n|From[\: ].*\\n', html) diff --git a/tests/data/html/cnn_article.html b/tests/data/html/cnn_article.html index eaf5b227..53c1629b 100644 --- a/tests/data/html/cnn_article.html +++ b/tests/data/html/cnn_article.html @@ -13,7 +13,7 @@ - + @@ -68,7 +68,7 @@ cnnBrandingValue="default"; cnnPartnerValue=""; cnnOmniBranding="", -cnnAuthor="Dana Ford and Tom Watkins, CNN", +cnnAuthor="Dana A. Ford, James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN", disqus_category_id=207582, disqus_identifier="/2013/11/27/travel/weather-thanksgiving/index.html", disqus_title="After storm, forecasters see smooth sailing for Thanksgiving", @@ -87,7 +87,7 @@ business: { cnn: { page: { -author: "Dana Ford and Tom Watkins, CNN", +author: "Dana A. Ford, James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN", broadcast_franchise: "", video_embed_count: "4", publish_date: "2013/11/27", @@ -294,7 +294,7 @@

After storm, forecasters see smooth sailing for Thanksgiving

-
By Dana Ford and Tom Watkins, CNN
+
By Dana A. Ford , James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN
November 28, 2013 -- Updated 0203 GMT (1003 HKT)
@@ -1022,7 +1022,7 @@

cnnsocial.share.setconfig(cnn_shareconfig); $j(document).ready(function () { 'use strict'; -loadChartbeat("travel", "Dana Ford and Tom Watkins, CNN"); +loadChartbeat("travel", "Dana A. Ford, James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN"); CNN.initFlipperTicker(); /* initialize cnnsocial */ cnnsocial.init(); @@ -1041,4 +1041,4 @@

- \ No newline at end of file + diff --git a/tests/unit_tests.py b/tests/unit_tests.py index e17b241b..3e485b09 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -158,7 +158,7 @@ def test_url(self): def test_download_html(self): html = mock_resource_with('cnn_article', 'html') self.article.download(html) - assert len(self.article.html) == 75175 + assert len(self.article.html) == 75406 @print_test def test_pre_download_parse(self): @@ -169,7 +169,7 @@ def test_pre_download_parse(self): @print_test def test_parse_html(self): - AUTHORS = ['Dana Ford', 'Tom Watkins'] + AUTHORS = ['Dana A. Ford', 'James S.A. Corey', 'Chien-Ming Wang', 'Tom Watkins'] TITLE = 'After storm, forecasters see smooth sailing for Thanksgiving' LEN_IMGS = 46 META_LANG = 'en' @@ -187,7 +187,7 @@ def test_parse_html(self): '01-weather-1128-story-top.jpg') assert self.article.top_img == TOP_IMG - assert sorted(self.article.authors) == AUTHORS + assert self.article.authors == AUTHORS assert self.article.title == TITLE assert len(self.article.imgs) == LEN_IMGS assert self.article.meta_lang == META_LANG @@ -209,7 +209,7 @@ def test_meta_extraction(self): 'title': 'After storm, forecasters see smooth sailing for Thanksgiving - CNN.com', 'og': {'site_name': 'CNN','description': 'A strong storm struck much of the eastern United States on Wednesday, complicating holiday plans for many of the 43 million Americans expected to travel.', 'title': 'After storm, forecasters see smooth sailing for Thanksgiving', 'url': 'http://www.cnn.com/2013/11/27/travel/weather-thanksgiving/index.html', 'image': 'http://i2.cdn.turner.com/cnn/dam/assets/131129200805-01-weather-1128-story-top.jpg', 'type': 'article'}, 'section': 'travel', - 'author': 'Dana Ford and Tom Watkins, CNN', + 'author': 'Dana A. Ford, James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN', 'robots': 'index,follow', 'vr': {'canonical': 'http://edition.cnn.com/2013/11/27/travel/weather-thanksgiving/index.html'}, 'source': 'CNN', From f727e16e549abab5ee68afd8abaea5d3bb977338 Mon Sep 17 00:00:00 2001 From: Dave Crumbacher Date: Fri, 27 Mar 2015 22:48:54 -0400 Subject: [PATCH 06/28] Ignore case when removing duplicate authors --- newspaper/extractors.py | 6 +++--- tests/unit_tests.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 1f62a1c8..ff7e9949 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -86,9 +86,9 @@ def uniqify_list(l): seen = {} result = [] for item in l: - if item in seen: continue - seen[item] = 1 - result.append(item) + if item.lower() in seen: continue + seen[item.lower()] = 1 + result.append(item.title()) return result def parse_byline(search_str): diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 3e485b09..61c1e923 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -169,7 +169,7 @@ def test_pre_download_parse(self): @print_test def test_parse_html(self): - AUTHORS = ['Dana A. Ford', 'James S.A. Corey', 'Chien-Ming Wang', 'Tom Watkins'] + AUTHORS = ['Chien-Ming Wang', 'Dana A. Ford', 'James S.A. Corey', 'Tom Watkins'] TITLE = 'After storm, forecasters see smooth sailing for Thanksgiving' LEN_IMGS = 46 META_LANG = 'en' @@ -187,7 +187,7 @@ def test_parse_html(self): '01-weather-1128-story-top.jpg') assert self.article.top_img == TOP_IMG - assert self.article.authors == AUTHORS + assert sorted(self.article.authors) == AUTHORS assert self.article.title == TITLE assert len(self.article.imgs) == LEN_IMGS assert self.article.meta_lang == META_LANG From 4a6e004a8649e4daab529b08e1ba9635eef24e5e Mon Sep 17 00:00:00 2001 From: Dave Crumbacher Date: Sun, 29 Mar 2015 17:40:19 -0400 Subject: [PATCH 07/28] Add dc.creator as an author meta tag --- newspaper/extractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index ff7e9949..b2246e33 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -136,7 +136,7 @@ def parse_byline(search_str): # Try 1: Search popular author tags for authors ATTRS = ['name', 'rel', 'itemprop', 'class', 'id'] - VALS = ['author', 'byline'] + VALS = ['author', 'byline', 'dc.creator'] matches = [] authors = [] From aa7b618987f2eaf00c8cce71108ed1fcab3bfb96 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Wed, 13 May 2015 21:48:15 +0300 Subject: [PATCH 08/28] Handle lxml raising ValueError on node.itertext() The error: ValueError: Input object has no element: HtmlProcessingInstruction Ref #143 --- newspaper/outputformatters.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/newspaper/outputformatters.py b/newspaper/outputformatters.py index c5726717..90cb8e2b 100644 --- a/newspaper/outputformatters.py +++ b/newspaper/outputformatters.py @@ -59,7 +59,11 @@ def get_formatted(self, top_node): def convert_to_text(self): txts = [] for node in list(self.get_top_node()): - txt = self.parser.getText(node) + try: + txt = self.parser.getText(node) + except ValueError: # lxml error + txt = None + if txt: txt = HTMLParser().unescape(txt) txt_lis = innerTrim(txt).split(r'\n') From 39297e4fdeea2525b1d0192ddf8cda15a3d17077 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Mon, 11 Jan 2016 12:57:06 +0200 Subject: [PATCH 09/28] Add logging on failure to parse an lxml node Conflicts: newspaper/outputformatters.py --- newspaper/outputformatters.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/newspaper/outputformatters.py b/newspaper/outputformatters.py index 90cb8e2b..02b8d225 100644 --- a/newspaper/outputformatters.py +++ b/newspaper/outputformatters.py @@ -8,10 +8,14 @@ __copyright__ = 'Copyright 2014, Lucas Ou-Yang' from html.parser import HTMLParser +import logging from .text import innerTrim +log = logging.getLogger(__name__) + + class OutputFormatter(object): def __init__(self, config): @@ -62,6 +66,7 @@ def convert_to_text(self): try: txt = self.parser.getText(node) except ValueError: # lxml error + log.warning('Error parsing lxml node', exc_info=True) txt = None if txt: From 4118a84a09b981f29633548fda04ed95fc09a747 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Thu, 14 Jan 2016 10:33:05 +0200 Subject: [PATCH 10/28] requirements.txt - Use minimal instead of exact versions Fixes #174 Fixes #138 --- requirements.txt | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2bfec338..f81232a0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,13 @@ -beautifulsoup4==4.4.1 -Pillow==2.6.1 -PyYAML==3.11 -cssselect==0.9.1 -lxml==3.3.5 -nltk==3.0.1 -requests==2.3.0 -six==1.7.3 -feedparser==5.1.3 -tldextract==1.5.1 -feedfinder2==0.0.1 -jieba3k==0.35.1 -python-dateutil==2.4.0 +beautifulsoup4>=4.4.1 +Pillow>=2.6.1 +PyYAML>=3.11 +cssselect>=0.9.1 +lxml>=3.3.5 +nltk>=3.0.1 +requests>=2.3.0 +six>=1.7.3 +feedparser>=5.1.3 +tldextract>=1.5.1 +feedfinder2>=0.0.1 +jieba3k>=0.35.1 +python-dateutil>=2.4.0 From 346385ed516780c942a2fad99f47f6fe82314a25 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Fri, 15 Jan 2016 10:27:41 +0200 Subject: [PATCH 11/28] Add .travis.yml, test against multiply Python versions --- .travis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..a4d0022e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +language: python +python: + - "3.2" + - "3.3" + - "3.4" + - "3.5" +install: + - pip install -r requirements.txt +script: + - python tests/unit_tests.py From 0a8db7dbcdd97db380107d3cc00e13f750f2c4ee Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Fri, 15 Jan 2016 11:09:17 +0200 Subject: [PATCH 12/28] Add travis.ci badge to the readme --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 87309793..5452c9c2 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,11 @@ Newspaper3k: Article scraping & curation :target: http://badge.fury.io/py/newspaper3k.svg :alt: Latest version +.. image:: https://secure.travis-ci.org/codelucas/newspaper.png + :target: http://travis-ci.org/codelucas/newspaper/ + :alt: Build status + + Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed: "Newspaper is an amazing python library for extracting & curating articles." From 7b0ae544f50ec859b62b15fec562ef6def1be8a2 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Fri, 15 Jan 2016 11:33:23 +0200 Subject: [PATCH 13/28] unit_test - exit with status code This way CI tools / test runners can know if the tests passed or failed --- tests/unit_tests.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 61c1e923..c26794e9 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -519,7 +519,9 @@ def test_spanish_fulltext_extract(self): suite.addTest(UrlTestCase()) suite.addTest(ArticleTestCase()) suite.addTest(APITestCase()) - unittest.TextTestRunner().run(suite) + result = unittest.TextTestRunner().run(suite) + exit_code = 0 if result.wasSuccessful() else 1 + sys.exit(exit_code) # TODO: suite.addTest(SourceTestCase()) # suite.addTest(MThreadingTestCase()) From 24675e9c44a6c13ce159e34b800c9b60fdd5ffb9 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Fri, 15 Jan 2016 11:39:16 +0200 Subject: [PATCH 14/28] travis.yml - download nltk corpora before running tests --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index a4d0022e..1dc89bf0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,5 +6,6 @@ python: - "3.5" install: - pip install -r requirements.txt + - python -c "import nltk ; nltk.download('punkt')" script: - python tests/unit_tests.py From d4073f1b9d1e4f4ae746494a0aa416f05052c070 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Fri, 15 Jan 2016 11:43:54 +0200 Subject: [PATCH 15/28] Disable Python3.5 support (for now) --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1dc89bf0..571f4f4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,6 @@ python: - "3.2" - "3.3" - "3.4" - - "3.5" install: - pip install -r requirements.txt - python -c "import nltk ; nltk.download('punkt')" From 5af033376b110cb497ee8045494ee8e3bb72d375 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Fri, 15 Jan 2016 20:17:27 +0200 Subject: [PATCH 16/28] Reuse download_corpora.py in Travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 571f4f4e..ee37c324 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,6 @@ python: - "3.4" install: - pip install -r requirements.txt - - python -c "import nltk ; nltk.download('punkt')" + - python download_corpora.py script: - python tests/unit_tests.py From ed0cd3e0ceb00c85a2b70e3a0b293991c3a715e9 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Thu, 21 Jan 2016 10:59:13 +0200 Subject: [PATCH 17/28] Installation docs - change according to README * Note about Python2 and 3. * Change package name in pip install example to newspaper3k. --- docs/user_guide/install.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/user_guide/install.rst b/docs/user_guide/install.rst index 1e85af86..7d7f645a 100644 --- a/docs/user_guide/install.rst +++ b/docs/user_guide/install.rst @@ -55,9 +55,12 @@ NOTE: You will still most likely need to install the following libraries via you - lxml: ``libxml2-dev`` ``libxslt-dev`` - Python Development version: ``python-dev`` +Note that the Python3 package name is ``newspaper3k`` while our Python2 +package name is ``newspaper``. + :: - $ pip install newspaper + $ pip install newspaper3k $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python2.7 From ea631dfbf44bf969786b79c7ea2d0fabb1b17694 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Thu, 21 Jan 2016 12:00:04 +0200 Subject: [PATCH 18/28] Travis badge - switch to svg --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5452c9c2..7e1ed6d7 100644 --- a/README.rst +++ b/README.rst @@ -5,7 +5,7 @@ Newspaper3k: Article scraping & curation :target: http://badge.fury.io/py/newspaper3k.svg :alt: Latest version -.. image:: https://secure.travis-ci.org/codelucas/newspaper.png +.. image:: https://travis-ci.org/codelucas/newspaper.svg :target: http://travis-ci.org/codelucas/newspaper/ :alt: Build status From 060d4886217a6e5b95fa06e42256b8add9010d9a Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Mon, 25 Jan 2016 13:06:06 +0200 Subject: [PATCH 19/28] Freeze feedfinder2 version to fix installation * Due to: https://github.com/dfm/feedfinder2/issues/5 * Should fix the installation / build issues. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f81232a0..257ffa51 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,6 @@ requests>=2.3.0 six>=1.7.3 feedparser>=5.1.3 tldextract>=1.5.1 -feedfinder2>=0.0.1 +feedfinder2==0.0.1 jieba3k>=0.35.1 python-dateutil>=2.4.0 From 5ddaf460bd608c27477f77c3340d9621514ff997 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Mon, 25 Jan 2016 13:47:05 +0200 Subject: [PATCH 20/28] Drop support for Python 3.2 (End of life in February 2016 and causing errors with some packages) --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ee37c324..5da77035 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python python: - - "3.2" - "3.3" - "3.4" install: From 00423b4a8d6810b620f3cd9ec63e33e63258a132 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Mon, 25 Jan 2016 14:06:42 +0200 Subject: [PATCH 21/28] Test on Python 3.5 --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5da77035..fa6480b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - "3.3" - "3.4" + - "3.5" install: - pip install -r requirements.txt - python download_corpora.py From 3b982d4454df2035ec88ae2a03e124e8e43ea506 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Mon, 25 Jan 2016 20:04:37 +0200 Subject: [PATCH 22/28] Upgrade feedfinder2 version to a non broken one --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 257ffa51..4b53dde1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,6 @@ requests>=2.3.0 six>=1.7.3 feedparser>=5.1.3 tldextract>=1.5.1 -feedfinder2==0.0.1 +feedfinder2>=0.0.4 jieba3k>=0.35.1 python-dateutil>=2.4.0 From 8a36b90201bfeb4f4660f40a15c9fa6f75edb065 Mon Sep 17 00:00:00 2001 From: alon7 Date: Wed, 27 Jan 2016 01:47:35 +0200 Subject: [PATCH 23/28] Added Hebrew stop words for language support --- README.rst | 1 + docs/index.rst | 1 + docs/user_guide/quickstart.rst | 1 + newspaper/resources/text/stopwords-he.txt | 221 ++++++++++++++++++++++ 4 files changed, 224 insertions(+) create mode 100644 newspaper/resources/text/stopwords-he.txt diff --git a/README.rst b/README.rst index 87309793..f60a97b3 100644 --- a/README.rst +++ b/README.rst @@ -190,6 +190,7 @@ Features en English es Spanish fr French + he Hebrew it Italian ko Korean no Norwegian diff --git a/docs/index.rst b/docs/index.rst index 108471e7..aaf60fc6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,6 +30,7 @@ Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed. en English es Spanish fr French + he Hebrew it Italian ko Korean no Norwegian diff --git a/docs/user_guide/quickstart.rst b/docs/user_guide/quickstart.rst index d7a12c57..a0fdaab6 100644 --- a/docs/user_guide/quickstart.rst +++ b/docs/user_guide/quickstart.rst @@ -252,6 +252,7 @@ of popular news source urls.. In case you need help choosing a news source! en English es Spanish fr French + he Hebrew it Italian ko Korean no Norwegian diff --git a/newspaper/resources/text/stopwords-he.txt b/newspaper/resources/text/stopwords-he.txt new file mode 100644 index 00000000..8ac77858 --- /dev/null +++ b/newspaper/resources/text/stopwords-he.txt @@ -0,0 +1,221 @@ +אני +את +אתה +אנחנו +אתן +אתם +הם +הן +היא +הוא +שלי +שלו +שלך +שלה +שלנו +שלכם +שלכן +שלהם +שלהן +לי +לו +לה +לנו +לכם +לכן +להם +להן +אותה +אותו +זה +זאת +אלה +אלו +תחת +מתחת +מעל +בין +עם +עד +נגר +על +אל +מול +של +אצל +כמו +אחר +אותו +בלי +לפני +אחרי +מאחורי +עלי +עליו +עליה +עליך +עלינו +עליכם +לעיכן +עליהם +עליהן +כל +כולם +כולן +כך +ככה +כזה +זה +זות +אותי +אותה +אותם +אותך +אותו +אותן +אותנו +ואת +את +אתכם +אתכן +איתי +איתו +איתך +איתה +איתם +איתן +איתנו +איתכם +איתכן +יהיה +תהיה +היתי +היתה +היה +להיות +עצמי +עצמו +עצמה +עצמם +עצמן +עצמנו +עצמהם +עצמהן +מי +מה +איפה +היכן +במקום שבו +אם +לאן +למקום שבו +מקום בו +איזה +מהיכן +איך +כיצד +באיזו מידה +מתי +בשעה ש +כאשר +כש +למרות +לפני +אחרי +מאיזו סיבה +הסיבה שבגללה +למה +מדוע +לאיזו תכלית +כי +יש +אין +אך +מנין +מאין +מאיפה +יכל +יכלה +יכלו +יכול +יכולה +יכולים +יכולות +יוכלו +יוכל +מסוגל +לא +רק +אולי +אין +לאו +אי +כלל +נגד +אם +עם +אל +אלה +אלו +אף +על +מעל +מתחת +מצד +בשביל +לבין +באמצע +בתוך +דרך +מבעד +באמצעות +למעלה +למטה +מחוץ +מן +לעבר +מכאן +כאן +הנה +הרי +פה +שם +אך +ברם +שוב +אבל +מבלי +בלי +מלבד +רק +בגלל +מכיוון +עד +אשר +ואילו +למרות +אס +כמו +כפי +אז +אחרי +כן +לכן +לפיכך +מאד +עז +מעט +מעטים +במידה +שוב +יותר +מדי +גם +כן +נו +אחר +אחרת +אחרים +אחרות +אשר +או \ No newline at end of file From 10e1a0c854b96d5f76acb47ea96b6578a6155b1b Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Wed, 27 Jan 2016 17:54:31 +0200 Subject: [PATCH 24/28] Fail on error http responses Fixes #142 --- docs/user_guide/advanced.rst | 2 ++ newspaper/configuration.py | 3 +++ newspaper/network.py | 7 +++++++ 3 files changed, 12 insertions(+) diff --git a/docs/user_guide/advanced.rst b/docs/user_guide/advanced.rst index 4aac8508..3cd34358 100644 --- a/docs/user_guide/advanced.rst +++ b/docs/user_guide/advanced.rst @@ -195,6 +195,8 @@ Here is a full list of the configuration options: ``keep_article_html``, default False, "set to True if you want to preserve html of body text" +``http_success_only``, default True, "set to False to capture non 2XX responses as well" + ``MIN_WORD_COUNT``, default 300, "num of word tokens in article text" ``MIN_SENT_COUNT``, default 7, "num of sentence tokens" diff --git a/newspaper/configuration.py b/newspaper/configuration.py index 374620c7..43dd4387 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -51,6 +51,9 @@ def __init__(self): # You may keep the html of just the main article body self.keep_article_html = False + # Fail for error respones (e.g. 404 page) + self.http_success_only = True + # English is the fallback self._language = 'en' diff --git a/newspaper/network.py b/newspaper/network.py index b12fb69f..4af11a0c 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -50,14 +50,21 @@ def get_html(url, config=None, response=None): try: html = None + response = requests.get( url=url, **get_request_kwargs(timeout, useragent)) + if response.encoding != FAIL_ENCODING: html = response.text else: html = response.content + + if config.http_success_only: + response.raise_for_status() # fail if other than "ok" response + if html is None: html = '' + return html except Exception as e: log.debug('%s on %s' % (e, url)) From 6f026341cf87396505a8f04822f881be41f43903 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Wed, 27 Jan 2016 17:58:08 +0200 Subject: [PATCH 25/28] More specific exception handling --- newspaper/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newspaper/network.py b/newspaper/network.py index 4af11a0c..f127e038 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -66,7 +66,7 @@ def get_html(url, config=None, response=None): html = '' return html - except Exception as e: + except requests.exceptions.RequestException as e: log.debug('%s on %s' % (e, url)) return '' From 1e7471d0911ba8eb4661dce311625067e33880f0 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Wed, 27 Jan 2016 18:00:16 +0200 Subject: [PATCH 26/28] http_success_only for async request as well --- newspaper/network.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/newspaper/network.py b/newspaper/network.py index f127e038..917eb9b7 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -79,6 +79,7 @@ class MRequest(object): """ def __init__(self, url, config=None): self.url = url + self.config = config config = config or Configuration() self.useragent = config.browser_user_agent self.timeout = config.request_timeout @@ -88,8 +89,9 @@ def send(self): try: self.resp = requests.get(self.url, **get_request_kwargs( self.timeout, self.useragent)) - except Exception as e: - pass + if self.config.http_success_only: + self.resp.raise_for_status() + except requests.exceptions.RequestException as e: log.critical('[REQUEST FAILED] ' + str(e)) From 29d62bad332f4097dd7f717059e958770fbadbef Mon Sep 17 00:00:00 2001 From: alon7 Date: Thu, 28 Jan 2016 21:27:28 +0200 Subject: [PATCH 27/28] - Added Hebrew to utils print_available_languages --- newspaper/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/newspaper/utils.py b/newspaper/utils.py index 86f299cf..e22ef078 100644 --- a/newspaper/utils.py +++ b/newspaper/utils.py @@ -336,6 +336,7 @@ def print_available_languages(): 'en': 'English', 'es': 'Spanish', 'fr': 'French', + 'he': 'Hebrew', 'it': 'Italian', 'ko': 'Korean', 'no': 'Norwegian', From 5b8aba774f30c7b33a88c8bfb0e52aa008dd8c5d Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Sat, 30 Jan 2016 01:34:31 -0800 Subject: [PATCH 28/28] Version bump -- also test travis ci --- newspaper/version.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/newspaper/version.py b/newspaper/version.py index ece3503d..c3e2cf7e 100644 --- a/newspaper/version.py +++ b/newspaper/version.py @@ -7,5 +7,5 @@ __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' -version_info = (0, 1, 6) +version_info = (0, 1, 7) __version__ = ".".join(map(str, version_info)) diff --git a/setup.py b/setup.py index 7b184cb0..380abfa8 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( name='newspaper3k', - version='0.1.6', + version='0.1.7', description='Simplified python article discovery & extraction.', long_description=readme, author='Lucas Ou-Yang',