diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..fa6480b6 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,10 @@ +language: python +python: + - "3.3" + - "3.4" + - "3.5" +install: + - pip install -r requirements.txt + - python download_corpora.py +script: + - python tests/unit_tests.py diff --git a/README.rst b/README.rst index 87309793..94554add 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,11 @@ Newspaper3k: Article scraping & curation :target: http://badge.fury.io/py/newspaper3k.svg :alt: Latest version +.. image:: https://travis-ci.org/codelucas/newspaper.svg + :target: http://travis-ci.org/codelucas/newspaper/ + :alt: Build status + + Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed: "Newspaper is an amazing python library for extracting & curating articles." @@ -190,6 +195,7 @@ Features en English es Spanish fr French + he Hebrew it Italian ko Korean no Norwegian diff --git a/docs/index.rst b/docs/index.rst index 108471e7..aaf60fc6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,6 +30,7 @@ Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed. en English es Spanish fr French + he Hebrew it Italian ko Korean no Norwegian diff --git a/docs/user_guide/advanced.rst b/docs/user_guide/advanced.rst index 4aac8508..3cd34358 100644 --- a/docs/user_guide/advanced.rst +++ b/docs/user_guide/advanced.rst @@ -195,6 +195,8 @@ Here is a full list of the configuration options: ``keep_article_html``, default False, "set to True if you want to preserve html of body text" +``http_success_only``, default True, "set to False to capture non 2XX responses as well" + ``MIN_WORD_COUNT``, default 300, "num of word tokens in article text" ``MIN_SENT_COUNT``, default 7, "num of sentence tokens" diff --git a/docs/user_guide/install.rst b/docs/user_guide/install.rst index 1e85af86..7d7f645a 100644 --- a/docs/user_guide/install.rst +++ b/docs/user_guide/install.rst @@ -55,9 +55,12 @@ NOTE: You will still most likely need to install the following libraries via you - lxml: ``libxml2-dev`` ``libxslt-dev`` - Python Development version: ``python-dev`` +Note that the Python3 package name is ``newspaper3k`` while our Python2 +package name is ``newspaper``. + :: - $ pip install newspaper + $ pip install newspaper3k $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python2.7 diff --git a/docs/user_guide/quickstart.rst b/docs/user_guide/quickstart.rst index d7a12c57..a0fdaab6 100644 --- a/docs/user_guide/quickstart.rst +++ b/docs/user_guide/quickstart.rst @@ -252,6 +252,7 @@ of popular news source urls.. In case you need help choosing a news source! en English es Spanish fr French + he Hebrew it Italian ko Korean no Norwegian diff --git a/newspaper/configuration.py b/newspaper/configuration.py index 374620c7..43dd4387 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -51,6 +51,9 @@ def __init__(self): # You may keep the html of just the main article body self.keep_article_html = False + # Fail for error respones (e.g. 404 page) + self.http_success_only = True + # English is the fallback self._language = 'en' diff --git a/newspaper/extractors.py b/newspaper/extractors.py index ec25b7d3..b2246e33 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -79,6 +79,18 @@ def get_authors(self, doc): def contains_digits(d): return bool(_digits.search(d)) + def uniqify_list(l): + """Remove duplicates from provided list but maintain original order. + Derived from http://www.peterbe.com/plog/uniqifiers-benchmark + """ + seen = {} + result = [] + for item in l: + if item.lower() in seen: continue + seen[item.lower()] = 1 + result.append(item.title()) + return result + def parse_byline(search_str): """Takes a candidate line of html or text and extracts out the name(s) in list form @@ -95,21 +107,19 @@ def parse_byline(search_str): search_str = search_str.strip() # Chunk the line by non alphanumeric tokens (few name exceptions) - # >>> re.split("[^\w\'\-]", "Lucas Ou, Dean O'Brian and Ronald") - # ['Lucas Ou', '', 'Dean O'Brian', 'and', 'Ronald'] - name_tokens = re.split("[^\w\'\-]", search_str) + # >>> re.split("[^\w\'\-\.]", "Tyler G. Jones, Lucas Ou, Dean O'Brian and Ronald") + # ['Tyler', 'G.', 'Jones', '', 'Lucas', 'Ou', '', 'Dean', "O'Brian", 'and', 'Ronald'] + name_tokens = re.split("[^\w\'\-\.]", search_str) name_tokens = [s.strip() for s in name_tokens] _authors = [] # List of first, last name tokens curname = [] - DELIM = ['and', ''] + DELIM = ['and', ',', ''] for token in name_tokens: if token in DELIM: - # should we allow middle names? - valid_name = (len(curname) == 2) - if valid_name: + if len(curname) > 0: _authors.append(' '.join(curname)) curname = [] @@ -126,9 +136,9 @@ def parse_byline(search_str): # Try 1: Search popular author tags for authors ATTRS = ['name', 'rel', 'itemprop', 'class', 'id'] - VALS = ['author', 'byline'] + VALS = ['author', 'byline', 'dc.creator'] matches = [] - _authors, authors = [], [] + authors = [] for attr in ATTRS: for val in VALS: @@ -145,13 +155,9 @@ def parse_byline(search_str): else: content = match.text or '' if len(content) > 0: - _authors.extend(parse_byline(content)) + authors.extend(parse_byline(content)) - uniq = list(set([s.lower() for s in _authors])) - for name in uniq: - names = [w.capitalize() for w in name.split(' ')] - authors.append(' '.join(names)) - return authors or [] + return uniqify_list(authors) # TODO Method 2: Search raw html for a by-line # match = re.search('By[\: ].*\\n|From[\: ].*\\n', html) diff --git a/newspaper/network.py b/newspaper/network.py index b12fb69f..917eb9b7 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -50,16 +50,23 @@ def get_html(url, config=None, response=None): try: html = None + response = requests.get( url=url, **get_request_kwargs(timeout, useragent)) + if response.encoding != FAIL_ENCODING: html = response.text else: html = response.content + + if config.http_success_only: + response.raise_for_status() # fail if other than "ok" response + if html is None: html = '' + return html - except Exception as e: + except requests.exceptions.RequestException as e: log.debug('%s on %s' % (e, url)) return '' @@ -72,6 +79,7 @@ class MRequest(object): """ def __init__(self, url, config=None): self.url = url + self.config = config config = config or Configuration() self.useragent = config.browser_user_agent self.timeout = config.request_timeout @@ -81,8 +89,9 @@ def send(self): try: self.resp = requests.get(self.url, **get_request_kwargs( self.timeout, self.useragent)) - except Exception as e: - pass + if self.config.http_success_only: + self.resp.raise_for_status() + except requests.exceptions.RequestException as e: log.critical('[REQUEST FAILED] ' + str(e)) diff --git a/newspaper/outputformatters.py b/newspaper/outputformatters.py index c5726717..02b8d225 100644 --- a/newspaper/outputformatters.py +++ b/newspaper/outputformatters.py @@ -8,10 +8,14 @@ __copyright__ = 'Copyright 2014, Lucas Ou-Yang' from html.parser import HTMLParser +import logging from .text import innerTrim +log = logging.getLogger(__name__) + + class OutputFormatter(object): def __init__(self, config): @@ -59,7 +63,12 @@ def get_formatted(self, top_node): def convert_to_text(self): txts = [] for node in list(self.get_top_node()): - txt = self.parser.getText(node) + try: + txt = self.parser.getText(node) + except ValueError: # lxml error + log.warning('Error parsing lxml node', exc_info=True) + txt = None + if txt: txt = HTMLParser().unescape(txt) txt_lis = innerTrim(txt).split(r'\n') diff --git a/newspaper/resources/text/stopwords-he.txt b/newspaper/resources/text/stopwords-he.txt new file mode 100644 index 00000000..8ac77858 --- /dev/null +++ b/newspaper/resources/text/stopwords-he.txt @@ -0,0 +1,221 @@ +אני +את +אתה +אנחנו +אתן +אתם +הם +הן +היא +הוא +שלי +שלו +שלך +שלה +שלנו +שלכם +שלכן +שלהם +שלהן +לי +לו +לה +לנו +לכם +לכן +להם +להן +אותה +אותו +זה +זאת +אלה +אלו +תחת +מתחת +מעל +בין +עם +עד +נגר +על +אל +מול +של +אצל +כמו +אחר +אותו +בלי +לפני +אחרי +מאחורי +עלי +עליו +עליה +עליך +עלינו +עליכם +לעיכן +עליהם +עליהן +כל +כולם +כולן +כך +ככה +כזה +זה +זות +אותי +אותה +אותם +אותך +אותו +אותן +אותנו +ואת +את +אתכם +אתכן +איתי +איתו +איתך +איתה +איתם +איתן +איתנו +איתכם +איתכן +יהיה +תהיה +היתי +היתה +היה +להיות +עצמי +עצמו +עצמה +עצמם +עצמן +עצמנו +עצמהם +עצמהן +מי +מה +איפה +היכן +במקום שבו +אם +לאן +למקום שבו +מקום בו +איזה +מהיכן +איך +כיצד +באיזו מידה +מתי +בשעה ש +כאשר +כש +למרות +לפני +אחרי +מאיזו סיבה +הסיבה שבגללה +למה +מדוע +לאיזו תכלית +כי +יש +אין +אך +מנין +מאין +מאיפה +יכל +יכלה +יכלו +יכול +יכולה +יכולים +יכולות +יוכלו +יוכל +מסוגל +לא +רק +אולי +אין +לאו +אי +כלל +נגד +אם +עם +אל +אלה +אלו +אף +על +מעל +מתחת +מצד +בשביל +לבין +באמצע +בתוך +דרך +מבעד +באמצעות +למעלה +למטה +מחוץ +מן +לעבר +מכאן +כאן +הנה +הרי +פה +שם +אך +ברם +שוב +אבל +מבלי +בלי +מלבד +רק +בגלל +מכיוון +עד +אשר +ואילו +למרות +אס +כמו +כפי +אז +אחרי +כן +לכן +לפיכך +מאד +עז +מעט +מעטים +במידה +שוב +יותר +מדי +גם +כן +נו +אחר +אחרת +אחרים +אחרות +אשר +או \ No newline at end of file diff --git a/newspaper/utils.py b/newspaper/utils.py index 86f299cf..e22ef078 100644 --- a/newspaper/utils.py +++ b/newspaper/utils.py @@ -336,6 +336,7 @@ def print_available_languages(): 'en': 'English', 'es': 'Spanish', 'fr': 'French', + 'he': 'Hebrew', 'it': 'Italian', 'ko': 'Korean', 'no': 'Norwegian', diff --git a/newspaper/version.py b/newspaper/version.py index ece3503d..c3e2cf7e 100644 --- a/newspaper/version.py +++ b/newspaper/version.py @@ -7,5 +7,5 @@ __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' -version_info = (0, 1, 6) +version_info = (0, 1, 7) __version__ = ".".join(map(str, version_info)) diff --git a/requirements.txt b/requirements.txt index 2bfec338..4b53dde1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,13 @@ -beautifulsoup4==4.4.1 -Pillow==2.6.1 -PyYAML==3.11 -cssselect==0.9.1 -lxml==3.3.5 -nltk==3.0.1 -requests==2.3.0 -six==1.7.3 -feedparser==5.1.3 -tldextract==1.5.1 -feedfinder2==0.0.1 -jieba3k==0.35.1 -python-dateutil==2.4.0 +beautifulsoup4>=4.4.1 +Pillow>=2.6.1 +PyYAML>=3.11 +cssselect>=0.9.1 +lxml>=3.3.5 +nltk>=3.0.1 +requests>=2.3.0 +six>=1.7.3 +feedparser>=5.1.3 +tldextract>=1.5.1 +feedfinder2>=0.0.4 +jieba3k>=0.35.1 +python-dateutil>=2.4.0 diff --git a/setup.py b/setup.py index 7b184cb0..380abfa8 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( name='newspaper3k', - version='0.1.6', + version='0.1.7', description='Simplified python article discovery & extraction.', long_description=readme, author='Lucas Ou-Yang', diff --git a/tests/data/html/cnn_article.html b/tests/data/html/cnn_article.html index eaf5b227..53c1629b 100644 --- a/tests/data/html/cnn_article.html +++ b/tests/data/html/cnn_article.html @@ -13,7 +13,7 @@ - + @@ -68,7 +68,7 @@ cnnBrandingValue="default"; cnnPartnerValue=""; cnnOmniBranding="", -cnnAuthor="Dana Ford and Tom Watkins, CNN", +cnnAuthor="Dana A. Ford, James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN", disqus_category_id=207582, disqus_identifier="/2013/11/27/travel/weather-thanksgiving/index.html", disqus_title="After storm, forecasters see smooth sailing for Thanksgiving", @@ -87,7 +87,7 @@ business: { cnn: { page: { -author: "Dana Ford and Tom Watkins, CNN", +author: "Dana A. Ford, James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN", broadcast_franchise: "", video_embed_count: "4", publish_date: "2013/11/27", @@ -294,7 +294,7 @@

After storm, forecasters see smooth sailing for Thanksgiving

-
By Dana Ford and Tom Watkins, CNN
+
By Dana A. Ford , James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN
November 28, 2013 -- Updated 0203 GMT (1003 HKT)
@@ -1022,7 +1022,7 @@

cnnsocial.share.setconfig(cnn_shareconfig); $j(document).ready(function () { 'use strict'; -loadChartbeat("travel", "Dana Ford and Tom Watkins, CNN"); +loadChartbeat("travel", "Dana A. Ford, James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN"); CNN.initFlipperTicker(); /* initialize cnnsocial */ cnnsocial.init(); @@ -1041,4 +1041,4 @@

- \ No newline at end of file + diff --git a/tests/unit_tests.py b/tests/unit_tests.py index e17b241b..c26794e9 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -158,7 +158,7 @@ def test_url(self): def test_download_html(self): html = mock_resource_with('cnn_article', 'html') self.article.download(html) - assert len(self.article.html) == 75175 + assert len(self.article.html) == 75406 @print_test def test_pre_download_parse(self): @@ -169,7 +169,7 @@ def test_pre_download_parse(self): @print_test def test_parse_html(self): - AUTHORS = ['Dana Ford', 'Tom Watkins'] + AUTHORS = ['Chien-Ming Wang', 'Dana A. Ford', 'James S.A. Corey', 'Tom Watkins'] TITLE = 'After storm, forecasters see smooth sailing for Thanksgiving' LEN_IMGS = 46 META_LANG = 'en' @@ -209,7 +209,7 @@ def test_meta_extraction(self): 'title': 'After storm, forecasters see smooth sailing for Thanksgiving - CNN.com', 'og': {'site_name': 'CNN','description': 'A strong storm struck much of the eastern United States on Wednesday, complicating holiday plans for many of the 43 million Americans expected to travel.', 'title': 'After storm, forecasters see smooth sailing for Thanksgiving', 'url': 'http://www.cnn.com/2013/11/27/travel/weather-thanksgiving/index.html', 'image': 'http://i2.cdn.turner.com/cnn/dam/assets/131129200805-01-weather-1128-story-top.jpg', 'type': 'article'}, 'section': 'travel', - 'author': 'Dana Ford and Tom Watkins, CNN', + 'author': 'Dana A. Ford, James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN', 'robots': 'index,follow', 'vr': {'canonical': 'http://edition.cnn.com/2013/11/27/travel/weather-thanksgiving/index.html'}, 'source': 'CNN', @@ -519,7 +519,9 @@ def test_spanish_fulltext_extract(self): suite.addTest(UrlTestCase()) suite.addTest(ArticleTestCase()) suite.addTest(APITestCase()) - unittest.TextTestRunner().run(suite) + result = unittest.TextTestRunner().run(suite) + exit_code = 0 if result.wasSuccessful() else 1 + sys.exit(exit_code) # TODO: suite.addTest(SourceTestCase()) # suite.addTest(MThreadingTestCase())