From 95f66da2e0e1b4ade435ac3a940dd3541eb7383d Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Thu, 15 Jan 2015 17:50:18 -0800 Subject: [PATCH 1/6] [bugfix] source `.build()` is broken, #111 --- newspaper/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newspaper/source.py b/newspaper/source.py index f4eed76b..9cc7174b 100644 --- a/newspaper/source.py +++ b/newspaper/source.py @@ -89,7 +89,7 @@ def build(self, response=None): """Encapsulates download and basic parsing with lxml. May be a good idea to split this into download() and parse() methods. """ - self.download(response) + self.download() self.parse() self.set_categories() From 73f77280aebd1794bb34d04207d52bdffd66a43a Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 29 Jan 2015 11:31:44 -0300 Subject: [PATCH 2/6] Fix incorrect log call while generating articles --- newspaper/source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/newspaper/source.py b/newspaper/source.py index 9cc7174b..c371511c 100644 --- a/newspaper/source.py +++ b/newspaper/source.py @@ -305,7 +305,8 @@ def generate_articles(self, limit=5000): """ articles = self._generate_articles() self.articles = articles[:limit] - log.debug(len(articles), 'articles generated and cutoff at', limit) + log.debug('%d articles generated and cutoff at %d', + len(articles), limit) def download_articles(self, threads=1): """Downloads all articles attached to self From 83775a73bcb3687ce2b385bbc1cd76d4b4640202 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Sun, 1 Feb 2015 00:44:33 -0800 Subject: [PATCH 3/6] Add fulltext(..), an API for direct HTML => fulltext --- newspaper/__init__.py | 4 ++-- newspaper/api.py | 25 +++++++++++++++++++++++++ tests/unit_tests.py | 6 +++++- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/newspaper/__init__.py b/newspaper/__init__.py index 53df549a..eaef8855 100644 --- a/newspaper/__init__.py +++ b/newspaper/__init__.py @@ -8,8 +8,8 @@ __copyright__ = 'Copyright 2014, Lucas Ou-Yang' from .article import Article, ArticleException -from .api import (build, build_article, hot, languages, popular_urls, - NewsPool, Configuration as Config) +from .api import (build, build_article, fulltext, hot, languages, + popular_urls, NewsPool, Configuration as Config) from .source import Source from .version import __version__ diff --git a/newspaper/api.py b/newspaper/api.py index c308d1f4..f69338f9 100644 --- a/newspaper/api.py +++ b/newspaper/api.py @@ -67,3 +67,28 @@ def hot(): except Exception as e: print('ERR hot terms failed!', str(e)) return None + + +def fulltext(html, language='en'): + """Takes article HTML string input and outputs the fulltext + Input string is decoded via UnicodeDammit if needed + """ + from .cleaners import DocumentCleaner + from .configuration import Configuration + from .extractors import ContentExtractor + from .outputformatters import OutputFormatter + + config = Configuration() + config.language = language + + extractor = ContentExtractor(config) + document_cleaner = DocumentCleaner(config) + output_formatter = OutputFormatter(config) + + doc = config.get_parser().fromstring(html) + doc = document_cleaner.clean(doc) + + top_node = extractor.calculate_best_node(doc) + top_node = extractor.post_cleanup(top_node) + text, article_html = output_formatter.get_formatted(top_node) + return text diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 409ff3ab..39ca5164 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -23,7 +23,7 @@ import newspaper from newspaper import ( - Article, Source, ArticleException, news_pool) + Article, fulltext, Source, ArticleException, news_pool) from newspaper.configuration import Configuration from newspaper.urls import get_domain @@ -172,6 +172,7 @@ def test_parse_html(self): text = mock_resource_with('cnn', 'txt') assert self.article.text == text + assert fulltext(self.article.html) == text # NOTE: top_img extraction requires an internet connection # unlike the rest of this test file @@ -469,6 +470,7 @@ def test_chinese_fulltext_extract(self): article.parse() text = mock_resource_with('chinese', 'txt') assert article.text == text + assert fulltext(article.html, 'zh') == text @print_test def test_arabic_fulltext_extract(self): @@ -481,6 +483,7 @@ def test_arabic_fulltext_extract(self): assert article.meta_lang == 'ar' text = mock_resource_with('arabic', 'txt') assert article.text == text + assert fulltext(article.html, 'ar') == text @print_test def test_spanish_fulltext_extract(self): @@ -492,6 +495,7 @@ def test_spanish_fulltext_extract(self): article.parse() text = mock_resource_with('spanish', 'txt') assert article.text == text + assert fulltext(article.html, 'es') == text if __name__ == '__main__': From 30eab17a49eb484420c65f347aa728b2515969ae Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Wed, 4 Feb 2015 03:09:50 -0800 Subject: [PATCH 4/6] Add publishing date extraction --- README.rst | 19 ++++++++------- newspaper/article.py | 7 +++--- newspaper/extractors.py | 53 +++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + tests/unit_tests.py | 14 ++++++++--- 5 files changed, 80 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index bcfb6507..3bc6aa11 100644 --- a/README.rst +++ b/README.rst @@ -43,6 +43,9 @@ A Glance: >>> article.authors ['Leigh Ann Caldwell', 'John Honway'] + >>> article.publish_date + datetime.datetime(2013, 12, 30, 0, 0) + >>> article.text 'Washington (CNN) -- Not everyone subscribes to a New Year's resolution...' @@ -70,16 +73,16 @@ A Glance: >>> for article in cnn_paper.articles: >>> print(article.url) - 'http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/' - 'http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html' + http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/ + http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html ... >>> for category in cnn_paper.category_urls(): >>> print(category) - 'http://lifestyle.cnn.com' - 'http://cnn.com/world' - 'http://tech.cnn.com' + http://lifestyle.cnn.com + http://cnn.com/world + http://tech.cnn.com ... .. code-block:: pycon @@ -125,9 +128,9 @@ If you are certain that an *entire* news source is in one language, **go ahead a >>> for category in sina_paper.category_urls(): >>> print(category) - 'http://health.sina.com.cn' - 'http://eladies.sina.com.cn' - 'http://english.sina.com' + http://health.sina.com.cn + http://eladies.sina.com.cn + http://english.sina.com ... >>> article = sina_paper.articles[0] diff --git a/newspaper/article.py b/newspaper/article.py index 99df1473..5fe75d88 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -82,8 +82,7 @@ def __init__(self, url, title='', source_url='', config=None, **kwargs): # List of authors who have published the article, via parse() self.authors = [] - # TODO: Date of when this article was published - self.published_date = '' + self.publish_date = '' # Summary generated from the article's body txt self.summary = '' @@ -204,7 +203,9 @@ def parse(self): meta_data = self.extractor.get_meta_data(self.clean_doc) self.set_meta_data(meta_data) - # TODO self.publish_date = ... + self.publish_date = self.extractor.get_publishing_date( + self.url, + self.clean_doc) # Before any computations on the body, clean DOM object self.doc = document_cleaner.clean(self.doc) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 253742b3..f91229d2 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -13,6 +13,7 @@ from collections import defaultdict import copy +from dateutil.parser import parse as date_parser import logging import re import urllib.parse @@ -162,6 +163,58 @@ def parse_byline(search_str): # return [] # Failed to find anything # return authors + def get_publishing_date(self, url, doc): + """3 strategies for publishing date extraction. The strategies + are descending in accuracy and the next strategy is only + attempted if a preferred one fails. + + 1. Pubdate from URL + 2. Pubdate from metadata + 3. Raw regex searches in the HTML + added heuristics + """ + + def parse_date_str(date_str): + try: + datetime_obj = date_parser(date_str) + return datetime_obj + except: + # near all parse failures are due to URL dates without a day + # specifier, e.g. /2014/04/ + return None + + date_match = re.search(urls.DATE_REGEX, url) + if date_match: + date_str = date_match.group(0) + datetime_obj = parse_date_str(date_str) + if datetime_obj: + return datetime_obj + + PUBLISH_DATE_TAGS = [ + {'attribute': 'property', 'value': 'rnews:datePublished', 'content': 'content'}, + {'attribute': 'property', 'value': 'article:published_time', 'content': 'content'}, + {'attribute': 'name', 'value': 'OriginalPublicationDate', 'content': 'content'}, + {'attribute': 'itemprop', 'value': 'datePublished', 'content': 'datetime'}, + {'attribute': 'property', 'value': 'og:published_time', 'content': 'content'}, + {'attribute': 'name', 'value': 'article_date_original', 'content': 'content'}, + {'attribute': 'name', 'value': 'publication_date', 'content': 'content'}, + {'attribute': 'name', 'value': 'sailthru.date', 'content': 'content'}, + {'attribute': 'name', 'value': 'PublishDate', 'content': 'content'}, + ] + for known_meta_tag in PUBLISH_DATE_TAGS: + meta_tags = self.parser.getElementsByTag( + doc, + attr=known_meta_tag['attribute'], + value=known_meta_tag['value']) + if meta_tags: + date_str = self.parser.getAttribute( + meta_tags[0], + known_meta_tag['content']) + datetime_obj = parse_date_str(date_str) + if datetime_obj: + return datetime_obj + + return None + def get_title(self, doc): """Fetch the article title and analyze it """ diff --git a/requirements.txt b/requirements.txt index da49828d..9f7bc216 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ feedparser==5.1.3 tldextract==1.5.1 feedfinder2==0.0.1 jieba3k==0.35.1 +python-dateutil==2.4.0 diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 39ca5164..e17b241b 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -82,7 +82,8 @@ def runTest(self): with open(URLS_FILE, 'r') as f: urls = [d.strip() for d in f.readlines() if d.strip()] - failed = 0 + fulltext_failed = 0 + pubdates_failed = 0 for url in urls: domain = get_base_domain(url) if domain in domain_counters: @@ -96,6 +97,8 @@ def runTest(self): a = Article(url) a.download(html) a.parse() + if a.publish_date is None: + pubdates_failed += 1 except Exception: print('<< URL: %s parse ERROR >>' % url) traceback.print_exc() @@ -107,13 +110,17 @@ def runTest(self): # `correct_text` holds the reason of failure if failure print('%s -- %s -- %s' % ('Fulltext failed', res_filename, correct_text.strip())) - failed += 1 + fulltext_failed += 1 # TODO: assert statements are commented out for full-text # extraction tests because we are constantly tweaking the # algorithm and improving # assert a.text == correct_text print('%s fulltext extractions failed out of %s' % - (failed, len(urls))) + (fulltext_failed, len(urls))) + print('%s pubdate extractions failed out of %s' % + (pubdates_failed, len(urls))) + assert pubdates_failed == 47 + assert fulltext_failed == 20 class ArticleTestCase(unittest.TestCase): @@ -184,6 +191,7 @@ def test_parse_html(self): assert self.article.title == TITLE assert len(self.article.imgs) == LEN_IMGS assert self.article.meta_lang == META_LANG + assert str(self.article.publish_date) == '2013-11-27 00:00:00' @print_test def test_meta_type_extraction(self): From 9439e9ddca5a16da5bbf7df19edf72c81c7fa3b1 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Wed, 4 Feb 2015 03:12:16 -0800 Subject: [PATCH 5/6] Add parse.ly to README, add fulltext API --- README.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 3bc6aa11..92bd2ba0 100644 --- a/README.rst +++ b/README.rst @@ -85,14 +85,19 @@ A Glance: http://tech.cnn.com ... -.. code-block:: pycon - >>> cnn_article = cnn_paper.articles[0] >>> cnn_article.download() >>> cnn_article.parse() >>> cnn_article.nlp() ... +.. code-block:: pycon + + >>> from newspaper import fulltext + + >>> html = requests.get(...).text + >>> text = fulltext(html) + Newspaper has *seamless* language extraction and detection. If no language is specified, Newspaper will attempt to auto detect a language. @@ -293,6 +298,8 @@ LICENSE ------- Authored and maintained by `Lucas Ou-Yang`_. +`Parse.ly`_ sponsored some work on newspaper, specifically focused on +automatic extraction. Newspaper uses a lot of `python-goose's`_ parsing code. View their license `here`_. @@ -308,3 +315,4 @@ to talk about the future of this library and news extraction in general! .. _`The Documentation`: http://newspaper.readthedocs.org .. _`lxml`: http://lxml.de/ .. _`requests`: https://github.com/kennethreitz/requests +.. _`Parse.ly`: http://parse.ly From baef15a1ecf2e89c3ad92daf8ab45744532c145e Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Wed, 4 Feb 2015 03:19:21 -0800 Subject: [PATCH 6/6] Version bump: 0.1.3 => 0.1.4 --- README.rst | 1 + newspaper/version.py | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 92bd2ba0..a3b7406a 100644 --- a/README.rst +++ b/README.rst @@ -298,6 +298,7 @@ LICENSE ------- Authored and maintained by `Lucas Ou-Yang`_. + `Parse.ly`_ sponsored some work on newspaper, specifically focused on automatic extraction. diff --git a/newspaper/version.py b/newspaper/version.py index 886da7bf..23245d84 100644 --- a/newspaper/version.py +++ b/newspaper/version.py @@ -7,5 +7,5 @@ __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' -version_info = (0, 1, 3) +version_info = (0, 1, 4) __version__ = ".".join(map(str, version_info)) diff --git a/setup.py b/setup.py index 1a8a860e..6433e830 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( name='newspaper3k', - version='0.1.3', + version='0.1.4', description='Simplified python article discovery & extraction.', long_description=readme, author='Lucas Ou-Yang',