diff --git a/README.rst b/README.rst index bcfb6507..a3b7406a 100644 --- a/README.rst +++ b/README.rst @@ -43,6 +43,9 @@ A Glance: >>> article.authors ['Leigh Ann Caldwell', 'John Honway'] + >>> article.publish_date + datetime.datetime(2013, 12, 30, 0, 0) + >>> article.text 'Washington (CNN) -- Not everyone subscribes to a New Year's resolution...' @@ -70,26 +73,31 @@ A Glance: >>> for article in cnn_paper.articles: >>> print(article.url) - 'http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/' - 'http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html' + http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/ + http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html ... >>> for category in cnn_paper.category_urls(): >>> print(category) - 'http://lifestyle.cnn.com' - 'http://cnn.com/world' - 'http://tech.cnn.com' + http://lifestyle.cnn.com + http://cnn.com/world + http://tech.cnn.com ... -.. code-block:: pycon - >>> cnn_article = cnn_paper.articles[0] >>> cnn_article.download() >>> cnn_article.parse() >>> cnn_article.nlp() ... +.. code-block:: pycon + + >>> from newspaper import fulltext + + >>> html = requests.get(...).text + >>> text = fulltext(html) + Newspaper has *seamless* language extraction and detection. If no language is specified, Newspaper will attempt to auto detect a language. @@ -125,9 +133,9 @@ If you are certain that an *entire* news source is in one language, **go ahead a >>> for category in sina_paper.category_urls(): >>> print(category) - 'http://health.sina.com.cn' - 'http://eladies.sina.com.cn' - 'http://english.sina.com' + http://health.sina.com.cn + http://eladies.sina.com.cn + http://english.sina.com ... >>> article = sina_paper.articles[0] @@ -291,6 +299,9 @@ LICENSE Authored and maintained by `Lucas Ou-Yang`_. +`Parse.ly`_ sponsored some work on newspaper, specifically focused on +automatic extraction. + Newspaper uses a lot of `python-goose's`_ parsing code. View their license `here`_. Please feel free to `email & contact me`_ if you run into issues or just would like @@ -305,3 +316,4 @@ to talk about the future of this library and news extraction in general! .. _`The Documentation`: http://newspaper.readthedocs.org .. _`lxml`: http://lxml.de/ .. _`requests`: https://github.com/kennethreitz/requests +.. _`Parse.ly`: http://parse.ly diff --git a/newspaper/__init__.py b/newspaper/__init__.py index 53df549a..eaef8855 100644 --- a/newspaper/__init__.py +++ b/newspaper/__init__.py @@ -8,8 +8,8 @@ __copyright__ = 'Copyright 2014, Lucas Ou-Yang' from .article import Article, ArticleException -from .api import (build, build_article, hot, languages, popular_urls, - NewsPool, Configuration as Config) +from .api import (build, build_article, fulltext, hot, languages, + popular_urls, NewsPool, Configuration as Config) from .source import Source from .version import __version__ diff --git a/newspaper/api.py b/newspaper/api.py index c308d1f4..f69338f9 100644 --- a/newspaper/api.py +++ b/newspaper/api.py @@ -67,3 +67,28 @@ def hot(): except Exception as e: print('ERR hot terms failed!', str(e)) return None + + +def fulltext(html, language='en'): + """Takes article HTML string input and outputs the fulltext + Input string is decoded via UnicodeDammit if needed + """ + from .cleaners import DocumentCleaner + from .configuration import Configuration + from .extractors import ContentExtractor + from .outputformatters import OutputFormatter + + config = Configuration() + config.language = language + + extractor = ContentExtractor(config) + document_cleaner = DocumentCleaner(config) + output_formatter = OutputFormatter(config) + + doc = config.get_parser().fromstring(html) + doc = document_cleaner.clean(doc) + + top_node = extractor.calculate_best_node(doc) + top_node = extractor.post_cleanup(top_node) + text, article_html = output_formatter.get_formatted(top_node) + return text diff --git a/newspaper/article.py b/newspaper/article.py index 99df1473..5fe75d88 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -82,8 +82,7 @@ def __init__(self, url, title='', source_url='', config=None, **kwargs): # List of authors who have published the article, via parse() self.authors = [] - # TODO: Date of when this article was published - self.published_date = '' + self.publish_date = '' # Summary generated from the article's body txt self.summary = '' @@ -204,7 +203,9 @@ def parse(self): meta_data = self.extractor.get_meta_data(self.clean_doc) self.set_meta_data(meta_data) - # TODO self.publish_date = ... + self.publish_date = self.extractor.get_publishing_date( + self.url, + self.clean_doc) # Before any computations on the body, clean DOM object self.doc = document_cleaner.clean(self.doc) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 253742b3..f91229d2 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -13,6 +13,7 @@ from collections import defaultdict import copy +from dateutil.parser import parse as date_parser import logging import re import urllib.parse @@ -162,6 +163,58 @@ def parse_byline(search_str): # return [] # Failed to find anything # return authors + def get_publishing_date(self, url, doc): + """3 strategies for publishing date extraction. The strategies + are descending in accuracy and the next strategy is only + attempted if a preferred one fails. + + 1. Pubdate from URL + 2. Pubdate from metadata + 3. Raw regex searches in the HTML + added heuristics + """ + + def parse_date_str(date_str): + try: + datetime_obj = date_parser(date_str) + return datetime_obj + except: + # near all parse failures are due to URL dates without a day + # specifier, e.g. /2014/04/ + return None + + date_match = re.search(urls.DATE_REGEX, url) + if date_match: + date_str = date_match.group(0) + datetime_obj = parse_date_str(date_str) + if datetime_obj: + return datetime_obj + + PUBLISH_DATE_TAGS = [ + {'attribute': 'property', 'value': 'rnews:datePublished', 'content': 'content'}, + {'attribute': 'property', 'value': 'article:published_time', 'content': 'content'}, + {'attribute': 'name', 'value': 'OriginalPublicationDate', 'content': 'content'}, + {'attribute': 'itemprop', 'value': 'datePublished', 'content': 'datetime'}, + {'attribute': 'property', 'value': 'og:published_time', 'content': 'content'}, + {'attribute': 'name', 'value': 'article_date_original', 'content': 'content'}, + {'attribute': 'name', 'value': 'publication_date', 'content': 'content'}, + {'attribute': 'name', 'value': 'sailthru.date', 'content': 'content'}, + {'attribute': 'name', 'value': 'PublishDate', 'content': 'content'}, + ] + for known_meta_tag in PUBLISH_DATE_TAGS: + meta_tags = self.parser.getElementsByTag( + doc, + attr=known_meta_tag['attribute'], + value=known_meta_tag['value']) + if meta_tags: + date_str = self.parser.getAttribute( + meta_tags[0], + known_meta_tag['content']) + datetime_obj = parse_date_str(date_str) + if datetime_obj: + return datetime_obj + + return None + def get_title(self, doc): """Fetch the article title and analyze it """ diff --git a/newspaper/source.py b/newspaper/source.py index f4eed76b..c371511c 100644 --- a/newspaper/source.py +++ b/newspaper/source.py @@ -89,7 +89,7 @@ def build(self, response=None): """Encapsulates download and basic parsing with lxml. May be a good idea to split this into download() and parse() methods. """ - self.download(response) + self.download() self.parse() self.set_categories() @@ -305,7 +305,8 @@ def generate_articles(self, limit=5000): """ articles = self._generate_articles() self.articles = articles[:limit] - log.debug(len(articles), 'articles generated and cutoff at', limit) + log.debug('%d articles generated and cutoff at %d', + len(articles), limit) def download_articles(self, threads=1): """Downloads all articles attached to self diff --git a/newspaper/version.py b/newspaper/version.py index 886da7bf..23245d84 100644 --- a/newspaper/version.py +++ b/newspaper/version.py @@ -7,5 +7,5 @@ __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' -version_info = (0, 1, 3) +version_info = (0, 1, 4) __version__ = ".".join(map(str, version_info)) diff --git a/requirements.txt b/requirements.txt index da49828d..9f7bc216 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ feedparser==5.1.3 tldextract==1.5.1 feedfinder2==0.0.1 jieba3k==0.35.1 +python-dateutil==2.4.0 diff --git a/setup.py b/setup.py index 1a8a860e..6433e830 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( name='newspaper3k', - version='0.1.3', + version='0.1.4', description='Simplified python article discovery & extraction.', long_description=readme, author='Lucas Ou-Yang', diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 409ff3ab..e17b241b 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -23,7 +23,7 @@ import newspaper from newspaper import ( - Article, Source, ArticleException, news_pool) + Article, fulltext, Source, ArticleException, news_pool) from newspaper.configuration import Configuration from newspaper.urls import get_domain @@ -82,7 +82,8 @@ def runTest(self): with open(URLS_FILE, 'r') as f: urls = [d.strip() for d in f.readlines() if d.strip()] - failed = 0 + fulltext_failed = 0 + pubdates_failed = 0 for url in urls: domain = get_base_domain(url) if domain in domain_counters: @@ -96,6 +97,8 @@ def runTest(self): a = Article(url) a.download(html) a.parse() + if a.publish_date is None: + pubdates_failed += 1 except Exception: print('<< URL: %s parse ERROR >>' % url) traceback.print_exc() @@ -107,13 +110,17 @@ def runTest(self): # `correct_text` holds the reason of failure if failure print('%s -- %s -- %s' % ('Fulltext failed', res_filename, correct_text.strip())) - failed += 1 + fulltext_failed += 1 # TODO: assert statements are commented out for full-text # extraction tests because we are constantly tweaking the # algorithm and improving # assert a.text == correct_text print('%s fulltext extractions failed out of %s' % - (failed, len(urls))) + (fulltext_failed, len(urls))) + print('%s pubdate extractions failed out of %s' % + (pubdates_failed, len(urls))) + assert pubdates_failed == 47 + assert fulltext_failed == 20 class ArticleTestCase(unittest.TestCase): @@ -172,6 +179,7 @@ def test_parse_html(self): text = mock_resource_with('cnn', 'txt') assert self.article.text == text + assert fulltext(self.article.html) == text # NOTE: top_img extraction requires an internet connection # unlike the rest of this test file @@ -183,6 +191,7 @@ def test_parse_html(self): assert self.article.title == TITLE assert len(self.article.imgs) == LEN_IMGS assert self.article.meta_lang == META_LANG + assert str(self.article.publish_date) == '2013-11-27 00:00:00' @print_test def test_meta_type_extraction(self): @@ -469,6 +478,7 @@ def test_chinese_fulltext_extract(self): article.parse() text = mock_resource_with('chinese', 'txt') assert article.text == text + assert fulltext(article.html, 'zh') == text @print_test def test_arabic_fulltext_extract(self): @@ -481,6 +491,7 @@ def test_arabic_fulltext_extract(self): assert article.meta_lang == 'ar' text = mock_resource_with('arabic', 'txt') assert article.text == text + assert fulltext(article.html, 'ar') == text @print_test def test_spanish_fulltext_extract(self): @@ -492,6 +503,7 @@ def test_spanish_fulltext_extract(self): article.parse() text = mock_resource_with('spanish', 'txt') assert article.text == text + assert fulltext(article.html, 'es') == text if __name__ == '__main__':