diff --git a/newspaper/extractors.py b/newspaper/extractors.py index ec25b7d3..b2246e33 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -79,6 +79,18 @@ def get_authors(self, doc): def contains_digits(d): return bool(_digits.search(d)) + def uniqify_list(l): + """Remove duplicates from provided list but maintain original order. + Derived from http://www.peterbe.com/plog/uniqifiers-benchmark + """ + seen = {} + result = [] + for item in l: + if item.lower() in seen: continue + seen[item.lower()] = 1 + result.append(item.title()) + return result + def parse_byline(search_str): """Takes a candidate line of html or text and extracts out the name(s) in list form @@ -95,21 +107,19 @@ def parse_byline(search_str): search_str = search_str.strip() # Chunk the line by non alphanumeric tokens (few name exceptions) - # >>> re.split("[^\w\'\-]", "Lucas Ou, Dean O'Brian and Ronald") - # ['Lucas Ou', '', 'Dean O'Brian', 'and', 'Ronald'] - name_tokens = re.split("[^\w\'\-]", search_str) + # >>> re.split("[^\w\'\-\.]", "Tyler G. Jones, Lucas Ou, Dean O'Brian and Ronald") + # ['Tyler', 'G.', 'Jones', '', 'Lucas', 'Ou', '', 'Dean', "O'Brian", 'and', 'Ronald'] + name_tokens = re.split("[^\w\'\-\.]", search_str) name_tokens = [s.strip() for s in name_tokens] _authors = [] # List of first, last name tokens curname = [] - DELIM = ['and', ''] + DELIM = ['and', ',', ''] for token in name_tokens: if token in DELIM: - # should we allow middle names? - valid_name = (len(curname) == 2) - if valid_name: + if len(curname) > 0: _authors.append(' '.join(curname)) curname = [] @@ -126,9 +136,9 @@ def parse_byline(search_str): # Try 1: Search popular author tags for authors ATTRS = ['name', 'rel', 'itemprop', 'class', 'id'] - VALS = ['author', 'byline'] + VALS = ['author', 'byline', 'dc.creator'] matches = [] - _authors, authors = [], [] + authors = [] for attr in ATTRS: for val in VALS: @@ -145,13 +155,9 @@ def parse_byline(search_str): else: content = match.text or '' if len(content) > 0: - _authors.extend(parse_byline(content)) + authors.extend(parse_byline(content)) - uniq = list(set([s.lower() for s in _authors])) - for name in uniq: - names = [w.capitalize() for w in name.split(' ')] - authors.append(' '.join(names)) - return authors or [] + return uniqify_list(authors) # TODO Method 2: Search raw html for a by-line # match = re.search('By[\: ].*\\n|From[\: ].*\\n', html) diff --git a/tests/data/html/cnn_article.html b/tests/data/html/cnn_article.html index eaf5b227..53c1629b 100644 --- a/tests/data/html/cnn_article.html +++ b/tests/data/html/cnn_article.html @@ -13,7 +13,7 @@ - + @@ -68,7 +68,7 @@ cnnBrandingValue="default"; cnnPartnerValue=""; cnnOmniBranding="", -cnnAuthor="Dana Ford and Tom Watkins, CNN", +cnnAuthor="Dana A. Ford, James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN", disqus_category_id=207582, disqus_identifier="/2013/11/27/travel/weather-thanksgiving/index.html", disqus_title="After storm, forecasters see smooth sailing for Thanksgiving", @@ -87,7 +87,7 @@ business: { cnn: { page: { -author: "Dana Ford and Tom Watkins, CNN", +author: "Dana A. Ford, James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN", broadcast_franchise: "", video_embed_count: "4", publish_date: "2013/11/27", @@ -294,7 +294,7 @@

After storm, forecasters see smooth sailing for Thanksgiving

-
By Dana Ford and Tom Watkins, CNN
+
By Dana A. Ford , James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN
November 28, 2013 -- Updated 0203 GMT (1003 HKT)
@@ -1022,7 +1022,7 @@

cnnsocial.share.setconfig(cnn_shareconfig); $j(document).ready(function () { 'use strict'; -loadChartbeat("travel", "Dana Ford and Tom Watkins, CNN"); +loadChartbeat("travel", "Dana A. Ford, James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN"); CNN.initFlipperTicker(); /* initialize cnnsocial */ cnnsocial.init(); @@ -1041,4 +1041,4 @@

- \ No newline at end of file + diff --git a/tests/unit_tests.py b/tests/unit_tests.py index e17b241b..61c1e923 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -158,7 +158,7 @@ def test_url(self): def test_download_html(self): html = mock_resource_with('cnn_article', 'html') self.article.download(html) - assert len(self.article.html) == 75175 + assert len(self.article.html) == 75406 @print_test def test_pre_download_parse(self): @@ -169,7 +169,7 @@ def test_pre_download_parse(self): @print_test def test_parse_html(self): - AUTHORS = ['Dana Ford', 'Tom Watkins'] + AUTHORS = ['Chien-Ming Wang', 'Dana A. Ford', 'James S.A. Corey', 'Tom Watkins'] TITLE = 'After storm, forecasters see smooth sailing for Thanksgiving' LEN_IMGS = 46 META_LANG = 'en' @@ -209,7 +209,7 @@ def test_meta_extraction(self): 'title': 'After storm, forecasters see smooth sailing for Thanksgiving - CNN.com', 'og': {'site_name': 'CNN','description': 'A strong storm struck much of the eastern United States on Wednesday, complicating holiday plans for many of the 43 million Americans expected to travel.', 'title': 'After storm, forecasters see smooth sailing for Thanksgiving', 'url': 'http://www.cnn.com/2013/11/27/travel/weather-thanksgiving/index.html', 'image': 'http://i2.cdn.turner.com/cnn/dam/assets/131129200805-01-weather-1128-story-top.jpg', 'type': 'article'}, 'section': 'travel', - 'author': 'Dana Ford and Tom Watkins, CNN', + 'author': 'Dana A. Ford, James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN', 'robots': 'index,follow', 'vr': {'canonical': 'http://edition.cnn.com/2013/11/27/travel/weather-thanksgiving/index.html'}, 'source': 'CNN',