Skip to content

Commit 4fd94de

Browse files
author
Xavier Grangier
committed
Merge branch 'feature/title-extraction-137' into develop
2 parents 502053d + 148ce9b commit 4fd94de

8 files changed

Lines changed: 90 additions & 41 deletions

File tree

‎goose/crawler.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -95,9 +95,9 @@ def crawl(self, crawl_candidate):
9595
self.article.raw_html = raw_html
9696
self.article.doc = doc
9797
self.article.raw_doc = deepcopy(doc)
98+
self.article.opengraph = self.extractor.extract_opengraph()
9899
self.article.publish_date = self.extractor.get_publish_date()
99100
# self.article.additional_data = config.get_additionaldata_extractor.extract(doc)
100-
self.article.title = self.extractor.get_title()
101101
self.article.meta_lang = self.extractor.get_meta_lang()
102102
self.article.meta_favicon = self.extractor.get_favicon()
103103
self.article.meta_description = self.extractor.get_meta_description()
@@ -106,9 +106,7 @@ def crawl(self, crawl_candidate):
106106
self.article.domain = self.extractor.get_domain()
107107
self.article.tags = self.extractor.extract_tags()
108108
self.article.authors = self.extractor.extract_authors()
109-
110-
# opengraph
111-
self.article.opengraph = self.extractor.extract_opengraph()
109+
self.article.title = self.extractor.get_title()
112110

113111
# check for an articleBody
114112
# if we find one force the article.doc to be the articleBody node

‎goose/extractors.py‎

Lines changed: 56 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
MOTLEY_REPLACEMENT = StringReplacement("�", "")
3131
ESCAPED_FRAGMENT_REPLACEMENT = StringReplacement(u"#!", u"?_escaped_fragment_=")
3232
TITLE_REPLACEMENTS = ReplaceSequence().create(u"»").append(u"»")
33+
TITLE_SPLITTERS = [u"|", u"-", u"»", u":"]
3334
PIPE_SPLITTER = StringSplitter("\\|")
3435
DASH_SPLITTER = StringSplitter(" - ")
3536
ARROWS_SPLITTER = StringSplitter("»")
@@ -65,44 +66,69 @@ def __init__(self, config, article):
6566
# stopwords class
6667
self.stopwords_class = config.stopwords_class
6768

69+
def clean_title(self, title):
70+
"""Clean title with the use of og:site_name
71+
in this case try to get ride of site name
72+
and use TITLE_SPLITTERS to reformat title
73+
"""
74+
# check if we have the site name in opengraph data
75+
if "site_name" in self.article.opengraph.keys():
76+
site_name = self.article.opengraph['site_name']
77+
# remove the site name from title
78+
title = title.replace(site_name, '').strip()
79+
80+
# try to remove the domain from url
81+
if self.article.domain:
82+
pattern = re.compile(self.article.domain, re.IGNORECASE)
83+
title = pattern.sub("", title).strip()
84+
85+
# split the title in words
86+
# TechCrunch | my wonderfull article
87+
# my wonderfull article | TechCrunch
88+
title_words = title.split()
89+
90+
# check if first letter is in TITLE_SPLITTERS
91+
# if so remove it
92+
if title_words[0] in TITLE_SPLITTERS:
93+
title_words.pop(0)
94+
95+
# check if last letter is in TITLE_SPLITTERS
96+
# if so remove it
97+
if title_words[-1] in TITLE_SPLITTERS:
98+
title_words.pop(-1)
99+
100+
# rebuild the title
101+
title = u" ".join(title_words).strip()
102+
103+
return title
104+
68105
def get_title(self):
69106
"""\
70107
Fetch the article title and analyze it
71108
"""
72-
73109
title = ''
74-
doc = self.article.doc
75110

76-
title_element = self.parser.getElementsByTag(doc, tag='title')
77-
# no title found
78-
if title_element is None or len(title_element) == 0:
79-
return title
111+
# rely on opengraph in case we have the data
112+
if "title" in self.article.opengraph.keys():
113+
title = self.article.opengraph['title']
114+
return self.clean_title(title)
80115

81-
# title elem found
82-
title_text = self.parser.getText(title_element[0])
83-
used_delimeter = False
84-
85-
# split title with |
86-
if '|' in title_text:
87-
title_text = self.split_title(title_text, PIPE_SPLITTER)
88-
used_delimeter = True
89-
90-
# split title with -
91-
if not used_delimeter and '-' in title_text:
92-
title_text = self.split_title(title_text, DASH_SPLITTER)
93-
used_delimeter = True
94-
95-
# split title with »
96-
if not used_delimeter and u'»' in title_text:
97-
title_text = self.split_title(title_text, ARROWS_SPLITTER)
98-
used_delimeter = True
99-
100-
# split title with :
101-
if not used_delimeter and ':' in title_text:
102-
title_text = self.split_title(title_text, COLON_SPLITTER)
103-
used_delimeter = True
116+
# try to fetch the meta headline
117+
meta_headline = self.parser.getElementsByTag(
118+
self.article.doc,
119+
tag="meta",
120+
attr="name",
121+
value="headline")
122+
if meta_headline is not None and len(meta_headline) > 0:
123+
title = self.parser.getAttribute(meta_headline[0], 'content')
124+
return self.clean_title(title)
125+
126+
# otherwise use the title meta
127+
title_element = self.parser.getElementsByTag(self.article.doc, tag='title')
128+
if title_element is not None and len(title_element) > 0:
129+
title = self.parser.getText(title_element[0])
130+
return self.clean_title(title)
104131

105-
title = MOTLEY_REPLACEMENT.replaceAll(title_text)
106132
return title
107133

108134
def split_title(self, title, splitter):

‎tests/data/extractors/test_allnewlyrics1.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010
"PJ Morton",
1111
"Stevie Wonder"
1212
],
13-
"title": "PJ Morton (Ft. Stevie Wonder)",
13+
"title": "\u201cOnly One\u201d Lyrics : PJ Morton (Ft. Stevie Wonder)",
1414
"meta_favicon": "",
1515
"meta_lang": "en"
1616
}
17-
}
17+
}

‎tests/data/extractors/test_cnn1.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
"final_url": "http://www.cnn.com/2010/POLITICS/08/13/democrats.social.security/index.html",
77
"meta_keywords": "",
88
"cleaned_text": "Washington (CNN) -- Democrats pledged ",
9-
"title": "Democrats to use Social Security against GOP this fall",
9+
"title": "Democrats to use Social Security against GOP this fall - CNN.com",
1010
"meta_favicon": "http://i.cdn.turner.com/cnn/.element/img/3.0/global/misc/apple-touch-icon.png",
1111
"meta_lang": "en"
1212
}
13-
}
13+
}

‎tests/data/extractors/test_time.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
"final_url": "http://www.time.com/time/health/article/0,8599,2011497,00.html",
77
"meta_keywords": "bp, oil, spill, gulf, mexico, invisible, dispersed, deepwater horizon, Charles Hopkinson",
88
"cleaned_text": "This month, the federal government released",
9-
"title": "Invisible Oil from BP Spill May Threaten Gulf Aquatic Life",
9+
"title": "Oil from Spill Could Still Pose Major Threat",
1010
"meta_favicon": "http://img.timeinc.net/time/favicon.ico",
1111
"meta_lang": null
1212
}
13-
}
13+
}

‎tests/data/extractors/test_title_opengraph.html‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<html>
2+
<head>
3+
<meta property="og:site_name" content="TechCrunch"/>
4+
<meta property="og:title" content="Good article title | TechCrunch"/>
5+
<title>Wrong article title - website</title>
6+
</head>
7+
<body>
8+
<div>
9+
<p>
10+
TextNode 1 - The Scala supported IDE is one of the few pain points of developers who want to start using Scala in their Java project. On existing long term project developed by a team its hard to step in and introduce a new language that is not supported by the existing IDE. On way to go about it is to hid the fact that you use Scala from the Java world by using one way dependency injection. Still, if you wish to truly absorb Scala into your existing java environment then you'll soon introduced cross language dependencies.
11+
</p>
12+
</div>
13+
</body>
14+
</html>

‎tests/data/extractors/test_title_opengraph.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"url": "http://exemple.com/test_opengraphcontent",
3+
"expected": {
4+
"title": "Good article title"
5+
}
6+
}

‎tests/extractors.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ def runArticleAssertions(self, article, fields):
125125
continue
126126

127127
# default assertion
128-
msg = u"Error %s" % field
128+
msg = u"Error %s \nexpected: %s\nresult: %s" % (field, expected_value, result_value)
129129
self.assertEqual(expected_value, result_value, msg=msg)
130130

131131
def extract(self, instance):
@@ -365,6 +365,11 @@ def test_opengraph(self):
365365
fields = ['opengraph']
366366
self.runArticleAssertions(article=article, fields=fields)
367367

368+
def test_title_opengraph(self):
369+
article = self.getArticle()
370+
fields = ['title']
371+
self.runArticleAssertions(article=article, fields=fields)
372+
368373
def test_issue129(self):
369374
article = self.getArticle()
370375
fields = ['cleaned_text']

0 commit comments

Comments
 (0)