diff --git a/newspaper/extractors.py b/newspaper/extractors.py
index 7b4b74b3..9fd6cee9 100644
--- a/newspaper/extractors.py
+++ b/newspaper/extractors.py
@@ -235,6 +235,24 @@ def parse_date_str(date_str):
return None
+ def _choose_title_candidate(self, candidates, og_title):
+ """Pick the best
when a document has more than one.
+
+ Prefer a candidate whose filtered text contains the og:title (the
+ descriptive article title), otherwise fall back to the longest
+ candidate. This avoids picking bare site names (e.g. Medium's second
+ Medium).
+ """
+ if len(candidates) == 1:
+ return candidates[0]
+ filter_regex = re.compile(r'[^\u4e00-\u9fa5a-zA-Z0-9\ ]')
+ og_filtered = filter_regex.sub('', og_title or '').lower().strip()
+ if og_filtered:
+ for candidate in candidates:
+ if og_filtered in filter_regex.sub('', candidate).lower():
+ return candidate
+ return max(candidates, key=len)
+
def get_title(self, doc):
"""Fetch the article title and analyze it
@@ -265,8 +283,20 @@ def get_title(self, doc):
if not title_text:
return title
else:
- # title elem found
- title_text = self.parser.getText(title_element[0])
+ # some sites (e.g. Medium) emit multiple tags, one of which
+ # is a bare site name ("Medium"). Blindly taking the first element
+ # can yield the useless site name, so pick the best candidate:
+ # prefer the one matching og:title, otherwise the longest text.
+ title_candidates = [self.parser.getText(el).strip()
+ for el in title_element]
+ title_candidates = [c for c in title_candidates if c]
+ if not title_candidates:
+ title_text = title_text_fb
+ if not title_text:
+ return title
+ else:
+ title_text = self._choose_title_candidate(title_candidates,
+ title_text_fb)
used_delimeter = False
# title from h1