diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 41dfa201..309b4efe 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -30,6 +30,7 @@ TITLE_REPLACEMENTS = ReplaceSequence().create(u"»").append(u"»") PIPE_SPLITTER = StringSplitter("\\|") DASH_SPLITTER = StringSplitter(" - ") +UNDERSCORE_SPLITTER = StringSplitter("_") ARROWS_SPLITTER = StringSplitter("»") COLON_SPLITTER = StringSplitter(":") SPACE_SPLITTER = StringSplitter(' ') @@ -182,6 +183,11 @@ def get_title(self, doc): title_text = self.split_title(title_text, DASH_SPLITTER) used_delimeter = True + # split title with _ + if not used_delimeter and '_' in title_text: + title_text = self.split_title(title_text, UNDERSCORE_SPLITTER) + used_delimeter = True + # split title with » if not used_delimeter and u'»' in title_text: title_text = self.split_title(title_text, ARROWS_SPLITTER)