From f5a6c037449307efc4790952a07b5fccf6ccc662 Mon Sep 17 00:00:00 2001 From: duw Date: Mon, 17 Nov 2014 16:36:02 +0800 Subject: [PATCH] split title with _ --- newspaper/extractors.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 41dfa201..309b4efe 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -30,6 +30,7 @@ TITLE_REPLACEMENTS = ReplaceSequence().create(u"»").append(u"»") PIPE_SPLITTER = StringSplitter("\\|") DASH_SPLITTER = StringSplitter(" - ") +UNDERSCORE_SPLITTER = StringSplitter("_") ARROWS_SPLITTER = StringSplitter("»") COLON_SPLITTER = StringSplitter(":") SPACE_SPLITTER = StringSplitter(' ') @@ -182,6 +183,11 @@ def get_title(self, doc): title_text = self.split_title(title_text, DASH_SPLITTER) used_delimeter = True + # split title with _ + if not used_delimeter and '_' in title_text: + title_text = self.split_title(title_text, UNDERSCORE_SPLITTER) + used_delimeter = True + # split title with » if not used_delimeter and u'»' in title_text: title_text = self.split_title(title_text, ARROWS_SPLITTER)