grangier#137 - refactor title extraction based on opengraph, meta headling and title element

Author: Xavier Grangier

goose/extractors.py

@@ -30,6 +30,7 @@
 MOTLEY_REPLACEMENT = StringReplacement("�", "")
 ESCAPED_FRAGMENT_REPLACEMENT = StringReplacement(u"#!", u"?_escaped_fragment_=")
 TITLE_REPLACEMENTS = ReplaceSequence().create(u"»").append(u"»")
+TITLE_SPLITTERS = [u"|", u"-", u"»", u":"]
 PIPE_SPLITTER = StringSplitter("\\|")
 DASH_SPLITTER = StringSplitter(" - ")
 ARROWS_SPLITTER = StringSplitter("»")
@@ -65,44 +66,69 @@ def __init__(self, config, article):
         # stopwords class
         self.stopwords_class = config.stopwords_class
 
+    def clean_title(self, title):
+        """Clean title with the use of og:site_name
+        in this case try to get ride of site name
+        and use TITLE_SPLITTERS to reformat title
+        """
+        # check if we have the site name in opengraph data
+        if "site_name" in self.article.opengraph.keys():
+            site_name = self.article.opengraph['site_name']
+            # remove the site name from title
+            title = title.replace(site_name, '').strip()
+
+        # try to remove the domain from url
+        if self.article.domain:
+            pattern = re.compile(self.article.domain, re.IGNORECASE)
+            title = pattern.sub("", title).strip()
+
+        # split the title in words
+        # TechCrunch | my wonderfull article
+        # my wonderfull article | TechCrunch
+        title_words = title.split()
+
+        # check if first letter is in TITLE_SPLITTERS
+        # if so remove it
+        if title_words[0] in TITLE_SPLITTERS:
+            title_words.pop(0)
+
+        # check if last letter is in TITLE_SPLITTERS
+        # if so remove it
+        if title_words[-1] in TITLE_SPLITTERS:
+            title_words.pop(-1)
+
+        # rebuild the title
+        title = u" ".join(title_words).strip()
+
+        return title
+
     def get_title(self):
         """\
         Fetch the article title and analyze it
         """
-
         title = ''
-        doc = self.article.doc
 
-        title_element = self.parser.getElementsByTag(doc, tag='title')
-        # no title found
-        if title_element is None or len(title_element) == 0:
-            return title
+        # rely on opengraph in case we have the data
+        if "title" in self.article.opengraph.keys():
+            title = self.article.opengraph['title']
+            return self.clean_title(title)
 
-        # title elem found
-        title_text = self.parser.getText(title_element[0])
-        used_delimeter = False
-
-        # split title with |
-        if '|' in title_text:
-            title_text = self.split_title(title_text, PIPE_SPLITTER)
-            used_delimeter = True
-
-        # split title with -
-        if not used_delimeter and '-' in title_text:
-            title_text = self.split_title(title_text, DASH_SPLITTER)
-            used_delimeter = True
-
-        # split title with »
-        if not used_delimeter and u'»' in title_text:
-            title_text = self.split_title(title_text, ARROWS_SPLITTER)
-            used_delimeter = True
-
-        # split title with :
-        if not used_delimeter and ':' in title_text:
-            title_text = self.split_title(title_text, COLON_SPLITTER)
-            used_delimeter = True
+        # try to fetch the meta headline
+        meta_headline = self.parser.getElementsByTag(
+                            self.article.doc,
+                            tag="meta",
+                            attr="name",
+                            value="headline")
+        if meta_headline is not None and len(meta_headline) > 0:
+            title = self.parser.getAttribute(meta_headline[0], 'content')
+            return self.clean_title(title)
+
+        # otherwise use the title meta
+        title_element = self.parser.getElementsByTag(self.article.doc, tag='title')
+        if title_element is not None and len(title_element) > 0:
+            title = self.parser.getText(title_element[0])
+            return self.clean_title(title)
 
-        title = MOTLEY_REPLACEMENT.replaceAll(title_text)
         return title
 
     def split_title(self, title, splitter):