grangier#188 - move tweet extraction to TweetExtractor class

Author: Xavier Grangier

goose/crawler.py

@@ -30,6 +30,7 @@
 from goose.extractors.title import TitleExtractor
 from goose.extractors.images import ImageExtractor
 from goose.extractors.links import LinksExtractor
+from goose.extractors.tweets import TweetsExtractor
 from goose.cleaners import StandardDocumentCleaner
 from goose.outputformatters import StandardOutputFormatter
 
@@ -66,6 +67,9 @@ def __init__(self, config):
         # init the output formatter
         self.formatter = self.get_formatter()
 
+        # tweets extractor
+        self.tweets_extractor = self.get_tweets_extractor()
+
         # links extractor
         self.links_extractor = self.get_links_extractor()
 
@@ -138,7 +142,7 @@ def crawl(self, crawl_candidate):
             self.article.links = self.links_extractor.extract()
 
             # tweets
-            self.article.tweets = self.extractor.extract_tweets()
+            self.article.tweets = self.tweets_extractor.extract()
 
             # video handling
             self.video_extractor.get_videos()
@@ -183,6 +187,9 @@ def get_html(self, crawl_candidate, parsing_candidate):
             })
         return html
 
+    def get_tweets_extractor(self):
+        return TweetsExtractor(self.config, self.article)
+
     def get_links_extractor(self):
         return LinksExtractor(self.config, self.article)
 

goose/extractors/content.py

@@ -192,21 +192,6 @@ def extract_opengraph(self):
                 opengraph_dict.update({attr.split(":")[1]: value})
         return opengraph_dict
 
-    def extract_tweets(self):
-        tweets = []
-        items = self.parser.getElementsByTag(
-                        self.article.top_node,
-                        tag='blockquote',
-                        attr="class",
-                        value="twitter-tweet")
-
-        for i in items:
-            for attr in ['gravityScore', 'gravityNodes']:
-                self.parser.delAttribute(i, attr)
-            tweets.append(self.parser.nodeToString(i))
-
-        return tweets
-
     def extract_authors(self):
         authors = []
         author_nodes = self.parser.getElementsByTag(

goose/extractors/tweets.py

@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+"""\
+This is a python port of "Goose" orignialy licensed to Gravity.com
+under one or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.
+
+Python port was written by Xavier Grangier for Recrutae
+
+Gravity.com licenses this file
+to you under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from goose.extractors import BaseExtractor
+
+
+class TweetsExtractor(BaseExtractor):
+
+    def extract(self):
+        tweets = []
+        items = self.parser.getElementsByTag(
+                        self.article.top_node,
+                        tag='blockquote',
+                        attr="class",
+                        value="twitter-tweet")
+
+        for i in items:
+            for attr in ['gravityScore', 'gravityNodes']:
+                self.parser.delAttribute(i, attr)
+            tweets.append(self.parser.nodeToString(i))
+
+        return tweets