grangier#188 - move meta extraction to MetasExtractor class

Author: Xavier Grangier

goose/crawler.py

@@ -35,6 +35,7 @@
 from goose.extractors.tags import TagsExtractor
 from goose.extractors.opengraph import OpenGraphExtractor
 from goose.extractors.publishdate import PublishDateExtractor
+from goose.extractors.metas import MetasExtractor
 from goose.cleaners import StandardDocumentCleaner
 from goose.outputformatters import StandardOutputFormatter
 
@@ -71,6 +72,9 @@ def __init__(self, config):
         # init the output formatter
         self.formatter = self.get_formatter()
 
+        # metas extractor
+        self.metas_extractor = self.get_metas_extractor()
+
         # publishdate extractor
         self.publishdate_extractor = self.get_publishdate_extractor()
 
@@ -131,12 +135,15 @@ def crawl(self, crawl_candidate):
         # publishdate
         self.article.publish_date = self.publishdate_extractor.extract()
 
-        # self.article.additional_data = config.get_additionaldata_extractor.extract(doc)
-        self.article.meta_lang = self.extractor.get_meta_lang()
-        self.article.meta_favicon = self.extractor.get_favicon()
-        self.article.meta_description = self.extractor.get_meta_description()
-        self.article.meta_keywords = self.extractor.get_meta_keywords()
-        self.article.canonical_link = self.extractor.get_canonical_link()
+        # meta
+        metas = self.metas_extractor.extract()
+        self.article.meta_lang = metas['lang']
+        self.article.meta_favicon = metas['favicon']
+        self.article.meta_description = metas['description']
+        self.article.meta_keywords = metas['keywords']
+        self.article.canonical_link = metas['canonical']
+
+        # domain
         self.article.domain = self.extractor.get_domain()
 
         # tags
@@ -214,6 +221,9 @@ def get_html(self, crawl_candidate, parsing_candidate):
             })
         return html
 
+    def get_metas_extractor(self):
+        return MetasExtractor(self.config, self.article)
+
     def get_publishdate_extractor(self):
         return PublishDateExtractor(self.config, self.article)
 

goose/extractors/content.py

@@ -20,14 +20,10 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
-import re
-
 from copy import deepcopy
-from urlparse import urlparse, urljoin
 
 from goose.extractors import BaseExtractor
 
-RE_LANG = r'^[A-Za-z]{2}$'
 
 KNOWN_ARTICLE_CONTENT_TAGS = [
     {'attr': 'itemprop', 'value': 'articleBody'},
@@ -50,91 +46,6 @@ def get_language(self):
                 return self.article.meta_lang[:2]
         return self.config.target_language
 
-    def get_favicon(self):
-        """\
-        Extract the favicon from a website
-        http://en.wikipedia.org/wiki/Favicon
-        <link rel="shortcut icon" type="image/png" href="favicon.png" />
-        <link rel="icon" type="image/png" href="favicon.png" />
-        """
-        kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'icon'}
-        meta = self.parser.getElementsByTag(self.article.doc, **kwargs)
-        if meta:
-            favicon = self.parser.getAttribute(meta[0], 'href')
-            return favicon
-        return ''
-
-    def get_meta_lang(self):
-        """\
-        Extract content language from meta
-        """
-        # we have a lang attribute in html
-        attr = self.parser.getAttribute(self.article.doc, attr='lang')
-        if attr is None:
-            # look up for a Content-Language in meta
-            items = [
-                {'tag': 'meta', 'attr': 'http-equiv', 'value': 'content-language'},
-                {'tag': 'meta', 'attr': 'name', 'value': 'lang'}
-            ]
-            for item in items:
-                meta = self.parser.getElementsByTag(self.article.doc, **item)
-                if meta:
-                    attr = self.parser.getAttribute(meta[0], attr='content')
-                    break
-
-        if attr:
-            value = attr[:2]
-            if re.search(RE_LANG, value):
-                return value.lower()
-
-        return None
-
-    def get_meta_content(self, doc, metaName):
-        """\
-        Extract a given meta content form document
-        """
-        meta = self.parser.css_select(doc, metaName)
-        content = None
-
-        if meta is not None and len(meta) > 0:
-            content = self.parser.getAttribute(meta[0], 'content')
-
-        if content:
-            return content.strip()
-
-        return ''
-
-    def get_meta_description(self):
-        """\
-        if the article has meta description set in the source, use that
-        """
-        return self.get_meta_content(self.article.doc, "meta[name=description]")
-
-    def get_meta_keywords(self):
-        """\
-        if the article has meta keywords set in the source, use that
-        """
-        return self.get_meta_content(self.article.doc, "meta[name=keywords]")
-
-    def get_canonical_link(self):
-        """\
-        if the article has meta canonical link set in the url
-        """
-        if self.article.final_url:
-            kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'canonical'}
-            meta = self.parser.getElementsByTag(self.article.doc, **kwargs)
-            if meta is not None and len(meta) > 0:
-                href = self.parser.getAttribute(meta[0], 'href')
-                if href:
-                    href = href.strip()
-                    o = urlparse(href)
-                    if not o.hostname:
-                        z = urlparse(self.article.final_url)
-                        domain = '%s://%s' % (z.scheme, z.hostname)
-                        href = urljoin(domain, href)
-                    return href
-        return self.article.final_url
-
     def get_domain(self):
         if self.article.final_url:
             o = urlparse(self.article.final_url)

goose/extractors/meta.py

@@ -21,8 +21,108 @@
 limitations under the License.
 """
 
+import re
+from urlparse import urljoin
+from urlparse import urlparse
+
 from goose.extractors import BaseExtractor
 
 
-class ContentMetaExtractor(BaseExtractor):
-    pass
+RE_LANG = r'^[A-Za-z]{2}$'
+
+
+class MetasExtractor(BaseExtractor):
+
+    def get_favicon(self):
+        """\
+        Extract the favicon from a website
+        http://en.wikipedia.org/wiki/Favicon
+        <link rel="shortcut icon" type="image/png" href="favicon.png" />
+        <link rel="icon" type="image/png" href="favicon.png" />
+        """
+        kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'icon'}
+        meta = self.parser.getElementsByTag(self.article.doc, **kwargs)
+        if meta:
+            favicon = self.parser.getAttribute(meta[0], 'href')
+            return favicon
+        return ''
+
+    def get_canonical_link(self):
+        """\
+        if the article has meta canonical link set in the url
+        """
+        if self.article.final_url:
+            kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'canonical'}
+            meta = self.parser.getElementsByTag(self.article.doc, **kwargs)
+            if meta is not None and len(meta) > 0:
+                href = self.parser.getAttribute(meta[0], 'href')
+                if href:
+                    href = href.strip()
+                    o = urlparse(href)
+                    if not o.hostname:
+                        z = urlparse(self.article.final_url)
+                        domain = '%s://%s' % (z.scheme, z.hostname)
+                        href = urljoin(domain, href)
+                    return href
+        return self.article.final_url
+
+    def get_meta_lang(self):
+        """\
+        Extract content language from meta
+        """
+        # we have a lang attribute in html
+        attr = self.parser.getAttribute(self.article.doc, attr='lang')
+        if attr is None:
+            # look up for a Content-Language in meta
+            items = [
+                {'tag': 'meta', 'attr': 'http-equiv', 'value': 'content-language'},
+                {'tag': 'meta', 'attr': 'name', 'value': 'lang'}
+            ]
+            for item in items:
+                meta = self.parser.getElementsByTag(self.article.doc, **item)
+                if meta:
+                    attr = self.parser.getAttribute(meta[0], attr='content')
+                    break
+
+        if attr:
+            value = attr[:2]
+            if re.search(RE_LANG, value):
+                return value.lower()
+
+        return None
+
+    def get_meta_content(self, metaName):
+        """\
+        Extract a given meta content form document
+        """
+        meta = self.parser.css_select(self.article.doc, metaName)
+        content = None
+
+        if meta is not None and len(meta) > 0:
+            content = self.parser.getAttribute(meta[0], 'content')
+
+        if content:
+            return content.strip()
+
+        return ''
+
+    def get_meta_description(self):
+        """\
+        if the article has meta description set in the source, use that
+        """
+        return self.get_meta_content("meta[name=description]")
+
+    def get_meta_keywords(self):
+        """\
+        if the article has meta keywords set in the source, use that
+        """
+        return self.get_meta_content("meta[name=keywords]")
+
+    def extract(self):
+        return {
+            "description": self.get_meta_description(),
+            "keywords": self.get_meta_keywords(),
+            "lang": self.get_meta_lang(),
+            "favicon": self.get_favicon(),
+            "canonical": self.get_canonical_link()
+        }