Extract Meta Tags in structured way

newspaper/article.py

@@ -101,6 +101,9 @@ def __init__(self, url, title=u'', source_url=u'', config=None, **kwargs):
         # meta favicon field in HTML source
         self.meta_favicon = u""
 
+        # Meta tags contain a lot of structured data like OpenGraph
+        self.meta_data = {}
+
         # The canonical link of this article if found in the meta data
         self.canonical_link = u""
 
@@ -180,6 +183,9 @@ def parse(self):
         meta_keywords = self.extractor.get_meta_keywords(self)
         self.set_meta_keywords(meta_keywords)
 
+        meta_data = self.extractor.get_meta_data(self)
+        self.set_meta_data(meta_data)
+
         # TODO self.publish_date = self.config.publishDateExtractor.extract(self.doc)
 
         # before we do any computations on the body itself, we must clean up the document
@@ -456,6 +462,9 @@ def set_meta_description(self, meta_description):
         """
         self.meta_description = meta_description
 
+    def set_meta_data(self, meta_data):
+        self.meta_data = meta_data
+
     def set_canonical_link(self, canonical_link):
         """
         """

newspaper/extractors.py

@@ -9,6 +9,7 @@
 import re
 import copy
 import urlparse
+from collections import defaultdict
 
 from .packages.tldextract import tldextract
 from .utils import (
@@ -285,6 +286,47 @@ def get_meta_keywords(self, article):
         """
         return self.get_meta_content(article.doc, "meta[name=keywords]")
 
+    def get_meta_data(self, article):
+        data = defaultdict(dict)
+        props = self.parser.css_select(article.doc, 'meta')
+
+        for prop in props:
+            key = prop.attrib.get('property')
+            if not key:
+                key = prop.attrib.get('name')
+
+            if not key:
+                continue
+
+            key = key.split(':')
+
+            value = prop.attrib.get('content')
+            if not value:
+                value = prop.attrib.get('value')
+
+            if not value:
+                continue
+
+            value = value.strip()
+
+            if value.isdigit():
+                value = int(value)
+
+            ref = data[key.pop(0)]
+
+            for idx, part in enumerate(key):
+                if not key[idx:-1]:  # no next values
+                    ref[part] = value
+                    break
+                if not ref.get(part):
+                    ref[part] = dict()
+                else:
+                    if isinstance(ref.get(part), basestring):
+                        ref[part] = {'url': ref[part]}
+                ref = ref[part]
+
+        return data
+
     def get_canonical_link(self, article):
         """
         If the article has meta canonical link set in the url.