diff --git a/newspaper/article.py b/newspaper/article.py index abe9e292..1519aec8 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -101,6 +101,9 @@ def __init__(self, url, title=u'', source_url=u'', config=None, **kwargs): # meta favicon field in HTML source self.meta_favicon = u"" + # Meta tags contain a lot of structured data like OpenGraph + self.meta_data = {} + # The canonical link of this article if found in the meta data self.canonical_link = u"" @@ -180,6 +183,9 @@ def parse(self): meta_keywords = self.extractor.get_meta_keywords(self) self.set_meta_keywords(meta_keywords) + meta_data = self.extractor.get_meta_data(self) + self.set_meta_data(meta_data) + # TODO self.publish_date = self.config.publishDateExtractor.extract(self.doc) # before we do any computations on the body itself, we must clean up the document @@ -456,6 +462,9 @@ def set_meta_description(self, meta_description): """ self.meta_description = meta_description + def set_meta_data(self, meta_data): + self.meta_data = meta_data + def set_canonical_link(self, canonical_link): """ """ diff --git a/newspaper/extractors.py b/newspaper/extractors.py index f38585ce..9ebff9c4 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -9,6 +9,7 @@ import re import copy import urlparse +from collections import defaultdict from .packages.tldextract import tldextract from .utils import ( @@ -285,6 +286,47 @@ def get_meta_keywords(self, article): """ return self.get_meta_content(article.doc, "meta[name=keywords]") + def get_meta_data(self, article): + data = defaultdict(dict) + props = self.parser.css_select(article.doc, 'meta') + + for prop in props: + key = prop.attrib.get('property') + if not key: + key = prop.attrib.get('name') + + if not key: + continue + + key = key.split(':') + + value = prop.attrib.get('content') + if not value: + value = prop.attrib.get('value') + + if not value: + continue + + value = value.strip() + + if value.isdigit(): + value = int(value) + + ref = data[key.pop(0)] + + for idx, part in enumerate(key): + if not key[idx:-1]: # no next values + ref[part] = value + break + if not ref.get(part): + ref[part] = dict() + else: + if isinstance(ref.get(part), basestring): + ref[part] = {'url': ref[part]} + ref = ref[part] + + return data + def get_canonical_link(self, article): """ If the article has meta canonical link set in the url.