From 6ca095a06d1b1a80bf4a0890e9f925ad8ba20026 Mon Sep 17 00:00:00 2001 From: Miguel Almeida Date: Thu, 20 Jul 2017 19:11:21 +0100 Subject: [PATCH 1/3] change content tag name for datePublished To be inline with http://schema.org/datePublished --- newspaper/extractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 237bc929..bd8eb162 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -203,7 +203,7 @@ def parse_date_str(date_str): {'attribute': 'name', 'value': 'OriginalPublicationDate', 'content': 'content'}, {'attribute': 'itemprop', 'value': 'datePublished', - 'content': 'datetime'}, + 'content': 'content'}, {'attribute': 'property', 'value': 'og:published_time', 'content': 'content'}, {'attribute': 'name', 'value': 'article_date_original', From 108bf89e5230b5b5b73da98132b70ea5ded87d77 Mon Sep 17 00:00:00 2001 From: Miguel Almeida Date: Fri, 21 Jul 2017 11:50:52 +0100 Subject: [PATCH 2/3] Simplify image extractor logic --- newspaper/extractors.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 237bc929..bed6c81b 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -445,23 +445,21 @@ def get_meta_content(self, doc, metaname): def get_meta_img_url(self, article_url, doc): """Returns the 'top img' as specified by the website """ - top_meta_image, try_one, try_two, try_three, try_four = [None] * 5 - try_one = self.get_meta_content(doc, 'meta[property="og:image"]') - if try_one is None: + top_meta_image = self.get_meta_content(doc, 'meta[property="og:image"]') + if top_meta_image is None: link_icon_kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'icon'} elems = self.parser.getElementsByTag(doc, **link_icon_kwargs) - try_two = elems[0].get('href') if elems else None + top_meta_image = elems[0].get('href') if elems else None - if try_two is None: + if top_meta_image is None: link_img_src_kwargs = \ {'tag': 'link', 'attr': 'rel', 'value': 'img_src'} elems = self.parser.getElementsByTag(doc, **link_img_src_kwargs) - try_three = elems[0].get('href') if elems else None + top_meta_image = elems[0].get('href') if elems else None - if try_three is None: - try_four = self.get_meta_content(doc, 'meta[name="og:image"]') + if top_meta_image is None: + top_meta_image = self.get_meta_content(doc, 'meta[name="og:image"]') - top_meta_image = try_one or try_two or try_three or try_four if top_meta_image: return urljoin(article_url, top_meta_image) From aab3ddf363a52feb2839be4b448b1f67b8ee3fdf Mon Sep 17 00:00:00 2001 From: Miguel Almeida Date: Fri, 21 Jul 2017 11:51:31 +0100 Subject: [PATCH 3/3] Add og:image:secure_url and og:image:url as image scraping sources --- newspaper/extractors.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index bed6c81b..4f312357 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -460,6 +460,11 @@ def get_meta_img_url(self, article_url, doc): if top_meta_image is None: top_meta_image = self.get_meta_content(doc, 'meta[name="og:image"]') + if top_meta_image is None: + top_meta_image = self.get_meta_content(doc, 'meta[name="og:image:secure_url"]') + + if top_meta_image is None: + top_meta_image = self.get_meta_content(doc, 'meta[name="og:image:url"]') if top_meta_image: return urljoin(article_url, top_meta_image)