diff --git a/newspaper/configuration.py b/newspaper/configuration.py index bcaf52cb..301dd179 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -60,6 +60,7 @@ def __init__(self): # Allow redirects (enabled by default) self.allow_redirects = True + self.ignored_images_suffix_list = [] # English is the fallback self._language = 'en' diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 28d1dd5b..d47eb488 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -13,6 +13,7 @@ import copy import logging +import os.path import re import re from collections import defaultdict @@ -449,19 +450,21 @@ def get_meta_img_url(self, article_url, doc): """ top_meta_image, try_one, try_two, try_three, try_four = [None] * 5 try_one = self.get_meta_content(doc, 'meta[property="og:image"]') + try_one = None if self.image_is_ignored(try_one) else try_one if not try_one: link_img_src_kwargs = \ {'tag': 'link', 'attr': 'rel', 'value': 'img_src|image_src'} elems = self.parser.getElementsByTag(doc, use_regex=True, **link_img_src_kwargs) try_two = elems[0].get('href') if elems else None - + try_two = None if self.image_is_ignored(try_two) else try_two if not try_two: try_three = self.get_meta_content(doc, 'meta[name="og:image"]') - + try_three = None if self.image_is_ignored(try_three) else try_three if not try_three: link_icon_kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'icon'} elems = self.parser.getElementsByTag(doc, **link_icon_kwargs) try_four = elems[0].get('href') if elems else None + try_four = None if self.image_is_ignored(try_four) else try_four top_meta_image = try_one or try_two or try_three or try_four @@ -469,6 +472,12 @@ def get_meta_img_url(self, article_url, doc): return urljoin(article_url, top_meta_image) return '' + def image_is_ignored(self, image): + return len([True for x in self.config.ignored_images_suffix_list if image and self.match_image(x, os.path.basename(image))]) > 0 + + def match_image(self, pattern, image): + return re.search(pattern, image) is not None + def get_meta_type(self, doc): """Returns meta type of article, open graph protocol """