From b94e1347bc27f5d3bb748cc73679eefd05d0069a Mon Sep 17 00:00:00 2001 From: Oleg Temnov Date: Thu, 30 Jan 2014 02:39:34 +0400 Subject: [PATCH 1/2] Remove duplicate image urls --- newspaper/extractors.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index f38585ce..ed514ea4 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -309,8 +309,7 @@ def get_img_urls(self, article): """ doc = article.raw_doc urls = self.parser.get_img_urls(doc) - img_links = [ urlparse.urljoin(article.url, url) for url in urls ] - + img_links = set([ urlparse.urljoin(article.url, url) for url in urls ]) return img_links def get_top_img_url(self, article): From 393601dc4a57d8ed422aaa155e8029516192a33b Mon Sep 17 00:00:00 2001 From: Oleg Temnov Date: Thu, 30 Jan 2014 03:39:21 +0400 Subject: [PATCH 2/2] Configuration for ignore excessively long/wide images ration --- newspaper/configuration.py | 1 + newspaper/images.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/newspaper/configuration.py b/newspaper/configuration.py index ec3ff26c..2beaa8b3 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -46,6 +46,7 @@ def __init__(self): # set this to false if you don't care about getting images self.fetch_images = True + self.image_dimension_ration = 16/9 # don't toggle this variable self.use_meta_language = True diff --git a/newspaper/images.py b/newspaper/images.py index af53a5f5..8734d083 100644 --- a/newspaper/images.py +++ b/newspaper/images.py @@ -124,6 +124,10 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False): 'installation this machine: %s' % str(e)) p = None break + except ValueError, ve: + log.debug('cant read image format: %s' % url) + p = None + break new_data = open_req.read(chunk_size) content += new_data @@ -189,7 +193,7 @@ def largest_image_url(self): continue # ignore excessively long/wide images - if max(size) / min(size) > 1.5: + if max(size) / min(size) > self.config.image_dimension_ration: log.debug('ignore dims %s' % img_url) continue