Original

newspaper/configuration.py

@@ -46,6 +46,7 @@ def __init__(self):
 
         # set this to false if you don't care about getting images
         self.fetch_images = True
+        self.image_dimension_ration = 16/9
 
         # don't toggle this variable
         self.use_meta_language = True

newspaper/extractors.py

@@ -309,8 +309,7 @@ def get_img_urls(self, article):
         """
         doc = article.raw_doc
         urls = self.parser.get_img_urls(doc)
-        img_links = [ urlparse.urljoin(article.url, url) for url in urls ]
-
+        img_links = set([ urlparse.urljoin(article.url, url) for url in urls ])
         return img_links
 
     def get_top_img_url(self, article):

newspaper/images.py

@@ -124,6 +124,10 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False):
                                 'installation this machine: %s' % str(e))
                         p = None
                         break
+                    except  ValueError, ve:
+                        log.debug('cant read image format: %s' % url)
+                        p = None
+                        break
                     new_data = open_req.read(chunk_size)
                     content += new_data
 
@@ -189,7 +193,7 @@ def largest_image_url(self):
                 continue
 
             # ignore excessively long/wide images
-            if max(size) / min(size) > 1.5:
+            if max(size) / min(size) > self.config.image_dimension_ration:
                 log.debug('ignore dims %s' % img_url)
                 continue