Use first image from article top_node

newspaper/article.py

@@ -59,9 +59,12 @@ def __init__(self, url, title=u'', source_url=u'', config=None, **kwargs):
         self.title = encodeValue(title)
 
         # the url of the "best image" to represent this article, via reddit algorithm
-        self.top_img = u''
+        self.top_img = self.top_image = u''
 
-        self.imgs = [] # all image urls
+        # stores image provided by metadata
+        self.meta_img = u''
+
+        self.imgs = self.images = [] # all image urls
         self.movies = [] # youtube, vimeo, etc
 
         # pure text from the article
@@ -110,6 +113,9 @@ def __init__(self, url, title=u'', source_url=u'', config=None, **kwargs):
         # Holds the top Element we think is a candidate for the main body
         self.top_node = None
 
+        # Holds clean version of top Element
+        self.clean_top_node = None
+
         # the lxml doc object
         self.doc = None
 
@@ -198,24 +204,35 @@ def parse(self):
             self.set_movies(video_extractor.get_videos())
 
             self.top_node = self.extractor.post_cleanup(self.top_node)
+            self.clean_top_node = copy.deepcopy(self.top_node)
+
             text, article_html = output_formatter.get_formatted(self)
             self.set_article_html(article_html)
             self.set_text(text)
 
+        if self.config.fetch_images:
+            self.fetch_images()
+
+        self.is_parsed = True
+        self.release_resources()
+
+    def fetch_images(self):
         if self.raw_doc is not None:
-            if self.config.fetch_images:
-                img_url = self.extractor.get_top_img_url(self)
-                self.set_top_img(img_url)
+            meta_img_url = self.extractor.get_meta_img_url(self)
+            self.set_meta_img(meta_img_url)
 
-                if self.config.fetch_images:
-                    top_imgs = self.extractor.get_img_urls(self)
-                    self.set_imgs(top_imgs)
+            imgs = self.extractor.get_img_urls(self)
+            self.set_imgs(imgs)
+
+        if self.clean_top_node is not None and not self.has_top_image():
+            first_img = self.extractor.get_first_img_url(self)
+            self.set_top_img(first_img)
 
-        if self.config.fetch_images:
+        if not self.has_top_image():
             self.set_reddit_top_img()
 
-        self.is_parsed = True
-        self.release_resources()
+    def has_top_image(self):
+        return self.top_img is not None and self.top_img != u''
 
     def is_valid_url(self):
         """
@@ -351,16 +368,15 @@ def set_reddit_top_img(self, test_run=False):
         first, uses Reddit's img algorithm as a fallback.
         """
 
+        #todo: move tests from here
         if test_run:
             s = images.Scraper(self)
             img = s.largest_image_url()
             print 'it worked, the img is', img
 
-        if self.top_img != u'': # if we already have a top img...
-            return
         try:
             s = images.Scraper(self)
-            self.set_top_img(s.largest_image_url())
+            self.set_top_img_no_ckeck(s.largest_image_url())
         except Exception, e:
             log.critical('jpeg error with PIL, %s' % e)
 
@@ -401,8 +417,18 @@ def set_article_html(self, article_html):
         """
         if article_html:
             self.article_html = encodeValue(article_html)
+
+    def set_meta_img(self, src_url):
+        self.meta_img = encodeValue(src_url)
+        self.set_top_img(src_url)
 
     def set_top_img(self, src_url):
+        if src_url is not None:
+            s = images.Scraper(self)
+            if s.satisfies_requirements(src_url):
+                self.set_top_img_no_ckeck(src_url)
+
+    def set_top_img_no_ckeck(self, src_url):
         """
         We want to provide 2 api's for images. One at
         "top_img", "imgs" and one at "top_image", "images".

newspaper/extractors.py

@@ -357,14 +357,23 @@ def get_img_urls(self, article):
         doc = article.raw_doc
         urls = self.parser.get_img_urls(doc)
         img_links = set([ urlparse.urljoin(article.url, url) for url in urls ])
+        if article.meta_img:
+            img_links.add(article.meta_img)
         return img_links
 
-    def get_top_img_url(self, article):
+    def get_first_img_url(self, article):
+        node_images = self.parser.get_img_urls(article.clean_top_node)
+        if node_images:
+            return urlparse.urljoin(article.url, node_images[0])
+        return u''
+
+    def get_meta_img_url(self, article):
         """
         """
         # !important, we must use raw_doc because at this point doc has been cleaned
         doc = article.raw_doc
-        return self.parser.get_top_img_url(doc)
+        meta_img_url = self.parser.get_meta_img_url(doc)
+        return urlparse.urljoin(article.url, meta_img_url)
 
     def get_category_urls(self, source, source_url=None, page_urls=None):
         """

newspaper/images.py

@@ -21,6 +21,7 @@
 
 chunk_size = 1024
 thumbnail_size = 90, 90
+minimal_area = 5000
 
 def image_to_str(image):
     s = StringIO.StringIO()
@@ -153,7 +154,7 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False):
             if 'open_req' in locals():
                 open_req.close()
 
-def fetch_size(url, useragent, referer=None, retries=1):
+def fetch_image_dimension(url, useragent, referer=None, retries=1):
     return fetch_url(url, useragent, referer, retries, dimension=True)
 
 class Scraper:
@@ -166,48 +167,60 @@ def __init__(self, article):
         self.useragent = self.config.browser_user_agent
 
     def largest_image_url(self):
+        #todo: remove. it is not responsibility of Scrapper
         if not self.imgs and not self.top_img:
             return None
-
         if self.top_img:
             return self.top_img
 
         max_area = 0
         max_url = None
 
         for img_url in self.imgs:
-            size = fetch_size(img_url, self.useragent, referer=self.url)
-            if not size:
-                continue
-
-            area = size[0] * size[1]
-
-            # ignore little images
-            if area < 5000:
-                log.debug('ignore little %s' % img_url)
-                continue
-
-            # PIL won't scale up, so we set a min width and
-            # maintain the aspect ratio
-            if size[0] < thumbnail_size[0]:
-                continue
-
-            # ignore excessively long/wide images
-            if max(size) / min(size) > self.config.image_dimension_ration:
-                log.debug('ignore dims %s' % img_url)
-                continue
-
-            # penalize images with "sprite" in their name
-            if 'sprite' in img_url.lower():
-                log.debug('penalizing sprite %s' % img_url)
-                area /= 10
+            dimension = fetch_image_dimension(img_url, self.useragent, referer=self.url)
+            area = self.calculate_area(img_url, dimension)
 
             if area > max_area:
                 max_area = area
                 max_url = img_url
 
         log.debug('using max img ' + max_url)
         return max_url
+
+    def calculate_area(self, img_url, dimension):
+        if not dimension:
+            return 0
+
+        area = dimension[0] * dimension[1]
+
+        #todo: introduce filter classes for each case
+        # ignore little images
+        if area < minimal_area:
+            log.debug('ignore little %s' % img_url)
+            return 0
+
+        # PIL won't scale up, so we set a min width and
+        # maintain the aspect ratio
+        if dimension[0] < thumbnail_size[0]:
+            return 0
+
+        # ignore excessively long/wide images
+        if max(dimension) / min(dimension) > self.config.image_dimension_ration:
+            log.debug('ignore dims %s' % img_url)
+            return 0
+
+        # penalize images with "sprite" in their name
+        lower_case_url = img_url.lower()
+        if 'sprite' in lower_case_url or 'logo' in lower_case_url:
+            log.debug('penalizing sprite %s' % img_url)
+            area /= 10
+
+        return area
+
+    def satisfies_requirements(self, img_url):
+        dimension = fetch_image_dimension(img_url, self.useragent, referer=self.url)
+        area = self.calculate_area(img_url, dimension)
+        return area > minimal_area
 
     def thumbnail(self):
         """

newspaper/parsers.py

@@ -74,7 +74,7 @@ def get_urls(cls, _input, titles=False, regex=False):
         return cls.root_to_urls(doc, titles)
 
     @classmethod
-    def get_top_img_url(cls, doc):
+    def get_meta_img_url(cls, doc):
         """
         Takes an lxml doc and returns the top img url
         running as method == 'soup' assumes lxml's soupparser.

newspaper/text.py

@@ -71,9 +71,14 @@ def __init__(self, language='en'):
     def remove_punctuation(self, content):
         # code taken form
         # http://stackoverflow.com/questions/265960/best-way-to-strip-punctuation-from-a-string-in-python
-        if isinstance(content, unicode):
+        content_is_unicode = isinstance(content, unicode)
+        if content_is_unicode:
             content = content.encode('utf-8')
-        return content.translate(self.TRANS_TABLE, string.punctuation)
+        stripped_input = content.translate(self.TRANS_TABLE, string.punctuation)
+
+        if content_is_unicode:
+            return stripped_input.decode('utf-8')
+        return stripped_input
 
     def candidate_words(self, stripped_input):
         return stripped_input.split(' ')