diff --git a/newspaper/article.py b/newspaper/article.py index 2714eabe..64ef10d7 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -59,9 +59,12 @@ def __init__(self, url, title=u'', source_url=u'', config=None, **kwargs): self.title = encodeValue(title) # the url of the "best image" to represent this article, via reddit algorithm - self.top_img = u'' + self.top_img = self.top_image = u'' - self.imgs = [] # all image urls + # stores image provided by metadata + self.meta_img = u'' + + self.imgs = self.images = [] # all image urls self.movies = [] # youtube, vimeo, etc # pure text from the article @@ -110,6 +113,9 @@ def __init__(self, url, title=u'', source_url=u'', config=None, **kwargs): # Holds the top Element we think is a candidate for the main body self.top_node = None + # Holds clean version of top Element + self.clean_top_node = None + # the lxml doc object self.doc = None @@ -198,24 +204,35 @@ def parse(self): self.set_movies(video_extractor.get_videos()) self.top_node = self.extractor.post_cleanup(self.top_node) + self.clean_top_node = copy.deepcopy(self.top_node) + text, article_html = output_formatter.get_formatted(self) self.set_article_html(article_html) self.set_text(text) + if self.config.fetch_images: + self.fetch_images() + + self.is_parsed = True + self.release_resources() + + def fetch_images(self): if self.raw_doc is not None: - if self.config.fetch_images: - img_url = self.extractor.get_top_img_url(self) - self.set_top_img(img_url) + meta_img_url = self.extractor.get_meta_img_url(self) + self.set_meta_img(meta_img_url) - if self.config.fetch_images: - top_imgs = self.extractor.get_img_urls(self) - self.set_imgs(top_imgs) + imgs = self.extractor.get_img_urls(self) + self.set_imgs(imgs) + + if self.clean_top_node is not None and not self.has_top_image(): + first_img = self.extractor.get_first_img_url(self) + self.set_top_img(first_img) - if self.config.fetch_images: + if not self.has_top_image(): self.set_reddit_top_img() - self.is_parsed = True - self.release_resources() + def has_top_image(self): + return self.top_img is not None and self.top_img != u'' def is_valid_url(self): """ @@ -351,16 +368,15 @@ def set_reddit_top_img(self, test_run=False): first, uses Reddit's img algorithm as a fallback. """ + #todo: move tests from here if test_run: s = images.Scraper(self) img = s.largest_image_url() print 'it worked, the img is', img - if self.top_img != u'': # if we already have a top img... - return try: s = images.Scraper(self) - self.set_top_img(s.largest_image_url()) + self.set_top_img_no_ckeck(s.largest_image_url()) except Exception, e: log.critical('jpeg error with PIL, %s' % e) @@ -401,8 +417,18 @@ def set_article_html(self, article_html): """ if article_html: self.article_html = encodeValue(article_html) + + def set_meta_img(self, src_url): + self.meta_img = encodeValue(src_url) + self.set_top_img(src_url) def set_top_img(self, src_url): + if src_url is not None: + s = images.Scraper(self) + if s.satisfies_requirements(src_url): + self.set_top_img_no_ckeck(src_url) + + def set_top_img_no_ckeck(self, src_url): """ We want to provide 2 api's for images. One at "top_img", "imgs" and one at "top_image", "images". diff --git a/newspaper/extractors.py b/newspaper/extractors.py index e057ccd9..7564c399 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -357,14 +357,23 @@ def get_img_urls(self, article): doc = article.raw_doc urls = self.parser.get_img_urls(doc) img_links = set([ urlparse.urljoin(article.url, url) for url in urls ]) + if article.meta_img: + img_links.add(article.meta_img) return img_links - def get_top_img_url(self, article): + def get_first_img_url(self, article): + node_images = self.parser.get_img_urls(article.clean_top_node) + if node_images: + return urlparse.urljoin(article.url, node_images[0]) + return u'' + + def get_meta_img_url(self, article): """ """ # !important, we must use raw_doc because at this point doc has been cleaned doc = article.raw_doc - return self.parser.get_top_img_url(doc) + meta_img_url = self.parser.get_meta_img_url(doc) + return urlparse.urljoin(article.url, meta_img_url) def get_category_urls(self, source, source_url=None, page_urls=None): """ diff --git a/newspaper/images.py b/newspaper/images.py index 8734d083..76434cff 100644 --- a/newspaper/images.py +++ b/newspaper/images.py @@ -21,6 +21,7 @@ chunk_size = 1024 thumbnail_size = 90, 90 +minimal_area = 5000 def image_to_str(image): s = StringIO.StringIO() @@ -153,7 +154,7 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False): if 'open_req' in locals(): open_req.close() -def fetch_size(url, useragent, referer=None, retries=1): +def fetch_image_dimension(url, useragent, referer=None, retries=1): return fetch_url(url, useragent, referer, retries, dimension=True) class Scraper: @@ -166,9 +167,9 @@ def __init__(self, article): self.useragent = self.config.browser_user_agent def largest_image_url(self): + #todo: remove. it is not responsibility of Scrapper if not self.imgs and not self.top_img: return None - if self.top_img: return self.top_img @@ -176,31 +177,8 @@ def largest_image_url(self): max_url = None for img_url in self.imgs: - size = fetch_size(img_url, self.useragent, referer=self.url) - if not size: - continue - - area = size[0] * size[1] - - # ignore little images - if area < 5000: - log.debug('ignore little %s' % img_url) - continue - - # PIL won't scale up, so we set a min width and - # maintain the aspect ratio - if size[0] < thumbnail_size[0]: - continue - - # ignore excessively long/wide images - if max(size) / min(size) > self.config.image_dimension_ration: - log.debug('ignore dims %s' % img_url) - continue - - # penalize images with "sprite" in their name - if 'sprite' in img_url.lower(): - log.debug('penalizing sprite %s' % img_url) - area /= 10 + dimension = fetch_image_dimension(img_url, self.useragent, referer=self.url) + area = self.calculate_area(img_url, dimension) if area > max_area: max_area = area @@ -208,6 +186,41 @@ def largest_image_url(self): log.debug('using max img ' + max_url) return max_url + + def calculate_area(self, img_url, dimension): + if not dimension: + return 0 + + area = dimension[0] * dimension[1] + + #todo: introduce filter classes for each case + # ignore little images + if area < minimal_area: + log.debug('ignore little %s' % img_url) + return 0 + + # PIL won't scale up, so we set a min width and + # maintain the aspect ratio + if dimension[0] < thumbnail_size[0]: + return 0 + + # ignore excessively long/wide images + if max(dimension) / min(dimension) > self.config.image_dimension_ration: + log.debug('ignore dims %s' % img_url) + return 0 + + # penalize images with "sprite" in their name + lower_case_url = img_url.lower() + if 'sprite' in lower_case_url or 'logo' in lower_case_url: + log.debug('penalizing sprite %s' % img_url) + area /= 10 + + return area + + def satisfies_requirements(self, img_url): + dimension = fetch_image_dimension(img_url, self.useragent, referer=self.url) + area = self.calculate_area(img_url, dimension) + return area > minimal_area def thumbnail(self): """ diff --git a/newspaper/parsers.py b/newspaper/parsers.py index 1cd7adae..3749983f 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -74,7 +74,7 @@ def get_urls(cls, _input, titles=False, regex=False): return cls.root_to_urls(doc, titles) @classmethod - def get_top_img_url(cls, doc): + def get_meta_img_url(cls, doc): """ Takes an lxml doc and returns the top img url running as method == 'soup' assumes lxml's soupparser. diff --git a/newspaper/text.py b/newspaper/text.py index 8c4d88ad..cc69a1ae 100644 --- a/newspaper/text.py +++ b/newspaper/text.py @@ -71,9 +71,14 @@ def __init__(self, language='en'): def remove_punctuation(self, content): # code taken form # http://stackoverflow.com/questions/265960/best-way-to-strip-punctuation-from-a-string-in-python - if isinstance(content, unicode): + content_is_unicode = isinstance(content, unicode) + if content_is_unicode: content = content.encode('utf-8') - return content.translate(self.TRANS_TABLE, string.punctuation) + stripped_input = content.translate(self.TRANS_TABLE, string.punctuation) + + if content_is_unicode: + return stripped_input.decode('utf-8') + return stripped_input def candidate_words(self, stripped_input): return stripped_input.split(' ')