From 6aa62d096c47ce80cb3c5fc82a1cc88c0b6ff7fc Mon Sep 17 00:00:00 2001 From: Oleg Temnov Date: Thu, 30 Jan 2014 05:19:09 +0400 Subject: [PATCH 1/6] Use first image from article top_node --- newspaper/article.py | 35 +++++++++++++++++++++++------------ newspaper/extractors.py | 6 ++++++ 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/newspaper/article.py b/newspaper/article.py index 2714eabe..ee58fdcf 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -110,6 +110,9 @@ def __init__(self, url, title=u'', source_url=u'', config=None, **kwargs): # Holds the top Element we think is a candidate for the main body self.top_node = None + # Holds clean version of top Element + self.clean_top_node = None + # the lxml doc object self.doc = None @@ -198,24 +201,35 @@ def parse(self): self.set_movies(video_extractor.get_videos()) self.top_node = self.extractor.post_cleanup(self.top_node) + self.clean_top_node = copy.deepcopy(self.top_node) + text, article_html = output_formatter.get_formatted(self) self.set_article_html(article_html) self.set_text(text) + if self.config.fetch_images: + self.fetch_images() + + self.is_parsed = True + self.release_resources() + + def fetch_images(self): if self.raw_doc is not None: - if self.config.fetch_images: - img_url = self.extractor.get_top_img_url(self) - self.set_top_img(img_url) + img_url = self.extractor.get_top_img_url(self) + self.set_top_img(img_url) - if self.config.fetch_images: - top_imgs = self.extractor.get_img_urls(self) - self.set_imgs(top_imgs) + top_imgs = self.extractor.get_img_urls(self) + self.set_imgs(top_imgs) + + if self.clean_top_node is not None and not self.has_top_image(): + first_img = self.extractor.get_first_img_url(self.clean_top_node) + self.set_top_img(first_img) - if self.config.fetch_images: + if not self.has_top_image(): self.set_reddit_top_img() - self.is_parsed = True - self.release_resources() + def has_top_image(self): + return self.top_img is not None and self.top_img != u'' def is_valid_url(self): """ @@ -350,14 +364,11 @@ def set_reddit_top_img(self, test_run=False): Wrapper for setting images, queries known image attributes first, uses Reddit's img algorithm as a fallback. """ - if test_run: s = images.Scraper(self) img = s.largest_image_url() print 'it worked, the img is', img - if self.top_img != u'': # if we already have a top img... - return try: s = images.Scraper(self) self.set_top_img(s.largest_image_url()) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index e057ccd9..00570bc8 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -359,6 +359,12 @@ def get_img_urls(self, article): img_links = set([ urlparse.urljoin(article.url, url) for url in urls ]) return img_links + def get_first_img_url(self, node): + node_images = self.parser.get_img_urls(node) + if node_images: + return node_images[0] + return u'' + def get_top_img_url(self, article): """ """ From 0997aeaa811a9351c68cf98616074ad8f5eabd44 Mon Sep 17 00:00:00 2001 From: Oleg Temnov Date: Sun, 2 Feb 2014 23:06:32 +0400 Subject: [PATCH 2/6] Check image requirements before set --- newspaper/article.py | 10 ++++++- newspaper/images.py | 67 ++++++++++++++++++++++++++------------------ 2 files changed, 49 insertions(+), 28 deletions(-) diff --git a/newspaper/article.py b/newspaper/article.py index ee58fdcf..299a73dc 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -364,6 +364,8 @@ def set_reddit_top_img(self, test_run=False): Wrapper for setting images, queries known image attributes first, uses Reddit's img algorithm as a fallback. """ + + #todo: move tests from here if test_run: s = images.Scraper(self) img = s.largest_image_url() @@ -371,7 +373,7 @@ def set_reddit_top_img(self, test_run=False): try: s = images.Scraper(self) - self.set_top_img(s.largest_image_url()) + self.set_top_img_no_ckeck(s.largest_image_url()) except Exception, e: log.critical('jpeg error with PIL, %s' % e) @@ -414,6 +416,12 @@ def set_article_html(self, article_html): self.article_html = encodeValue(article_html) def set_top_img(self, src_url): + if src_url is not None: + s = images.Scraper(self) + if s.satisfies_requirements(src_url): + self.set_top_img_no_ckeck(src_url) + + def set_top_img_no_ckeck(self, src_url): """ We want to provide 2 api's for images. One at "top_img", "imgs" and one at "top_image", "images". diff --git a/newspaper/images.py b/newspaper/images.py index 8734d083..76434cff 100644 --- a/newspaper/images.py +++ b/newspaper/images.py @@ -21,6 +21,7 @@ chunk_size = 1024 thumbnail_size = 90, 90 +minimal_area = 5000 def image_to_str(image): s = StringIO.StringIO() @@ -153,7 +154,7 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False): if 'open_req' in locals(): open_req.close() -def fetch_size(url, useragent, referer=None, retries=1): +def fetch_image_dimension(url, useragent, referer=None, retries=1): return fetch_url(url, useragent, referer, retries, dimension=True) class Scraper: @@ -166,9 +167,9 @@ def __init__(self, article): self.useragent = self.config.browser_user_agent def largest_image_url(self): + #todo: remove. it is not responsibility of Scrapper if not self.imgs and not self.top_img: return None - if self.top_img: return self.top_img @@ -176,31 +177,8 @@ def largest_image_url(self): max_url = None for img_url in self.imgs: - size = fetch_size(img_url, self.useragent, referer=self.url) - if not size: - continue - - area = size[0] * size[1] - - # ignore little images - if area < 5000: - log.debug('ignore little %s' % img_url) - continue - - # PIL won't scale up, so we set a min width and - # maintain the aspect ratio - if size[0] < thumbnail_size[0]: - continue - - # ignore excessively long/wide images - if max(size) / min(size) > self.config.image_dimension_ration: - log.debug('ignore dims %s' % img_url) - continue - - # penalize images with "sprite" in their name - if 'sprite' in img_url.lower(): - log.debug('penalizing sprite %s' % img_url) - area /= 10 + dimension = fetch_image_dimension(img_url, self.useragent, referer=self.url) + area = self.calculate_area(img_url, dimension) if area > max_area: max_area = area @@ -208,6 +186,41 @@ def largest_image_url(self): log.debug('using max img ' + max_url) return max_url + + def calculate_area(self, img_url, dimension): + if not dimension: + return 0 + + area = dimension[0] * dimension[1] + + #todo: introduce filter classes for each case + # ignore little images + if area < minimal_area: + log.debug('ignore little %s' % img_url) + return 0 + + # PIL won't scale up, so we set a min width and + # maintain the aspect ratio + if dimension[0] < thumbnail_size[0]: + return 0 + + # ignore excessively long/wide images + if max(dimension) / min(dimension) > self.config.image_dimension_ration: + log.debug('ignore dims %s' % img_url) + return 0 + + # penalize images with "sprite" in their name + lower_case_url = img_url.lower() + if 'sprite' in lower_case_url or 'logo' in lower_case_url: + log.debug('penalizing sprite %s' % img_url) + area /= 10 + + return area + + def satisfies_requirements(self, img_url): + dimension = fetch_image_dimension(img_url, self.useragent, referer=self.url) + area = self.calculate_area(img_url, dimension) + return area > minimal_area def thumbnail(self): """ From 1f134ac632a291e78f71327e4fee7184f1da72ad Mon Sep 17 00:00:00 2001 From: Oleg Temnov Date: Mon, 3 Feb 2014 10:41:05 +0400 Subject: [PATCH 3/6] Initialize images and top_mage --- newspaper/article.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/newspaper/article.py b/newspaper/article.py index 299a73dc..aa624bc7 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -59,9 +59,9 @@ def __init__(self, url, title=u'', source_url=u'', config=None, **kwargs): self.title = encodeValue(title) # the url of the "best image" to represent this article, via reddit algorithm - self.top_img = u'' + self.top_img = self.top_image = u'' - self.imgs = [] # all image urls + self.imgs = self.images = [] # all image urls self.movies = [] # youtube, vimeo, etc # pure text from the article From e278edf7042ae7fd4e780ae387f1cb196ffaf9a0 Mon Sep 17 00:00:00 2001 From: Oleg Temnov Date: Mon, 3 Feb 2014 11:38:51 +0400 Subject: [PATCH 4/6] Include meta image url to images collection --- newspaper/article.py | 15 +++++++++++---- newspaper/extractors.py | 6 ++++-- newspaper/parsers.py | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/newspaper/article.py b/newspaper/article.py index aa624bc7..c5a06532 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -61,6 +61,9 @@ def __init__(self, url, title=u'', source_url=u'', config=None, **kwargs): # the url of the "best image" to represent this article, via reddit algorithm self.top_img = self.top_image = u'' + # stores image provided by metadata + self.meta_img = u'' + self.imgs = self.images = [] # all image urls self.movies = [] # youtube, vimeo, etc @@ -215,11 +218,11 @@ def parse(self): def fetch_images(self): if self.raw_doc is not None: - img_url = self.extractor.get_top_img_url(self) - self.set_top_img(img_url) + meta_img_url = self.extractor.get_meta_img_url(self) + self.set_meta_img(meta_img_url) - top_imgs = self.extractor.get_img_urls(self) - self.set_imgs(top_imgs) + imgs = self.extractor.get_img_urls(self) + self.set_imgs(imgs) if self.clean_top_node is not None and not self.has_top_image(): first_img = self.extractor.get_first_img_url(self.clean_top_node) @@ -414,6 +417,10 @@ def set_article_html(self, article_html): """ if article_html: self.article_html = encodeValue(article_html) + + def set_meta_img(self, src_url): + self.meta_img = encodeValue(src_url) + self.set_top_img(src_url) def set_top_img(self, src_url): if src_url is not None: diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 00570bc8..26db840f 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -357,6 +357,8 @@ def get_img_urls(self, article): doc = article.raw_doc urls = self.parser.get_img_urls(doc) img_links = set([ urlparse.urljoin(article.url, url) for url in urls ]) + if article.meta_img: + img_links.add(article.meta_img) return img_links def get_first_img_url(self, node): @@ -365,12 +367,12 @@ def get_first_img_url(self, node): return node_images[0] return u'' - def get_top_img_url(self, article): + def get_meta_img_url(self, article): """ """ # !important, we must use raw_doc because at this point doc has been cleaned doc = article.raw_doc - return self.parser.get_top_img_url(doc) + return self.parser.get_meta_img_url(doc) def get_category_urls(self, source, source_url=None, page_urls=None): """ diff --git a/newspaper/parsers.py b/newspaper/parsers.py index 1cd7adae..3749983f 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -74,7 +74,7 @@ def get_urls(cls, _input, titles=False, regex=False): return cls.root_to_urls(doc, titles) @classmethod - def get_top_img_url(cls, doc): + def get_meta_img_url(cls, doc): """ Takes an lxml doc and returns the top img url running as method == 'soup' assumes lxml's soupparser. From b0ec588877fcbb5d97d1c46d3ac246613bdc9a7f Mon Sep 17 00:00:00 2001 From: Oleg Temnov Date: Mon, 3 Feb 2014 11:59:41 +0400 Subject: [PATCH 5/6] Resolve relative image urls --- newspaper/article.py | 2 +- newspaper/extractors.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/newspaper/article.py b/newspaper/article.py index c5a06532..64ef10d7 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -225,7 +225,7 @@ def fetch_images(self): self.set_imgs(imgs) if self.clean_top_node is not None and not self.has_top_image(): - first_img = self.extractor.get_first_img_url(self.clean_top_node) + first_img = self.extractor.get_first_img_url(self) self.set_top_img(first_img) if not self.has_top_image(): diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 26db840f..7564c399 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -361,10 +361,10 @@ def get_img_urls(self, article): img_links.add(article.meta_img) return img_links - def get_first_img_url(self, node): - node_images = self.parser.get_img_urls(node) + def get_first_img_url(self, article): + node_images = self.parser.get_img_urls(article.clean_top_node) if node_images: - return node_images[0] + return urlparse.urljoin(article.url, node_images[0]) return u'' def get_meta_img_url(self, article): @@ -372,7 +372,8 @@ def get_meta_img_url(self, article): """ # !important, we must use raw_doc because at this point doc has been cleaned doc = article.raw_doc - return self.parser.get_meta_img_url(doc) + meta_img_url = self.parser.get_meta_img_url(doc) + return urlparse.urljoin(article.url, meta_img_url) def get_category_urls(self, source, source_url=None, page_urls=None): """ From d77c3a3434f207215bfc1bf082c4e72456deb908 Mon Sep 17 00:00:00 2001 From: Oleg Temnov Date: Wed, 5 Feb 2014 04:28:12 +0400 Subject: [PATCH 6/6] Fix for stopwords - wrongly checks bytestring vs unicode set Example: http://top.rbc.ru/economics/04/02/2014/903127.shtml --- newspaper/text.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/newspaper/text.py b/newspaper/text.py index 8c4d88ad..cc69a1ae 100644 --- a/newspaper/text.py +++ b/newspaper/text.py @@ -71,9 +71,14 @@ def __init__(self, language='en'): def remove_punctuation(self, content): # code taken form # http://stackoverflow.com/questions/265960/best-way-to-strip-punctuation-from-a-string-in-python - if isinstance(content, unicode): + content_is_unicode = isinstance(content, unicode) + if content_is_unicode: content = content.encode('utf-8') - return content.translate(self.TRANS_TABLE, string.punctuation) + stripped_input = content.translate(self.TRANS_TABLE, string.punctuation) + + if content_is_unicode: + return stripped_input.decode('utf-8') + return stripped_input def candidate_words(self, stripped_input): return stripped_input.split(' ')