diff --git a/newspaper/__init__.py b/newspaper/__init__.py index 08cc8e20..53df549a 100644 --- a/newspaper/__init__.py +++ b/newspaper/__init__.py @@ -8,15 +8,13 @@ __copyright__ = 'Copyright 2014, Lucas Ou-Yang' from .article import Article, ArticleException +from .api import (build, build_article, hot, languages, popular_urls, + NewsPool, Configuration as Config) from .source import Source - -from .api import build, build_article, popular_urls, hot, languages -from .api import NewsPool, Configuration as Config +from .version import __version__ news_pool = NewsPool() -from .version import __version__ - # Set default logging handler to avoid "No handler found" warnings. import logging diff --git a/newspaper/api.py b/newspaper/api.py index 71211e77..c45a0955 100644 --- a/newspaper/api.py +++ b/newspaper/api.py @@ -1,65 +1,63 @@ # -*- coding: utf-8 -*- """ +Ignore the unused imports, this file's purpose is to make visible +anything which a user might need to import from newspaper. +View newspaper/__init__.py for its usage. """ __title__ = 'newspaper' __author__ = 'Lucas Ou-Yang' __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' -from .packages.feedparser import feedparser -from .source import Source from .article import Article -from .settings import POPULAR_URLS, TRENDING_URL from .configuration import Configuration from .mthreading import NewsPool -from .configuration import Configuration -from .utils import print_available_languages, extend_config +from .packages.feedparser import feedparser +from .settings import POPULAR_URLS, TRENDING_URL +from .source import Source +from .utils import extend_config, print_available_languages + def build(url=u'', dry=False, config=None, **kwargs): + """Returns a constructed source object without + downloading or parsing the articles """ - Returns a constructed source object without - downloading or parsing the articles. - """ - config = config or Configuration() # Order matters + config = config or Configuration() config = extend_config(config, kwargs) - url = url or '' s = Source(url, config=config) - - # dry means we are just testing, don't actually build source if not dry: s.build() return s + def build_article(url=u'', config=None, **kwargs): + """Returns a constructed article object without downloading + or parsing """ - Returns a constructed article object without - downloading or parsing. - """ - config = config or Configuration() # Order matters + config = config or Configuration() config = extend_config(config, kwargs) - url = url or '' a = Article(url, config=config) return a + def languages(): - """ - Returns a list of the supported languages. + """Returns a list of the supported languages """ print_available_languages() + def popular_urls(): - """ - Returns a list of pre-extracted popular source urls. + """Returns a list of pre-extracted popular source urls """ with open(POPULAR_URLS) as f: urls = ['http://' + u.strip() for u in f.readlines()] return urls + def hot(): - """ - Returns a list of hit terms via google trends. + """Returns a list of hit terms via google trends """ try: listing = feedparser.parse(TRENDING_URL)['entries'] diff --git a/newspaper/article.py b/newspaper/article.py index 41231721..3cddb440 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -1,7 +1,4 @@ # -*- coding: utf-8 -*- -""" -Article objects abstract an online news article page. -""" __title__ = 'newspaper' __author__ = 'Lucas Ou-Yang' __license__ = 'MIT' @@ -12,152 +9,153 @@ import os import glob -from . import nlp from . import images from . import network +from . import nlp from . import settings +from . import urls + +from .cleaners import DocumentCleaner from .configuration import Configuration -from .extractors import StandardContentExtractor -from .utils import (URLHelper, - encodeValue, - RawHelper, - extend_config, +from .extractors import ContentExtractor +from .outputformatters import OutputFormatter +from .utils import (URLHelper, encodeValue, RawHelper, extend_config, get_available_languages) -from .cleaners import StandardDocumentCleaner -from .outputformatters import StandardOutputFormatter from .videos.extractors import VideoExtractor -from .urls import (prepare_url, - get_domain, - get_scheme, - valid_url) log = logging.getLogger(__name__) + class ArticleException(Exception): pass + class Article(object): - """ + """Article objects abstract an online news article page """ def __init__(self, url, title=u'', source_url=u'', config=None, **kwargs): - """ - The **kwargs arguement can be filled with config values which we then - push in. + """The **kwargs argument may be filled with config values, which + is added into the config object """ self.config = config or Configuration() self.config = extend_config(self.config, kwargs) - self.parser = self.config.get_parser() - self.extractor = self.get_extractor() + self.extractor = ContentExtractor(self.config) if source_url == u'': - source_url = get_scheme(url) + '://' + get_domain(url) + source_url = urls.get_scheme(url) + '://' + urls.get_domain(url) if source_url is None or source_url == '': raise ArticleException('input url bad format') - # if no attached source object, we just fallback on scheme + domain of url + # URL to the main page of the news source which owns this article self.source_url = encodeValue(source_url) url = encodeValue(url) - self.url = prepare_url(url, self.source_url) + self.url = urls.prepare_url(url, self.source_url) self.title = encodeValue(title) - # the url of the "best image" to represent this article, via reddit algorithm + # URL of the "best image" to represent this article self.top_img = self.top_image = u'' # stores image provided by metadata self.meta_img = u'' - self.imgs = self.images = [] # all image urls - self.movies = [] # youtube, vimeo, etc + # All image urls in this article + self.imgs = self.images = [] - # pure text from the article + # All videos in this article: youtube, vimeo, etc + self.movies = [] + + # Body text from this article self.text = u'' - # keywords extracted via nlp() from the body text - # meta_keywords are via parse() from tags - # tags are related terms via parse() in the tags + # `keywords` are extracted via nlp() from the body text self.keywords = [] + + # `meta_keywords` are extracted via parse() from tags self.meta_keywords = [] + + # `tags` are also extracted via parse() from tags self.tags = set() - # list of authors who have published the article, via parse() + # List of authors who have published the article, via parse() self.authors = [] - self.published_date = u'' # TODO + # TODO: Date of when this article was published + self.published_date = u'' - # summary generated from the article's body txt + # Summary generated from the article's body txt self.summary = u'' - # the article's unchanged and raw html + # This article's unchanged and raw HTML self.html = u'' - # The html of the main article node + # The HTML of this article's main node (most important part) self.article_html = u'' - # flags warning users in-case they forget to download() or parse() + # Flags warning users in-case they forget to download() or parse() + # or if they call methods out of order self.is_parsed = False self.is_downloaded = False - # meta description field in HTML source + # Meta description field in the HTML source self.meta_description = u"" - # meta lang field in HTML source + # Meta language field in HTML source self.meta_lang = u"" - # meta favicon field in HTML source + # Meta favicon field in HTML source self.meta_favicon = u"" - # Meta tags contain a lot of structured data like OpenGraph + # Meta tags contain a lot of structured data, e.g. OpenGraph self.meta_data = {} # The canonical link of this article if found in the meta data self.canonical_link = u"" - # Holds the top Element we think is a candidate for the main body + # Holds the top element of the DOM that we determine is a candidate + # for the main body of the article self.top_node = None - # Holds clean version of top Element + # A deepcopied clone of the above object before heavy parsing + # operations, useful for users to query data in the + # "most important part of the page" self.clean_top_node = None - # the lxml doc object + # lxml DOM object generated from HTML self.doc = None - # a pure object from the orig html without any cleaning options done on it + # A deepcopied clone of the above object before undergoing heavy + # cleaning operations, serves as an API if users need to query the DOM self.clean_doc = None - # A property bucket for consumers of goose to store custom data extractions. + # A property dict for users to store custom data. self.additional_data = {} - def build(self): - """ - Build a lone article from a url independent of the - source (newspaper). We won't normally call this method b/c - we want to multithread articles on a source (newspaper) level. + """Build a lone article from a URL independent of the source (newspaper). + Don't normally call this method b/c it's good to multithread articles + on a source (newspaper) level. """ self.download() self.parse() self.nlp() def download(self): - """ - Downloads the link's html content, don't use if we are async - downloading batch articles. + """Downloads the link's HTML content, don't use if you are batch async + downloading articles """ html = network.get_html(self.url, self.config) self.set_html(html) def parse(self): - """ - """ if not self.is_downloaded: print 'You must download() an article before parsing it!' raise ArticleException() - self.doc = self.parser.fromstring(self.html) + self.doc = self.config.get_parser().fromstring(self.html) self.clean_doc = copy.deepcopy(self.doc) if self.doc is None: @@ -166,56 +164,61 @@ def parse(self): # TODO: Fix this, sync in our fix_url() method parse_candidate = self.get_parse_candidate() - self.link_hash = parse_candidate.link_hash # MD5 + self.link_hash = parse_candidate.link_hash # MD5 - document_cleaner = self.get_document_cleaner() - output_formatter = self.get_output_formatter() + document_cleaner = DocumentCleaner(self.config) + output_formatter = OutputFormatter(self.config) - title = self.extractor.get_title(self) + title = self.extractor.get_title(self.clean_doc) self.set_title(title) - authors = self.extractor.get_authors(self) + authors = self.extractor.get_authors(self.clean_doc) self.set_authors(authors) - meta_lang = self.extractor.get_meta_lang(self) + meta_lang = self.extractor.get_meta_lang(self.clean_doc) self.set_meta_language(meta_lang) - self.extractor.update_language(self) - output_formatter.update_language(self) + if self.config.use_meta_language: + self.extractor.update_language(self.meta_lang) + output_formatter.update_language(self.meta_lang) - meta_favicon = self.extractor.get_favicon(self) + meta_favicon = self.extractor.get_favicon(self.clean_doc) self.set_meta_favicon(meta_favicon) - meta_description = self.extractor.get_meta_description(self) + meta_description = \ + self.extractor.get_meta_description(self.clean_doc) self.set_meta_description(meta_description) - canonical_link = self.extractor.get_canonical_link(self) + canonical_link = self.extractor.get_canonical_link( + self.url, self.clean_doc) self.set_canonical_link(canonical_link) - tags = self.extractor.extract_tags(self) + tags = self.extractor.extract_tags(self.clean_doc) self.set_tags(tags) - meta_keywords = self.extractor.get_meta_keywords(self) + meta_keywords = self.extractor.get_meta_keywords( + self.clean_doc) self.set_meta_keywords(meta_keywords) - meta_data = self.extractor.get_meta_data(self) + meta_data = self.extractor.get_meta_data(self.clean_doc) self.set_meta_data(meta_data) - # TODO self.publish_date = self.config.publishDateExtractor.extract(self.doc) + # TODO self.publish_date = ... - # before we do any computations on the body itself, we must clean up the document - self.doc = document_cleaner.clean(self) + # Before any computations on the body, clean DOM object + self.doc = document_cleaner.clean(self.doc) text = u'' - self.top_node = self.extractor.calculate_best_node(self) + self.top_node = self.extractor.calculate_best_node(self.doc) if self.top_node is not None: - video_extractor = self.get_video_extractor(self) + video_extractor = VideoExtractor(self.config, self.top_node) self.set_movies(video_extractor.get_videos()) self.top_node = self.extractor.post_cleanup(self.top_node) self.clean_top_node = copy.deepcopy(self.top_node) - text, article_html = output_formatter.get_formatted(self) + text, article_html = output_formatter.get_formatted( + self.top_node) self.set_article_html(article_html) self.set_text(text) @@ -227,14 +230,18 @@ def parse(self): def fetch_images(self): if self.clean_doc is not None: - meta_img_url = self.extractor.get_meta_img_url(self) + meta_img_url = self.extractor.get_meta_img_url( + self.url, self.clean_doc) self.set_meta_img(meta_img_url) - imgs = self.extractor.get_img_urls(self) + imgs = self.extractor.get_img_urls(self.url, self.clean_doc) + if self.meta_img: + imgs.add(self.meta_img) self.set_imgs(imgs) if self.clean_top_node is not None and not self.has_top_image(): - first_img = self.extractor.get_first_img_url(self) + first_img = self.extractor.get_first_img_url( + self.url, self.clean_top_node) self.set_top_img(first_img) if not self.has_top_image(): @@ -244,25 +251,23 @@ def has_top_image(self): return self.top_img is not None and self.top_img != u'' def is_valid_url(self): + """Performs a check on the url of this link to determine if article + is a real news article or not """ - Performs a check on the url of this link to - determine if a real news article or not. - """ - return valid_url(self.url) + return urls.valid_url(self.url) def is_valid_body(self): - """ - If the article's body text is long enough to meet - standard article requirements, we keep the article. + """If the article's body text is long enough to meet + standard article requirements, keep the article """ if not self.is_parsed: raise ArticleException('must parse article before checking \ if it\'s body is valid!') - meta_type = self.extractor.get_meta_type(self) + meta_type = self.extractor.get_meta_type(self.clean_doc) wordcount = self.text.split(' ') sentcount = self.text.split('.') - if meta_type == 'article' and wordcount > (self.config.MIN_WORD_COUNT - 50): + if meta_type == 'article' and wordcount > (self.config.MIN_WORD_COUNT): log.debug('%s verified for article and wc' % self.url) return True @@ -290,8 +295,8 @@ def is_valid_body(self): return True def is_media_news(self): - """ - If the article is a gallery, video, etc related. + """If the article is related heavily to media: + gallery, video, big pictures, etc """ safe_urls = ['/video', '/slide', '/gallery', '/powerpoint', '/fashion', '/glamour', '/cloth'] @@ -301,8 +306,7 @@ def is_media_news(self): return False def nlp(self): - """ - Keyword extraction wrapper. + """Keyword extraction wrapper """ if not self.is_downloaded or not self.is_parsed: print 'You must download and parse an article before parsing it!' @@ -318,39 +322,23 @@ def nlp(self): self.set_summary(summary) def get_parse_candidate(self): + """A parse candidate is a wrapper object holding a link hash of this + article and a final_url of the article """ - A parse candidate is a wrapper object holding a link hash of this - article and a final_url. - """ - # TODO: Should we actually compute a hash using the html? It is more inconvenient if we do that if self.html: return RawHelper.get_parsing_candidate(self.url, self.html) return URLHelper.get_parsing_candidate(self.url) - def get_video_extractor(self, article): - return VideoExtractor(article, self.config) - - def get_output_formatter(self): - return StandardOutputFormatter(self.config) - - def get_document_cleaner(self): - return StandardDocumentCleaner(self.config) - - def get_extractor(self): - return StandardContentExtractor(self.config) - def build_resource_path(self): - """ - Must be called after we compute html/final url. + """Must be called after computing HTML/final URL """ res_path = self.get_resource_path() if not os.path.exists(res_path): os.mkdir(res_path) def get_resource_path(self): - """ - Every article object has a special directory to store data in from - initialization to garbage collection. + """Every article object has a special directory to store data in from + initialization to garbage collection """ res_dir_fn = 'article_resources' resource_directory = os.path.join(settings.TOP_DIRECTORY, res_dir_fn) @@ -360,9 +348,7 @@ def get_resource_path(self): return dir_path def release_resources(self): - """ - TODO: Actually implement this properly. - """ + # TODO: implement in entirety path = self.get_resource_path() for fname in glob.glob(path): try: @@ -371,18 +357,10 @@ def release_resources(self): pass # os.remove(path) - def set_reddit_top_img(self, test_run=False): + def set_reddit_top_img(self): + """Wrapper for setting images. Queries known image attributes + first, then uses Reddit's imgage algorithm as a fallback. """ - Wrapper for setting images, queries known image attributes - first, uses Reddit's img algorithm as a fallback. - """ - - #todo: move tests from here - if test_run: - s = images.Scraper(self) - img = s.largest_image_url() - print 'it worked, the img is', img - try: s = images.Scraper(self) self.set_top_img_no_ckeck(s.largest_image_url()) @@ -390,13 +368,9 @@ def set_reddit_top_img(self, test_run=False): log.critical('jpeg error with PIL, %s' % e) def set_title(self, title): - """ - The prechecked_title boolean is important for cases where our - educated guess of an article's title works and is actually - better than the actual title being extracted. - """ - prechecked_title = (self.title and not title) - if prechecked_title: + if self.title and not title: + # Title has already been set by an educated guess and + # extraction failed return title = title[:self.config.MAX_TITLE] title = encodeValue(title) @@ -404,25 +378,20 @@ def set_title(self, title): self.title = title def set_text(self, text): - """ - """ - text = text[:self.config.MAX_TEXT-5] + text = text[:self.config.MAX_TEXT] text = encodeValue(text) if text: self.text = text def set_html(self, html): - """ - This method is quite important because many other objects - besides this one will be modifying and setting the html. + """Encode HTML before setting it """ self.is_downloaded = True if html: - self.html = encodeValue(html) + self.html = encodeValue(html) def set_article_html(self, article_html): - """ - Sets the html of just our article body, the "top node". + """Sets the HTML of just the article's `top_node` """ if article_html: self.article_html = encodeValue(article_html) @@ -438,35 +407,32 @@ def set_top_img(self, src_url): self.set_top_img_no_ckeck(src_url) def set_top_img_no_ckeck(self, src_url): - """ - We want to provide 2 api's for images. One at - "top_img", "imgs" and one at "top_image", "images". + """Provide 2 APIs for images. One at "top_img", "imgs" + and one at "top_image", "images" """ src_url = encodeValue(src_url) self.top_img = src_url self.top_image = src_url def set_imgs(self, imgs): - """ - The motive for this method is the same as above, we want - to provide apis for both "imgs" and "images". + """The motive for this method is the same as above, provide APIs + for both `article.imgs` and `article.images` """ imgs = [encodeValue(i) for i in imgs] self.images = imgs self.imgs = imgs def set_keywords(self, keywords): - """ - Keys are stored in list format. + """Keys are stored in list format """ if not isinstance(keywords, list): raise Exception("Keyword input must be list!") if keywords: - self.keywords = [encodeValue(k) for k in keywords[:self.config.MAX_KEYWORDS]] + self.keywords = [encodeValue(k) + for k in keywords[:self.config.MAX_KEYWORDS]] def set_authors(self, authors): - """ - Authors are in ["firstName lastName", "firstName lastName"] format. + """Authors are in ["firstName lastName", "firstName lastName"] format """ if not isinstance(authors, list): raise Exception("authors input must be list!") @@ -475,53 +441,41 @@ def set_authors(self, authors): self.authors = [encodeValue(author) for author in authors] def set_summary(self, summary): - """ - Summary is a paragraph of text from the title + body text. + """Summary here refers to a paragraph of text from the + title text and body text """ summary = summary[:self.config.MAX_SUMMARY] self.summary = encodeValue(summary) def set_meta_language(self, meta_lang): - """ - Save langauges in their ISO 2 char form + """Save langauges in their ISO 2-character form """ if meta_lang and len(meta_lang) >= 2 and \ meta_lang in get_available_languages(): self.meta_lang = meta_lang[:2] def set_meta_keywords(self, meta_keywords): - """ - Store the keys in list form. + """Store the keys in list form """ self.meta_keywords = [k.strip() for k in meta_keywords.split(',')] def set_meta_favicon(self, meta_favicon): - """ - """ self.meta_favicon = meta_favicon def set_meta_description(self, meta_description): - """ - """ self.meta_description = meta_description def set_meta_data(self, meta_data): self.meta_data = meta_data def set_canonical_link(self, canonical_link): - """ - """ self.canonical_link = canonical_link def set_tags(self, tags): - """ - """ self.tags = tags def set_movies(self, movie_objects): - """ - Trim goose's movie objects into just urls for us. + """Trim video objects into just urls """ movie_urls = [o.src for o in movie_objects if o and o.src] self.movies = movie_urls - diff --git a/newspaper/cleaners.py b/newspaper/cleaners.py index dd1b9203..5de87b06 100644 --- a/newspaper/cleaners.py +++ b/newspaper/cleaners.py @@ -5,30 +5,36 @@ """ from .utils import ReplaceSequence + class DocumentCleaner(object): def __init__(self, config): + """Set appropriate tag names and regexes of tags to remove + from the HTML + """ self.config = config self.parser = self.config.get_parser() - self.remove_nodes_re = ( - "^side$|combx|retweet|mediaarticlerelated|menucontainer|" - "navbar|storytopbar-bucket|utility-bar|inline-share-tools" - "|comment|PopularQuestions|contact|foot|footer|Footer|footnote" - "|cnn_strycaptiontxt|cnn_html_slideshow|cnn_strylftcntnt" - "|links|meta$|shoutbox|sponsor" - "|tags|socialnetworking|socialNetworking|cnnStryHghLght" - "|cnn_stryspcvbx|^inset$|pagetools|post-attributes" - "|welcome_form|contentTools2|the_answers" - "|communitypromo|runaroundLeft|subscribe|vcard|articleheadings" - "|date|^print$|popup|author-dropdown|tools|socialtools|byline" - "|konafilter|KonaFilter|breadcrumbs|^fn$|wp-caption-text" - "|legende|ajoutVideo|timestamp|js_replies" + "^side$|combx|retweet|mediaarticlerelated|menucontainer|" + "navbar|storytopbar-bucket|utility-bar|inline-share-tools" + "|comment|PopularQuestions|contact|foot|footer|Footer|footnote" + "|cnn_strycaptiontxt|cnn_html_slideshow|cnn_strylftcntnt" + "|links|meta$|shoutbox|sponsor" + "|tags|socialnetworking|socialNetworking|cnnStryHghLght" + "|cnn_stryspcvbx|^inset$|pagetools|post-attributes" + "|welcome_form|contentTools2|the_answers" + "|communitypromo|runaroundLeft|subscribe|vcard|articleheadings" + "|date|^print$|popup|author-dropdown|tools|socialtools|byline" + "|konafilter|KonaFilter|breadcrumbs|^fn$|wp-caption-text" + "|legende|ajoutVideo|timestamp|js_replies" ) self.regexp_namespace = "http://exslt.org/regular-expressions" - self.nauthy_ids_re = "//*[re:test(@id, '%s', 'i')]" % self.remove_nodes_re - self.nauthy_classes_re = "//*[re:test(@class, '%s', 'i')]" % self.remove_nodes_re - self.nauthy_names_re = "//*[re:test(@name, '%s', 'i')]" % self.remove_nodes_re + self.nauthy_ids_re = ("//*[re:test(@id, '%s', 'i')]" % + self.remove_nodes_re) + self.nauthy_classes_re = ("//*[re:test(@class, '%s', 'i')]" % + self.remove_nodes_re) + self.nauthy_names_re = ("//*[re:test(@name, '%s', 'i')]" % + self.remove_nodes_re) self.div_to_p_re = r"<(a|blockquote|dl|div|img|ol|p|pre|table|ul)" self.caption_re = "^caption$" self.google_re = " google " @@ -37,14 +43,13 @@ def __init__(self, config): self.facebook_braodcasting_re = "facebook-broadcasting" self.twitter_re = "[^-]twitter" self.tablines_replacements = ReplaceSequence()\ - .create("\n", "\n\n")\ - .append("\t")\ - .append("^\\s+$") + .create("\n", "\n\n")\ + .append("\t")\ + .append("^\\s+$") - def clean(self, article): + def clean(self, doc_to_clean): + """Remove chunks of the DOM as specified """ - """ - doc_to_clean = article.doc doc_to_clean = self.clean_body_classes(doc_to_clean) doc_to_clean = self.clean_article_tags(doc_to_clean) doc_to_clean = self.clean_em_tags(doc_to_clean) @@ -55,7 +60,8 @@ def clean(self, article): doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.google_re) doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.entries_re) doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.facebook_re) - doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.facebook_braodcasting_re) + doc_to_clean = self.remove_nodes_regex(doc_to_clean, + self.facebook_braodcasting_re) doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.twitter_re) doc_to_clean = self.clean_para_spans(doc_to_clean) doc_to_clean = self.div_to_para(doc_to_clean, 'div') @@ -63,9 +69,9 @@ def clean(self, article): return doc_to_clean def clean_body_classes(self, doc): - # we don't need body classes - # in case it matches an unwanted class all the document - # will be empty + """Removes the `class` attribute from the <body> tag because + if there is a bad match, the entire DOM will be empty! + """ elements = self.parser.getElementsByTag(doc, tag="body") if elements: self.parser.delAttribute(elements[0], attr="class") @@ -87,10 +93,10 @@ def clean_em_tags(self, doc): return doc def remove_drop_caps(self, doc): - items = self.parser.css_select(doc, "span[class~=dropcap], span[class~=drop_cap]") + items = self.parser.css_select(doc, 'span[class~=dropcap], ' + 'span[class~=drop_cap]') for item in items: self.parser.drop_tag(item) - return doc def remove_scripts_styles(self, doc): @@ -98,12 +104,10 @@ def remove_scripts_styles(self, doc): scripts = self.parser.getElementsByTag(doc, tag='script') for item in scripts: self.parser.remove(item) - # remove styles styles = self.parser.getElementsByTag(doc, tag='style') for item in styles: self.parser.remove(item) - # remove comments comments = self.parser.getComments(doc) for item in comments: @@ -116,17 +120,14 @@ def clean_bad_tags(self, doc): naughty_list = self.parser.xpath_re(doc, self.nauthy_ids_re) for node in naughty_list: self.parser.remove(node) - # class naughty_classes = self.parser.xpath_re(doc, self.nauthy_classes_re) for node in naughty_classes: self.parser.remove(node) - # name naughty_names = self.parser.xpath_re(doc, self.nauthy_names_re) for node in naughty_names: self.parser.remove(node) - return doc def remove_nodes_regex(self, doc, pattern): @@ -146,53 +147,54 @@ def clean_para_spans(self, doc): def get_flushed_buffer(self, replacement_text, doc): return self.parser.textToPara(replacement_text) + def replace_walk_left_right(self, kid, kid_text, + replacement_text, nodes_to_remove): + kid_text_node = kid + replace_text = self.tablines_replacements.replaceAll(kid_text) + if len(replace_text) > 1: + prev_node = self.parser.previousSibling(kid_text_node) + while prev_node is not None \ + and self.parser.getTag(prev_node) == "a" \ + and self.parser.getAttribute( + prev_node, 'grv-usedalready') != 'yes': + outer = " " + self.parser.outerHtml(prev_node) + " " + replacement_text.append(outer) + nodes_to_remove.append(prev_node) + self.parser.setAttribute(prev_node, attr='grv-usedalready', + value='yes') + prev_node = self.parser.previousSibling(prev_node) + + replacement_text.append(replace_text) + next_node = self.parser.nextSibling(kid_text_node) + while next_node is not None \ + and self.parser.getTag(next_node) == "a" \ + and self.parser.getAttribute( + next_node, 'grv-usedalready') != 'yes': + outer = " " + self.parser.outerHtml(next_node) + " " + replacement_text.append(outer) + nodes_to_remove.append(next_node) + self.parser.setAttribute(next_node, attr='grv-usedalready', + value='yes') + next_node = self.parser.nextSibling(next_node) + def get_replacement_nodes(self, doc, div): replacement_text = [] nodes_to_return = [] nodes_to_remove = [] - childs = self.parser.childNodesWithText(div) - - for kid in childs: - # node is a p - # and already have some replacement text + kids = self.parser.childNodesWithText(div) + for kid in kids: + # The node is a <p> and already has some replacement text if self.parser.getTag(kid) == 'p' and len(replacement_text) > 0: - newNode = self.get_flushed_buffer(''.join(replacement_text), doc) - nodes_to_return.append(newNode) + new_node = self.get_flushed_buffer( + ''.join(replacement_text), doc) + nodes_to_return.append(new_node) replacement_text = [] nodes_to_return.append(kid) - # node is a text node + # The node is a text node elif self.parser.isTextNode(kid): - kid_text_node = kid kid_text = self.parser.getText(kid) - replace_text = self.tablines_replacements.replaceAll(kid_text) - if(len(replace_text)) > 1: - previous_sibling_node = self.parser.previousSibling(kid_text_node) - while previous_sibling_node is not None \ - and self.parser.getTag(previous_sibling_node) == "a" \ - and self.parser.getAttribute(previous_sibling_node, 'grv-usedalready') != 'yes': - outer = " " + self.parser.outerHtml(previous_sibling_node) + " " - replacement_text.append(outer) - nodes_to_remove.append(previous_sibling_node) - self.parser.setAttribute(previous_sibling_node, - attr='grv-usedalready', value='yes') - prev = self.parser.previousSibling(previous_sibling_node) - previous_sibling_node = prev if prev is not None else None - # append replace_text - replacement_text.append(replace_text) - # - next_sibling_node = self.parser.nextSibling(kid_text_node) - while next_sibling_node is not None \ - and self.parser.getTag(next_sibling_node) == "a" \ - and self.parser.getAttribute(next_sibling_node, 'grv-usedalready') != 'yes': - outer = " " + self.parser.outerHtml(next_sibling_node) + " " - replacement_text.append(outer) - nodes_to_remove.append(next_sibling_node) - self.parser.setAttribute(next_sibling_node, - attr='grv-usedalready', value='yes') - next = self.parser.nextSibling(next_sibling_node) - previous_sibling_node = next if next is not None else None - - # otherwise + self.replace_walk_left_right(kid, kid_text, replacement_text, + nodes_to_remove) else: nodes_to_return.append(kid) @@ -214,8 +216,8 @@ def div_to_para(self, doc, dom_type): bad_divs = 0 else_divs = 0 divs = self.parser.getElementsByTag(doc, tag=dom_type) - tags = ['a', 'blockquote', 'dl', 'div', 'img', 'ol', 'p', 'pre', 'table', 'ul'] - + tags = ['a', 'blockquote', 'dl', 'div', 'img', 'ol', 'p', + 'pre', 'table', 'ul'] for div in divs: items = self.parser.getElementsByTags(div, tags) if div is not None and len(items) == 0: @@ -224,14 +226,7 @@ def div_to_para(self, doc, dom_type): elif div is not None: replaceNodes = self.get_replacement_nodes(doc, div) div.clear() - for c, n in enumerate(replaceNodes): div.insert(c, n) - else_divs += 1 - return doc - - -class StandardDocumentCleaner(DocumentCleaner): - pass diff --git a/newspaper/configuration.py b/newspaper/configuration.py index 451db8a0..fa5d6d67 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -12,12 +12,9 @@ import logging -from .text import StopWords -from .text import StopWordsChinese -from .text import StopWordsArabic -from .text import StopWordsKorean from .parsers import Parser, ParserSoup -from .urls import is_abs_url, get_domain +from .text import (StopWords, StopWordsArabic, StopWordsChinese, + StopWordsKorean) from .version import __version__ log = logging.getLogger(__name__) @@ -30,49 +27,48 @@ def __init__(self): Modify any of these Article / Source properties TODO: Have a seperate ArticleConfig and SourceConfig extend this! """ - self.MIN_WORD_COUNT = 300 # num of word tokens in text - self.MIN_SENT_COUNT = 7 # num of sentence tokens - self.MAX_TITLE = 200 # num of chars - self.MAX_TEXT = 100000 # num of chars - self.MAX_KEYWORDS = 35 # num of strings in list - self.MAX_AUTHORS = 10 # num strings in list - self.MAX_SUMMARY = 5000 # num of chars + self.MIN_WORD_COUNT = 300 # num of word tokens in text + self.MIN_SENT_COUNT = 7 # num of sentence tokens + self.MAX_TITLE = 200 # num of chars + self.MAX_TEXT = 100000 # num of chars + self.MAX_KEYWORDS = 35 # num of strings in list + self.MAX_AUTHORS = 10 # num strings in list + self.MAX_SUMMARY = 5000 # num of chars # max number of urls we cache for each news source self.MAX_FILE_MEMO = 20000 - self.parser_class = 'lxml' # lxml vs soup + self.parser_class = 'lxml' # 'lxml' or 'soup' - # cache and save articles run after run + # Cache and save articles run after run self.memoize_articles = True - # set this to false if you don't care about getting images + # Set this to false if you don't care about getting images self.fetch_images = True self.image_dimension_ration = 16/9.0 - # don't toggle this variable + # Don't toggle this variable, done internally self.use_meta_language = True - # you may keep the html of just the main article body + # You may keep the html of just the main article body self.keep_article_html = False - # english is our fallback + # English is the fallback self._language = 'en' - # unique stopword classes for oriental languages, don't toggle + # Unique stopword classes for oriental languages, don't toggle self.stopwords_class = StopWords self.browser_user_agent = 'newspaper/%s' % __version__ self.request_timeout = 7 - self.number_threads = 10 # number of threads when mthreading + self.number_threads = 10 - self.verbose = False # turn this on when debugging - - # set this to False if you want to recompute the categories *every* time - # self.use_cached_categories = True # TODO: Make this work - - # self.hints = None TODO: Maybe a future release? + self.verbose = False # for debugging + # Set this to False if you want to recompute the categories + # *every* time you build a `Source` object + # TODO: Actually make this work + # self.use_cached_categories = True def get_language(self): return self._language @@ -81,21 +77,22 @@ def del_language(self): raise Exception('wtf are you doing?') def set_language(self, language): - """ - Language setting must be done in this method because non-occidental + """Language setting must be set in this method b/c non-occidental (western) langauges require a seperate stopwords class. """ if not language or len(language) != 2: raise Exception("Your input language must be a 2 char langauge code, \ for example: english-->en \n and german-->de") - self.use_meta_language = False # if explicitly set langauge, don't use meta + # If explicitly set langauge, don't use meta + self.use_meta_language = False - # Set oriental language stopword class. + # Set oriental language stopword class self._language = language self.stopwords_class = self.get_stopwords_class(language) - language = property(get_language, set_language, del_language, "langauge prop") + language = property(get_language, set_language, + del_language, "langauge prop") def get_stopwords_class(self, language): if language == 'ko': @@ -109,40 +106,10 @@ def get_stopwords_class(self, language): def get_parser(self): return Parser if self.parser_class == 'lxml' else ParserSoup - def get_publishdate_extractor(self): - """ - """ - return self.extract_publishdate - - def set_publishdate_extractor(self, extractor): - """ - Pass in to extract article publish dates. - @param extractor a concrete instance of PublishDateExtractor - """ - if not extractor: - raise ValueError("extractor must not be null!") - self.extract_publishdate = extractor - - def get_additionaldata_extractor(self): - """ - """ - return self.additional_data_extractor - - def set_additionaldata_extractor(self, extractor): - """ - Pass in to extract any additional data not defined within - @param extractor a concrete instance of AdditionalDataExtractor - """ - if not extractor: - raise ValueError("extractor must not be null!") - self.additional_data_extractor = extractor - - -# TODO: Since we have Source() and Article() objects, we should split up -# TODO: the config options for both class ArticleConfiguration(Configuration): pass + class SourceConfiguration(Configuration): pass diff --git a/newspaper/extractors.py b/newspaper/extractors.py index dc40111f..2bccc2e3 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -1,30 +1,32 @@ # -*- coding: utf-8 -*- """ -Newspaper uses a lot of python-goose's extraction code. View their -license here: https://github.com/codelucas/newspaper/blob/master/GOOSE-LICENSE.txt +Newspaper uses much of python-goose's extraction code. View their license: +https://github.com/codelucas/newspaper/blob/master/GOOSE-LICENSE.txt -Keep all html page extraction code within this file. PLEASE abstract any -lxml or soup parsing mechanisms in the parsers.py file! +Keep all html page extraction code within this file. Abstract any +lxml or soup parsing code in the parsers.py file! """ __title__ = 'newspaper' __author__ = 'Lucas Ou-Yang' __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' -import re +from collections import defaultdict import copy +import logging +import re import urlparse -from collections import defaultdict + +from . import urls from .packages.tldextract import tldextract -from .utils import ( - StringSplitter, StringReplacement, ReplaceSequence) -from .urls import ( - get_path, get_domain, get_scheme, prepare_url) +from .utils import ReplaceSequence, StringReplacement, StringSplitter +log = logging.getLogger(__name__) MOTLEY_REPLACEMENT = StringReplacement("�", "") -ESCAPED_FRAGMENT_REPLACEMENT = StringReplacement(u"#!", u"?_escaped_fragment_=") +ESCAPED_FRAGMENT_REPLACEMENT = StringReplacement( + u"#!", u"?_escaped_fragment_=") TITLE_REPLACEMENTS = ReplaceSequence().create(u"»").append(u"ยป") PIPE_SPLITTER = StringSplitter("\\|") DASH_SPLITTER = StringSplitter(" - ") @@ -33,7 +35,8 @@ SPACE_SPLITTER = StringSplitter(' ') NO_STRINGS = set() A_REL_TAG_SELECTOR = "a[rel=tag]" -A_HREF_TAG_SELECTOR = "a[href*='/tag/'], a[href*='/tags/'], a[href*='/topic/'], a[href*='?keyword=']" +A_HREF_TAG_SELECTOR = ("a[href*='/tag/'], a[href*='/tags/'], " + "a[href*='/topic/'], a[href*='?keyword=']") RE_LANG = r'^[A-Za-z]{2}$' good_paths = ['story', 'article', 'feature', 'featured', 'slides', @@ -53,29 +56,27 @@ def __init__(self, config): self.language = config.language self.stopwords_class = config.stopwords_class - def update_language(self, article): - ''' - Required to be called before the extraction process in some + def update_language(self, meta_lang): + '''Required to be called before the extraction process in some cases because the stopwords_class has to set incase the lang - is not latin based. + is not latin based ''' - if article.config.use_meta_language and article.meta_lang: - self.language = article.meta_lang - self.stopwords_class = article.config.\ - get_stopwords_class(article.meta_lang) + if meta_lang: + self.language = meta_lang + self.stopwords_class = \ + self.config.get_stopwords_class(meta_lang) - def get_authors(self, article): - """ - Fetch the authors of the article, return as a list - Only works for english articles. + def get_authors(self, doc): + """Fetch the authors of the article, return as a list + Only works for english articles """ _digits = re.compile('\d') + def contains_digits(d): return bool(_digits.search(d)) def parse_byline(search_str): - """ - Takes a candidate line of html or text and + """Takes a candidate line of html or text and extracts out the name(s) in list form >>> search_str('<div>By: <strong>Lucas Ou-Yang</strong>, \ <strong>Alex Smith</strong></div>') @@ -90,15 +91,14 @@ def parse_byline(search_str): search_str = search_str.strip() # Chunk the line by non alphanumeric tokens (few name exceptions) - - # >>> re.split("[^\w\'\-]", "Lucas Ou-Yang, Dean O'Brian and Ronald") - # ['Lucas Ou-Yang', '', 'Dean O'Brian', 'and', 'Ronald'] - + # >>> re.split("[^\w\'\-]", "Lucas Ou, Dean O'Brian and Ronald") + # ['Lucas Ou', '', 'Dean O'Brian', 'and', 'Ronald'] name_tokens = re.split("[^\w\'\-]", search_str) name_tokens = [s.strip() for s in name_tokens] _authors = [] - curname = [] # List of first, last name tokens + # List of first, last name tokens + curname = [] DELIM = ['and', ''] for token in name_tokens: @@ -125,8 +125,6 @@ def parse_byline(search_str): VALS = ['author', 'byline'] matches = [] _authors, authors = [], [] - doc = article.clean_doc - html = article.html for attr in ATTRS: for val in VALS: @@ -136,46 +134,35 @@ def parse_byline(search_str): for match in matches: content = u'' - if match.tag == 'meta': mm = match.xpath('@content') if len(mm) > 0: content = mm[0] - - else: # match.tag == <any other tag> - content = match.text or u'' # text_content() - + else: + content = match.text or u'' if len(content) > 0: _authors.extend(parse_byline(content)) uniq = list(set([s.lower() for s in _authors])) - for name in uniq: names = [w.capitalize() for w in name.split(' ')] authors.append(' '.join(names)) - return authors or [] # TODO Method 2: Search raw html for a by-line - # match = re.search('By[\: ].*\\n|From[\: ].*\\n', html) - # try: # # Don't let zone be too long # line = match.group(0)[:100] # authors = parse_byline(line) # except: # return [] # Failed to find anything - # return authors - def get_title(self, article): - """ - Fetch the article title and analyze it. + def get_title(self, doc): + """Fetch the article title and analyze it """ title = '' - doc = article.clean_doc - title_element = self.parser.getElementsByTag(doc, tag='title') # no title found if title_element is None or len(title_element) == 0: @@ -209,8 +196,7 @@ def get_title(self, article): return title def split_title(self, title, splitter): - """ - Split the title to best part possible. + """Split the title to best part possible """ large_text_length = 0 large_text_index = 0 @@ -227,59 +213,53 @@ def split_title(self, title, splitter): title = title_pieces[large_text_index] return TITLE_REPLACEMENTS.replaceAll(title).strip() - def get_feed_urls(self, source_or_category): - """ - Returns list of feed urls on a source or category object. - """ - # This feels really weird..., needs more refactoring - is_source = source_or_category.__class__.__name__ == 'Source' - - if not is_source: # recursively call method with Source's categories - feed_urls = [] - for category in source_or_category.categories: - feed_urls.extend(self.get_feed_urls(category)) - feed_urls = feed_urls[:50] - feed_urls = [prepare_url(f, source.url) for f in feed_urls] - - feed_urls = list(set(feed_urls)) - return feed_urls - - doc = source_or_category.doc # it's a Category - kwargs = {'attr': 'type', 'value': 'application/rss+xml'} - feed_elements = self.parser.getElementsByTag(doc, **kwargs) - feed_urls = [e.get('href') for e in feed_elements if e.get('href')] - return feed_urls - - def get_favicon(self, article): - """ - Extract the favicon from a website - http://en.wikipedia.org/wiki/Favicon + def get_feed_urls(self, source_url, categories): + """Takes a source url and a list of category objects and returns + a list of feed urls + """ + total_feed_urls = [] + for category in categories: + kwargs = {'attr': 'type', 'value': 'application\/rss\+xml'} + feed_elements = self.parser.getElementsByTag( + category.doc, **kwargs) + feed_urls = [e.get('href') for e in feed_elements if e.get('href')] + total_feed_urls.extend(feed_urls) + + total_feed_urls = total_feed_urls[:50] + total_feed_urls = [urls.prepare_url(f, source_url) + for f in total_feed_urls] + total_feed_urls = list(set(total_feed_urls)) + return total_feed_urls + + def get_favicon(self, doc): + """Extract the favicon from a website http://en.wikipedia.org/wiki/Favicon <link rel="shortcut icon" type="image/png" href="favicon.png" /> <link rel="icon" type="image/png" href="favicon.png" /> """ kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'icon'} - meta = self.parser.getElementsByTag(article.clean_doc, **kwargs) + meta = self.parser.getElementsByTag(doc, **kwargs) if meta: favicon = self.parser.getAttribute(meta[0], 'href') return favicon return '' - def get_meta_lang(self, article): - """ - Extract content language from meta. + def get_meta_lang(self, doc): + """Extract content language from meta """ # we have a lang attribute in html - attr = self.parser.getAttribute(article.clean_doc, attr='lang') + attr = self.parser.getAttribute(doc, attr='lang') if attr is None: # look up for a Content-Language in meta items = [ - {'tag': 'meta', 'attr': 'http-equiv', 'value': 'content-language'}, + {'tag': 'meta', 'attr': 'http-equiv', + 'value': 'content-language'}, {'tag': 'meta', 'attr': 'name', 'value': 'lang'} ] for item in items: - meta = self.parser.getElementsByTag(article.clean_doc, **item) + meta = self.parser.getElementsByTag(doc, **item) if meta: - attr = self.parser.getAttribute(meta[0], attr='content') + attr = self.parser.getAttribute( + meta[0], attr='content') break if attr: value = attr[:2] @@ -289,8 +269,7 @@ def get_meta_lang(self, article): return None def get_meta_content(self, doc, metaName): - """ - Extract a given meta content form document. + """Extract a given meta content form document. Example metaNames: "meta[name=description]" "meta[name=keywords]" @@ -298,22 +277,16 @@ def get_meta_content(self, doc, metaName): """ meta = self.parser.css_select(doc, metaName) content = None - if meta is not None and len(meta) > 0: content = self.parser.getAttribute(meta[0], 'content') - if content: return content.strip() - return '' - def get_meta_img_url(self, article): - """ - Returns the 'top img' as specified by the website. + def get_meta_img_url(self, article_url, doc): + """Returns the 'top img' as specified by the website """ top_meta_image, try_one, try_two, try_three, try_four = [None] * 5 - doc = article.clean_doc - try_one = self.get_meta_content(doc, 'meta[property="og:image"]') if try_one is None: link_icon_kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'icon'} @@ -321,44 +294,36 @@ def get_meta_img_url(self, article): try_two = elems[0].get('href') if elems else None if try_two is None: - link_img_src_kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'img_src'} + link_img_src_kwargs = \ + {'tag': 'link', 'attr': 'rel', 'value': 'img_src'} elems = self.parser.getElementsByTag(doc, **link_img_src_kwargs) try_three = elems[0].get('href') if elems else None if try_three is None: try_four = self.get_meta_content(doc, 'meta[name="og:image"]') - top_meta_image = try_one or try_two or try_three or try_four # :) + top_meta_image = try_one or try_two or try_three or try_four - return urlparse.urljoin(article.url, top_meta_image) + return urlparse.urljoin(article_url, top_meta_image) - def get_meta_type(self, article): - """ - Returns meta type of article, open graph protocol. + def get_meta_type(self, doc): + """Returns meta type of article, open graph protocol """ - return self.get_meta_content(article.clean_doc, 'meta[property="og:type"]') + return self.get_meta_content(doc, 'meta[property="og:type"]') - def get_meta_description(self, article_or_source): + def get_meta_description(self, doc): + """If the article has meta description set in the source, use that """ - If the article has meta description set in the source, use that. - """ - # Since <source> objects use this particular method and sources don't - # have a 'clean_doc' we just use doc - try: # "easier to ask for forgiveness than permission" - doc = article_or_source.clean_doc - except: - doc = article_or_source.doc return self.get_meta_content(doc, "meta[name=description]") - def get_meta_keywords(self, article): - """ - If the article has meta keywords set in the source, use that. + def get_meta_keywords(self, doc): + """If the article has meta keywords set in the source, use that """ - return self.get_meta_content(article.clean_doc, "meta[name=keywords]") + return self.get_meta_content(doc, "meta[name=keywords]") - def get_meta_data(self, article): + def get_meta_data(self, doc): data = defaultdict(dict) - properties = self.parser.css_select(article.clean_doc, 'meta') + properties = self.parser.css_select(doc, 'meta') for prop in properties: key = prop.attrib.get('property') or prop.attrib.get('name') value = prop.attrib.get('content') or prop.attrib.get('value') @@ -389,174 +354,164 @@ def get_meta_data(self, article): ref = ref[part] return data - def get_canonical_link(self, article): - """ - If the article has meta canonical link set in the url. + def get_canonical_link(self, article_url, doc): + """If the article has meta canonical link set in the url """ kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'canonical'} - meta = self.parser.getElementsByTag(article.clean_doc, **kwargs) + meta = self.parser.getElementsByTag(doc, **kwargs) if meta is not None and len(meta) > 0: href = self.parser.getAttribute(meta[0], 'href') if href: href = href.strip() o = urlparse.urlparse(href) if not o.hostname: - z = urlparse.urlparse(article.url) + z = urlparse.urlparse(article_url) domain = '%s://%s' % (z.scheme, z.hostname) href = urlparse.urljoin(domain, href) return href return u'' - def get_img_urls(self, article, use_top_node=False): - """ - Return all of the images on an html page, lxml root. + def get_img_urls(self, article_url, doc): + """Return all of the images on an html page, lxml root """ - doc = article.clean_top_node if use_top_node else article.clean_doc - img_kwargs = {'tag': 'img'} img_tags = self.parser.getElementsByTag(doc, **img_kwargs) - urls = [img_tag.get('src') for img_tag in img_tags if img_tag.get('src')] - img_links = set([ urlparse.urljoin(article.url, url) for url in urls ]) - - if article.meta_img: - img_links.add(article.meta_img) + urls = [img_tag.get('src') + for img_tag in img_tags if img_tag.get('src')] + img_links = set([urlparse.urljoin(article_url, url) for url in urls]) return img_links - def get_first_img_url(self, article): - """ - Retrieves the first image in the 'top_node' - The top node is essentially the HTML - markdown where the main article lies and the first image - in that area is probably signifigcant. + def get_first_img_url(self, article_url, top_node): + """Retrieves the first image in the 'top_node' + The top node is essentially the HTML markdown where the main + article lies and the first image in that area is probably signifigcant. """ - node_images = self.get_img_urls(article, use_top_node=True) + node_images = self.get_img_urls(article_url, top_node) node_images = list(node_images) if node_images: - return urlparse.urljoin(article.url, node_images[0]) + return urlparse.urljoin(article_url, node_images[0]) return u'' def _get_urls(self, doc, titles): - """ - Return a list of urls or a list of (url, title_text) tuples if specified. + """Return a list of urls or a list of (url, title_text) tuples + if specified. """ if doc is None: return [] a_kwargs = {'tag': 'a'} - a_tags = self.parser.getElementsByTag(doc, **a_kwargs) # doc.xpath('//a') + a_tags = self.parser.getElementsByTag(doc, **a_kwargs) - # TODO this should be refactored! We should have a seperate method which - # siphones the titles our of a list of <a> tags. + # TODO: this should be refactored! We should have a seperate + # method which siphones the titles our of a list of <a> tags. if titles: - return [ (a.get('href'), a.text) for a in a_tags if a.get('href') ] - - return [ a.get('href') for a in a_tags if a.get('href') ] + return [(a.get('href'), a.text) for a in a_tags if a.get('href')] + return [a.get('href') for a in a_tags if a.get('href')] def get_urls(self, doc_or_html, titles=False, regex=False): - """ - doc_or_htmls html page or doc and returns list of urls, the regex + """`doc_or_html`s html page or doc and returns list of urls, the regex flag indicates we don't parse via lxml and just search the html. """ if doc_or_html is None: log.critical('Must extract urls from either html, text or doc!') return [] - # If we are extracting from raw text if regex: doc_or_html = re.sub('<[^<]+?>', ' ', doc_or_html) - doc_or_html = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', doc_or_html) + doc_or_html = re.findall( + 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|' + '(?:%[0-9a-fA-F][0-9a-fA-F]))+', doc_or_html) doc_or_html = [i.strip() for i in doc_or_html] return doc_or_html or [] - # If the doc_or_html is html, parse it into a root if isinstance(doc_or_html, str) or isinstance(doc_or_html, unicode): doc = self.parser.fromstring(doc_or_html) else: doc = doc_or_html - return self._get_urls(doc, titles) - def get_category_urls(self, source, source_url=None, page_urls=None): - """ - Requires: source lxml root and source url takes a domain and finds all of the - top level urls, we are assuming that these are the category urls. - + def get_category_urls(self, source_url, doc): + """Inputs source lxml root and source url, extracts domain and + finds all of the top level urls, we are assuming that these are + the category urls. cnn.com --> [cnn.com/latest, world.cnn.com, cnn.com/asia] """ - - source_url = source.url if not source_url else source_url - page_urls = self.get_urls(source.doc) if not page_urls else page_urls + page_urls = self.get_urls(doc) valid_categories = [] for p_url in page_urls: - scheme = get_scheme(p_url, allow_fragments=False) - domain = get_domain(p_url, allow_fragments=False) - path = get_path(p_url, allow_fragments=False) + scheme = urls.get_scheme(p_url, allow_fragments=False) + domain = urls.get_domain(p_url, allow_fragments=False) + path = urls.get_path(p_url, allow_fragments=False) if not domain and not path: - if source.config.verbose: + if self.config.verbose: print 'elim category url %s for no domain and path' % p_url continue if path and path.startswith('#'): - if source.config.verbose: + if self.config.verbose: print 'elim category url %s path starts with #' % p_url continue - if scheme and (scheme!='http' and scheme!='https'): - if source.config.verbose: - print 'elim category url %s for bad scheme, not http nor https' % p_url + if scheme and (scheme != 'http' and scheme != 'https'): + if self.config.verbose: + print ('elim category url %s for bad scheme, ' + 'not http nor https' % p_url) continue if domain: child_tld = tldextract.extract(p_url) domain_tld = tldextract.extract(source_url) - child_subdomain_parts = child_tld.subdomain.split('.') subdomain_contains = False for part in child_subdomain_parts: if part == domain_tld.domain: - if source.config.verbose: - print 'subdomain contains at %s and %s' % (str(part), str(domain_tld.domain)) + if self.config.verbose: + print ('subdomain contains at %s and %s' % + (str(part), str(domain_tld.domain))) subdomain_contains = True break - # microsoft.com is definitely not related to espn.com, but espn.go.com is probably - # related to espn.com - if not subdomain_contains and (child_tld.domain != domain_tld.domain): - if source.config.verbose: - print 'elim category url %s for domain mismatch' % p_url + # Ex. microsoft.com is definitely not related to + # espn.com, but espn.go.com is probably related to espn.com + if not subdomain_contains and \ + (child_tld.domain != domain_tld.domain): + if self.config.verbose: + print ('elim category url %s for domain ' + 'mismatch' % p_url) continue elif child_tld.subdomain in ['m', 'i']: - if source.config.verbose: - print 'elim category url %s for mobile subdomain' % p_url + if self.config.verbose: + print ('elim category url %s for mobile ' + 'subdomain' % p_url) continue else: valid_categories.append(scheme+'://'+domain) - # TODO account for case where category is in form http://subdomain.domain.tld/category/ <-- it's still legal! + # TODO account for case where category is in form + # http://subdomain.domain.tld/category/ <-- still legal! else: # we want a path with just one subdir # cnn.com/world and cnn.com/world/ are both valid_categories - path_chunks = [ x for x in path.split('/') if len(x) > 0 ] - + path_chunks = [x for x in path.split('/') if len(x) > 0] if 'index.html' in path_chunks: path_chunks.remove('index.html') if len(path_chunks) == 1 and len(path_chunks[0]) < 14: valid_categories.append(domain+path) else: - if source.config.verbose: - print 'elim category url %s for >1 path chunks or size path chunks' % p_url - - + if self.config.verbose: + print ('elim category url %s for >1 path chunks ' + 'or size path chunks' % p_url) stopwords = [ 'about', 'help', 'privacy', 'legal', 'feedback', 'sitemap', 'profile', 'account', 'mobile', 'sitemap', 'facebook', 'myspace', - 'twitter', 'linkedin', 'bebo', 'friendster', 'stumbleupon', 'youtube', - 'vimeo', 'store', 'mail', 'preferences', 'maps', 'password', 'imgur', - 'flickr', 'search', 'subscription', 'itunes', 'siteindex', 'events', - 'stop', 'jobs', 'careers', 'newsletter', 'subscribe', 'academy', - 'shopping', 'purchase', 'site-map', 'shop', 'donate', 'newsletter', - 'product', 'advert', 'info', 'tickets', 'coupons', 'forum', 'board', - 'archive', 'browse', 'howto', 'how to', 'faq', 'terms', 'charts', - 'services', 'contact', 'plus', 'admin', 'login', 'signup', 'register', + 'twitter', 'linkedin', 'bebo', 'friendster', 'stumbleupon', + 'youtube', 'vimeo', 'store', 'mail', 'preferences', 'maps', + 'password', 'imgur', 'flickr', 'search', 'subscription', 'itunes', + 'siteindex', 'events', 'stop', 'jobs', 'careers', 'newsletter', + 'subscribe', 'academy', 'shopping', 'purchase', 'site-map', + 'shop', 'donate', 'newsletter', 'product', 'advert', 'info', + 'tickets', 'coupons', 'forum', 'board', 'archive', 'browse', + 'howto', 'how to', 'faq', 'terms', 'charts', 'services', + 'contact', 'plus', 'admin', 'login', 'signup', 'register', 'developer', 'proxy'] _valid_categories = [] @@ -564,23 +519,24 @@ def get_category_urls(self, source, source_url=None, page_urls=None): # TODO Stop spamming urlparse and tldextract calls... for p_url in valid_categories: - path = get_path(p_url) + path = urls.get_path(p_url) subdomain = tldextract.extract(p_url).subdomain conjunction = path + ' ' + subdomain bad = False for badword in stopwords: if badword.lower() in conjunction.lower(): - if source.config.verbose: - print 'elim category url %s for subdomain contain stopword!' % p_url - bad=True + if self.config.verbose: + print ('elim category url %s for subdomain ' + 'contain stopword!' % p_url) + bad = True break if not bad: _valid_categories.append(p_url) - _valid_categories.append('/') # add the root! + _valid_categories.append('/') # add the root for i, p_url in enumerate(_valid_categories): - if p_url.startswith('://') : + if p_url.startswith('://'): p_url = 'http' + p_url _valid_categories[i] = p_url @@ -594,20 +550,19 @@ def get_category_urls(self, source, source_url=None, page_urls=None): _valid_categories = list(set(_valid_categories)) - category_urls = [prepare_url(p_url, source_url) for p_url in _valid_categories] + category_urls = [urls.prepare_url(p_url, source_url) + for p_url in _valid_categories] category_urls = [c for c in category_urls if c is not None] return category_urls - def extract_tags(self, article): - node = article.clean_doc - - # node doesn't have chidren - if len(list(node)) == 0: + def extract_tags(self, doc): + if len(list(doc)) == 0: return NO_STRINGS - - elements = self.parser.css_select(node, A_REL_TAG_SELECTOR) + elements = self.parser.css_select( + doc, A_REL_TAG_SELECTOR) if not elements: - elements = self.parser.css_select(node, A_HREF_TAG_SELECTOR) + elements = self.parser.css_select( + doc, A_HREF_TAG_SELECTOR) if not elements: return NO_STRINGS @@ -616,19 +571,11 @@ def extract_tags(self, article): tag = self.parser.getText(el) if tag: tags.append(tag) - return set(tags) - def calculate_best_node(self, article): - doc = article.doc - - # if article.config.hints: TODO: Maybe a future release - # rules_kw = article.config.hints.get(article.source_url) - # doc = article.parser.getElementsByTag(doc, **rules_kw)[0] - + def calculate_best_node(self, doc): top_node = None nodes_to_check = self.nodes_to_check(doc) - starting_boost = float(1.0) cnt = 0 i = 0 @@ -637,7 +584,8 @@ def calculate_best_node(self, article): for node in nodes_to_check: text_node = self.parser.getText(node) - word_stats = self.stopwords_class(language=self.language).get_stopword_count(text_node) + word_stats = self.stopwords_class(language=self.language).\ + get_stopword_count(text_node) high_link_density = self.is_highlink_density(node) if word_stats.get_stopword_count() > 2 and not high_link_density: nodes_with_text.append(node) @@ -656,17 +604,18 @@ def calculate_best_node(self, article): # nodes_number if nodes_number > 15: if (nodes_number - i) <= bottom_negativescore_nodes: - booster = float(bottom_negativescore_nodes - (nodes_number - i)) + booster = float( + bottom_negativescore_nodes - (nodes_number - i)) boost_score = float(-pow(booster, float(2))) negscore = abs(boost_score) + negative_scoring if negscore > 40: boost_score = float(5) text_node = self.parser.getText(node) - word_stats = self.stopwords_class(language=self.language).get_stopword_count(text_node) + word_stats = self.stopwords_class(language=self.language).\ + get_stopword_count(text_node) upscore = int(word_stats.get_stopword_count() + boost_score) - # parent node parent_node = self.parser.getParent(node) self.update_score(parent_node, upscore) self.update_node_count(parent_node, 1) @@ -674,7 +623,7 @@ def calculate_best_node(self, article): if parent_node not in parent_nodes: parent_nodes.append(parent_node) - # parent of parent node + # Parent of parent node parent_parent_node = self.parser.getParent(parent_node) if parent_parent_node is not None: self.update_node_count(parent_parent_node, 1) @@ -694,17 +643,14 @@ def calculate_best_node(self, article): if top_node is None: top_node = e - return top_node def is_boostable(self, node): - """ - Alot of times the first paragraph might be the caption under an image + """Alot of times the first paragraph might be the caption under an image so we'll want to make sure if we're going to boost a parent node that - it should be connected to other paragraphs, - at least for the first n paragraphs so we'll want to make sure that - the next sibling is a paragraph and has at - least some substantial weight to it. + it should be connected to other paragraphs, at least for the first n + paragraphs so we'll want to make sure that the next sibling is a + paragraph and has at least some substantial weight to it. """ para = "p" steps_away = 0 @@ -713,13 +659,14 @@ def is_boostable(self, node): nodes = self.walk_siblings(node) for current_node in nodes: - # p + # <p> current_node_tag = self.parser.getTag(current_node) if current_node_tag == para: if steps_away >= max_stepsaway_from_node: return False paraText = self.parser.getText(current_node) - word_stats = self.stopwords_class(language=self.language).get_stopword_count(paraText) + word_stats = self.stopwords_class(language=self.language).\ + get_stopword_count(paraText) if word_stats.get_stopword_count() > minimum_stopword_count: return True steps_away += 1 @@ -730,31 +677,33 @@ def walk_siblings(self, node): b = [] while current_sibling is not None: b.append(current_sibling) - previousSibling = self.parser.previousSibling(current_sibling) - current_sibling = None if previousSibling is None else previousSibling + current_sibling = self.parser.previousSibling(current_sibling) return b def add_siblings(self, top_node): baselinescore_siblings_para = self.get_siblings_score(top_node) results = self.walk_siblings(top_node) for current_node in results: - ps = self.get_siblings_content(current_node, baselinescore_siblings_para) + ps = self.get_siblings_content( + current_node, baselinescore_siblings_para) for p in ps: top_node.insert(0, p) return top_node - def get_siblings_content(self, current_sibling, baselinescore_siblings_para): - """ - Adds any siblings that may have a decent score to this node. + def get_siblings_content( + self, current_sibling, baselinescore_siblings_para): + """Adds any siblings that may have a decent score to this node """ - if current_sibling.tag == 'p' and len(self.parser.getText(current_sibling)) > 0: + if current_sibling.tag == 'p' and \ + len(self.parser.getText(current_sibling)) > 0: e0 = current_sibling if e0.tail: e0 = copy.deepcopy(e0) e0.tail = '' return [e0] else: - potential_paragraphs = self.parser.getElementsByTag(current_sibling, tag='p') + potential_paragraphs = self.parser.getElementsByTag( + current_sibling, tag='p') if potential_paragraphs is None: return None else: @@ -762,19 +711,22 @@ def get_siblings_content(self, current_sibling, baselinescore_siblings_para): for first_paragraph in potential_paragraphs: text = self.parser.getText(first_paragraph) if len(text) > 0: - word_stats = self.stopwords_class(language=self.language).get_stopword_count(text) + word_stats = self.stopwords_class(language=self.language).\ + get_stopword_count(text) paragraph_score = word_stats.get_stopword_count() sibling_baseline_score = float(.30) - high_link_density = self.is_highlink_density(first_paragraph) - score = float(baselinescore_siblings_para * sibling_baseline_score) + high_link_density = self.is_highlink_density( + first_paragraph) + score = float(baselinescore_siblings_para * + sibling_baseline_score) if score < paragraph_score and not high_link_density: - p = self.parser.createElement(tag='p', text=text, tail=None) + p = self.parser.createElement( + tag='p', text=text, tail=None) ps.append(p) return ps def get_siblings_score(self, top_node): - """ - We could have long articles that have tons of paragraphs + """We could have long articles that have tons of paragraphs so if we tried to calculate the base score against the total text score of those paragraphs it would be unfair. So we need to normalize the score based on the average scoring @@ -789,7 +741,8 @@ def get_siblings_score(self, top_node): for node in nodes_to_check: text_node = self.parser.getText(node) - word_stats = self.stopwords_class(language=self.language).get_stopword_count(text_node) + word_stats = self.stopwords_class(language=self.language).\ + get_stopword_count(text_node) high_link_density = self.is_highlink_density(node) if word_stats.get_stopword_count() > 2 and not high_link_density: paragraphs_number += 1 @@ -801,10 +754,9 @@ def get_siblings_score(self, top_node): return base def update_score(self, node, addToScore): - """ - Adds a score to the gravityScore Attribute we put on divs - we'll get the current score then add the score - we're passing in to the current. + """Adds a score to the gravityScore Attribute we put on divs + we'll get the current score then add the score we're passing + in to the current. """ current_score = 0 score_string = self.parser.getAttribute(node, 'gravityScore') @@ -815,8 +767,7 @@ def update_score(self, node, addToScore): self.parser.setAttribute(node, "gravityScore", str(new_score)) def update_node_count(self, node, add_to_count): - """ - Stores how many decent nodes are under a parent node. + """Stores how many decent nodes are under a parent node """ current_score = 0 count_string = self.parser.getAttribute(node, 'gravityNodes') @@ -827,10 +778,8 @@ def update_node_count(self, node, add_to_count): self.parser.setAttribute(node, "gravityNodes", str(new_score)) def is_highlink_density(self, e): - """ - Checks the density of links within a node, - is there not much text and most of it contains linky shit? - if so it's no good. + """Checks the density of links within a node, if there is a high + link to text ratio, then the text is less likely to be relevant """ links = self.parser.getElementsByTag(e, tag='a') if links is None or len(links) == 0: @@ -855,8 +804,7 @@ def is_highlink_density(self, e): # return True if score > 1.0 else False def get_score(self, node): - """ - Returns the gravityScore as an integer from this node. + """Returns the gravityScore as an integer from this node """ return self.get_node_gravity_score(node) or 0 @@ -867,9 +815,8 @@ def get_node_gravity_score(self, node): return int(grvScoreString) def nodes_to_check(self, doc): - """ - Returns a list of nodes we want to search - on like paragraphs and tables. + """Returns a list of nodes we want to search + on like paragraphs and tables """ nodes_to_check = [] for tag in ['p', 'pre', 'td']: @@ -898,21 +845,16 @@ def is_nodescore_threshold_met(self, node, e): return False return True - def post_cleanup(self, targetNode): - """ - Remove any divs that looks like non-content, - clusters of links, or paras with no gusto. + def post_cleanup(self, top_node): + """Remove any divs that looks like non-content, + clusters of links, or paras with no gusto """ - node = self.add_siblings(targetNode) + node = self.add_siblings(top_node) for e in self.parser.getChildren(node): e_tag = self.parser.getTag(e) if e_tag != 'p': if self.is_highlink_density(e) \ - or self.is_table_and_no_para_exist(e) \ - or not self.is_nodescore_threshold_met(node, e): + or self.is_table_and_no_para_exist(e) \ + or not self.is_nodescore_threshold_met(node, e): self.parser.remove(e) return node - - -class StandardContentExtractor(ContentExtractor): - pass diff --git a/newspaper/images.py b/newspaper/images.py index 9ceae28e..ea80fc9e 100644 --- a/newspaper/images.py +++ b/newspaper/images.py @@ -9,13 +9,13 @@ __copyright__ = 'Copyright 2014, Lucas Ou-Yang' import logging -import urllib -import StringIO import math +import StringIO +import urllib +import urllib2 -from PIL import Image, ImageFile -from urllib2 import Request, HTTPError, URLError, build_opener from httplib import InvalidURL +from PIL import Image, ImageFile from . import urls @@ -25,81 +25,77 @@ thumbnail_size = 90, 90 minimal_area = 5000 + def image_to_str(image): s = StringIO.StringIO() image.save(s, image.format) s.seek(0) return s.read() + def str_to_image(s): s = StringIO.StringIO(s) s.seek(0) image = Image.open(s) return image + def prepare_image(image): image = square_image(image) - image.thumbnail(thumbnail_size, Image.ANTIALIAS) # inplace + image.thumbnail(thumbnail_size, Image.ANTIALIAS) return image + def image_entropy(img): - """ - Calculate the entropy of an image. + """ Calculate the entropy of an image """ hist = img.histogram() hist_size = sum(hist) hist = [float(h) / hist_size for h in hist] return -sum([p * math.log(p, 2) for p in hist if p != 0]) + def square_image(img): + """If the image is taller than it is wide, square it off. determine + which pieces to cut off based on the entropy pieces """ - If the image is taller than it is wide, square it off. determine - which pieces to cut off based on the entropy pieces. - """ - x,y = img.size + x, y = img.size while y > x: - # slice 10px at a time until square + # Slice 10px at a time until square slice_height = min(y - x, 10) - bottom = img.crop((0, y - slice_height, x, y)) top = img.crop((0, 0, x, slice_height)) - # remove the slice with the least entropy if image_entropy(bottom) < image_entropy(top): img = img.crop((0, 0, x, y - slice_height)) else: img = img.crop((0, slice_height, x, y)) - - x,y = img.size - + x, y = img.size return img + def clean_url(url): - """ - Url quotes unicode data out of urls. + """Url quotes unicode data out of urls """ url = url.encode('utf8') url = ''.join([urllib.quote(c) if ord(c) >= 127 else c for c in url]) return url + def fetch_url(url, useragent, referer=None, retries=1, dimension=False): - """ - """ cur_try = 0 nothing = None if dimension else (None, None) url = clean_url(url) - if not url.startswith(('http://', 'https://')): return nothing - while True: try: - req = Request(url) + req = urllib2.Request(url) req.add_header('User-Agent', useragent) if referer: req.add_header('Referer', referer) - opener = build_opener() + opener = urllib2.build_opener() open_req = opener.open(req, timeout=5) # if we only need the dimension of the image, we may not @@ -161,29 +157,32 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False): return content_type, content - except (URLError, HTTPError, InvalidURL), e: + except (urllib2.URLError, urllib2.HTTPError, InvalidURL), e: cur_try += 1 if cur_try >= retries: - log.debug('error while fetching: %s refer: %s' % (url, referer)) + log.debug('error while fetching: %s refer: %s' % + (url, referer)) return nothing finally: if 'open_req' in locals(): open_req.close() + def fetch_image_dimension(url, useragent, referer=None, retries=1): return fetch_url(url, useragent, referer, retries, dimension=True) + class Scraper: def __init__(self, article): - self.url = article.url # if not url else url - self.imgs = article.imgs # if not imgs else imgs - self.top_img = article.top_img # if not top_img else top_img + self.url = article.url + self.imgs = article.imgs + self.top_img = article.top_img self.config = article.config self.useragent = self.config.browser_user_agent def largest_image_url(self): - #todo: remove. it is not responsibility of Scrapper + # TODO: remove. it is not responsibility of Scrapper if not self.imgs and not self.top_img: return None if self.top_img: @@ -191,56 +190,48 @@ def largest_image_url(self): max_area = 0 max_url = None - for img_url in self.imgs: - dimension = fetch_image_dimension(img_url, self.useragent, referer=self.url) + dimension = fetch_image_dimension( + img_url, self.useragent, referer=self.url) area = self.calculate_area(img_url, dimension) - if area > max_area: max_area = area max_url = img_url - log.debug('using max img ' + max_url) return max_url def calculate_area(self, img_url, dimension): if not dimension: return 0 - area = dimension[0] * dimension[1] - - #todo: introduce filter classes for each case - # ignore little images + # Ignore tiny images if area < minimal_area: log.debug('ignore little %s' % img_url) return 0 - - # PIL won't scale up, so we set a min width and + # PIL won't scale up, so set a min width and # maintain the aspect ratio if dimension[0] < thumbnail_size[0]: return 0 - - # ignore excessively long/wide images - if max(dimension) / min(dimension) > self.config.image_dimension_ration: + # Ignore excessively long/wide images + current_ratio = max(dimension) / min(dimension) + if current_ratio > self.config.image_dimension_ration: log.debug('ignore dims %s' % img_url) return 0 - - # penalize images with "sprite" in their name + # Penalize images with "sprite" in their name lower_case_url = img_url.lower() if 'sprite' in lower_case_url or 'logo' in lower_case_url: log.debug('penalizing sprite %s' % img_url) area /= 10 - return area def satisfies_requirements(self, img_url): - dimension = fetch_image_dimension(img_url, self.useragent, referer=self.url) + dimension = fetch_image_dimension( + img_url, self.useragent, referer=self.url) area = self.calculate_area(img_url, dimension) return area > minimal_area def thumbnail(self): - """ - Identifies top image, trims out a thumbnail and also has a url. + """Identifies top image, trims out a thumbnail and also has a url """ image_url = self.largest_image_url() if image_url: @@ -252,9 +243,5 @@ def thumbnail(self): except IOError, e: if 'interlaced' in e.message: return None - # raise return image, image_url - return None, None - - diff --git a/newspaper/mthreading.py b/newspaper/mthreading.py index 62df790e..f3e3738b 100644 --- a/newspaper/mthreading.py +++ b/newspaper/mthreading.py @@ -100,7 +100,7 @@ def join(self): resets the task. """ if self.pool is None: - print 'Please call set(..) with a list of source objects before .join(..)' + print 'Call set(..) with a list of source objects before .join(..)' raise self.pool.wait_completion() self.papers = [] diff --git a/newspaper/network.py b/newspaper/network.py index 94e1f214..ee0c9ab8 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -11,27 +11,27 @@ import logging import requests -from .settings import cj from .configuration import Configuration from .mthreading import ThreadPool +from .settings import cj log = logging.getLogger(__name__) + def get_request_kwargs(timeout, useragent): - """ - This Wrapper method exists b/c some values in req_kwargs dict - are methods which need to be called every time we make a request. + """This Wrapper method exists b/c some values in req_kwargs dict + are methods which need to be called every time we make a request """ return { - 'headers' : {'User-Agent': useragent}, - 'cookies' : cj(), - 'timeout' : timeout, - 'allow_redirects' : True + 'headers': {'User-Agent': useragent}, + 'cookies': cj(), + 'timeout': timeout, + 'allow_redirects': True } + def get_html(url, config=None, response=None): - """ - Retrieves the html for either a url or a response object. All html + """Retrieves the html for either a url or a response object. All html extractions MUST come from this method due to some intricies in the requests module. To get the encoding, requests only uses the HTTP header encoding declaration requests.utils.get_encoding_from_headers() and reverts @@ -39,34 +39,34 @@ def get_html(url, config=None, response=None): encoding in a lot of cases. """ FAIL_ENCODING = 'ISO-8859-1' - config = config or Configuration() + config = config or Configuration() useragent = config.browser_user_agent timeout = config.request_timeout if response is not None: if response.encoding != FAIL_ENCODING: return response.text - return response.content # not unicode, fix later + return response.content try: html = None - response = requests.get(url=url, **get_request_kwargs(timeout, useragent)) + response = requests.get(url=url, + **get_request_kwargs(timeout, useragent)) if response.encoding != FAIL_ENCODING: html = response.text else: - html = response.content # not unicode, fix later + html = response.content if html is None: html = u'' return html except Exception, e: - # print '[REQUEST FAILED]', str(e) log.debug('%s on %s' % (e, url)) return u'' + class MRequest(object): - """ - Wrapper for request object for multithreading. If the domain we are + """Wrapper for request object for multithreading. If the domain we are crawling is under heavy load, the self.resp will be left as None. If this is the case, we still want to report the url which has failed so (perhaps) we can try again later. @@ -81,23 +81,20 @@ def __init__(self, url, config=None): def send(self): try: self.resp = requests.get(self.url, **get_request_kwargs( - self.timeout, self.useragent)) + self.timeout, self.useragent)) except Exception, e: pass log.critical('[REQUEST FAILED] ' + str(e)) - # TODO, do something with url when we fail! - # print '[REQUEST FAILED]', str(e) + def multithread_request(urls, config=None): - """ - Request multiple urls via mthreading, order of urls & requests is stable + """Request multiple urls via mthreading, order of urls & requests is stable returns same requests but with response variables filled. """ config = config or Configuration() num_threads = config.number_threads pool = ThreadPool(num_threads) - # print 'beginning of mthreading, %s threads running' % activeCount() m_requests = [] for url in urls: @@ -113,7 +110,8 @@ def multithread_request(urls, config=None): # """receives a list of requests and sends them all # asynchronously at once""" # -# rs = (grequests.request('GET', url, **get_request_kwargs(timeout)) for url in urls) +# rs = (grequests.request('GET', url, +# **get_request_kwargs(timeout)) for url in urls) # responses = grequests.map(rs, size=10) # # return responses @@ -131,5 +129,5 @@ def multithread_request(urls, config=None): # for url in urls_or_url] # return resps # else: -# return requests.get(urls_or_url, **get_request_kwargs(timeout, useragent)) - +# return requests.get(urls_or_url, +# **get_request_kwargs(timeout, useragent)) diff --git a/newspaper/nlp.py b/newspaper/nlp.py index d43d8e60..daa2552e 100644 --- a/newspaper/nlp.py +++ b/newspaper/nlp.py @@ -9,25 +9,24 @@ import re import math +import operator -from collections import Counter, OrderedDict -from . import settings +from collections import Counter +from . import settings with open(settings.NLP_STOPWORDS_EN, 'r') as f: stopwords = set([w.strip() for w in f.readlines()]) ideal = 20.0 + def summarize(url='', title='', text=''): - """ - """ if (text == '' or title == ''): return [] if isinstance(title, unicode): title = title.encode('utf-8', 'ignore') - if isinstance(text, unicode): text = text.encode('utf-8', 'ignore') @@ -36,16 +35,15 @@ def summarize(url='', title='', text=''): keys = keywords(text) titleWords = split_words(title) - # score setences, and use the top 5 sentences + # Score setences, and use the top 5 sentences ranks = score(sentences, titleWords, keys).most_common(5) for rank in ranks: summaries.append(rank[0]) - return summaries + def score(sentences, titleWords, keywords): - """ - Score sentences based on different features. + """Score sentences based on different features """ senSize = len(sentences) ranks = Counter() @@ -57,16 +55,14 @@ def score(sentences, titleWords, keywords): sbsFeature = sbs(sentence, keywords) dbsFeature = dbs(sentence, keywords) frequency = (sbsFeature + dbsFeature) / 2.0 * 10.0 - - # weighted average of scores from four categories + # Weighted average of scores from four categories totalScore = (titleFeature*1.5 + frequency*2.0 + sentenceLength*1.0 + sentencePosition*1.0)/4.0 ranks[s] = totalScore return ranks + def sbs(words, keywords): - """ - """ score = 0.0 if (len(words) == 0): return 0 @@ -75,10 +71,9 @@ def sbs(words, keywords): score += keywords[word] return (1.0 / math.fabs(len(words)) * score)/10.0 + def dbs(words, keywords): - """ - """ - if (len(words)==0): + if (len(words) == 0): return 0 summ = 0 first = [] @@ -87,46 +82,45 @@ def dbs(words, keywords): for i, word in enumerate(words): if word in keywords: score = keywords[word] - if first==[]: + if first == []: first = [i, score] else: second = first first = [i, score] dif = first[0] - second[0] - summ+=(first[1]*second[1]) / (dif ** 2) - - # number of intersections + summ += (first[1]*second[1]) / (dif ** 2) + # Number of intersections k = len(set(keywords.keys()).intersection(set(words)))+1 return (1/(k*(k+1.0))*summ) + def split_words(text): - """ - Split a string into array of words. + """Split a string into array of words """ try: - text = re.sub(r'[^\w ]', '', text) #strip special chars + text = re.sub(r'[^\w ]', '', text) # strip special chars return [x.strip('.').lower() for x in text.split()] except TypeError: return None + def keywords(text): - """ - Get the top 10 keywords and their frequency scores ignores blacklisted + """Get the top 10 keywords and their frequency scores ignores blacklisted words in stopwords, counts the number of occurrences of each word, and - sorts them in reverse natural order (so descending) by number of occurrences. + sorts them in reverse natural order (so descending) by number of + occurrences. """ - import operator # sorting text = split_words(text) # of words before removing blacklist words num_words = len(text) text = [x for x in text if x not in stopwords] freq = Counter() for word in text: - freq[word]+=1 + freq[word] += 1 - minSize = min(10, len(freq)) - keywords = tuple(freq.most_common(minSize)) # get first 10 - keywords = dict((x,y) for x, y in keywords) # recreate a dict + min_size = min(10, len(freq)) + keywords = tuple(freq.most_common(min_size)) + keywords = dict((x, y) for x, y in keywords) for k in keywords: articleScore = keywords[k]*1.0 / max(num_words, 1) @@ -136,40 +130,37 @@ def keywords(text): keywords.reverse() return dict(keywords) + def split_sentences(text): - """ - Split a large string into sentences. + """Split a large string into sentences """ import nltk.data tokenizer = nltk.data.load('tokenizers/punkt/english.pickle') - # text = re.sub(r'[^\w .]', '', text) sentences = tokenizer.tokenize(text) - sentences = [x.replace('\n','') for x in sentences if len(x)>10] + sentences = [x.replace('\n', '') for x in sentences if len(x) > 10] return sentences + def length_score(sentence_len): - """ - """ - return 1- math.fabs(ideal - sentence_len) / ideal + return 1 - math.fabs(ideal - sentence_len) / ideal + def title_score(title, sentence): - """ - """ title = [x for x in title if x not in stopwords] count = 0.0 for word in sentence: if (word not in stopwords and word in title): - count+=1.0 + count += 1.0 return count / max(len(title), 1) + def sentence_position(i, size): - """ - Different sentence positions indicate different + """Different sentence positions indicate different probability of being an important sentence. """ - normalized = i*1.0 / size - if (normalized > 1.0): #just in case + normalized = i * 1.0 / size + if (normalized > 1.0): return 0 elif (normalized > 0.9): return 0.15 @@ -193,4 +184,3 @@ def sentence_position(i, size): return 0.17 else: return 0 - diff --git a/newspaper/outputformatters.py b/newspaper/outputformatters.py index 906324b9..8cbf488a 100644 --- a/newspaper/outputformatters.py +++ b/newspaper/outputformatters.py @@ -8,9 +8,9 @@ __copyright__ = 'Copyright 2014, Lucas Ou-Yang' from HTMLParser import HTMLParser + from .text import innerTrim -import lxml class OutputFormatter(object): @@ -21,36 +21,35 @@ def __init__(self, config): self.language = config.language self.stopwords_class = config.stopwords_class - def update_language(self, article): - """ - Called before formatting the top node to ensure the stopwords_class - has been updated incase a non-latin language code is extracted. - """ - if article.config.use_meta_language and article.meta_lang: - self.language = article.meta_lang - self.stopwords_class = article.config.\ - get_stopwords_class(article.meta_lang) + def update_language(self, meta_lang): + '''Required to be called before the extraction process in some + cases because the stopwords_class has to set incase the lang + is not latin based + ''' + if meta_lang: + self.language = meta_lang + self.stopwords_class = \ + self.config.get_stopwords_class(meta_lang) def get_top_node(self): return self.top_node - def get_formatted(self, article): - """ - Returns the body text of an article, and also the body article - html if specified. Returns in (text, html) form. + def get_formatted(self, top_node): + """Returns the body text of an article, and also the body article + html if specified. Returns in (text, html) form """ - self.top_node = article.top_node + self.top_node = top_node html, text = u'', u'' self.remove_negativescores_nodes() - if article.config.keep_article_html: + if self.config.keep_article_html: html = self.convert_to_html() self.links_to_text() self.add_newline_to_br() self.replace_with_text() - self.remove_fewwords_paragraphs(article) + self.remove_fewwords_paragraphs() text = self.convert_to_text() return (text, html) @@ -74,17 +73,17 @@ def add_newline_to_br(self): e.text = r'\n' def links_to_text(self): - """ - Cleans up and converts any nodes that should be considered text into text. + """Cleans up and converts any nodes that should be considered + text into text. """ self.parser.stripTags(self.get_top_node(), 'a') def remove_negativescores_nodes(self): + """If there are elements inside our top node that have a + negative gravity score, let's give em the boot. """ - If there are elements inside our top node that have a negative gravity score, - let's give em the boot. - """ - gravity_items = self.parser.css_select(self.top_node, "*[gravityScore]") + gravity_items = self.parser.css_select( + self.top_node, "*[gravityScore]") for item in gravity_items: score = self.parser.getAttribute(item, 'gravityScore') score = int(score, 0) @@ -98,9 +97,10 @@ def replace_with_text(self): With whatever text is inside them. code : http://lxml.de/api/lxml.etree-module.html#strip_tags """ - self.parser.stripTags(self.get_top_node(), 'b', 'strong', 'i', 'br', 'sup') + self.parser.stripTags( + self.get_top_node(), 'b', 'strong', 'i', 'br', 'sup') - def remove_fewwords_paragraphs(self, article): + def remove_fewwords_paragraphs(self): """ Remove paragraphs that have less than x number of words, would indicate that it's some sort of link. @@ -110,10 +110,14 @@ def remove_fewwords_paragraphs(self, article): for el in all_nodes: tag = self.parser.getTag(el) text = self.parser.getText(el) - stop_words = self.stopwords_class(language=self.language).get_stopword_count(text) - if (tag != 'br' or text != '\\r') and stop_words.get_stopword_count() < 3 \ - and len(self.parser.getElementsByTag(el, tag='object')) == 0 \ - and len(self.parser.getElementsByTag(el, tag='embed')) == 0: + stop_words = self.stopwords_class(language=self.language).\ + get_stopword_count(text) + if (tag != 'br' or text != '\\r') \ + and stop_words.get_stopword_count() < 3 \ + and len(self.parser.getElementsByTag( + el, tag='object')) == 0 \ + and len(self.parser.getElementsByTag( + el, tag='embed')) == 0: self.parser.remove(el) # TODO # check if it is in the right place @@ -121,7 +125,3 @@ def remove_fewwords_paragraphs(self, article): trimmed = self.parser.getText(el) if trimmed.startswith("(") and trimmed.endswith(")"): self.parser.remove(el) - - -class StandardOutputFormatter(OutputFormatter): - pass diff --git a/newspaper/parsers.py b/newspaper/parsers.py index cfdc0cce..07e2dbd1 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -1,25 +1,23 @@ # -*- coding: utf-8 -*- """ -Newspaper uses a lot of python-goose's extraction code. View their -license here: https://github.com/codelucas/newspaper/blob/master/GOOSE-LICENSE.txt +Newspaper uses a lot of python-goose's parsing code. View theirlicense: +https://github.com/codelucas/newspaper/blob/master/GOOSE-LICENSE.txt Parser objects will only contain operations that manipulate or query an lxml or soup dom object generated from an article's html. """ -import re import logging - +import lxml.etree import lxml.html -from lxml.html import soupparser -from lxml.html.clean import Cleaner -from lxml import etree from copy import deepcopy -from .text import innerTrim -from .utils import encodeValue + +from . import text +from . import utils log = logging.getLogger(__name__) + class Parser(object): @classmethod @@ -42,7 +40,7 @@ def css_select(cls, node, selector): @classmethod def fromstring(cls, html): - html = encodeValue(html) + html = utils.encodeValue(html) try: cls.doc = lxml.html.fromstring(html) except Exception, e: @@ -61,18 +59,19 @@ def node_to_string(cls, node): @classmethod def clean_article_html(cls, node): - article_cleaner = Cleaner() + article_cleaner = lxml.html.clean.Cleaner() article_cleaner.javascript = True article_cleaner.style = True - article_cleaner.allow_tags = ['a', 'span', 'p', 'br', 'strong', 'b', - 'em', 'i', 'tt', 'code', 'pre', 'blockquote', 'img', 'h1', - 'h2', 'h3', 'h4', 'h5', 'h6'] + article_cleaner.allow_tags = [ + 'a', 'span', 'p', 'br', 'strong', 'b', + 'em', 'i', 'tt', 'code', 'pre', 'blockquote', 'img', 'h1', + 'h2', 'h3', 'h4', 'h5', 'h6'] article_cleaner.remove_unknown_tags = False return article_cleaner.clean_html(node) @classmethod def nodeToString(cls, node): - return etree.tostring(node) + return lxml.etree.tostring(node) @classmethod def replaceTag(cls, node, tag): @@ -80,7 +79,7 @@ def replaceTag(cls, node, tag): @classmethod def stripTags(cls, node, *tags): - etree.strip_tags(node, *tags) + lxml.etree.strip_tags(node, *tags) @classmethod def getElementById(cls, node, idd): @@ -91,7 +90,8 @@ def getElementById(cls, node, idd): return None @classmethod - def getElementsByTag(cls, node, tag=None, attr=None, value=None, childs=False): + def getElementsByTag( + cls, node, tag=None, attr=None, value=None, childs=False): NS = "http://exslt.org/regular-expressions" # selector = tag or '*' selector = 'descendant-or-self::%s' % (tag or '*') @@ -193,7 +193,7 @@ def getTag(cls, node): @classmethod def getText(cls, node): txts = [i for i in node.itertext()] - return innerTrim(u' '.join(txts).strip()) + return text.innerTrim(u' '.join(txts).strip()) @classmethod def previousSiblings(cls, node): @@ -254,7 +254,6 @@ def outerHtml(cls, node): class ParserSoup(Parser): @classmethod def fromstring(cls, html): - html = encodeValue(html) - cls.doc = soupparser.fromstring(html) + html = utils.encodeValue(html) + cls.doc = lxml.html.soupparser.fromstring(html) return cls.doc - diff --git a/newspaper/settings.py b/newspaper/settings.py index 257334c4..db3c4b22 100644 --- a/newspaper/settings.py +++ b/newspaper/settings.py @@ -11,21 +11,23 @@ import logging import os -from .version import __version__ - from cookielib import CookieJar as cj +from .version import __version__ + log = logging.getLogger(__name__) PARENT_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) -POPULAR_URLS = os.path.join(PARENT_DIRECTORY, 'resources/misc/popular_sources.txt') +POPULAR_URLS = os.path.join( + PARENT_DIRECTORY, 'resources/misc/popular_sources.txt') USERAGENTS = os.path.join(PARENT_DIRECTORY, 'resources/misc/useragents.txt') STOPWORDS_DIR = os.path.join(PARENT_DIRECTORY, 'resources/text') # NLP stopwords are != regular stopwords for now... -NLP_STOPWORDS_EN = os.path.join(PARENT_DIRECTORY, 'resources/misc/stopwords-nlp-en.txt') +NLP_STOPWORDS_EN = os.path.join( + PARENT_DIRECTORY, 'resources/misc/stopwords-nlp-en.txt') DATA_DIRECTORY = '.newspaper_scraper' @@ -35,7 +37,8 @@ # Error log LOGFILE = os.path.join(TOP_DIRECTORY, 'newspaper_errors_%s.log' % __version__) -MONITOR_LOGFILE = os.path.join(TOP_DIRECTORY, 'newspaper_monitors_%s.log' % __version__) +MONITOR_LOGFILE = os.path.join( + TOP_DIRECTORY, 'newspaper_monitors_%s.log' % __version__) # Memo directory (same for all concur crawlers) MEMO_FILE = 'memoized' @@ -52,4 +55,3 @@ os.mkdir(ANCHOR_DIRECTORY) TRENDING_URL = 'http://www.google.com/trends/hottrends/atom/feed?pn=p1' - diff --git a/newspaper/source.py b/newspaper/source.py index 6f73ab18..4f2269ed 100644 --- a/newspaper/source.py +++ b/newspaper/source.py @@ -11,16 +11,15 @@ import logging from . import network +from . import urls +from . import utils + from .article import Article -from .settings import ANCHOR_DIRECTORY -from .packages.tldextract import tldextract -from .packages.feedparser import feedparser +from .extractors import ContentExtractor from .configuration import Configuration -from .extractors import StandardContentExtractor -from .urls import ( - get_domain, get_scheme, prepare_url) -from .utils import ( - memoize_articles, cache_disk, clear_memo_cache, encodeValue, extend_config) +from .packages.feedparser import feedparser +from .packages.tldextract import tldextract +from .settings import ANCHOR_DIRECTORY log = logging.getLogger(__name__) @@ -28,7 +27,7 @@ class Category(object): def __init__(self, url): - self.url = encodeValue(url) + self.url = utils.encodeValue(url) self.html = None self.doc = None @@ -36,15 +35,13 @@ def __init__(self, url): class Feed(object): def __init__(self, url): - self.url = encodeValue(url) + self.url = utils.encodeValue(url) self.rss = None # TODO self.dom = None, speed up Feedparser class Source(object): - """ - Sources are abstractions of online news vendors like huffpost or cnn. - + """Sources are abstractions of online news vendors like huffpost or cnn. domain = 'www.cnn.com' scheme = 'http' categories = ['http://cnn.com/world', 'http://money.cnn.com'] @@ -53,24 +50,22 @@ class Source(object): brand = 'cnn' """ def __init__(self, url, config=None, **kwargs): - """ - **The config object for this source will be passed into all of this - source's children articles unless specified otherwise or re-set.** + """The config object for this source will be passed into all of this + source's children articles unless specified otherwise or re-set. """ if (url is None) or ('://' not in url) or (url[:4] != 'http'): raise Exception('Input url is bad!') - self.config = config or Configuration() # Order matters - self.config = extend_config(self.config, kwargs) + self.config = config or Configuration() + self.config = utils.extend_config(self.config, kwargs) - self.parser = self.config.get_parser() - self.extractor = StandardContentExtractor(config=self.config) + self.extractor = ContentExtractor(self.config) - self.url = encodeValue(url) - self.url = prepare_url(url) + self.url = utils.encodeValue(url) + self.url = urls.prepare_url(url) - self.domain = get_domain(self.url) - self.scheme = get_scheme(self.url) + self.domain = urls.get_domain(self.url) + self.scheme = urls.get_scheme(self.url) self.categories = [] self.feeds = [] @@ -84,35 +79,31 @@ def __init__(self, url, config=None, **kwargs): self.brand = tldextract.extract(self.url).domain self.description = u'' - self.is_parsed = False # flags to warn users if they forgot to - self.is_downloaded = False # download() or parse() + self.is_parsed = False + self.is_downloaded = False def build(self): - """ - Encapsulates download and basic parsing with lxml. May be a + """Encapsulates download and basic parsing with lxml. May be a good idea to split this into download() and parse() methods. """ self.download() self.parse() - # Can not merge category and feed tasks together because - # computing feed urls relies on the category urls! self.set_categories() - self.download_categories() # mthread + self.download_categories() # mthread self.parse_categories() self.set_feeds() - self.download_feeds() # mthread - # self.parse_feeds() # TODO regexing out feeds until we fix feedparser! + self.download_feeds() # mthread + # TODO: self.parse_feeds() # regex for now self.generate_articles() def purge_articles(self, reason, articles): - """ - Delete rejected articles, if there is an articles param, we - purge from there, otherwise purge from our source instance. + """Delete rejected articles, if there is an articles param, + purge from there, otherwise purge from source instance. - Reference this excellent StackOverflow post for some of the wonky + Reference this StackOverflow post for some of the wonky syntax below: http://stackoverflow.com/questions/1207406/remove-items-from-a- list-while-iterating-in-python @@ -123,46 +114,39 @@ def purge_articles(self, reason, articles): articles[:] = [a for a in articles if a.is_valid_body()] return articles - @cache_disk(seconds=(86400*1), cache_folder=ANCHOR_DIRECTORY) + @utils.cache_disk(seconds=(86400*1), cache_folder=ANCHOR_DIRECTORY) def _get_category_urls(self, domain): + """The domain param is **necessary**, see .utils.cache_disk for reasons. + the boilerplate method is so we can use this decorator right. + We are caching categories for 1 day. """ - The domain param is **necessary**, see .utils.cache_disk for reasons. - the boilerplate method is so we can use this decorator right. We are caching - categories for 1 day. - """ - return self.extractor.get_category_urls(self) + return self.extractor.get_category_urls(self.url, self.doc) def set_categories(self): - """ - """ urls = self._get_category_urls(self.domain) self.categories = [Category(url=url) for url in urls] def set_feeds(self): + """Don't need to cache getting feed urls, it's almost + instant with xpath """ - Don't need to cache getting feed urls, it's almost - instant with xpath. - """ - urls = self.extractor.get_feed_urls(self) + urls = self.extractor.get_feed_urls(self.url, self.categories) self.feeds = [Feed(url=url) for url in urls] def set_description(self): + """Sets a blurb for this source, for now we just query the + desc html attribute """ - Sets a blurb for this source, for now we just - query the desc html attribute. - """ - desc = self.extractor.get_meta_description(self) - self.description = encodeValue(desc) + desc = self.extractor.get_meta_description(self.doc) + self.description = utils.encodeValue(desc) def download(self): - """ - Downloads html of source. + """Downloads html of source """ self.html = network.get_html(self.url, config=self.config) def download_categories(self): - """ - Download all category html, can use mthreading. + """Download all category html, can use mthreading """ category_urls = [c.url for c in self.categories] requests = network.multithread_request(category_urls, self.config) @@ -170,15 +154,16 @@ def download_categories(self): for index, _ in enumerate(self.categories): req = requests[index] if req.resp is not None: - self.categories[index].html = network.get_html(req.url, response=req.resp) + self.categories[index].html = network.get_html( + req.url, response=req.resp) else: if self.config.verbose: - print 'deleting category', self.categories[index].url, 'due to download err' + print ('deleting category', + self.categories[index].url, 'due to download err') self.categories = [c for c in self.categories if c.html] def download_feeds(self): - """ - Download all feed html, can use mthreading. + """Download all feed html, can use mthreading """ feed_urls = [f.url for f in self.feeds] requests = network.multithread_request(feed_urls, self.config) @@ -186,31 +171,32 @@ def download_feeds(self): for index, _ in enumerate(self.feeds): req = requests[index] if req.resp is not None: - self.feeds[index].rss = network.get_html(req.url, response=req.resp) + self.feeds[index].rss = network.get_html( + req.url, response=req.resp) else: if self.config.verbose: - print 'deleting feed', self.categories[index].url, 'due to download err' + print ('deleting feed', + self.categories[index].url, 'due to download err') self.feeds = [f for f in self.feeds if f.rss] def parse(self): - """ - Sets the lxml root, also sets lxml roots of all - children links, also sets description. + """Sets the lxml root, also sets lxml roots of all + children links, also sets description """ # TODO: This is a terrible idea, ill try to fix it when i'm more rested - self.doc = self.parser.fromstring(self.html) + self.doc = self.config.get_parser().fromstring(self.html) if self.doc is None: print '[Source parse ERR]', self.url return self.set_description() def parse_categories(self): + """Parse out the lxml root in each category """ - Parse out the lxml root in each category. - """ - log.debug('We are extracting from %d categories' % len(self.categories)) + log.debug('We are extracting from %d categories' % + len(self.categories)) for category in self.categories: - doc = self.parser.fromstring(category.html) + doc = self.config.get_parser().fromstring(category.html) category.doc = doc if category.doc is None: print '[Category parse ERR]', category.url @@ -218,8 +204,7 @@ def parse_categories(self): self.categories = [c for c in self.categories if c.doc is not None] def parse_feeds(self): - """ - **THIS METHOD IS CURRENTLY RETIRED** + """DEPRECATED Due to the slow speed of feedparser, we won't be dom parsing our .rss feeds, but rather regex searching for urls in the .rss text and then relying on our article logic to detect false urls. @@ -235,8 +220,7 @@ def parse_feeds(self): self.feeds = [feed for feed in self.feeds if feed.dom is not None] def feeds_to_articles(self): - """ - Returns articles given the url of a feed. + """Returns articles given the url of a feed """ articles = [] for feed in self.feeds: @@ -248,29 +232,28 @@ def feeds_to_articles(self): article = Article( url=url, source_url=self.url, - config=self.config - # (pre) title=? # TODO Fast title regexing? - ) + config=self.config) cur_articles.append(article) cur_articles = self.purge_articles('url', cur_articles) after_purge = len(cur_articles) if self.config.memoize_articles: - cur_articles = memoize_articles(self, cur_articles) + cur_articles = utils.memoize_articles(self, cur_articles) after_memo = len(cur_articles) articles.extend(cur_articles) if self.config.verbose: - print '%d->%d->%d for %s' % (before_purge, after_purge, after_memo, feed.url) - log.debug('%d->%d->%d for %s' % (before_purge, after_purge, after_memo, feed.url)) + print('%d->%d->%d for %s' % + (before_purge, after_purge, after_memo, feed.url)) + log.debug('%d->%d->%d for %s' % + (before_purge, after_purge, after_memo, feed.url)) return articles def categories_to_articles(self): - """ - Takes the categories, splays them into a big list of urls and churns - the articles out of each url with the url_to_article method. + """Takes the categories, splays them into a big list of urls and churns + the articles out of each url with the url_to_article method """ articles = [] for category in self.categories: @@ -294,39 +277,37 @@ def categories_to_articles(self): after_purge = len(cur_articles) if self.config.memoize_articles: - cur_articles = memoize_articles(self, cur_articles) + cur_articles = utils.memoize_articles(self, cur_articles) after_memo = len(cur_articles) articles.extend(cur_articles) if self.config.verbose: - print '%d->%d->%d for %s' % (before_purge, after_purge, after_memo, category.url) - log.debug('%d->%d->%d for %s' % (before_purge, after_purge, after_memo, category.url)) - + print ('%d->%d->%d for %s' % + (before_purge, after_purge, after_memo, category.url)) + log.debug('%d->%d->%d for %s' % + (before_purge, after_purge, after_memo, category.url)) return articles def _generate_articles(self): - """ - Returns a list of all articles, from both categories and feeds. + """Returns a list of all articles, from both categories and feeds """ category_articles = self.categories_to_articles() feed_articles = self.feeds_to_articles() articles = feed_articles + category_articles - uniq = { article.url:article for article in articles } + uniq = {article.url: article for article in articles} return uniq.values() def generate_articles(self, limit=5000): - """ - Saves all current articles of news source, filter out bad urls. + """Saves all current articles of news source, filter out bad urls """ articles = self._generate_articles() self.articles = articles[:limit] log.debug(len(articles), 'articles generated and cutoff at', limit) def download_articles(self, threads=1): - """ - Downloads all articles attached to self. + """Downloads all articles attached to self """ # TODO fix how the article's is_downloaded is not set! urls = [a.url for a in self.articles] @@ -342,7 +323,8 @@ def download_articles(self, threads=1): self.articles = [a for a in self.articles if a.html] else: if threads > 5: - print 'Using 5+ threads on a single source may get you rate limited!' + print ('Using 5+ threads on a single source ' + 'may get you rate limited!') filled_requests = network.multithread_request(urls, self.config) # Note that the responses are returned in original order for index, req in enumerate(filled_requests): @@ -359,8 +341,7 @@ def download_articles(self, threads=1): [a.url for a in failed_articles] def parse_articles(self): - """ - Parse all articles, delete if too small. + """Parse all articles, delete if too small """ for index, article in enumerate(self.articles): article.parse() @@ -369,56 +350,48 @@ def parse_articles(self): self.is_parsed = True def size(self): - """ - Number of articles linked to this news source. + """Number of articles linked to this news source """ if self.articles is None: return 0 return len(self.articles) def clean_memo_cache(self): + """Clears the memoization cache for this specific news domain """ - Clears the memoization cache for this specific news domain. - """ - clear_memo_cache(self) + utils.clear_memo_cache(self) def feed_urls(self): - """ - Returns a list of feed urls. + """Returns a list of feed urls """ return [feed.url for feed in self.feeds] def category_urls(self): - """ - Returns a list of category urls. + """Returns a list of category urls """ return [category.url for category in self.categories] def article_urls(self): - """ - Returns a list of article urls. + """Returns a list of article urls """ return [article.url for article in self.articles] def get_key(self): - """ - """ # TODO pass def clear_anchor_directory(self): + """Clears out all files in our directory where we cache anchors + the key is sha1(self.domain).hexdigest() fn is ANCHOR_DIR/key. """ - Clears out all files in our directory where we cache anchors - the key is sha1(self.domain).hexdigest() - fn is ANCHOR_DIR/key. - """ - # TODO tomorow - #d_pth = os.path.join(settings.MEMO_DIR, domain_to_filename(source_domain)) - #os.path.remove(ANCHOR_DIRECTORY) + pass + # TODO: + # d_pth = os.path.join( + # settings.MEMO_DIR, domain_to_filename(source_domain)) + # os.path.remove(ANCHOR_DIRECTORY) def print_summary(self): - """ - Prints out a summary of the data in our source instance. + """Prints out a summary of the data in our source instance """ print '[source url]:', self.url print '[source brand]:', self.brand @@ -439,4 +412,3 @@ def print_summary(self): print 'feed_urls:', self.feed_urls() print '\r\n' print 'category_urls:', self.category_urls() - diff --git a/newspaper/text.py b/newspaper/text.py index f83c48ae..f3b40a43 100644 --- a/newspaper/text.py +++ b/newspaper/text.py @@ -15,6 +15,7 @@ TABSSPACE = re.compile(r'[\s\t]+') + def innerTrim(value): if isinstance(value, (unicode, str)): # remove tab and white space @@ -23,6 +24,7 @@ def innerTrim(value): return value.strip() return '' + class WordStats(object): def __init__(self): @@ -56,14 +58,16 @@ def set_word_count(self, cnt): class StopWords(object): - PUNCTUATION = re.compile("[^\\p{Ll}\\p{Lu}\\p{Lt}\\p{Lo}\\p{Nd}\\p{Pc}\\s]") + PUNCTUATION = re.compile( + "[^\\p{Ll}\\p{Lu}\\p{Lt}\\p{Lo}\\p{Nd}\\p{Pc}\\s]") TRANS_TABLE = string.maketrans('', '') _cached_stop_words = {} def __init__(self, language='en'): - if not language in self._cached_stop_words: + if language not in self._cached_stop_words: path = os.path.join('text', 'stopwords-%s.txt' % language) - self._cached_stop_words[language] = set(FileHelper.loadResourceFile(path).splitlines()) + self._cached_stop_words[language] = \ + set(FileHelper.loadResourceFile(path).splitlines()) self.STOP_WORDS = self._cached_stop_words[language] def remove_punctuation(self, content): @@ -72,7 +76,8 @@ def remove_punctuation(self, content): content_is_unicode = isinstance(content, unicode) if content_is_unicode: content = content.encode('utf-8') - stripped_input = content.translate(self.TRANS_TABLE, string.punctuation) + stripped_input = content.translate( + self.TRANS_TABLE, string.punctuation) if content_is_unicode: return stripped_input.decode('utf-8') @@ -101,8 +106,7 @@ def get_stopword_count(self, content): class StopWordsChinese(StopWords): - """ - Chinese segmentation. + """Chinese segmentation """ def __init__(self, language='zh'): super(StopWordsChinese, self).__init__(language='zh') @@ -115,8 +119,7 @@ def candidate_words(self, stripped_input): class StopWordsArabic(StopWords): - """ - Arabic segmentation. + """Arabic segmentation """ def __init__(self, language='ar'): # force ar languahe code @@ -135,8 +138,7 @@ def candidate_words(self, stripped_input): class StopWordsKorean(StopWords): - """ - Korean segmentation. + """Korean segmentation """ def __init__(self, language='ko'): super(StopWordsKorean, self).__init__(language='ko') @@ -158,4 +160,3 @@ def get_stopword_count(self, content): ws.set_stopword_count(len(overlapping_stopwords)) ws.set_stop_words(overlapping_stopwords) return ws - diff --git a/newspaper/urls.py b/newspaper/urls.py index bcc202b4..62a74c8a 100644 --- a/newspaper/urls.py +++ b/newspaper/urls.py @@ -11,8 +11,7 @@ import logging import re -from urlparse import ( - urlparse, urljoin, urlsplit, urlunsplit, parse_qs) +from urlparse import parse_qs, urljoin, urlparse, urlsplit, urlunsplit from .packages.tldextract import tldextract diff --git a/newspaper/utils/__init__.py b/newspaper/utils/__init__.py index bd21715b..30e69cf6 100644 --- a/newspaper/utils/__init__.py +++ b/newspaper/utils/__init__.py @@ -8,21 +8,21 @@ __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' -import time +import codecs import hashlib -import re +import logging import os -import random -import codecs -import threading -import sys import pickle -import logging +import random +import re import string +import sys +import threading +import time from hashlib import sha1 -from .encoding import ( - smart_unicode, smart_str, DjangoUnicodeDecodeError) + +from . import encoding from .. import settings log = logging.getLogger(__name__) @@ -35,7 +35,7 @@ def loadResourceFile(self, filename): if not os.path.isabs(filename): # _PARENT_DIR = os.path.join(_TEST_DIR, '../..') # packages/goose # dirpath = os.path.dirname(goose.__file__) - dirpath = os.path.abspath(os.path.dirname(__file__)) # goose/utils + dirpath = os.path.abspath(os.path.dirname(__file__)) path = os.path.join(dirpath, '../resources', filename) else: path = filename @@ -67,16 +67,14 @@ def get_parsing_candidate(self, url, raw_html): class URLHelper(object): @classmethod def get_parsing_candidate(self, url_to_crawl): - # replace shebang in urls + # Replace shebang in urls final_url = url_to_crawl.replace('#!', '?_escaped_fragment_=') \ - if '#!' in url_to_crawl else url_to_crawl + if '#!' in url_to_crawl else url_to_crawl link_hash = '%s.%s' % (hashlib.md5(final_url).hexdigest(), time.time()) return ParsingCandidate(final_url, link_hash) class StringSplitter(object): - """ - """ def __init__(self, pattern): self.pattern = re.compile(pattern) @@ -101,7 +99,6 @@ class ReplaceSequence(object): def __init__(self): self.replacements = [] - #@classmethod def create(self, firstPattern, replaceWith=None): result = StringReplacement(firstPattern, replaceWith or u'') self.replacements.append(result) @@ -115,7 +112,6 @@ def replaceAll(self, string): return u'' mutatedString = string - for rp in self.replacements: mutatedString = rp.replaceAll(mutatedString) return mutatedString @@ -126,8 +122,7 @@ class TimeoutError(Exception): def timelimit(timeout): - """ - Borrowed from web.py, rip Aaron Swartz. + """Borrowed from web.py, rip Aaron Swartz """ def _1(function): def _2(*args, **kw): @@ -145,7 +140,6 @@ def run(self): self.result = function(*args, **kw) except: self.error = sys.exc_info() - c = Dispatch() c.join(timeout) if c.isAlive(): @@ -156,26 +150,26 @@ def run(self): return _2 return _1 + def domain_to_filename(domain): + """All '/' are turned into '-', no trailing. schema's + are gone, only the raw domain + ".txt" remains """ - All '/' are turned into '-', no trailing. schema's - are gone, only the raw domain + ".txt" remains. - """ - filename = domain.replace('/', '-') + filename = domain.replace('/', '-') if filename[-1] == '-': filename = filename[:-1] filename += ".txt" return filename + def filename_to_domain(filename): - """ - [:-4] for the .txt at end. + """[:-4] for the .txt at end """ return filename.replace('-', '/')[:-4] + def is_ascii(word): - """ - True if a word is only ascii chars. + """True if a word is only ascii chars """ def onlyascii(char): if ord(char) > 127: @@ -187,28 +181,28 @@ def onlyascii(char): return False return True + def to_valid_filename(s): - """ - Converts arbitrary string (for us domain name) - into a valid file name for caching. + """Converts arbitrary string (for us domain name) + into a valid file name for caching """ valid_chars = "-_.() %s%s" % (string.ascii_letters, string.digits) return ''.join(c for c in s if c in valid_chars) + def cache_disk(seconds=(86400*5), cache_folder="/tmp"): - """ - Caching extracting category locations & rss feeds for 5 days. + """Caching extracting category locations & rss feeds for 5 days """ def do_cache(function): def inner_function(*args, **kwargs): - """ - Calculate a cache key based on the decorated method signature + """Calculate a cache key based on the decorated method signature args[1] indicates the domain of the inputs, we hash on domain! """ key = sha1(str(args[1]) + str(kwargs)).hexdigest() filepath = os.path.join(cache_folder, key) - # verify that the cached object exists and is less than X seconds old + # verify that the cached object exists and is less than + # X seconds old if os.path.exists(filepath): modified = os.path.getmtime(filepath) age_seconds = time.time() - modified @@ -220,13 +214,12 @@ def inner_function(*args, **kwargs): # ... and save the cached object for next time pickle.dump(result, open(filepath, "wb")) return result - return inner_function return do_cache + def print_duration(method): - """ - Prints out the runtime duration of a method in seconds. + """Prints out the runtime duration of a method in seconds """ def timed(*args, **kw): ts = time.time() @@ -234,30 +227,28 @@ def timed(*args, **kw): te = time.time() print '%r %2.2f sec' % (method.__name__, te-ts) return result - return timed + def chunks(l, n): - """ - Yield n successive chunks from l. + """Yield n successive chunks from l """ newn = int(len(l) / n) for i in xrange(0, n-1): yield l[i*newn:i*newn+newn] yield l[n*newn-newn:] + def purge(fn, pattern): + """Delete files in a dir matching pattern """ - Delete files in a dir matching pattern. - """ - import os, re for f in os.listdir(fn): if re.search(pattern, f): os.remove(os.path.join(fn, f)) + def clear_memo_cache(source): - """ - Clears the memoization cache for this specific news domain. + """Clears the memoization cache for this specific news domain """ d_pth = os.path.join(settings.MEMO_DIR, domain_to_filename(source.domain)) if os.path.exists(d_pth): @@ -265,23 +256,22 @@ def clear_memo_cache(source): else: print 'memo file for', source.domain, 'has already been deleted!' + def encodeValue(value): - """ - """ if value is None: return u'' string_org = value try: - value = smart_unicode(value) - except (UnicodeEncodeError, DjangoUnicodeDecodeError): - value = smart_str(value) + value = encoding.smart_unicode(value) + except (UnicodeEncodeError, encoding.DjangoUnicodeDecodeError): + value = encoding.smart_str(value) except: value = string_org return value.strip() + def memoize_articles(source, articles): - """ - When we parse the <a> links in an <html> page, on the 2nd run + """When we parse the <a> links in an <html> page, on the 2nd run and later, check the <a> links of previous runs. If they match, it means the link must not be an article, because article urls change as time passes. This method also uniquifies articles. @@ -292,19 +282,17 @@ def memoize_articles(source, articles): if len(articles) == 0: return [] - cur_articles = { article.url:article for article in articles } memo = {} - # print '******current article urls!', cur_articles.keys()[:10] - + cur_articles = {article.url: article for article in articles} d_pth = os.path.join(settings.MEMO_DIR, domain_to_filename(source_domain)) if os.path.exists(d_pth): f = codecs.open(d_pth, 'r', 'utf8') - urls = f.readlines() # list of urls, unicode + urls = f.readlines() f.close() urls = [u.strip() for u in urls] - memo = { url:True for url in urls } + memo = {url: True for url in urls} # prev_length = len(memo) for url, article in cur_articles.items(): if memo.get(url): @@ -320,7 +308,6 @@ def memoize_articles(source, articles): [encodeValue(href.strip()) for href in cur_articles.keys()]) # new_length = len(cur_articles) - if len(memo) > config.MAX_FILE_MEMO: # We still keep current batch of articles though! log.critical('memo overflow, dumping') @@ -330,12 +317,11 @@ def memoize_articles(source, articles): ff = codecs.open(d_pth, 'w', 'utf-8') ff.write(memo_text) ff.close() - # print '***** final article urls', cur_articles.keys()[:10] - return cur_articles.values() # articles returned + return cur_articles.values() + def get_useragent(): - """ - Uses generator to return next useragent in saved file. + """Uses generator to return next useragent in saved file """ with open(settings.USERAGENTS, 'r') as f: agents = f.readlines() @@ -343,9 +329,9 @@ def get_useragent(): agent = agents[selection] return agent.strip() + def get_available_languages(): - """ - Returns a list of available languages and their 2 char input codes. + """Returns a list of available languages and their 2 char input codes """ stopword_files = os.listdir(os.path.join(settings.STOPWORDS_DIR)) two_dig_codes = [f.split('-')[1].split('.')[0] for f in stopword_files] @@ -353,9 +339,9 @@ def get_available_languages(): assert len(d) == 2 return two_dig_codes + def print_available_languages(): - """ - Prints available languages with their full names + """Prints available languages with their full names """ language_dict = { 'ar': 'Arabic', @@ -386,6 +372,7 @@ def print_available_languages(): print ' %s\t\t\t %s' % (code, language_dict[code]) print + def extend_config(config, config_items): """ We are handling config value setting like this for a cleaner api. @@ -397,4 +384,3 @@ def extend_config(config, config_items): setattr(config, key, val) return config - diff --git a/newspaper/utils/encoding.py b/newspaper/utils/encoding.py index 2d6ba530..dce9cba2 100644 --- a/newspaper/utils/encoding.py +++ b/newspaper/utils/encoding.py @@ -3,10 +3,12 @@ Byte string <---> unicode conversions take place here, pretty much anything encoding related """ -import types import datetime +import types + from decimal import Decimal + class DjangoUnicodeDecodeError(UnicodeDecodeError): def __init__(self, obj, *args): self.obj = obj @@ -15,13 +17,11 @@ def __init__(self, obj, *args): def __str__(self): original = UnicodeDecodeError.__str__(self) return '%s. You passed in %r (%s)' % (original, self.obj, - type(self.obj)) + type(self.obj)) class StrAndUnicode(object): - """ - A class whose __str__ returns its __unicode__ as a UTF-8 bytestring. - + """A class whose __str__ returns its __unicode__ as a UTF-8 bytestring. Useful as a mix-in. """ def __str__(self): @@ -29,10 +29,8 @@ def __str__(self): def smart_unicode(s, encoding='utf-8', strings_only=False, errors='strict'): - """ - Returns a unicode object representing 's'. Treats bytestrings using the + """Returns a unicode object representing 's'. Treats bytestrings using the 'encoding' codec. - If strings_only is True, don't convert (some) non-string-like objects. """ # if isinstance(s, Promise): @@ -56,12 +54,9 @@ def is_protected_type(obj): def force_unicode(s, encoding='utf-8', strings_only=False, errors='strict'): - """ - Similar to smart_unicode, except that lazy instances are resolved to + """Similar to smart_unicode, except that lazy instances are resolved to strings, rather than kept as lazy objects. - If strings_only is True, don't convert (some) non-string-like objects. - """ # Handle the common case first, saves 30-40% in performance when s # is an instance of unicode. This function gets called often in that @@ -87,7 +82,7 @@ def force_unicode(s, encoding='utf-8', strings_only=False, errors='strict'): # approximation to what the Exception's standard str() # output should be. s = u' '.join([force_unicode(arg, encoding, strings_only, - errors) for arg in s]) + errors) for arg in s]) elif not isinstance(s, unicode): # Note: We use .decode() here, instead of unicode(s, encoding, # errors), so that if s is a SafeString, it ends up being a @@ -103,14 +98,12 @@ def force_unicode(s, encoding='utf-8', strings_only=False, errors='strict'): # further exception by individually forcing the exception args # to unicode. s = u' '.join([force_unicode(arg, encoding, strings_only, - errors) for arg in s]) + errors) for arg in s]) return s def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): - """ - Returns a bytestring version of 's', encoded as specified in 'encoding'. - + """Returns a bytestring version of 's', encoded as specified in 'encoding'. If strings_only is True, don't convert (some) non-string-like objects. """ if strings_only and isinstance(s, (types.NoneType, int)): @@ -126,7 +119,7 @@ def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): # know how to print itself properly. We shouldn't raise a # further exception. return ' '.join([smart_str(arg, encoding, strings_only, - errors) for arg in s]) + errors) for arg in s]) return unicode(s).encode(encoding, errors) elif isinstance(s, unicode): return s.encode(encoding, errors) diff --git a/newspaper/videos/extractors.py b/newspaper/videos/extractors.py index f5739c9d..ffbcbb54 100644 --- a/newspaper/videos/extractors.py +++ b/newspaper/videos/extractors.py @@ -1,7 +1,4 @@ # -*- coding: utf-8 -*- -""" -""" - from .videos import Video VIDEOS_TAGS = ['iframe', 'embed', 'object', 'video'] @@ -9,27 +6,19 @@ class VideoExtractor(object): + """Extracts a list of video from Article top node """ - Extracts a list of video from Article top node - """ - def __init__(self, article, config): - # article - self.article = article - - # config + def __init__(self, config, top_node): self.config = config - - # parser self.parser = self.config.get_parser() - - # candidates + self.top_node = top_node self.candidates = [] - - # movies self.movies = [] def get_embed_code(self, node): - return "".join([line.strip() for line in self.parser.nodeToString(node).splitlines()]) + return "".join([ + line.strip() + for line in self.parser.nodeToString(node).splitlines()]) def get_embed_type(self, node): return self.parser.getTag(node) @@ -51,8 +40,7 @@ def get_provider(self, src): return None def get_video(self, node): - """ - Create a video object from a video embed + """Create a video object from a video embed """ video = Video() video.embed_code = self.get_embed_code(node) @@ -67,7 +55,8 @@ def get_iframe_tag(self, node): return self.get_video(node) def get_video_tag(self, node): - """extract html video tags""" + """Extract html video tags + """ return Video() def get_embed_tag(self, node): @@ -91,7 +80,8 @@ def get_object_tag(self, node): # get the object source # if wa don't have a src node don't coninue - src_node = self.parser.getElementsByTag(node, tag="param", attr="name", value="movie") + src_node = self.parser.getElementsByTag( + node, tag="param", attr="name", value="movie") if not src_node: return None @@ -108,9 +98,8 @@ def get_object_tag(self, node): return video def get_videos(self): - # candidates node - self.candidates = self.parser.getElementsByTags(self.article.top_node, VIDEOS_TAGS) - + self.candidates = self.parser.getElementsByTags( + self.top_node, VIDEOS_TAGS) # loop all candidates # and check if src attribute belongs to a video provider for candidate in self.candidates: diff --git a/newspaper/videos/videos.py b/newspaper/videos/videos.py index 7a8e4aa0..01e2061e 100644 --- a/newspaper/videos/videos.py +++ b/newspaper/videos/videos.py @@ -1,28 +1,20 @@ # -*- coding: utf-8 -*- -""" -""" + class Video(object): - """ - Video object + """Video object """ def __init__(self): - # type of embed # embed, object, iframe self.embed_type = None - # video provider name self.provider = None - # width self.width = None - # height self.height = None - # embed code self.embed_code = None - # src self.src = None diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 09b30878..f1598f0f 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -27,14 +27,16 @@ from newspaper import Config from newspaper.network import multithread_request from newspaper.configuration import Configuration -from newspaper.text import StopWords, StopWordsArabic, StopWordsKorean, StopWordsChinese +from newspaper.text import (StopWords, StopWordsArabic, + StopWordsKorean, StopWordsChinese) from newspaper.utils.encoding import smart_str, smart_unicode from newspaper.utils import encodeValue def print_test(method): - """utility method for print verbalizing test suite, prints out - time taken for test and functions name, and status""" + """Utility method for print verbalizing test suite, prints out + time taken for test and functions name, and status + """ def run(*args, **kw): ts = time.time() print '\ttesting function %r' % method.__name__ @@ -43,14 +45,16 @@ def run(*args, **kw): print '\t[OK] in %r %2.2f sec' % (method.__name__, te-ts) return run -def read_urls(base_fn=URLS_FN, amount=100): - """utility funct which extracts out a listing of sample urls""" +def read_urls(base_fn=URLS_FN, amount=100): + """Utility funct which extracts out a listing of sample urls + """ f = codecs.open(base_fn, 'r', 'utf8') lines = f.readlines() lines = [l.strip() for l in lines] return lines[:amount] + def mock_response_with(url, response_file): response_path = os.path.join(TEST_DIR, "data/html/%s.html" % response_file) with open(response_path, 'r') as f: @@ -59,6 +63,7 @@ def mock_response_with(url, response_file): responses.add(responses.GET, url, body=body, status=200, content_type='text/html') + class ArticleTestCase(unittest.TestCase): def runTest(self): print 'testing article unit' @@ -75,15 +80,19 @@ def setUp(self): """called before the first test case of this unit begins""" self.article = Article( - url='http://www.cnn.com/2013/11/27/travel/weather-thanksgiving/index.html?iref=allsearch') + url='http://www.cnn.com/2013/11/27/travel/weather-' + 'thanksgiving/index.html?iref=allsearch') def tearDown(self): - """called after all test cases finish of this unit""" + """Called after all test cases finish of this unit + """ pass @print_test def test_url(self): - assert self.article.url == u'http://www.cnn.com/2013/11/27/travel/weather-thanksgiving/index.html' + assert self.article.url == ( + u'http://www.cnn.com/2013/11/27/travel/weather-' + 'thanksgiving/index.html') @print_test @responses.activate @@ -94,9 +103,10 @@ def test_download_html(self): @print_test def test_pre_download_parse(self): - """before we download an article you should not be parsing!""" - + """Before we download an article you should not be parsing! + """ article = Article(self.article.url) + def failfunc(): article.parse() self.assertRaises(ArticleException, failfunc) @@ -104,12 +114,13 @@ def failfunc(): @print_test @responses.activate def test_parse_html(self): - TOP_IMG = 'http://i2.cdn.turner.com/cnn/dam/assets/131129200805-01-weather-1128-story-top.jpg' + TOP_IMG = ('http://i2.cdn.turner.com/cnn/dam/assets/131129200805-' + '01-weather-1128-story-top.jpg') DOMAIN = 'www.cnn.com' SCHEME = 'http' AUTHORS = ['Dana Ford', 'Tom Watkins'] TITLE = 'After storm, forecasters see smooth sailing for Thanksgiving' - LEN_IMGS = 46 # list is too big, we just check size of images arr + LEN_IMGS = 46 META_LANG = 'en' mock_response_with(self.article.url, 'cnn_article') @@ -128,17 +139,17 @@ def test_meta_type_extraction(self): mock_response_with(self.article.url, 'cnn_article') self.article.build() - meta_type = self.article.extractor.get_meta_type(self.article) + meta_type = self.article.extractor.get_meta_type( + self.article.clean_doc) assert 'article' == meta_type - @print_test @responses.activate def test_meta_extraction(self): mock_response_with(self.article.url, 'cnn_article') self.article.build() - meta = self.article.extractor.get_meta_data(self.article) + meta = self.article.extractor.get_meta_data(self.article.clean_doc) META_DATA = defaultdict(dict, { 'medium': 'news', 'googlebot': 'noarchive', @@ -180,6 +191,7 @@ def test_pre_download_nlp(self): """Test running NLP algos before even downloading the article""" mock_response_with(self.article.url, 'cnn_article') + def failfunc(): self.article.nlp() self.assertRaises(ArticleException, failfunc) @@ -190,6 +202,7 @@ def test_pre_parse_nlp(self): article = Article(self.article.url) article.download() + def failfunc(): article.nlp() self.assertRaises(ArticleException, failfunc) @@ -199,8 +212,10 @@ def failfunc(): def test_nlp_body(self): SUMMARY = """Wish the forecasters were wrong all the time :)"Though the worst of the storm has passed, winds could still pose a problem.\r\nForecasters see mostly smooth sailing into Thanksgiving.\r\nThe forecast has left up in the air the fate of the balloons in Macy's Thanksgiving Day Parade.\r\nThe storm caused some complications and inconveniences, but no major delays or breakdowns.\r\n"That's good news for people like Latasha Abney, who joined the more than 43 million Americans expected by AAA to travel over the Thanksgiving holiday weekend.""" - KEYWORDS = [u'great', u'good', u'flight', u'sailing', u'delays', u'smooth', u'thanksgiving', - u'snow', u'weather', u'york', u'storm', u'winds', u'balloons', u'forecasters'] + KEYWORDS = [ + u'great', u'good', u'flight', u'sailing', u'delays', + u'smooth', u'thanksgiving', u'snow', u'weather', u'york', + u'storm', u'winds', u'balloons', u'forecasters'] mock_response_with(self.article.url, 'cnn_article') self.article.build() @@ -210,6 +225,7 @@ def test_nlp_body(self): assert self.article.summary == SUMMARY assert self.article.keywords == KEYWORDS + class SourceTestCase(unittest.TestCase): def runTest(self): print 'testing source unit' @@ -230,13 +246,34 @@ def test_source_build(self): builds a source object, validates it has no errors, prints out all valid categories and feed urls """ - DESC = """CNN.com International delivers breaking news from across the globe and information on the latest top stories, business, sports and entertainment headlines. Follow the news as it happens through: special reports, videos, audio, photo galleries plus interactive maps and timelines.""" - CATEGORY_URLS = [u'http://cnn.com/ASIA', u'http://connecttheworld.blogs.cnn.com', u'http://cnn.com/HLN', u'http://cnn.com/MIDDLEEAST', u'http://cnn.com', u'http://ireport.cnn.com', u'http://cnn.com/video', u'http://transcripts.cnn.com', u'http://cnn.com/espanol', u'http://partners.cnn.com', u'http://www.cnn.com', u'http://cnn.com/US', u'http://cnn.com/EUROPE', u'http://cnn.com/TRAVEL', u'http://cnn.com/cnni', u'http://cnn.com/SPORT', u'http://cnn.com/mostpopular', u'http://arabic.cnn.com', u'http://cnn.com/WORLD', u'http://cnn.com/LATINAMERICA', u'http://us.cnn.com', u'http://travel.cnn.com', u'http://mexico.cnn.com', u'http://cnn.com/SHOWBIZ', u'http://edition.cnn.com', u'http://amanpour.blogs.cnn.com', u'http://money.cnn.com', u'http://cnn.com/tools/index.html', u'http://cnnespanol.cnn.com', u'http://cnn.com/CNNI', u'http://business.blogs.cnn.com', u'http://cnn.com/AFRICA', u'http://cnn.com/TECH', u'http://cnn.com/BUSINESS'] + DESC = ('CNN.com International delivers breaking news from across ' + 'the globe and information on the latest top stories, ' + 'business, sports and entertainment headlines. Follow the ' + 'news as it happens through: special reports, videos, ' + 'audio, photo galleries plus interactive maps and timelines.') + CATEGORY_URLS = [ + u'http://cnn.com/ASIA', u'http://connecttheworld.blogs.cnn.com', + u'http://cnn.com/HLN', u'http://cnn.com/MIDDLEEAST', + u'http://cnn.com', u'http://ireport.cnn.com', + u'http://cnn.com/video', u'http://transcripts.cnn.com', + u'http://cnn.com/espanol', + u'http://partners.cnn.com', u'http://www.cnn.com', + u'http://cnn.com/US', u'http://cnn.com/EUROPE', + u'http://cnn.com/TRAVEL', u'http://cnn.com/cnni', + u'http://cnn.com/SPORT', u'http://cnn.com/mostpopular', + u'http://arabic.cnn.com', u'http://cnn.com/WORLD', + u'http://cnn.com/LATINAMERICA', u'http://us.cnn.com', + u'http://travel.cnn.com', u'http://mexico.cnn.com', + u'http://cnn.com/SHOWBIZ', u'http://edition.cnn.com', + u'http://amanpour.blogs.cnn.com', u'http://money.cnn.com', + u'http://cnn.com/tools/index.html', u'http://cnnespanol.cnn.com', + u'http://cnn.com/CNNI', u'http://business.blogs.cnn.com', + u'http://cnn.com/AFRICA', u'http://cnn.com/TECH', + u'http://cnn.com/BUSINESS'] + FEEDS = [u'http://rss.cnn.com/rss/edition.rss'] BRAND = 'cnn' - config = Configuration() - config.verbose = False - s = Source('http://cnn.com', config=config) + s = Source('http://cnn.com', verbose=False, memoize_articles=False) url_re = re.compile(".*cnn\.com") mock_response_with(url_re, 'cnn_main_site') s.clean_memo_cache() @@ -246,6 +283,10 @@ def test_source_build(self): assert s.description == DESC assert s.size() == 241 assert s.category_urls() == CATEGORY_URLS + # TODO: A lot of the feed extraction is NOT being tested because feeds + # are primarly extracted from the HTML of category URLs. We lose this + # effect by just mocking CNN's main page HTML. Warning: tedious fix. + assert s.feed_urls() == FEEDS @print_test @responses.activate @@ -261,10 +302,11 @@ def test_cache_categories(self): s.set_categories() saved_urls = s.category_urls() - s.categories = [] # reset and try again with caching + s.categories = [] s.set_categories() assert sorted(s.category_urls()) == sorted(saved_urls) + class UrlTestCase(unittest.TestCase): def runTest(self): print 'testing url unit' @@ -296,12 +338,12 @@ def test_valid_urls(self): @print_test def test_prepare_url(self): - """ - normalizes a url, removes arguments, hashtags. If a relative url, it + """Normalizes a url, removes arguments, hashtags. If a relative url, it merges it with the source domain to make an abs url, etc """ pass + class APITestCase(unittest.TestCase): def runTest(self): print 'testing API unit' @@ -312,14 +354,17 @@ def runTest(self): @print_test def test_source_build(self): - huff_paper = newspaper.build('http://www.huffingtonpost.com/', dry=True) - assert isinstance(huff_paper, Source) == True + huff_paper = newspaper.build( + 'http://www.huffingtonpost.com/', dry=True) + assert isinstance(huff_paper, Source) is True @print_test def test_article_build(self): - url = 'http://abcnews.go.com/blogs/politics/2013/12/states-cite-surge-in-obamacare-sign-ups-ahead-of-first-deadline/' + url = ('http://abcnews.go.com/blogs/politics/2013/12/' + 'states-cite-surge-in-obamacare-sign-ups-ahead' + '-of-first-deadline/') article = newspaper.build_article(url) - assert isinstance(article, Article) == True + assert isinstance(article, Article) is True article.build() article.nlp() @@ -337,6 +382,7 @@ def test_popular_urls(self): """ newspaper.popular_urls() + class EncodingTestCase(unittest.TestCase): def runTest(self): self.test_encode_val() @@ -369,16 +415,14 @@ def runTest(self): @print_test def test_download_works(self): - """ - """ config = Configuration() config.memoize_articles = False slate_paper = newspaper.build('http://slate.com', config=config) tc_paper = newspaper.build('http://techcrunch.com', config=config) espn_paper = newspaper.build('http://espn.com', config=config) - print 'slate has %d articles tc has %d articles espn has %d articles' \ - % (slate_paper.size(), tc_paper.size(), espn_paper.size()) + print ('slate has %d articles tc has %d articles espn has %d articles' + % (slate_paper.size(), tc_paper.size(), espn_paper.size())) papers = [slate_paper, tc_paper, espn_paper] news_pool.set(papers, threads_per_source=2) @@ -396,51 +440,53 @@ def runTest(self): @print_test def test_config_build(self): + """Test if our **kwargs to config building setup actually works. """ - Test if our **kwargs to config building setup actually works. - """ - a = Article(url='http://www.cnn.com/2013/11/27/travel/weather-thanksgiving/index.html') + a = Article(url='http://www.cnn.com/2013/11/27/' + 'travel/weather-thanksgiving/index.html') assert a.config.language == 'en' - assert a.config.memoize_articles == True - assert a.config.use_meta_language == True + assert a.config.memoize_articles is True + assert a.config.use_meta_language is True - a = Article(url='http://www.cnn.com/2013/11/27/travel/weather-thanksgiving/index.html', - language='zh', memoize_articles=False) + a = Article(url='http://www.cnn.com/2013/11/27/travel/' + 'weather-thanksgiving/index.html', + language='zh', memoize_articles=False) assert a.config.language == 'zh' - assert a.config.memoize_articles == False - assert a.config.use_meta_language == False + assert a.config.memoize_articles is False + assert a.config.use_meta_language is False s = Source(url='http://cnn.com') assert s.config.language == 'en' assert s.config.MAX_FILE_MEMO == 20000 - assert s.config.memoize_articles == True - assert s.config.use_meta_language == True + assert s.config.memoize_articles is True + assert s.config.use_meta_language is True s = Source(url="http://cnn.com", memoize_articles=False, - MAX_FILE_MEMO=10000, language='en') - assert s.config.memoize_articles == False + MAX_FILE_MEMO=10000, language='en') + assert s.config.memoize_articles is False assert s.config.MAX_FILE_MEMO == 10000 assert s.config.language == 'en' - assert s.config.use_meta_language == False + assert s.config.use_meta_language is False s = newspaper.build('http://cnn.com', dry=True) assert s.config.language == 'en' assert s.config.MAX_FILE_MEMO == 20000 - assert s.config.memoize_articles == True - assert s.config.use_meta_language == True + assert s.config.memoize_articles is True + assert s.config.use_meta_language is True s = newspaper.build('http://cnn.com', dry=True, memoize_articles=False, - MAX_FILE_MEMO=10000, language='zh') + MAX_FILE_MEMO=10000, language='zh') assert s.config.language == 'zh' assert s.config.MAX_FILE_MEMO == 10000 - assert s.config.memoize_articles == False - assert s.config.use_meta_language == False + assert s.config.memoize_articles is False + assert s.config.use_meta_language is False + class MultiLanguageTestCase(unittest.TestCase): def runTest(self): self.test_chinese_fulltext_extract() self.test_arabic_fulltext_extract() - #self.test_spanish_fulltext_extract() + # self.test_spanish_fulltext_extract() @print_test def test_chinese_fulltext_extract(self):