do not pass article instance around everywhere take 1

Author: grangier

goose/cleaners.py

@@ -25,10 +25,17 @@
 
 class DocumentCleaner(object):
 
-    def __init__(self, config):
+    def __init__(self, config, article):
+        # config
         self.config = config
+
         # parser
         self.parser = self.config.get_parser()
+
+        # article
+        self.article = article
+
+        # nodes to remove regexp
         self.remove_nodes_re = (
         "^side$|combx|retweet|mediaarticlerelated|menucontainer|navbar"
         "|comment|PopularQuestions|contact|foot|footer|Footer|footnote"
@@ -57,9 +64,9 @@ def __init__(self, config):
                                             .append("\t")\
                                             .append("^\\s+$")
 
-    def clean(self, article):
+    def clean(self):
 
-        doc_to_clean = article.doc
+        doc_to_clean = self.article.doc
         doc_to_clean = self.clean_article_tags(doc_to_clean)
         doc_to_clean = self.clean_em_tags(doc_to_clean)
         doc_to_clean = self.remove_drop_caps(doc_to_clean)

goose/crawler.py

@@ -46,100 +46,136 @@ def __init__(self, config, url, raw_html):
 class Crawler(object):
 
     def __init__(self, config):
+        # config
         self.config = config
         # parser
         self.parser = self.config.get_parser()
+
+        # article
+        self.article = Article()
+
+        # init the extractor
+        self.extractor = self.get_extractor()
+
+        # init the document cleaner
+        self.cleaner = self.get_cleaner()
+
+        # init the output formatter
+        self.formatter = self.get_formatter()
+
+        # video extractor
+        self.video_extractor = self.get_video_extractor()
+
+        # image extrator
+        self.image_extractor = self.get_image_extractor()
+
+        # TODO : log prefix
         self.logPrefix = "crawler:"
 
     def crawl(self, crawl_candidate):
-        article = Article()
 
+        # parser candidate
         parse_candidate = self.get_parse_candidate(crawl_candidate)
+
+        # raw html
         raw_html = self.get_html(crawl_candidate, parse_candidate)
 
         if raw_html is None:
-            return article
+            return self.article
 
+        # create document
         doc = self.get_document(raw_html)
 
-        extractor = self.get_extractor()
-        document_cleaner = self.get_document_cleaner()
-        output_formatter = self.get_output_formatter()
-
         # article
-        article.final_url = parse_candidate.url
-        article.link_hash = parse_candidate.link_hash
-        article.raw_html = raw_html
-        article.doc = doc
-        article.raw_doc = deepcopy(doc)
-        article.title = extractor.get_title(article)
+        self.article.final_url = parse_candidate.url
+        self.article.link_hash = parse_candidate.link_hash
+        self.article.raw_html = raw_html
+        self.article.doc = doc
+        self.article.raw_doc = deepcopy(doc)
         # TODO
-        # article.publish_date = config.publishDateExtractor.extract(doc)
-        # article.additional_data = config.get_additionaldata_extractor.extract(doc)
-        article.meta_lang = extractor.get_meta_lang(article)
-        article.meta_favicon = extractor.get_favicon(article)
-        article.meta_description = extractor.get_meta_description(article)
-        article.meta_keywords = extractor.get_meta_keywords(article)
-        article.canonical_link = extractor.get_canonical_link(article)
-        article.domain = extractor.get_domain(article.final_url)
-        article.tags = extractor.extract_tags(article)
-        # # before we do any calcs on the body itself let's clean up the document
-        article.doc = document_cleaner.clean(article)
+        # self.article.publish_date = config.publishDateExtractor.extract(doc)
+        # self.article.additional_data = config.get_additionaldata_extractor.extract(doc)
+        self.article.title = self.extractor.get_title()
+        self.article.meta_lang = self.extractor.get_meta_lang()
+        self.article.meta_favicon = self.extractor.get_favicon()
+        self.article.meta_description = self.extractor.get_meta_description()
+        self.article.meta_keywords = self.extractor.get_meta_keywords()
+        self.article.canonical_link = self.extractor.get_canonical_link()
+        self.article.domain = self.extractor.get_domain()
+        self.article.tags = self.extractor.extract_tags()
+
+        # before we do any calcs on the body itself let's clean up the document
+        self.article.doc = self.cleaner.clean()
 
         # big stuff
-        article.top_node = extractor.calculate_best_node(article)
-        if article.top_node is not None:
+        self.article.top_node = self.extractor.calculate_best_node()
+
+        # if we have a top node
+        # let's process it
+        if self.article.top_node is not None:
+
             # video handeling
-            video_extractor = self.get_video_extractor(article)
-            video_extractor.get_videos()
+            self.video_extractor.get_videos()
+
             # image handeling
             if self.config.enable_image_fetching:
-                image_extractor = self.get_image_extractor(article)
-                article.top_image = image_extractor.get_best_image(article.raw_doc, article.top_node)
+                self.get_image()
+
             # post cleanup
-            article.top_node = extractor.post_cleanup(article.top_node)
+            self.article.top_node = self.extractor.post_cleanup()
+
             # clean_text
-            article.cleaned_text = output_formatter.get_formatted_text(article)
+            self.article.cleaned_text = self.formatter.get_formatted_text()
 
         # cleanup tmp file
-        self.relase_resources(article)
+        self.relase_resources()
 
-        return article
+        # return the article
+        return self.article
 
     def get_parse_candidate(self, crawl_candidate):
         if crawl_candidate.raw_html:
             return RawHelper.get_parsing_candidate(crawl_candidate.url, crawl_candidate.raw_html)
         return URLHelper.get_parsing_candidate(crawl_candidate.url)
 
+    def get_image(self):
+        doc = self.article.raw_doc
+        top_node = self.article.top_node
+        self.article.top_image = self.image_extractor.get_best_image(doc, top_node)
+
     def get_html(self, crawl_candidate, parsing_candidate):
+        # we got a raw_tml
+        # no need to fetch remote content
         if crawl_candidate.raw_html:
             return crawl_candidate.raw_html
+
         # fetch HTML
-        html = HtmlFetcher().get_html(self.config, parsing_candidate.url)
+        fetcher = HtmlFetcher(self.config, parsing_candidate.url)
+        html = fetcher.get_html()
+        #html = HtmlFetcher().get_html(self.config, parsing_candidate.url)
         return html
 
-    def get_image_extractor(self, article):
-        http_client = None
-        return UpgradedImageIExtractor(http_client, article, self.config)
+    def get_image_extractor(self):
+        return UpgradedImageIExtractor(self.config, self.article)
 
-    def get_video_extractor(self, article):
-        return VideoExtractor(article, self.config)
+    def get_video_extractor(self):
+        return VideoExtractor(self.config, self.article)
 
-    def get_output_formatter(self):
-        return StandardOutputFormatter(self.config)
+    def get_formatter(self):
+        return StandardOutputFormatter(self.config, self.article)
 
-    def get_document_cleaner(self):
-        return StandardDocumentCleaner(self.config)
+    def get_cleaner(self):
+        return StandardDocumentCleaner(self.config, self.article)
 
     def get_document(self, raw_html):
         doc = self.parser.fromstring(raw_html)
         return doc
 
     def get_extractor(self):
-        return StandardContentExtractor(self.config)
+        return StandardContentExtractor(self.config, self.article)
 
-    def relase_resources(self, article):
-        path = os.path.join(self.config.local_storage_path, '%s_*' % article.link_hash)
+    def relase_resources(self):
+        path = os.path.join(self.config.local_storage_path, '%s_*' % self.article.link_hash)
         for fname in glob.glob(path):
             try:
                 os.remove(fname)

goose/extractors.py

@@ -43,32 +43,29 @@
 
 class ContentExtractor(object):
 
-    def __init__(self, config):
+    def __init__(self, config, article):
+        # config
         self.config = config
+
         # parser
         self.parser = self.config.get_parser()
+
+        # article
+        self.article = article
+
+        # language
         self.language = config.target_language
+
+        # stopwords class
         self.stopwords_class = config.stopwords_class
 
-    def get_language(self, article):
-        """\
-        Returns the language is by the article or
-        the configuration language
-        """
-        # we don't want to force the target laguage
-        # so we use the article.meta_lang
-        if self.config.use_meta_language == True:
-            if article.meta_lang:
-                self.language = article.meta_lang[:2]
-        self.language = self.config.target_language
-
-    def get_title(self, article):
+    def get_title(self):
         """\
         Fetch the article title and analyze it
         """
 
         title = ''
-        doc = article.doc
+        doc = self.article.doc
 
         title_element = self.parser.getElementsByTag(doc, tag='title')
         # no title found
@@ -121,34 +118,34 @@ def split_title(self, title, splitter):
         title = title_pieces[large_text_index]
         return TITLE_REPLACEMENTS.replaceAll(title).strip()
 
-    def get_favicon(self, article):
+    def get_favicon(self):
         """\
         Extract the favicon from a website
         http://en.wikipedia.org/wiki/Favicon
         <link rel="shortcut icon" type="image/png" href="favicon.png" />
         <link rel="icon" type="image/png" href="favicon.png" />
         """
         kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'icon'}
-        meta = self.parser.getElementsByTag(article.doc, **kwargs)
+        meta = self.parser.getElementsByTag(self.article.doc, **kwargs)
         if meta:
             favicon = self.parser.getAttribute(meta[0], 'href')
             return favicon
         return ''
 
-    def get_meta_lang(self, article):
+    def get_meta_lang(self):
         """\
         Extract content language from meta
         """
         # we have a lang attribute in html
-        attr = self.parser.getAttribute(article.doc, attr='lang')
+        attr = self.parser.getAttribute(self.article.doc, attr='lang')
         if attr is None:
             # look up for a Content-Language in meta
             items = [
                 {'tag': 'meta', 'attr': 'http-equiv', 'value': 'content-language'},
                 {'tag': 'meta', 'attr': 'name', 'value': 'lang'}
             ]
             for item in items:
-                meta = self.parser.getElementsByTag(article.doc, **item)
+                meta = self.parser.getElementsByTag(self.article.doc, **item)
                 if meta:
                     attr = self.parser.getAttribute(meta[0], attr='content')
                     break
@@ -175,45 +172,45 @@ def get_meta_content(self, doc, metaName):
 
         return ''
 
-    def get_meta_description(self, article):
+    def get_meta_description(self):
         """\
         if the article has meta description set in the source, use that
         """
-        return self.get_meta_content(article.doc, "meta[name=description]")
+        return self.get_meta_content(self.article.doc, "meta[name=description]")
 
-    def get_meta_keywords(self, article):
+    def get_meta_keywords(self):
         """\
         if the article has meta keywords set in the source, use that
         """
-        return self.get_meta_content(article.doc, "meta[name=keywords]")
+        return self.get_meta_content(self.article.doc, "meta[name=keywords]")
 
-    def get_canonical_link(self, article):
+    def get_canonical_link(self):
         """\
         if the article has meta canonical link set in the url
         """
-        if article.final_url:
+        if self.article.final_url:
             kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'canonical'}
-            meta = self.parser.getElementsByTag(article.doc, **kwargs)
+            meta = self.parser.getElementsByTag(self.article.doc, **kwargs)
             if meta is not None and len(meta) > 0:
                 href = self.parser.getAttribute(meta[0], 'href')
                 if href:
                     href = href.strip()
                     o = urlparse(href)
                     if not o.hostname:
-                        z = urlparse(article.final_url)
+                        z = urlparse(self.article.final_url)
                         domain = '%s://%s' % (z.scheme, z.hostname)
                         href = urljoin(domain, href)
                     return href
-        return article.final_url
+        return self.article.final_url
 
-    def get_domain(self, url):
-        if url:
-            o = urlparse(url)
+    def get_domain(self):
+        if self.article.final_url:
+            o = urlparse(self.article.final_url)
             return o.hostname
         return None
 
-    def extract_tags(self, article):
-        node = article.doc
+    def extract_tags(self):
+        node = self.article.doc
 
         # node doesn't have chidren
         if len(list(node)) == 0:
@@ -233,8 +230,8 @@ def extract_tags(self, article):
 
         return set(tags)
 
-    def calculate_best_node(self, article):
-        doc = article.doc
+    def calculate_best_node(self):
+        doc = self.article.doc
         top_node = None
         nodes_to_check = self.nodes_to_check(doc)
 
@@ -507,11 +504,12 @@ def is_nodescore_threshold_met(self, node, e):
             return False
         return True
 
-    def post_cleanup(self, targetNode):
+    def post_cleanup(self):
         """\
         remove any divs that looks like non-content,
         clusters of links, or paras with no gusto
         """
+        targetNode = self.article.top_node
         node = self.add_siblings(targetNode)
         for e in self.parser.getChildren(node):
             e_tag = self.parser.getTag(e)

goose/images/extractors.py

@@ -49,7 +49,7 @@ class ImageExtractor(object):
 
 class UpgradedImageIExtractor(ImageExtractor):
 
-    def __init__(self, http_client, article, config):
+    def __init__(self, config, article):
         self.custom_site_mapping = {}
         self.load_customesite_mapping()