Fulltext extraction improvement #1

.gitignore

@@ -40,8 +40,3 @@ nosetests.xml
 .project
 .pydevproject
 venv
-
-# Utility files for generating resources for unit testing
-generate_fulltext.py
-generate_html.py
-generate_urls.py

README.rst

@@ -15,66 +15,19 @@ Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed:
 .. _`tweeted by`: https://twitter.com/kennethreitz/status/419520678862548992
 .. _`The Changelog`: http://thechangelog.com/newspaper-delivers-instapaper-style-article-extraction/
 
-
 **Newspaper is a Python3 library**! Or, view the `Python2 branch`_
 
 .. _`Python2 branch`: https://github.com/codelucas/newspaper/tree/python-2-head
 
-**We support 10+ languages and everything is in unicode!**
-
-.. code-block:: pycon
-
-    >>> import newspaper
-    >>> newspaper.languages()
-
-    Your available languages are:
-    input code      full name
-
-      ar              Arabic
-      ru              Russian
-      nl              Dutch
-      de              German
-      en              English
-      es              Spanish
-      fr              French
-      it              Italian
-      ko              Korean
-      no              Norwegian
-      pt              Portuguese
-      sv              Swedish
-      hu              Hungarian
-      fi              Finnish
-      da              Danish
-      zh              Chinese
-      id              Indonesian
-      vi              Vietnamese
-
 A Glance:
 ---------
 
 .. code-block:: pycon
 
-    >>> import newspaper
-
-    >>> cnn_paper = newspaper.build('http://cnn.com')
-
-    >>> for article in cnn_paper.articles:
-    >>>     print(article.url)
-    'http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/'
-    'http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html'
-    ...
-
-    >>> for category in cnn_paper.category_urls():
-    >>>     print(category)
-
-    'http://lifestyle.cnn.com'
-    'http://cnn.com/world'
-    'http://tech.cnn.com'
-    ...
-
-.. code-block:: pycon
+    >>> from newspaper import Article
 
-    >>> article = cnn_paper.articles[0]
+    >>> url = 'http://fox13now.com/2013/12/30/new-year-new-laws-obamacare-pot-guns-and-drones/'
+    >>> article = Article(url)
 
 .. code-block:: pycon
 
@@ -109,6 +62,34 @@ A Glance:
     >>> article.summary
     'The study shows that 93% of people ...'
 
+.. code-block:: pycon
+
+    >>> import newspaper
+
+    >>> cnn_paper = newspaper.build('http://cnn.com')
+
+    >>> for article in cnn_paper.articles:
+    >>>     print(article.url)
+    'http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/'
+    'http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html'
+    ...
+
+    >>> for category in cnn_paper.category_urls():
+    >>>     print(category)
+
+    'http://lifestyle.cnn.com'
+    'http://cnn.com/world'
+    'http://tech.cnn.com'
+    ...
+
+.. code-block:: pycon
+
+    >>> cnn_article = cnn_paper.articles[0]
+    >>> cnn_article.download()
+    >>> cnn_article.parse()
+    >>> cnn_article.nlp()
+    ...
+
 
 Newspaper has *seamless* language extraction and detection.
 If no language is specified, Newspaper will attempt to auto detect a language.
@@ -176,7 +157,6 @@ Features
 --------
 
 - Full Python3 and Python2 support
-- Works in 10+ languages (English, Chinese, German, Arabic, ...)
 - Multi-threaded article download framework
 - News url identification
 - Text extraction from html
@@ -186,6 +166,35 @@ Features
 - Summary extraction from text
 - Author extraction from text
 - Google trending terms extraction
+- Works in 10+ languages (English, Chinese, German, Arabic, ...)
+
+.. code-block:: pycon
+
+    >>> import newspaper
+    >>> newspaper.languages()
+
+    Your available languages are:
+    input code      full name
+
+      ar              Arabic
+      ru              Russian
+      nl              Dutch
+      de              German
+      en              English
+      es              Spanish
+      fr              French
+      it              Italian
+      ko              Korean
+      no              Norwegian
+      pt              Portuguese
+      sv              Swedish
+      hu              Hungarian
+      fi              Finnish
+      da              Danish
+      zh              Chinese
+      id              Indonesian
+      vi              Vietnamese
+
 
 Get it now
 ----------
@@ -267,6 +276,10 @@ Feel free to give our testing suite a shot, everything is mocked!::
 
     $ python3 tests/unit_tests.py
 
+Planning on tweaking our full-text algorithm? Add the ``fulltext`` parameter::
+
+    $ python3 tests/unit_tests.py fulltext
+
 
 Demo
 ----

newspaper/extractors.py

@@ -866,15 +866,13 @@ def is_nodescore_threshold_met(self, node, e):
         return True
 
     def post_cleanup(self, top_node):
-        """Remove any divs that looks like non-content,
-        clusters of links, or paras with no gusto
+        """Remove any divs that looks like non-content, clusters of links,
+        or paras with no gusto; add adjacent nodes which look contenty
         """
         node = self.add_siblings(top_node)
         for e in self.parser.getChildren(node):
             e_tag = self.parser.getTag(e)
             if e_tag != 'p':
-                if self.is_highlink_density(e) \
-                        or self.is_table_and_no_para_exist(e) \
-                        or not self.is_nodescore_threshold_met(node, e):
+                if self.is_highlink_density(e):
                     self.parser.remove(e)
         return node

newspaper/outputformatters.py

@@ -48,10 +48,12 @@ def get_formatted(self, top_node):
 
         self.links_to_text()
         self.add_newline_to_br()
+        self.add_newline_to_li()
         self.replace_with_text()
-        self.remove_fewwords_paragraphs()
-
+        self.remove_empty_tags()
+        self.remove_trailing_media_div()
         text = self.convert_to_text()
+        # print(self.parser.nodeToString(self.get_top_node()))
         return (text, html)
 
     def convert_to_text(self):
@@ -61,17 +63,26 @@ def convert_to_text(self):
             if txt:
                 txt = HTMLParser().unescape(txt)
                 txt_lis = innerTrim(txt).split(r'\n')
+                txt_lis = [n.strip(' ') for n in txt_lis]
                 txts.extend(txt_lis)
         return '\n\n'.join(txts)
 
     def convert_to_html(self):
         cleaned_node = self.parser.clean_article_html(self.get_top_node())
-        return self.parser.node_to_string(cleaned_node)
+        return self.parser.nodeToString(cleaned_node)
 
     def add_newline_to_br(self):
         for e in self.parser.getElementsByTag(self.top_node, tag='br'):
             e.text = r'\n'
 
+    def add_newline_to_li(self):
+        for e in self.parser.getElementsByTag(self.top_node, tag='ul'):
+            li_list = self.parser.getElementsByTag(e, tag='li')
+            for li in li_list[:-1]:
+                li.text = self.parser.getText(li) + r'\n'
+                for c in self.parser.getChildren(li):
+                    self.parser.remove(c)
+
     def links_to_text(self):
         """Cleans up and converts any nodes that should be considered
         text into text.
@@ -100,28 +111,48 @@ def replace_with_text(self):
         self.parser.stripTags(
             self.get_top_node(), 'b', 'strong', 'i', 'br', 'sup')
 
-    def remove_fewwords_paragraphs(self):
-        """
-        Remove paragraphs that have less than x number of words,
-        would indicate that it's some sort of link.
+    def remove_empty_tags(self):
+        """It's common in top_node to exit tags that are filled with data
+        within properties but not within the tags themselves, delete them
         """
-        all_nodes = self.parser.getElementsByTags(self.get_top_node(), ['*'])
+        all_nodes = self.parser.getElementsByTags(
+            self.get_top_node(), ['*'])
         all_nodes.reverse()
         for el in all_nodes:
             tag = self.parser.getTag(el)
             text = self.parser.getText(el)
-            stop_words = self.stopwords_class(language=self.language).\
-                get_stopword_count(text)
             if (tag != 'br' or text != '\\r') \
-                    and stop_words.get_stopword_count() < 3 \
+                    and not text \
                     and len(self.parser.getElementsByTag(
                         el, tag='object')) == 0 \
                     and len(self.parser.getElementsByTag(
                         el, tag='embed')) == 0:
                 self.parser.remove(el)
-            # TODO
-            # check if it is in the right place
-            else:
-                trimmed = self.parser.getText(el)
-                if trimmed.startswith("(") and trimmed.endswith(")"):
-                    self.parser.remove(el)
+
+    def remove_trailing_media_div(self):
+        """Punish the *last top level* node in the top_node if it's
+        DOM depth is too deep. Many media non-content links are
+        eliminated: "related", "loading gallery", etc
+        """
+
+        def get_depth(node, depth=1):
+            """Computes depth of an lxml element via BFS, this would be
+            in parser if it were used anywhere else besides this method
+            """
+            children = self.parser.getChildren(node)
+            if not children:
+                return depth
+            max_depth = 0
+            for c in children:
+                e_depth = get_depth(c, depth + 1)
+                if e_depth > max_depth:
+                    max_depth = e_depth
+            return max_depth
+
+        top_level_nodes = self.parser.getChildren(self.get_top_node())
+        if len(top_level_nodes) < 3:
+            return
+
+        last_node = top_level_nodes[-1]
+        if get_depth(last_node) >= 2:
+            self.parser.remove(last_node)

newspaper/parsers.py

@@ -68,10 +68,6 @@ def fromstring(cls, html):
             traceback.print_exc()
             return
 
-    @classmethod
-    def node_to_string(cls, node):
-        return lxml.html.tostring(node)
-
     @classmethod
     def clean_article_html(cls, node):
         article_cleaner = lxml.html.clean.Cleaner()