From 8e6d8aad5eab217a6edac535dfda16a1e50ffe53 Mon Sep 17 00:00:00 2001 From: Kevin Greenan Date: Fri, 18 Sep 2020 14:57:47 -0700 Subject: [PATCH 1/8] Issues 645 and 776 prevent text from being extracted from many sites. This updates the function that scores the subtrees to try to account for all content. --- .gitignore | 1 + newspaper/extractors.py | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 3d20afb1..0b9c4f9f 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ nosetests.xml .project .pydevproject venv +Pipfile* diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 96255401..c8dd522f 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -820,13 +820,23 @@ def calculate_best_node(self, doc): if parent_node not in parent_nodes: parent_nodes.append(parent_node) - # Parent of parent node - parent_parent_node = self.parser.getParent(parent_node) - if parent_parent_node is not None: - self.update_node_count(parent_parent_node, 1) - self.update_score(parent_parent_node, upscore / 2) - if parent_parent_node not in parent_nodes: - parent_nodes.append(parent_parent_node) + # Issue 645, 776 - Some text was left out due to "deep" nodes + # containing content not being accounted for. This should result + # in the "heaviest" content subtree as the "top node." + # + # Percolate weights up through the tree. Not really sure + # what the upscore is, but seems that it should decay, so + # decaying linearly. + ancestor_node = self.parser.getParent(parent_node) + j = 2 + while ancestor_node is not None: + self.update_node_count(ancestor_node, 1) + self.update_score(ancestor_node, upscore / (float(j))) + if ancestor_node not in parent_nodes: + parent_nodes.append(ancestor_node) + ancestor_node = self.parser.getParent(ancestor_node) + j += 1 + cnt += 1 i += 1 From d01f6b5bfb007f6e99a9d2fcb6b6a58c14c0f183 Mon Sep 17 00:00:00 2001 From: Kevin Greenan Date: Fri, 18 Sep 2020 15:25:11 -0700 Subject: [PATCH 2/8] Remove node count from percolating up, because it could lead to the root of the page always being selected. --- newspaper/extractors.py | 1 - 1 file changed, 1 deletion(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index c8dd522f..ed5964cc 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -830,7 +830,6 @@ def calculate_best_node(self, doc): ancestor_node = self.parser.getParent(parent_node) j = 2 while ancestor_node is not None: - self.update_node_count(ancestor_node, 1) self.update_score(ancestor_node, upscore / (float(j))) if ancestor_node not in parent_nodes: parent_nodes.append(ancestor_node) From 67c74c3e97d26fed14031abccbe81c0885782fb7 Mon Sep 17 00:00:00 2001 From: Kevin Greenan Date: Sat, 19 Feb 2022 07:57:21 -0800 Subject: [PATCH 3/8] Print exception reason when HTML parsing fails --- newspaper/parsers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/newspaper/parsers.py b/newspaper/parsers.py index a4d030c0..a9ac973f 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -67,8 +67,8 @@ def fromstring(cls, html): html = re.sub(r'^\<\?.*?\?\>', '', html, flags=re.DOTALL) cls.doc = lxml.html.fromstring(html) return cls.doc - except Exception: - log.warn('fromstring() returned an invalid string: %s...', html[:20]) + except Exception as e: + log.warn('fromstring() returned an invalid string (%s): %s...', str(e), html[:20]) return @classmethod From b9e188390e84422c41a2157f32c89aea8cbdb509 Mon Sep 17 00:00:00 2001 From: Kevin Greenan Date: Sun, 20 Feb 2022 07:27:40 -0800 Subject: [PATCH 4/8] Adding more logging --- newspaper/parsers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newspaper/parsers.py b/newspaper/parsers.py index a9ac973f..23c09fce 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -68,7 +68,7 @@ def fromstring(cls, html): cls.doc = lxml.html.fromstring(html) return cls.doc except Exception as e: - log.warn('fromstring() returned an invalid string (%s): %s...', str(e), html[:20]) + log.warn('fromstring() returned an invalid string (%s): %s...', str(e), html[:256]) return @classmethod From 4c86bc6be644fc6bf3e277f4cb280f4454bff82d Mon Sep 17 00:00:00 2001 From: Kevin Greenan Date: Sun, 20 Feb 2022 15:25:56 -0800 Subject: [PATCH 5/8] Use soupparser and reduce output string length of failed parse --- newspaper/parsers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/newspaper/parsers.py b/newspaper/parsers.py index 23c09fce..9af3686e 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -9,6 +9,7 @@ import logging import lxml.etree import lxml.html +import lxml.html.soupparser import lxml.html.clean import re from html import unescape @@ -65,10 +66,10 @@ def fromstring(cls, html): # lxml does not play well with encoding tags if html.startswith('', '', html, flags=re.DOTALL) - cls.doc = lxml.html.fromstring(html) + cls.doc = lxml.html.soupparser.fromstring(html) return cls.doc except Exception as e: - log.warn('fromstring() returned an invalid string (%s): %s...', str(e), html[:256]) + log.warn('fromstring() returned an invalid string (%s): %s...', str(e), html[:20]) return @classmethod From 0a7a436a2fb811857e7b7f17d79e4f1389069819 Mon Sep 17 00:00:00 2001 From: Kevin Greenan Date: Mon, 21 Feb 2022 09:07:00 -0800 Subject: [PATCH 6/8] Print traceback on parser failures --- newspaper/parsers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/newspaper/parsers.py b/newspaper/parsers.py index 9af3686e..9ad38a8e 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -7,6 +7,8 @@ or query an lxml or soup dom object generated from an article's html. """ import logging +import traceback + import lxml.etree import lxml.html import lxml.html.soupparser @@ -69,7 +71,9 @@ def fromstring(cls, html): cls.doc = lxml.html.soupparser.fromstring(html) return cls.doc except Exception as e: + tb = traceback.format_exc() log.warn('fromstring() returned an invalid string (%s): %s...', str(e), html[:20]) + log.warn(tb) return @classmethod From 5d9fd1d696ed632154462586ec40b0184f1b45a3 Mon Sep 17 00:00:00 2001 From: Kevin Greenan Date: Mon, 21 Feb 2022 09:07:00 -0800 Subject: [PATCH 7/8] Print traceback on parser failures --- newspaper/parsers.py | 2 +- requirements.txt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/newspaper/parsers.py b/newspaper/parsers.py index 9ad38a8e..56ec707c 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -68,7 +68,7 @@ def fromstring(cls, html): # lxml does not play well with encoding tags if html.startswith('', '', html, flags=re.DOTALL) - cls.doc = lxml.html.soupparser.fromstring(html) + cls.doc = lxml.html.soupparser.fromstring(html, None, None, features='html5lib') return cls.doc except Exception as e: tb = traceback.format_exc() diff --git a/requirements.txt b/requirements.txt index 61974601..b225d10b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,5 @@ python-dateutil>=2.5.3 PyYAML>=3.11 requests>=2.10.0 tinysegmenter==0.3 # TODO(codelucas): Investigate making this >=0.3 -tldextract>=2.0.1 \ No newline at end of file +tldextract>=2.0.1 +html5lib>=1.1 From 1167d8581eef8e6d9b8beff4025c4b32d39a6cee Mon Sep 17 00:00:00 2001 From: Kevin Greenan Date: Sun, 13 Mar 2022 12:13:54 -0700 Subject: [PATCH 8/8] Use BS4 as a fallback --- newspaper/parsers.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/newspaper/parsers.py b/newspaper/parsers.py index 56ec707c..1c554a8b 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -68,7 +68,12 @@ def fromstring(cls, html): # lxml does not play well with encoding tags if html.startswith('', '', html, flags=re.DOTALL) - cls.doc = lxml.html.soupparser.fromstring(html, None, None, features='html5lib') + root = lxml.html.fromstring(html) + try: + ignore = lxml.html.tostring(root, encoding='unicode') + cls.doc = root + except UnicodeDecodeError: + cls.doc = lxml.html.soupparser.fromstring(html, None, None, features='html5lib') return cls.doc except Exception as e: tb = traceback.format_exc()