From c96c9f1453480d0b71761cd75486bcaaace98886 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Wed, 13 May 2015 21:48:15 +0300 Subject: [PATCH 1/2] Handle lxml raising ValueError on node.itertext() The error: ValueError: Input object has no element: HtmlProcessingInstruction Ref #143 --- newspaper/outputformatters.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/newspaper/outputformatters.py b/newspaper/outputformatters.py index 56bea437..3711bf2f 100644 --- a/newspaper/outputformatters.py +++ b/newspaper/outputformatters.py @@ -60,7 +60,11 @@ def get_formatted(self, top_node): def convert_to_text(self): txts = [] for node in list(self.get_top_node()): - txt = self.parser.getText(node) + try: + txt = self.parser.getText(node) + except ValueError: # lxml error + txt = None + if txt: txt = HTMLParser().unescape(txt) txt_lis = innerTrim(txt).split(r'\n') From b7ee7ed1f445b971496d73df3a56229d46f09928 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Mon, 11 Jan 2016 12:57:06 +0200 Subject: [PATCH 2/2] Add logging on failure to parse an lxml node --- newspaper/outputformatters.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/newspaper/outputformatters.py b/newspaper/outputformatters.py index 3711bf2f..dc9d7610 100644 --- a/newspaper/outputformatters.py +++ b/newspaper/outputformatters.py @@ -8,10 +8,14 @@ __copyright__ = 'Copyright 2014, Lucas Ou-Yang' from HTMLParser import HTMLParser +import logging from .text import innerTrim +log = logging.getLogger(__name__) + + class OutputFormatter(object): def __init__(self, config): @@ -63,6 +67,7 @@ def convert_to_text(self): try: txt = self.parser.getText(node) except ValueError: # lxml error + log.warning('Error parsing lxml node', exc_info=True) txt = None if txt: