diff --git a/newspaper/outputformatters.py b/newspaper/outputformatters.py index 56bea437..dc9d7610 100644 --- a/newspaper/outputformatters.py +++ b/newspaper/outputformatters.py @@ -8,10 +8,14 @@ __copyright__ = 'Copyright 2014, Lucas Ou-Yang' from HTMLParser import HTMLParser +import logging from .text import innerTrim +log = logging.getLogger(__name__) + + class OutputFormatter(object): def __init__(self, config): @@ -60,7 +64,12 @@ def get_formatted(self, top_node): def convert_to_text(self): txts = [] for node in list(self.get_top_node()): - txt = self.parser.getText(node) + try: + txt = self.parser.getText(node) + except ValueError: # lxml error + log.warning('Error parsing lxml node', exc_info=True) + txt = None + if txt: txt = HTMLParser().unescape(txt) txt_lis = innerTrim(txt).split(r'\n')