From 81c2075b442984aea767dfe20431732d150cb8f0 Mon Sep 17 00:00:00 2001 From: Michael Hall Date: Sat, 20 Dec 2014 05:04:12 -0700 Subject: [PATCH] Fixed #78: Remove encoding tag because lxml won't accept it for unicode objects lxml apparently doesn't accept unicode objects with both an encoding specified and an encoding tag in the HTML document, because they're paranoid that the two won't match. Stripped encoding tag before calling lxml.html.fromstring() --- newspaper/parsers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/newspaper/parsers.py b/newspaper/parsers.py index 222c45b2..3debcc0f 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -10,6 +10,7 @@ import lxml.etree import lxml.html import lxml.html.clean +import re from copy import deepcopy @@ -43,6 +44,10 @@ def css_select(cls, node, selector): def fromstring(cls, html): html = utils.encodeValue(html) try: + # Remove encoding tag because lxml won't accept it for unicode objects (Issue #78) + if html.startswith('', '', html, flags=re.DOTALL) + cls.doc = lxml.html.fromstring(html) except Exception, e: print '[Parse lxml ERR]', str(e)