From fd4f1acae5304c7df5bc715356a2841924ca0a46 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Fri, 26 Dec 2014 23:51:50 -0800 Subject: [PATCH 1/2] Add top image extraction unit test --- tests/unit_tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 02f57275..50288066 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -119,11 +119,11 @@ def test_parse_html(self): with open(os.path.join(TEXT_FN, 'cnn.txt'), 'r') as f: assert self.article.text == f.read() - # TOP_IMG = ('http://i2.cdn.turner.com/cnn/dam/assets/131129200805-' - # '01-weather-1128-story-top.jpg') - # `top_img` isn't tested because it is extracted for this particular - # article with the "reddit method", it requires internet connection - # TODO: assert self.article.top_img == TOP_IMG + # NOTE: top_img extraction requires an internet connection + # unlike the rest of this test file + TOP_IMG = ('http://i2.cdn.turner.com/cnn/dam/assets/131129200805-' + '01-weather-1128-story-top.jpg') + assert self.article.top_img == TOP_IMG assert sorted(self.article.authors) == AUTHORS assert self.article.title == TITLE From f2d0681c3b55c8314b087129454b39e215a192d0 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Sat, 27 Dec 2014 00:04:42 -0800 Subject: [PATCH 2/2] [bugfix] decode bytestring returned from lxml's `toString` before sending it out to outer code --- newspaper/cleaners.py | 2 +- newspaper/parsers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/newspaper/cleaners.py b/newspaper/cleaners.py index d9b84096..5de87b06 100644 --- a/newspaper/cleaners.py +++ b/newspaper/cleaners.py @@ -170,7 +170,7 @@ def replace_walk_left_right(self, kid, kid_text, and self.parser.getTag(next_node) == "a" \ and self.parser.getAttribute( next_node, 'grv-usedalready') != 'yes': - outer = " " + self.parser.outerHtml(next_node).decode('utf-8') + " " + outer = " " + self.parser.outerHtml(next_node) + " " replacement_text.append(outer) nodes_to_remove.append(next_node) self.parser.setAttribute(next_node, attr='grv-usedalready', diff --git a/newspaper/parsers.py b/newspaper/parsers.py index 06d7879d..10bbd540 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -72,7 +72,7 @@ def clean_article_html(cls, node): @classmethod def nodeToString(cls, node): - return lxml.etree.tostring(node) + return lxml.etree.tostring(node).decode() @classmethod def replaceTag(cls, node, tag):