From e408b8644e28d3edc0131112606f346d359cc0e2 Mon Sep 17 00:00:00 2001
From: Emanuele Cesena <emanuele.cesena@gmail.com>
Date: Thu, 15 Jan 2015 11:40:25 -0800
Subject: [PATCH 1/7] Fixed empty return in top_meta_image

Return empty unicode in ContentExtractor.top_meta_image() when no image is found. It was returning the article's url.

Example:
http://www.usnews.com/news/entertainment/articles/2015/01/15/boyhood-keaton-moore-are-oscar-favorites-in-vegas
(note that USNews often updates its articles, e.g. by adding an image -- at the time of writing it was without images)
---
 newspaper/extractors.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/newspaper/extractors.py b/newspaper/extractors.py
index 253742b3..6a9a417e 100644
--- a/newspaper/extractors.py
+++ b/newspaper/extractors.py
@@ -316,7 +316,9 @@ def get_meta_img_url(self, article_url, doc):
 
         top_meta_image = try_one or try_two or try_three or try_four
 
-        return urllib.parse.urljoin(article_url, top_meta_image)
+        if top_meta_image:
+            return urllib.parse.urljoin(article_url, top_meta_image)
+        return u''
 
     def get_meta_type(self, doc):
         """Returns meta type of article, open graph protocol

From 5cdc5d015d68f569bc7c0fbab3fe0cafcd699147 Mon Sep 17 00:00:00 2001
From: Emanuele Cesena <emanuele.cesena@gmail.com>
Date: Thu, 15 Jan 2015 15:35:08 -0800
Subject: [PATCH 2/7] Fix nodeToString() to return valid HTML

In some cases the output of nodeToString() is not valid HTML, e.g. for iframe nodes.

Solution found here: http://stackoverflow.com/questions/27020950/lxml-modify-tags-prevent

An example of bad behavior is an article with a youtube video inside. The video embed code is transformed into `<iframe ... />`, that is not valid, and thus the result in the browser is a page with no visible content after the video.
---
 newspaper/parsers.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/newspaper/parsers.py b/newspaper/parsers.py
index f81c7177..c40e09d6 100644
--- a/newspaper/parsers.py
+++ b/newspaper/parsers.py
@@ -85,7 +85,7 @@ def nodeToString(cls, node):
         """`decode` is needed at the end because `etree.tostring`
         returns a python bytestring
         """
-        return lxml.etree.tostring(node).decode()
+        return lxml.etree.tostring(node, method='html').decode()
 
     @classmethod
     def replaceTag(cls, node, tag):

From b4711d63798fe4d5871e43e14ec91e359af17718 Mon Sep 17 00:00:00 2001
From: Emanuele Cesena <emanuele.cesena@gmail.com>
Date: Sun, 18 Jan 2015 15:21:06 -0800
Subject: [PATCH 3/7] Allow lists in clean_article_html() - fixes #108

---
 newspaper/parsers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/newspaper/parsers.py b/newspaper/parsers.py
index f81c7177..a204e64e 100644
--- a/newspaper/parsers.py
+++ b/newspaper/parsers.py
@@ -76,7 +76,8 @@ def clean_article_html(cls, node):
         article_cleaner.allow_tags = [
             'a', 'span', 'p', 'br', 'strong', 'b',
             'em', 'i', 'tt', 'code', 'pre', 'blockquote', 'img', 'h1',
-            'h2', 'h3', 'h4', 'h5', 'h6']
+            'h2', 'h3', 'h4', 'h5', 'h6',
+            'ul', 'ol', 'li', 'dl', 'dt', 'dd']
         article_cleaner.remove_unknown_tags = False
         return article_cleaner.clean_html(node)
 

From f01fda94c6077bd13a390455cb41cde7a3497710 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang <lucasyangpersonal@gmail.com>
Date: Fri, 6 Feb 2015 03:44:41 -0800
Subject: [PATCH 4/7] [bugfix] Handle None and empty str in get_unicode_html

---
 newspaper/parsers.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/newspaper/parsers.py b/newspaper/parsers.py
index f81c7177..d7bedd55 100644
--- a/newspaper/parsers.py
+++ b/newspaper/parsers.py
@@ -45,6 +45,8 @@ def css_select(cls, node, selector):
     def get_unicode_html(cls, html):
         if isinstance(html, str):
             return html
+        if not html:
+            return html
         converted = UnicodeDammit(html, is_html=True)
         if not converted.unicode_markup:
             raise Exception(

From 9beb5aaa892edf80573bf1c2dbd48c7606cead8d Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang <lucasyangpersonal@gmail.com>
Date: Fri, 6 Feb 2015 04:25:01 -0800
Subject: [PATCH 5/7] Fix default return str in get_meta_img_url

---
 newspaper/extractors.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/newspaper/extractors.py b/newspaper/extractors.py
index bf2acdce..ec25b7d3 100644
--- a/newspaper/extractors.py
+++ b/newspaper/extractors.py
@@ -371,7 +371,7 @@ def get_meta_img_url(self, article_url, doc):
 
         if top_meta_image:
             return urllib.parse.urljoin(article_url, top_meta_image)
-        return u''
+        return ''
 
     def get_meta_type(self, doc):
         """Returns meta type of article, open graph protocol

From 11885fd72093e54e2e227b0133b75019ccf57d3a Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang <lucasyangpersonal@gmail.com>
Date: Wed, 4 Mar 2015 00:20:28 -0800
Subject: [PATCH 6/7] Update nltk 3.0.0 => 3.0.1, issue #121

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 9f7bc216..f40f012e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,7 @@ Pillow==2.6.1
 PyYAML==3.11
 cssselect==0.9.1
 lxml==3.3.5
-nltk==3.0.0
+nltk==3.0.1
 requests==2.3.0
 six==1.7.3
 feedparser==5.1.3

From 48d6667f386005be0147a840299ea9bf207c06a4 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang <lucasyangpersonal@gmail.com>
Date: Wed, 4 Mar 2015 00:23:30 -0800
Subject: [PATCH 7/7] Version bump (just for the NLTK build fix)

---
 newspaper/version.py | 2 +-
 setup.py             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/newspaper/version.py b/newspaper/version.py
index 23245d84..9778534c 100644
--- a/newspaper/version.py
+++ b/newspaper/version.py
@@ -7,5 +7,5 @@
 __license__ = 'MIT'
 __copyright__ = 'Copyright 2014, Lucas Ou-Yang'
 
-version_info = (0, 1, 4)
+version_info = (0, 1, 5)
 __version__ = ".".join(map(str, version_info))
diff --git a/setup.py b/setup.py
index 6433e830..082afbea 100755
--- a/setup.py
+++ b/setup.py
@@ -34,7 +34,7 @@
 
 setup(
     name='newspaper3k',
-    version='0.1.4',
+    version='0.1.5',
     description='Simplified python article discovery & extraction.',
     long_description=readme,
     author='Lucas Ou-Yang',