diff --git a/newspaper/extractors.py b/newspaper/extractors.py index f91229d2..ec25b7d3 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -369,7 +369,9 @@ def get_meta_img_url(self, article_url, doc): top_meta_image = try_one or try_two or try_three or try_four - return urllib.parse.urljoin(article_url, top_meta_image) + if top_meta_image: + return urllib.parse.urljoin(article_url, top_meta_image) + return '' def get_meta_type(self, doc): """Returns meta type of article, open graph protocol diff --git a/newspaper/parsers.py b/newspaper/parsers.py index f81c7177..b30d5632 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -45,6 +45,8 @@ def css_select(cls, node, selector): def get_unicode_html(cls, html): if isinstance(html, str): return html + if not html: + return html converted = UnicodeDammit(html, is_html=True) if not converted.unicode_markup: raise Exception( @@ -76,7 +78,8 @@ def clean_article_html(cls, node): article_cleaner.allow_tags = [ 'a', 'span', 'p', 'br', 'strong', 'b', 'em', 'i', 'tt', 'code', 'pre', 'blockquote', 'img', 'h1', - 'h2', 'h3', 'h4', 'h5', 'h6'] + 'h2', 'h3', 'h4', 'h5', 'h6', + 'ul', 'ol', 'li', 'dl', 'dt', 'dd'] article_cleaner.remove_unknown_tags = False return article_cleaner.clean_html(node) @@ -85,7 +88,7 @@ def nodeToString(cls, node): """`decode` is needed at the end because `etree.tostring` returns a python bytestring """ - return lxml.etree.tostring(node).decode() + return lxml.etree.tostring(node, method='html').decode() @classmethod def replaceTag(cls, node, tag): diff --git a/newspaper/version.py b/newspaper/version.py index 23245d84..9778534c 100644 --- a/newspaper/version.py +++ b/newspaper/version.py @@ -7,5 +7,5 @@ __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' -version_info = (0, 1, 4) +version_info = (0, 1, 5) __version__ = ".".join(map(str, version_info)) diff --git a/requirements.txt b/requirements.txt index 9f7bc216..f40f012e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ Pillow==2.6.1 PyYAML==3.11 cssselect==0.9.1 lxml==3.3.5 -nltk==3.0.0 +nltk==3.0.1 requests==2.3.0 six==1.7.3 feedparser==5.1.3 diff --git a/setup.py b/setup.py index 6433e830..082afbea 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( name='newspaper3k', - version='0.1.4', + version='0.1.5', description='Simplified python article discovery & extraction.', long_description=readme, author='Lucas Ou-Yang',