From f2de9788989ec3c9c664c2c23b2d19ecc5567bf7 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Sat, 27 Dec 2014 01:35:42 -0800 Subject: [PATCH 1/2] Handle exception logging using tracebacks, removed custom exception alert stdout print statements --- newspaper/article.py | 2 +- newspaper/images.py | 18 ++++++------------ newspaper/mthreading.py | 7 ++++--- newspaper/parsers.py | 10 ++++++---- newspaper/source.py | 2 -- 5 files changed, 17 insertions(+), 22 deletions(-) diff --git a/newspaper/article.py b/newspaper/article.py index a33b770d..4c7a8290 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -160,7 +160,7 @@ def parse(self): self.clean_doc = copy.deepcopy(self.doc) if self.doc is None: - print('[Article parse ERR] %s' % self.url) + # `parse` call failed, return nothing return # TODO: Fix this, sync in our fix_url() method diff --git a/newspaper/images.py b/newspaper/images.py index 09115084..1ae5a6ab 100644 --- a/newspaper/images.py +++ b/newspaper/images.py @@ -11,6 +11,7 @@ import logging import math import io +import traceback import urllib.request, urllib.parse, urllib.error import urllib.request, urllib.error, urllib.parse @@ -116,28 +117,21 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False): while not p.image and new_data: try: p.feed(new_data) - except IOError as e: - # pil failed to install, jpeg codec broken - # **should work if you install via pillow - print(('***jpeg misconfiguration! check pillow or pil' - 'installation this machine: %s' % str(e))) + except IOError: + traceback.print_exc() p = None break - except ValueError as ve: - log.debug('cant read image format: %s' % url) + except ValueError: + traceback.print_exc() p = None break except Exception as e: # For some favicon.ico images, the image is so small # that our PIL feed() method fails a length test. - # We add a check below for this. is_favicon = (urls.url_to_filetype(url) == 'ico') if is_favicon: - print('we caught a favicon!: %s' % url) + pass else: - # import traceback - # print(traceback.format_exc()) - print('PIL feed() failure for image:', url, str(e)) raise e p = None break diff --git a/newspaper/mthreading.py b/newspaper/mthreading.py index 8113d2ff..c1cb355c 100644 --- a/newspaper/mthreading.py +++ b/newspaper/mthreading.py @@ -10,6 +10,7 @@ __copyright__ = 'Copyright 2014, Lucas Ou-Yang' import queue +import traceback from threading import Thread @@ -28,12 +29,12 @@ def run(self): try: func, args, kargs = self.tasks.get() except queue.Empty: - print('thread breaking b/c queue is empty') + traceback.print_exc() break try: func(*args, **kargs) - except Exception as e: - print('critical multi-thread err %s' % e) + except Exception: + traceback.print_exc() self.tasks.task_done() diff --git a/newspaper/parsers.py b/newspaper/parsers.py index 10bbd540..95326af4 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -10,6 +10,7 @@ import lxml.etree import lxml.html import lxml.html.clean +import traceback from copy import deepcopy @@ -42,14 +43,15 @@ def css_select(cls, node, selector): @classmethod def fromstring(cls, html): html = utils.encodeValue(html) + # don't bring the entire library down because one article + # or article failed to parse try: cls.doc = lxml.html.fromstring(html) - except Exception as e: - print('[Parse lxml ERR]', str(e)) + return cls.doc + except Exception: + traceback.print_exc() return None - return cls.doc - # @classmethod # def set_doc(cls, html): # cls.doc = cls.fromstring(html) diff --git a/newspaper/source.py b/newspaper/source.py index 74338755..0a0200ff 100644 --- a/newspaper/source.py +++ b/newspaper/source.py @@ -201,8 +201,6 @@ def parse_categories(self): for category in self.categories: doc = self.config.get_parser().fromstring(category.html) category.doc = doc - if category.doc is None: - print('[Category parse ERR]', category.url) self.categories = [c for c in self.categories if c.doc is not None] From 800651c08fdf5d1832b1a8ff56ed22896599865c Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Sat, 27 Dec 2014 02:34:51 -0800 Subject: [PATCH 2/2] Remove encoding tag in HTML before having lxml parse a DOM out This was a very annoying bug in the past, which caused DOM parse failures: Reference https://github.com/codelucas/newspaper/pull/97 https://github.com/codelucas/newspaper/issues/78 --- newspaper/parsers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/newspaper/parsers.py b/newspaper/parsers.py index 95326af4..cbc86216 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -10,6 +10,7 @@ import lxml.etree import lxml.html import lxml.html.clean +import re import traceback from copy import deepcopy @@ -46,6 +47,14 @@ def fromstring(cls, html): # don't bring the entire library down because one article # or article failed to parse try: + # remove encoding tag because lxml won't accept it for + # unicode objects (Issue #78) + if isinstance(html, bytes): + if html.startswith(b'', b'', html, flags=re.DOTALL) + else: + if html.startswith('', '', html, flags=re.DOTALL) cls.doc = lxml.html.fromstring(html) return cls.doc except Exception: