Huge bugfix to aid lxml DOM parsing + remove unhelpful and excess exception messages and added tracebacks to exception logging

newspaper/article.py

@@ -160,7 +160,7 @@ def parse(self):
         self.clean_doc = copy.deepcopy(self.doc)
 
         if self.doc is None:
-            print('[Article parse ERR] %s' % self.url)
+            # `parse` call failed, return nothing
             return
 
         # TODO: Fix this, sync in our fix_url() method

newspaper/images.py

@@ -11,6 +11,7 @@
 import logging
 import math
 import io
+import traceback
 import urllib.request, urllib.parse, urllib.error
 import urllib.request, urllib.error, urllib.parse
 
@@ -116,28 +117,21 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False):
                 while not p.image and new_data:
                     try:
                         p.feed(new_data)
-                    except IOError as e:
-                        # pil failed to install, jpeg codec broken
-                        # **should work if you install via pillow
-                        print(('***jpeg misconfiguration! check pillow or pil'
-                               'installation this machine: %s' % str(e)))
+                    except IOError:
+                        traceback.print_exc()
                         p = None
                         break
-                    except ValueError as ve:
-                        log.debug('cant read image format: %s' % url)
+                    except ValueError:
+                        traceback.print_exc()
                         p = None
                         break
                     except Exception as e:
                         # For some favicon.ico images, the image is so small
                         # that our PIL feed() method fails a length test.
-                        # We add a check below for this.
                         is_favicon = (urls.url_to_filetype(url) == 'ico')
                         if is_favicon:
-                            print('we caught a favicon!: %s' % url)
+                            pass
                         else:
-                            # import traceback
-                            # print(traceback.format_exc())
-                            print('PIL feed() failure for image:', url, str(e))
                             raise e
                         p = None
                         break

newspaper/mthreading.py

@@ -10,6 +10,7 @@
 __copyright__ = 'Copyright 2014, Lucas Ou-Yang'
 
 import queue
+import traceback
 from threading import Thread
 
 
@@ -28,12 +29,12 @@ def run(self):
             try:
                 func, args, kargs = self.tasks.get()
             except queue.Empty:
-                print('thread breaking b/c queue is empty')
+                traceback.print_exc()
                 break
             try:
                 func(*args, **kargs)
-            except Exception as e:
-                print('critical multi-thread err %s' % e)
+            except Exception:
+                traceback.print_exc()
 
             self.tasks.task_done()
 

newspaper/parsers.py

@@ -10,6 +10,8 @@
 import lxml.etree
 import lxml.html
 import lxml.html.clean
+import re
+import traceback
 
 from copy import deepcopy
 
@@ -42,14 +44,23 @@ def css_select(cls, node, selector):
     @classmethod
     def fromstring(cls, html):
         html = utils.encodeValue(html)
+        # don't bring the entire library down because one article
+        # or article failed to parse
         try:
+            # remove encoding tag because lxml won't accept it for
+            # unicode objects (Issue #78)
+            if isinstance(html, bytes):
+                if html.startswith(b'<?'):
+                    html = re.sub(b'^\<\?.*?\?\>', b'', html, flags=re.DOTALL)
+            else:
+                if html.startswith('<?'):
+                    html = re.sub(r'^\<\?.*?\?\>', '', html, flags=re.DOTALL)
             cls.doc = lxml.html.fromstring(html)
-        except Exception as e:
-            print('[Parse lxml ERR]', str(e))
+            return cls.doc
+        except Exception:
+            traceback.print_exc()
             return None
 
-        return cls.doc
-
     # @classmethod
     # def set_doc(cls, html):
     #    cls.doc = cls.fromstring(html)

newspaper/source.py

@@ -201,8 +201,6 @@ def parse_categories(self):
         for category in self.categories:
             doc = self.config.get_parser().fromstring(category.html)
             category.doc = doc
-            if category.doc is None:
-                print('[Category parse ERR]', category.url)
 
         self.categories = [c for c in self.categories if c.doc is not None]