Don't scrape error pages

docs/user_guide/advanced.rst

@@ -195,6 +195,8 @@ Here is a full list of the configuration options:
 
 ``keep_article_html``, default False, "set to True if you want to preserve html of body text"
 
+``http_success_only``, default True, "set to False to capture non 2XX responses as well"
+
 ``MIN_WORD_COUNT``, default 300, "num of word tokens in article text"
 
 ``MIN_SENT_COUNT``, default 7, "num of sentence tokens"

newspaper/configuration.py

@@ -51,6 +51,9 @@ def __init__(self):
         # You may keep the html of just the main article body
         self.keep_article_html = False
 
+        # Fail for error respones (e.g. 404 page)
+        self.http_success_only = True
+
         # English is the fallback
         self._language = 'en'
 

newspaper/network.py

@@ -50,16 +50,23 @@ def get_html(url, config=None, response=None):
 
     try:
         html = None
+
         response = requests.get(
             url=url, **get_request_kwargs(timeout, useragent))
+
         if response.encoding != FAIL_ENCODING:
             html = response.text
         else:
             html = response.content
+
+        if config.http_success_only:
+            response.raise_for_status()  # fail if other than "ok" response
+
         if html is None:
             html = ''
+
         return html
-    except Exception as e:
+    except requests.exceptions.RequestException as e:
         log.debug('%s on %s' % (e, url))
         return ''
 
@@ -72,6 +79,7 @@ class MRequest(object):
     """
     def __init__(self, url, config=None):
         self.url = url
+        self.config = config
         config = config or Configuration()
         self.useragent = config.browser_user_agent
         self.timeout = config.request_timeout
@@ -81,8 +89,9 @@ def send(self):
         try:
             self.resp = requests.get(self.url, **get_request_kwargs(
                                      self.timeout, self.useragent))
-        except Exception as e:
-            pass
+            if self.config.http_success_only:
+                self.resp.raise_for_status()
+        except requests.exceptions.RequestException as e:
             log.critical('[REQUEST FAILED] ' + str(e))