diff --git a/docs/user_guide/advanced.rst b/docs/user_guide/advanced.rst index 4aac8508..3cd34358 100644 --- a/docs/user_guide/advanced.rst +++ b/docs/user_guide/advanced.rst @@ -195,6 +195,8 @@ Here is a full list of the configuration options: ``keep_article_html``, default False, "set to True if you want to preserve html of body text" +``http_success_only``, default True, "set to False to capture non 2XX responses as well" + ``MIN_WORD_COUNT``, default 300, "num of word tokens in article text" ``MIN_SENT_COUNT``, default 7, "num of sentence tokens" diff --git a/newspaper/configuration.py b/newspaper/configuration.py index 374620c7..43dd4387 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -51,6 +51,9 @@ def __init__(self): # You may keep the html of just the main article body self.keep_article_html = False + # Fail for error respones (e.g. 404 page) + self.http_success_only = True + # English is the fallback self._language = 'en' diff --git a/newspaper/network.py b/newspaper/network.py index b12fb69f..917eb9b7 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -50,16 +50,23 @@ def get_html(url, config=None, response=None): try: html = None + response = requests.get( url=url, **get_request_kwargs(timeout, useragent)) + if response.encoding != FAIL_ENCODING: html = response.text else: html = response.content + + if config.http_success_only: + response.raise_for_status() # fail if other than "ok" response + if html is None: html = '' + return html - except Exception as e: + except requests.exceptions.RequestException as e: log.debug('%s on %s' % (e, url)) return '' @@ -72,6 +79,7 @@ class MRequest(object): """ def __init__(self, url, config=None): self.url = url + self.config = config config = config or Configuration() self.useragent = config.browser_user_agent self.timeout = config.request_timeout @@ -81,8 +89,9 @@ def send(self): try: self.resp = requests.get(self.url, **get_request_kwargs( self.timeout, self.useragent)) - except Exception as e: - pass + if self.config.http_success_only: + self.resp.raise_for_status() + except requests.exceptions.RequestException as e: log.critical('[REQUEST FAILED] ' + str(e))