From 10e1a0c854b96d5f76acb47ea96b6578a6155b1b Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Wed, 27 Jan 2016 17:54:31 +0200 Subject: [PATCH 1/3] Fail on error http responses Fixes #142 --- docs/user_guide/advanced.rst | 2 ++ newspaper/configuration.py | 3 +++ newspaper/network.py | 7 +++++++ 3 files changed, 12 insertions(+) diff --git a/docs/user_guide/advanced.rst b/docs/user_guide/advanced.rst index 4aac8508..3cd34358 100644 --- a/docs/user_guide/advanced.rst +++ b/docs/user_guide/advanced.rst @@ -195,6 +195,8 @@ Here is a full list of the configuration options: ``keep_article_html``, default False, "set to True if you want to preserve html of body text" +``http_success_only``, default True, "set to False to capture non 2XX responses as well" + ``MIN_WORD_COUNT``, default 300, "num of word tokens in article text" ``MIN_SENT_COUNT``, default 7, "num of sentence tokens" diff --git a/newspaper/configuration.py b/newspaper/configuration.py index 374620c7..43dd4387 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -51,6 +51,9 @@ def __init__(self): # You may keep the html of just the main article body self.keep_article_html = False + # Fail for error respones (e.g. 404 page) + self.http_success_only = True + # English is the fallback self._language = 'en' diff --git a/newspaper/network.py b/newspaper/network.py index b12fb69f..4af11a0c 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -50,14 +50,21 @@ def get_html(url, config=None, response=None): try: html = None + response = requests.get( url=url, **get_request_kwargs(timeout, useragent)) + if response.encoding != FAIL_ENCODING: html = response.text else: html = response.content + + if config.http_success_only: + response.raise_for_status() # fail if other than "ok" response + if html is None: html = '' + return html except Exception as e: log.debug('%s on %s' % (e, url)) From 6f026341cf87396505a8f04822f881be41f43903 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Wed, 27 Jan 2016 17:58:08 +0200 Subject: [PATCH 2/3] More specific exception handling --- newspaper/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newspaper/network.py b/newspaper/network.py index 4af11a0c..f127e038 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -66,7 +66,7 @@ def get_html(url, config=None, response=None): html = '' return html - except Exception as e: + except requests.exceptions.RequestException as e: log.debug('%s on %s' % (e, url)) return '' From 1e7471d0911ba8eb4661dce311625067e33880f0 Mon Sep 17 00:00:00 2001 From: Yuri Prezument Date: Wed, 27 Jan 2016 18:00:16 +0200 Subject: [PATCH 3/3] http_success_only for async request as well --- newspaper/network.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/newspaper/network.py b/newspaper/network.py index f127e038..917eb9b7 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -79,6 +79,7 @@ class MRequest(object): """ def __init__(self, url, config=None): self.url = url + self.config = config config = config or Configuration() self.useragent = config.browser_user_agent self.timeout = config.request_timeout @@ -88,8 +89,9 @@ def send(self): try: self.resp = requests.get(self.url, **get_request_kwargs( self.timeout, self.useragent)) - except Exception as e: - pass + if self.config.http_success_only: + self.resp.raise_for_status() + except requests.exceptions.RequestException as e: log.critical('[REQUEST FAILED] ' + str(e))