diff --git a/newspaper/configuration.py b/newspaper/configuration.py index 94688e70..bcaf52cb 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -57,6 +57,9 @@ def __init__(self): # Fail for error responses (e.g. 404 page) self.http_success_only = True + # Allow redirects (enabled by default) + self.allow_redirects = True + # English is the fallback self._language = 'en' diff --git a/newspaper/network.py b/newspaper/network.py index 29f0e699..8ba02aa7 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -21,7 +21,7 @@ FAIL_ENCODING = 'ISO-8859-1' -def get_request_kwargs(timeout, useragent, proxies, headers): +def get_request_kwargs(timeout, useragent, proxies, headers, allow_redirects): """This Wrapper method exists b/c some values in req_kwargs dict are methods which need to be called every time we make a request """ @@ -29,7 +29,7 @@ def get_request_kwargs(timeout, useragent, proxies, headers): 'headers': headers if headers else {'User-Agent': useragent}, 'cookies': cj(), 'timeout': timeout, - 'allow_redirects': True, + 'allow_redirects': allow_redirects, 'proxies': proxies } @@ -55,12 +55,13 @@ def get_html_2XX_only(url, config=None, response=None): timeout = config.request_timeout proxies = config.proxies headers = config.headers + allow_redirects = config.allow_redirects if response is not None: return _get_html_from_response(response, config) response = requests.get( - url=url, **get_request_kwargs(timeout, useragent, proxies, headers)) + url=url, **get_request_kwargs(timeout, useragent, proxies, headers, allow_redirects)) html = _get_html_from_response(response, config) @@ -107,7 +108,7 @@ def __init__(self, url, config=None): def send(self): try: self.resp = requests.get(self.url, **get_request_kwargs( - self.timeout, self.useragent, self.proxies, self.headers)) + self.timeout, self.useragent, self.proxies, self.headers, self.config.allow_redirects)) if self.config.http_success_only: self.resp.raise_for_status() except requests.exceptions.RequestException as e: