diff --git a/newspaper/configuration.py b/newspaper/configuration.py index 301dd179..d5c6d998 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -72,6 +72,7 @@ def __init__(self): self.request_timeout = 7 self.proxies = {} self.number_threads = 10 + self.verify_ssl_cert = True self.verbose = False # for debugging diff --git a/newspaper/network.py b/newspaper/network.py index 3fb448f0..52b80748 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -21,7 +21,7 @@ FAIL_ENCODING = 'ISO-8859-1' -def get_request_kwargs(timeout, useragent, proxies, headers, allow_redirects): +def get_request_kwargs(timeout, useragent, proxies, headers, allow_redirects, verify_ssl_cert): """This Wrapper method exists b/c some values in req_kwargs dict are methods which need to be called every time we make a request """ @@ -30,7 +30,8 @@ def get_request_kwargs(timeout, useragent, proxies, headers, allow_redirects): 'cookies': cj(), 'timeout': timeout, 'allow_redirects': allow_redirects, - 'proxies': proxies + 'proxies': proxies, + 'verify': verify_ssl_cert, } @@ -55,6 +56,7 @@ def get_html_2XX_only(url, config=None, response=None, return_final_url=False): timeout = config.request_timeout proxies = config.proxies headers = config.headers + verify_ssl_cert = config.verify_ssl_cert allow_redirects = config.allow_redirects if response is not None: @@ -64,7 +66,7 @@ def get_html_2XX_only(url, config=None, response=None, return_final_url=False): return html response = requests.get( - url=url, **get_request_kwargs(timeout, useragent, proxies, headers, allow_redirects)) + url=url, **get_request_kwargs(timeout, useragent, proxies, headers, allow_redirects, verify_ssl_cert)) html = _get_html_from_response(response, config) final_url = response.url