From 7bb0c76a99fdc7cd74bc79f896ce3940a05246fa Mon Sep 17 00:00:00 2001 From: Ido Shamun Date: Thu, 17 Jun 2021 17:55:36 +0300 Subject: [PATCH 01/19] fix cleaning the wrong top node Before this change, `top_node` was cleaned and then copied to the `clean_top_node`. I believe this is not the original intent and should be fixed because it creates confusion and there's no way to get the raw `top_node` --- newspaper/article.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newspaper/article.py b/newspaper/article.py index df0d9c43..d7c2cd34 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -275,8 +275,8 @@ def parse(self): video_extractor = VideoExtractor(self.config, self.top_node) self.set_movies(video_extractor.get_videos()) - self.top_node = self.extractor.post_cleanup(self.top_node) self.clean_top_node = copy.deepcopy(self.top_node) + self.clean_top_node = self.extractor.post_cleanup(self.clean_top_node) text, article_html = output_formatter.get_formatted( self.top_node) From e4beb8f1531155d7c9fff1e8f93dfcf7c1f42697 Mon Sep 17 00:00:00 2001 From: Ido Shamun Date: Wed, 28 Jul 2021 13:55:03 +0300 Subject: [PATCH 02/19] add ul and ol to the list of nodes to check --- newspaper/extractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 96255401..aa81f7fc 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -1014,7 +1014,7 @@ def nodes_to_check(self, doc): on like paragraphs and tables """ nodes_to_check = [] - for tag in ['p', 'pre', 'td']: + for tag in ['p', 'pre', 'td', 'ol', 'ul']: items = self.parser.getElementsByTag(doc, tag=tag) nodes_to_check += items return nodes_to_check From a626a41032015fd32b49bf03471654b44fae6044 Mon Sep 17 00:00:00 2001 From: Ido Shamun Date: Wed, 28 Jul 2021 13:56:15 +0300 Subject: [PATCH 03/19] deep copy top node in the output formatter --- newspaper/outputformatters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/newspaper/outputformatters.py b/newspaper/outputformatters.py index 47a76467..620dce63 100644 --- a/newspaper/outputformatters.py +++ b/newspaper/outputformatters.py @@ -9,6 +9,7 @@ from html import unescape import logging +import copy from .text import innerTrim @@ -42,7 +43,7 @@ def get_formatted(self, top_node): """Returns the body text of an article, and also the body article html if specified. Returns in (text, html) form """ - self.top_node = top_node + self.top_node = copy.deepcopy(top_node) html, text = '', '' self.remove_negativescores_nodes() From a5b15ff3d057618f770353e084d399bf251db626 Mon Sep 17 00:00:00 2001 From: Ido Shamun Date: Tue, 19 Oct 2021 17:44:49 +0300 Subject: [PATCH 04/19] add support for section based articles --- newspaper/extractors.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index aa81f7fc..31eee574 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -1017,6 +1017,10 @@ def nodes_to_check(self, doc): for tag in ['p', 'pre', 'td', 'ol', 'ul']: items = self.parser.getElementsByTag(doc, tag=tag) nodes_to_check += items + for tag in ['section']: + items = self.parser.getElementsByTag(doc, tag=tag) + if len(items) > 1: + nodes_to_check = items return nodes_to_check def is_table_and_no_para_exist(self, e): From a737681a1a1bdfd17bd97bd78deea7791b431ffe Mon Sep 17 00:00:00 2001 From: Ido Shamun Date: Mon, 25 Oct 2021 16:04:06 +0300 Subject: [PATCH 05/19] fix medium heuristic for finding the content node --- newspaper/extractors.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 31eee574..f6f9406a 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -1014,13 +1014,16 @@ def nodes_to_check(self, doc): on like paragraphs and tables """ nodes_to_check = [] - for tag in ['p', 'pre', 'td', 'ol', 'ul']: - items = self.parser.getElementsByTag(doc, tag=tag) - nodes_to_check += items - for tag in ['section']: - items = self.parser.getElementsByTag(doc, tag=tag) - if len(items) > 1: - nodes_to_check = items + articles = self.parser.getElementsByTag(doc, tag='article') + if len(articles) > 0: + # Specific heuristic for Medium articles + sections = self.parser.getElementsByTag(articles[0], tag='section') + if len(sections) > 1: + nodes_to_check = sections + if len(nodes_to_check) == 0: + for tag in ['p', 'pre', 'td', 'ol', 'ul']: + items = self.parser.getElementsByTag(doc, tag=tag) + nodes_to_check += items return nodes_to_check def is_table_and_no_para_exist(self, e): From a169a56c9845b1b60c4b62bc164cffacbade5857 Mon Sep 17 00:00:00 2001 From: Ido Shamun Date: Tue, 26 Oct 2021 12:33:07 +0300 Subject: [PATCH 06/19] don't clean the default doc variable instead use clean_doc --- newspaper/article.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/newspaper/article.py b/newspaper/article.py index d7c2cd34..85c5f362 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -213,19 +213,22 @@ def parse(self): self.throw_if_not_downloaded_verbose() self.doc = self.config.get_parser().fromstring(self.html) - self.clean_doc = copy.deepcopy(self.doc) if self.doc is None: # `parse` call failed, return nothing return + document_cleaner = DocumentCleaner(self.config) + output_formatter = OutputFormatter(self.config) + + self.clean_doc = copy.deepcopy(self.doc) + # Before any computations on the body, clean DOM object + self.clean_doc = document_cleaner.clean(self.clean_doc) + # TODO: Fix this, sync in our fix_url() method parse_candidate = self.get_parse_candidate() self.link_hash = parse_candidate.link_hash # MD5 - document_cleaner = DocumentCleaner(self.config) - output_formatter = OutputFormatter(self.config) - title = self.extractor.get_title(self.clean_doc) self.set_title(title) @@ -267,9 +270,6 @@ def parse(self): self.url, self.clean_doc) - # Before any computations on the body, clean DOM object - self.doc = document_cleaner.clean(self.doc) - self.top_node = self.extractor.calculate_best_node(self.doc) if self.top_node is not None: video_extractor = VideoExtractor(self.config, self.top_node) From c01bdec73c950660b88e8301412d215a236918c9 Mon Sep 17 00:00:00 2001 From: Ido Shamun Date: Tue, 26 Oct 2021 16:25:03 +0300 Subject: [PATCH 07/19] force medium heuristic on medium articles only --- newspaper/extractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index f6f9406a..28d1dd5b 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -1015,7 +1015,7 @@ def nodes_to_check(self, doc): """ nodes_to_check = [] articles = self.parser.getElementsByTag(doc, tag='article') - if len(articles) > 0: + if len(articles) > 0 and self.get_meta_site_name(doc) == 'Medium': # Specific heuristic for Medium articles sections = self.parser.getElementsByTag(articles[0], tag='section') if len(sections) > 1: From a4dcbe45b84839d9ece18be8f698f8f50daa6aa3 Mon Sep 17 00:00:00 2001 From: Ido Shamun Date: Tue, 26 Oct 2021 16:47:46 +0300 Subject: [PATCH 08/19] fallback to clean doc if it wasn't possible to find top node --- newspaper/article.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/newspaper/article.py b/newspaper/article.py index 85c5f362..620cb097 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -271,6 +271,8 @@ def parse(self): self.clean_doc) self.top_node = self.extractor.calculate_best_node(self.doc) + if self.top_node is None: + self.top_node = self.extractor.calculate_best_node(self.clean_doc) if self.top_node is not None: video_extractor = VideoExtractor(self.config, self.top_node) self.set_movies(video_extractor.get_videos()) From 7b95eb6c51800d27bedafc3ece94bad148dc9832 Mon Sep 17 00:00:00 2001 From: Ido Shamun Date: Wed, 27 Oct 2021 17:04:45 +0300 Subject: [PATCH 09/19] add fallback heuristics when can't find top node --- newspaper/article.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/newspaper/article.py b/newspaper/article.py index 620cb097..dd3a25e5 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -273,6 +273,14 @@ def parse(self): self.top_node = self.extractor.calculate_best_node(self.doc) if self.top_node is None: self.top_node = self.extractor.calculate_best_node(self.clean_doc) + if self.top_node is None: + self.top_node = self.extractor.parser.getElementById(self.doc, 'content') + if self.top_node is None: + for tag in ['article', 'main']: + nodes = self.extractor.parser.getElementsByTag(self.doc, tag=tag) + if len(nodes) > 0: + self.top_node = nodes[0] + break if self.top_node is not None: video_extractor = VideoExtractor(self.config, self.top_node) self.set_movies(video_extractor.get_videos()) From 06551769ffaaba3edd5ecf327c40705d0cb2f750 Mon Sep 17 00:00:00 2001 From: vpol Date: Thu, 14 Jul 2022 07:52:29 +0100 Subject: [PATCH 10/19] allow_redirects config option (#2) --- newspaper/configuration.py | 3 +++ newspaper/network.py | 9 +++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/newspaper/configuration.py b/newspaper/configuration.py index 94688e70..bcaf52cb 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -57,6 +57,9 @@ def __init__(self): # Fail for error responses (e.g. 404 page) self.http_success_only = True + # Allow redirects (enabled by default) + self.allow_redirects = True + # English is the fallback self._language = 'en' diff --git a/newspaper/network.py b/newspaper/network.py index 29f0e699..8ba02aa7 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -21,7 +21,7 @@ FAIL_ENCODING = 'ISO-8859-1' -def get_request_kwargs(timeout, useragent, proxies, headers): +def get_request_kwargs(timeout, useragent, proxies, headers, allow_redirects): """This Wrapper method exists b/c some values in req_kwargs dict are methods which need to be called every time we make a request """ @@ -29,7 +29,7 @@ def get_request_kwargs(timeout, useragent, proxies, headers): 'headers': headers if headers else {'User-Agent': useragent}, 'cookies': cj(), 'timeout': timeout, - 'allow_redirects': True, + 'allow_redirects': allow_redirects, 'proxies': proxies } @@ -55,12 +55,13 @@ def get_html_2XX_only(url, config=None, response=None): timeout = config.request_timeout proxies = config.proxies headers = config.headers + allow_redirects = config.allow_redirects if response is not None: return _get_html_from_response(response, config) response = requests.get( - url=url, **get_request_kwargs(timeout, useragent, proxies, headers)) + url=url, **get_request_kwargs(timeout, useragent, proxies, headers, allow_redirects)) html = _get_html_from_response(response, config) @@ -107,7 +108,7 @@ def __init__(self, url, config=None): def send(self): try: self.resp = requests.get(self.url, **get_request_kwargs( - self.timeout, self.useragent, self.proxies, self.headers)) + self.timeout, self.useragent, self.proxies, self.headers, self.config.allow_redirects)) if self.config.http_success_only: self.resp.raise_for_status() except requests.exceptions.RequestException as e: From e149436510ac32dfa34e150373246400aaf5370f Mon Sep 17 00:00:00 2001 From: Viktor Poluksht Date: Thu, 13 Apr 2023 12:05:10 +0100 Subject: [PATCH 11/19] feat: allow to ignore certain basename regex (#3) Signed-off-by: Viktor Poluksht --- newspaper/configuration.py | 1 + newspaper/extractors.py | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/newspaper/configuration.py b/newspaper/configuration.py index bcaf52cb..301dd179 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -60,6 +60,7 @@ def __init__(self): # Allow redirects (enabled by default) self.allow_redirects = True + self.ignored_images_suffix_list = [] # English is the fallback self._language = 'en' diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 28d1dd5b..d47eb488 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -13,6 +13,7 @@ import copy import logging +import os.path import re import re from collections import defaultdict @@ -449,19 +450,21 @@ def get_meta_img_url(self, article_url, doc): """ top_meta_image, try_one, try_two, try_three, try_four = [None] * 5 try_one = self.get_meta_content(doc, 'meta[property="og:image"]') + try_one = None if self.image_is_ignored(try_one) else try_one if not try_one: link_img_src_kwargs = \ {'tag': 'link', 'attr': 'rel', 'value': 'img_src|image_src'} elems = self.parser.getElementsByTag(doc, use_regex=True, **link_img_src_kwargs) try_two = elems[0].get('href') if elems else None - + try_two = None if self.image_is_ignored(try_two) else try_two if not try_two: try_three = self.get_meta_content(doc, 'meta[name="og:image"]') - + try_three = None if self.image_is_ignored(try_three) else try_three if not try_three: link_icon_kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'icon'} elems = self.parser.getElementsByTag(doc, **link_icon_kwargs) try_four = elems[0].get('href') if elems else None + try_four = None if self.image_is_ignored(try_four) else try_four top_meta_image = try_one or try_two or try_three or try_four @@ -469,6 +472,12 @@ def get_meta_img_url(self, article_url, doc): return urljoin(article_url, top_meta_image) return '' + def image_is_ignored(self, image): + return len([True for x in self.config.ignored_images_suffix_list if image and self.match_image(x, os.path.basename(image))]) > 0 + + def match_image(self, pattern, image): + return re.search(pattern, image) is not None + def get_meta_type(self, doc): """Returns meta type of article, open graph protocol """ From 7d34fc99ed3fbe7636cbf4d156bcb8c1e9abd5e4 Mon Sep 17 00:00:00 2001 From: Viktor Poluksht Date: Thu, 13 Apr 2023 15:29:17 +0100 Subject: [PATCH 12/19] fix: more places where we have to check images Signed-off-by: Viktor Poluksht --- newspaper/extractors.py | 3 ++- tests/unit_tests.py | 23 +++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index d47eb488..cfc2bca5 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -473,7 +473,7 @@ def get_meta_img_url(self, article_url, doc): return '' def image_is_ignored(self, image): - return len([True for x in self.config.ignored_images_suffix_list if image and self.match_image(x, os.path.basename(image))]) > 0 + return any([True for x in self.config.ignored_images_suffix_list if image and image != '' and self.match_image(x, os.path.basename(image))]) def match_image(self, pattern, image): return re.search(pattern, image) is not None @@ -584,6 +584,7 @@ def get_img_urls(self, article_url, doc): for img_tag in img_tags if img_tag.get('src')] img_links = set([urljoin(article_url, url) for url in urls]) + img_links = [x for x in img_links if not self.image_is_ignored(x)] return img_links def get_first_img_url(self, article_url, top_node): diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 69c05adf..76e54f9c 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -24,7 +24,7 @@ URLS_FILE = os.path.join(TEST_DIR, 'data', 'fulltext_url_list.txt') import newspaper -from newspaper import Article, fulltext, Source, ArticleException, news_pool +from newspaper import Article, Config, fulltext, Source, ArticleException, news_pool from newspaper.article import ArticleDownloadState from newspaper.configuration import Configuration from newspaper.urls import get_domain @@ -406,9 +406,9 @@ def test_get_top_image_from_meta(self): html = '' \ '' html_empty_og_content = '' \ - '' + '' html_empty_all = '' \ - '' + '' html_rel_img_src = html_empty_all + '' html_rel_img_src2 = html_empty_all + '' html_rel_icon = html_empty_all + '' @@ -544,7 +544,6 @@ def test_valid_urls(self): print('\t\turl: %s is supposed to be %s' % (url, truth_val)) raise - @print_test def test_pubdate(self): """Checks that irrelevant data in url isn't considered as publishing date""" @@ -568,7 +567,6 @@ def test_pubdate(self): print('\t\tpublishing date in %s should not be present' % (url)) raise - @unittest.skip("Need to write an actual test") @print_test def test_prepare_url(self): @@ -635,9 +633,9 @@ class ConfigBuildTestCase(unittest.TestCase): NOTE: No need to mock responses as we are just initializing the objects, not actually calling download(..) """ + @print_test def test_article_default_params(self): - a = Article(url='http://www.cnn.com/2013/11/27/' 'travel/weather-thanksgiving/index.html') self.assertEqual('en', a.config.language) @@ -767,6 +765,19 @@ def test_article_pdf_fetching(self): a.download() self.assertNotEqual('%PDF-', a.html) + +class TestIgnoreImages(unittest.TestCase): + + @print_test + def test_config_ignore_images(self): + config = Config() + config.ignored_images_suffix_list = ['think.png', '(.*)\.ico'] + a = Article('https://www.reillywood.com/blog/why-nu/', config=config) + a.download() + a.parse() + self.assertEqual('https://d33wubrfki0l68.cloudfront.net/77d3013f91800257b3ca2adfb995ae24e49fff4e/b3086/img/main/headshot.jpg', a.top_img) + + if __name__ == '__main__': argv = list(sys.argv) if 'fulltext' in argv: From b75385a7d26378a7d8b04b6a243706d6042b4c1c Mon Sep 17 00:00:00 2001 From: Viktor Poluksht Date: Tue, 25 Apr 2023 08:57:02 +0100 Subject: [PATCH 13/19] fix: type conversion list -> set Signed-off-by: Viktor Poluksht --- newspaper/extractors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index cfc2bca5..d1caada0 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -584,7 +584,7 @@ def get_img_urls(self, article_url, doc): for img_tag in img_tags if img_tag.get('src')] img_links = set([urljoin(article_url, url) for url in urls]) - img_links = [x for x in img_links if not self.image_is_ignored(x)] + img_links = set([x for x in img_links if not self.image_is_ignored(x)]) return img_links def get_first_img_url(self, article_url, top_node): From d25229e2dd87d8b9fc243621acaf83150226af55 Mon Sep 17 00:00:00 2001 From: denis <54359969+denisb0@users.noreply.github.com> Date: Wed, 3 Apr 2024 08:54:15 +0300 Subject: [PATCH 14/19] chore: set lxml version to supported (#6) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 61974601..0ea8c095 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cssselect>=0.9.2 feedfinder2>=0.0.4 feedparser>=5.2.1 jieba3k>=0.35.1 -lxml>=3.6.0 +lxml==5.1.0 # https://lxml.de/5.2/changes-5.2.0.html nltk>=3.2.1 Pillow>=3.3.0 pythainlp>=1.7.2 From 46fed320c254e0ec6db1e8b31566e4f382c95954 Mon Sep 17 00:00:00 2001 From: denis <54359969+denisb0@users.noreply.github.com> Date: Wed, 14 May 2025 08:54:07 +0300 Subject: [PATCH 15/19] feat: add a fallback to image download fail case (#7) --- newspaper/images.py | 35 ++++++++++++++++++++++++++++++++++- tests/unit_tests.py | 14 +++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/newspaper/images.py b/newspaper/images.py index 77476625..bead0cc3 100644 --- a/newspaper/images.py +++ b/newspaper/images.py @@ -83,6 +83,37 @@ def clean_url(url): return url +def get_full_image_dimensions(image_url): + """Fallback in case PIL can't open the streamed image + """ + try: + response = requests.get(image_url) # No stream=True needed + response.raise_for_status() # Raise an exception for bad status codes + + # Use io.BytesIO to treat the response content (bytes) as a file + image_bytes = io.BytesIO(response.content) + + # Open the image directly from the bytes stream + img = Image.open(image_bytes) + + sz = img.size + + # It's good practice to close the image when done + img.close() + + return sz + + except requests.exceptions.RequestException as e: + log.warning(f"Method 2 (Direct): Error fetching the image via requests: {e}") + return None + except FileNotFoundError: + log.warning("Method 2 (Direct): Error: io.BytesIO did not behave as expected (treated as file not found).") + return None + except Exception as e: + log.warning(f"Method 2 (Direct): An unexpected error occurred while opening image: {e}") + return None + + def fetch_url(url, useragent, referer=None, retries=1, dimension=False): cur_try = 0 nothing = None if dimension else (None, None) @@ -143,7 +174,9 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False): if dimension and p.image: return p.image.size elif dimension: - return nothing + # we did read the image, but it failed to parse for some reason + # try to download it in one go + return get_full_image_dimensions(url) elif dimension: # expected an image, but didn't get one return nothing diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 76e54f9c..1e867d33 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -24,7 +24,7 @@ URLS_FILE = os.path.join(TEST_DIR, 'data', 'fulltext_url_list.txt') import newspaper -from newspaper import Article, Config, fulltext, Source, ArticleException, news_pool +from newspaper import Article, Config, fulltext, Source, ArticleException, news_pool, images from newspaper.article import ArticleDownloadState from newspaper.configuration import Configuration from newspaper.urls import get_domain @@ -778,6 +778,18 @@ def test_config_ignore_images(self): self.assertEqual('https://d33wubrfki0l68.cloudfront.net/77d3013f91800257b3ca2adfb995ae24e49fff4e/b3086/img/main/headshot.jpg', a.top_img) +class TestGetImageDimensionFallback(unittest.TestCase): + + @print_test + def test_get_image_dimension_fallback(self): + config = Config() + config.image_dimension_ration = 32 / 9 + a = Article('https://appwrite.io/blog/post/add-figma-oauth2-appwrite', config=config) + s = images.Scraper(a) + sr = s.satisfies_requirements('https://appwrite.io/images/blog/add-figma-oauth2-appwrite/cover.png') + self.assertTrue(sr) + + if __name__ == '__main__': argv = list(sys.argv) if 'fulltext' in argv: From bc8fd3294ff7bf9fbb41b2ae0ace135682dcfb71 Mon Sep 17 00:00:00 2001 From: Ido Shamun Date: Mon, 25 Aug 2025 20:45:29 +0300 Subject: [PATCH 16/19] feat: add final_url property to article final_url stores the actual url used to fetch the html after redirects and meta refresh --- newspaper/article.py | 16 +++++++++++++++- newspaper/network.py | 10 ++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/newspaper/article.py b/newspaper/article.py index dd3a25e5..4a1b6ad1 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -153,6 +153,9 @@ def __init__(self, url, title='', source_url='', config=None, **kwargs): # A property dict for users to store custom data. self.additional_data = {} + # The final URL after redirects and meta refresh + self.final_url = None + def build(self): """Build a lone article from a URL independent of the source (newspaper). Don't normally call this method b/c it's good to multithread articles @@ -173,7 +176,9 @@ def _parse_scheme_file(self, path): def _parse_scheme_http(self): try: - return network.get_html_2XX_only(self.url, self.config) + html, final_url = network.get_html_2XX_only(self.url, self.config, return_final_url=True) + self.final_url = final_url + return html except requests.exceptions.RequestException as e: self.download_state = ArticleDownloadState.FAILED_RESPONSE self.download_exception_msg = str(e) @@ -190,18 +195,27 @@ def download(self, input_html=None, title=None, recursion_counter=0): parsed_url = urlparse(self.url) if parsed_url.scheme == "file": html = self._parse_scheme_file(parsed_url.path) + # For file scheme, the final URL is the same as the initial URL + if self.final_url is None: + self.final_url = self.url else: html = self._parse_scheme_http() + # final_url is already set in _parse_scheme_http if html is None: log.debug('Download failed on URL %s because of %s' % (self.url, self.download_exception_msg)) return else: html = input_html + # If HTML is provided directly and final_url not set, use the current URL + if self.final_url is None: + self.final_url = self.url if self.config.follow_meta_refresh: meta_refresh_url = extract_meta_refresh(html) if meta_refresh_url and recursion_counter < 1: + # Update final_url to the meta refresh URL + self.final_url = meta_refresh_url return self.download( input_html=network.get_html(meta_refresh_url), recursion_counter=recursion_counter + 1) diff --git a/newspaper/network.py b/newspaper/network.py index 8ba02aa7..3fb448f0 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -44,7 +44,7 @@ def get_html(url, config=None, response=None): return '' -def get_html_2XX_only(url, config=None, response=None): +def get_html_2XX_only(url, config=None, response=None, return_final_url=False): """Consolidated logic for http requests from newspaper. We handle error cases: - Attempt to find encoding of the html by using HTTP header. Fallback to 'ISO-8859-1' if not provided. @@ -58,17 +58,23 @@ def get_html_2XX_only(url, config=None, response=None): allow_redirects = config.allow_redirects if response is not None: - return _get_html_from_response(response, config) + html = _get_html_from_response(response, config) + if return_final_url: + return html, getattr(response, 'url', url) + return html response = requests.get( url=url, **get_request_kwargs(timeout, useragent, proxies, headers, allow_redirects)) html = _get_html_from_response(response, config) + final_url = response.url if config.http_success_only: # fail if HTTP sends a non 2XX response response.raise_for_status() + if return_final_url: + return html, final_url return html From e69b5f88d2bd5a0534a849f06c125db3aeeec694 Mon Sep 17 00:00:00 2001 From: Viktor Poluksht Date: Sun, 30 Nov 2025 13:06:28 +0000 Subject: [PATCH 17/19] feat: added verify_ssl_cert option to pass to requests for proxy use --- newspaper/configuration.py | 1 + newspaper/network.py | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/newspaper/configuration.py b/newspaper/configuration.py index 301dd179..d5c6d998 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -72,6 +72,7 @@ def __init__(self): self.request_timeout = 7 self.proxies = {} self.number_threads = 10 + self.verify_ssl_cert = True self.verbose = False # for debugging diff --git a/newspaper/network.py b/newspaper/network.py index 3fb448f0..52b80748 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -21,7 +21,7 @@ FAIL_ENCODING = 'ISO-8859-1' -def get_request_kwargs(timeout, useragent, proxies, headers, allow_redirects): +def get_request_kwargs(timeout, useragent, proxies, headers, allow_redirects, verify_ssl_cert): """This Wrapper method exists b/c some values in req_kwargs dict are methods which need to be called every time we make a request """ @@ -30,7 +30,8 @@ def get_request_kwargs(timeout, useragent, proxies, headers, allow_redirects): 'cookies': cj(), 'timeout': timeout, 'allow_redirects': allow_redirects, - 'proxies': proxies + 'proxies': proxies, + 'verify': verify_ssl_cert, } @@ -55,6 +56,7 @@ def get_html_2XX_only(url, config=None, response=None, return_final_url=False): timeout = config.request_timeout proxies = config.proxies headers = config.headers + verify_ssl_cert = config.verify_ssl_cert allow_redirects = config.allow_redirects if response is not None: @@ -64,7 +66,7 @@ def get_html_2XX_only(url, config=None, response=None, return_final_url=False): return html response = requests.get( - url=url, **get_request_kwargs(timeout, useragent, proxies, headers, allow_redirects)) + url=url, **get_request_kwargs(timeout, useragent, proxies, headers, allow_redirects, verify_ssl_cert)) html = _get_html_from_response(response, config) final_url = response.url From cfa7b659a28bcde68307283cb36160cdb3d1ced8 Mon Sep 17 00:00:00 2001 From: Viktor Poluksht Date: Tue, 31 Mar 2026 18:27:00 +0100 Subject: [PATCH 18/19] feat: use og:title if title is absent Signed-off-by: Viktor Poluksht --- newspaper/extractors.py | 22 ++++++++++++---------- tests/unit_tests.py | 4 ++++ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index d1caada0..7b4b74b3 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -254,12 +254,19 @@ def get_title(self, doc): """ title = '' title_element = self.parser.getElementsByTag(doc, tag='title') - # no title found - if title_element is None or len(title_element) == 0: - return title + title_text_fb = ( + self.get_meta_content(doc, 'meta[property="og:title"]') or + self.get_meta_content(doc, 'meta[name="og:title"]') or '' + ) - # title elem found - title_text = self.parser.getText(title_element[0]) + # no title found, fallback to og:title + if title_element is None or len(title_element) == 0: + title_text = title_text_fb + if not title_text: + return title + else: + # title elem found + title_text = self.parser.getText(title_element[0]) used_delimeter = False # title from h1 @@ -281,11 +288,6 @@ def get_title(self, doc): # clean double spaces title_text_h1 = ' '.join([x for x in title_text_h1.split() if x]) - # title from og:title - title_text_fb = ( - self.get_meta_content(doc, 'meta[property="og:title"]') or - self.get_meta_content(doc, 'meta[name="og:title"]') or '') - # create filtered versions of title_text, title_text_h1, title_text_fb # for finer comparison filter_regex = re.compile(r'[^\u4e00-\u9fa5a-zA-Z0-9\ ]') diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 1e867d33..56583171 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -375,6 +375,10 @@ def test_get_title_quotes(self): html = '{}'.format(title) self.assertEqual(self._get_title(html), title) + def test_get_title_fallback_to_og_title_when_title_missing(self): + html = '' + self.assertEqual(self._get_title(html), 'Fallback title from og') + def _get_canonical_link(self, article_url, html): doc = self.parser.fromstring(html) return self.extractor.get_canonical_link(article_url, doc) From 51b75961a58c7d77f12922bae6647a36b9de801d Mon Sep 17 00:00:00 2001 From: rebelchris Date: Thu, 23 Jul 2026 13:53:51 +0000 Subject: [PATCH 19/19] fix(title): pick best when a page has multiple Medium serves two <title> tags on article pages, one of which is a bare site name (<title>Medium). In the non-JS HTML our scraper receives that bare tag can come first, so get_title() picked "Medium" as the post title. Instead of blindly using title_element[0], choose the best candidate among all tags: prefer the one whose filtered text contains og:title, otherwise the longest. Single-title pages are unaffected. --- newspaper/extractors.py | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 7b4b74b3..9fd6cee9 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -235,6 +235,24 @@ def parse_date_str(date_str): return None + def _choose_title_candidate(self, candidates, og_title): + """Pick the best <title> when a document has more than one. + + Prefer a candidate whose filtered text contains the og:title (the + descriptive article title), otherwise fall back to the longest + candidate. This avoids picking bare site names (e.g. Medium's second + <title>Medium). + """ + if len(candidates) == 1: + return candidates[0] + filter_regex = re.compile(r'[^\u4e00-\u9fa5a-zA-Z0-9\ ]') + og_filtered = filter_regex.sub('', og_title or '').lower().strip() + if og_filtered: + for candidate in candidates: + if og_filtered in filter_regex.sub('', candidate).lower(): + return candidate + return max(candidates, key=len) + def get_title(self, doc): """Fetch the article title and analyze it @@ -265,8 +283,20 @@ def get_title(self, doc): if not title_text: return title else: - # title elem found - title_text = self.parser.getText(title_element[0]) + # some sites (e.g. Medium) emit multiple tags, one of which + # is a bare site name ("Medium"). Blindly taking the first element + # can yield the useless site name, so pick the best candidate: + # prefer the one matching og:title, otherwise the longest text. + title_candidates = [self.parser.getText(el).strip() + for el in title_element] + title_candidates = [c for c in title_candidates if c] + if not title_candidates: + title_text = title_text_fb + if not title_text: + return title + else: + title_text = self._choose_title_candidate(title_candidates, + title_text_fb) used_delimeter = False # title from h1