feat: add a fallback to image download fail case

newspaper/images.py

@@ -83,6 +83,37 @@ def clean_url(url):
     return url
 
 
+def get_full_image_dimensions(image_url):
+    """Fallback in case PIL can't open the streamed image
+    """
+    try:
+        response = requests.get(image_url) # No stream=True needed
+        response.raise_for_status() # Raise an exception for bad status codes
+
+        # Use io.BytesIO to treat the response content (bytes) as a file
+        image_bytes = io.BytesIO(response.content)
+
+        # Open the image directly from the bytes stream
+        img = Image.open(image_bytes)
+
+        sz = img.size
+
+        # It's good practice to close the image when done
+        img.close()
+
+        return sz
+
+    except requests.exceptions.RequestException as e:
+        log.warning(f"Method 2 (Direct): Error fetching the image via requests: {e}")
+        return None
+    except FileNotFoundError:
+        log.warning("Method 2 (Direct): Error: io.BytesIO did not behave as expected (treated as file not found).")
+        return None
+    except Exception as e:
+        log.warning(f"Method 2 (Direct): An unexpected error occurred while opening image: {e}")
+        return None
+
+
 def fetch_url(url, useragent, referer=None, retries=1, dimension=False):
     cur_try = 0
     nothing = None if dimension else (None, None)
@@ -143,7 +174,9 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False):
                 if dimension and p.image:
                     return p.image.size
                 elif dimension:
-                    return nothing
+                    # we did read the image, but it failed to parse for some reason
+                    # try to download it in one go
+                    return get_full_image_dimensions(url)
             elif dimension:
                 # expected an image, but didn't get one
                 return nothing

tests/unit_tests.py

@@ -24,7 +24,7 @@
 URLS_FILE = os.path.join(TEST_DIR, 'data', 'fulltext_url_list.txt')
 
 import newspaper
-from newspaper import Article, Config, fulltext, Source, ArticleException, news_pool
+from newspaper import Article, Config, fulltext, Source, ArticleException, news_pool, images
 from newspaper.article import ArticleDownloadState
 from newspaper.configuration import Configuration
 from newspaper.urls import get_domain
@@ -778,6 +778,18 @@ def test_config_ignore_images(self):
         self.assertEqual('https://d33wubrfki0l68.cloudfront.net/77d3013f91800257b3ca2adfb995ae24e49fff4e/b3086/img/main/headshot.jpg', a.top_img)
 
 
+class TestGetImageDimensionFallback(unittest.TestCase):
+
+    @print_test
+    def test_get_image_dimension_fallback(self):
+        config = Config()
+        config.image_dimension_ration = 32 / 9
+        a = Article('https://appwrite.io/blog/post/add-figma-oauth2-appwrite', config=config)
+        s = images.Scraper(a)
+        sr = s.satisfies_requirements('https://appwrite.io/images/blog/add-figma-oauth2-appwrite/cover.png')
+        self.assertTrue(sr)
+
+
 if __name__ == '__main__':
     argv = list(sys.argv)
     if 'fulltext' in argv: