+
+
+
\ No newline at end of file
diff --git a/tests/data/text/thai.txt b/tests/data/text/thai.txt
new file mode 100644
index 00000000..b2ce780d
--- /dev/null
+++ b/tests/data/text/thai.txt
@@ -0,0 +1,15 @@
+21 ม.ค.2561 เวลาประมาณ 12.00 น. ผู้สื่อข่าวได้รับแจ้งจากนาย ต.(ไม่ประสงค์เปิดเผยชื่อ) บุตรชายของ 'ภูชนะ' คนสนิทของนายสุรชัย แซ่ด่าน ที่หายไปว่า ผลการตรวจ DNA โดยใช้เนื้อเยื่อของศพที่ถูกสังหารด้วยการมัดแขน รัดคอ ทุบจนใบหน้าเละ และท้องถูกผ่ายัดเสาปูนที่ลอยมาติดที่ อ.ธาตุพนม จ.นครพนม นั้น เมื่อตรวจสอบแล้วมีความเกี่ยวพันทางสายเลือดกับนาย ต.จริง
+
+ภูชนะ (นามแฝง) เป็นคนใกล้ชิดของนายสุรชัย ด่านวัฒนานุสรณ์ หรือ สุรชัย แซ่ด่าน นักเคลื่อนไหวทางการเมืองที่ลี้ภัยออกจากประเทศไทยและได้หายตัวไปจากที่พักในประเทศเพื่อนบ้านในช่วงคืนวันที่ 12-13 ธ.ค.2561 พร้อมกันกับสุรชัยและคนสนิทอีกคนหนึ่ง
+
+ปรานี ด่านวัฒนานุสรณ์ ภรรยาของสุรชัยกล่าวว่า ได้ทราบข่าวจากบุตรชายของภูชนะแล้ว แต่ยังไม่ขอพูดอะไร โดยเบื้องต้นได้ทำใจแต่แรกแล้วว่าเหตุการณ์ลักษณะนี้จะต้องเกิดขึ้นสักวันหนึ่ง
+
+ขณะที่ นาย ว. พี่เขยของ 'กาสะลอง' อีกหนึ่งผู้ลี้ภัยที่ได้หายไปพร้อมกับนายสุรชัยกล่าวกับผู้สื่อข่าวว่า หลังจากได้ทราบผลการตรวจสอบ DNA จากลูกชายของภูชนะ ทางญาติของกาสะลองก็ได้ติดต่อไปที่พนักงานสอบสวนเจ้าของคดีและได้คำตอบว่า จะได้ทราบผลการตรวจสอบ DNA ภายในเวลา 2-3 วันนี้
+
+สุรชัย (78 ปี) ภูชนะ (54 ปี) กาสะลอง (47 ปี) เป็นนักเคลื่อนไหวทางการเมือง และเป็นผู้ลี้ภัยจากเหตุการณ์รัฐประหาร 2557 ไปยังประเทศเพื่อนบ้านได้หายออกจากที่พักในประเทศเพื่อนบ้านโดยที่ไม่มีใครสามารถติดต่อได้จนปัจจุบันนับเป็นเวลานาน 1 เดือน กับอีก 10 วัน
+
+เรื่องที่เกี่ยวข้อง
+
+ลืออุ้ม อ.สุรชัย ผู้ลี้ภัยหายจากที่พัก 12 วันแล้ว
+
+ตร.เร่งหาเบาะแสคดีฆ่ายัดเสาถ่วงน้ำโขง - เมียไม่เชื่อเป็น 'สุรชัย แซ่ด่าน'
\ No newline at end of file
diff --git a/tests/unit_tests.py b/tests/unit_tests.py
index 01140109..058a6deb 100644
--- a/tests/unit_tests.py
+++ b/tests/unit_tests.py
@@ -730,6 +730,17 @@ def test_japanese_fulltext_extract2(self):
self.assertEqual(text, article.text)
self.assertEqual(text, fulltext(article.html, 'ja'))
+ @print_test
+ def test_thai_fulltext_extract(self):
+ url = 'https://prachatai.com/journal/2019/01/80642'
+ article = Article(url=url, language='th')
+ html = mock_resource_with('thai_article', 'html')
+ article.download(html)
+ article.parse()
+ text = mock_resource_with('thai', 'txt')
+ self.assertEqual(text, article.text)
+ self.assertEqual(text, fulltext(article.html, 'th'))
+
class TestNewspaperLanguagesApi(unittest.TestCase):
@print_test
From 069a4379206a2cec087fe1c45999ee8e69aa023d Mon Sep 17 00:00:00 2001
From: Guy Rosin
Date: Sun, 17 Mar 2019 02:56:20 +0200
Subject: [PATCH 26/59] Update extractors.py (#688)
Added a date tag
---
newspaper/extractors.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/newspaper/extractors.py b/newspaper/extractors.py
index c097df2a..96255401 100644
--- a/newspaper/extractors.py
+++ b/newspaper/extractors.py
@@ -216,6 +216,8 @@ def parse_date_str(date_str):
'content': 'content'},
{'attribute': 'pubdate', 'value': 'pubdate',
'content': 'datetime'},
+ {'attribute': 'name', 'value': 'publish_date',
+ 'content': 'content'},
]
for known_meta_tag in PUBLISH_DATE_TAGS:
meta_tags = self.parser.getElementsByTag(
From c258db1e540bf7c86a1cee56c5cb2b56c1f4f68d Mon Sep 17 00:00:00 2001
From: Akash Nidhi P S
Date: Sun, 17 Mar 2019 06:28:27 +0530
Subject: [PATCH 27/59] Added more stopwords for stopwords-hi.txt (#675)
---
newspaper/resources/text/stopwords-hi.txt | 220 ++++++++++++++++++++--
1 file changed, 207 insertions(+), 13 deletions(-)
diff --git a/newspaper/resources/text/stopwords-hi.txt b/newspaper/resources/text/stopwords-hi.txt
index d02051a8..2cf52d70 100644
--- a/newspaper/resources/text/stopwords-hi.txt
+++ b/newspaper/resources/text/stopwords-hi.txt
@@ -1,3 +1,209 @@
+अंदर
+अत
+अदि
+अप
+अपना
+अपनि
+अपनी
+अपने
+अभि
+अभी
+आदि
+इंहिं
+इंहें
+इंहों
+इतयादि
+इत्यादि
+इन
+इनका
+इन्हीं
+इन्हें
+इन्हों
+इस
+इसका
+इसकि
+इसकी
+इसके
+इसमें
+इसि
+इसी
+इसे
+उंहिं
+उंहें
+उंहों
+उन
+उनका
+उनकि
+उनकी
+उनको
+उन्हीं
+उन्हें
+उन्हों
+उस
+उसके
+उसि
+उसी
+एक
+एवं
+एस
+एसे
+ऐसे
+ओर
+कइ
+कई
+करता
+करते
+करना
+करने
+करें
+कहते
+कहा
+का
+काफि
+काफ़ी
+कि
+किंहें
+किंहों
+कितना
+किन्हें
+किन्हों
+किया
+किर
+किस
+किसि
+किसी
+किसे
+की
+कुछ
+कुल
+के
+कोइ
+कोई
+कोन
+कोनसा
+कौनसा
+गया
+घर
+जब
+जहाँ
+जहां
+जा
+जिंहें
+जिंहों
+जितना
+जिधर
+जिन
+जिन्हें
+जिन्हों
+जिस
+जिसे
+जीधर
+जेसा
+जेसे
+जैसा
+जो
+तक
+तब
+तरह
+तिंहें
+तिंहों
+तिन
+तिन्हें
+तिन्हों
+तिस
+तिसे
+थि
+थी
+थे
+दबारा
+दवारा
+दिया
+दुसरा
+दुसरे
+दूसरे
+दो
+द्वारा
+न
+नहिं
+नहीं
+ना
+निचे
+निहायत
+ने
+पहले
+पुरा
+पूरा
+पे
+फिर
+बनि
+बनी
+बहि
+बही
+बहुत
+बाद
+बाला
+बिलकुल
+भि
+भितर
+भी
+भीतर
+मगर
+मानो
+मे
+यदि
+यहाँ
+यहां
+यहि
+यही
+या
+यिह
+ये
+रखें
+रवासा
+रहा
+रहे
+ऱ्वासा
+लिए
+लिये
+लेकिन
+व
+वगेरह
+वरग
+वर्ग
+वहां
+वहिं
+वहीं
+वाले
+वुह
+वे
+वग़ैरह
+संग
+सकता
+सबसे
+सभि
+सभी
+साथ
+साबुत
+साभ
+सारा
+से
+सो
+हि
+ही
+हुअ
+हुआ
+हुइ
+हुई
+हुए
+हे
+हें
+हो
+होता
+होति
+होती
+होते
+होना
+होने
को
नीचे
सब
@@ -6,7 +212,6 @@
तो
कौन
यह
-एक
और
वहाँ
था
@@ -22,22 +227,11 @@
सकते
हैं
मुझे
-एक
-साथ
-मुझे
-अभी
-तक
अब
-या
हमें
-भी
-को
-से
-था
क्या
हम
करेगा
-वे
-कहा
इतना
होगा
+
From 2788a2fdcdbca6b1c153ebea72f931fba945afad Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Sun, 7 Apr 2019 16:00:31 -0400
Subject: [PATCH 28/59] Replace patreon with consulting
---
README.rst | 18 ++++++------------
1 file changed, 6 insertions(+), 12 deletions(-)
diff --git a/README.rst b/README.rst
index ef5dabc6..9f04726a 100644
--- a/README.rst
+++ b/README.rst
@@ -298,18 +298,14 @@ NOTE: You will still most likely need to install the following libraries via you
$ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3
-Support Newspaper3K financially
--------------------------------
-
-Help support me via Patreon so I can dedicate more of my time towards Newspaper3K! I plan
-on adding things like advanced machine learned based techniques for news, image, and full-text
-extraction. We even have lofty long-term goals such as tackling the fake news problem!
-`View our Patreon support page here`_
+Consulting
+----------
*This service is already used around the world* by startups, top news organizations (CNN, NYT, etc),
-graduate school researchers, and, of course, hackers like you :) Perhaps one day we can even tackle and be
-at the forefront of the news intelligence domain; tackling new problems such as fake news, nlp, author
-credibility. Your support goes a long way, thank you.
+graduate school researchers, and, of course, hackers like you :) If you or your company are interested
+in more advanced features like: increased NLP and scraping accuracy, mis-information, fake news, author
+credibility, boosted coverage and accuracy for your use case, and etc; feel free to contact me for
+consulting.
Development
-----------
@@ -359,8 +355,6 @@ to talk about the future of this library and news extraction in general!
.. _`python-goose's`: https://github.com/grangier/python-goose
.. _`here`: https://github.com/codelucas/newspaper/blob/master/GOOSE-LICENSE.txt
-.. _`View our Patreon support page here`: https://www.patreon.com/codelucas
-
.. _`Quickstart guide`: https://newspaper.readthedocs.io/en/latest/
.. _`The Documentation`: https://newspaper.readthedocs.io
.. _`lxml`: http://lxml.de/
From cf85a7eadf4ae26ca54c95afd8f404edaa4795ac Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Sun, 7 Apr 2019 16:31:52 -0400
Subject: [PATCH 29/59] Modify readme
---
README.rst | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/README.rst b/README.rst
index 9f04726a..aa62a1f0 100644
--- a/README.rst
+++ b/README.rst
@@ -304,8 +304,8 @@ Consulting
*This service is already used around the world* by startups, top news organizations (CNN, NYT, etc),
graduate school researchers, and, of course, hackers like you :) If you or your company are interested
in more advanced features like: increased NLP and scraping accuracy, mis-information, fake news, author
-credibility, boosted coverage and accuracy for your use case, and etc; feel free to contact me for
-consulting.
+credibility, boosted coverage and accuracy for your use case, and etc; feel free to `email & contact me`_
+for consulting.
Development
-----------
From 9b89046d076b244fef006aee1a485b1f6e537bdf Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Fri, 12 Apr 2019 08:23:13 -0400
Subject: [PATCH 30/59] Add donations links in readme
---
README.rst | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/README.rst b/README.rst
index aa62a1f0..8908df6d 100644
--- a/README.rst
+++ b/README.rst
@@ -307,6 +307,16 @@ in more advanced features like: increased NLP and scraping accuracy, mis-informa
credibility, boosted coverage and accuracy for your use case, and etc; feel free to `email & contact me`_
for consulting.
+Donations
+---------
+
+Your donations are greatly appreciated! They will free me up to work on this project more,
+to take on things like: adding new features, bug-fix support, addressing concerns with the library.
+
+- My PayPal link: `https://www.paypal.me/codelucas`_
+- My `Venmo`_ handle: @Lucas-Ou-Yang
+
+
Development
-----------
@@ -355,6 +365,9 @@ to talk about the future of this library and news extraction in general!
.. _`python-goose's`: https://github.com/grangier/python-goose
.. _`here`: https://github.com/codelucas/newspaper/blob/master/GOOSE-LICENSE.txt
+.. _`https://www.paypal.me/codelucas`: https://www.paypal.me/codelucas
+.. _`Venmo`: https://www.venmo.com/Lucas-Ou-Yang
+
.. _`Quickstart guide`: https://newspaper.readthedocs.io/en/latest/
.. _`The Documentation`: https://newspaper.readthedocs.io
.. _`lxml`: http://lxml.de/
From 1c7feb1c5524c70ce6038aa86c454d8a18f78d14 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Sat, 20 Jun 2020 11:59:42 -0700
Subject: [PATCH 31/59] Update README, gitad setup
---
README.rst | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
diff --git a/README.rst b/README.rst
index 8908df6d..a0f2ca85 100644
--- a/README.rst
+++ b/README.rst
@@ -298,14 +298,9 @@ NOTE: You will still most likely need to install the following libraries via you
$ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3
-Consulting
-----------
-*This service is already used around the world* by startups, top news organizations (CNN, NYT, etc),
-graduate school researchers, and, of course, hackers like you :) If you or your company are interested
-in more advanced features like: increased NLP and scraping accuracy, mis-information, fake news, author
-credibility, boosted coverage and accuracy for your use case, and etc; feel free to `email & contact me`_
-for consulting.
+.. image:: https://lh3.googleusercontent.com/7qtWmNNg3NDOf1pIuCaWbUJ2TtnMgVveXs-nkf4Ikoq0iIZcu9fevPpykFTCP8M8adnUVvPqBqyqGvmgAfA_tB-r7Q1U8ERIvPi3Xjds_lzdLBUcNO609bfbOibU0oIBHuv82f07lTB-SVzfctkG4w0Vnjw89QLMWjadnBnOu7mH0mZ6P3zNXPdLClTcvRKGI3K2F0ZATNMO2UfewcHpR8jI75066IItlFBS6NUUl0BRiUaFUCKOBqR79DULKZm5JFSx2-n73fPzpYRWwViHHDaA9GDKy55pBqq-JhpKwdlg6MUtHMWTh1XmYRCum5XEzzauB3j8qkqsizi-q3oaXCOH7kgrdm0misoV83eqa9CrC8lC83-2qb3hYgPzVmnTeX2VEnwwu8A6zwpvIn-VzufvOONa9OXK08UKMshtlR3Xy8toNujqa4hdq2JXsh_ux6NhgvwJ38OG_pIADzw0PR6rGLLi1_HZ97vE2e54oIz-GqU154ZZojEQ4Jqbf2uCjK3F83pfOC4EHnuJgI59lmhx14Guz8Br4KWYfrOMG37KZwFC-xxLSI-bzNs6eAtaB-JqryYENV8lDOX9wrCfTh0VByZNOW8SyZyAOfMB5HhdwNOotn35KHrrfKIgV_Se0gvJlI9s16BVufwBUFv7fnLZ_DfMx3j_Vn-OCJcZUTqc6Iew4Eu0iBmFvEsgYQs7E8Yz=w3360-h1832-ft
+ :target: https://tracking.gitads.io/?campaign=gitads&repo=newspaper&redirect=gitads.io
Donations
---------
From a0f725333af7a9abad996b1f49976006a6a93c7c Mon Sep 17 00:00:00 2001
From: Kyle Jones
Date: Mon, 22 Jun 2020 21:38:44 +0100
Subject: [PATCH 32/59] Dropping python 3.4 support (#768)
* Dropping python 3.4 support
* Fixing build issues
* Changed version number - incremented major version due to breaking change
* Removing pandas dependency
---
.travis.yml | 2 +-
newspaper/version.py | 2 +-
requirements.txt | 2 +-
setup.py | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index f5cd2e33..8af415d8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,8 +1,8 @@
language: python
python:
- - "3.4"
- "3.5"
- "3.6"
+ - "3.7"
install:
- pip install -r requirements.txt coverage coveralls
- python download_corpora.py
diff --git a/newspaper/version.py b/newspaper/version.py
index f89d0d3e..e2eab31a 100644
--- a/newspaper/version.py
+++ b/newspaper/version.py
@@ -7,5 +7,5 @@
__license__ = 'MIT'
__copyright__ = 'Copyright 2014, Lucas Ou-Yang'
-version_info = (0, 2, 8)
+version_info = (0, 3, 0)
__version__ = ".".join(map(str, version_info))
diff --git a/requirements.txt b/requirements.txt
index 9d5c744e..61974601 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -11,4 +11,4 @@ python-dateutil>=2.5.3
PyYAML>=3.11
requests>=2.10.0
tinysegmenter==0.3 # TODO(codelucas): Investigate making this >=0.3
-tldextract>=2.0.1
+tldextract>=2.0.1
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 91dc24fd..5569c7cb 100755
--- a/setup.py
+++ b/setup.py
@@ -47,7 +47,7 @@
setup(
name='newspaper3k',
- version='0.2.8',
+ version='0.3.0',
description='Simplified python article discovery & extraction.',
long_description=readme,
author='Lucas Ou-Yang',
From cba065801162011c134883c841f37325546eefea Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Mon, 22 Jun 2020 16:35:16 -0700
Subject: [PATCH 33/59] Update README.rst
---
README.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.rst b/README.rst
index a0f2ca85..34243c94 100644
--- a/README.rst
+++ b/README.rst
@@ -299,7 +299,7 @@ NOTE: You will still most likely need to install the following libraries via you
$ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3
-.. image:: https://lh3.googleusercontent.com/7qtWmNNg3NDOf1pIuCaWbUJ2TtnMgVveXs-nkf4Ikoq0iIZcu9fevPpykFTCP8M8adnUVvPqBqyqGvmgAfA_tB-r7Q1U8ERIvPi3Xjds_lzdLBUcNO609bfbOibU0oIBHuv82f07lTB-SVzfctkG4w0Vnjw89QLMWjadnBnOu7mH0mZ6P3zNXPdLClTcvRKGI3K2F0ZATNMO2UfewcHpR8jI75066IItlFBS6NUUl0BRiUaFUCKOBqR79DULKZm5JFSx2-n73fPzpYRWwViHHDaA9GDKy55pBqq-JhpKwdlg6MUtHMWTh1XmYRCum5XEzzauB3j8qkqsizi-q3oaXCOH7kgrdm0misoV83eqa9CrC8lC83-2qb3hYgPzVmnTeX2VEnwwu8A6zwpvIn-VzufvOONa9OXK08UKMshtlR3Xy8toNujqa4hdq2JXsh_ux6NhgvwJ38OG_pIADzw0PR6rGLLi1_HZ97vE2e54oIz-GqU154ZZojEQ4Jqbf2uCjK3F83pfOC4EHnuJgI59lmhx14Guz8Br4KWYfrOMG37KZwFC-xxLSI-bzNs6eAtaB-JqryYENV8lDOX9wrCfTh0VByZNOW8SyZyAOfMB5HhdwNOotn35KHrrfKIgV_Se0gvJlI9s16BVufwBUFv7fnLZ_DfMx3j_Vn-OCJcZUTqc6Iew4Eu0iBmFvEsgYQs7E8Yz=w3360-h1832-ft
+.. image:: https://images.gitads.io/newspaper
:target: https://tracking.gitads.io/?campaign=gitads&repo=newspaper&redirect=gitads.io
Donations
From 56de65af9efbfea6293c82c0b1821e2ca9fbddaa Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Mon, 22 Jun 2020 16:36:27 -0700
Subject: [PATCH 34/59] Update README.rst
---
README.rst | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/README.rst b/README.rst
index 34243c94..dc122100 100644
--- a/README.rst
+++ b/README.rst
@@ -302,15 +302,6 @@ NOTE: You will still most likely need to install the following libraries via you
.. image:: https://images.gitads.io/newspaper
:target: https://tracking.gitads.io/?campaign=gitads&repo=newspaper&redirect=gitads.io
-Donations
----------
-
-Your donations are greatly appreciated! They will free me up to work on this project more,
-to take on things like: adding new features, bug-fix support, addressing concerns with the library.
-
-- My PayPal link: `https://www.paypal.me/codelucas`_
-- My `Venmo`_ handle: @Lucas-Ou-Yang
-
Development
-----------
@@ -342,6 +333,15 @@ View a working online demo here: http://newspaper-demo.herokuapp.com
This is another working online demo: http://newspaper.chinazt.cc/
+Donations
+---------
+
+Your donations are greatly appreciated! They will free me up to work on this project more,
+to take on things like: adding new features, bug-fix support, addressing concerns with the library.
+
+- My PayPal link: `https://www.paypal.me/codelucas`_
+- My `Venmo`_ handle: @Lucas-Ou-Yang
+
LICENSE
-------
From 837bd13e96083074d8daf99e1457d7d5482ada67 Mon Sep 17 00:00:00 2001
From: Bachstelze
Date: Fri, 26 Jun 2020 07:47:35 +0200
Subject: [PATCH 35/59] changed 404 url (#819)
---
tests/unit_tests.py | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/tests/unit_tests.py b/tests/unit_tests.py
index 058a6deb..69c05adf 100644
--- a/tests/unit_tests.py
+++ b/tests/unit_tests.py
@@ -753,8 +753,7 @@ class TestDownloadPdf(unittest.TestCase):
@print_test
def test_article_pdf_ignoring(self):
empty_pdf = "%PDF-" # empty PDF constant
- a = Article(url='http://www.technik-medien.at/ePaper_Download/'
- 'IoT4Industry+Business_2018-10-31_2018-03.pdf',
+ a = Article(url='https://www.adobe.com/pdf/pdfs/ISO32000-1PublicPatentLicense.pdf',
ignored_content_types_defaults={"application/pdf": empty_pdf,
"application/x-pdf": empty_pdf,
"application/x-bzpdf": empty_pdf,
From 1c27e6da19a7b5d9b885eee0584dffd3759bb934 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Sat, 27 Jun 2020 19:49:08 -0700
Subject: [PATCH 36/59] Update README.rst
Project support
---
README.rst | 3 +++
1 file changed, 3 insertions(+)
diff --git a/README.rst b/README.rst
index dc122100..34e3cffb 100644
--- a/README.rst
+++ b/README.rst
@@ -298,6 +298,9 @@ NOTE: You will still most likely need to install the following libraries via you
$ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3
+Your support
+------------
+You can support the project easily by checking out our sponsor page. It takes only one click:
.. image:: https://images.gitads.io/newspaper
:target: https://tracking.gitads.io/?campaign=gitads&repo=newspaper&redirect=gitads.io
From db81b55aabec735041359d13951b868b68806e61 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Fri, 3 Jul 2020 12:04:24 -0700
Subject: [PATCH 37/59] Update README.rst
---
README.rst | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/README.rst b/README.rst
index 34e3cffb..40ef5048 100644
--- a/README.rst
+++ b/README.rst
@@ -300,7 +300,7 @@ NOTE: You will still most likely need to install the following libraries via you
Your support
------------
-You can support the project easily by checking out our sponsor page. It takes only one click:
+You can support the project by checking out our sponsor. `It takes only one click`_
.. image:: https://images.gitads.io/newspaper
:target: https://tracking.gitads.io/?campaign=gitads&repo=newspaper&redirect=gitads.io
@@ -371,3 +371,4 @@ to talk about the future of this library and news extraction in general!
.. _`lxml`: http://lxml.de/
.. _`requests`: https://github.com/kennethreitz/requests
.. _`Parse.ly`: http://parse.ly
+.. _`It takes only one click`: https://tracking.gitads.io/?campaign=gitads&repo=newspaper&redirect=gitads.io
From 2f6ca8fa635541fc9c7562559725737451fe3578 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Fri, 3 Jul 2020 12:09:51 -0700
Subject: [PATCH 38/59] Update README.rst
---
README.rst | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/README.rst b/README.rst
index 40ef5048..ce75c90c 100644
--- a/README.rst
+++ b/README.rst
@@ -163,10 +163,14 @@ If you are certain that an *entire* news source is in one language, **go ahead a
两年双免0手续0利率 科鲁兹掀背金融轻松购_武汉车市_武汉汽
车网_新浪汽车_新浪网
-Documentation
--------------
+Support our library
+-------------------
+`It takes only one click`_
-Check out `The Documentation`_ for full and detailed guides using newspaper.
+Docs
+----
+
+Check out `The Docs`_ for full and detailed guides using newspaper.
Interested in adding a new language for us? Refer to: `Docs - Adding new languages `_
@@ -298,9 +302,6 @@ NOTE: You will still most likely need to install the following libraries via you
$ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3
-Your support
-------------
-You can support the project by checking out our sponsor. `It takes only one click`_
.. image:: https://images.gitads.io/newspaper
:target: https://tracking.gitads.io/?campaign=gitads&repo=newspaper&redirect=gitads.io
From 4b35117e7e055e9eff6d18fb3e68827a6cab27a8 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Fri, 3 Jul 2020 12:10:45 -0700
Subject: [PATCH 39/59] Update README.rst
---
README.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.rst b/README.rst
index ce75c90c..c8d4e9af 100644
--- a/README.rst
+++ b/README.rst
@@ -368,7 +368,7 @@ to talk about the future of this library and news extraction in general!
.. _`Venmo`: https://www.venmo.com/Lucas-Ou-Yang
.. _`Quickstart guide`: https://newspaper.readthedocs.io/en/latest/
-.. _`The Documentation`: https://newspaper.readthedocs.io
+.. _`The Docs`: https://newspaper.readthedocs.io
.. _`lxml`: http://lxml.de/
.. _`requests`: https://github.com/kennethreitz/requests
.. _`Parse.ly`: http://parse.ly
From b0cc1278c40829ce7156f2e2901f7fe77af21e3f Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Sat, 4 Jul 2020 19:34:46 -0700
Subject: [PATCH 40/59] Update README.rst
---
README.rst | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/README.rst b/README.rst
index c8d4e9af..0b88c893 100644
--- a/README.rst
+++ b/README.rst
@@ -1,3 +1,8 @@
+Newspaper is being sponsored by the following tool; please help to support us by taking a look!
+
+.. image:: https://images.gitads.io/newspaper
+ :target: https://tracking.gitads.io/?campaign=gitads&repo=newspaper&redirect=gitads.io
+
Newspaper3k: Article scraping & curation
========================================
@@ -303,10 +308,6 @@ NOTE: You will still most likely need to install the following libraries via you
$ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3
-.. image:: https://images.gitads.io/newspaper
- :target: https://tracking.gitads.io/?campaign=gitads&repo=newspaper&redirect=gitads.io
-
-
Development
-----------
From 5af1bea20f394bb511689ed33f5b4accf83ab19d Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Sun, 12 Jul 2020 18:16:14 -0700
Subject: [PATCH 41/59] Update README.rst
---
README.rst | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/README.rst b/README.rst
index 0b88c893..7afb3572 100644
--- a/README.rst
+++ b/README.rst
@@ -1,7 +1,7 @@
-Newspaper is being sponsored by the following tool; please help to support us by taking a look!
+Newspaper is being sponsored by the following tool; please help to support us by taking a look and signing up for a free trial.
.. image:: https://images.gitads.io/newspaper
- :target: https://tracking.gitads.io/?campaign=gitads&repo=newspaper&redirect=gitads.io
+ :target: https://tracking.gitads.io/?repo=newspaper
Newspaper3k: Article scraping & curation
========================================
From f622011177f6c2e95e48d6076561e21c016f08c3 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Tue, 1 Sep 2020 23:54:25 -0700
Subject: [PATCH 42/59] Update README.rst
---
README.rst | 22 ++++++++--------------
1 file changed, 8 insertions(+), 14 deletions(-)
diff --git a/README.rst b/README.rst
index 7afb3572..21373ba7 100644
--- a/README.rst
+++ b/README.rst
@@ -1,8 +1,3 @@
-Newspaper is being sponsored by the following tool; please help to support us by taking a look and signing up for a free trial.
-
-.. image:: https://images.gitads.io/newspaper
- :target: https://tracking.gitads.io/?repo=newspaper
-
Newspaper3k: Article scraping & curation
========================================
@@ -307,6 +302,14 @@ NOTE: You will still most likely need to install the following libraries via you
$ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3
+Donations
+---------
+
+Your donations are greatly appreciated! They will free me up to work on this project more,
+to take on things like: adding new features, bug-fix support, addressing concerns with the library.
+
+- My PayPal link: `https://www.paypal.me/codelucas`_
+- My `Venmo`_ handle: @Lucas-Ou-Yang
Development
-----------
@@ -338,15 +341,6 @@ View a working online demo here: http://newspaper-demo.herokuapp.com
This is another working online demo: http://newspaper.chinazt.cc/
-Donations
----------
-
-Your donations are greatly appreciated! They will free me up to work on this project more,
-to take on things like: adding new features, bug-fix support, addressing concerns with the library.
-
-- My PayPal link: `https://www.paypal.me/codelucas`_
-- My `Venmo`_ handle: @Lucas-Ou-Yang
-
LICENSE
-------
From ba8d2f41be9618a6f1112355082c892d3c3e1177 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Thu, 6 Mar 2025 17:44:47 -0800
Subject: [PATCH 43/59] Update README.rst
---
README.rst | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/README.rst b/README.rst
index 21373ba7..0b25308e 100644
--- a/README.rst
+++ b/README.rst
@@ -163,9 +163,18 @@ If you are certain that an *entire* news source is in one language, **go ahead a
两年双免0手续0利率 科鲁兹掀背金融轻松购_武汉车市_武汉汽
车网_新浪汽车_新浪网
-Support our library
--------------------
-`It takes only one click`_
+
+
+Enjoy Newspaper3k? You'll also love BrightData
+----------------------------------------------
+`Click here to see BrightData`_, the premier company offering the internet's most trusted web data.
+They have AI-powered web scrapers and datasets you may find useful.
+
+.. image:: https://github.com/user-attachments/assets/ca540b8b-7888-49d2-89df-2c07760f6d9f
+ :target: https://get.brightdata.com/hm89s0nowetp
+ :alt: Bright Data's Scraping Browser Product
+
+.. _`Click here to see BrightData`: https://get.brightdata.com/hm89s0nowetp
Docs
----
From b39a4b407fd322eb4d256d29170ad69a0a39eb7a Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Thu, 10 Jul 2025 23:25:16 -0700
Subject: [PATCH 44/59] Update README.rst include sponsors
---
README.rst | 38 ++++++++++++++++++++------------------
1 file changed, 20 insertions(+), 18 deletions(-)
diff --git a/README.rst b/README.rst
index 0b25308e..1d0f7b5b 100644
--- a/README.rst
+++ b/README.rst
@@ -164,17 +164,16 @@ If you are certain that an *entire* news source is in one language, **go ahead a
车网_新浪汽车_新浪网
+Enjoy Newspaper3k?
+------------------
+`Click here to see SerpAPI`_, scrape search engines easily with SerpAPI - Search API.
+They scrape Google Search, Google News, Google Maps, and more!
-Enjoy Newspaper3k? You'll also love BrightData
-----------------------------------------------
-`Click here to see BrightData`_, the premier company offering the internet's most trusted web data.
-They have AI-powered web scrapers and datasets you may find useful.
+.. image:: https://github.com/user-attachments/assets/9a80eeb4-72a8-43f1-9413-93c7a47b2bf6
+ :target: https://serpapi.com/google-news-api?utm_source=newspaper3k_github
+ :alt: Scrape search engines easily with SerpAPI - Search API.
-.. image:: https://github.com/user-attachments/assets/ca540b8b-7888-49d2-89df-2c07760f6d9f
- :target: https://get.brightdata.com/hm89s0nowetp
- :alt: Bright Data's Scraping Browser Product
-
-.. _`Click here to see BrightData`: https://get.brightdata.com/hm89s0nowetp
+.. _`Click here to see SerpAPI`: https://serpapi.com?utm_source=newspaper3k_github
Docs
----
@@ -245,6 +244,18 @@ Features
zh Chinese
+BrightData offers top quality web scraping with little effort!
+--------------------------------------------------------------
+`Click here to see BrightData`_, the premier company offering the internet's most trusted web data.
+They have AI-powered web scrapers and datasets you may find useful.
+
+.. image:: https://github.com/user-attachments/assets/ca540b8b-7888-49d2-89df-2c07760f6d9f
+ :target: https://get.brightdata.com/hm89s0nowetp
+ :alt: Bright Data's Scraping Browser Product
+
+.. _`Click here to see BrightData`: https://get.brightdata.com/hm89s0nowetp
+
+
Get it now
----------
@@ -311,15 +322,6 @@ NOTE: You will still most likely need to install the following libraries via you
$ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3
-Donations
----------
-
-Your donations are greatly appreciated! They will free me up to work on this project more,
-to take on things like: adding new features, bug-fix support, addressing concerns with the library.
-
-- My PayPal link: `https://www.paypal.me/codelucas`_
-- My `Venmo`_ handle: @Lucas-Ou-Yang
-
Development
-----------
From 1b5fce18d4ce8a75ae7f6da26b874354bd9ce243 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Thu, 10 Jul 2025 23:35:11 -0700
Subject: [PATCH 45/59] Update README.rst - more sponsors
---
README.rst | 15 ++++++++++-----
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git a/README.rst b/README.rst
index 1d0f7b5b..774c116c 100644
--- a/README.rst
+++ b/README.rst
@@ -244,17 +244,22 @@ Features
zh Chinese
-BrightData offers top quality web scraping with little effort!
---------------------------------------------------------------
-`Click here to see BrightData`_, the premier company offering the internet's most trusted web data.
-They have AI-powered web scrapers and datasets you may find useful.
+BrightData: Scrape Any Website, at Scale, Without Getting Blocked
+-----------------------------------------------------------------
+👉 `Get instant access to BrightData’s AI-powered web scraping tools and datasets.`_
+
+* ✅ AI-powered scraping browser
+* ✅ 72M+ rotating IPs
+* ✅ Zero bans, faster extraction
+* `👉 Get access now.`_
.. image:: https://github.com/user-attachments/assets/ca540b8b-7888-49d2-89df-2c07760f6d9f
:target: https://get.brightdata.com/hm89s0nowetp
:alt: Bright Data's Scraping Browser Product
-.. _`Click here to see BrightData`: https://get.brightdata.com/hm89s0nowetp
+.. _`Get instant access to BrightData’s AI-powered web scraping tools and datasets.`: https://get.brightdata.com/hm89s0nowetp
+.. _`👉 Get access now.`: https://get.brightdata.com/hm89s0nowetp
Get it now
----------
From bde5d50f5de80dd0beaa6ed3044b07a6f3888417 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Fri, 11 Jul 2025 07:52:35 -0700
Subject: [PATCH 46/59] Update README.rst - sponsors
---
README.rst | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/README.rst b/README.rst
index 774c116c..b1c05e53 100644
--- a/README.rst
+++ b/README.rst
@@ -166,14 +166,14 @@ If you are certain that an *entire* news source is in one language, **go ahead a
Enjoy Newspaper3k?
------------------
-`Click here to see SerpAPI`_, scrape search engines easily with SerpAPI - Search API.
-They scrape Google Search, Google News, Google Maps, and more!
+`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`_.
+Scrape Google Search, Google News, Google Maps, and more!
.. image:: https://github.com/user-attachments/assets/9a80eeb4-72a8-43f1-9413-93c7a47b2bf6
:target: https://serpapi.com/google-news-api?utm_source=newspaper3k_github
- :alt: Scrape search engines easily with SerpAPI - Search API.
+ :alt: Scrape search engines easily with SerpApi - Search API.
-.. _`Click here to see SerpAPI`: https://serpapi.com?utm_source=newspaper3k_github
+.. _`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`: https://serpapi.com?utm_source=newspaper3k_github
Docs
----
From 6c1dcddd3af2373dfe17f9646558e60e2d74688d Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Thu, 14 Aug 2025 15:56:11 +0700
Subject: [PATCH 47/59] Update README.rst
---
README.rst | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/README.rst b/README.rst
index b1c05e53..04d56692 100644
--- a/README.rst
+++ b/README.rst
@@ -175,6 +175,18 @@ Scrape Google Search, Google News, Google Maps, and more!
.. _`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`: https://serpapi.com?utm_source=newspaper3k_github
+Interested in proxies?
+----------------------
+
+`Click here to explore BestProxy`_, your go-to solution for premium residential proxies. BestProxy's proxies ensure smooth browsing, fast speeds, and total anonymity. `Get Started`_ today and experience the difference!
+
+.. image:: https://github.com/user-attachments/assets/1c6ef38c-f0c0-4db0-aad2-3ed9d6adf0b5
+ :target: https://bestproxy.com/?keyword=b2vgzl0r
+ :alt: Experience BestProxy, smooth browsing, fast speeds, and total anonymity.
+
+.. _`Click here to explore BestProxy`: https://bestproxy.com/?keyword=b2vgzl0r
+.. _`Get Started`: https://bestproxy.com/?keyword=b2vgzl0r
+
Docs
----
From 8901de7451320ce616aea613ab91e81e7cedd289 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Tue, 14 Oct 2025 22:41:54 +0700
Subject: [PATCH 48/59] remove bd from readme ads
---
README.rst | 18 ------------------
1 file changed, 18 deletions(-)
diff --git a/README.rst b/README.rst
index 04d56692..2ff3c022 100644
--- a/README.rst
+++ b/README.rst
@@ -255,24 +255,6 @@ Features
vi Vietnamese
zh Chinese
-
-BrightData: Scrape Any Website, at Scale, Without Getting Blocked
------------------------------------------------------------------
-👉 `Get instant access to BrightData’s AI-powered web scraping tools and datasets.`_
-
-* ✅ AI-powered scraping browser
-* ✅ 72M+ rotating IPs
-* ✅ Zero bans, faster extraction
-* `👉 Get access now.`_
-
-.. image:: https://github.com/user-attachments/assets/ca540b8b-7888-49d2-89df-2c07760f6d9f
- :target: https://get.brightdata.com/hm89s0nowetp
- :alt: Bright Data's Scraping Browser Product
-
-.. _`Get instant access to BrightData’s AI-powered web scraping tools and datasets.`: https://get.brightdata.com/hm89s0nowetp
-
-.. _`👉 Get access now.`: https://get.brightdata.com/hm89s0nowetp
-
Get it now
----------
From 546c4cd6179f31264ec732fb30fb90f083746b1f Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Wed, 22 Oct 2025 20:52:44 -0700
Subject: [PATCH 49/59] Add Thordata + proxy example
---
README.rst | 62 +++++++++++++++++++++++++++++++++++++++++++-----------
1 file changed, 50 insertions(+), 12 deletions(-)
diff --git a/README.rst b/README.rst
index 2ff3c022..9421b019 100644
--- a/README.rst
+++ b/README.rst
@@ -133,6 +133,40 @@ If no language is specified, Newspaper will attempt to auto detect a language.
港特首梁振英就住宅违建事件道歉
+Proxy Usage
+===========
+Newspaper3K supports proxies. Here is an example using `Thordata Proxies`_, a reliable proxy service suitable for AI and global web scraping scenarios.
+
+.. code-block:: pycon
+
+ from newspaper import Article
+ from newspaper import Configuration
+
+ # Configure ThorData proxy
+ config = Configuration()
+ config.proxies = {
+ 'https': 'https://td-customer-USERNAME:PASSWORD@t.pr.thordata.net:9999'
+ }
+
+ # Download and parse an article through the proxy
+ article = Article('https://www.nytimes.com/2024/01/15/technology/ai-trends.html', config=config)
+ article.download()
+ article.parse()
+ print(f"Title: {article.title}")
+
+`Click here to get a free trial of ThorData`_
+
+.. image:: https://github.com/user-attachments/assets/27c4288e-4507-4100-9127-ae2960af287a
+ :target: https://affiliate.thordata.com/tzc4taw66x3b
+ :alt: ThorData proxies
+
+.. _`Thordata Proxies`: https://affiliate.thordata.com/tzc4taw66x3b
+.. _`Click here to get a free trial of ThorData`: https://affiliate.thordata.com/tzc4taw66x3b
+
+
+Multi-lingual
+=============
+
If you are certain that an *entire* news source is in one language, **go ahead and use the same api :)**
.. code-block:: pycon
@@ -164,20 +198,11 @@ If you are certain that an *entire* news source is in one language, **go ahead a
车网_新浪汽车_新浪网
-Enjoy Newspaper3k?
-------------------
-`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`_.
-Scrape Google Search, Google News, Google Maps, and more!
-
-.. image:: https://github.com/user-attachments/assets/9a80eeb4-72a8-43f1-9413-93c7a47b2bf6
- :target: https://serpapi.com/google-news-api?utm_source=newspaper3k_github
- :alt: Scrape search engines easily with SerpApi - Search API.
-
-.. _`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`: https://serpapi.com?utm_source=newspaper3k_github
-
Interested in proxies?
-----------------------
+======================
+Stay private, fast, and fully in control
+----------------------------------------
`Click here to explore BestProxy`_, your go-to solution for premium residential proxies. BestProxy's proxies ensure smooth browsing, fast speeds, and total anonymity. `Get Started`_ today and experience the difference!
.. image:: https://github.com/user-attachments/assets/1c6ef38c-f0c0-4db0-aad2-3ed9d6adf0b5
@@ -187,6 +212,19 @@ Interested in proxies?
.. _`Click here to explore BestProxy`: https://bestproxy.com/?keyword=b2vgzl0r
.. _`Get Started`: https://bestproxy.com/?keyword=b2vgzl0r
+
+Unlock the Web — the Smart Way
+------------------------------
+`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`_.
+Scrape Google Search, Google News, Google Maps, and more!
+
+.. image:: https://github.com/user-attachments/assets/9a80eeb4-72a8-43f1-9413-93c7a47b2bf6
+ :target: https://serpapi.com/google-news-api?utm_source=newspaper3k_github
+ :alt: Scrape search engines easily with SerpApi - Search API.
+
+.. _`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`: https://serpapi.com?utm_source=newspaper3k_github
+
+
Docs
----
From 90441f027e6fee3d75b9966d2af90e4dcb4f7577 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Mon, 3 Nov 2025 00:32:49 -0800
Subject: [PATCH 50/59] Update README with new proxy services and remove old
content
---
README.rst | 65 +++++++++++++++++++++++++++++++-----------------------
1 file changed, 37 insertions(+), 28 deletions(-)
diff --git a/README.rst b/README.rst
index 9421b019..73b93687 100644
--- a/README.rst
+++ b/README.rst
@@ -133,6 +133,16 @@ If no language is specified, Newspaper will attempt to auto detect a language.
港特首梁振英就住宅违建事件道歉
+Newspaper works better with proxies
+===================================
+Click here to try `RapidProxy`_, your all-in-one proxy solution. Enjoy unlimited, never-expiring traffic, fast and stable connections, and full anonymity. Start your free test today and experience hassle-free, reliable proxy services worldwide!
+
+.. image:: https://github.com/user-attachments/assets/719da34a-a0d1-4ee0-81fb-c626d7163616
+ :target: https://www.rapidproxy.io/?ref=gitnew
+ :alt: Experience RapidProxy, your all-in-one proxy solution. Enjoy unlimited, never-expiring traffic, fast and stable connections.
+
+.. _`RapidProxy`: https://www.rapidproxy.io/?ref=gitnew
+
Proxy Usage
===========
Newspaper3K supports proxies. Here is an example using `Thordata Proxies`_, a reliable proxy service suitable for AI and global web scraping scenarios.
@@ -198,33 +208,6 @@ If you are certain that an *entire* news source is in one language, **go ahead a
车网_新浪汽车_新浪网
-Interested in proxies?
-======================
-
-Stay private, fast, and fully in control
-----------------------------------------
-`Click here to explore BestProxy`_, your go-to solution for premium residential proxies. BestProxy's proxies ensure smooth browsing, fast speeds, and total anonymity. `Get Started`_ today and experience the difference!
-
-.. image:: https://github.com/user-attachments/assets/1c6ef38c-f0c0-4db0-aad2-3ed9d6adf0b5
- :target: https://bestproxy.com/?keyword=b2vgzl0r
- :alt: Experience BestProxy, smooth browsing, fast speeds, and total anonymity.
-
-.. _`Click here to explore BestProxy`: https://bestproxy.com/?keyword=b2vgzl0r
-.. _`Get Started`: https://bestproxy.com/?keyword=b2vgzl0r
-
-
-Unlock the Web — the Smart Way
-------------------------------
-`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`_.
-Scrape Google Search, Google News, Google Maps, and more!
-
-.. image:: https://github.com/user-attachments/assets/9a80eeb4-72a8-43f1-9413-93c7a47b2bf6
- :target: https://serpapi.com/google-news-api?utm_source=newspaper3k_github
- :alt: Scrape search engines easily with SerpApi - Search API.
-
-.. _`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`: https://serpapi.com?utm_source=newspaper3k_github
-
-
Docs
----
@@ -381,7 +364,6 @@ Planning on tweaking our full-text algorithm? Add the ``fulltext`` parameter::
$ python3 tests/unit_tests.py fulltext
-
Demo
----
@@ -389,6 +371,33 @@ View a working online demo here: http://newspaper-demo.herokuapp.com
This is another working online demo: http://newspaper.chinazt.cc/
+
+Interested in proxies?
+======================
+
+Stay private, fast, and fully in control
+----------------------------------------
+`Click here to explore BestProxy`_, your go-to solution for premium residential proxies. BestProxy's proxies ensure smooth browsing, fast speeds, and total anonymity. `Get Started`_ today and experience the difference!
+
+.. image:: https://github.com/user-attachments/assets/1c6ef38c-f0c0-4db0-aad2-3ed9d6adf0b5
+ :target: https://bestproxy.com/?keyword=b2vgzl0r
+ :alt: Experience BestProxy, smooth browsing, fast speeds, and total anonymity.
+
+.. _`Click here to explore BestProxy`: https://bestproxy.com/?keyword=b2vgzl0r
+.. _`Get Started`: https://bestproxy.com/?keyword=b2vgzl0r
+
+
+Unlock the Web — the Smart Way
+------------------------------
+`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`_.
+Scrape Google Search, Google News, Google Maps, and more!
+
+.. image:: https://github.com/user-attachments/assets/9a80eeb4-72a8-43f1-9413-93c7a47b2bf6
+ :target: https://serpapi.com/google-news-api?utm_source=newspaper3k_github
+ :alt: Scrape search engines easily with SerpApi - Search API.
+
+.. _`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`: https://serpapi.com?utm_source=newspaper3k_github
+
LICENSE
-------
From 4ab043bf08765b3b5d3c141367cd7bf036e8102b Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Sun, 23 Nov 2025 21:43:38 -0800
Subject: [PATCH 51/59] remove thordata advert from readme
---
README.rst | 30 ------------------------------
1 file changed, 30 deletions(-)
diff --git a/README.rst b/README.rst
index 73b93687..c2a1198d 100644
--- a/README.rst
+++ b/README.rst
@@ -143,36 +143,6 @@ Click here to try `RapidProxy`_, your all-in-one proxy solution. Enjoy unlimited
.. _`RapidProxy`: https://www.rapidproxy.io/?ref=gitnew
-Proxy Usage
-===========
-Newspaper3K supports proxies. Here is an example using `Thordata Proxies`_, a reliable proxy service suitable for AI and global web scraping scenarios.
-
-.. code-block:: pycon
-
- from newspaper import Article
- from newspaper import Configuration
-
- # Configure ThorData proxy
- config = Configuration()
- config.proxies = {
- 'https': 'https://td-customer-USERNAME:PASSWORD@t.pr.thordata.net:9999'
- }
-
- # Download and parse an article through the proxy
- article = Article('https://www.nytimes.com/2024/01/15/technology/ai-trends.html', config=config)
- article.download()
- article.parse()
- print(f"Title: {article.title}")
-
-`Click here to get a free trial of ThorData`_
-
-.. image:: https://github.com/user-attachments/assets/27c4288e-4507-4100-9127-ae2960af287a
- :target: https://affiliate.thordata.com/tzc4taw66x3b
- :alt: ThorData proxies
-
-.. _`Thordata Proxies`: https://affiliate.thordata.com/tzc4taw66x3b
-.. _`Click here to get a free trial of ThorData`: https://affiliate.thordata.com/tzc4taw66x3b
-
Multi-lingual
=============
From 648fb2a18bccb330ce8a0791308fb98936c3de14 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Sat, 6 Dec 2025 08:29:46 +0800
Subject: [PATCH 52/59] remove RapidProxy
---
README.rst | 12 ------------
1 file changed, 12 deletions(-)
diff --git a/README.rst b/README.rst
index c2a1198d..7ab6805f 100644
--- a/README.rst
+++ b/README.rst
@@ -132,18 +132,6 @@ If no language is specified, Newspaper will attempt to auto detect a language.
>>> print(a.title)
港特首梁振英就住宅违建事件道歉
-
-Newspaper works better with proxies
-===================================
-Click here to try `RapidProxy`_, your all-in-one proxy solution. Enjoy unlimited, never-expiring traffic, fast and stable connections, and full anonymity. Start your free test today and experience hassle-free, reliable proxy services worldwide!
-
-.. image:: https://github.com/user-attachments/assets/719da34a-a0d1-4ee0-81fb-c626d7163616
- :target: https://www.rapidproxy.io/?ref=gitnew
- :alt: Experience RapidProxy, your all-in-one proxy solution. Enjoy unlimited, never-expiring traffic, fast and stable connections.
-
-.. _`RapidProxy`: https://www.rapidproxy.io/?ref=gitnew
-
-
Multi-lingual
=============
From 1f6e1770c95468d862d86084d290aa8b0e8b5db7 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Mon, 23 Mar 2026 09:04:29 +0800
Subject: [PATCH 53/59] change git attributes
---
.gitattributes | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitattributes b/.gitattributes
index 4cb342ff..f8c996b6 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,2 +1,2 @@
docs/* linguist-documentation
-tests/* linguist-vendored
+tests/** linguist-vendored
From 72ba63387685b5735ced3a582858596714e52773 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Fri, 10 Apr 2026 21:52:46 +0800
Subject: [PATCH 54/59] add ceointerviews
---
README.rst | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/README.rst b/README.rst
index 7ab6805f..dfd3f398 100644
--- a/README.rst
+++ b/README.rst
@@ -13,6 +13,23 @@ Newspaper3k: Article scraping & curation
:target: https://coveralls.io/github/codelucas/newspaper
:alt: Coverage status
+.. raw:: html
+
+
+
From the author of newspaper:CEOInterviews — the world's largest verified database of executive interviews. 20,000+ CEOs, 1M+ quotes, and full transcripts. Every quote is AI + human verified to the original primary source, so your research tools never hallucinate a CEO soundbite again.
+
+
+
❌ Other generic news APIs
+
✅ CEOInterviews
+
+
+
+
+
+
+
Ask "What has Elon Musk said about tariffs?" — you get speaker-authenticated transcripts with source URLs, not SEO-spam blog paraphrases.
+
Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed:
From bfd7f5cc39f151e4d403fcb37d7e687f44d0e353 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Thu, 16 Apr 2026 18:09:58 +0700
Subject: [PATCH 55/59] webshare proxy
---
README.rst | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/README.rst b/README.rst
index dfd3f398..2ce8b217 100644
--- a/README.rst
+++ b/README.rst
@@ -350,6 +350,16 @@ This is another working online demo: http://newspaper.chinazt.cc/
Interested in proxies?
======================
+Skip the scraping headaches — get proxies that actually work
+------------------------------------------------------------
+`Click here to try Webshare`_, the proxy network built for devs who scrape at scale. Get 80M+ residential IPs, blazing-fast datacenter proxies, and a free tier that actually lets you test before you commit. No contracts, no nonsense — just proxies that don't get blocked.
+
+.. image:: https://github.com/user-attachments/assets/af3431bb-6436-485b-b354-10828692803d
+ :target: https://www.webshare.io/?referral_code=eb53spijl2ih
+ :alt: Webshare — reliable proxies built for developers and scrapers.
+.. _`Click here to try Webshare`: https://www.webshare.io/?referral_code=eb53spijl2ih
+.. _`Start Scraping`: https://www.webshare.io/?referral_code=eb53spijl2ih
+
Stay private, fast, and fully in control
----------------------------------------
`Click here to explore BestProxy`_, your go-to solution for premium residential proxies. BestProxy's proxies ensure smooth browsing, fast speeds, and total anonymity. `Get Started`_ today and experience the difference!
From dc5c2a3c7cb1883ae611afa51aa0e43039c44943 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Sat, 9 May 2026 19:33:37 +0700
Subject: [PATCH 56/59] remove ceointerviews
---
README.rst | 16 ----------------
1 file changed, 16 deletions(-)
diff --git a/README.rst b/README.rst
index 2ce8b217..f72b076b 100644
--- a/README.rst
+++ b/README.rst
@@ -15,22 +15,6 @@ Newspaper3k: Article scraping & curation
.. raw:: html
-
-
From the author of newspaper:CEOInterviews — the world's largest verified database of executive interviews. 20,000+ CEOs, 1M+ quotes, and full transcripts. Every quote is AI + human verified to the original primary source, so your research tools never hallucinate a CEO soundbite again.
-
-
-
❌ Other generic news APIs
-
✅ CEOInterviews
-
-
-
-
-
-
-
Ask "What has Elon Musk said about tariffs?" — you get speaker-authenticated transcripts with source URLs, not SEO-spam blog paraphrases.
-
-
Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed:
"Newspaper is an amazing python library for extracting & curating articles."
From b97791d8f5187387782ca0f8b5de19b996f7f258 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Wed, 13 May 2026 11:02:33 +0700
Subject: [PATCH 57/59] add swiftproxy
---
README.rst | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 54 insertions(+)
diff --git a/README.rst b/README.rst
index f72b076b..462f2a69 100644
--- a/README.rst
+++ b/README.rst
@@ -167,6 +167,50 @@ If you are certain that an *entire* news source is in one language, **go ahead a
车网_新浪汽车_新浪网
+Scraping at scale: avoiding IP blocks
+=====================================
+
+Once you move past scraping a handful of articles, you'll hit the same wall every news scraper hits: 403s, captchas, rate limits, and silent shadow bans. Your code is fine — your IP is the problem. The fix is rotating residential proxies.
+
+I personally route my own newspaper3k pipelines through `Swiftproxy`_ — 80M+ residential IPs across 195+ countries, a 99.89% success rate, non-expiring traffic, and a free trial so you can pressure-test it before paying. Plugging it into newspaper3k takes about four lines:
+
+.. code-block:: python
+
+ from newspaper import Article, Config
+
+ config = Config()
+ config.proxies = {
+ 'http': 'http://USERNAME:PASSWORD@gate.swiftproxy.net:7777',
+ 'https': 'http://USERNAME:PASSWORD@gate.swiftproxy.net:7777',
+ }
+ # a real browser UA helps too
+ config.browser_user_agent = (
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
+ 'AppleWebKit/537.36 (KHTML, like Gecko) '
+ 'Chrome/124.0.0.0 Safari/537.36'
+ )
+ config.request_timeout = 20
+
+ article = Article('https://example.com/some-news-story', config=config)
+ article.download()
+ article.parse()
+ print(article.title)
+
+The same ``config`` object works with ``newspaper.build()`` — every article fetched by the source will rotate through residential IPs automatically:
+
+.. code-block:: python
+
+ import newspaper
+ paper = newspaper.build('http://cnn.com', config=config, memoize_articles=False)
+ for article in paper.articles:
+ article.download()
+ article.parse()
+
+Grab credentials and a free trial at `swiftproxy.net `_. Use code ``PROXY90`` for 10% off your first plan.
+
+.. _`Swiftproxy`: https://www.swiftproxy.net/?ref=codelucas
+
+
Docs
----
@@ -333,6 +377,16 @@ This is another working online demo: http://newspaper.chinazt.cc/
Interested in proxies?
======================
+Power your scraping and automation at real-world scale
+------------------------------------------------------
+`Click here to try Swiftproxy`_ — built for developers running scraping, automation, and data collection workflows at scale. Access 80M+ residential IPs from $0.7/GB, fast ISP proxies from $6/IP, global coverage across 195+ countries, non-expiring traffic, and a 99.89% success rate. Free trial available — use code ``PROXY90`` for 10% off.
+
+.. image:: https://github.com/user-attachments/assets/913f1fd6-20e9-4f37-89b7-ba6b0bd0724a
+ :target: https://www.swiftproxy.net/?ref=codelucas
+ :alt: Swiftproxy — residential and ISP proxies built for scrapers and developers.
+
+.. _`Click here to try Swiftproxy`: https://www.swiftproxy.net/?ref=codelucas
+
Skip the scraping headaches — get proxies that actually work
------------------------------------------------------------
From 15f6701e3a8dea3eff3dd09e6cab4b025ff46ee9 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Thu, 9 Jul 2026 02:33:52 +0800
Subject: [PATCH 58/59] adjustments to readme and readthedocs
---
.readthedocs.yaml | 9 +++++++
README.rst | 63 +++++++++++++++++++++++++++++++++++------------
docs/index.rst | 15 +++++++++++
3 files changed, 71 insertions(+), 16 deletions(-)
create mode 100644 .readthedocs.yaml
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
new file mode 100644
index 00000000..fb179079
--- /dev/null
+++ b/.readthedocs.yaml
@@ -0,0 +1,9 @@
+version: 2
+
+build:
+ os: ubuntu-24.04
+ tools:
+ python: "3.12"
+
+sphinx:
+ configuration: docs/conf.py
diff --git a/README.rst b/README.rst
index 462f2a69..6832767f 100644
--- a/README.rst
+++ b/README.rst
@@ -13,8 +13,6 @@ Newspaper3k: Article scraping & curation
:target: https://coveralls.io/github/codelucas/newspaper
:alt: Coverage status
-.. raw:: html
-
Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed:
"Newspaper is an amazing python library for extracting & curating articles."
@@ -167,6 +165,38 @@ If you are certain that an *entire* news source is in one language, **go ahead a
车网_新浪汽车_新浪网
+Scraping by topic: where do the URLs come from?
+===============================================
+
+``newspaper.build()`` is perfect when you know *which sites* to crawl. But the other question I get constantly is: "I want every article about **X**, across all publications — where do I get the URLs?" The answer is to search Google News for your keyword first, then feed the result links straight into newspaper for extraction.
+
+The easiest way to query Google News programmatically is the `Google News API`_ from `SerpApi - Search API`_ (they also cover Google Search, Google Maps, and more). The two libraries snap together in a few lines:
+
+.. code-block:: python
+
+ # pip3 install google-search-results
+ from serpapi import GoogleSearch
+ from newspaper import Article
+
+ search = GoogleSearch({
+ "engine": "google_news",
+ "q": "electric vehicles",
+ "api_key": "YOUR_SERPAPI_KEY", # free plan at serpapi.com
+ })
+
+ for result in search.get_dict()["news_results"]:
+ article = Article(result["link"])
+ article.download()
+ article.parse()
+ article.nlp()
+ print(article.title, "--", article.summary[:120])
+
+This pattern of SerpApi for *discovery*, newspaper3k for *extraction*, is how most production news-monitoring pipelines are built, and it sidesteps writing a crawler for every source you care about.
+
+.. _`SerpApi - Search API`: https://serpapi.com?utm_source=newspaper3k_github
+.. _`Google News API`: https://serpapi.com/google-news-api?utm_source=newspaper3k_github
+
+
Scraping at scale: avoiding IP blocks
=====================================
@@ -375,8 +405,21 @@ View a working online demo here: http://newspaper-demo.herokuapp.com
This is another working online demo: http://newspaper.chinazt.cc/
-Interested in proxies?
-======================
+Interested in scraping APIs & proxies?
+======================================
+
+Unlock the Web — the Smart Way
+------------------------------
+`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`_.
+Scrape Google Search, Google News, Google Maps, and more!
+
+.. image:: https://github.com/user-attachments/assets/9a80eeb4-72a8-43f1-9413-93c7a47b2bf6
+ :target: https://serpapi.com/google-news-api?utm_source=newspaper3k_github
+ :alt: Scrape search engines easily with SerpApi - Search API.
+
+.. _`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`: https://serpapi.com?utm_source=newspaper3k_github
+
+
Power your scraping and automation at real-world scale
------------------------------------------------------
`Click here to try Swiftproxy`_ — built for developers running scraping, automation, and data collection workflows at scale. Access 80M+ residential IPs from $0.7/GB, fast ISP proxies from $6/IP, global coverage across 195+ countries, non-expiring traffic, and a 99.89% success rate. Free trial available — use code ``PROXY90`` for 10% off.
@@ -409,18 +452,6 @@ Stay private, fast, and fully in control
.. _`Click here to explore BestProxy`: https://bestproxy.com/?keyword=b2vgzl0r
.. _`Get Started`: https://bestproxy.com/?keyword=b2vgzl0r
-
-Unlock the Web — the Smart Way
-------------------------------
-`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`_.
-Scrape Google Search, Google News, Google Maps, and more!
-
-.. image:: https://github.com/user-attachments/assets/9a80eeb4-72a8-43f1-9413-93c7a47b2bf6
- :target: https://serpapi.com/google-news-api?utm_source=newspaper3k_github
- :alt: Scrape search engines easily with SerpApi - Search API.
-
-.. _`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`: https://serpapi.com?utm_source=newspaper3k_github
-
LICENSE
-------
diff --git a/docs/index.rst b/docs/index.rst
index 111ae633..b4e0bae7 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -352,6 +352,21 @@ Newspaper uses a lot of `python-goose's`_ parsing code. View their license `here
Please feel free to `email & contact me`_ if you run into issues or just would like
to talk about the future of this library and news extraction in general!
+Sponsored by SerpApi
+--------------------
+
+`Scrape search engines easily with SerpApi - Search API`_.
+Scrape Google Search, Google News, Google Maps, and more! Their `Google News API`_
+pairs perfectly with newspaper: use it to discover article URLs by keyword, then
+extract them with ``Article``.
+
+.. image:: https://github.com/user-attachments/assets/9a80eeb4-72a8-43f1-9413-93c7a47b2bf6
+ :target: https://serpapi.com/google-news-api?utm_source=newspaper3k_docs
+ :alt: Scrape search engines easily with SerpApi - Search API.
+
+.. _`Scrape search engines easily with SerpApi - Search API`: https://serpapi.com?utm_source=newspaper3k_docs
+.. _`Google News API`: https://serpapi.com/google-news-api?utm_source=newspaper3k_docs
+
.. _`Lucas Ou-Yang`: http://codelucas.com
.. _`email & contact me`: mailto:lucasyangpersonal@gmail.com
.. _`python-goose's`: https://github.com/grangier/python-goose
From 8f501b693bfe5c5847ebff755204b9451937d372 Mon Sep 17 00:00:00 2001
From: Lucas Ou-Yang
Date: Tue, 21 Jul 2026 20:26:35 +0700
Subject: [PATCH 59/59] remove webshare
---
README.rst | 10 ----------
1 file changed, 10 deletions(-)
diff --git a/README.rst b/README.rst
index 6832767f..dcbee056 100644
--- a/README.rst
+++ b/README.rst
@@ -431,16 +431,6 @@ Power your scraping and automation at real-world scale
.. _`Click here to try Swiftproxy`: https://www.swiftproxy.net/?ref=codelucas
-Skip the scraping headaches — get proxies that actually work
-------------------------------------------------------------
-`Click here to try Webshare`_, the proxy network built for devs who scrape at scale. Get 80M+ residential IPs, blazing-fast datacenter proxies, and a free tier that actually lets you test before you commit. No contracts, no nonsense — just proxies that don't get blocked.
-
-.. image:: https://github.com/user-attachments/assets/af3431bb-6436-485b-b354-10828692803d
- :target: https://www.webshare.io/?referral_code=eb53spijl2ih
- :alt: Webshare — reliable proxies built for developers and scrapers.
-.. _`Click here to try Webshare`: https://www.webshare.io/?referral_code=eb53spijl2ih
-.. _`Start Scraping`: https://www.webshare.io/?referral_code=eb53spijl2ih
-
Stay private, fast, and fully in control
----------------------------------------
`Click here to explore BestProxy`_, your go-to solution for premium residential proxies. BestProxy's proxies ensure smooth browsing, fast speeds, and total anonymity. `Get Started`_ today and experience the difference!
แสดงความคิดเห็น
+ +