From f50ceaf29877fb552f72f5b1108b99e5ddfef3a0 Mon Sep 17 00:00:00 2001 From: Daniel Shashko Date: Wed, 1 Apr 2026 17:22:39 +0300 Subject: [PATCH 1/2] fix: try punkt fallback to punkt_tab in split_sentences (NLTK >= 3.9) --- newspaper/nlp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/newspaper/nlp.py b/newspaper/nlp.py index 23505255..782afb72 100644 --- a/newspaper/nlp.py +++ b/newspaper/nlp.py @@ -154,7 +154,10 @@ def split_sentences(text): """Split a large string into sentences """ import nltk.data - tokenizer = nltk.data.load('tokenizers/punkt/english.pickle') + try: + tokenizer = nltk.data.load('tokenizers/punkt/english.pickle') + except LookupError: + tokenizer = nltk.data.load('tokenizers/punkt_tab/english') sentences = tokenizer.tokenize(text) sentences = [x.replace('\n', '') for x in sentences if len(x) > 10] From 958ec7d7292a6c37af3245d3b563e6eb14bf9595 Mon Sep 17 00:00:00 2001 From: Daniel Shashko Date: Wed, 1 Apr 2026 17:22:41 +0300 Subject: [PATCH 2/2] fix: add punkt_tab to REQUIRED_CORPORA for NLTK >= 3.9 --- download_corpora.py | 1 + 1 file changed, 1 insertion(+) diff --git a/download_corpora.py b/download_corpora.py index 94abfc70..733b1d19 100644 --- a/download_corpora.py +++ b/download_corpora.py @@ -8,6 +8,7 @@ REQUIRED_CORPORA = [ 'brown', # Required for FastNPExtractor 'punkt', # Required for WordTokenizer + 'punkt_tab', # Required for WordTokenizer (NLTK >= 3.9) 'maxent_treebank_pos_tagger', # Required for NLTKTagger 'movie_reviews', # Required for NaiveBayesAnalyzer 'wordnet', # Required for lemmatization and Wordnet