diff --git a/download_corpora.py b/download_corpora.py index 94abfc70..733b1d19 100644 --- a/download_corpora.py +++ b/download_corpora.py @@ -8,6 +8,7 @@ REQUIRED_CORPORA = [ 'brown', # Required for FastNPExtractor 'punkt', # Required for WordTokenizer + 'punkt_tab', # Required for WordTokenizer (NLTK >= 3.9) 'maxent_treebank_pos_tagger', # Required for NLTKTagger 'movie_reviews', # Required for NaiveBayesAnalyzer 'wordnet', # Required for lemmatization and Wordnet diff --git a/newspaper/nlp.py b/newspaper/nlp.py index 23505255..782afb72 100644 --- a/newspaper/nlp.py +++ b/newspaper/nlp.py @@ -154,7 +154,10 @@ def split_sentences(text): """Split a large string into sentences """ import nltk.data - tokenizer = nltk.data.load('tokenizers/punkt/english.pickle') + try: + tokenizer = nltk.data.load('tokenizers/punkt/english.pickle') + except LookupError: + tokenizer = nltk.data.load('tokenizers/punkt_tab/english') sentences = tokenizer.tokenize(text) sentences = [x.replace('\n', '') for x in sentences if len(x) > 10]