diff --git a/newspaper/article.py b/newspaper/article.py index 6feb2dc5..9835ba81 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -8,7 +8,7 @@ import copy import os import glob - +import operator import requests from . import images @@ -343,10 +343,26 @@ def nlp(self): self.throw_if_not_parsed_verbose() nlp.load_stopwords(self.config.get_language()) - text_keyws = list(nlp.keywords(self.text).keys()) - title_keyws = list(nlp.keywords(self.title).keys()) - keyws = list(set(title_keyws + text_keyws)) - self.set_keywords(keyws) + text_keyws = nlp.keywords(self.text, self.url) + title_keyws = nlp.keywords(self.title, self.url) + keyws = {} + count = 0 + for word in list(title_keyws.keys()): + if word in text_keyws: + keyws[word] = 1.5 + else: + keyws[word] = 1.15 + count += 1 + tcount = 0 + end = len(text_keyws.keys()) + while tcount < end: + # find the word with the largest weight amoung the rest + word = max(text_keyws, key=lambda i: text_keyws[i]) + if word not in keyws: + keyws[word] = text_keyws[word] + del text_keyws[word] + tcount += 1 + self.set_keywords(sorted(keyws.items(), key=lambda item: item[1], reverse=True)) max_sents = self.config.MAX_SUMMARY_SENT @@ -462,7 +478,7 @@ def set_keywords(self, keywords): """Keys are stored in list format """ if not isinstance(keywords, list): - raise Exception("Keyword input must be list!") + raise Exception("Keyword input must be dictionary!") if keywords: self.keywords = keywords[:self.config.MAX_KEYWORDS] diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 425ece59..2bd11ff6 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -212,6 +212,8 @@ def parse_date_str(date_str): 'content': 'content'}, {'attribute': 'name', 'value': 'sailthru.date', 'content': 'content'}, + {'attribute': 'name', 'value': 'article.published', + 'content': 'content'}, {'attribute': 'name', 'value': 'PublishDate', 'content': 'content'}, {'attribute': 'pubdate', 'value': 'pubdate', diff --git a/newspaper/nlp.py b/newspaper/nlp.py index 23505255..66bd10d7 100644 --- a/newspaper/nlp.py +++ b/newspaper/nlp.py @@ -12,9 +12,11 @@ from os import path from collections import Counter - +from rake_nltk import Rake from . import settings - +import nltk +# from nltk.corpus import stopwords +# s=set(stopwords.words('english')) ideal = 20.0 stopwords = set() @@ -43,7 +45,7 @@ def summarize(url='', title='', text='', max_sents=5): summaries = [] sentences = split_sentences(text) - keys = keywords(text) + keys = keywords(text, url) titleWords = split_words(title) # Score sentences, and use the top 5 or max_sents sentences @@ -116,36 +118,46 @@ def split_words(text): return None -def keywords(text): +def keywords(text, url, titleSig = False): """Get the top 10 keywords and their frequency scores ignores blacklisted words in stopwords, counts the number of occurrences of each word, and sorts them in reverse natural order (so descending) by number of occurrences. """ - NUM_KEYWORDS = 10 + NUM_KEYWORDS = 20 text = split_words(text) + # tokens = nltk.word_tokenize(' '.join(text)) + # tagged = nltk.pos_tag(tokens) # of words before removing blacklist words + innerStopwords = list(stopwords)[:] if text: num_words = len(text) - text = [x for x in text if x not in stopwords] + # validText = [item[0] for item in tagged if item[1].find('NN') != -1 || item[1] == 'JJ'] + if url.find('nytimes') != -1: + innerStopwords += ['new', 'york'] + text = [x for x in text if x not in set(innerStopwords)] freq = {} for word in text: if word in freq: freq[word] += 1 else: freq[word] = 1 - min_size = min(NUM_KEYWORDS, len(freq)) keywords = sorted(freq.items(), - key=lambda x: (x[1], x[0]), - reverse=True) + key=lambda x: (x[1], x[0]), + reverse=True) keywords = keywords[:min_size] - keywords = dict((x, y) for x, y in keywords) - + keywords = dict((x, y) for x, y in keywords if not x.isdigit()) for k in keywords: articleScore = keywords[k] * 1.0 / max(num_words, 1) keywords[k] = articleScore * 1.5 + 1 return dict(keywords) + # else: + # r = Rake() + # print (' '.join(text)) + # r.extract_keywords_from_text(' '.join(text)) + # print ('from content') + # print (r.get_ranked_phrases()) else: return dict() diff --git a/newspaper/resources/text/stopwords-en.txt b/newspaper/resources/text/stopwords-en.txt index d3a39543..bfce3e02 100644 --- a/newspaper/resources/text/stopwords-en.txt +++ b/newspaper/resources/text/stopwords-en.txt @@ -1,4 +1,15 @@ +- + +, +. +a +e +i +o +u +t a's +as able about above @@ -11,6 +22,7 @@ afterwards again against ain't +aint all allow allows @@ -41,6 +53,7 @@ appreciate appropriate are aren't +arent around as aside @@ -77,10 +90,12 @@ by c c'mon c's +cs came campaign can can't +cant cannot cant cause @@ -110,12 +125,15 @@ described despite did didn't +didnt different do does doesn't +doesnt doing don't +dont done down downwards @@ -174,21 +192,26 @@ gotten greetings had hadn't +hadnt happens hardly has hasn't +hasnt have haven't +havent having he he's +hes hello help hence her here here's +heres hereafter hereby herein @@ -205,9 +228,13 @@ how howbeit however i'd +id i'll +ill i'm +im i've +ive if ignored immediate @@ -225,11 +252,15 @@ into inward is isn't +isnt it it'd +itd it'll +itll it's its +its itself just keep @@ -248,6 +279,7 @@ less lest let let's +lets like liked likely @@ -399,6 +431,7 @@ such sup sure t's +ts take taken tell @@ -419,6 +452,7 @@ then thence there there's +theres thereafter thereby therefore @@ -428,9 +462,13 @@ thereupon these they they'd +theyd they'll +theyll they're +theyre they've +theyve think third this @@ -485,22 +523,29 @@ wasn't way we we'd +wed we'll +well we're +were we've +weve welcome well went were weren't +werent what what's +whats whatever when whence whenever where where's +wheres whereafter whereas whereby @@ -525,6 +570,7 @@ with within without won't +wont wonder would would @@ -533,9 +579,13 @@ yes yet you you'd +youd you'll +youll you're +youre you've +youve your yours yourself @@ -543,4 +593,345 @@ yourselves zero official sharply -criticized \ No newline at end of file +criticized +sports +1 +10 +2012 +sa +says +tweet +pm +home +sports +section +newsinfo +stories +story +photo +2013 +na +ng +ang +year +years +week +weeks +day +days +percent +ko +ako +yung +yun +2 +3 +4 +5 +6 +7 +8 +9 +0 +time +january +february +march +april +may +june +july +august +september +october +november +december +police +2014 +mr +things +awesome +great +good +people +million +man +woman +men +female +male +time +times +worst +best +wrong +month +months +mattered +builds +build +built +building +anybodys +everybodys +somebodys +nobodys +schedules +problem +popular +npr +politico +fivethirtyeight +trumps +take +takes +taking +took +data +bigger +cool +write +writes +writing +wrote +talk +talks +needed +wanted +strange +issues +issue +make +makes +making +made +ahead +worries +worry +worrying +amid +conclusion +lessons +lesson +visits +visit +visited +visiting +trillion +trillions +sensational +belongs +belong +belonging +belonged +quiet +anger +angers +top +bloomberg +exaggerates +exaggerate +exaggerating +fivethirtyeights +stuff +hopefully +hope +show +showing +shows +showed +places +place +placed +big +small +hell +smaller +older +picture +weak +sees +see +seeing +saw +interview +interviews +excerpt +excerpts +win +wins +won +lose +loses +lost +reading +read +read +work +working +worked +impact +watch +kind +lot +cbn +news +thing +sort +monday +tuesday +wednesday +thursday +friday +saturday +sunday +start +told +seek +seeking +seeked +heavily +smoothing +focus +podcasts +podcast +future +limitations +limitation +power +view +views +viewed +today +night +began +begin +begins +asks +ask +asked +askking +earlier +early +spoke +speak +speaks +suggestive +demeaning +call +calling +called +dismiss +dismissed +dismissing +dismissive +previously +previous +renew +heartbreaking +claims +claim +claimed +claiming +meet +met +foward +ruined +ruin +ruining +badly +bad +lie +lies +lying +weekend +weekdays +weekday +weekly +abc +backed +including +videos +guilty +shame +fill +filling +heard +hear +elevate +conversation +conversations +feel +feels +felt +feeling +young +old +thought +think +thinking +comments +comment +current +currently +minute +stay +staying +stayed +photograph +worse +proud +turning +reject +morning +evening +afternoon +support +binges +binge +special +leading +lead +poor +rich +poorer +richer +uproar +expensive +loophole +wealthy +richest +though +paint +meeting +question +absolute +fair +tired +tricky +manage +manages +managed +trip +snubs +snub +snub +needless +todo +warns +warn +thousands +thousand +hundred +hundreds +accelerated +billion +frustrated +questions +rhetoric +happening +happen +happened +mercilessly +heckles +means \ No newline at end of file diff --git a/newspaper/settings.py b/newspaper/settings.py index 5add9eeb..f012592a 100644 --- a/newspaper/settings.py +++ b/newspaper/settings.py @@ -27,7 +27,7 @@ # NLP stopwords are != regular stopwords for now... NLP_STOPWORDS_EN = os.path.join( - PARENT_DIRECTORY, 'resources/misc/stopwords-nlp-en.txt') + PARENT_DIRECTORY, 'resources/text/stopwords-en.txt') DATA_DIRECTORY = '.newspaper_scraper'