diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..4cb342ff --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +docs/* linguist-documentation +tests/* linguist-vendored diff --git a/README.rst b/README.rst index a3b7406a..87309793 100644 --- a/README.rst +++ b/README.rst @@ -15,7 +15,7 @@ Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed: .. _`tweeted by`: https://twitter.com/kennethreitz/status/419520678862548992 .. _`The Changelog`: http://thechangelog.com/newspaper-delivers-instapaper-style-article-extraction/ -**Newspaper is a Python3 library**! Or, view the `Python2 branch`_ +**Newspaper is a Python3 library**! Or, view our **deprecated and buggy** `Python2 branch`_ .. _`Python2 branch`: https://github.com/codelucas/newspaper/tree/python-2-head @@ -164,7 +164,6 @@ Interested in adding a new language for us? Refer to: `Docs - Adding new languag Features -------- -- Full Python3 and Python2 support - Multi-threaded article download framework - News url identification - Text extraction from html @@ -202,6 +201,7 @@ Features zh Chinese id Indonesian vi Vietnamese + tr Turkish Get it now @@ -210,11 +210,15 @@ Get it now Installing newspaper is simple with `pip `_. However, you will run into fixable issues if you are trying to install on ubuntu. -Note that our Python3 package name is ``newspaper3k`` while our Python2 +Note that the Python3 package name is ``newspaper3k`` while our Python2 package name is ``newspaper``. **If you are on Debian / Ubuntu**, install using the following: +- Install ``pip3`` command needed to install ``newspaper3k`` package:: + + $ sudo apt-get install python3-pip + - Python development version, needed for Python.h:: $ sudo apt-get install python-dev @@ -266,9 +270,6 @@ NOTE: You will still most likely need to install the following libraries via you Development ----------- -Newspaper has two branches up for development. *This* branch, the master, is our Python3 -codebase while our Python2 branch is located on *python-2-head*. - If you'd like to contribute and hack on the newspaper project, feel free to clone a development version of this repository locally:: diff --git a/newspaper/article.py b/newspaper/article.py index 5fe75d88..aec41c40 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -269,7 +269,8 @@ def is_valid_body(self): wordcount = self.text.split(' ') sentcount = self.text.split('.') - if meta_type == 'article' and wordcount > (self.config.MIN_WORD_COUNT): + if (meta_type == 'article' and len(wordcount) > + (self.config.MIN_WORD_COUNT)): log.debug('%s verified for article and wc' % self.url) return True @@ -367,6 +368,11 @@ def set_reddit_top_img(self): try: s = images.Scraper(self) self.set_top_img(s.largest_image_url()) + except TypeError as e: + if "Can't convert 'NoneType' object to str implicitly" in e.args[0]: + log.debug("No pictures found. Top image not set, %s" % e) + else: + log.critical('jpeg error with PIL, %s' % e) except Exception as e: log.critical('jpeg error with PIL, %s' % e) diff --git a/newspaper/configuration.py b/newspaper/configuration.py index c0960a6b..374620c7 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -63,6 +63,8 @@ def __init__(self): self.verbose = False # for debugging + self.thread_timeout_seconds = 1 + # Set this to False if you want to recompute the categories # *every* time you build a `Source` object # TODO: Actually make this work @@ -76,13 +78,13 @@ def del_language(self): def set_language(self, language): """Language setting must be set in this method b/c non-occidental - (western) langauges require a seperate stopwords class. + (western) languages require a seperate stopwords class. """ if not language or len(language) != 2: - raise Exception("Your input language must be a 2 char langauge code, \ + raise Exception("Your input language must be a 2 char language code, \ for example: english-->en \n and german-->de") - # If explicitly set langauge, don't use meta + # If explicitly set language, don't use meta self.use_meta_language = False # Set oriental language stopword class @@ -90,7 +92,7 @@ def set_language(self, language): self.stopwords_class = self.get_stopwords_class(language) language = property(get_language, set_language, - del_language, "langauge prop") + del_language, "language prop") def get_stopwords_class(self, language): if language == 'ko': diff --git a/newspaper/images.py b/newspaper/images.py index 54e5b1e1..79fbccbd 100644 --- a/newspaper/images.py +++ b/newspaper/images.py @@ -195,7 +195,7 @@ def largest_image_url(self): if area > max_area: max_area = area max_url = img_url - log.debug('using max img ' + max_url) + log.debug('using max img {}'.format(max_url)) return max_url def calculate_area(self, img_url, dimension): diff --git a/newspaper/mthreading.py b/newspaper/mthreading.py index 5164635a..92ebfb29 100644 --- a/newspaper/mthreading.py +++ b/newspaper/mthreading.py @@ -13,23 +13,26 @@ import traceback from threading import Thread +from .configuration import Configuration + class Worker(Thread): """ Thread executing tasks from a given tasks queue. """ - def __init__(self, tasks): + def __init__(self, tasks, timeout_seconds): Thread.__init__(self) self.tasks = tasks + self.timeout = timeout_seconds self.daemon = True self.start() def run(self): while True: try: - func, args, kargs = self.tasks.get() + func, args, kargs = self.tasks.get(timeout=self.timeout) except queue.Empty: - traceback.print_exc() + # Extra thread allocated, no job, exit gracefully break try: func(*args, **kargs) @@ -40,35 +43,21 @@ def run(self): class ThreadPool: - """ - Pool of threads consuming tasks from a queue. - """ - def __init__(self, num_threads): + def __init__(self, num_threads, timeout_seconds): self.tasks = queue.Queue(num_threads) for _ in range(num_threads): - Worker(self.tasks) + Worker(self.tasks, timeout_seconds) def add_task(self, func, *args, **kargs): - """ - Add a task to the queue. - """ self.tasks.put((func, args, kargs)) def wait_completion(self): - """ - Wait for completion of all the tasks in the queue. - """ self.tasks.join() - def clear_threads(self): - """ - """ - pass - class NewsPool(object): - def __init__(self): + def __init__(self, config=None): """ Abstraction of a threadpool. A newspool can accept any number of source OR article objects together in a list. It allocates one @@ -94,6 +83,7 @@ def __init__(self): """ self.papers = [] self.pool = None + self.config = config or Configuration() def join(self): """ @@ -109,12 +99,10 @@ def join(self): self.pool = None def set(self, paper_list, threads_per_source=1): - """ - Sets the job batch. - """ self.papers = paper_list num_threads = threads_per_source * len(self.papers) - self.pool = ThreadPool(num_threads) + timeout = self.config.thread_timeout_seconds + self.pool = ThreadPool(num_threads, timeout) for paper in self.papers: self.pool.add_task(paper.download_articles) diff --git a/newspaper/network.py b/newspaper/network.py index b4ee4c9e..b12fb69f 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -92,8 +92,9 @@ def multithread_request(urls, config=None): """ config = config or Configuration() num_threads = config.number_threads + timeout = config.thread_timeout_seconds - pool = ThreadPool(num_threads) + pool = ThreadPool(num_threads, timeout) m_requests = [] for url in urls: diff --git a/newspaper/resources/text/stopwords-mk.txt b/newspaper/resources/text/stopwords-mk.txt new file mode 100644 index 00000000..c128a1f2 --- /dev/null +++ b/newspaper/resources/text/stopwords-mk.txt @@ -0,0 +1,173 @@ +а +е +од +до +без +со +за +на +ја +го +ги +низ +исто +истото +под +над +да +ќе +во +него +неа +тој +таа +тоа +ние +вие +тие +кој +која +кои +дали +се +не +беше +еден +едно +јас +рече +сите +сум +сме +овде +така +и +што +или +по +но +му +па +нив +ни +итн +бил +кај +ова +врз +a +две +би +она +си +кое +ако +два +има +в +како +само +дека +една +туку +кога +сега +ми +потоа +ти +кон +додека +веќе +нешто +уште +таму +ли +беа +ме +некој +ништо +тука +пред +им +каде +повторно +ниту +биде +толку +никогаш +мене +тогаш +своите +сето +нема +околу +многу +полека +секогаш +зошто +те +добро +можеби +колку +можеше +нивните +преку +миг +знам +малку +вратата +ох +навистина +оваа +покрај +повеќе +овој +сеуште +имаше +својата +неговите +неговата +друго +зашто +немаше +воопшто +понекогаш +ах +зад +еднаш +својот +дури +себе +ви +токму +зарем +сте +било +сосема +секој +неговото +друг +мошне +ајде +можел +може +при +пак +сè +други +треба +ама +после +некоја +нас +бе +никој +одма +сѐ +ај +нѐ +неколку +оние +мора +оди +еј diff --git a/newspaper/resources/text/stopwords-tr.txt b/newspaper/resources/text/stopwords-tr.txt new file mode 100644 index 00000000..43b8a0ff --- /dev/null +++ b/newspaper/resources/text/stopwords-tr.txt @@ -0,0 +1,210 @@ +acaba +altmış +altı +ama +ancak +arada +aslında +ayrıca +bana +bazı +belki +ben +benden +beni +benim +beri +beş +bile +bin +bir +birçok +biri +birkaç +birkez +birşey +birşeyi +biz +bize +bizden +bizi +bizim +böyle +böylece +bu +buna +bunda +bundan +bunlar +bunları +bunların +bunu +bunun +burada +çok +çünkü +da +daha +dahi +de +defa +değil +diğer +diye +doksan +dokuz +dolayı +dolayısıyla +dört +edecek +eden +ederek +edilecek +ediliyor +edilmesi +ediyor +eğer +elli +en +etmesi +etti +ettiği +ettiğini +gibi +göre +halen +hangi +hatta +hem +henüz +hep +hepsi +her +herhangi +herkes +herkesin +hiç +hiçbir +için +iki +ile +ilgili +ise +işte +itibaren +itibariyle +kadar +karşın +katrilyon +kendi +kendilerine +kendini +kendisi +kendisine +kendisini +kez +ki +kim +kimden +kime +kimi +kimse +kırk +milyar +milyon +mu +mü +mı +nasıl +ne +neden +nedenle +nerde +nerede +nereye +niye +niçin +o +olan +olarak +oldu +olduğu +olduğunu +olduklarını +olmadı +olmadığı +olmak +olması +olmayan +olmaz +olsa +olsun +olup +olur +olursa +oluyor +on +ona +ondan +onlar +onlardan +onları +onların +onu +onun +otuz +oysa +öyle +pek +rağmen +sadece +sanki +sekiz +seksen +sen +senden +seni +senin +siz +sizden +sizi +sizin +şey +şeyden +şeyi +şeyler +şöyle +şu +şuna +şunda +şundan +şunları +şunu +tarafından +trilyon +tüm +üç +üzere +var +vardı +ve +veya +ya +yani +yapacak +yapılan +yapılması +yapıyor +yapmak +yaptı +yaptığı +yaptığını +yaptıkları +yedi +yerine +yetmiş +yine +yirmi +yoksa +yüz +zaten diff --git a/newspaper/urls.py b/newspaper/urls.py index 2d414882..33fd3bda 100644 --- a/newspaper/urls.py +++ b/newspaper/urls.py @@ -245,8 +245,13 @@ def url_to_filetype(abs_url): path = path[:-1] path_chunks = [x for x in path.split('/') if len(x) > 0] last_chunk = path_chunks[-1].split('.') # last chunk == file usually - file_type = last_chunk[-1] if len(last_chunk) >= 2 else None - return file_type or None + if len(last_chunk) < 2: + return None + file_type = last_chunk[-1] + # Assume that file extension is maximum 5 characters long + if len(file_type) <= 5 or file_type.lower() in ALLOWED_TYPES: + return file_type.lower() + return None def get_domain(abs_url, **kwargs): """ diff --git a/newspaper/utils.py b/newspaper/utils.py index aede89b5..86f299cf 100644 --- a/newspaper/utils.py +++ b/newspaper/utils.py @@ -348,6 +348,8 @@ def print_available_languages(): 'zh': 'Chinese', 'id': 'Indonesian', 'vi': 'Vietnamese', + 'mk': 'Macedonian', + 'tr': 'Turkish', } codes = get_available_languages() diff --git a/newspaper/version.py b/newspaper/version.py index 9778534c..ece3503d 100644 --- a/newspaper/version.py +++ b/newspaper/version.py @@ -7,5 +7,5 @@ __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' -version_info = (0, 1, 5) +version_info = (0, 1, 6) __version__ = ".".join(map(str, version_info)) diff --git a/requirements.txt b/requirements.txt index f40f012e..2bfec338 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -beautifulsoup4==4.3.2 +beautifulsoup4==4.4.1 Pillow==2.6.1 PyYAML==3.11 cssselect==0.9.1 diff --git a/setup.py b/setup.py index 082afbea..7b184cb0 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( name='newspaper3k', - version='0.1.5', + version='0.1.6', description='Simplified python article discovery & extraction.', long_description=readme, author='Lucas Ou-Yang', diff --git a/tests/data/test_urls.txt b/tests/data/test_urls.txt index 8807b349..b851ed46 100644 --- a/tests/data/test_urls.txt +++ b/tests/data/test_urls.txt @@ -37,3 +37,4 @@ 1 http://www.alarabiya.net/ar/arab-and-world/syria/2014/02/02/%D8%A7%D9%84%D9%86%D8%A7%D8%AA%D9%88-%D9%8A%D8%B9%D9%84%D9%86-%D8%A7%D8%B3%D8%AA%D8%B9%D8%AF%D8%A7%D8%AF%D9%87-%D9%84%D8%AA%D8%AF%D9%85%D9%8A%D8%B1-%D8%A7%D9%84%D8%A3%D8%B3%D9%84%D8%AD%D8%A9-%D8%A7%D9%84%D9%83%D9%8A%D9%85%D9%8A%D8%A7%D9%88%D9%8A%D8%A9-%D8%A7%D9%84%D8%B3%D9%88%D8%B1%D9%8A%D8%A9.html 1 http://www.almasryalyoum.com/news/details/387323 1 http://tahrirnews.com/news/view.aspx?cdate=02022014&id=990c7733-9293-4441-982e-a44af000bb19 +1 http://www.enfieldindependent.co.uk/news/14140506.Review_of_the_year__July/