From 20796fcc6a2d22fd900a5caeda155cc0cd23eef8 Mon Sep 17 00:00:00 2001 From: Karl Sutt Date: Wed, 23 Jul 2014 17:16:02 +0100 Subject: [PATCH 01/11] Make tests/ directory a package. --- tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b From 6ef0d28445be93022ab07e512eab8ca13a79c314 Mon Sep 17 00:00:00 2001 From: Karl Sutt Date: Wed, 23 Jul 2014 17:17:19 +0100 Subject: [PATCH 02/11] Remove stray newlines at the end of files --- newspaper/configuration.py | 1 - newspaper/mthreading.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/newspaper/configuration.py b/newspaper/configuration.py index 744f4cff..f01676dc 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -181,4 +181,3 @@ class ArticleConfiguration(Configuration): class SourceConfiguration(Configuration): pass - diff --git a/newspaper/mthreading.py b/newspaper/mthreading.py index b4837602..62df790e 100644 --- a/newspaper/mthreading.py +++ b/newspaper/mthreading.py @@ -116,5 +116,3 @@ def set(self, paper_list, threads_per_source=1): for paper in self.papers: self.pool.add_task(paper.download_articles) - - From 69b00c8554dc763f42d4c82d2287865268b1482e Mon Sep 17 00:00:00 2001 From: Karl Sutt Date: Wed, 23 Jul 2014 17:18:15 +0100 Subject: [PATCH 03/11] Fix a multilingual test failure (due to newline EOF) --- tests/data/body_text/spanish_text_1.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/data/body_text/spanish_text_1.txt b/tests/data/body_text/spanish_text_1.txt index f106063a..080a09fd 100644 --- a/tests/data/body_text/spanish_text_1.txt +++ b/tests/data/body_text/spanish_text_1.txt @@ -1,5 +1,3 @@ -Las discrepancias jurídicas entre el juez José Castro y el fiscal anticorrupción Pedro Horrach han quedado de manifiesto en la resolución dictada por el titular del Juzgado de Instrucción 3 de Palma, en la que cita a declarar a la hija del Rey. 27-06-2012 | Efe - El fiscal Pedro Horrach está perfilando con el jefe de la Fiscalía Anticorrupción, Antonio Salinas, en Madrid, la estrategia jurídica que seguirá tras el auto dictado por el juez José Castro en el que acuerda citar en calidad de imputada a la infanta Cristina de Borbón y Grecia el próximo 8 de marzo en Palma por los delitos de blanqueo de capitales y contra la Hacienda Pública. Horrach duda si insistir en su postura de dejar al margen a la hija menor del Rey de los indicios incriminatorios citados por el instructor del ‘caso Nóos’, como ya lo hizo por escrito el pasado mes de noviembre o, por el contrario, se limita exponer de manera sucinta su punto de vista para apoyar el recurso que ya ha anunciado el abogado de la Infanta, Miquel Roca. From 2f201adbc7394df4133079a945cfa80fa58792e6 Mon Sep 17 00:00:00 2001 From: Karl Sutt Date: Wed, 23 Jul 2014 17:19:59 +0100 Subject: [PATCH 04/11] Meta tag extraction fix (need to build article first) --- tests/unit_tests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 8ad91096..a93dabf6 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -106,6 +106,8 @@ def test_parse_html(self): @print_test def test_meta_tag_extraction(self): + self.article.build() + meta_type = self.article.extractor.get_meta_type(self.article) # print 'meta type is---------', meta_type assert 'article' == meta_type From 223447670cd980c2587243d8209f437d587dee7c Mon Sep 17 00:00:00 2001 From: Karl Sutt Date: Wed, 23 Jul 2014 17:21:35 +0100 Subject: [PATCH 05/11] Move setting up the test to setUp() (otherwise fails) --- tests/unit_tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index a93dabf6..bb99b99f 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -275,12 +275,14 @@ def test_popular_urls(self): class EncodingTestCase(unittest.TestCase): def runTest(self): - self.uni_string = u"∆ˆˆø∆ßåßlucas yang˜" - self.normal_string = "∆ƒˆƒ´´lucas yang" self.test_encode_val() self.test_smart_unicode() self.test_smart_str() + def setUp(self): + self.uni_string = u"∆ˆˆø∆ßåßlucas yang˜" + self.normal_string = "∆ƒˆƒ´´lucas yang" + @print_test def test_encode_val(self): assert encodeValue(self.uni_string) == self.uni_string From 94d51bdf088e78c9732fbed47aeb8b7f3a890b56 Mon Sep 17 00:00:00 2001 From: Karl Sutt Date: Wed, 23 Jul 2014 17:22:58 +0100 Subject: [PATCH 06/11] Use build() instead of download() and parse() --- tests/unit_tests.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index bb99b99f..4e960484 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -252,8 +252,7 @@ def test_article_build(self): url = 'http://abcnews.go.com/blogs/politics/2013/12/states-cite-surge-in-obamacare-sign-ups-ahead-of-first-deadline/' article = newspaper.build_article(url) assert isinstance(article, Article) == True - article.download() - article.parse() + article.build() article.nlp() # print article.title # print article.summary @@ -382,8 +381,7 @@ def runTest(self): def test_chinese_fulltext_extract(self): url = 'http://www.bbc.co.uk/zhongwen/simp/chinese_news/2012/12/121210_hongkong_politics.shtml' article = Article(url=url, language='zh') - article.download() - article.parse() + article.build() with codecs.open(os.path.join(TEXT_FN, 'chinese_text_1.txt'), 'r', 'utf8') as f: assert article.text == f.read() @@ -394,8 +392,7 @@ def test_chinese_fulltext_extract(self): def test_arabic_fulltext_extract(self): url = 'http://arabic.cnn.com/2013/middle_east/8/3/syria.clashes/index.html' article = Article(url=url, language='ar') - article.download() - article.parse() + article.build() with codecs.open(os.path.join(TEXT_FN, 'arabic_text_1.txt'), 'r', 'utf8') as f: assert article.text == f.read() @@ -406,8 +403,7 @@ def test_arabic_fulltext_extract(self): def test_spanish_fulltext_extract(self): url = 'http://ultimahora.es/mallorca/noticia/noticias/local/fiscalia-anticorrupcion-estudia-recurre-imputacion-infanta.html' article = Article(url=url, language='es') - article.download() - article.parse() + article.build() with codecs.open(os.path.join(TEXT_FN, 'spanish_text_1.txt'), 'r', 'utf8') as f: assert article.text == f.read() From 634f0990de38c18cf6bc3393ffc67dda4a88e524 Mon Sep 17 00:00:00 2001 From: Karl Sutt Date: Wed, 23 Jul 2014 17:24:10 +0100 Subject: [PATCH 07/11] Fix threading tests -- config passed incorrectly --- tests/unit_tests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 4e960484..19f07968 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -308,9 +308,9 @@ def test_download_works(self): """ config = Configuration() config.memoize_articles = False - slate_paper = newspaper.build('http://slate.com', config) - tc_paper = newspaper.build('http://techcrunch.com', config) - espn_paper = newspaper.build('http://espn.com', config) + slate_paper = newspaper.build('http://slate.com', config=config) + tc_paper = newspaper.build('http://techcrunch.com', config=config) + espn_paper = newspaper.build('http://espn.com', config=config) print 'slate has %d articles tc has %d articles espn has %d articles' \ % (slate_paper.size(), tc_paper.size(), espn_paper.size()) From f5e6f19ff82eef507c7100f7b93501d9820f9cf1 Mon Sep 17 00:00:00 2001 From: Karl Sutt Date: Wed, 23 Jul 2014 17:25:16 +0100 Subject: [PATCH 08/11] Download before parsing. Fix number of images assertion --- tests/unit_tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 19f07968..c7b2d292 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -93,9 +93,9 @@ def test_parse_html(self): SCHEME = 'http' AUTHORS = ['Dana Ford', 'Tom Watkins'] TITLE = 'After storm, forecasters see smooth sailing for Thanksgiving' - LEN_IMGS = 47 # list is too big, we just check size of images arr + LEN_IMGS = 46 # list is too big, we just check size of images arr - self.article.parse() + self.article.build() with open(os.path.join(TEST_DIR, 'data/body_example.txt'), 'r') as f: assert self.article.text == f.read() assert self.article.top_img == TOP_IMG From bf7437b27aa93e9a4dc99806faae6774b61a30eb Mon Sep 17 00:00:00 2001 From: Karl Sutt Date: Wed, 23 Jul 2014 17:26:09 +0100 Subject: [PATCH 09/11] Download and parse article before doing any NLP --- tests/unit_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index c7b2d292..73d0c87c 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -131,6 +131,7 @@ def test_nlp_body(self): KEYWORDS = [u'great', u'good', u'flight', u'sailing', u'delays', u'smooth', u'thanksgiving', u'snow', u'weather', u'york', u'storm', u'winds', u'balloons', u'forecasters'] + self.article.build() self.article.nlp() # print self.article.summary # print self.article.keywords From 1973a9bd53fd6eae87e9c7e1671f70444108294d Mon Sep 17 00:00:00 2001 From: Karl Sutt Date: Wed, 23 Jul 2014 17:26:46 +0100 Subject: [PATCH 10/11] CNN's description has changed. Fix test accordingly --- tests/unit_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 73d0c87c..02d8b3ef 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -157,7 +157,7 @@ def test_source_build(self): builds a source object, validates it has no errors, prints out all valid categories and feed urls """ - DESC = """CNN.com delivers the latest breaking news and information on the latest top stories, weather, business, entertainment, politics, and more. For in-depth coverage, CNN.com provides special reports, video, audio, photo galleries, and interactive guides.""" + DESC = """CNN.com International delivers breaking news from across the globe and information on the latest top stories, business, sports and entertainment headlines. Follow the news as it happens through: special reports, videos, audio, photo galleries plus interactive maps and timelines.""" BRAND = 'cnn' config = Configuration() From 0d02da4d50d346e402c4797ea454c418a3a52684 Mon Sep 17 00:00:00 2001 From: Karl Sutt Date: Wed, 23 Jul 2014 17:27:15 +0100 Subject: [PATCH 11/11] :scissors: whitespace EOF --- tests/unit_tests.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 02d8b3ef..14cb28df 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -425,6 +425,3 @@ def test_spanish_fulltext_extract(self): suite.addTest(ArticleTestCase()) suite.addTest(APITestCase()) unittest.TextTestRunner().run(suite) # run custom subset - - -