From 9a458721fbdc95a9779ba0f871042db645804c5a Mon Sep 17 00:00:00 2001 From: codelucas Date: Mon, 29 Dec 2014 02:38:26 -0800 Subject: [PATCH 01/13] [bugfix] spelling correction Conflicts: newspaper/article.py --- newspaper/article.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/newspaper/article.py b/newspaper/article.py index 4c7a8290..59b4ce4c 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -365,7 +365,7 @@ def set_reddit_top_img(self): """ try: s = images.Scraper(self) - self.set_top_img_no_ckeck(s.largest_image_url()) + self.set_top_img_no_check(s.largest_image_url()) except Exception as e: log.critical('jpeg error with PIL, %s' % e) @@ -406,9 +406,9 @@ def set_top_img(self, src_url): if src_url is not None: s = images.Scraper(self) if s.satisfies_requirements(src_url): - self.set_top_img_no_ckeck(src_url) + self.set_top_img_no_check(src_url) - def set_top_img_no_ckeck(self, src_url): + def set_top_img_no_check(self, src_url): """Provide 2 APIs for images. One at "top_img", "imgs" and one at "top_image", "images" """ From fd012dae71e2458027c63784cb0f546672320f73 Mon Sep 17 00:00:00 2001 From: codelucas Date: Mon, 29 Dec 2014 03:03:04 -0800 Subject: [PATCH 02/13] [bugfix] meta/og "top image" extractions should not be filtered by size/dimensions as the server specifies that it's the "top image" Conflicts: newspaper/article.py --- newspaper/article.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/newspaper/article.py b/newspaper/article.py index 59b4ce4c..a9ec169f 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -365,7 +365,7 @@ def set_reddit_top_img(self): """ try: s = images.Scraper(self) - self.set_top_img_no_check(s.largest_image_url()) + self.set_top_img(s.largest_image_url()) except Exception as e: log.critical('jpeg error with PIL, %s' % e) @@ -400,7 +400,7 @@ def set_article_html(self, article_html): def set_meta_img(self, src_url): self.meta_img = encodeValue(src_url) - self.set_top_img(src_url) + self.set_top_img_no_check(src_url) def set_top_img(self, src_url): if src_url is not None: From c42c904829989441249a63514d59ad9bd84065d3 Mon Sep 17 00:00:00 2001 From: codelucas Date: Mon, 29 Dec 2014 03:46:20 -0800 Subject: [PATCH 03/13] Don't filter url GET params, it may ID the article Also includes unit test updates. Example article: http://www.hr-online.de/website/rubriken/nachrichten/indexhessen34938.jsp?rubrik=36094&key=standard_document_53891717 Conflicts: newspaper/urls.py tests/unit_tests.py --- newspaper/urls.py | 6 +++--- tests/unit_tests.py | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/newspaper/urls.py b/newspaper/urls.py index e83c319e..2d414882 100644 --- a/newspaper/urls.py +++ b/newspaper/urls.py @@ -83,12 +83,12 @@ def prepare_url(url, source_url=None): source_domain = urlparse(source_url).netloc proper_url = urljoin(source_url, url) proper_url = redirect_back(proper_url, source_domain) - proper_url = remove_args(proper_url) + # proper_url = remove_args(proper_url) else: - proper_url = remove_args(url) + # proper_url = remove_args(url) + proper_url = url except ValueError as e: log.critical('url %s failed on err %s' % (url, str(e))) - # print('url %s failed on err %s' % (url, str(e))) proper_url = '' return proper_url diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 50288066..389be4c0 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -91,11 +91,13 @@ def tearDown(self): def test_url(self): assert self.article.url == ( 'http://www.cnn.com/2013/11/27/travel/weather-' - 'thanksgiving/index.html') + 'thanksgiving/index.html?iref=allsearch') @print_test def test_download_html(self): - resp = mock_response_with(self.article.url, 'cnn_article') + self.canon_url = ('http://www.cnn.com/2013/11/27/travel/' + 'weather-thanksgiving/index.html') + resp = mock_response_with(self.canon_url, 'cnn_article') self.article.download(resp) assert len(self.article.html) == 75176 @@ -186,7 +188,7 @@ def test_pre_parse_nlp(self): """Test running NLP algos before parsing the article """ new_article = Article(self.article.url) - resp = mock_response_with(new_article.url, 'cnn_article') + resp = mock_response_with(self.canon_url, 'cnn_article') new_article.download(resp) self.assertRaises(ArticleException, new_article.nlp) @@ -259,8 +261,9 @@ def test_source_build(self): # assert s.brand == BRAND # assert s.description == DESC - # assert s.size() == 241 + # assert s.size() == 266 # assert s.category_urls() == CATEGORY_URLS + # TODO: A lot of the feed extraction is NOT being tested because feeds # are primarly extracted from the HTML of category URLs. We lose this # effect by just mocking CNN's main page HTML. Warning: tedious fix. From b5064bd661c4efbf1fc3df7352b8fc463e71cdb3 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Tue, 30 Dec 2014 03:21:27 -0800 Subject: [PATCH 04/13] Scaffolding for improving full-text-extraction -- tons of new unit tests --- tests/data/fulltext_domain_list.txt | 87 ++++++++++ tests/data/fulltext_url_list.txt | 166 +++++++++++++++++++ tests/data/text/247wallst.com1.txt | 0 tests/data/text/247wallst.com2.txt | 0 tests/data/text/about.com1.txt | 21 +++ tests/data/text/about.com2.txt | 43 +++++ tests/data/text/adoption.com1.txt | 29 ++++ tests/data/text/adoption.com2.txt | 11 ++ tests/data/text/al.com1.txt | 21 +++ tests/data/text/al.com2.txt | 3 + tests/data/text/apartmenttherapy.com1.txt | 1 + tests/data/text/apartmenttherapy.com2.txt | 3 + tests/data/text/architecturaldigest.com1.txt | 11 ++ tests/data/text/architecturaldigest.com2.txt | 1 + tests/data/text/avclub.com1.txt | 67 ++++++++ tests/data/text/avclub.com2.txt | 1 + tests/data/text/backstage.com1.txt | 1 + tests/data/text/backstage.com2.txt | 7 + tests/data/text/bhg.com1.txt | 21 +++ tests/data/text/bhg.com2.txt | 3 + tests/data/text/bloomberg.com1.txt | 77 +++++++++ tests/data/text/bloomberg.com2.txt | 1 + tests/data/text/bostonherald.com1.txt | 49 ++++++ tests/data/text/bostonherald.com2.txt | 5 + tests/data/text/businessinsider.com1.txt | 13 ++ tests/data/text/businessinsider.com2.txt | 35 ++++ tests/data/text/businessweek.com1.txt | 93 +++++++++++ tests/data/text/businessweek.com2.txt | 9 + tests/data/text/cleveland.com1.txt | 16 ++ tests/data/text/cleveland.com2.txt | 75 +++++++++ tests/data/text/cntraveler.com1.txt | 1 + tests/data/text/cntraveler.com2.txt | 1 + tests/data/text/coolhunting.com1.txt | 15 ++ tests/data/text/coolhunting.com2.txt | 15 ++ tests/data/text/cricket.com.au1.txt | 75 +++++++++ tests/data/text/cricket.com.au2.txt | 1 + tests/data/text/dailycaller.com1.txt | 25 +++ tests/data/text/dailycaller.com2.txt | 13 ++ tests/data/text/dailystar.co.uk1.txt | 57 +++++++ tests/data/text/dailystar.co.uk2.txt | 11 ++ tests/data/text/dallasnews.com1.txt | 3 + tests/data/text/dallasnews.com2.txt | 5 + tests/data/text/details.com1.txt | 0 tests/data/text/details.com2.txt | 1 + tests/data/text/elle.com1.txt | 1 + tests/data/text/elle.com2.txt | 65 ++++++++ tests/data/text/flavorwire.com1.txt | 27 +++ tests/data/text/flavorwire.com2.txt | 23 +++ tests/data/text/fool.com1.txt | 55 ++++++ tests/data/text/fool.com2.txt | 0 tests/data/text/foxbusiness.com1.txt | 57 +++++++ tests/data/text/foxbusiness.com2.txt | 23 +++ tests/data/text/foxnews.com1.txt | 45 +++++ tests/data/text/foxnews.com2.txt | 19 +++ tests/data/text/foxnews.com3.txt | 0 tests/data/text/foxnews.com4.txt | 3 + tests/data/text/glamour.com1.txt | 17 ++ tests/data/text/glamour.com2.txt | 5 + tests/data/text/globalnews.ca1.txt | 1 + tests/data/text/globalnews.ca2.txt | 1 + tests/data/text/gq.com1.txt | 19 +++ tests/data/text/gq.com2.txt | 43 +++++ tests/data/text/graziadaily.co.uk1.txt | 7 + tests/data/text/graziadaily.co.uk2.txt | 9 + tests/data/text/gulflive.com1.txt | 73 ++++++++ tests/data/text/gulflive.com2.txt | 3 + tests/data/text/huffingtonpost.de1.txt | 1 + tests/data/text/huffingtonpost.de2.txt | 1 + tests/data/text/lifebuzz.com1.txt | 5 + tests/data/text/lifebuzz.com2.txt | 5 + tests/data/text/livescience.com1.txt | 29 ++++ tests/data/text/livescience.com2.txt | 33 ++++ tests/data/text/mashable.com1.txt | 5 + tests/data/text/mashable.com2.txt | 17 ++ tests/data/text/mlive.com1.txt | 15 ++ tests/data/text/mlive.com2.txt | 35 ++++ tests/data/text/newyorker.com1.txt | 23 +++ tests/data/text/newyorker.com2.txt | 5 + tests/data/text/nj.com1.txt | 1 + tests/data/text/nj.com2.txt | 1 + tests/data/text/nola.com1.txt | 33 ++++ tests/data/text/nola.com2.txt | 1 + tests/data/text/nydailynews.com1.txt | 3 + tests/data/text/nydailynews.com2.txt | 27 +++ tests/data/text/nypost.com1.txt | 29 ++++ tests/data/text/nypost.com2.txt | 17 ++ tests/data/text/ok.co.uk1.txt | 19 +++ tests/data/text/ok.co.uk2.txt | 19 +++ tests/data/text/oregonlive.com1.txt | 33 ++++ tests/data/text/oregonlive.com2.txt | 13 ++ tests/data/text/parsely.com1.txt | 25 +++ tests/data/text/parsely.com2.txt | 49 ++++++ tests/data/text/pe.com1.txt | 1 + tests/data/text/pe.com2.txt | 69 ++++++++ tests/data/text/pewresearch.org1.txt | 25 +++ tests/data/text/pewresearch.org2.txt | 13 ++ tests/data/text/pixable.com1.txt | 1 + tests/data/text/pixable.com2.txt | 1 + tests/data/text/pixelmonkey.org1.txt | 35 ++++ tests/data/text/pixelmonkey.org2.txt | 23 +++ tests/data/text/readwrite.com1.txt | 0 tests/data/text/readwrite.com2.txt | 0 tests/data/text/recipe.com1.txt | 17 ++ tests/data/text/recipe.com2.txt | 29 ++++ tests/data/text/reuters.com1.txt | 17 ++ tests/data/text/reuters.com2.txt | 61 +++++++ tests/data/text/reuters.com3.txt | 47 ++++++ tests/data/text/reuters.com4.txt | 19 +++ tests/data/text/reuters.com5.txt | 31 ++++ tests/data/text/reuters.com6.txt | 0 tests/data/text/self.com1.txt | 7 + tests/data/text/self.com2.txt | 7 + tests/data/text/sitepoint.com1.txt | 0 tests/data/text/sitepoint.com2.txt | 81 +++++++++ tests/data/text/slate.com1.txt | 0 tests/data/text/slate.com2.txt | 3 + tests/data/text/space.com1.txt | 83 ++++++++++ tests/data/text/space.com2.txt | 25 +++ tests/data/text/syracuse.com1.txt | 1 + tests/data/text/syracuse.com2.txt | 9 + tests/data/text/talkingpointsmemo.com1.txt | 1 + tests/data/text/talkingpointsmemo.com2.txt | 37 +++++ tests/data/text/technologyreview.com1.txt | 45 +++++ tests/data/text/technologyreview.com2.txt | 15 ++ tests/data/text/teenvogue.com1.txt | 3 + tests/data/text/teenvogue.com2.txt | 3 + tests/data/text/telegraph.co.uk1.txt | 62 +++++++ tests/data/text/telegraph.co.uk2.txt | 45 +++++ tests/data/text/theatlantic.com1.txt | 21 +++ tests/data/text/theatlantic.com2.txt | 25 +++ tests/data/text/theatlanticcities.com1.txt | 11 ++ tests/data/text/theatlanticcities.com2.txt | 17 ++ tests/data/text/thedailybeast.com1.txt | 25 +++ tests/data/text/thedailybeast.com2.txt | 43 +++++ tests/data/text/thedebrief.co.uk1.txt | 23 +++ tests/data/text/thedebrief.co.uk2.txt | 39 +++++ tests/data/text/theglobeandmail.com1.txt | 55 ++++++ tests/data/text/theglobeandmail.com2.txt | 33 ++++ tests/data/text/thekitchn.com1.txt | 1 + tests/data/text/thekitchn.com2.txt | 1 + tests/data/text/thenextweb.com1.txt | 3 + tests/data/text/thenextweb.com2.txt | 3 + tests/data/text/theonion.com1.txt | 0 tests/data/text/theonion.com2.txt | 1 + tests/data/text/theroot.com1.txt | 19 +++ tests/data/text/theroot.com2.txt | 0 tests/data/text/tnr.com1.txt | 33 ++++ tests/data/text/tnr.com2.txt | 23 +++ tests/data/text/uproxx.com1.txt | 13 ++ tests/data/text/uproxx.com2.txt | 0 tests/data/text/upworthy.com1.txt | 35 ++++ tests/data/text/upworthy.com2.txt | 17 ++ tests/data/text/usnews.com1.txt | 21 +++ tests/data/text/usnews.com2.txt | 1 + tests/data/text/vanityfair.com1.txt | 0 tests/data/text/vanityfair.com2.txt | 0 tests/data/text/vogue.com1.txt | 7 + tests/data/text/vogue.com2.txt | 7 + tests/data/text/vogue.de1.txt | 3 + tests/data/text/vogue.de2.txt | 1 + tests/data/text/wetpaint.com1.txt | 21 +++ tests/data/text/wetpaint.com2.txt | 1 + tests/data/text/wired.com1.txt | 19 +++ tests/data/text/wired.com2.txt | 25 +++ tests/data/text/wnet.org1.txt | 9 + tests/data/text/wnet.org2.txt | 5 + tests/data/text/youbeauty.com1.txt | 27 +++ tests/data/text/youbeauty.com2.txt | 9 + tests/generate_fulltext.py | 57 +++++++ tests/generate_urls.py | 55 ++++++ tests/unit_tests.py | 66 +++++++- 171 files changed, 3618 insertions(+), 6 deletions(-) create mode 100644 tests/data/fulltext_domain_list.txt create mode 100644 tests/data/fulltext_url_list.txt create mode 100644 tests/data/text/247wallst.com1.txt create mode 100644 tests/data/text/247wallst.com2.txt create mode 100644 tests/data/text/about.com1.txt create mode 100644 tests/data/text/about.com2.txt create mode 100644 tests/data/text/adoption.com1.txt create mode 100644 tests/data/text/adoption.com2.txt create mode 100644 tests/data/text/al.com1.txt create mode 100644 tests/data/text/al.com2.txt create mode 100644 tests/data/text/apartmenttherapy.com1.txt create mode 100644 tests/data/text/apartmenttherapy.com2.txt create mode 100644 tests/data/text/architecturaldigest.com1.txt create mode 100644 tests/data/text/architecturaldigest.com2.txt create mode 100644 tests/data/text/avclub.com1.txt create mode 100644 tests/data/text/avclub.com2.txt create mode 100644 tests/data/text/backstage.com1.txt create mode 100644 tests/data/text/backstage.com2.txt create mode 100644 tests/data/text/bhg.com1.txt create mode 100644 tests/data/text/bhg.com2.txt create mode 100644 tests/data/text/bloomberg.com1.txt create mode 100644 tests/data/text/bloomberg.com2.txt create mode 100644 tests/data/text/bostonherald.com1.txt create mode 100644 tests/data/text/bostonherald.com2.txt create mode 100644 tests/data/text/businessinsider.com1.txt create mode 100644 tests/data/text/businessinsider.com2.txt create mode 100644 tests/data/text/businessweek.com1.txt create mode 100644 tests/data/text/businessweek.com2.txt create mode 100644 tests/data/text/cleveland.com1.txt create mode 100644 tests/data/text/cleveland.com2.txt create mode 100644 tests/data/text/cntraveler.com1.txt create mode 100644 tests/data/text/cntraveler.com2.txt create mode 100644 tests/data/text/coolhunting.com1.txt create mode 100644 tests/data/text/coolhunting.com2.txt create mode 100644 tests/data/text/cricket.com.au1.txt create mode 100644 tests/data/text/cricket.com.au2.txt create mode 100644 tests/data/text/dailycaller.com1.txt create mode 100644 tests/data/text/dailycaller.com2.txt create mode 100644 tests/data/text/dailystar.co.uk1.txt create mode 100644 tests/data/text/dailystar.co.uk2.txt create mode 100644 tests/data/text/dallasnews.com1.txt create mode 100644 tests/data/text/dallasnews.com2.txt create mode 100644 tests/data/text/details.com1.txt create mode 100644 tests/data/text/details.com2.txt create mode 100644 tests/data/text/elle.com1.txt create mode 100644 tests/data/text/elle.com2.txt create mode 100644 tests/data/text/flavorwire.com1.txt create mode 100644 tests/data/text/flavorwire.com2.txt create mode 100644 tests/data/text/fool.com1.txt create mode 100644 tests/data/text/fool.com2.txt create mode 100644 tests/data/text/foxbusiness.com1.txt create mode 100644 tests/data/text/foxbusiness.com2.txt create mode 100644 tests/data/text/foxnews.com1.txt create mode 100644 tests/data/text/foxnews.com2.txt create mode 100644 tests/data/text/foxnews.com3.txt create mode 100644 tests/data/text/foxnews.com4.txt create mode 100644 tests/data/text/glamour.com1.txt create mode 100644 tests/data/text/glamour.com2.txt create mode 100644 tests/data/text/globalnews.ca1.txt create mode 100644 tests/data/text/globalnews.ca2.txt create mode 100644 tests/data/text/gq.com1.txt create mode 100644 tests/data/text/gq.com2.txt create mode 100644 tests/data/text/graziadaily.co.uk1.txt create mode 100644 tests/data/text/graziadaily.co.uk2.txt create mode 100644 tests/data/text/gulflive.com1.txt create mode 100644 tests/data/text/gulflive.com2.txt create mode 100644 tests/data/text/huffingtonpost.de1.txt create mode 100644 tests/data/text/huffingtonpost.de2.txt create mode 100644 tests/data/text/lifebuzz.com1.txt create mode 100644 tests/data/text/lifebuzz.com2.txt create mode 100644 tests/data/text/livescience.com1.txt create mode 100644 tests/data/text/livescience.com2.txt create mode 100644 tests/data/text/mashable.com1.txt create mode 100644 tests/data/text/mashable.com2.txt create mode 100644 tests/data/text/mlive.com1.txt create mode 100644 tests/data/text/mlive.com2.txt create mode 100644 tests/data/text/newyorker.com1.txt create mode 100644 tests/data/text/newyorker.com2.txt create mode 100644 tests/data/text/nj.com1.txt create mode 100644 tests/data/text/nj.com2.txt create mode 100644 tests/data/text/nola.com1.txt create mode 100644 tests/data/text/nola.com2.txt create mode 100644 tests/data/text/nydailynews.com1.txt create mode 100644 tests/data/text/nydailynews.com2.txt create mode 100644 tests/data/text/nypost.com1.txt create mode 100644 tests/data/text/nypost.com2.txt create mode 100644 tests/data/text/ok.co.uk1.txt create mode 100644 tests/data/text/ok.co.uk2.txt create mode 100644 tests/data/text/oregonlive.com1.txt create mode 100644 tests/data/text/oregonlive.com2.txt create mode 100644 tests/data/text/parsely.com1.txt create mode 100644 tests/data/text/parsely.com2.txt create mode 100644 tests/data/text/pe.com1.txt create mode 100644 tests/data/text/pe.com2.txt create mode 100644 tests/data/text/pewresearch.org1.txt create mode 100644 tests/data/text/pewresearch.org2.txt create mode 100644 tests/data/text/pixable.com1.txt create mode 100644 tests/data/text/pixable.com2.txt create mode 100644 tests/data/text/pixelmonkey.org1.txt create mode 100644 tests/data/text/pixelmonkey.org2.txt create mode 100644 tests/data/text/readwrite.com1.txt create mode 100644 tests/data/text/readwrite.com2.txt create mode 100644 tests/data/text/recipe.com1.txt create mode 100644 tests/data/text/recipe.com2.txt create mode 100644 tests/data/text/reuters.com1.txt create mode 100644 tests/data/text/reuters.com2.txt create mode 100644 tests/data/text/reuters.com3.txt create mode 100644 tests/data/text/reuters.com4.txt create mode 100644 tests/data/text/reuters.com5.txt create mode 100644 tests/data/text/reuters.com6.txt create mode 100644 tests/data/text/self.com1.txt create mode 100644 tests/data/text/self.com2.txt create mode 100644 tests/data/text/sitepoint.com1.txt create mode 100644 tests/data/text/sitepoint.com2.txt create mode 100644 tests/data/text/slate.com1.txt create mode 100644 tests/data/text/slate.com2.txt create mode 100644 tests/data/text/space.com1.txt create mode 100644 tests/data/text/space.com2.txt create mode 100644 tests/data/text/syracuse.com1.txt create mode 100644 tests/data/text/syracuse.com2.txt create mode 100644 tests/data/text/talkingpointsmemo.com1.txt create mode 100644 tests/data/text/talkingpointsmemo.com2.txt create mode 100644 tests/data/text/technologyreview.com1.txt create mode 100644 tests/data/text/technologyreview.com2.txt create mode 100644 tests/data/text/teenvogue.com1.txt create mode 100644 tests/data/text/teenvogue.com2.txt create mode 100644 tests/data/text/telegraph.co.uk1.txt create mode 100644 tests/data/text/telegraph.co.uk2.txt create mode 100644 tests/data/text/theatlantic.com1.txt create mode 100644 tests/data/text/theatlantic.com2.txt create mode 100644 tests/data/text/theatlanticcities.com1.txt create mode 100644 tests/data/text/theatlanticcities.com2.txt create mode 100644 tests/data/text/thedailybeast.com1.txt create mode 100644 tests/data/text/thedailybeast.com2.txt create mode 100644 tests/data/text/thedebrief.co.uk1.txt create mode 100644 tests/data/text/thedebrief.co.uk2.txt create mode 100644 tests/data/text/theglobeandmail.com1.txt create mode 100644 tests/data/text/theglobeandmail.com2.txt create mode 100644 tests/data/text/thekitchn.com1.txt create mode 100644 tests/data/text/thekitchn.com2.txt create mode 100644 tests/data/text/thenextweb.com1.txt create mode 100644 tests/data/text/thenextweb.com2.txt create mode 100644 tests/data/text/theonion.com1.txt create mode 100644 tests/data/text/theonion.com2.txt create mode 100644 tests/data/text/theroot.com1.txt create mode 100644 tests/data/text/theroot.com2.txt create mode 100644 tests/data/text/tnr.com1.txt create mode 100644 tests/data/text/tnr.com2.txt create mode 100644 tests/data/text/uproxx.com1.txt create mode 100644 tests/data/text/uproxx.com2.txt create mode 100644 tests/data/text/upworthy.com1.txt create mode 100644 tests/data/text/upworthy.com2.txt create mode 100644 tests/data/text/usnews.com1.txt create mode 100644 tests/data/text/usnews.com2.txt create mode 100644 tests/data/text/vanityfair.com1.txt create mode 100644 tests/data/text/vanityfair.com2.txt create mode 100644 tests/data/text/vogue.com1.txt create mode 100644 tests/data/text/vogue.com2.txt create mode 100644 tests/data/text/vogue.de1.txt create mode 100644 tests/data/text/vogue.de2.txt create mode 100644 tests/data/text/wetpaint.com1.txt create mode 100644 tests/data/text/wetpaint.com2.txt create mode 100644 tests/data/text/wired.com1.txt create mode 100644 tests/data/text/wired.com2.txt create mode 100644 tests/data/text/wnet.org1.txt create mode 100644 tests/data/text/wnet.org2.txt create mode 100644 tests/data/text/youbeauty.com1.txt create mode 100644 tests/data/text/youbeauty.com2.txt create mode 100644 tests/generate_fulltext.py create mode 100644 tests/generate_urls.py diff --git a/tests/data/fulltext_domain_list.txt b/tests/data/fulltext_domain_list.txt new file mode 100644 index 00000000..94c72d8b --- /dev/null +++ b/tests/data/fulltext_domain_list.txt @@ -0,0 +1,87 @@ +about.com +al.com +annarbor.com +apartmenttherapy.com +architecturaldigest.com +blog.parsely.com +pixelmonkey.org +bostonherald.com +businessinsider.com +cleveland.com +cntraveler.com +coolhunting.com +dallasnews.com +deseretdigital.com +details.com +elle.com +fool.com +foxbusiness.com +foxnews.com +latino.foxnews.com +smallbusiness.foxnews.com +globeandmail.com +huffingtonpost.de +in.reuters.com +reuters.com +uk.reuters.com +mashable.com +mlive.com +oregonlive.com +nydailynews.com +nola.com +nj.com +newyorker.com +readwrite.com +slate.com +thenextweb.com +thekitchn.com +thedailybeast.com +theatlantic.com +theatlanticcities.com +theonion.com +upworthy.com +wired.com +wetpaint.com +vogue.de +vogue.com +vanityfair.com +telegraph.co.uk +pewresearch.org +dailystar.co.uk +backstage.com +syracuse.com +slive.com +globalnews.ca +uproxx.com +dailycaller.com +pe.com +avclub.com +usnews.com +talkingpointsmemo.com +ok.co.uk +glamour.com +gulflive.com +247wallst.com +flavorwire.com +businessweek.com +bloomberg.com +self.com +thedebrief.co.uk +graziadaily.co.uk +theroot.com +livescience.com +cricket.com.au +lifebuzz.com +wnet.org +gq.com +tnr.com +teenvogue.com +pixable.com +space.com +adoption.com +youbeauty.com +sitepoint.com +technologyreview.com +nypost.com +recipe.com +bhg.com diff --git a/tests/data/fulltext_url_list.txt b/tests/data/fulltext_url_list.txt new file mode 100644 index 00000000..b8db2777 --- /dev/null +++ b/tests/data/fulltext_url_list.txt @@ -0,0 +1,166 @@ +http://bandb.about.com/od/illinois/fl/Galena-IL-A-BampB-Kinda-Town.htm?utm_source=google&utm_medium=social&utm_campaign=shareurlbuttons +http://lungcancer.about.com/od/Targeted-Therapy-Lung-Cancer/fl/Genetic-Testing-for-People-with-Lung-Cancer.htm?utm_source=google&utm_medium=social&utm_campaign=shareurlbuttons +http://www.al.com/sports/index.ssf/2014/12/lou_holtz_ohio_state_defender.html +http://www.al.com/business/index.ssf/2014/12/hockey_boat_show_and_future_ci.html +http://www.mlive.com/news/ann-arbor/index.ssf/2014/12/police_release_surveillance_ph.html#incart_river +http://www.mlive.com/lions/index.ssf/2014/12/detroit_lions_ndamukong_suh_wi_2.html#incart_most-read +http://blog.parsely.com/post/1539/facebook-and-publishers-a-fix-for-direct-traffic-from-facebook-mobiles-app/ +http://blog.parsely.com/post/1511/the-ultimate-referral-guide-to-your-audience/ +http://latino.foxnews.com/latino/entertainment/2014/12/29/how-oscar-isaac-and-jessica-chastain-went-about-bringing-to-life-most-violent/ +http://latino.foxnews.com/latino/news/2014/12/23/teen-desperate-for-baby-kills-8-month-pregnant-woman-and-unborn-child-in-mexico/ +http://www.theglobeandmail.com/news/world/airasia-search/article22224253/ +http://www.theglobeandmail.com/report-on-business/international-business/oil-prices-and-interest-rates-what-to-watch-in-2015/article22233509/ +http://in.reuters.com/article/2014/12/30/india-sensex-nifty-idINKBN0K80A020141230 +http://in.reuters.com/article/2014/12/30/indonesia-airplane-idINKBN0K703120141230 +http://uk.reuters.com/article/2014/12/30/uk-indonesia-airplane-idUKKBN0K601G20141230 +http://uk.reuters.com/article/2014/12/29/uk-health-ebola-britain-idUKKBN0K71C820141229 +http://www.telegraph.co.uk/news/worldnews/asia/indonesia/11315001/AirAsia-plane-reported-missing-with-162-passengers-onboard-latest.html +http://www.telegraph.co.uk/news/politics/margaret-thatcher/11313354/Margaret-Thatcher-feared-GCSEs-would-lower-school-standards.html +http://www.dailystar.co.uk/news/latest-news/417544/Ebola-virus-Glasgow-Scotland-Sierra-Leone +http://www.dailystar.co.uk/showbiz/417443/Helen-Wood-cheating-boyfriend-column +http://www.ok.co.uk/celebrity-news/sean-penn-charlize-theron-engaged +http://www.ok.co.uk/celebrity-news/eastenders-kat-sex-offender-uncle-harry +http://www.self.com/flash/beauty-blog/2014/12/taylor-swift-headband/ +http://www.self.com/flash/beauty-blog/2014/12/easy-hairstyles-new-years-eve/ +http://www.thedebrief.co.uk/2014/12/ed-sheeran-wants-to-set-up-taylor-swift-but-we-re-not-so-sure-about-his-matchmaking-skills +http://www.thedebrief.co.uk/2014/11/in-which-we-discuss-whether-taylor-s-or-beyonce-s-lyrics-are-better-break-up-material#.VKJZfsCsA +http://www.graziadaily.co.uk/2014/12/lady-gaga-shiseido-campaign#.VKJZhsCsA +http://www.graziadaily.co.uk/2014/12/inside-our-trip-to-gstaad-with-a-small-world-and-carey-mulligan#.VKJZosCsA +http://www.cricket.com.au/news/match-report/third-test-day-five-australia-v-india-mcg/2014-12-30 +http://www.cricket.com.au/video/video-glenn-maxwell-leaves-gets-bowled-big-bash-league-stars-v-heat/2014-12-28 +http://www.teenvogue.com/entertainment/2014-12/disney-channel-original-movies-zenon-smart-house/?slide=1 +http://www.teenvogue.com/celebrity-style/red-carpet/2014-12/breakout-style-stars-of-the-year/?slide=1 +http://www.pixable.com/article/sofia-vergara-and-joe-manganiello-are-engaged-54364 +http://www.pixable.com/article/pop-culture-trends-we-hope-stay-in-2014 +http://www.recipe.com/cheesy-potato-bake-with-eggs/ +http://www.recipe.com/blogs/cooking/make-ahead-side-sweet-potato-parsnip-and-celery-root-mash/ +http://www.apartmenttherapy.com/knockyoursocksoff-interactive-childrens-books-holiday-gift-guide-from-apartment-therapy-213998#comments +http://www.apartmenttherapy.com/before-after-a-year-of-ikea-hacks-best-of-2014-214389 +http://architecturaldigest.com/decor/2015-01/marwan-al-sayed-jan-showers-designed-arizona-home-article +http://architecturaldigest.com/video/promotion/jenn-air-video +http://www.pixelmonkey.org/2012/12/30/mobile-in-1998 +http://www.pixelmonkey.org/2013/09/03/parse-ly-funding-coverage +http://bostonherald.com/entertainment/movies/movie_news/2014/12/selma_ava_duvernay_and_oprah_winfrey_turn_60s_civil_unrest +http://bostonherald.com/gift_guide/sports_items/2014/12/chrome_industries_mini_buran_laptop_messenger_bag_is_meant_to_last +http://www.businessinsider.com/david-cenciotti-argentinas-fighter-purchase-may-threaten-britain-2014-12 +http://www.businessinsider.com/financial-advisor-insights-december-29-2014-12#comments +http://www.cleveland.com/food/index.ssf/2014/12/let_me_google_that_for_you_201.html#comments +http://www.cleveland.com/books/index.ssf/2014/12/best_books_for_young_readers_e.html +http://cntraveler.com/galleries/2014-12-08/marrakech-morocco-best-shopping-guide +http://cntraveler.com/galleries/2014-12-24/what-the-airlines-did-well-in-2014-american-jetblue-ryanair +http://www.coolhunting.com/travel/monastero-santa-rosa-hotel-spa-on-the-almafi-coast +http://coolhunting.com/travel/monastero-santa-rosa-hotel-spa-on-the-almafi-coast +http://letterstotheeditorblog.dallasnews.com/2014/12/kindhearted-police-with-holiday-spirit.html/' +http://dallasmorningviewsblog.dallasnews.com/2014/12/footballs-growing-caste-system-as-more-parents-say-no.html/feed/ +http://details.com/watch/black-on-black-watches-for-men +http://details.com/fashion-style/perfect-wardrobe/201408/gray-shirts-sweaters-pants-suits-coats-for-fall +http://elle.com/runway/ready-to-wear/spring-2015-rtw/nicole-miller/collection/ +http://www.elle.com/news/lifestyle/passive-aggressive-holiday-digs +http://fool.com/investing/general/2014/12/22/7-top-stocks-to-buy-for-2015.aspx#1024571 +http://fool.com/../../personal-finance/2010/07/30/5-fixes-for-female-money-problems.aspx +http://smallbusiness.foxbusiness.com/marketing-sales/2014/12/16/7-powerful-ways-to-convert-neutral-traffic-into-paying-customers/ +http://smallbusiness.foxbusiness.com/marketing-sales/2014/12/23/how-to-deal-with-scrooge-client/ +http://foxnews.com/2014/12/23/he-cannot-run-city-new-york-oreilly-rips-incompetent-mayor-de-blasio +http://radio.foxnews.com/2014/12/25/afmw-michael-w-smith-michael-w-smith-friends-the-spirit-of-christmas-video/ +http://huffingtonpost.de/news/missing-airasia-plane/ +http://huffingtonpost.de/news/inspiration +http://reuters.com/article/2014/12/29/uk-google-china-idUKKBN0K70BJ20141229 +http://reuters.com/article/businessNews/idFRKBN0K70N920141229 +http://mashable.com/2010/01/01/im-feeling-lucky-google-new-year/ +http://mashable.com/2014/12/22/stansted-power-outage/ +http://www.oregonlive.com/business/index.ssf/2014/12/alzheimers_foundation_comcast.html +http://www.oregonlive.com/pacific-northwest-news/index.ssf/2014/12/missionary_from_oregon_pleads.html#comments +http://nydailynews.com/autos/news +http://www.nydailynews.com/photos/news +http://www.nola.com/jazzfest/index.ssf/2014/12/multi-million-dollar_george_an.html +http://www.nola.com/business/index.ssf/2014/11/new_orleans_business_names_in_20.html +http://nj.com/news/article/3875334678772745563/bayonne-46-at-st-mary-eliz-77-in-season-tournament-boys-basketball/#comments +http://videos.nj.com/2013/09/raw_footage_seaside_boardwalk.html +http://www.newyorker.com/magazine/2014/12/22/climate-change-christmas-carols +http://video.newyorker.com/watch/annals-of-sound-at-home-with-meredith-monk +http://readwrite.com/2014/12/29/microsoft-browser-spartan-internet-explorer-chrome +http://readwrite.com/2014/12/29/pinterest-opens-up-promoted-pins +http://www.slate.com/articles/technology/the_back_end/2014/10/the_new_seven_wonders_defining_the_top_technological_marvels_of_the_contemporary.html +http://www.slate.com/blogs/the_spot/2014/06/10/brazil_world_cup_protests_striking_images_of_the_country_s_anti_world_cup.html +https://deals.thenextweb.com/sales/free-anime-for-30-days-from-crunchyroll?utm_campaign=feed&utm_medium=RSS&utm_source=thenextweb +http://thenextweb.com/apps/2014/12/12/google-maps-finally-adds-lane-guidance-confused-european-drivers/feed/ +http://www.thekitchn.com/before-after-the-details-make-a-difference-reader-kitchen-remodel-214061#comments +http://www.thekitchn.com/drink-up-15-good-ideas-for-beer-and-wine-the-kitchns-best-of-2014-214146#comments +http://www.thedailybeast.com/articles/2014/12/16/the-vatican-s-rare-nod-to-american-nuns.html +http://www.thedailybeast.com/articles/2014/12/22/justice-league-vigil-for-slain-nypd-officers-asks-whose-life-matters.html +http://theatlantic.com/entertainment/archive/2014/12/cracking-the-sitcom-code/384068/ +http://www.theatlantic.com/entertainment/archive/2014/12/the-interview-is-a-pretty-smart-movie/384082/#disqus_thread +http://theatlanticcities.com/crime/2014/12/tourism-to-antarctica-could-be-making-penguins-sick/383979/#disqus_thread +http://theatlanticcities.com/weather/2014/11/a-climate-model-of-white-christmasesthrough-2100/383143/#disqus_thread +http://theonion.com/articles/i-dont-see-race-i-only-see-grayishbrown-vaguely-hu,37667/ +http://theonion.com/video/greyhound-now-offering-premium-upgrade-to-slightly,37261/ +http://upworthy.com/what-a-service-dog-really-thinks-of-the-injured-vet-who-owns-him?c=tpstream +http://www.upworthy.com/theyre-innocent-looking-enough-but-actually-quite-dangerous +http://www.wired.com/2014/12/butterflies-get-shine/ +http://www.wired.com/2014/12/finney-swat/ +http://www.wetpaint.com/the-bachelor/articles/2014-11-04-whitney-bischoff-eliminated-season-19 +http://www.wetpaint.com/the-bachelor/articles/2014-12-29-alissa-giambrone-contestant-season-19 +http://vogue.de/beauty/beauty-blog/lieblingsstueck-die-iphone-huelle-von-iphoria +http://vogue.de/mode/mode-trends/weihnachten-die-redaktion-zeigt-ihre-christmas-sweater +http://www.vogue.com/slideshow/3619755/ +http://vogue.com/collection/fallwinter2014/ready-to-wear/ +http://vanityfair.com/show/cinema/14/12/18/baci-piu-belli-del-2014-film-serie-tv +http://vanityfair.com/show/tv/14/12/22/serie-tv-coppie-piu-belle-del-2014 +http://www.pewresearch.org/fact-tank/2014/11/20/those-from-mexico-will-benefit-most-from-obamas-executive-action/ +http://www.pewresearch.org/fact-tank/2014/12/16/gas-prices-consumer-sentiment/ +http://backstage.com/casting/coney-island-55854/actors-208609/ +http://backstage.com/advice-for-actors/resources/los-angeles-acting-schools-and-coaches-2014/ +http://www.syracuse.com/living/index.ssf/2014/12/kwanzaa_is_celebrated_at_beauchamp_library.html#comments +http://www.syracuse.com/living/index.ssf/2014/12/echos_of_the_past_the_hotel_syracuse_historic_ghostwalk.html#comments +http://globalnews.ca/video/1747657/video-of-the-gardners-hours-before-birth-of-quadruplets +http://globalnews.ca/video/1747659/quad-squad-born-in-utah/ +http://uproxx.com/webculture/2014/12/a-reddit-post-about-a-fake-ebola-like-illness-sparked-a-panic-in-and-around-an-arizona-town/ +http://uproxx.com/filmdrunk/2014/12/simon-pegg-gave-his-blessing-for-a-touring-shaun-of-the-dead-stage-show/feed/ +http://dailycaller.com/2014/12/25/happy-kwanzaa-the-holiday-brought-to-you-by-the-fbi/ +http://dailycaller.com/2014/12/27/jennifer-lawrence-made-an-appearance-at-the-louisville-kentucky-flop-fest-video/ +http://pe.com/los-angeles-CA/legal/legal-services/The-Law-Offices-of-Jacob-Emrani-951-268-7555 +http://pe.com/sections/news/riverside-county/lake-elsinore/ +http://avclub.com/article/best-film-scenes-2014-212903 +http://avclub.com/article/finally-mail-kimp-remix-you-never-knew-you-wanted-212941 +http://www.usnews.com/news/blogs/run-2016/2014/10/31/tracking-the-2016-presidential-candidates +http://video.usnews.com/Cigar-rollers-in-Havana-thrilled-with-warming-USCuba-policy-28285271 +http://media.talkingpointsmemo.com/slideshow/astronomy-photographer-competition-2013 +http://talkingpointsmemo.com/muckraker/mark-shurtleff-john-swallow-wildest-details +http://glamour.com/fashion/blogs/dressed/2014/09/rihanna-leaves-pants-at-home-f +http://glamour.com/weddings/blogs/save-the-date/2014/12/simple-wedding-dresses +http://blog.gulflive.com/mississippi-press-living/2014/12/sampling_history_new_year_offe.html +http://videos.gulflive.com/2013/10/crown_equity_holdings_could_be.html +http://247wallst.com/investing/2014/12/29/the-4-stocks-that-slimmed-the-djia-monday-slide/feed/ +http://247wallst.com/investing/2014/12/29/the-52-week-low-club-for-monday-19/feed/ +http://flavorwire.com/495982/13-media-moments-from-2014-we-could-have-lived-without#comments +http://flavorwire.com/495418/2014-the-year-the-outrage-machine-started-eating-the-real-world +http://www.businessweek.com/articles/2014-12-18/afghanistans-new-rich-navigate-u-dot-s-dot-pullout#r=nav-fst +http://www.businessweek.com/articles/2014-12-22/cuomos-cap-on-hedge-funds-in-pensions-wont-solve-the-real-problem#r=nav-f-story +http://www.bloomberg.com/news/2014-09-30/job-woes-linger-in-29-states-as-u-s-recovers-unevenly.html +http://bloomberg.com/infographics/2014-08-22/ukraine-russia-map.html +http://www.theroot.com/articles/culture/2014/12/rev_sharpton_responds_to_his_critics.html +http://theroot.com/list/The-Root-100/video/The-Root-100-Panel-Discussion-2 +http://livescience.com/48502-magic-mushrooms-change-brain-networks.html +http://livescience.com/48543-how-zombies-evolved-in-pop-culture.html +http://lifebuzz.com/colbie-caillat-got-tired-of-being-photoshopped-so-heres-what-she-did-about-it/ +http://lifebuzz.com/what-do-you-think-this-guy-is-doing-you-will-never-guess-and-its-going-to-break-your-heart/ +http://www.wnet.org/blog/2013/12/16/audit-committee/ +http://www.wnet.org/blog/2013/12/16/wnet-interactive-technology-committee-meetings/ +http://www.gq.com/style/blogs/the-gq-eye/2014/05/how-to-pack-everything-for-your-3-day-weekend-trip.html +http://gq.com/blogs/the-feed/2014/06/motorcycle-bikes-gear-buying-guide.html +http://tnr.com/article/120169/canadas-former-liberal-party-leader-offers-advice-young-liberals +http://tnr.com/article/120578/global-warming-threshold-what-2-degrees-celsius-36-f-looks +http://space.com/47-mars-the-red-planet-fourth-planet-from-the-sun.html +http://space.com/27998-nasa-18-billion-omnibus-spending-bill.html +http://adoption.com/what-not-to-say-to-a-birth-mom-or-adoptee/ +http://forums.adoption.com/search-birthfamily-adoptee/222937-if-you-looking-siblings-please-post-here.html +http://youbeauty.com/skin/our-favorite-pop-culture-nail-art +http://youbeauty.com/face/we-tried-it-clinique-acne-solutions-powder-makeup-in-golden +http://www.sitepoint.com/average-page-weight-increases-15-2014/feed/ +http://www.sitepoint.com/3-ways-implement-embeddable-custom-badges/ +http://www.technologyreview.com/featuredstory/532796/who-owns-the-biggest-biotech-discovery-of-the-century/ +http://www.technologyreview.com/news/532896/discarded-laptop-batteries-keep-the-lights-on/ +http://nypost.com/2014/12/18/amazon-macmillan-make-peace-over-book-pricing/ +http://nypost.com/2014/12/29/surfer-survives-jaws-moment-with-great-white-shark/ +http://bhg.com/thanksgiving/indoor-decorating/centerpiece-and-tabletop-decoration-ideas-fall/ +http://www.bhg.com/videos/m/93269792/how-to-hang-a-christmas-wreath-two-no-fail-secrets.htm diff --git a/tests/data/text/247wallst.com1.txt b/tests/data/text/247wallst.com1.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/text/247wallst.com2.txt b/tests/data/text/247wallst.com2.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/text/about.com1.txt b/tests/data/text/about.com1.txt new file mode 100644 index 00000000..6332a69f --- /dev/null +++ b/tests/data/text/about.com1.txt @@ -0,0 +1,21 @@ +Which US town is popular with generals? Galena in Jo Daviess County, Illinois, of course! Nine Civil War generals who settled in this town, including Galena’s favorite son - Ulysses S. Grant. After his victory in 1865, General Grant received a hero’s welcome from the town and was presented with a handsome Italianate-style red brick mansion. Grant’s Home is now a National Historic Landmark and you can view its original furnishings including Grant’s favorite chair. To find out more about the 18th President of the United States, you can also the town's Grant’s Leather Store and the Galena & U.S. Grant Museum. + +But generals aren't the only ones who like this quinetessential small town. "Conde Nast Taveler" rated it as the “Second Friendliest City in the US” and “14th Friendliest City in the World”, while "Forbes" Magazine listed it one of “America’s Prettiest Towns.” + +Galena derived its name from the rich lead ore or galena (in Latin) deposits, which the original Native Americans used for their body paints. The town is rich in history with eighty-five percent of its buildings listed on the National Register of Historic Places. Step back in time to roam its streets where Abraham Lincoln and Ulysses S. Grant once walked. Admire its well-preserved unique period architecture heritage ranging from 19th-century commercial-style buildings lining Main Street to the grand residential mansions showcasing the Greek Revival and later Victorian styles. + +One historic landmark not to be missed is the DeSoto House Hotel on Main Street, the oldest operating hotel in Illinois. Since its opening in 1850, the hotel has hosted many major historical events and famous guests such as Mark Twain, Theodore Roosevelt and Ralph Waldo Emerson. Take a free self-guided tour to visit the balcony decorated with red, white and blue bunting from where Abraham Lincoln once gave his speech; rooms 209 and 211 from which Ulysses S. Grant planned and conducted his presidential campaigns; and the Quiet Room where you watch videos about the rich history of Galena and the hotel. + +When visiting Galena, don’t miss the opportunity to professionally decorate your own personal dining set and enjoy good food at the same time. Book at the Stone House Pottery and Gallery. Master potter Charles Fach and his wife Sandra will get you a three-piece dining set of dinner plate, salad/dessert plate and bowl. They then teach you how to decorate, glaze and fire your dinnerware. The next day, a Galena restaurant provides dinner consisting of an appetizer, salad, entrée and dessert that you will enjoy on your newly created plates and bowl. + +Galena boasts many vineyards and wineries. Join the Blackhawk Wine Tour to swirl, sniff and sip the best wines in Illinois at three different wineries and explore the scenic views of Jo Daviess County as well. The tour begins at Galena Cellars Vineyard & Winery, the next stop is Rocky Waters Winery and the final call is the Massbach Ridge Winery. Besides wine tasting, you'll also have the opportunity to buy limited-edition wines to take back home. Included in the tour is lunch at Procento’s Pizzeria, one of Galena’s best restaurants, along with complimentary water bottles, and pickup and drop-off at your B&B. + +You don’t need to go on a tour to sample Galena Cellar’s award-winning red, white and fruit wines. Just visit their tasting room and gift shop in a restored 1840’s grain building at downtown Galena. Alternatively, beer lovers can head to Galena Brewery on Main Street to taste their flagship and seasonal brews while enjoying tapas, soups, sandwiches and live entertainment. + +Great wines should be paired with great food. Steak aficionados will love Log Cabin Steakhouse on Main Street, famous for its big and premium aged Angus beef, hand-cut daily. Sample their excellent 16-oz New York Cut or tender and delicious 24-ounce T-bone or the grand 32-ounce porterhouse steak and you won’t be disappointed. + +Enjoy French and German cooking? Then head to Fritz and Frites to try the French onion soup with baked Gruyere and Parmesan cheese crust; escargots de Bourgogne; the coq au-vin (braised chicken with red wine sauce); citron et bierre mussels (with white ale and lemon); steak frites (rib-eye steak with parsley butter and pommes frites). From the German side of the menu there's kassler rippchen (smoked pork chop with a cider glaze); Wiener schnitzels (breaded veal cutlets), and sauerbraten (beef marinated in a sweet and sour sauce) served with red cabbage or sauerkraut. The family-owned bistro serves European meals with imported wines and beers at affordable prices. + +For your treats, walk into The Great American Popcorn Company on Main Street and you'll be welcomed with a friendly smile and a fresh, warm sample of their famous old-fashioned caramel corn. Choose from a daily selection of 50 delicious flavors of gourmet popcorn. In addition, the store offers handmade chocolates, candies, home-made fudge and ice cream. The store was featured on MSNBC’s Your Business and The Today Show. + +Find out more reasons why Galena is a B&B Kinda Town here. \ No newline at end of file diff --git a/tests/data/text/about.com2.txt b/tests/data/text/about.com2.txt new file mode 100644 index 00000000..68418aaa --- /dev/null +++ b/tests/data/text/about.com2.txt @@ -0,0 +1,43 @@ +Updated August 02, 2014. + + + +Written or reviewed by a board-certified physician. See About.com's Medical Review Board. + +If you’ve been recently diagnosed with lung cancer, especially lung adenocarcinoma, your oncologist may have talked to you about genetic testing (otherwise known as molecular profiling or biomarker testing) of your tumor. It's now recommended that all lung cancer patients with advanced or metastatic lung adenocarcinoma (a type of non-small cell lung cancer) have biomarker testing to look for EGFR mutations and ALK rearrangements. In addition, patients with other forms of non-small cell lung cancer (for example, adenosquamous carcinoma in non-smokers) should also be considered for testing. What does this mean? + +Genetic testing involves tests that a pathologist performs in the lab using a sample of your cancer tissue - tests that look at the cancer from a molecular level. This tissue may come from a biopsy of your tumor, or from tissue removed during surgery for lung cancer. The reason behind this is that cancers have gene mutations that "drive" or control the growth of the cancer. Simplistically, if these mutations can be identified, then treatments can be used which "target" these mutations, hence stopping the growth of the cancer. It is these mutations that lead to the development of a cancer in the first place. + +Before going further it's helpful to address something that is confusing for many people. There are two primary types of gene mutations. + +One type of mutations are hereditary mutations (also called germline mutations,) meaning you inherit genes with mutations from one or more parents. Common examples of these mutations include hemophilia associated as well as mutations that may predispose someone to developing breast cancer (BRCA1 and BRCA2 mutations.) + +The type of mutations that scientists look for in people with lung cancer are instead called acquired mutations (also called somatic mutations.) These mutations are not present at birth (they do not run in families) but rather develop in the process of cells becoming cancerous. + +Gene mutations are changes to a particular gene in a chromosome. All genes are made up of variable sequences of 4 amino acids (called bases); adenine, tyrosine, cytosine, and guanine. When a gene is exposed to toxins in the environment, or when an accident occurs in cell division, a mutation (change) may occur. In some cases it may mean that one base is substituted for another, say adenine instead of guanine. In other cases bases may be inserted, or deleted, or genes may be rearranged in some way. + +Why are oncologists interested in acquired gene mutations in a tumor? First, we should talk about the two types of acquired mutations found in lung cancers. One type of mutation is termed a driver mutation. These mutations, via several mechanisms, “drive” the growth of a tumor. In lung cancer the number of driver mutations is variable. In one study, an average of 11 driver mutations per cancer were found. Another type of mutation is termed a passenger mutation. Just as someone may be a passenger in a car, these genes do not drive the cancer and are basically along for the ride. Again we don’t know exactly how many passenger mutations are present in a tumor (and the number varies from tumor to tumor) but some tumors may have more than 1,000 of these mutations. + +Driver mutations not only initiate the development of a cancer, but work to maintain the growth of a cancer as well. + +There are many mutations that are being studied by scientists looking at lung tumors. So far driver mutations have been identified in approximately 60% of lung adenocarcinomas. Researchers are now finding driver mutations in squamous cell lung cancer as well. + +These 4 mutations are in general mutually exclusive and are only rarely seen in the same tumor. + +The use of "targeted therapies" - that is medications that target particular genetic abnormalities in a tumor -- has been coined personalized medicine. What this means is that rather than a conventional chemotherapy drug that attacks all rapidly dividing cells, a targeted drug instead attacks a particular abnormality present only in your cancer cells. In general targeted treatments have fewer side effects than traditional chemotherapy. To date, targeted therapies that have been approved for people with lung cancer include: + +Other medications are being studied in clinical trials, including targeted therapies for those whose tumor becomes resistant to Tarceva or Xalkori.. + +A challenging problem with current targeted treatments is that nearly everyone inevitably becomes resistant to treatments we have. There are many mechanisms by which this occurs making it difficult to find one solution. Research is ongoing in clinical trials; evaluating both the use of substituting a second drug to target the mutations, and drugs that use different targets or mechanisms to attack the cancer cell. + +The ability to understand the molecular profile of lung tumors is an extremely exciting area of research, and it’s likely that new treatments for other mutations will soon be available. An example of how rapidly this area of medicine is advancing is the ALK4-EML gene rearrangment. This gene "mutation" (actually a rearranglement) was discovered as recently as 2007. Through a rapid process, the medication Xalkori (crizotinib) was approved in 2011 for general use by the FDA for those patients whose tumors have this rearrangment. There are clinical trials currently in progress evaluating the use of second generation drugs for those who have become resistant to Xalkori. + +If you have been diagnosed with non-small cell lung cancer, especially lung adenocarcinoma or squamous cell lung cancer, talk to your doctor about genetic testing. Although testing is now recommended for everyone with advanced non-small cell lung cancer, a recent study reported that only 60% of oncologists are currently ordering testing. You may also wish to talk to your doctor about clinical trials that may be an option for you. If you are interested in looking into trials evaluating these treatments worldwide, check out the article below on how to find clinical trials. It can be confusing as you check out these databases, but help is near. Recently a lung cancer clinical trial matching service backed by several lung cancer organizations has become available. With this free service a trained nurse navigator can help you locate any clinical trials that may be an option for you. + +Hensing, T., Chawla, A., Batra, R., and R. Salgia. A personalized treatment for lung cancer: molecular pathways, targeted therapies, and genomic characterization. Advances in Experimental Medicine and Biology. 2014. 799:85-117. + +Kim, H., Mitsudomi, T., Soo, R., and B. Cho. Personalized therapy on the horizon for squamous cell carcinoma of the lung. Lung Cancer. 2013. 80(3):249-55. + +Li, T., Kung, H., Mack, P., and D. Gandara. Genotyping and genomic profiling of non-small-cell lung cancer: implications for current and future therapies. Journal of Clinical Oncology. 2013. 31(8):1039-49. + +Villaruz, L., Burns, T., Ramfidis, V., and M. Socinski. Personalizing therapy in advanced non-small cell lung cancer. Seminars in Respiratory and Critical Care Medicine. 2013. 34(6):822-36. \ No newline at end of file diff --git a/tests/data/text/adoption.com1.txt b/tests/data/text/adoption.com1.txt new file mode 100644 index 00000000..0a9a9f7b --- /dev/null +++ b/tests/data/text/adoption.com1.txt @@ -0,0 +1,29 @@ +This is solely my opinion. Every person is different, so the things that I have experienced in no way reflect EVERY birth mother or adoptee. + +First, the biggest mistake I often hear in adoption talk is “giving a baby up for adoption.” I know this a common mistake. I’ve said it too, and I’m a birth mom AND an adoptee! This phrase is such a stab in the heart to most birth moms because it presumes we just gave our baby away because we did not want him or her. Giving something away typically means you did not want whatever it was. In the adoption world, we use the term “place.” I placed my birth son for adoption. This phrase is much more loving and just sounds better. We placed our baby in the arms of their parents. We placed them in a good home. Placing something somewhere is usually done with care and caution because you have love and concern for what you are placing. In the adoption world, placing our children is done with love and concern. + +I know someone who used to have this notion about birth moms. He said he thought birth mother simply did not want the responsibility of taking care of a child. Being a birth mother is much more difficult than one might think. Choosing to be a birth mom is choosing an emotionally difficult path, a path far from lazy. I hope not many people think this, but if you do, do not say it to a birth mom. Go talk to one so she can change your mind! + +“So, did you not want your baby?” + +Yes, people actually ask this. I know for many people, they really cannot comprehend why I, or any other birth mother, would allow someone else to raise their child. The reasons are usually very personal. Every birth mom I have met all wanted their children, but for their personal and very emotional reasons, chose adoption for their baby. + +“Do you think your birth son will be mad at you for choosing adoption?” + +He might be mad. He might question why I did choose adoption. I also trust his parents. I trust that they will tell him how much I loved him and that I wanted him to have the very best life from the start. Will this ease his questioning? I do not know. I do know that it eased mine. I was never angry with my own birth mother because my parents always told me the great love she had for me. There was never a question or doubt of her love for me. I trust that my birth son will feel the same. + +This question is hurtful, but it allows a birth mother to share her testimony of why she chose adoption. We can never determine what the future holds. A birth mom cannot determine the future effects that an adoption will have on a child. + +“Can you get your baby back?” + +I hate to think that people really do not understand what adoption is. Adoption is much different from foster care. In the foster care system, most children were taken out of the home due to the parents not doing what was best for their child. Yes, they can get their children back after they get things in order and clean up their life. However, adoption is different. An expectant parent chooses a family for their child and when that child is born, the birth mother then signs all parental rights away to the new parents. It varies by state, but usually the birth mother only has a limited amount of time to change her mind (in California it is 24 hours). I do not think this question is appropriate to ask any birth mother. The choice she made was not easy and a question like this may generate painful feelings. + +“Do you know your real parents?” + +I love my birth mother and she is just that: my birth mother. My mom is the woman that raised me–my adoptive mother. So, to ask an adoptee if they know their real parents is thoughtless and can be hurtful. Many adoptees do not even know their birth parents. All they have ever known is their adoptive parents: their real parents. + +Someone once said this too me. It was someone who did not know me but knew I was adopted. I did not let this comment get to me because I knew without any doubt that my birth mom did love me. I knew because my parents had told me and my birth mother had written me a letter when I was just hours old. A comment like this is completely horrible and heartless. Yes, my birth mother loved me. That is why she chose life for me and gave me a wonderful family. + +I have heard that my siblings and I are the exception to this rule because someone knows one family who has one adopted kid that has some problems. This must mean all adopted kids are messed up right? How many families do you know who have biological children who are “messed up”? I know plenty! Therefore, I do not think there is any merit to this comment. The child might have had problems if he or she was not adopted. Maybe their bad behavior is just their personality. If children are adopted at an older age and have experienced traumatic things, then yes, they may have some issues to be addressed. However, categorizing all adoptees into being “messed up” is wrong and hurtful. + +These comments and questions are just a few of the things not to say to a birth mom or adoptee. I picked these particular comments and questions because they have been directed at me at some point. Just remember: When speaking to an adoptee or birth mom, it is okay to ask questions. I am an open book! However, be sensitive. I know many people are curious, but maybe the adoptee or birth mom is not ready to answer certain questions. Let certain things stay personal. If the adoptee or birth mom chooses to share, they will. \ No newline at end of file diff --git a/tests/data/text/adoption.com2.txt b/tests/data/text/adoption.com2.txt new file mode 100644 index 00000000..e5d097d4 --- /dev/null +++ b/tests/data/text/adoption.com2.txt @@ -0,0 +1,11 @@ +Thanks and good luck in your search. I have found that many mothers from an era do not actively search for various reasons but have made mention of their placed children to their other children or other family members. I am currently searching for a birth cousin (I know birth mother and father and half siblings names) and aiding a friend in the quest of her birth family. Without a name of either mother or father it is much like searching for a needle in a haystack! Thus the ideal of this thread was created. + + + + + + My paternal male cousin was born May 24, 1962 in Hayti, Missouri. Possibly placed in Illinois, the Chicago area. His birth mother's name is Elain M. Strock or Strack. She had 2 older sons, Ricky and Garry Steigman.The birth father, my uncle, is Thelbert (Peck) Cole. He has 2 younger children. + + + + My female friend was born in St Louis, MO on August 4, 1963 at Bethseda Hospital. She was placed at the Children's home Society and the adoption was finalized within the City of St. Louis. She grew up in rural Missouri. Last edited by southemissouri : 11-04-2005 at . \ No newline at end of file diff --git a/tests/data/text/al.com1.txt b/tests/data/text/al.com1.txt new file mode 100644 index 00000000..7272c2c7 --- /dev/null +++ b/tests/data/text/al.com1.txt @@ -0,0 +1,21 @@ +NEW ORLEANS - Both college football analysts on ESPN, Lou Holtz and Mark May are paid to discuss their unbiased, wide-ranging opinions about the sport on the air. + +They disagree a lot, especially when it comes to Ohio State. + +Holtz, a Notre Dame coaching legend, typically has been pro-Ohio State this year while May, a former player at Pittsburgh, has been down on the Buckeyes and the Big Ten. + +As No. 4 Ohio State prepares to take on No. 1 Alabama in the Sugar Bowl - a semifinal game in the inaugural College Football Playoff - Holtz didn't mind boasting a little about being right about the Buckeyes this year. + +"Mark May is a great guy - We have no teleprompter, no script, no rehearsal, but we have a difference opinion," Holtz said. "I love him, but he was a player, I was a coach. He made suggestions, I made decisions. He showered after work, I showered before work. I signed the the paycheck on the front, he signed the back. + +"We just have a different way of looking at things." + +During a segment called "Final Verdict" on ESPN's College Football Final, Holtz bantered with May about whether or not the Big Ten would have a team in the playoff and whether Ohio State had a shot of cracking the top four. + +In those segments, co-host Rece Davis, dressed like a judge, rules either in favor of Holtz or May. Both times, May, who said Ohio State and the Big Ten were out of the College Football Playoff hunt, got the ruling. + +"I lost two 'Final Verdicts' and doggone it both of them turned out that Rece was wrong," Holtz said. "No. 1 the Big Ten would have somebody in (the playoff) and Ohio State had a chance. Both times he ruled against me." + +As for why May tends to have an anti-Big Ten opinion - something many Ohio State fans feel is a trend - Holtz decided to sidestep that question. + +"You would have to ask Mark May," Holtz said. "One thing I learned, I don't speak for Mark May. I have a hard time speaking for Lou Holtz." \ No newline at end of file diff --git a/tests/data/text/al.com2.txt b/tests/data/text/al.com2.txt new file mode 100644 index 00000000..8afc77c5 --- /dev/null +++ b/tests/data/text/al.com2.txt @@ -0,0 +1,3 @@ +HUNTSVILLE, Alabama - Hockey players, roller skating enthusiasts and Future City competitors are among groups expected to bring more than 5,900 people together next month in Huntsville/Madison County. + +Here is a Huntsville/Madison County Convention & Visitors Bureau calendar of January events and conventions with host hotel if applicable and number of expected attendees: \ No newline at end of file diff --git a/tests/data/text/apartmenttherapy.com1.txt b/tests/data/text/apartmenttherapy.com1.txt new file mode 100644 index 00000000..69696bd5 --- /dev/null +++ b/tests/data/text/apartmenttherapy.com1.txt @@ -0,0 +1 @@ +I truly believe that all a children's book needs to be magical is a well illustrated wonderful story, but sometimes, as a gift,...a book can fall flat. These interactive books, on the other hand, have so many bells and whistles, they won't be cast aside on Christmas morning in favor of "real" toys. They are super creative, super fun and meant to be played with, over and over. 1. What's Inside by OKIDO. This special twist of this book is that when certain pages are held up to light, more pictures are revealed giving children a look at what's inside bodies, buildings, cars and more. A really unique, enjoyable experience. You can see a video here. 2. My First Keyboard Book by Sam Taplin. Part picture book, part sheet music, part working keyboard, this book is a wonderful introduction to an instrument or just plain fun for young children to experiment with. You can watch a video demo here. 3. I'm So Glad You're Here Giant Book by Michelle Romo and J. Betrue. Perhaps hard to tell from the photo collage, this soft book is actually child-sized. And by that I don't mean it's sized for a child; I mean that it is the size of a small child. Measuring in at 25" x 22", the book features beautiful appliqué and embroidery. 4. Is There a Dog in This Book? by Viviane Schwarz. A long-awaited follow-up to the hilarious "There are Cats in this Book" and "There are NO Cats in this Book", the cat trio of Tiny, Moonpie, and Andre are back for more interactive fun. Children will be amused by the story and delighted in the surprises awaiting them behind all the flaps. 5. Playbook Farm by Corina Fletcher and Britta Teckentrup. This is a fantastic pop-up book on its own, but in the (probably trademarked) words of Ron Popeil, "But wait, there's more!", the book unfolds to become a 3-D playmat. Really inventive format that has been continued in Playbook Pirates and Playbook Castle. You can see customer photos and a video here. 6. My Little Blue Robot by Stephen T. Johnson. Perfect for kids who love to build, or who love robots, this book includes sturdy cardboard parts to build your own robot (who even talks!). All the pieces use slots and tabs so you don't need glue or any extraneous materials. You can see a video of one being built here. 7. The Ultimate Book of Vehicles From Around the World by Anne-Sophie Baumann and Didier Balicevic. Not only is this a big book and filled with wonderful illustrations of all kinds of vehicles - cars, motorcycles, buses, cement trucks — but it has 60 moving parts: from simple flaps to open, tabs to pull and more complex parts a child can manipulate (see this video to get a better idea). This is truly a memorable book that a child can spend hours and hours with. There's also a follow-up: The Ultimate Construction Site Book. 8. Ocean: A Photicular Book created by Dan Kainen, written by Carol Kaufmann. If you haven't yet experienced a "photicular" book - you must. When this book arrived at the office everyone crowded around to marvel at the seemingly moving pictures. Kainen's first creation, Safari, is wonderful and his latest, Ocean, takes readers under the sea. This video gives you a peek behind the photicular technology. 9. My First Computer by Anne-Sophie Baumann and Marion Billet. This book won't keep your toddler from wanting to bang on your computer keyboard, but it will certainly keep them occupied and delighted, perhaps while you do your own work. With flaps to open and, of course, things to slide, ala a tablet, this book simulates the discovery of an app, but in a beautifully low tech/high creativity way. Watch a demo here. 10. Presto Change-O! A Book of Animal Magic by Édouard Manceau. This is one of the most creative interactive books I've ever seen. Children can literally move the pictures around to transform one scene into another. A teapot turns into an elephant, a hot air balloon turns into a rabbit - and back again, if you wish. Check out this video of the book in action. \ No newline at end of file diff --git a/tests/data/text/apartmenttherapy.com2.txt b/tests/data/text/apartmenttherapy.com2.txt new file mode 100644 index 00000000..40816aef --- /dev/null +++ b/tests/data/text/apartmenttherapy.com2.txt @@ -0,0 +1,3 @@ +Nothing stirs the DIY spirit like a good IKEA hack. IKEA is so accessible — anybody could stroll in there and buy a BEKVAM or two — but its Scandinavian design is deceptively simple. With a little ingenuity a box, or a table, or a stool from IKEA could become pretty much anything. Here's proof. + +Click on the slideshow to see the projects. To learn more about each project (and see more pictures), click the links in the captions. diff --git a/tests/data/text/architecturaldigest.com1.txt b/tests/data/text/architecturaldigest.com1.txt new file mode 100644 index 00000000..34899de4 --- /dev/null +++ b/tests/data/text/architecturaldigest.com1.txt @@ -0,0 +1,11 @@ +Not very often does one hear a contemporary architect allude to the mortuary temple of the Egyptian pharaoh Hatshepsut when describing the inspiration for a new house. Ditto the rather obscure Majorcan cliff-top villa Can Lis, designed by Sydney Opera House mastermind Jørn Utzon. But in the context of the extraordinary Arizona residence that architect Marwan Al-Sayed and decorator Jan Showers created for Joann and Paul Delaney, these seemingly arcane reference points actually make perfect sense. In addition to its similar setting (sunbaked rocky landscape), building material (stone), and overall form (rectilinear), the retreat shares another, less tangible quality with those unexpected antecedents—timeless, otherworldly serenity. + +"I remember Paul telling me that he wanted the house to last a thousand years," says Al-Sayed, a recent Phoenix-to–Los Angeles transplant who was part of the design triumvirate responsible for the astonishing Amangiri resort in Utah. "So I was intrigued by the idea of ancient architecture—its weight, proportion, grandeur, and materiality." + +The 9,000-square-foot, single-level dwelling he ultimately devised sits on nine acres of desert terrain at the foot of Mummy Mountain (did someone say Egypt?) in the aptly named Phoenix suburb of Paradise Valley, a place where rugged red hills cast craggy shadows across a landscape of saguaro cacti, aloe vera plants, and creosote bushes. Approaching the house from the front drive, one is greeted by a poker-faced exterior of limestone blocks. With little hint of what lies beyond, the elevation might easily be mistaken for the façade of a formulaic modernist box. + +Any such notions quickly vanish, however, as the entry procession leads through a semienclosed passageway directly into a glorious courtyard. Straight ahead lie the main entertaining areas—the living and dining rooms as well as an art-lined gallery—but visitors are meant to pause in this oasis-like reception space, planted with mesquite trees and highlighted by what seems to be a vast reflecting pool. Framed in black granite, the water feature is, in fact, a ten-foot-deep infinity swimming pool that cascades over its far wall into a shallow basin below. + +Flanking the pool are two loggias—one off the master suite and the other off the guest quarters—delineated by limestone brise-soleils that orchestrate an ever-changing dance of reflected light. "I used limestone for both the courtyard floor and most of the walls to underscore the idea of the house as a configuration of interconnected pavilions with varying degrees of exposure," the architect says, noting the stone's luminous yet earthy quality. "Using one material throughout has a calming effect—it gives you the luxury of tuning out the cacophony of the outside world." + +For the full story and more photos, subscribe now and get the digital edition immediately. \ No newline at end of file diff --git a/tests/data/text/architecturaldigest.com2.txt b/tests/data/text/architecturaldigest.com2.txt new file mode 100644 index 00000000..e4b6f627 --- /dev/null +++ b/tests/data/text/architecturaldigest.com2.txt @@ -0,0 +1 @@ +Sign up for our newsletter to get the latest in design and decorating, celebrity style, shopping, and more. \ No newline at end of file diff --git a/tests/data/text/avclub.com1.txt b/tests/data/text/avclub.com1.txt new file mode 100644 index 00000000..f6ae988e --- /dev/null +++ b/tests/data/text/avclub.com1.txt @@ -0,0 +1,67 @@ +Great scenes, like great movies, adhere to no single formula. Some of them work only in relation to the moments that occur before and/or after them. Others play like miniature movies themselves, their appeal independent of the feature-length films that house them. What all great scenes have in common, though, is their ability to imprint themselves on a viewer’s brain. In anticipation of our best films of 2014 list, which drops on Thursday, we’ve singled out 22 of our favorite scenes from the year in cinema. They’re in no particular order—save for the first one, which several contributors cited and we’ve hence decided is the year’s best. Fair warning: Some of these entries, including the first one below, disclose major plot points. Proceed with caution. + +Those who haven’t seen Whiplash yet should probably skip right past this glowing appraisal of its final minutes, and rush to the nearest theater showing it. Those who have seen the film should finally catch their breath and read on. In the closing scene of the movie, aspiring jazz drummer Andrew Neyman (Miles Teller) suffers one last humiliation at the hands of his abusive mentor, Terence Fletcher (J.K. Simmons), who takes revenge on his star pupil by providing him with the wrong sheet music for a climactic concert at Carnegie Hall. Rather than accept defeat, however, the musician returns to his kit, answering this cruel betrayal the only way he knows how: by drumming his ass off—this time to his own beat, guiding the ensemble with an improvised burst of virtuosic playing. Nearly all of Whiplash operates on a level of pure anxiety, entwining the nerves of its audience and protagonist. And so this parting display of showboating talent, which director Damien Chazelle stages with all the kinetic verve of a car chase or a battle sequence, feels downright liberating in its sense of cathartic release. A true marvel of editing, composition, and performance, the scene would be a contender for the year’s finest even if seen completely out of context. What clinches its victory, though, is the troubling ambivalence lurking beneath the awe-inspiring spectacle: We’re watching not just the birth of a future jazz legend, but also the consummation of a truly toxic relationship—the moment, in other words, when one obsessive sociopath finally rebuilds another to his exact specifications. It’s equal parts disturbing and rousing, a thunderous cymbal crack on the beating hearts of its audience. [A.A. Dowd] + + + +The first half of Jonathan Glazer’s Under The Skin establishes a hypnotic pattern: An alien in human guise (Scarlett Johansson) picks up unsuspecting men, seduces them into accompanying her into a dilapidated-looking house, and, once inside, does away with them, in a fashion disturbing and visually striking in equal measure. But just around the film’s halfway mark, the alien picks up a man with a facial deformity. Their conversation, in a moving car, first proceeds in simple alternating one-shots as she asks him questions and repeatedly compliments his hands. But when she asks him to touch her face, they share the frame, and later one-shots are closer, more intimate. At first, she seems to simply assume a gentler seduction tactic. While she does bring the man back to her lair, she eventually allows him to leave without suffering the same fate as the others. When she stares at her own shadowy reflection that night, she may be catching a glimpse of empathy. The sequence is fascinating in its own right, but even more so for the way it reverberates through the rest of the movie, knocking the Johansson character off her axis and setting the second half of her quiet story in motion. [Jesse Hassenger] + + + +The year’s best, purest chase scene—technically a street racing scene—is one part old-school style, one part newfangled tech. Shot on Canon C500s—beefier, 4K variants of the low-light-friendly cameras used to shoot micro-budgeted projects like Blue Ruin and Blue Is The Warmest Color—it follows five gleaming cars as they rip through a vivid small-town nightscape, saturated in sodium-vapor orange. The camera style is self-consciously 1980: slow zoom-ins at the starting line; passenger-side handheld shots of the drivers; the kind of natural, unaffected shaking that happens when a cameraman is trying to keep a speeding car in the frame. And, for the bulk of the scene, there’s no music, only the snarl and squeal of the cars. Five engines, half-a-dozen vicious turns, and a train barreling into the distance—nothing more is necessary. [Ignatiy Vishnevetsky] + +Ruben Östlund’s Force Majeure is a stinging critique of the male ego, and it reaches a crescendo during a late scene in which a pitiful husband and father (Johannes Kuhnke)—having already proven his spinelessness by ditching his clan in order to save himself during an apparent avalanche at their ski-vacation resort—enjoys a beer with his best mate (Kristofer Hivju) at the bottom of the mountain. There, the two men are approached by a woman who tells them that her friend thinks they’re the best-looking guys at the place, news that naturally boosts their self-esteem. That high doesn’t last long, however, since just as they’re enjoying the praise, the woman returns to apologize for having relayed the compliment to them; it was really intended for two other nearby gentlemen. Left deflated beyond repair, they vacillate between anger and embarrassment, which Östlund depicts in a protracted single shot that reveals the clownish emptiness of their macho pretenses. [Nick Schager] + + + +Just out of prison following a 12-year stretch, expert safecracker Dom Hemingway (Jude Law) is looking for work, among other things, until a former associate named Lestor (Jumayn Hunter) tells Dom his skills are now useless, as newfangled electronic safes are impossible to crack via methods of a dozen years past. Since Lestor despises Dom, he offers him a deal: If Dom can open Lestor’s personal safe in less than 10 minutes, he’ll give him a highly lucrative job. But if Dom fails, Lestor gets to cut off his dick, right on the spot. Writer-director Richard Shepard plays this ludicrous ticking-clock scenario to the hilt, devising both a wholly unexpected safecracking method—no gentle taps and slowly twisted dials here—and a diabolical punchline. Mostly, though, it’s just a hoot to watch Law’s high-octane performance dovetail with a rare moment of concentrated focus for his character. Dom is ostensibly working feverishly to save his penis, but the expression on Law’s face throughout is pure, uncut fun. [Mike D’Angelo] + +Darren Aronofsky’s Noah is a work of conflicting aspirations—a Biblical epic that wants to be intimate and spectacular, “realistic” and fantastical, accommodating of both serious spiritual inquiry and talking, CGI rock monsters. The film’s most fruitful attempt at reconciling seemingly contradictory positions arrives during the centerpiece sequence, when Noah (Russell Crowe) recounts to his family the story of God creating the universe. Billions of years pass through simulated time-lapse photography, and as single-cell organisms transform into fish, which soon slither onto dry land and turn into something else entirely, it becomes clear that Aronofsky has incorporated evolutionary theory into his retelling of Genesis 1. Fundamentalists might balk at this revisionist take on scripture, but it’s hard to imagine anyone shrugging off the grandeur of the scene—a montage of blooming nebulas, scampering species, throbbing forbidden fruit, a menacing serpent, a glowing Adam and Eve, and Cain murdering Abel in striking silhouette. Noah has no shortage of grand imagery, but only this flashback to the beginning of time inspires a religious (or at least near-religious) awe. Cinephiles both devout and secular should give thanks. [A.A. Dowd] + + + +Lars Von Trier’s epic study in carnal (and non-carnal) knowledge features plenty of explicit sex, but it’s a fully dressed, gatecrashing Uma Thurman who provides its most blistering, passionate sequence. After more than an hour spent exploring heroine Joe’s sexual self-indulgence, Nymphomaniac abruptly serves up a comically ghastly reminder that her actions have consequences. Into Joe’s apartment strides Thurman’s Mrs. H, her three young sons in tow, in order to confront their philandering father. “Confront” isn’t quite the word, though, since Mrs. H opts to channel her anger through relentless bitter sarcasm, congratulating her husband and Joe on their shared happiness and taking the kids on a tour of the premises, so that they can see what Dad will be up to from now on. “Would it be all right if I show the children the whoring bed?” she politely asks Joe, in the scene’s signature moment. Thurman plays this minor role (it’s the character’s sole appearance) without an ounce of vanity, digging so deep into Mrs. H’s feelings of debasement that all she can finally do, at the end, is emit a truly bloodcurdling shriek. [Mike D’Angelo] + +Phil Lord and Christopher Miller’s 22 Jump Street is the rare comedy sequel to equal its predecessor. Even more impressive still, it manages to surpass the original at that least likely of moments, the end credits. Having already spent its entire feature-length runtime making self-referential jokes about the derivativeness of sequels, 22 Jump Street goes that extra step—and then a few more—by delivering a montage of phony upcoming follow-ups, replete with clips and poster art. From “Culinary School” and “Foreign Exchange Students” to “A Semester At Sea” and “Traffic School,” it’s a sequence that pushes the material’s auto-critique craziness into outright absurdity, especially when Seth Rogen momentarily appears in “Sunday School” as a replacement for Jonah Hill (apparently due to a “contract dispute”). At once dim-bulb silly and cannily critical of its own inherent existence, it’s perhaps the best end-credits sequence in movie comedy history—as well as a capper that mocks (to the point of negating) the need for any further franchise installments. [Nick Schager] + + + +For much of its running time, comedian Bobcat Goldthwait’s first stab at found-footage horror seems content to operate like an affectionate goof on The Blair Witch Project. The film does, however, make a late attempt at eliciting goosebumps instead of giggles, and the results are quite effective. Having retreated to the woods in search of the elusive Bigfoot, amateur filmmaker Jim (Bryce Johnson) and his patient girlfriend Kelly (Alexie Gilmore) are awoken in the middle of the night by strange sounds coming from outside of their tent. Goldthwait captures the subsequent gauntlet of terror in a single, 20-minute take, locking his camera on the faces of his increasingly alarmed characters, who sit paralyzed with fear as the noises get louder, closer, and weirder. It’s a marvelously suspenseful sequence, relying on not just the credible distress of the actors, but also the claustrophobia of the tent—a structure that limits our view of the surroundings, allowing the imagination to run wild with thoughts of the beasts surely lurking on the other side of its flimsy, nylon walls. As in Blair Witch, what we envision is much scarier than anything we could be shown. [A.A. Dowd] + +God Help The Girl has half a dozen lovely little production numbers in a variety of moods, but Stuart Murdoch’s musical is never more delightful than when it engages with the pure joy of making and experiencing music, crystallized in the scene scored to the song “I’ll Have To Dance With Cassie.” James (Olly Alexander) brings troubled singer-songwriter Eve (Emily Browning) to some kind of civic center, where the supporting members of their band are playing an afternoon dance attended mostly by the elderly. When Eve joins them onstage to sing and her song kicks into full-band ebullience, minor miracles abound: Dancers who look ready for a community-college production of Grease appear from nowhere; bandmate Cassie (Hannah Murray) arrives at just the right time; and Eve seems to forget her problems, however briefly. Murdoch cuts around madly; it’s exactly the kind of dance number that makes purists complain about not being able to see the dancing. But intricate choreography isn’t the point here; Murdoch stages a dance party halfway between homemade reality and music-video dreams. “Hell do I care what I look like when I feel this good?” the song asks—though as it happens, everyone looks pretty great. [Jesse Hassenger] + + + +Bong-Joon Ho’s Snowpiercer was the summer’s best action film, and it peaks when Chris Evans’ rebel leader, guiding his insurgency through a train that houses the last survivors of a global environmental apocalypse, arrives at the car where young children are being lectured by their teacher (Alison Pill) on the myth of their train’s creator. After the bleakness of their prior environments, Evans and company’s arrival in this brightly colored elementary-school compartment is jarring, and made more so by Pill’s overly cheery demeanor, which carries with it more than a whiff of madness. Basing its art-deco designs (both in terms of the classroom, and the historical video that the kids watch) on those found in the popular Bioshock first-person-shooter videogame series, the sequence has an unsettling strangeness that eventually erupts in a paroxysm of violence. Evil is rarely more chilling than when it comes in the form of a sunshiny mentor. [Nick Schager] + +A man (Mark Ruffalo) slumps over a bar, his left shoulder jutted toward the camera. He hears a guitar and a woman’s voice; slowly, he begins to peek over his arm, as though he were a sun reluctant to rise. The camera stays on him for almost 45 seconds, only revealing the singer—a songstress (Keira Knightley) with an acoustic guitar, awkwardly perched on a stool—once the chorus starts. This is actually the second time this scene has played out in Begin Again; the first time was at the very beginning of the movie, from her perspective, with the camera landing on his big goofy grin as she got off the stage. Now, the viewer is seeing it—and hearing it—from his angle, as he eyes the next act’s instruments and starts imaging a full-blown, Starbucks-playlist arrangement. Begin Again—a music-business musical that serves as a kind of spiritual sequel to director John Carney’s earlier Once—is largely bogus, but this sequence, its only lapse into overt fantasy, feels completely authentic. That’s thanks in so small part to Ruffalo’s performance, which turns what should be a simple reaction shot into a glimpse into a character’s soul. [Ignatiy Vishnevetsky] + + + +For all the praise being heaped on Steve Carell’s transformation into a beaked, aristocratic sociopath, the best performance in Foxcatcher required no prosthetic noses. As Dave, the older and more decorated of the two Olympic-champion Schultz brothers, a bulked-up Mark Ruffalo finds layers of feeling his co-stars aren’t quite afforded. His showcase scene—and arguably the film’s single strongest moment—is the one in which Dave is forced to speak about the influence his sponsor and coach, John Du Pont (Carell), has had on his athletic career. At the behest of a filmmaker, who’s making a presumably propagandistic documentary on Du Pont, Dave struggles to explain the man’s coaching strategies (which are basically useless) and to muster up a single word of praise. When the director requests that he describe Du Pont as a “mentor,” Dave valiantly attempts to swallow his pride—and Ruffalo makes his thought process palpable, a small storm of emotions passing across his face. It’s a miniature master class in acting, and one of the brief, promising instances in which Foxcatcher threatens to burst its bubble of oppressive melancholia and become a stealth comedy of discomfort. [A.A. Dowd] + +Movie musicals may have gone out of style, but a dancing interlude can still elevate the pulse of just about any film. In Bloom, a Georgian drama about two 14-year-old girls and a gun, reaches its emotional apex during the mid-film scene in which one of them, Natia (Mariam Bokeria), marries a man she barely knows, at her family’s behest. Her best friend, Eka (Lika Babluani), is visibly upset during the reception, and even calls Natia into the bathroom at one point to ask whether she loves her new husband. (The answer: “I guess I do.”) Upon emerging, however, Eka suddenly takes the center of the room and performs a lengthy solo dance as the other guests stand around her in a circle, clapping and cheering. What’s remarkable about this routine is the wealth of contradictory feelings it somehow conveys—Eka’s decision to elbow past everyone and take command of the dance floor is at once a defiant, fuck-you gesture; a burst of self-liberation; and an ardent declaration of love for Natia. Plus, Georgians really know how to dance—when a standing Eka picks up a napkin from the floor with her teeth, you’ll feel the urge to whoop and holler, too. [Mike D’Angelo] + + + +Paul W.S. Anderson—the widely maligned English director behind Resident Evil, Alien Vs. Predator, and The Three Musketeers 3D—has a real knack for organizing and diagramming space, and though Pompeii, his first blockbuster-budget production, is slow-going at first, it hits its stride once it comes time to depict the destruction of the titular city street by street, neighborhood by neighborhood. Obsessed with bunkers, mazes, and caverns, Anderson can’t help but turn the city grid into yet another of his deathtrap tunnel systems. He envisions the destruction of Pompeii as one long set piece, with characters scampering over bodies and ruins, trying to outrun a ship that’s been forced inland while fiery debris rains down in the foreground. It’s brisk and breathtaking. [Ignatiy Vishnevetsky] + +Recently, every year has had at least three or four superhero movies, and all of those superhero movies have at least three or four action sequences, many of which involve gigantic airships and/or collapsing buildings. It makes sense, then, that the most memorable superhero action of the year would be a bit more granular, focusing on the use of a single superpower. In Bryan Singer’s mutant-packed X-Men sequel, Professor X, Beast, and Wolverine enlist super-fast mutant Quicksilver (Evan Peters) to help break Magneto out of his metal-free holding cell in the Pentagon. When the group is discovered and security guards open fire, the movie switches to Quicksilver’s point of view as he flips on his Walkman and springs into action, running up and down the walls with the nonchalance of a morning jogger, scored to the quiet strains of Jim Croce’s “Time In A Bottle.” With a few gentle tweaks, he knocks out cops and sets bullets off course; when the film returns to normal speed, the conflict is over in a matter of seconds. This interlude is a sustained moment of playfulness in an otherwise fairly serious superhero epic, and a reminder that the X-Men movies distinguish themselves by toying with the idiosyncrasies and possibilities of their universe’s vast array of mutants, not by getting into a building-destruction contest. [Jesse Hassenger] + + + +Gone Girl’s first half is a slow-drip mystery, but it’s the second half of David Fincher’s adaptation—after the big twist is revealed—that taps a truly sinister vein. The story’s malevolence reaches a fever pitch during the scene when Amy (Rosamund Pike), now a captive of her creepy ex-boyfriend (Neil Patrick Harris), finally “gives in” to his carnal wishes and takes him to bed. It’s an encounter of sexual aggression and manipulation that Fincher stages with mounting unease, until the moment the white lingerie-clad Amy, beneath her lover on the bed, suddenly and swiftly slices his throat, coating herself in blood at the very moment that he climaxes inside her. More chilling still: After letting him bleed out, Pike straddles him and then flips her blood-soaked hair out of her face—an offhand gesture of aggravation that speaks volumes about the depths of her mercilessness. [Nick Schager] + +“Even with nuclear weapons there is no guarantee that the creatures will succumb,” declares the witty opening credits sequence for Hollywood’s latest iteration of Japan’s favorite monster. “Evidence show [sic] that it is likely the creatures will come back with David Strathairn’s head.” That’s what the text reads for a split second, anyway—only by freeze-framing the DVD can one catch more than a few words—before everything other than DAVID STRATHAIRN is redacted. Superimposed over stills ranging from Darwin’s Origin Of Species to news coverage of the Bikini Atoll tests, all of the credits are accompanied by alarmist phrases that are swiftly blacked out, creating a deliciously paranoid mood (augmented by Alexandre Desplat’s urgent score) before the story has even begun. There are even some in-jokes among the barely visible text: Bryan Cranston’s credit includes the phrase “Walter Malcolm has claimed that government men dressed in white lab coats routinely appear at site,” which is redacted in a way that leaves the words “Walter” and “white” visible by themselves for a fraction of a second. With “Malcolm” in the middle. [Mike D’Angelo] + + + +Christopher Nolan has developed an inaccurate reputation as a chilly filmmaker with a Kubrickian detachment from human emotion. Despite superficial 2001 influences, Interstellar seems almost designed to correct that assumption, never more effectively than in a single scene that essentially involves space pilot Cooper (Matthew McConaughey) sitting down to check his messages. Cooper has been on a planet that experiences time far more slowly than Earth, and upon his return to his ship, he watches a series of video messages from his children, received in the hour or so that he was gone. In a few moments, he watches them age over two decades. Because it’s a Nolan movie, this scene also contains plenty of information: about what’s happened on Earth while Cooper has been gone and where his plot-crucial daughter Murph is now. But the exposition feels secondary to the emotional wallop of Cooper’s family life accelerating without him. In a movie of amazing sights, it’s the simple passage of time that hits the hardest, culminating in the final video message, from now-grown Murph (Jessica Chastain). When she finishes a goodbye to her father, thought to be lost in space, Nolan cuts to the other side of her camera and follows her as she returns to her job—bringing the movie to this decades-forward Earth for the first time. The transition doesn’t span anywhere near the amount of time traversed in the most famous cut of 2001, but it’s a similar technique, hurtling the audience ahead into the next phase of the story. [Jesse Hassenger] + +Set in a depressed (and depressing) post-apocalyptic Australia, The Rover adopts a mood of consistent despair, rarely allowing for even the faintest trace of levity. Maybe that’s why the belated arrival of a bona fide pop anthem on the soundtrack—briefly replacing the ambient, dread-infused hum of Antony Partos’ original score—qualifies as downright triumphant. As the film’s mismatched road warriors (played by Guy Pearce and Robert Pattinson) wander into the Outback during a protracted wide shot, director David Michôd cues up Keri Hilson’s narcissistic earworm single “Pretty Girl Rock.” It seems like a bitterly ironic song selection, until the scene cuts to a nighttime image of Pattinson’s tragic simpleton sitting alone in a jeep, softly singing along to the suddenly diegetic music. From here, the tune takes on a melancholy quality, with Michôd employing it as a creature comfort from another era—a bittersweet blast of nostalgia, a sonic relic of the more hopeful world that now exists only in the rearview mirror of these characters’ lives. As needle drops go, it’s eccentric and weirdly, powerfully affecting. Also, good luck getting that damn song out of your head. [A.A. Dowd] + + + +Chad Stahelski and David Leitch are both veteran stuntmen, which explains why their debut feature, the superbly entertaining John Wick, happens to contain some of the most energetic, best-directed action sequences in recent memory, with the standout being the movie’s centerpiece, in which Keanu Reeves’ eponymous ex-hitman shoots his way through a nightclub/bathhouse called the Red Circle. Starting in the back of the club—where silhouetted henchmen are dispatched one by one while Kaleida’s “Think” slinks on the soundtrack—and then bursting into the main floor, Collateral-style, it’s as much a dance piece as a gunfight. Reeves and an ensemble of stunt performers roll, tumble, shoot, and reload, over and over, moving through pools of blue and magenta light, always oriented around the camera and the geometry of the space; this is amped-up action as kinetic art. [Ignatiy Vishnevetsky] + +Bill Hader and Kristen Wiig prove themselves adept serious actors in The Skeleton Twins, starring as siblings reunited after a 10-year estrangement. The movie itself doesn’t often transcend its indie-dramedy roots, but it generates more feeling than it might have otherwise due to its stars’ chemistry, and the SNL-honed comic timing they bring to their depressive characters. The movie’s strongest juxtaposition of angst and crowd-pleasing comedy comes when Milo (Hader) comes home to a frustrated Maggie (Wiig) demanding him to get his shit together. He attempts to cheer her up by silently putting Jefferson Starship’s “Nothing’s Gonna Stop Us Now” on the stereo and beginning to lip-sync, imploring her to join him. Maggie resists for well over a minute before finally, hilariously giving in to the emotive cheesiness—the mugging equivalent of an expertly delayed chorus. It’s the kind of goofy bit Wiig and Hader could’ve sold on SNL, and the movie draws on that history to make Maggie and Milo especially convincing, and touching, as family. [Jesse Hassenger] \ No newline at end of file diff --git a/tests/data/text/avclub.com2.txt b/tests/data/text/avclub.com2.txt new file mode 100644 index 00000000..f2cabae0 --- /dev/null +++ b/tests/data/text/avclub.com2.txt @@ -0,0 +1 @@ +By now the Serial theme song has been chopped a screwed a million different ways by a million different people, including us for our Serial Serial podcast. But that little kid that says “mail kimp?” That’s a different story. Soundcloud user kpffkl has created the “Mail Kimp Remix” to give that little kid some glory, and it’s both annoying and mesmerizing, as most remixes of a single sound are, but it’s still new and potentially interesting Serial-related content, something the Internet is certainly hungry for this holiday season. \ No newline at end of file diff --git a/tests/data/text/backstage.com1.txt b/tests/data/text/backstage.com1.txt new file mode 100644 index 00000000..0d11d7b2 --- /dev/null +++ b/tests/data/text/backstage.com1.txt @@ -0,0 +1 @@ +This casting/job notice is still under review by Backstage's editors. However, you can apply right away! Scams and inappropriate content will be promptly removed. \ No newline at end of file diff --git a/tests/data/text/backstage.com2.txt b/tests/data/text/backstage.com2.txt new file mode 100644 index 00000000..47b56eaa --- /dev/null +++ b/tests/data/text/backstage.com2.txt @@ -0,0 +1,7 @@ +To find Los Angeles–area stage and film acting schools, teachers, and coaches, click here to search Backstage's Acting Schools database. + +Each of the entries contains the following information, if applicable: name of teacher or school, address, phone and fax numbers, email address and/or website, average number of students per class, whether beginning, intermediate, or advanced students are taught, whether auditing is permitted, whether classes are ongoing or by sessions, any special emphasis used in classes or coaching, whether a work/study program is offered. Descriptions of the class, school, or coaching are provided by the instructor or institution and edited by Backstage. + +Also, you can find additional schools and coaches in the Backstage Yellow Pages. + +Schools and coaches who have been omitted may contact listings {at} backstage.com regarding inclusion in the Acting Schools database. Schools and coaches can also self-post listings and media-enhanced ads in the Backstage Yellow Pages. Contact Backstage's Advertising Department for additional marketing options. \ No newline at end of file diff --git a/tests/data/text/bhg.com1.txt b/tests/data/text/bhg.com1.txt new file mode 100644 index 00000000..e67f22db --- /dev/null +++ b/tests/data/text/bhg.com1.txt @@ -0,0 +1,21 @@ +Going simple is the best decorating solution for fall centerpieces and table toppers. Here, deeply hued roses and dahlias play off the more muted collection of faux tomatoes and fall foliage. Editor's Tip: Choose a vase color that recedes and doesn't compete with the blooms. + +Without carved faces, pumpkins make a dramatic tabletop. Start with lengths of grapevine wrapped around and up and down pumpkins in different patterns. Display smaller gourds under glass and scatter a base of leaves underneath, and include a tall vase filled with wheat or dried lavender for subtle color complement. + +It's easy to craft a no-sew table runner in the shapes and colors of fall. Find and enlarge a fallen leaf shape and cut it out from a piece of felt. Glue ribbon or single-fold bias tape to the edge and place on a contrasting piece of felt or complementary table runner. + +These colorful vegetables are the perfect fall tabletop decorations when stuffed with flowers and foliage. Cut each pepper lengthwise an inch from the top to make an opening (don't cut the top completely off), hollow out the pepper and fill with different types of flowers. Set the peppers on small plates to finish the decoration. + +Inexpensive tools make this uniquely carved pumpkin easy to recreate. Use a large leaf to trace onto a pumpkin and use a scraping tool to cut away only the outermost layer. Set it on a bed of moss and a pedestal for a centerpiece that will last all season. + +The vibrant, diverse colors of Indian corn perfectly represent the color palette of fall, and you can put them to use in a no-fuss candleholder. Place a candle at the center and hot-glue corncobs together in a circle around it to make this pretty tabletop decoration. + +Pumpkins typically are front and center at doorways and on tables, but this elegant collection of corn and wheat stalks fits in perfectly with the season. Include a variety of colors of corn and a few different sizes and include an individual cob at each place setting. + +Small gourds and pumpkins are nearly sculptural, and a cluster of two or three on a pretty plain plate resembles a still life. Tuck in a pinecone and a few leaves, and wrap a bit of raffia or twine around one of the stems. + +This decorative arrangement of cattails, pheasant feathers (found at crafts stores), and Purple Majesty ornamental millet doesn't need water, so it will last all season long. Editor's Tip: Place chicken wire or floral netting inside an urn to create structure within the vase. Mold the wire into a ball to fit the bottom of the vase and stick the stems into the holes to complete the arrangement. + +Pumpkins like the ones pictured can be used as decorations all season long. Elevate a few -- chosen for the beauty of their contrasting colors -- on a pedestal. Underneath, use a table runner or cloth that picks up on the hues, and tuck in few bittersweet branches for texture. + +Repetition of color -- whether with pumpkins, foliage, or flowers -- creates a unified fall decorating scheme that accents all of fall's bounty. Dominant tones of orange and yellow are present in a vase of blooms and a cluster of branches on a sideboard, in addition to the main pumpkin display. \ No newline at end of file diff --git a/tests/data/text/bhg.com2.txt b/tests/data/text/bhg.com2.txt new file mode 100644 index 00000000..574eae5d --- /dev/null +++ b/tests/data/text/bhg.com2.txt @@ -0,0 +1,3 @@ +It's not a holiday home until you've hung a Christmas wreath. This quick video offers nifty tricks for damage-free wreath hanging on doors and windows. Get our secrets to success! + +A pretty wreath, is just the trick to take your Christmas decorations from basic to bold. Here are the no fail secrets, for hanging window, and door wreaths. For outdoor wreaths, choose a ribbon that's strong, durable, and will make a high impact statement. Your best bet, is a two and half inch satin ribbon. To avoid a droopy look, hang your wreath in the top half of the window. On a door, center the wreath at about eye level. To hang a wreath in a window, lower the top window sash and place the wreath outside of the window while holding the ends of the length of the hanging ribbon. [MUSIC] A piece of paint-friendly tape provides stability. Tape just the very end of the ribbon before sliding your window securely close. When hanging on a door, use a staple gun, to staple the ends of the hanging ribbon to the center top of the door. That way, the staples will never be seen. To keep a lightweight wreath in place, use double-sided foam tape on the back. It's that simple. Use this method every Christmas, to hang your wreath without damaging your home. [MUSIC] \ No newline at end of file diff --git a/tests/data/text/bloomberg.com1.txt b/tests/data/text/bloomberg.com1.txt new file mode 100644 index 00000000..b015a297 --- /dev/null +++ b/tests/data/text/bloomberg.com1.txt @@ -0,0 +1,77 @@ +Nevada, Arizona and Florida are among those furthest from their peak employment during the December 2007-June 2009 downturn. Those states, along with five others -- Alabama, Illinois, Michigan, New Jersey and Ohio -- remain more than 50,000 positions short of that level. + +Nevada, Arizona and Florida are among those furthest from their peak employment during... Read More + +Nevada, Arizona and Florida are among those furthest from their peak employment during the December 2007-June 2009 downturn. Those states, along with five others -- Alabama, Illinois, Michigan, New Jersey and Ohio -- remain more than 50,000 positions short of that level. Close + +Kevin Yearout has added about 80 jobs to his Albuquerque, New Mexico, contracting company since July of last year. That still leaves him with less than half the number he employed in 2009, at the end of the deepest downturn since the Great Depression. + +“It has been a very slow climb back,” said Yearout, 51, co-owner and chief executive officer of Yearout Mechanical Inc. “The economy went very south, very quickly.” With his commercial construction business hobbled by government funding cutbacks, “I never see the local economy getting back” to justify the prior level of jobs. + +Even as the U.S. economy reached a milestone in May with employment exceeding the prerecession peak, 29 of 50 states have yet to match that accomplishment, according to Labor Department data compiled by Bloomberg. New Mexico, for instance, still had 4 percent fewer employed workers, ranking among the bottom 10 percent of states. + +“This is not like any other recovery,” said John Herrmann, director of U.S. rate strategy at Mitsubishi UFJ Securities USA Inc., who tracks the gross domestic products of states. “There is a tremendous disparity, not a uniform recovery at all, with the performance of the economy much more skewed on a regional basis.” + +The weakest jobs rebound has been in the states central to the 2002-2006 housing bubble and the subsequent price collapse. + +Nevada, Arizona and Florida are among those furthest from their peak employment during the December 2007-June 2009 downturn. Those states, along with five others -- Alabama, Illinois, Michigan, New Jersey and Ohio -- remain more than 50,000 positions short of that level. + +Yet the regional woes are far broader: New Jersey has been hurt by the loss of casino and pharmaceutical industry positions, New Mexico by U.S. government defense cutbacks, Alabama by weakness in manufacturing, and Michigan by a loss in auto jobs. + +The energy industry is driving the economic expansion in 12 of the 13 states leading growth since the recession ended, Herrmann said. Leaders include Texas, North Dakota, Oklahoma and Louisiana, with Oregon the only non-energy state among the standouts. Oregon has been boosted by technology manufacturing and fast growth in exports. + +Excluding the 1.1 million jobs created in Texas, which has led the expansion, the nation would be 350,000 below the prerecession peak, according to Federal Reserve Bank of Dallas research cited by its president, Richard Fisher. + +The spotty regional recovery meshes with the Federal Open Market Committee’s view there is “significant underutilization of labor resources,” according to the statement after its September meeting, and continued room to keep interest rates near zero long after the central bank ends its bond buying as planned in October. + +“There is no pressure to raise interest rates this year and into next given that many parts of the country have a surfeit of unemployed and underemployed workers,” said Mark Zandi, chief economist at Moody’s Analytics Inc. in West Chester, Pennsylvania. + +IHS Global Insight economists in Lexington, Massachusetts, project that every state won’t have returned to its peak employment until Michigan achieves that level in 2019. + +While the auto industry has recovered and sales in August reached the highest level since January 2006, employment levels in Michigan haven’t. + +“The auto industry has been moving out of Michigan,” said James Diffley, IHS Global Insight chief U.S. regional economist, who is based in Philadelphia. “Job totals have been declining.” + +Michigan regained 34,185 automotive manufacturing jobs between 2010 and 2013. Yet that doesn’t come close to replacing the 174,429 industry positions lost from 2001 to 2010, a 59 percent decline, according to data calculated for Bloomberg by the Ann Arbor-based Center for Automotive Research. + +While the auto industry remains dominant in Michigan, “education, tourism, green technologies” and information technology “will become increasingly important,” according to a Moody’s Analytics’ forecast in June. + +Weak wage growth outside a few areas, such as energy jobs in North Dakota and Texas, supports the view that most of the country remains far from fully recovered, said Gary Burtless, a senior fellow at the Brookings Institution in Washington and a former Labor Department official. + +Hourly earnings nationwide were up 2.1 percent over the past 12 months, compared to 3.9 percent in June 2007, six months prior to the past recession, Labor Department data show. + +Young adults 25 to 34 years old, who are most likely to relocate for jobs, have been moving at decreasing rates even after the recession ended with migration at historically low levels in 2013, according to a Brookings analysis. + +That mobility remains low suggests “that local job markets have been basically lousy almost everywhere” except a few areas benefiting from the energy boom, Burtless said. “Wages are behaving as though there’s still a lot of labor-market slack.” Wages gains have been basically flat for the past several years, he said. + +While housing prices have regained part of their losses, Nevada was 5.9 percent below its peak employment, Arizona 4.1 percent, Florida 1.5 percent and Georgia 1.1 percent. The data compiled by Bloomberg compares the top employment level for any month during 2014 with the maximum during the downturn. + +“The economy is still sluggish in Nevada,” said Jim Mason, president and co-owner of Taylor International Corp. in Las Vegas. ‘We still have a lot of excess capacity in the employment market.’’ + +In 2007, Taylor employed 500 people managing $1 billion in construction primarily of hotels and casinos. Today, the company employs 30 in the state and gets 20 applications for each job opening. + +In New Jersey, Trump Entertainment Resorts Inc., which owns two properties in Atlantic City, filed for bankruptcy protection in September. New Jersey has lost more than 5,000 casino hotel jobs in the past two years, according to state figures. + +Casinos there have faced new competition from Pennsylvania, Connecticut and New York, and there has been “very little private sector investment” in new manufacturing and commercial construction, said Joseph Seneca, a Rutgers University economist in New Brunswick. + +“We have a lot of white elephants -- large empty casinos,” he said. “This is a significant problem for the state going forward.” + +Alabama’s job level remains 4.6 percent below the peak during the downturn, according to data compiled by Bloomberg. Job creation has been hurt by weakness in construction, manufacturing and government spending, said Ahmad Ijaz, an economist at the University of Alabama in Tuscaloosa. + +“Commercial construction is still relatively weak due to sluggish consumer and business spending,” he said. “Nondurables, particularly the textile and apparel industry, will not be adding any jobs anyway because of off-shoring and automation. Government spending is weak because of cutbacks in federal spending which in turn also impacts state and local government spending.” + +Eddie Foreman, 40, of Opelika, Alabama, who has a part-time job at a fast-food restaurant, said better-paying and permanent industrial positions aren’t coming back. + +“The economy is tough and there are no jobs for us here,” said Foreman, who has put out 20 applications in the past year and had no interviews. He worked in a variety of manufacturing and construction jobs before the recession. “Eventually I think it has to get better. It can’t get any worse.” + +New Mexico’s economy was slowed as reverberations from last year’s federal government shutdown rippled through the state, according to a Moody’s Analytics report in April. The state, home of the Los Alamos National Laboratory, has government as its largest employer, with 24 percent of all workers compared to 16 percent in the U.S., according to Moody’s Analytics. There were 30,000 federal workers among the 193,000 public sector employees in the state last year, according to the Moody’s report. + +“Government has been a drag,” said Michael O’Donnell, research scientist with the Bureau of Business and Economic Research at the University of New Mexico. “Many of the private sector jobs rely on government funds and grants.” + +While governments and companies have started to seek bids for new construction projects, Yearout said, there’s no backlog of work that ensures rising employment a year from now. At about 280 now, he doesn’t see returning to the 750 workers the company employed in 2009 anytime soon. + +“There is a cautious optimism that things are turning around,” he said, yet private businesses sometimes back out even after seeking bids. “The owner is afraid to pull the trigger. There is no long-term visibility or momentum. Everyone is a little bit afraid.” + +To contact the reporter on this story: Steve Matthews in Atlanta at smatthews@bloomberg.net + +To contact the editors responsible for this story: Chris Wellisz at cwellisz@bloomberg.net Gail DeGeorge, Carlos Torres \ No newline at end of file diff --git a/tests/data/text/bloomberg.com2.txt b/tests/data/text/bloomberg.com2.txt new file mode 100644 index 00000000..26bd9102 --- /dev/null +++ b/tests/data/text/bloomberg.com2.txt @@ -0,0 +1 @@ +This site uses cookies. By continuing to browse the site you are agreeing to our use of cookies. X \ No newline at end of file diff --git a/tests/data/text/bostonherald.com1.txt b/tests/data/text/bostonherald.com1.txt new file mode 100644 index 00000000..a98ad9e6 --- /dev/null +++ b/tests/data/text/bostonherald.com1.txt @@ -0,0 +1,49 @@ +, a 54-year-old motel maid, simply wanted to vote. But in 1965, as one of 100-plus black locals in line to register in , she confronted a segregationist sheriff - and ended up on the ground, beaten viciously with a billy club. When asked to portray the unlikely freedom fighter in the big-screen drama , hesitated: "I didn't want to do it because in every movie I'm hitting somebody!" she tells Us. (Previous wallops: The Color Purple and The Butler.) But director , 42, persuaded her (more on that below), and Winfrey, 60, joined the release - about ( ) and his -to­Montgomery marches for civil rights - as a star and producer. Shortly after DuVernay became the first black female director nominated for a Golden Globe (and the film's Oscar buzz began building), the two sat down with Us to talk, laugh and, in Winfrey's case, get a little misty-eyed. + +Ava, how did Oprah do when she had to punch the sheriff? + +AD She was all in. She had no problem working with the stunt guys to be taken down. I kept saying to them, "Please, can you just be a little more careful?" And they're like, "She's falling on her own!" + +OW I did it every time. + +With in the headlines, is particularly meaningful now? + +OW It's a jaw-dropping thing that this piece of art can meet this cultural moment that's so rich, so robust, so bursting with the energy of people finding their voices. This film's about being heard. I feel like this film, not to overstate it, but it is here for a reason in this moment. + +Will you be screening this for your close friend ? + +OW For sure. Our desire is to go to the and show it to him. + +With such intense scenes - marchers were teargassed - was it tough keeping your emotions in check? + +OW The very first time I saw it with the other producers, they're all taking notes. And I'm, like, sobbing. And I thought, OK, I guess as a producer you're not supposed to sob (continuing in a choked voice) so let me just take some notes too. I'm gonna blow my nose and stop crying. + +Oprah, how did Ava try to woo you for ? + +OW I was just gonna be in the background going, "Yay!" But she sent me the link to a story about when she turned 100, and it said every day she watched the Oprah show eating a tuna-fish sandwich! And Ava says, "Don't you think it would make her proud to know that you played her?" And I go, "Yeeesss, yes, it would." + +It probably also helped that as a girl you ­wanted to be Dr. King! + +OW What made me think I was gonna do that? I was 12, 13. I remember being at our yellow Formica kitchen table filling out one of those "What are you gonna be when you grow up?" forms, and my father was saying, "You can't be Dr. King because Dr. King is a man!" "Well, I'm gonna have me a church," I said. + +What about you, Ava? Who was your idol? + +AD (Pointing at Winfrey.) My mom had magazines with you on the cover on our coffee table since as long as I can remember, to the point where I thought you were my family member. She would tape your show at work and watch it at night. My mom would just say, "Look at her. Don't be like me, be like her," and get emotional. + +OW Oh, God, I'm emo­tional about that (tearing up). You never told me that! (She playfully smacks DuVernay.) Don't tell me that in an interview for the first time! + +Let's talk about something lighter. Ava, you worked on Scandal! + +AD Imagine a geek fan getting a chance to direct her favorite show! I can make be Fitz, and I can do whatever I want: "Walk over there. Now walk back. Turn. Oh, that looks good." You know what I mean? (Laughs.) So that was fun-fun. + +So you're Team Fitz, then - not Team Jake? + +AD Oh, Fitz. + +OW Fitz. + +AD Sorry, Scott Foley! + +The Real-Life Heroes + +Oyelowo says he gained insight from King friend , who revealed "the prankster … the man who was at times unsure." As for Ejogo, Winfrey says and sister Bernice found she "depicted their mother beautifully." diff --git a/tests/data/text/bostonherald.com2.txt b/tests/data/text/bostonherald.com2.txt new file mode 100644 index 00000000..7b946e19 --- /dev/null +++ b/tests/data/text/bostonherald.com2.txt @@ -0,0 +1,5 @@ +Chrome Industries' Mini Buran laptop messenger bag ($140) is a good investment for anyone who needs a sturdy bag that is built to last and can contain all of your stuff, whether it be a thin laptop or iPad. There are also small pockets for lighter objects. + +There are four pockets in the front under the flap, a large pocket inside and a hidden lined pocket where a thin laptop or iPad and rest safe and snug​. This bag is large enough to carry a lot of stuff, but not bulky. + +Seemingly indestructible, the bag features a seatbelt-buckle strap that grips your body nice and tight, but is ridiculously easy to get on and off. \ No newline at end of file diff --git a/tests/data/text/businessinsider.com1.txt b/tests/data/text/businessinsider.com1.txt new file mode 100644 index 00000000..e188e70c --- /dev/null +++ b/tests/data/text/businessinsider.com1.txt @@ -0,0 +1,13 @@ +The UK may be forced to review its Falkland Islands air defenses to face a renewed threat in the South Atlantic. + +According to a report in the Daily Express newspaper, the Argentine Air Force is set to get a dozen Sukhoi Su-24 Fencer attack planes from Russia in return for foodstuff. + +Due to this, the UK Ministry of Defense is in the process of reviewing the Falkland Islands air defenses. The delivery of the supersonic, all-weather attack aircraft could pose a threat to the islands, referred to as “Malvinas” by Argentina. + +According to Jane’s, the islands current British air defenses include four Eurofighter Typhoon jets, Rapier SAM (Surface to Air Missile) systems, along with about 1,200 troops permanently stationed in the South Atlantic base. + +Even though the Typhoons are modern enough to deal with a dozen Su-24s, the Soviet-era twin-engined two-seater are able to perform ultra-low level surface and maritime strike missions. The planes can be outfitted with a wide variety of General Purpose as well as Laser Guided Bombs and stand-off missiles, such as the Kh-31 (AS-17 “Krypton”) anti-radiation and anti-shipping sea-skimming missiles. + +We don’t know whether the potential deal includes armament; still the possible delivery of Su-24s to Argentina makes the Falkland Islands a bit more vulnerable to an attack by the Fuerza Aérea Argentina. + +This article originally appeared at The Aviationist. Copyright 2014. Follow The Aviationist on Twitter. \ No newline at end of file diff --git a/tests/data/text/businessinsider.com2.txt b/tests/data/text/businessinsider.com2.txt new file mode 100644 index 00000000..9d85d88a --- /dev/null +++ b/tests/data/text/businessinsider.com2.txt @@ -0,0 +1,35 @@ +FA Insights is a daily newsletter from Business Insider that delivers the top news and commentary for financial advisors. + +Russia's seen a volatile December: hiked interest rates, increased inflation, and a plunging ruble. Morningstar's Karin Anderson suggests that Russia's problems reflect the larger risks associated with emerging markets. + +"Emerging markets' fundamentals can change fairly quickly with changes in political regimes, commodities pricing, or geopolitical risk. Second, currency fluctuations can have a quick and pronounced impact on investors' assessment of the risks and valuations of emerging-markets bonds," writes Anderson. + +"Because of those risks, Russia is a reminder that emerging markets remain subject to swift and meaningful changes in capital flows," Anderson adds. + +"F-Squared Investments Inc., which builds investment portfolios out of exchange-traded funds, admitted it misled clients about its track record and agreed to pay $35 million in a settlement with regulators," reports Corrie Driebusch. + +The SEC filed separate civil charges against the former CEO. He stepped down from his position last month, but his lawyers maintain that the SEC's allegations "are misdirected and meritless." + +Analysts Aren't Feeling Great About Commodities In 2015 (Think Advisor) + +Thanks in part to the strong US dollar, commodities have seen a tough year. And analysts don't think that things will get any better in the next one. + +A recent Bloomberg survey sees Brent sliding to $50 a barrel, and Goldman Sachs analysts "see further declines in prices for oil and for other commodities with anticipated strengthening of the US dollar in 2015 and weaker demand for commodities in China," reports Janet Levaux. + +Additionally, analysts have mixed feelings about agri-commodities in the next year. + +An SEC Investor Advocate Will Push For Spending More On RIA Exams (Financial Advisor Magazine) + +"Securities and Exchange Commission Investor Advocate Rick Fleming said Tuesday he will push SEC chairman Mary Jo White to use a 'significant' portion of its extra $150 million for 2015 to go to increase financial advisor oversight," reports Ted Knutson. + +"Fraudulent or abusive practices by an investment advisor," said Fleming, "can be very difficult for an individual investor to detect, particularly if the advisor goes so far as to falsify account statements or other records." + +Fleming reportedly has limited clout within the agency. + +There Are 4 Things Advisors Need To Know To Avoid Getting Fired (Financial Planning) + +Veteran Wall Streeter Charlotte Beyer has four tips for how to keep wealthy investors as clients. + +The first two are related: advisors should probe and get feedback, and they should overcommunicate. The biggest reason advisors get fired, writes Beyer, is "it's always about communication — usually a lack of — every time." + +Additionally, advisors should be diplomatic about demands by anticipating a client's needs. And finally, advisors should be consistent by emphasizing the long-term approach. \ No newline at end of file diff --git a/tests/data/text/businessweek.com1.txt b/tests/data/text/businessweek.com1.txt new file mode 100644 index 00000000..c067a21b --- /dev/null +++ b/tests/data/text/businessweek.com1.txt @@ -0,0 +1,93 @@ +“Do you want to listen to Taliban cassette?” Matiullah Matie asks as he steers his white Toyota Corolla along a narrow road surrounded by cornfields and mud huts. He keeps the tapes in the car for long drives, Matie explains, just in case he picks up a hitchhiker who looks like a Talib. “They think I am such a pious mujahid man,” the round, bearded businessman laughs. “They don’t know I am screwing them all.” + +We are driving to the Nawa district, just 30 minutes outside Lashkar Gah, the capital of Helmand province in the southwest corner of Afghanistan. Matie is going to show us how he first became a millionaire. + +Earlier that morning, photographer Lorenzo Tugnoli and I found Matie sprawled on his office floor. He’d spent the night Facebooking—until he passed out. In the corner, on the armrest of a brown couch, a Dell laptop flashed an error message. Stacks of blue posters for the cell phone company Salaam lay against the wall. Matie had recently bought the local Salaam distribution license. It’s his latest project. + +When we drive into the bazaar at Nawa, people recognize Matie immediately. Many wave at him. He’s done business here before—and he’s already brought Salaam to the district. That’s the reason one man with a neatly trimmed beard approaches the car and leans in to chat. Matie curses his luck under his breath. + +“I have bought 100 SIM cards but no one buys,” says the guy, a retailer representing Matie’s franchise. Matie tells him to be patient. It’s a new company, he explains, business will pick up. + +“Will you come back for lunch, all of you be my guests?” the man asks. + +“Sure,” Matie says. “Make some chicken for lunch once we drive back from Garmsir.” + +Matie has no intention of going to Garmsir or lunch with the man. “The bastard’s son still has links to the Taliban,” he says as we drive on. “You really can’t trust anyone.” + +In a few minutes we reach the compound of the 1st Battalion 9th Marines—“The Walking Dead,” as a yellow logo proclaims inside one of its rooms. The U.S. Marines packed up a year ago, and all that’s left is a series of shipping-container offices that once housed U.S. Agency for International Development contractors. The desks and furniture are locked inside; the windows are covered in dust and cobwebs. But when the Marines ruled Nawa—the district governor’s office was within their compound—the Americans started Matie on his road to prosperity. In the U.S., wartime contracting is often associated with such names as Blackwater (now known as Academi), DynCorp International, Triple Canopy, and others, but on the ground in Afghanistan, the Pentagon depended on a small army of locals. And as hundreds of billions of dollars in U.S. taxpayer money poured into the country, it created a new class of wealthy, entrepreneurial Afghans. + +The October 2001 U.S.-led invasion and the subsequent allied military campaigns transformed the country. At the end of 2014, however, as the American troop presence draws down to 10,000 from a height of 98,000, it’s becoming clear that the U.S. dollar has reshaped Afghanistan even more than the military did. In private, U.S. officials admit they don’t know how much they’ve spent on the Afghan war. Independent analysts estimate its cost at about $1.6 trillion—factoring in inflation and long-term care for veterans. The money found its way not just into the hands of ruthless oligarchs, as in post-Soviet Russia, but also into those of teachers, translators, restaurant owners, and drivers who tapped into the gusher of cash to become millionaires and multimillionaires. + +In the five years that Mullah Omar and his Taliban regime dominated Afghanistan, “foreign currency was rare. There probably wasn’t even $2 million in the market,” says Khan Mohammad Baz, the bespectacled head of the currency exchange union at Sarai Shahzada, Afghanistan’s central exchange market. “By 2003 there was probably $1 billion circulating.” These days, Baz says, about $20 million worth of business deals are made in a day. The central bank alone pumps about $60 million a week into the market to buy back the Afghan currency and keep it stable. + +About 36 percent of Afghanistan’s 30 million people live below the poverty line. “If you ask people on the streets whether we have a billionaire, they will shrug and say no,” says a senior Afghan economic official, who asked to remain anonymous because he is privy to sensitive information. “But I can tell you with confidence we have many. If the top 10 wealthy men in Afghanistan—I would say 9 of them products of the past 10 years—came together, they could buy this government, the bank, this whole system.” + + + + + +Afghanistan’s megarich are not shy about their wealth. Many are driven around in $150,000 armored vehicles, trailed by convoys of cars and pickup trucks full of security guards. Several live in Wazir Akbar Khan, Kabul’s diplomatic enclave, but others have illegally carved up Sherpur, an historic hill district in the capital. Some have second homes in Dubai, Istanbul, or various European cities. Just like Russia’s oligarchs, many members of this wealthy class owe their fortunes to politics. Some are warlords who helped the U.S. topple the Taliban; others are technocrats who returned from abroad to work in the new government. Both groups enriched themselves through the country’s system of patronage and influence—and by drawing on the immense sums of American cash flowing into it. Afghanistan runs on connections, and many of the biggest dealmakers operate with impunity. A clan can have one brother in the administration, another in parliament, and yet another running a huge company or state enterprise. The family of former President Hamid Karzai was the object of much criticism for that reason. + +Trailing behind the politically influential is a much larger—and younger—class of nouveaux riches, which includes Matie. Spread around the country, they’ve made money by getting close to the American military and responding to its immediate needs. Many were translators who saw gaps in the Pentagon’s supply chain and took advantage of the situation by becoming contractors. Others were simply entrepreneurs who fed off the donor money being doled out to every sector of the post-Taliban society. + +There’s still money to be made from the American military—though it’s a much smaller pie, and more local contractors fight over it. The U.S. and its NATO allies will continue to provide Afghanistan with more than $5 billion annually for its security forces and $5 billion to $8 billion for reconstruction. But the largesse will now flow through the central government, with its propensity for playing favorites. From now on, ministries in Kabul will be in charge of dispensing the contracting cash. + +The new president, Ashraf Ghani, is promising to bring order to procurement and contracting, but transparency may be difficult to achieve. Among Ghani’s first appointments was Hazrat Omar Zakhilwal, a finance minister under Karzai who was entangled in the country’s biggest banking scandal, among other controversies. Zakhilwal, who denies any wrongdoing, now has oversight over the entire financial portfolio of the country. + + + + + +In 2009, Matie, then in his late 20s, trundled up to the Marine compound on a donkey, after treading slowly through a heavily mined field. “Out of control mines,” he recalls. The son of a religious studies teacher, he had already tried many jobs, including joining the Taliban, twice. He had been working as a customer-care representative—making a lucrative $300 a month—with one of the new telecom companies when he realized he wanted to start his own company. He was sitting through a business development training seminar conducted by Malaysians when he thought to himself, “I want to be my own boss.” He quit and got a license to start a construction company. He wrote up a company profile and fact sheet—as the Malaysians had taught him—and, two weeks before Ramadan, put the papers in a saddle, mounted his donkey, and headed for the Marines in Nawa. “Hi, sir! Is there anybody to talk to me?” he’d shouted in his elementary English at a Marine manning a watchtower. They were happy to let him in. + +The Marines were part of Obama’s surge to push back a Taliban onslaught that threatened to overwhelm the towns loyal to Karzai’s government. In Nawa the surge expanded the U.S. military presence from 100 troops to 1,100; a contingent of that size needed local logistical support. When the Marines arrived, they found the local bazaar deserted—except for a boy selling cans of Pepsi. “You couldn’t find a single contractor here, they were all too afraid,” recalls Abdul Manaf, the aging district governor, as he puts on his hearing aid. “Matiullah was the first to come.” + +The Marines had cash and lots of it. Congress has appropriated about $3.7 billion over the past 10 years for the Commanders Emergency Response Program (CERP), a fund that officers in Afghanistan and Iraq could draw on for “urgent humanitarian relief and reconstruction requirements in their areas of responsibility.” In Helmand province—one of the areas fiercely contested with the Taliban—the U.S. military would spend $153 million on 2,164 CERP projects. USAID also poured money into the area through foreign contractors who implemented so-called stabilization projects—such as rebuilding bazaars and supplying technology to district offices. For example, according to the Washington Post, USAID spent $30 million on agriculture in Nawa over the course of nine months in 2010. All of this created opportunities for enterprising Afghans such as Matie. + +His first project was the reconstruction of the district governor’s office. It needed new doors, windows, fresh plaster—and walls. When the Marine captain proposed the project to him, Matie calculated an estimate on the spot: $10,109. He asked for five days to get things going. + +Only later did he realize what he’d agreed to do. The area between Lashkar Gah and Nawa was strictly Taliban country—and Matie had to transport gravel, shovels, and barrels from the provincial capital to the Marine compound. So when Matie recruited laborers, he didn’t tell them he was sending them to Nawa but rather to another, safer district nearby. “I went ahead on a motorcycle,” he recalls. “When they got here, I said, ‘Don’t you worry. I will give you more money than you want. As much as you want.’ ” The workers stayed for the entire 15 days of the project. + +To deliver supplies, he rented a Mazda dump truck for $600. He didn’t have funds to hire a security escort, but this time he didn’t lie. “We are going to Nawa,” he said to the driver. “But I am riding alongside you, and whatever happens to you will happen to me first.” + +He found two motorcycles, one for himself and one for his assistant. He dressed in Taliban style: white clothing, a large paaj turban on his head, his beard oiled, black shades over his eyes. In his pocket he had a small radio that captured Taliban military signals, which he played loudly. As the motorcycles escorted the truck on the bumpy road to the base, they passed several Taliban. “Salaam u alikum,” he’d shout—the traditional “peace be with you”—with authority, and the Taliban would respond the same way, addressing him as Mullah sa’eb, a term of reverence. + +To keep up the masquerade, Matie says he’d curse at the driver. “Keep going, you son of a swine. You f-‍-‍-er, this is what you get for supplying infidels. Keep driving.” (“I had informed him I’d be cursing at him,” Matie explains. “I told him not to take it to heart.”) + +By 2012, Matie’s company had more than $2 million in the bank. It had delivered fertilizer and seeds; it had helped repair clinics, schools, and government buildings; and it had graded more than 77 kilometers of local roads, smoothing them with gravel. He also helped deliver USAID cash to far-flung districts as part of a jobs program called Cash for Work. He started other businesses as well, importing Iranian biscuits and shampoo from Nimroz, a large border province and hub for smugglers, distributing the products across the country. He invested his earnings abroad, including putting $100,000 into a bakery in the United Arab Emirates. He had become rich—thanks to American spending. + +Because the prosperity of Matie’s newly rich class often stems from loose American money, it can carry the odor of malfeasance and corruption. The office of the U.S. Special Inspector General for Afghanistan Reconstruction (SIGAR) is investigating several cases, following the money to see if American funds were misappropriated or even spent to support the insurgency. Double-dealing is almost instinctive here, part of a survive-at-all-cost mentality ingrained by decades of chaos and war. When the country emerged from Taliban rule at the end of 2001, “it’s like we were stuck in a dark well of isolation, then someone threw us a rope to pull us up,” says Naseem Akbar, a former economic official in the Afghan government. “But we somehow got all strangled up in that rope.” + +Corruption is pervasive and visible: the flashy car belonging to a tax clerk whose monthly salary is $200; the fancy bungalow of a precinct police chief. Money purchases status, buys protection—from the law and perhaps even from God, judging by the number of mosques built with ill-gotten funds and the many hajj pilgrimages financed by dirty money. “Has corruption become what holds everything together in Afghanistan?” one Western official asks. “Maybe.” + +Unlike some of the extremely rich—who put their profits into foreign bank accounts—the entrepreneurial class tends to keep much of its cash within the country’s borders. “There’s everything-to-myself corruption,” the same official explains, “and then there’s this Tammany Hall kind of corruption, a sort of Robin Hood style, where you are generous to the community.” + + + + + +Hikmatullah Shadman doesn’t look like Robin Hood, though the American investigators have grave suspicions about him. He works in what used to be the Kabul home of Ahmad Zahir, a legendary entertainer known as the Afghan Elvis. Shadman, 29, likes flowers. His pastel-colored compound in Wazir Akbar Khan looks like a dollhouse, with plastic flowers strung across the ceiling and framing paintings, mirrors, and photographs. “Flowers make me happy,” Shadman says as he sits down for an interview about his businesses and philanthropies—which are mainly in Kandahar, almost 300 miles to the south. He wears a black sports jacket over a black tunic embroidered in silver. On the table in front of us are platters of dried fruit and bottles of Gatorade, Starbucks frappuccino, and Ocean Spray cranberry juice. A cleanshaven elderly man—whom Shadman refers to as mama, or maternal uncle—is thumbing his prayer beads while lounging on a couch to the businessman’s right. + +Shadman prefers not to talk too much about his philanthropic activity—though it has earned him a degree of influence with the public as well as the government. The Afghan media says he arranged to set up dormitories for university students in the eastern city of Jalalabad; that he was one of the first to send a convoy of aid after mudslides devastated the northern province of Badakhshan in May; that he supports more than 60 students on scholarships, including 13 he sent to schools in India. Most recently, Shadman launched a Mr. Facebook contest in Kandahar to identify and award citizens who use the social media site for public good. He has supplied food—through the government—to 180 families in a Taliban-controlled village in Kandahar province. “We show that Talib means mines and explosions; government means aid,” he says. While he still instinctively refers to the ousted Taliban leader reverently as Mullah sa’eb, Shadman is trying to create a different kind of Afghan identity and nationalism—out from the shadows of the white-robed jihadis. In October he announced he would build a mausoleum for Malala of Maiwand, perhaps the most famous woman warrior in modern Afghan history, whose campaign against British invaders in 1880 led to her being described as the country’s Joan of Arc. + +Shadman also traces his riches back to U.S. military money. The son of a literature teacher in Kandahar, he sold almond sweets in the bazaar after school. When the U.S. ousted the Taliban, he went to work for a local mason rebuilding the airport. Soon, he became an interpreter for a U.S. Army Special Forces unit that, within six months, conducted more than 50 combat operations in the area. Accompanying the U.S. soldiers kept Shadman away from home for weeks at a time but allowed him to save much of his monthly salary. He bought a Land Rover for about $4,000 and leased it back to the Special Forces. “I was making two salaries after that. I made $800, and my vehicle made $800.” + +In a couple of years, he’d purchased hundreds of vehicles, renting them to the U.S. military and foreign contractors who came to Afghanistan. He also started doing construction projects for Canadian units that were part of the International Security Assistance Force, as the U.S. and its allied troops were called. His Special Forces bosses helped him with connections that got him contracts to supply propane to NATO bases in the south. Shadman’s main line of business, however, became trucking. Working first as a middleman for a Hungarian firm that provided the exuberantly decorated “jingle trucks” for ferrying goods throughout the country, Shadman quickly built a fleet of his own vehicles. Profits escalated with the U.S. surge. According to court documents, he carried out 5,421 transport missions for the ISAF. + +Shadman is accused of defrauding the U.S. government of $77 million. According to court documents, SIGAR alleges that Shadman managed to expand his trucking empire only because he “bribed and paid kickbacks” to managers of the Hungarian contractor, who then allegedly inflated prices for Shadman so he could charge the ISAF even more. In October 2012, at 4:30 a.m., the U.S. military raided his compound in Kandahar. He says they flashed a light in his eyes, blindfolded him, tied his hands, and flew him to the prison at the American military base at Bagram. He was held there for 74 days and accused of funding the enemy and supplying women to the Taliban and alcohol to U.S. soldiers. In a civil forfeiture lawsuit, SIGAR and the U.S. Department of Justice asked for Shadman’s accounts in an Afghan bank to be frozen. However, they were quickly unfrozen by Afghan authorities and some of the money has made its way to Dubai, where he has three homes. + +Shadman says he is heartbroken by the way the Americans turned against him. “I grew up with them, with their soldiers.” He insists he’s not afraid of litigation, because the evidence against him is flimsy. “My only wish is to prove to the American public that … in my case your tax money has not been wasted.” Then he turns from being politic to blunt. “My money is clean. I don’t hide it. It’s there in the open, for America to see it, for London to see it. I have no fear.” + + + + + +Shadman has been able, so far, to withstand the legal assault on his reputation. While he no longer has any contracts with the U.S. military, he imports German energy drinks, which are extremely popular among young Afghans. He is planning to build a pomegranate juice factory in Kandahar. + +Matie’s trajectory, however, has shifted. He’s gone from rags to riches to starting all over. In 2012 he decided to use some of his largesse to travel to Mecca for the hajj—one of the five “pillars of Islam” that pious Muslims are enjoined to do. When he returned from the monthlong trip, his money was gone. He says his partner had cooked up a scheme with locals, taking advantage of his absence to complain that 1,450 people hadn’t received their USAID Cash for Work payments because Matie was out of the country. His partner, he says, told him the laborers had already been paid. As USAID and the Marines tangled with Matie over details, the partner packed up and fled to Kabul. Matie says he’s appealed to the government for help, but so far nothing has happened to remedy the situation. “He will fight me by bribing the government with my own money,” Matie complains. “I can’t do anything in this government.” + +He says his fortunes fell so low that he didn’t even have gas money when he was stuck in the countryside, his fuel tank and his pockets empty. “I called Marine friends, and they sent me fuel for my car.” + +With U.S. money now being dispensed by the political elite, contractors such as Matie who aren’t at the top of the food chain have to change gears completely. That’s why he acquired the telecom distribution contract with some cash he saved from a couple of small projects for the U.S. embassy. What he makes now doesn’t compare to his income at the height of the surge. He’s philosophical about his new financial situation: “You make little, but it’s more sustainable.” + +He acknowledges the pain of losing so much money but says it’s easier on him than others who’ve also seen riches come and go. He never let money change his modest “nomadic” way of living, moving from town to town to do business and sell services. “When I had money, I lived like this also,” he says. “I have lived because of my honesty and my parents’ prayers.” He adds: “Those who stole money from me, I know what kind of wrath God will inflict on them.” \ No newline at end of file diff --git a/tests/data/text/businessweek.com2.txt b/tests/data/text/businessweek.com2.txt new file mode 100644 index 00000000..91c72154 --- /dev/null +++ b/tests/data/text/businessweek.com2.txt @@ -0,0 +1,9 @@ +New York Governor Andrew Cuomo vetoed a bill last week that would have allowed the New York state, city, and teachers’ pension funds to increase their investment in hedge funds from 25 percent to 30 percent of fund assets. He cited the high fees and risk associated with hedge funds. The now-dead bill contained a memo justifying its proposed increase: + +That’s a terrible reason to invest in hedge funds. Stocks are supposed to be riskier; that’s why they normally provide higher returns. If the investment board can’t handle volatility, it should invest in safer, lower-yielding assets—and while that may describe some hedge funds, cheaper options are out there. Hedge funds typically charge 2 percent of assets, plus 20 percent of gains, and rarely outperform the stock market. The figure below, from an article last year, plots an index of hedge fund strategy performance compared with the S&P 500 index. + +Underperformance is why such states as California are pulling out of hedge funds (though they are piling into risky alternatives, such as private equity). The sentiment is not uniform, though, and other states, including New Jersey, Ohio, New Mexico, and Illinois, are sticking with or increasing their hedge fund investment. + +Cuomo was right to veto the bill. Investing in hedge funds is extra risky for New York, which is also home to a large share of the hedge fund industry. Employees in the finance industry pay a large share of the state’s tax revenue. That would seem to give politicians an incentive to increase their hedge fund investment. But it’s a dangerous strategy. If the hedge fund industry has a bad year, the state would take a twofold hit: Pension fund assets would tank, along with the tax revenue from finance industry professionals whose compensation depends on performance. + +Like all states, New York’s pensions are underfunded. In 2013, its assets could cover only 87.3 percent of pension promises, a figure that assumes superior investment performance each and every year. More prudent measures estimate a larger deficit. Making up the difference will require either outstanding, consistent investment performance or more contributions from state employees and taxpayers. The governor made a good choice by realizing that investing in hedge funds isn’t the answer. He has made less progress on how to fill in the gap reliably. \ No newline at end of file diff --git a/tests/data/text/cleveland.com1.txt b/tests/data/text/cleveland.com1.txt new file mode 100644 index 00000000..bfbf0b4f --- /dev/null +++ b/tests/data/text/cleveland.com1.txt @@ -0,0 +1,16 @@ +Chia seeds and goji berries are the new kale and quinoa, according to Google's recent parsing of food-focused searches from 2014. Each year, the search giant pours through some of our more fascinating queries to come up with their Year in Search. + +Some of the other more interesting food-related data points include: + +Pizza was searched more than the World Cup. +The Cronut rose to 17th on the global recipe list after its arrival last year. +Our favorite ways to eat eggs are: 1) Deviled, 2) Scotch, 3) Scrambled, 4) Pickled, 5) Boiled. +This year we searched for 'recipes' less and 'restaurant' significantly more. +Our top slimming questions were 'how many calories should i eat in a day' and 'how to lose weight,' and the Paleo diet was the top searched way to trim down. +Foodies in Japan searched French food more than France. +Hungry folk in Australia searched Argentine food more than Argentina. +Spice-loving Brits searched Indian food more than India. +In 2014 'i am hungry' was searched a button-popping 7x more than 'i am thirsty.' +Oh, and nine million people watched a tiny hampster eating a tiny burrito. + +The lesson, as always: you are what you search. diff --git a/tests/data/text/cleveland.com2.txt b/tests/data/text/cleveland.com2.txt new file mode 100644 index 00000000..5a9e9e39 --- /dev/null +++ b/tests/data/text/cleveland.com2.txt @@ -0,0 +1,75 @@ +The Pilot and the Little Prince + +Peter Sís + + Foster Books, $18.99; ages 5-8 + +This exquisitely illustrated biography recounts the life of writer and aviator Antoine de Saint-Exupéry. Lovers of his classic tale "The Little Prince" will find that the masterful Sís has captured the wonder and dreaminess evoked in that book. -- Karen Sandstrom + +Goodrich finds hilarity and drama in "Mister Bud Wears the Cone," a story of one dog's humility in the confines of a plastic head cone. -- Sandstrom + + +Mister Bud Wears the Cone + +Carter Goodrich + +Simon & Schuster, $16.99; ages 4-8 + +Goodrich finds hilarity and drama in "Mister Bud Wears the Cone," a story of one dog's humility in the confines of a plastic head cone. -- Sandstrom + + + Abuelo + +Arthur Dorros; illustrated by Raúl Colón + + Harper, $17.99; ages 6-8 + +A young boy learns a lot from his grandfather in the country as they camp and ride horses under the big sky. The abuelo also teaches his grandson the Spanish words for important ideas. Colón's rich illustrations make each page dazzle. -- Sandstrom + +A girl and her mouse keep their friendship a secret from their parents in this gorgeously illustrated look at parallel lives. You'll marvel over the details McClintock gets just right as she lays out the two worlds in a soft, golden palette. -- Sandstrom + + +Where's My Mommy? + +Beverly Donofrio; illustrated by Barbara McClintock + +Schwartz & Wade, $17.99; ages 3-7 + +A girl and her mouse keep their friendship a secret from their parents in this gorgeously illustrated look at parallel lives. You'll marvel over the details McClintock gets just right as she lays out the two worlds in a soft, golden palette. -- Sandstrom + + + President Taft is Stuck in the Bath + +Mac Barnett; illustrated by Chris Van Dusen + + Candlewick Press, $16.99; ages 4-8 + +The story goes that our horizontally challenged 27th president once became trapped by in a bathtub. Branett's words and Van Dusen's silky smooth illustrations have the most possible fun with the idea, yet remain respectful. Historical notes at the end separate fact from fiction. -- Sandstrom + +Woodson won the National Book Award for this childhood memoir in verse. It begins with Woodson's birth in 1963. "I am born on a Tuesday at University Hospital / Columbus, Ohio, / USA – / a country caught / between Black and White." A wonderful model for young readers wanting to write their own family stories. -- Tricia Springstubb + +Weaving Norwegian folklore with actual hardships faced by 19th-century immigrants, Preus spins a riveting tale of love, sin, remorse, forgiveness and how stories give us courage. -- Springstubb + +Sports and life intertwine in this electric celebration of the game and of the power of brotherhood and family. The language sizzles and pops, ranging from free verse to hip-hop to poems for two voices. -- Springstubb + +Behind a museum door, Ophelia discovers a small boy with an unbelievable tale: Wizards have chosen him to defeat the wicked Snow Queen, and he needs help. Ophelia finds herself bravely battling evil, not just to save him, but her beloved father and sister, too, in this melancholy and beautiful book. -- Springstubb + +Maggie is smart, self-centered and friendless – not that she minds one bit. Her own brain and ambition are company enough. Resolving to fix her father's illness, she discovers multiple sclerosis has no cure. Maggie's family copes with courage, humor and love, and readers will not only root for them, but believe in them. -- Springstubb + +Mississippi, 1964. Sunny and her stepbrother sneak into their town's segregated pool one hot night. To their surprise, a "colored" boy is there, too. Wiles chronicles the Freedom Summer through a mixture of fiction and documentary, including photos, excerpts of speeches and song lyrics. -- Springstubb + +The Family Romanov: Murder, Rebellion, and the Fall of Imperial Russia + +No fiction is more fantastic than the true story of foolish Nicholas, fanatic Anastasia, and their five beautiful children. Geared toward middle-school readers, this superb account will also satisfy adults looking for an overview of this period of Russian history, when impoverished workers revolted, soldiers died by the millions, Lenin was on the rise, and the tsar, secluded in his opulent country palace, trusted in God to save his throne. -- Springstubb + +Why We Took the Car + +Boys (and girls, too) will love this joy ride of a story, featuring German teens Mike and Tschick, who "borrow" a car to escape the boredom of summer break. Adventures – and misadventures – ensue, until they end up in police custody. Fasten your seatbelt for a rollicking read. -- Susan Glaser + +The protagonist of this story is Cady Sinclair, a member of an upper-crust New England family that spends its summers on a private island off the coast of Massachusetts. Cady, together with same-age cousins Johnny and Mirren and love interest Gat Patil, make up "the Liars," tightknit teens who share adventures – and one big surprise. At age 17, Cady returns to Beechwood Island to piece together the details of an accident two years prior. The result: An ending you won't see coming. -- Glaser + +Twins Noah and Jude are the narrators of this tale, which alternates between their 13th and 16th years. In between: Their mother dies, their lives are upended and the two struggle to make peace with each other and the people around them. Their paths come back together at the end – but the real story is in the journey. -- Glaser + +Anderson, who has tackled date rape and eating disorders in previous works, turns her attention to post-traumatic stress disorder in this story about Hayley Kincain and her Iraq War vet father, Andy. Hoping for stability, the two return to Andy's hometown in upstate New York for Hayley's senior year of high school. Instead, the two are forced to deal with more chaos, including Andy's increasingly erratic behavior. A budding romance between Hayley and fellow high schooler Finn helps lighten the mood. Even so, Anderson maintains her status as one of the most serious storytellers in the young adult genre. -- Glaser + +Imagine that a second girl from Kansas was swept away to Oz after Dorothy left. But when Amy arrives, Oz is falling apart. The color is dull, and a huge hole in the middle of the land is getting bigger. What has happened to Oz? Short answer: Dorothy. Now it is up to Amy to fix things, and decide which of her unlikely companions are the most trustworthy. Is it the order of the wicked witches or the wingless flying monkeys, or both? Or are there other players in this game that we don't know about yet? I loved this book. -- Ami Bray, Loganberry Books diff --git a/tests/data/text/cntraveler.com1.txt b/tests/data/text/cntraveler.com1.txt new file mode 100644 index 00000000..27809b0c --- /dev/null +++ b/tests/data/text/cntraveler.com1.txt @@ -0,0 +1 @@ +Maryam Montague, the owner of this lovely, cheery boutique hotel works for a human rights NGO; runs her inn; writes and photographs; and sources rugs, textiles, accessories, and furniture from across Morocco, Central Asia, and Africa for a small clientele of high-profile fashion designers in New York and Europe. There’s a rotating selection of her finds available on her website, or you can stop by and see what she has in stock in person. (Kilometer 18, Route de Ouarzazate; 212-6644-14653; peacockpavilions.com) \ No newline at end of file diff --git a/tests/data/text/cntraveler.com2.txt b/tests/data/text/cntraveler.com2.txt new file mode 100644 index 00000000..63258874 --- /dev/null +++ b/tests/data/text/cntraveler.com2.txt @@ -0,0 +1 @@ +In 2013, the Federal Aviation Administration gave the green light for gadgets to remain on during an entire flight, but U.S. airlines still had to prove to the FAA that passengers could safely use their gadgets in Airplane Mode from gate to gate. In 2014, U.S. airlines made their case, and carriers in the U.K. and Europe followed. The new freedom to read a Kindle, use a travel app, or watch videos on takeoff is a victory over boredom. \ No newline at end of file diff --git a/tests/data/text/coolhunting.com1.txt b/tests/data/text/coolhunting.com1.txt new file mode 100644 index 00000000..5bc05dc5 --- /dev/null +++ b/tests/data/text/coolhunting.com1.txt @@ -0,0 +1,15 @@ +Vacationing in a former convent may sound a bit austere—that is, until you experience Monastero Santa Rosa on Italy’s famous Amalfi Coast. This monastery-turned-hotel offers all the peaceful seclusion the sisters enjoyed, with none of the personal sacrifices. And though the popular coastline is studded with hotels, this distinctive destination—with its unmatched combination of five-star luxuries, natural beauty and centuries of character—is a serious standout. + +Getting there might have you pressing your palms together in supplication. The main route through the craggy Amalfi coastline is Strada Statale 163—also known as "the road of 1,000 bends." The ancient Roman-built drive is full of hairpin twists and turns, careening over steep precipices past sun-bleached villages and lemon groves for 50 miles, and is just barely wide enough for two cars to pass at some points. About 18 miles in sits Monastero Santa Rosa, perched on a cliff above the small fishing village of Conca dei Marini. The venue's American owner Bianca Sharma spotted the nunnery’s ruins from a boat in 1999 and promptly bought the property. After a lengthy decade-long restoration, it opened as a luxury hotel for the 2012 season. + +Much of the convent’s original 17th century architecture remains intact, thanks to Sharma’s careful conservation. Inside, reconfigured spaces pay thoughtful homage to the hotel’s heritage. Nuns' quarters are cleverly combined into 20 unique rooms, some with private terraces or multiple levels. One of the largest is located in what was the sisters’ refectory. All feature fine Italian linens, an array of tasteful period furniture, and deep soaking tubs made from fine Jerusalem stone, in addition to everyday contemporary amenities like minibars, television, and WiFi. Some of the rooms have vaulted ceilings, private alfresco dining areas, and seafront balconies; others have housed famous names like Prince Albert and Princess Charlene of Monaco. The guest rooms exude comfort, and every conceivable wish is quickly taken care of by the attentive staff. + +The tasteful restoration continues into Monastero Santa Rosa’s massive 750-foot state-of-the-art spa. Cavernous treatment rooms are carved from spaces once devoted to silent prayers, while the spa’s centerpiece, a vaulted tepidarium (warm relaxation room), is where the sisters made wine. There’s also a sauna, steam room, hydrotherapy pool, whirlpool footbaths, ice fountain and even programmable showers. + +The real star of Monastero Santa Rosa is the knockout coastal view, and there’s a countless number of magical places for taking it in. From the convent’s highest point, an airy sunset terrace, guests can gaze as far as the buzzy nearby towns of Amalfi or Positano. There’s the cascade of tiered, expertly manicured semi-tropical gardens, beset with ocean-facing sun loungers, daybeds and cabanas. There’s the breezy dining terrace of the hotel Ristorante, where guests can enjoy the scenery while tasting exclusively sourced extra virgin olive oil produced right in Conca dei Marini. The most breathtaking spot is floating within the curved cliff’s edge infinity pool while surveying the Gulf of Salerno some 660-feet below. + +After whiling away the hours on these sacred grounds, be sure to unburden your soul at the large antique wooden confessional resting in the main hallway. It cheekily invites guests “make a confession” by way of a written feedback for the hotel. + +Monastero Santa Rosa is located in the town of Conca dei Marini on Italy’s Amalfi Coast. Nightly room and suite rates range from $500 to $2,400. The resort will open for the 2015 season on 17 April. + +See more photos in the gallery, images by Tanveer Badal \ No newline at end of file diff --git a/tests/data/text/coolhunting.com2.txt b/tests/data/text/coolhunting.com2.txt new file mode 100644 index 00000000..5bc05dc5 --- /dev/null +++ b/tests/data/text/coolhunting.com2.txt @@ -0,0 +1,15 @@ +Vacationing in a former convent may sound a bit austere—that is, until you experience Monastero Santa Rosa on Italy’s famous Amalfi Coast. This monastery-turned-hotel offers all the peaceful seclusion the sisters enjoyed, with none of the personal sacrifices. And though the popular coastline is studded with hotels, this distinctive destination—with its unmatched combination of five-star luxuries, natural beauty and centuries of character—is a serious standout. + +Getting there might have you pressing your palms together in supplication. The main route through the craggy Amalfi coastline is Strada Statale 163—also known as "the road of 1,000 bends." The ancient Roman-built drive is full of hairpin twists and turns, careening over steep precipices past sun-bleached villages and lemon groves for 50 miles, and is just barely wide enough for two cars to pass at some points. About 18 miles in sits Monastero Santa Rosa, perched on a cliff above the small fishing village of Conca dei Marini. The venue's American owner Bianca Sharma spotted the nunnery’s ruins from a boat in 1999 and promptly bought the property. After a lengthy decade-long restoration, it opened as a luxury hotel for the 2012 season. + +Much of the convent’s original 17th century architecture remains intact, thanks to Sharma’s careful conservation. Inside, reconfigured spaces pay thoughtful homage to the hotel’s heritage. Nuns' quarters are cleverly combined into 20 unique rooms, some with private terraces or multiple levels. One of the largest is located in what was the sisters’ refectory. All feature fine Italian linens, an array of tasteful period furniture, and deep soaking tubs made from fine Jerusalem stone, in addition to everyday contemporary amenities like minibars, television, and WiFi. Some of the rooms have vaulted ceilings, private alfresco dining areas, and seafront balconies; others have housed famous names like Prince Albert and Princess Charlene of Monaco. The guest rooms exude comfort, and every conceivable wish is quickly taken care of by the attentive staff. + +The tasteful restoration continues into Monastero Santa Rosa’s massive 750-foot state-of-the-art spa. Cavernous treatment rooms are carved from spaces once devoted to silent prayers, while the spa’s centerpiece, a vaulted tepidarium (warm relaxation room), is where the sisters made wine. There’s also a sauna, steam room, hydrotherapy pool, whirlpool footbaths, ice fountain and even programmable showers. + +The real star of Monastero Santa Rosa is the knockout coastal view, and there’s a countless number of magical places for taking it in. From the convent’s highest point, an airy sunset terrace, guests can gaze as far as the buzzy nearby towns of Amalfi or Positano. There’s the cascade of tiered, expertly manicured semi-tropical gardens, beset with ocean-facing sun loungers, daybeds and cabanas. There’s the breezy dining terrace of the hotel Ristorante, where guests can enjoy the scenery while tasting exclusively sourced extra virgin olive oil produced right in Conca dei Marini. The most breathtaking spot is floating within the curved cliff’s edge infinity pool while surveying the Gulf of Salerno some 660-feet below. + +After whiling away the hours on these sacred grounds, be sure to unburden your soul at the large antique wooden confessional resting in the main hallway. It cheekily invites guests “make a confession” by way of a written feedback for the hotel. + +Monastero Santa Rosa is located in the town of Conca dei Marini on Italy’s Amalfi Coast. Nightly room and suite rates range from $500 to $2,400. The resort will open for the 2015 season on 17 April. + +See more photos in the gallery, images by Tanveer Badal \ No newline at end of file diff --git a/tests/data/text/cricket.com.au1.txt b/tests/data/text/cricket.com.au1.txt new file mode 100644 index 00000000..8e6e776b --- /dev/null +++ b/tests/data/text/cricket.com.au1.txt @@ -0,0 +1,75 @@ +Heroics from Johnson, Harris not enough but Australia still regain Border-Gavaskar Trophy + +The series win that brings the Border-Gavaskar Trophy back to Australia after almost two years in India arrived courtesy of a mutually agreed draw rather than the emphatic statement the home team seemed primed to deliver. + +When rival captains Steve Smith and MS Dhoni decided no result was possible at 6.24pm this evening, Australia had a further four of their compulsory 15 final-hour overs to bowl and India was nowhere close to their notional victory target of 384 from a minimum 70 overs at 6-174. + +While the unassailable two-nil lead that Australia now takes into the last of the four Tests in Sydney next week was undoubtedly a factor in Smith opting to bat through today’s morning session and setting the improbable target, it raised more than a few questions due to its conservatism. + +The captain and his brains trust will doubtless argue there was nothing to gain by setting India a goal they might conceivably attain, regardless of the spectacle it might have yielded and the availability of an extra dozen overs or more might have made. + +And if the logic for delaying the declaration was to instil in India’s batsmen a sense of futility before the pursuit began, then the incident-laden first half hour of their second innings offered emphatic if ultimately premature vindication. + +With his second delivery Ryan Harris did for Shikhar Dhawan whose duck had him briefly being mentioned as a possible omission for next week’s final Commonwealth Bank Test in Sydney. + +That was until the man being mooted as his potential replacement – Lokesh Rahul – completed his debut Test with a second innings less meritorious than his forgettable first. + +On day three, the highly rated top-order batsman came in at No.6, lasted eight balls from which he was dropped once before holing out to an ugly slog sweep having scored three. + +Today, promoted to No.3 in place of out-of-favour and form Cheteshwar Pujara and endured just five deliveries for a single scored before an ambitious pull shot to a ball from Mitchell Johnson that was too full, too close and too fast landed in the hands of Shane Watson running back from slip. + +When India’s in-form opener Murali Vijay (11) was pinned lbw to a ball that the video review technology – being used as a novelty rather than a tool in this series – showed would have snuck past leg stump, the tourists were surging towards defeat at 3-19. + +And in light of the contribution the bottom half of India’s batting has managed thus far in the series, the end was realistically just one more breakthrough away. + +For almost two hours, the Australians searched for it only to fumble when it fell their way. + +Kohli should have been run out on four, his anxiety to get his innings underway failing to factor in Australia’s even stronger urge to restrain him as he pushed to mid-off and followed through for a single. + +It was only due to a badly-bruised right forearm that David Warner was fielding there rather than his customary spot in the cordon, and it was that same injury that invariably saved Kohli as the return Warner fired after a diving save with Kohli stranded mid-pitch failed to threaten the stumps. + +A more straightforward chance was offered by Rahane when he was on 22 and failed to get on top of a cut shot off Johnson that flew above head height to Chris Rogers at point who jumped, clutched and then jogged after the spill which meant his back was turned to the bowler’s displeasure. + +In the over prior to tea, Kohli’s impetuosity almost cost him again when he defied old-fashioned cricket wisdom and tried to steal an overthrow that Pujara clearly didn’t believe existed, and was this time spared by Nathan Lyon’s inability to gather the searing throw and break the stumps. + +But when the Australians finally secured the wicket they so desperately sought – the in-form, in-your-face Kohli – it came amid such anti-climactic circumstances that the home team couldn’t find it in themselves to give their pantomime villain a send-off. + +Perhaps they felt the Indian star was suffering enough, his shock at gently flicking the first ball after tea softly to an equally surprised Joe Burns at square leg reflected in the look he gave the pitch as if he had been done cold by one of those day five grubbers the MCG was once infamous for. + +However, the final key that was supposed to unlock India’s dysfunctional lower-order batting took a while to turn. + +And when it did, that turn came from an unlikely source. + +After shaking Pujara with a bouncer that slammed into the grille of his protective helmet and then another that soared over the batsman’s head, Johnson slipped in the sucker punch in the form of a ‘slow’ (125kph – gentle by his standards) orthodox spinner that would have done Derek Underwood proud. + +Delivered from wide with the bowler’s fingers dragged down the side of the ball to impart turn, it pitched on the perfect length to lure Pujara forward and turned sufficiently to beat the bat but not off stump. + +Four overs later Harris claimed the other prized scalp when Rahane aimed one of his few false strokes of this Test with a pull that hit high on the bat and looped to midwicket, leaving the Indian tail that had proved so hopelessly inadequate over the past year 15 overs to survive. + +But Dhoni and spinner Ravi Ashwin defied both recent history and the Australian attack to survive a final hour and eliminate the prospect of a series whitewash. + +The talking point at day’s beginning was at what stage of the morning Smith would shut down his second innings and let his bowlers loose on the tourists. + +As it transpired, it was so far into the afternoon the players were on their way to the lunch room beneath the MCC Members’ Stand when the flag was waved, the tourists handed an unappetising order of 384 runs from a minimum of 70 overs to keep the series afloat. + +The tardiness of Smith’s declaration had become topical to the point of obsession during a morning session that began 24 minutes early, lost 40 minutes to Melbourne’s traditional festive season drizzle and came within a few centimetres of delivering a highlight. + +That came when Shaun Marsh’s three-year wait to score a Test century on home soil was scuttled as he dived to complete his hundredth run, found short by Kohli (of course) who swooped, gathered and hit the stumps from close range as he sensed the Australian’s desperation for the milestone. + +A polarising figure due to the vast, unexplained fluctuations between his successes and failures, Marsh is unlikely to finish with a Bradman-esque average. + +But given how close he was to completing the run that distinguishes a very good innings from a great one he should perhaps be listed in the scorebook as ‘run out (Kohli) 99.94’. + +The mean-spiritedness of declaring on a batsman who has fought a long and public battle to forge an international career just as a definitive moment beckoned was one mitigating reason for Smith opting to bat India out of the series before declaring. + +The effect of the damp outfield on his bowlers’ capacity to grip the ball another. + +As was the prospect of India taking the second new-ball early in the day, thus raising the hope of a lift in the miserable scoring rate above the barely-two-runs-per-over the seemingly aimless session eventually yielded. + +And maybe even the tidal fluctuations of the nearby Yarra River. + +Whatever the rationale, Smith was so often and so closely scrutinised by television cameras as he sat in the dressing room casually tossing a ball from hand to hand he could have been excused for thinking he had become a Big Brother inmate. + +While the simple explanation – that in holding a two-nil series lead and having been bowled out twice in the first three Tests there was no conceivable reason to gift India the remotest chance of winning – was also the most credible, the morning’s tactics remained curious. + +Once the prospect of losing an early wicket and having the tailenders face a new ball had passed, why not up the tempo and grant yourself more than two sessions – albeit extended ones due to poor weather – to seal the series with a win rather than a damp draw? \ No newline at end of file diff --git a/tests/data/text/cricket.com.au2.txt b/tests/data/text/cricket.com.au2.txt new file mode 100644 index 00000000..b6bf9a86 --- /dev/null +++ b/tests/data/text/cricket.com.au2.txt @@ -0,0 +1 @@ +The Brisbane Heat's Ryan Duffield had his second wicket in as many balls when Glenn Maxwell left a ball swinging back into the stumps \ No newline at end of file diff --git a/tests/data/text/dailycaller.com1.txt b/tests/data/text/dailycaller.com1.txt new file mode 100644 index 00000000..0b5e2101 --- /dev/null +++ b/tests/data/text/dailycaller.com1.txt @@ -0,0 +1,25 @@ +I will not be shooting any Black Panthers this week because I am Kwanza-reform, and we are not that observant. Kwanzaa, celebrated exclusively by white liberals, is a fake holiday invented in 1966 by black radical/FBI stooge, Ron Karenga — aka Dr. Maulana Karenga, founder of United Slaves, the violent nationalist rival to the Black Panthers. In the annals of the American ’60s, Karenga was the Father Gapon, pawn of the czarist police. + +In what was ultimately a foolish gambit, during the madness of the ’60s, the FBI encouraged the most extreme black nationalist organizations in order to discredit and split the left. The more preposterous the group, the better. By that criterion, Karenga’s United Slaves was perfect. + +Despite modern perceptions that blend all the black activists of the ’60s, the Black Panthers did not hate whites. Although some of their most high-profile leaders were drug dealers and murderers, they did not seek armed revolution. + +Those were the precepts of Karenga’s United Slaves. The United Slaves were proto-fascists, walking around in dashikis, gunning down Black Panthers and adopting invented “African” names. + +And hasn’t that been a huge help to the black community? The black man who assassinated two New York City cops last week went by the name “Ismaaiyl Abdullah-Muhammad,” and the man who attempted to hatchet four NYPD officers to death in October had adopted the name “Zaim Farouq Abdul-Malik.” + +It’s as if David Duke invented a holiday called “Anglika,” which he based on the philosophy of “Mein Kampf” — and clueless public school teachers began celebrating the made-up, racist holiday. + +Whether Karenga was a willing FBI dupe, or just a dupe, remains unclear. + +In the category of the-gentleman-doth-protest-too-much, back in the ’70s, Karenga was quick to criticize Nigerian newspapers that claimed that certain American black radicals were CIA operatives. Karenga publicly denounced the idea, saying, “Africans must stop generalizing about the loyalties and motives of Afro-Americans, including the widespread suspicion of black Americans being CIA agents.” + +In a 1995 interview with Ethnic NewsWatch, Karenga matter-of-factly explained that the forces out to get O.J. Simpson for the “framed” murder of two whites included: “the FBI, the CIA, the State Department, Interpol, the Chicago Police Department” and so on. Karenga should know about FBI infiltration. (He further noted that the evidence against O.J. did not “eliminate unreasonable doubt” — an interesting standard of proof.) + +Now we know: The FBI fueled the bloody rivalry between the Panthers and United Slaves. In one barbarous outburst, Karenga’s United Slaves shot two Black Panthers to death on the UCLA campus: Al “Bunchy” Carter and John Huggins. Karenga himself served time, a useful stepping-stone for his current position as a black studies professor at California State University at Long Beach. + +Back to the esteemed Cal State professor: Karenga’s invented holiday is a nutty blend of schmaltzy ’60s rhetoric, black racism and Marxism. The seven principles of Kwanzaa are the very same seven principles of the Symbionese Liberation Army, another invention of the Worst Generation. + +In 1974, Patty Hearst, kidnap victim-cum-SLA revolutionary, posed next to the banner of her alleged captors, a seven-headed cobra. Each snakehead stood for one of the SLA’s revolutionary principles: Umoja, Kujichagulia, Ujima, Ujamaa, Nia, Kuumba and Imani. These are the exact same seven “principles” of Kwanzaa. And here’s something interesting: Kawaida, Kwanzaa and Kuumba are also the only three Kardashian sisters not to have their own shows on the E! Network. + +Kwanzaa praises collectivism in every possible area of life — economics, work, personality, even litter removal. (“Kuumba: Everyone should strive to improve the community and make it more beautiful.”) It takes a village to raise a police snitch. When Karenga was asked to distinguish Kawaida, the philosophy underlying Kwanzaa, from “classical Marxism,” he essentially said that, under Kawaida, we also hate whites. \ No newline at end of file diff --git a/tests/data/text/dailycaller.com2.txt b/tests/data/text/dailycaller.com2.txt new file mode 100644 index 00000000..f1d1bc9a --- /dev/null +++ b/tests/data/text/dailycaller.com2.txt @@ -0,0 +1,13 @@ +Dec 27, 2014; Louisville, KY, USA; Louisville Cardinals forward Montrezl Harrell (24) posts up against Kentucky Wildcats guard Aaron Harrison (2) during the first half at KFC Yum! Center. Mandatory Credit: Jamie Rhodes-USA TODAY Sports - RTR4JDW4 + +The Louisville Cardinals hosted the Kentucky Wildcats on Saturday for one of the 2014 season’s most highly anticipated college basketball games. + +Rick Pitino’s 4th ranked Cardinals have had a great start to the season, but they were still huge underdogs against John Calipari’s unbeaten powerhouse. (RELATED: Kentucky Basketball’s Christmas Card Must Be Pretty Intimidating For Opposing Teams) + +Louisville trailed by 4 points at halftime, but they couldn’t close the gap thanks to plays like this. + +Slow it down one time. + +In games like these, I usually cheer for the underdog, but that flop had me wanting Kentucky to win by fifty. (RELATED: Forget ISIS, China And Immigrants; Flopping Is Threatening To Destroy America) + +Like the rest of their games this season, the Wildcats eventually pulled away from their cross-state rival. UK won by a final score of 58-50. \ No newline at end of file diff --git a/tests/data/text/dailystar.co.uk1.txt b/tests/data/text/dailystar.co.uk1.txt new file mode 100644 index 00000000..0b765474 --- /dev/null +++ b/tests/data/text/dailystar.co.uk1.txt @@ -0,0 +1,57 @@ +The Scottish nurse arrived at the Royal Free Hospital in Hampstead, north London, for specialist treatment at the infectious diseases unit, surrounded by a convoy of ambulances and police cars. + +The woman returned to Scotland after working in the west African country and arrived on a British Airways flight at Glasgow airport at around 11.30pm on Sunday. + +The patient – who is understood to have been volunteering for Save The Children – was said to be displaying no symptoms on the plane and the risk to other patients is said to be "low". + +Public health officials are said to be looking to contact passengers who sat adjacent to the nurse as a precautionary approach. + +Meanwhile, another person is being tested in north west Scotland for the deadly virus but is said to be "low risk". + +Another patient, in Truro, Cornwall, has been placed in isolation and is also being tested for Ebola. + +The Scottish nurse was admitted to hospital early on Monday after feeling unwell. She is in a stable condition. + +She was accompanied to hospital by a convoy of ambulances and police cars as workers wore full protective suits. + +A tent has been set up around her bed so the infection can be contained while she is treated. + +The unit in London successfully treated British nurse WIlliam Pooley who was flown home from Sierra Leone in August after contracting the virus. + +Scotland's First Minister Nicola Sturgeon said: "Our first thoughts at this time must be with the patient diagnosed with Ebola and their friends and family. I wish them a speedy recovery. + +"Scotland has been preparing for this possibility from the beginning of the outbreak in West Africa and I am confident that we are well prepared. + +"We have the robust procedures in place to identify cases rapidly. Our health service also has the expertise and facilities to ensure that confirmed Ebola cases such as this are contained and isolated effectively minimising any potential spread of the disease. + +"Scotland’s NHS has proved it is well able to cope with infectious diseases in the past, such as swine flu, and I am confident we will be able to respond effectively again.” + +She has also spoken to Prime Minister David Cameron. + +Health Secretary Jeremy Hunt, who chaired an emergency Cobra meeting, vowed the Government was doing "absolutely everything" to keep the public safe. + +"We are reviewing our procedures and protocols for the other NHS workers who are working in Sierra Leone alongside colleagues from the Department for International Development and the Foreign Office," he said. + +"They are doing a very, very brave job, under very challenging circumstances. + +"We want to make absolutely sure that we are doing everything we can to keep them safe. + +"The clinical advice is that the risk is very low to other passengers. She wasn't exhibiting feverish symptoms. + +"The process has worked well because the moment she did exhibit those symptoms we were able to take her into isolation." + +A British Airways spokesman said: "We are working closely with the health authorities in England and Scotland and will offer assistance with any information they require. + +"Customers who flew from London Heathrow to Glasgow on BA1478 which departed at 2100 on Sunday December 28 and have concerns should contact the special number 08000 858531 set up by the Scottish Government. + +"The safety and security of our customers and crew is always our top priority and the risk to people on board that individual flight is extremely low." + +The current Ebola outbreak, which began in West Africa, is the largest epidemic of the virus in history. + +It began in Guinea in December 2013 and quickly spread to Liberia and Sierra Leone – the two worst-affected nations. + +The World Health Organisation (WHO) has reported a total of 19,980 suspected cases and 7,793 deaths. + +But the WHO believes these figures understate the true size of the deadly outbreak. + +Earlier this month Time magazine named those healthcare workers who have travelled to help with the crisis as their "Person of the Year". diff --git a/tests/data/text/dailystar.co.uk2.txt b/tests/data/text/dailystar.co.uk2.txt new file mode 100644 index 00000000..690016ea --- /dev/null +++ b/tests/data/text/dailystar.co.uk2.txt @@ -0,0 +1,11 @@ +Merry belated Christmas, 'tis the season to be jolly 'nd all that. And 'tis the season for all the dirty dawgs to get up to no good. + +The holidays come with many positives – families, no work, copious amounts of cheese (literally, eating it and the corny movies variety) – but they also come with a lot of naughtiness. When I say naughtiness, I don’t mean kids unwrapping their pressies early. I mean the parties where everyone is obliterated and mauling one another (with a wedding ring on their finger). Last week, I was sat in the local as an office party was going on beside me and my mates, just minding our own biz. Then a lad, about 25, sits down on the stool next to us before a rather trollied lass plonked her unsteady behind down on his knee. They were both slurring their words when we overheard the girl say, "Weeeell, you've got a girlfriend and I'm not single, but so what?!" Lovely. I especially loved her discretion, slap bang in the middle of the pub. She's basically got a wide-on over another guy while her other half was probably at home stuffing the turkey. Stats show that cheating increases at this time of year, instead of secret Santa we have the secret s***gers too. Jokes aside, I think it's disgusting. The classic excuse, 'it meant nothing'. Well, that's more of a f***ing insult! If someone is going to cheat, there must be some feelings or strong attraction, surely. Another lame backtrack is blaming dropping your chinos or lifting your skirt on being drunk. + +I don’t understand how anyone in their right mind gets THAT wasted that they don’t know what they're doing. Coming from someone who every time they go out they get sloshed, I still don’t wake up the next morning thinking, 'Oh my God, who have I slept with?' I've also never woken up after a night out thinking, 'S**t, did I s**g someone behind my partner's back last night?' I think if I ever did wake up after doing that, I'd give up alcohol altogether and reassess my morals. Alcohol is just used as an excuse, b*****ks do people only get these urges to cheat when under the influence. If you're going to cheat, you'll do it sober or drunk. Despite the trolls (who really should make getting out more their primary new year resolution) blasting me with 'You can't talk, homewrecker/s**g/blah blah' – I CAN talk. My views on cheating are extremely strong and always have been. Yeah, I've done my fair share of messing around and one-night stands, but I was never hurting anyone. When I was younger, it didn’t really bother me if I knew a guy had a girlfriend, if I didn’t know the girl, it didn’t effect me. That was my opinion at the time. + +Since getting older, my feelings have changed. I think if I gave my number out knowing that the lad has a chick, it'll bite me back on the a**e at some point in life. Although I've been the girl to cheat with men that are occupied, I'd never be able to do that myself. Having found cheap eyelashes, texts and lip-glosses in and around my ex’s things, it didn’t take a genius to work out what he’d been up to. Let's just say I've put that one to bed now. + +If I felt in any way sexual towards another guy, and I don’t mean looking at a fittie in a bar, it's time to leave. Wanting to bed another man clearly means my current partner is not the one I want to have sex with for the rest of my life – SO WHAT IS THE POINT IN BEING THERE?! One thing that actually grates on me, is when a couple 'move on' from cheating: how the f**k do you move on? I don’t care if it's texting, kissing, s***ging whatever, it's all still cheating. A text usually leads to sex, otherwise why else do people do this? You don’t start the ball rolling for a hot chocolate in Starbucks. + +I pity girls that stick with guys that mess around, I mean, I can never ever imagine getting back into bed with a bloke who had his penis in another girl. What do you talk about? 'So, erm, did she have a Brazilian or a Hollywood?' 'Does she spit or swallow?' 'Who went on top?' It's just bizarre, get a grip! I couldn’t even speak to a guy who had cheated on me, who had stripped me of dignity, trampled on my respect and took me for a fool. There is no moving on from there. With so many guys and girls in the world who are loyal, why waste your time on one who can't even keep an eyeball on you without the other wandering. We’re approaching a new year, thank God I'm approaching it single, unlike some of the desperadoes I know who will celebrate being a mug through to 2016 and onwards, not me thanks. Have a fab new year and keep it in your pants if you're otherwise engaged. Or if you're the victim of someone doing the dirty, you're worth more than that. Time to skedaddle. \ No newline at end of file diff --git a/tests/data/text/dallasnews.com1.txt b/tests/data/text/dallasnews.com1.txt new file mode 100644 index 00000000..b8215d3c --- /dev/null +++ b/tests/data/text/dallasnews.com1.txt @@ -0,0 +1,3 @@ +It seems we can’t find what you’re looking for. Perhaps searching, or one of the links below, can help. + +Try looking in the monthly archives. :) \ No newline at end of file diff --git a/tests/data/text/dallasnews.com2.txt b/tests/data/text/dallasnews.com2.txt new file mode 100644 index 00000000..a4b1a393 --- /dev/null +++ b/tests/data/text/dallasnews.com2.txt @@ -0,0 +1,5 @@ +There is a caste system already in football with potential performance measured up front by the Wonderlic test. It ranges from quarterbacks in the 30 range (Romo 37 and Stafford 38 which is right up there with engineers and chemists) to single digits in other positions which rank below janitors at 14. There was even a study which indicated that the higher the score was for some positions, the less desirable they may be for draft purposes. + +Football is just one of the ways we are no longer the country we once were. We just lost an orthopedic surgeon named Jimmy Swink who was a TCU All American in the days getting a college scholarship meant getting an education a lot of other people wouldn’t get. College football has since evolved into an NFL farm team model where attending class is an unfortunate, mind numbing byproduct you and your mentor just have to put up with to stay on the team. + +We have achieved Rome’s greatness with the military machine, highways, architecture, government, and of course the bread and circuses. TANF, EBT, Section 8, Free Cell Phones, Obamacare, et al gives us the Bread and Football and MMA the gladiators. You don’t have to read up on Gibbon to see the parallels between the Decline phase of the first superpower and the process currently underway within the former superpower. \ No newline at end of file diff --git a/tests/data/text/details.com1.txt b/tests/data/text/details.com1.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/text/details.com2.txt b/tests/data/text/details.com2.txt new file mode 100644 index 00000000..18fefce6 --- /dev/null +++ b/tests/data/text/details.com2.txt @@ -0,0 +1 @@ +This season, designers updated a classic color by way of modern silhouettes both tailored and casual. The verdict? The most effortless way to transition into fall is all in the shading. \ No newline at end of file diff --git a/tests/data/text/elle.com1.txt b/tests/data/text/elle.com1.txt new file mode 100644 index 00000000..707bf5ea --- /dev/null +++ b/tests/data/text/elle.com1.txt @@ -0,0 +1 @@ +From our editors to your inbox. Sign up for our newsletter today. \ No newline at end of file diff --git a/tests/data/text/elle.com2.txt b/tests/data/text/elle.com2.txt new file mode 100644 index 00000000..03c563f4 --- /dev/null +++ b/tests/data/text/elle.com2.txt @@ -0,0 +1,65 @@ +Ah, the holidays: A time for gift giving, family gathering, and passive aggressive rage flames. Here are some well-meaning but totally backhanded digs we heard over the holidays. Add your own in the comments. It feels good to vent. + +From: Grandma + +What she said: "Wouldn't it be nice to be in a relationship so you don't have to have roommates anymore?" + +What we said: "I like my apartment; did Mom show you the new bar cart I found?" + +What we wanted to say: "Literally the only thing harder than finding an affordable apartment in New York City is meeting a guy good enough to make you move out of it. If you know of a site that lists cute 1-bedrooms stocked with single guys, send it the hell over." + +From Mom: + +What she said: "It would be so nice to have some babies around here soon to open presents!" + +What we said: HAHAHAHAHAHA. + +What we wanted to say: HAHAHAHAHAHA. + +From Aunt: + +What she said: "Have you heard of this thing called Tinder?" + +What we said: "Um, yes, kind of." + +What we wanted to say: "If I hadn't heard of Tinder, I wouldn't have touched a male in over a year. So. Yes, yes I have." + +From: Mother in Law: + +What she said: "Did you REALLY cook all this yourself?" + +What we said:" I sure did, hope you enjoy!" + +What we wanted to say: "Just because I’m not giving up my career to cook three meals a day for your precious angel of a son doesn’t mean I can’t figure out which end of the turkey is which, lady." + +From: Mom + +What she said: "Maybe it’s time to see a therapist." + +What we said: "Yeah, maybe that could help." + +What we wanted to say: "I am already seeing a therapist to try and resolve my relationship issues before I’m doomed to die alone, THANKS!" + +From: Mom: + +What she said: "Did you go outside at all today?" + +What we said: "Oh, haha, I guess I haven't!" + +What we wanted to say: *crawls back under covers, fighting back shame waves* + +From: Brother + +What he said:"If she can wear that, then I don’t have to get dressed." + +What we said: "Then don’t get dressed." + +What we wanted to say: "Then don’t get dressed." + +From a STRANGER at a Christmas party: + +What she said: "I lived in New York and would have worked forever, like you, if my husband hadn’t moved us to Virginia. But really, you can’t work forever and wait to have kids. You should do Match.com. My sister did Match.com and she’s 42 and just got married and had a baby." + +What we said: *Awkward polite laughter* + +What we wanted to say: "Save some of that white wine for the rest of us " \ No newline at end of file diff --git a/tests/data/text/flavorwire.com1.txt b/tests/data/text/flavorwire.com1.txt new file mode 100644 index 00000000..937ba521 --- /dev/null +++ b/tests/data/text/flavorwire.com1.txt @@ -0,0 +1,27 @@ +Presenting; a mix of the morally execrable and the culturally annoying moments in 2014 that we sort wish had never happened, to be honest, although we dutifully thought and wrote about them (and to be even more honest, these ten moments are merely the tip-top of the Christmas tree of head-smacking moments from 2014.) + +1. Celebrity nude photo leaks. It was a bummer when actresses were targeted by hackers who released their nude photos. And when everyone slut-shamed them as a result. Sidenote: same went for celebrity plastic surgery gawking. + +2. Donald Sterling-gate. One of many examples of 2014 scandals that made us aware of how awfully racist people in power were, but this one was just the most epically depressing on a human level. + +3. The Newsroom‘s Campus Rape Plotline and Aaron’s Sorkin’s response. Just no, Aaron Sorkin. + +4. The Interview hacking brouhaha. Censorship is terrible and free speech is important, but why did we have to this discussion about a sub-par Rogen-Franco joint? + +5. Igloo Australia’s Twitter racism. While pop star feuds, beefs, and rivalries are usually enjoyable and beneficial for everyone involved (I kid), Iggy Azalea’s ignorance on matters of race only reinforced her clueless pop persona. + +6. Camille Cosby’s statement and Woody Allen’s letter to the New York Times. Alleged sexual predators and their wives: given a platform for victim-blaming since the beginning of modern media. + +7. “Rude.” You are genuinely terrible, “Rude,” we declared this year. Also, you are still stuck in our heads and will inevitably be heard on New Year’s Eve. + +8. Literary genre wars that weren’t. Did we really have to spend another year scolding each other for our reading choices? Even scolding the littlest readers? + +9. Accusing Lena Dunham of a crime for her writing about of her sister on the left and right and trying to discredit her own sexual assault story on the right. Rape culture is serious, and the freedom some (white) artists have to be messy while other artists are constrained by social prejudice informs a lot of the understandable ire directed at Dunham. But none of those factors, nor anything Dunham has written, indicated she had committed a crime, nor that the assault she wrote about herself hadn’t taken place — and watching feminists and right-wingers join in on this one broke our hearts a little bit. + +10. Normcore, Basic, and Health Goth. Stop trying to make fetch happen, bogus trend-pieces. + +11. Ariel Pink. Was his misogyny a joke? A nasty slice of his real personality? Trolling because he realized he got more attention for being a jerk than for his music? To be real, much of this whole conversation was unnecessary. + +13. The disrespectful, racist right wing media circus around Ferguson and the death of Michael Brown. + +Bonus: Sarah Palin’s Christmas Pie-baking instruction and garbled speech special that’s eerily reminiscent of SNL’s “Martha Stewart’s Topless Christmas.” Just kidding, we’re so glad this happened! Never leave us, ex-governor Palin. \ No newline at end of file diff --git a/tests/data/text/flavorwire.com2.txt b/tests/data/text/flavorwire.com2.txt new file mode 100644 index 00000000..a5b4436d --- /dev/null +++ b/tests/data/text/flavorwire.com2.txt @@ -0,0 +1,23 @@ +If you’re inclined to believe #slatepitches, then 2014 was the Year Of Outrage. The Internet’s favorite shrine to contrarianism published an interactive calendar earlier this month wherein one can track, day by day, the things about which we (“we” being liberal American adults on the Internet, basically) were outraged this year. I’m not so sure this year was any different to any other, though: the public has always been fond of being righteously outraged, and for the last few years, at least, the Internet has felt like (and been characterized as) a giant outrage machine. But 2014 did feel like a landmark in one respect: it was the year that the outrage machine proved its power to chew up and spit out people IRL as well as on Twitter or Tumblr. It was a year in which the precipitous fall taken by Bill Cosby, in particular, proved that it’s much, much harder for stars to bury unflattering narratives these days. But the ever-growing power of the angry mob also has pretty terrifying implications if you take an old-fashioned view of what constitutes justice. + +There have certainly been plenty of hand-wringing thinkpieces this year lamenting the rise of rage culture (and at least one about how the 300 Sandwiches kerfuffle may have saved us from invading Syria). But notably, two of the biggest stories of 2014 demonstrate the flip-side of that culture: instances where the Internet’s unparalleled ability to disseminate information, and to do so relentlessly, has shone light into hitherto dark places. I’m talking about the fall of Bill Cosby and the near-fall of R. Kelly. Both cases involved allegations that have been on the public record for years, but had nevertheless disappeared from public attention. (And, it should be noted, both involved black men: the sordid history of white celebrities tends to remain buried far more readily than that of men like Kelly and Cosby.) + +In both cases, you can trace the story’s reemergence back to a single moment. First there was Jessica Hopper’s interview with Jim DeRogatis in the Village Voice, right at the end of last year, which brought to light (again) the details of the multiple rape accusations against Kelly. Not long after, DeRogatis himself penned a piece entitled, “Why Are People Finally Paying Attention to R. Kelly’s Many Crimes?” DeRogatis’ piece was interesting and perceptive, but the answer could have been summarized in three words: the outrage machine. + +It was the same with Cosby: again, all it took was one person to say, “Wait, look at this.” In this case, it was Hannibal Buress flat-out calling Cosby a rapist on stage. The next morning, you could feel the entire Internet rubbing its eyes and saying, “Oh yeah, so what was the deal with Cosby?” As it turned out, the deal with Cosby was just as awful as the Kelly accusations: decades’ worth of women telling stories of having been drugged and assaulted. + +Not long after Buress’ performance, the Cosby thinkpieces started to appear. Then one of Cosby’s alleged victims, Barbara Bowman, wrote an op-ed for the Washington Post. And then… well, you can choose your own metaphor: the floodgates opening, the avalanche building, a shower turning into hurricane. However you want to put it, in the course of about three weeks, the attention given to the Cosby accusations increased exponentially, to the point that it could no longer be ignored by anyone (except Cosby himself, who’s still defiantly sticking to a strategy of stonewalling that simply no longer works). The fallout was dramatic: a canceled NBC project, a shelved Netflix special, pulled Cosby Show re-runs all over the place. This wasn’t a case of people online saying nasty things; this was quantifiable and significant damage to whatever’s left of Cosby’s career. + +Both with Cosby and with Kelly, we had public interest being reignited in accusations that had been actively suppressed or allowed to just fade away. In the past, it was relatively easy to bury a story: gaslight and discredit the accuser, cozy up to the press, call in some favors. There were only so many ways that news could get out. Once those channels were closed, a story would live on, at best, as a rumor, discussed in bars and over dinner, always laden with an air of doubt and hearsay. + +In 2014, it’s almost impossible for that to happen once the story gains any sort of momentum. Thanks in large part to social media, there’s an infinite number of leaks to plug. No doubt some stories are still suppressed before they ever get reported, but once there’s anything on the public record, it just takes one person to notice it and tweet it… and the story’s gone, beyond anyone’s control. + +Even if the outrage machine stuck exclusively to eviscerating celebrities who have been burying bad press for decades, its power to destroy would be kind of terrifying. But, of course, it doesn’t. The case that springs to mind immediately is that of the musician Conor Oberst, accused of rape by one Joanie Faircloth in a series of comments on an xoJane article. The pattern was the same: the original comments appeared, they sat idle for a couple of days, not garnering any attention beyond their original forum… then someone reported them, at which point they started doing the rounds on Twitter and Tumblr. And suddenly they were everywhere. + +It’s no coincidence, I’m sure, that all three cases — Kelly, Cosby, and Oberst — involved sexual assault. As we all know, it’s a crime that’s notoriously hard to prove and to prosecute, and even more so when years have passed since the alleged incident. In the cases of Kelly and Cosby, it appears from sheer weight of evidence that the Internet’s collective guilty verdict is most likely justified. The case of Oberst was quite a bit different. + +As I wrote at the time, the simple fact was that no one knew what had happened except Oberst and his accuser. As so often happens with rape, it was a game of he-said, she-said, except in this case neither he nor she was saying anything — Oberst released a brief statement denying the accusations, and Faircloth went to ground. There was no information beyond a couple of deleted comments made by a woman about which the world knew next to nothing. Of course, because the Internet abhors a vacuum of information, this only served to heighten the rage and speculation. Commentators pronounced Oberst guilty or innocent without the slightest hint of a doubt, on the basis of absolutely nothing beyond personal conviction and confirmation bias. + +As it transpired, Faircloth withdrew the allegations after Oberst threatened to sue her. But no one will come out of this story unscathed: there will always be people out there who now believe, no matter what, that Conor Oberst is a rapist. Faircloth, meanwhile, has her credibility left in shreds — there’ll be people to whom she’ll always be the crazy girl who made the false accusation. Either way, the Internet feeding frenzy has done her no favors: if her accusation was true, it seems almost impossible for her to ever pursue it, and if it was false, she most likely needs serious help. Having her story plastered all over the Internet (in violation of her express wishes) did her no more good than it did Oberst. + +None of this bothered the Internet judge/jury/executioners, though, who had already happily moved on. The Internet being the Internet, though, what’s forgotten isn’t necessarily gone. The Frisky’s hugely irresponsible “Why I Believe Conor Oberst’s Anonymous Rape Accuser” essay, for instance, is still online, prefaced only by a one-sentence “update” that’s essentially the journalistic equivalent of ¯\_(ツ)_/¯. In cyberspace, accusations never die: they just sit there, somewhere on Google, waiting for people to dredge them up again. It’s for this reason that there have been recent cases in the EU regarding a right to be forgotten. \ No newline at end of file diff --git a/tests/data/text/fool.com1.txt b/tests/data/text/fool.com1.txt new file mode 100644 index 00000000..80e4ac35 --- /dev/null +++ b/tests/data/text/fool.com1.txt @@ -0,0 +1,55 @@ +Need a reason to invest in stocks? How about the beginning of a new year. To help you find solid stock ideas we asked Fool.com contributors covering technology and consumer goods stocks to talk about top stocks for 2015. Read on to see what they had to say about Qualcomm (NASDAQ: QCOM ) , Facebook (NASDAQ: FB ) , SeaWorld Entertainment (NYSE: SEAS ) , WhiteWave Foods (NYSE: WWAV ) , Google (NASDAQ: GOOG ) (NASDAQ: GOOGL ) , Taiwan Semiconductor (NYSE: TSM ) , and Apple (NASDAQ: AAPL ) . + +Ashraf Eassa (Qualcomm): It's hard not to be pleased with the performances of technology and, in particular, semiconductor stocks in 2014. The Philadelphia Semiconductor Index is up over 28% year-to-date, handily crushing the S&P 500 and the Nasdaq, up 10.12% and 12.73%, respectively. However, one high-quality chip company that has underperformed pretty significantly during 2014 -- but one that I believe is set to do much better in 2015 -- is Qualcomm. + +First, Qualcomm's execution in developing and delivering a compelling range of mobile applications processor offerings looks unmatched. For example, the company revealed on Dec. 11 that it would be upgrading the baseband on its upcoming high-end Snapdragon 810 processor to offer 50% greater download speeds than had been previously announced. This further extends the company's leadership position in cellular baseband technology. + +It's this kind of execution in its chip business that not only keeps it ahead of major competitors like MediaTek, but also makes it very difficult for mobile device vendors to successfully develop their own in-house chip solutions in a bid to cut Qualcomm out. + +Further, Qualcomm's technology licensing business, which collects royalties on most 3G/4G devices sold, is extremely profitable and should continue to grow with overall smartphone growth. Now, it's well-known that Qualcomm is having issues collecting on royalties from some Chinese handset vendors (leading to pessimism around the business), but I think Qualcomm will be able to solve its issues there, as it has done in the past. + +All told, Qualcomm stock is cheap at just 16.36 times trailing-12-month earnings, it's a high-quality company, but the stock has underperformed during 2014. Qualcomm the company is a winner, and I think that during 2015, Qualcomm the stock will be, too. + +Andrés Cardenal (Google): Information is power and Google's mission statement, "to organize the world's information and make it universally accessible and useful" says a lot about the company and the role it plays in times of chaotically abundant information. + +Google is the undisputed king in online search; the company has a bigger market share than all its competitors combined. In addition, Google has built a massive portfolio of services and applications, including enormously valuable assets like Gmail, YouTube, and Chrome, to name a few remarkable examples. More than 80% of smartphones around the planet are powered by Android, so Google is in a position of strength to continue thriving under the mobile paradigm. + +The company generates tons of cash flows from its leadership position in online advertising, and management is not shy at all when it comes to investing that money in the search for breakthrough innovations. From self-driving cars to biotechnology solutions to fighting human aging and associated diseases, Google has plenty of exciting projects with disruptive potential in its pipeline. + +Investors are getting concerned about slowing revenue growth and rising expenses lately, and this may provide a buying opportunity in the online search giant. Google trades at a forward P/E ratio near 17.5, roughly in line with the S&P 500 Index. However, even during a "disappointing" third quarter, Google delivered a big increase of 20% in revenues, a level of performance which most companies in the index can only envy. Google has a lot of things going right and I think Google is a top stock to consider buying for 2015. + +Tamara Walsh (WhiteWave Foods): From smart acquisitions to promising opportunities in oversees markets such as China, WhiteWave Foods is one of my favorite stock picks heading into the new year. The packaged food and beverage company has enjoyed a nice run this year with the stock up more than 47% year-to-date. However, there should be plenty of growth ahead thanks to WhiteWave's partnership with Mengniu Dairy, one of China's largest dairy companies. + +As part of this joint venture, WhiteWave purchased a production facility where it plans to begin manufacturing its products for the Chinese market in the coming months. WhiteWave Foods owns a 49% stake in the deal, which will enable the company to sell its brands in China, one of the world's largest consumer markets with over 1.3 billion consumers and a rapidly growing middle class. Market-leading brands including Silk soy milk and almond milk, Land-o-Lakes butter, and International Delight coffee creamers, have already helped WhiteWave Foods make a name for itself in North America and Europe. The company celebrated a record third quarter recently, with net sales climbing 35% to $857 million in the period. I expect this momentum to carry over into the new year, and for the stock to continue to gain speed in the year ahead as the company expands into new markets and product categories. + +Rick Munarriz (SeaWorld Entertainment): I'm going to go full contrarian with a stock that everybody seems to hate. SeaWorld is in a bad spot these days. Activists have succeeded in keeping guests away from its marine life theme parks given the negative publicity about killer whales in captivity. Attendance across its 11 parks fell 4.1% in 2013 and is off by another 4.7% through the first nine months of 2014. This is the only theme park or regional amusement park operator that's experiencing lower turnstile clicks this year. The stock that went public at $27 early in 2013 is now all the way down to the mid-teens, and earlier this month it announced that it would have to postpone the dividend that was supposed to go out in December because it would violate its debt covenants. + +This all seems pretty grim, but changes are coming. SeaWorld's CEO is leaving in January, opening the door for an outsider who can help soften the battered brand. Along the way we have some favorable trends including an improving economy and lower gas prices that should deliver big boosts to the theme park industry in general. + +SeaWorld is in a bad spot, but it's also important to remember that just three of its parks are orca-housing SeaWorld attractions. The dividend should return in January, and guests will eventually follow as the chain either takes active steps to improve its image or fickle consumers move on to a new cause. With SeaWorld trading at a valuation discount to its peers there's plenty of upside in 2015 even if the market doesn't comply. The climate is ripe for SeaWorld to make a big splash in the year ahead. + +Dylan Lewis (Apple): It's far from a sexy pick, but most who have bet against Apple the past five years have come to regret it and I don't see 2015 being any different. + +Apple's brand cachet and customer loyalty hasn't wavered -- while the company's iPad line is faltering, Macs are selling well. In October the company revealed the segment posted its highest quarterly market share since 1995. + +Apple's brand prestige is even more valuable in the increasingly competitive smartphone market as Asian OEM's Xiaomi, Lenovo, and Huawei continue to produce low-cost devices. Skeptics need to look no further than Samsung's shrinking smartphone market share to appreciate the moat Apple enjoys due to its status, quality, and exclusive iOS and Mac operating systems. + +The recent December sell-off gives investors an even more attractive entry point, the company still trades at a TTM P/E of 17 and a forward P/E of 14. + +Even in the neighborhood of some of the more conservative analyst estimates of 5 million to 10 million units, the Apple Watch could provide a 1%-3% lift on projected revenue for CY 2015. The fledgling Apple Pay now supports credit cards that comprise 90% of the U.S. credit card purchase volume and could prove to be an even bigger catalyst than the new line of wearables. With each e-commerce security breach (it seems like there's one almost every other week), Apple Pay's tokenization system becomes increasingly appealing to consumers seeking security. + +Stability with its hardware stalwarts, new growth opportunities, a decent dividend yield (1.7%) and an average of $11 billion in stock repurchases each quarter for the past year and a half – there are simply too many reasons to ignore the Mac maker in 2015. + +Tim Brugger (Facebook): With its stock price up 38% so far this year, it may seem counterintuitive to include Facebook on a list of stocks to buy in 2015. However, there are a laundry list of revenue opportunities at Facebook's fingertips heading into the new year, and it appears a couple in particular are about ready to pay off. + +Though Twitter is loath to admit the importance of the milestone, the news that Instagram recently topped 300 million monthly active users (MAUs) is significant, to say the least. Perhaps most impressive is how quickly Instagram's MAUs grew. The number was hovering around 200 million users just nine months ago. No wonder Twitter's envious. + +Facebook COO Sheryl Sandberg made waves earlier this year when she said that there was no rush to monetize Instagram in any meaningful way, nor incorporate video spots as an advertising medium. Instead, Sandberg and CEO Mark Zuckerberg wanted to grow Instagram's user base, ensure a positive user experience, and test the video ad waters -- at a whopping cost of $1 million a day -- before making them available to its marketing partners. The MAU growth of Instagram is certainly there, and with the advent of video ads on both Facebook and Instagram, 2015 should be yet another banner year. + +Sean O’Reilly (Taiwan Semiconductor): Technology can be a tough business to be in for investors. The relentless competition and constant need to innovate frequently make long-term shareholder gains elusive. However, Taiwan Semiconductor not only dominates its market but happens to be leveraged to an increasingly important technological trend making it my top stock pick for 2015. + +Taiwan Semiconductor is the world’s largest fabricator of silicon chips. The company pioneered the dedicated semiconductor foundry model and operates primarily by partnering with fabless customers that don’t have the scale and operating expertise that Taiwan Semiconductor possesses. Cost advantage and scale are the name of the game and these happen to be things that Taiwan Semiconductor has in spades. These advantages will become all the more apparent as the world’s need for semiconductor chips grows exponentially in the coming years. + +As the world becomes more and more connected (a trend called the “Internet of Things”), Taiwan Semiconductor stands to benefit in a big way. Technology research organization Gartner estimates that 4.9 billion connected “things” will be in use in 2015, up from 3.75 billion in 2014. Gartner’s estimate for 2020? Try 25 billion connected devices. Investors have two ways to participate in this increasingly connected world: Focus on those that produce the connected devices, or the companies that make connecting these devices to the internet possible, like Taiwan Semiconductor. + +There’s a lot to like on Taiwan Semi’s balance sheet and income statement as well. The company trades for just under 14 times this year’s estimated EPS according to S&P Capital IQ estimates (high for a semiconductor fabricator but more than fair given its dominant industry position) and has a pristine balance sheet with very little debt. Add in the company’s exceptional return on equity, which has averaged 23.62% over the last five years and you get a company that should be on every Foolish investor's holiday wish list. \ No newline at end of file diff --git a/tests/data/text/fool.com2.txt b/tests/data/text/fool.com2.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/text/foxbusiness.com1.txt b/tests/data/text/foxbusiness.com1.txt new file mode 100644 index 00000000..b4b22f43 --- /dev/null +++ b/tests/data/text/foxbusiness.com1.txt @@ -0,0 +1,57 @@ +No matter how much traffic your site gets, what really matters is conversions. Convert neutral traffic – people who haven’t decided whether to buy or not – by investing in analyzing and incentivizing them to buy. Increasing conversions is more financially beneficial than driving more traffic. Instead of accepting your current conversion rate, focus on increases of 10% or greater using the methods below. + +Your site must grab visitors attention immediately, then lead them from where they land to checkout. Is it obvious how landing on your site will benefit your visitors? If they can’t immediately tell they will leave. Use analytics to check bounce rates and improve any pages that show high exit rates. + +Offering related products on product pages to sell more than one product is a common strategy known as cross-selling. Upselling is offering more expensive related products to increase the amount spent. + +New SaaS provider Fanplayr.com helps to make this process seamless for businesses. Their service analyzes your existing traffic, assessing the behavior and profiles of each visitor in real-time and automatically upsells or offer specifically tailored deals. This is executed based on defined rules and customized offers and incentives set by you for your target customers. This customized approached helps you to increase your average order value and conversions. + +Fanplayr analyzes traffic you drive so you can target different offers to segments of visitors based on: + + + +New visitors who place items with high profit margins in their carts can be offered a higher dollar discount. Reward the loyalty of your repeat buyers with percent off their current purchase. Be sure to let them know they received the discount because they have purchased from you before. + +Just as some sites offer free shipping for orders over $50, $75 or $100, increase visitor’s total spend by offering a higher discount based on the dollar amount they have in their carts or how much they have spent with you previously. Read more in Proven Online Cross-Selling and Upselling Techniques. + +Make sure your customers can find what they want quickly. Test your site search to ensure that all your products show up as expected and just as importantly confirm that irrelevant products are not appearing in searches. Consumers will not trudge through page after page of search results. + +Standard site search often returns all results for any word in the search. What this means is that if someone searches for ‘pink widgets’ the search will return every pink item on the site. This is unacceptable as not all users will realize the problem and change their search phrases. There are many easily installed third party site search solutions to resolve these issues. + +Traffic from AdWords can be offered multiple types of incentives which can be measured to see which increases sales more. For example, test whether your visitors would rather have: + +Free shipping if they spend $50 or more + + 10% off sales of $100 or more + + $20 off a $200 purchase + +Fanplayr not only analyzes, segments, and allows you to add incentives and offers; it integrates A/B testing into ecommerce software. Changes can be made to text campaigns on the fly. + +Conversion testing can increase conversion rates up to 25% ~ Up to 50% higher average order values ~ Targeted offers can generate up to 20% margin improvement. + +Solutions providers can use analysis and segmenting to decrease the time it takes to progress through their sales funnels. For example, Neil Patel at QuickSprout suggested in 7 Simple A/B Tests to Increase Conversions 10% or More to test: + +Improve your sales funnel by comparing the actions of visitors who have converted with those who haven’t. Check page views to determine whether visiting particular pages increases conversions. If it does, change site navigation to add that page to the sales funnel. + +Provide different incentives to repeat visitors or visitors who have downloaded specific white papers or watched specific videos. Analysis can provide your sales team insights into what a particular lead is most interested in. + +Unless you’re a household name, your site must convey how trustworthy your site is. Adding photos of the business location and people behind the brand to your about page can increase conversions. Your about page is usually visited just before a person checks out to make sure you are a “real” business. Optimize this page. Make it convey why they should buy from you. + +Online buyers are more sophisticated today. Many know to verify that your site is secure by looking for the https:// in your url and a Secure Shopping 256 bit icon. Display Trust Signals such as Better Business Bureau, TrustE, Norton Secured, TrustWave, Google Trusted Store. + +Real testimonials are another way to increase trust. If you sell a service, offer it to the leading influencers in your industry and ask for their endorsement. Use their photo, logo and text or even a video. Video testimonials are doable for any size business. + +Do your fellow consumers a favor and review the products and services you buy. Even though testimonials and reviews can be manipulated, they are still valuable during the decision-making process. Savvy buyers know to look at a reviewers other reviews or to search for the testimonial giver online to determine whether they can be trusted. + +Enable the ability for your buyers to review your products and services. What they write is feedback for you and your customers. No product or service is a perfect fit for everyone. Details left by buyers ensure buyers choose wisely. + +People don’t buy unless they’re confident they are making the right decision. Even if you have excellent product descriptions and multiple photos, they may still have a question. At a minimum be sure you have a contact page that is easy to find. Test it to make sure it works. + +Better yet, add live chat to your site. Businesses that provide live interactions at least during business hours save money and experience increased sales and conversions. + +“According to an article on Sitepoint, the top ten reasons buyers abandon their online shopping sessions are often related to confusion and complications at checkout. Confused customers may have a question that they want answered in real time. “ + +This is especially true if you are a very small brand. People want to know there is a real person they can contact behind the site. Answering them in real time increases confidence they can reach you should their package not arrive or they have problems with their purchase. + +Gail Gardner is the Small Business Marketing Strategist who founded and provides consulting at GrowMap.com. She also answers questions in live chat as Community Manager at SocialImplications.com. \ No newline at end of file diff --git a/tests/data/text/foxbusiness.com2.txt b/tests/data/text/foxbusiness.com2.txt new file mode 100644 index 00000000..e54cd313 --- /dev/null +++ b/tests/data/text/foxbusiness.com2.txt @@ -0,0 +1,23 @@ +If a Scrooge client is scaring the dickens out of you, don’t let it spoil your holiday season. A visit from the Spirit of Business Past, the Spirit of Business Present, and the Spirit of Business Future will make even the sourest customer as sweet as Tiny Tim. + +Scrooge clients have short memories. Pay them a holiday visit and review all of the ways you have helped them save money, become more efficient, increase their sales, avert a crisis, and whatever else had a meaningful impact. + +It takes more than a casual visit. The more visual and vivid your presentation is, the greater its effect — recall how the cold heart of Ebenezer Scrooge began to thaw when he saw and felt and heard his former self. Do you think the Spirit of Christmas Past would have elicited the same response by showing Ebenezer a PowerPoint crammed with wordy bullet points? + +The holiday season is a time of joy — do you remember what a merry old soul the Spirit of Christmas Present was? A small thing such as taking your Scrooge client to lunch or meeting him for an after-work cocktail can change everything. Simply getting your Scrooge away from that damp, cold, and dreary office will make his mood more festive and his business more secure. + +Along with a dollop of holiday cheer, offer your Scrooge a business-present present: a gift, a December discount, or extra rewards points — something that answers the question, “What have you done for me lately?” Nothing makes a Scrooge client more Scrooge-like than the feeling of being taken for granted. + +The outlook for a Scrooge client is grim indeed; you cannot help but feel that it’s only a matter of time before his business is buried in the graveyard of lost opportunities. But as the Spirit of Christmas Future revealed to Ebenezer, the future can be changed! + +Give your Scrooge scenario a hopeful future by introducing incentives that reward his future business. Annual rebates, volume discounts, and loyalty rewards programs make it easier — far easier — for your Scrooge to overlook the day-to-day hiccups that occur in any business relationship. Just as the goal of curing Tiny Tim gave Ebenezer a reason to live well, a goal of getting more “something” gives clients a reason to do business. + +Most businesses routinely do these activities, which require only a modest effort and investment. The plot twist here is to do them all in equal measure. + +Focusing on the past alone, or the present alone, or the future alone is not enough. If only one or two spirits had visited Ebenezer that Christmas Eve, the story would have been quite different; he would not have been redeemed. + +By demonstrating past value, present value, and future value, your Scrooge clients will be reclaimed. For you, and them, it will be a very merry holiday season indeed. + +“I am as light as a feather, I am as happy as an angel, I am as merry as a school-boy. I am as giddy as a drunken man. A merry Christmas to every-body! A happy New Year to all the world! Hallo here! Whoop! Hallo!” – Charles Dickens, A Christmas Carol + +Brad Shorr is Director of B2B Marketing for Straight North, an Internet marketing agency in the Chicago area. With in-house, freelance and agency experience, he writes frequently about content marketing, SEO, social media and small business strategy. \ No newline at end of file diff --git a/tests/data/text/foxnews.com1.txt b/tests/data/text/foxnews.com1.txt new file mode 100644 index 00000000..db75a005 --- /dev/null +++ b/tests/data/text/foxnews.com1.txt @@ -0,0 +1,45 @@ +In this image released by A24 Films, Jessica Chastain, left, and Oscar Isaac appear in a scene from "A Most Violent Year." Chastain was nominated for a Golden Globe for best supporting actress in a drama for her role in the film on Thursday, Dec. 11, 2014. The 72nd annual Golden Globe awards will air on NBC on Sunday, Jan. 11. (AP Photo/A24 Films, Atsushi Nishijima)AP2014 + +Oscar Isaac has had a pretty stellar year. It started off with a Golden Globe nomination for his portrayal of the title role in “Inside Llewyn Davis.” + +He just completed filming in the next highly-anticipated installment of the “Star Wars” franchise, and he finishes off the year with a masterful performance in J.C. Chandor’s “A Most Violent Year, ”out on Dec. 31 in limited release. + +Set in New York City in 1981, the 35-year-old Cuban-Guatemalan actor plays Abel Morales, a moral businessman in a very immoral time. He doesn't want to be a gangster and has to fight against himself, as well as everything around him, to make sure it never happens. + +It was this tension that attracted Isaac to the role, which allowed him to work alongside longtime friend, Jessica Chastain, who co-stars as his wife, Anna. + +“He doesn't want to be a gangster, and he has never wanted to be one. And he is afraid that if he starts down that path, that he will be dismissed as one – and also, possibly, that if he starts down that path, he will really like it,” Isaac told Fox News Latino recently during an interview that also included Chastain. “The tension that J.C. creates, it plays with the audience’s expectations. The audience gets a little bloodthirsty.” + +For this internal battle fighting against violence, Chandor asked his leading man to pick a time in Latin America that the character had lived through and escaped. + +“Scripts are vague on back-story, what you see on screen is what the actors get when they first (read it),” the director said. “Oscar’s character, I always believed, came to the U.S. sometime between 7 years old and 10 or 12 – in that window, so young enough that you are able to strip away your past and young enough that you really can become just an American in your adult life.” + +He said Isaac zeroed in on a period of civil unrest in Colombia after World War II that is known as “La Violencia” ("The Violence"). + +“I thought it was a cool opportunity to tie in what his character was then going to be facing, the equivalent that he would have come here in the late 1950s probably and seen this city and America climb, climb, climb through the '60s and then in the late '60s and into the '70s, see the city of New York fall into this life of crime and people leaving the city – falling back into decay.” + +Isaac said he chose that specific time because it would have been “really intense if (Abel) had left this incredibly violent situation to come to this country to escape the violence, and yet the violence follows him.” + +To further develop their characters' back-stories, Isaac and Chastian, 37, used their mutual Julliard School training to discuss every scene. + +“It’s pretty intense because the script itself is already quite dense, full of so many details, but also very mysterious 'cause there is not a lot of description of his past or even their past, how they got to where they are,” Isaac said. “We got together and went through every scene, every line and just talked about it. We started talking about possibilities of where this whole relationship started, when it started, how we met, when we decided to buy the business. Just to create the context so that when we started shooting, we had that bedrock.” + +Chastain said that the director "really left (the script) free for us to explore … because he doesn’t want to taint our natural instincts." + +Chandor attributed the two actors’ long off-screen friendship for making Abel and Anna’s onscreen relationship feel so real. + +“I’m not sure if I knew it at the time, but, looking back at it now, it was just amazing cause there’s this shared history. The fact that known each other almost as long as the characters had – it was just meant to be,” he said. + +“A Most Violent Year” hits select theaters on Dec. 31, followed by a wider release in January. + +Next year Isaac will also appear in two blockbuster films, though “Star Wars: Episode VII” could probably the most secretive. + +All known of Isaac’s character is that at the very least he pilots an X-wing fighter in the trailer. + +Although he said it was “wild” to see himself in the film’s teaser, Isaac stopped short of revealing anything about it. + +“I’ve signed away my organs,” he joked, adding that filming the iconic franchise was great. “J.J. Abrams is amazing.” + +Like us on Facebook + + Follow us on Twitter & Instagram \ No newline at end of file diff --git a/tests/data/text/foxnews.com2.txt b/tests/data/text/foxnews.com2.txt new file mode 100644 index 00000000..4fdf4e30 --- /dev/null +++ b/tests/data/text/foxnews.com2.txt @@ -0,0 +1,19 @@ +In a page taken from the most gruesome of books, a 17-year-old girl from Reynosa, Mexico lured an 8-months-pregnant woman to her home and killed her to try and steal her unborn baby. + +Nathaly Cartas Leon, 20, had “met” Guadalupe Salinas Hernández on Facebook and had gratefully accepted her offer to give the mom-to-be a few baby items she needed. They met at a mall and then went to Salinas’ home, where she said she had more stuff to give her. + +Once there, investigators said, Salinas beat Cartas to death with a blunt object and then proceeded to open her belly with a kitchen knife. The motive, police said: obsessive love. According to authorities, Salinas had an abortion back in June and had not told her boyfriend, who still believed she was pregnant and was looking forward to the birth. + +It is not clear whether Salinas’ abortion was intentional or not. + +After the gruesome killing and removal of the newborn, the teen hurried to the hospital and tried to save him, claiming the child was stillborn at home. But the fetus had stopped breathing when her mother died and could not be saved. + +It didn’t take much for the doctors at the hospital to realize that the baby had not come from Salinas’ womb and they contacted police. + +“I don’t regret it, I don’t regret,” local newspapers quote Salinas as saying. The body of the young mother was found in a shrub close to the killer’s house. Investigators are still trying to determine if she had an accomplice. + +Reynosa is a border city in the northern part of Tamaulipas, Mexico. It is located on the southern bank of the Rio Grande, directly across the border from Hidalgo, Texas. + +Like us on Facebook + + Follow us on Twitter & Instagram \ No newline at end of file diff --git a/tests/data/text/foxnews.com3.txt b/tests/data/text/foxnews.com3.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/text/foxnews.com4.txt b/tests/data/text/foxnews.com4.txt new file mode 100644 index 00000000..344a3f52 --- /dev/null +++ b/tests/data/text/foxnews.com4.txt @@ -0,0 +1,3 @@ +Christmas is upon us – and with that comes Christmas albums. Michael W. Smith brings us his latest holiday album – “Michael W. Smith and Friends: The Spirit of Christmas,” the new album from the multi-talented Smith. It includes a star studded list of guests who sing on the album – including: Vince Gill, Lady Antebellum, Little Big Town, Martina McBride, Amy Grant, Carrie Underwood, Jennifer Nettles, Bono and Michael McDonald. + +To find out more about Michael W. Smith, click HERE. \ No newline at end of file diff --git a/tests/data/text/glamour.com1.txt b/tests/data/text/glamour.com1.txt new file mode 100644 index 00000000..faba877b --- /dev/null +++ b/tests/data/text/glamour.com1.txt @@ -0,0 +1,17 @@ +Oh, RiRi. You truly are an icon. + +When Rihanna walks into a fashion show, even editors, publicists, other celebs gasp. Gasp they did on Friday and Saturday as the cooler-than-thou-crooner sashayed into her costume designer Adam Selman's show, as well as those of Joseph Altuzarra and Alexander Wang. And she did not disappoint. Let's take a look at what she wore: + + + + At Adam Selman, RiRi rocked an elegant and girly white A-line dress with pearls and my favorite Christian Louboutin strappy pumps. + + + + For Altuzarra, she glammed out in super sophistication with a plunging black jacket with fringe from the designer's collection—WITHOUT PANTS. I LOVE IT. + + + + At Alexander Wang, she brought out her inner '90s child with a matching midnight blue hoodie and skirt. + +Are you a Rihhana fan? I love her. Now where's my plunging black jacket? Not to fret—I will wear pants. \ No newline at end of file diff --git a/tests/data/text/glamour.com2.txt b/tests/data/text/glamour.com2.txt new file mode 100644 index 00000000..cf0f4a9a --- /dev/null +++ b/tests/data/text/glamour.com2.txt @@ -0,0 +1,5 @@ +A ton of brides are going for the pretty pretty princess vibe on their wedding day, so lots of wedding dress designers automatically reach for the tulle and the crinolines. {“Fluffier! Puffier! MORE! } If you’re a bride who isn’t feeling the cupcake vibe, I suggest checking out white evening dresses, which tend to be a lot more streamlined. And bonus: Evening gowns are generally less expensive than wedding gowns. (Four of the dresses below are under $1,000.) Here are six off-the-rack I like: + + + + Off-the-rack wedding dresses: Yes or no? Would you want to wear an evening gown on your wedding day? Or does it HAVE to be bridal? \ No newline at end of file diff --git a/tests/data/text/globalnews.ca1.txt b/tests/data/text/globalnews.ca1.txt new file mode 100644 index 00000000..f80c5806 --- /dev/null +++ b/tests/data/text/globalnews.ca1.txt @@ -0,0 +1 @@ +In a video posted to their Facebook page, Ashley and Tyson Gardner ask for prayers of support as they are about to welcome two sets of twins into the world 12 weeks early. \ No newline at end of file diff --git a/tests/data/text/globalnews.ca2.txt b/tests/data/text/globalnews.ca2.txt new file mode 100644 index 00000000..0c88185a --- /dev/null +++ b/tests/data/text/globalnews.ca2.txt @@ -0,0 +1 @@ +After a six-month roller-coaster ride that has taken them to parenthood — not to mention around the world in the news media — the Tyson and Ashley Gardner of Utah County celebrated the arrival of their highly-anticipated quadruplets on Sunday. \ No newline at end of file diff --git a/tests/data/text/gq.com1.txt b/tests/data/text/gq.com1.txt new file mode 100644 index 00000000..2eeb7adf --- /dev/null +++ b/tests/data/text/gq.com1.txt @@ -0,0 +1,19 @@ +Ahhh, at last, Memorial Day weekend. The sweet sweet reminder that summer has officially arrived. It's just crab cakes and football straight through Labor Day, right? Yeah. Well, if you find yourself celebrating the return of the season with a road trip, but have yet to pack, we've got your back. You'll be traveling with everything you need in no time. All you have to do is pay close attention to the following. + + + +First, make sure your trusty weekender is ready to go. If you're bringing a suit, better go with a garment bag. + +Grab your toiletry kit, a 3-day weekend is not the time to let your skin down. + +Make sure you bring sunscreen. It's about your health, it's important. + +Don't just throw everything in your weekender, guys. There's an art to folding. Learn it. Live it. Love it. + +Planning on going for a swim? You'll need one of these. Not the board shorts you bought a few summers ago. Forget about those. Forever. + +Skip the running shoes, you don't need those when you've got this. + +Throw in a good book for the beach, the train, or wherever. + +And remember, don't show up at somebody's house empty-handed. \ No newline at end of file diff --git a/tests/data/text/gq.com2.txt b/tests/data/text/gq.com2.txt new file mode 100644 index 00000000..e108f82c --- /dev/null +++ b/tests/data/text/gq.com2.txt @@ -0,0 +1,43 @@ +With summer finally upon us, it's hard not look wistfully from the inside of a car and long for the wind on your face—barreling down country roads, freed from the tethers of smartphones and chatty passengers. For a lot of folks, this time of year is when they finally take the plunge, shirk that four-wheeled cage, and buy a motorcycle. Bikes are simple: two wheels, a motor, and miles and miles of outlaw-style freedom. But, as with everything in life, that simplicity comes with a price. So, if heading off into the sunset like Brando in The Wild One is your new life's goal, then here's everything you need to know about it before you do: + +The Real Cost of Ownership + + It's not a surprise that when gas prices spike during the warmer months, so do motorcycle sales. And while it's true that some motorcycles do get better gas mileage than cars, and that they're often cheaper to buy, the fact is that the cost of bike ownership goes way beyond the MSRP and price at the pump: + +The Bike + + Motorcycle prices can vary wildly, but on average, if you're buying a new motorcycle fit for a beginner, you're probably spending anywhere between $5,000 and $10,000. + +Insurance + + If you are over 25 and have a spotless driving record, you can get a pretty decent rate on insurance, possibly under $500 a year. Unfortunately, there's a lot more involved than simply your age and driving record—the population density of where you live, the theft rate of the bike model, whether Christmas falls on a Tuesday...when it comes to insurance, it's Thunderdome. Shop around, obviously, but just know that you're definitely going to shell out some cash. + +Equipment and Maintenance + + This is where things can add up. Cars go a lot longer between service intervals, not to mention things like tire, spark plug, and belt replacement. Tires can be especially expensive on motorcycles, running between $400 and $600 for a set. And depending on how hard you ride, you may have to change at least the rear tire every 3,000 miles or so. Chains and drive belts need occasional replacing, and those can cost between $140 and $250. Maintenance intervals can run anywhere between 5,000 and 20,000 miles, depending on the motorcycle, but if there's a valve adjustment involved, expect to pay anywhere between $800 and $1,500. Add in regular oil changes, chain maintenance, and various other odds and ends and, if you ride often, you can expect to drop at least $1,000 per year just on maintenance. + +Gear + + At the very least you will need a helmet, which can run anywhere from $150 to $900. But if a helmet is all you think you need, you should stick to four wheels. The smart rider who values his skin will also wear a motor jacket, preferably high-abrasion grade leather, gloves, and boots at all times. And while most people ride in jeans, the truth is, if you go down at any speed above 15 mph, jeans will come off like a wet paper towel; protective pants are highly recommended. Conservatively, you should plan to initially spend at least $800 to $1,200 on new gear, which, of course, will eventually have to be replaced as items wear out. + +Getting Started + + All right, so you're still undaunted and have decided to take the plunge. So where to start? The best thing you can do for yourself, as well as everyone else on the road, is to sign up for the Basic RiderCourse at the Motorcycle Safety Foundation. It usually costs around $275-$350, depending on where you live, but it's a lot cheaper than a trip to the hospital because you had no idea what you were doing. The course consists of 10 hours of riding instruction, usually stretched over a weekend, and both motorcycles and helmets are provided (although if bowling shoe rentals give you the creeps, then you'll definitely want to bring your own helmet. Just make sure it's DOT or Snell approved). The class is generally taught in a big parking lot or other open space, so you have the benefit of making mistakes without cars barreling down on you. And in a lot of states, passing the class will count as your DMV riding test, which is worth the price of admission alone. + +And if you think classes are for sissies, consider that in 2011, there were 4,323 deaths on motorcycles. Then consider that there were 81,000 recorded injuries as well, and that per vehicle mile, motorcyclists were 30 times more likely to die in a crash than passenger car occupants. Long story short? Take the class. + +Getting a Bike + + The best buyer is an informed one—and there's this thing called the Internet with a lot of information on it, so you really have no excuse. Generally speaking, motorcycle salesmen are pretty knowledgeable, but they aren't shamans, so you need to have an idea of what you're looking for. And in the off chance that you get a clueless salesman, it's even more important to be informed. If you know what type of riding you plan on doing—track days, commuting, cruising, touring, or just tooling around town—you can usually narrow your options before even hitting the motorcycle shops. + +The first question is what size bike you should buy. As someone who's worked in a motorcycle shop, my suggestion is to aim for the middle. You can start with a 250cc bike, which will definitely be the easiest to learn on, but you'll probably outgrow it within a few months. Which means you'll just turn around and want to buy another bike, ending up spending far more money than you planned to. Something in the 500-600cc range, on the other hand, will stay fun for a long time, even as your skills improve. Engine size isn't the only factor, either: A Yamaha YZF-R6 is a 600cc, but it's essentially a race bike capable of serious acceleration and high top end speeds. This is not a bike for beginners. A 865cc Triumph Bonneville, on the other hand, is a fantastic bike to start on. + +The best thing to do when picking a motorcycle is to consider the seating position—standard is the easiest for learning—as well as the weight and power output, and how you plan to use it. And remember, just because your cousin let you ride his '92 Ninja once doesn't mean know how to handle a modern 180-horespower liter bike. When starting out, there is absolutely no reason to buy a big bike, whether a big Harley or a powerful sports bike. The learning curve is far too steep. If you need further proof of that, just check out these NHTSA statistics: + +Fatalities by engine size: + + Up to 500cc: 6% + + 500 to 1000cc: 40% + + 1001 to 1500cc: 30% \ No newline at end of file diff --git a/tests/data/text/graziadaily.co.uk1.txt b/tests/data/text/graziadaily.co.uk1.txt new file mode 100644 index 00000000..7eeb42f3 --- /dev/null +++ b/tests/data/text/graziadaily.co.uk1.txt @@ -0,0 +1,7 @@ +Lady Gaga's no stranger to the world of beauty, having worked on a MAC makeup collection before releasing two of her own fragrances. Now she's dipping her heel into the realm once more as the face of Shiseido's 2015 New Year's ad campaign. + +Not only that but Gaga turned her hand to photography for the Japanese beauty brand, shooting the campaign images herself. And yes, that means selfies ahoy! + +50 selfies to be exact. According to WWD, the pop star shot a different image for a whooping 50 different Japanese newspapers. And it's no surprise considering selfies have dominated social media in 2014 and there's no doubt Gaga's had plenty of practice - have you seen her selfie-packed Instagram profile? Exactly. + +Each image will appear across the New Year's busy shopping period, encouraging shoppers to look as selfie-perfect as Gaga and we have every faith they'll be as wacktacular as ever. Sadly, Shiseido has no plans to publish the campaign outside of Japan but if you'd like to see Gaga's 50 selfies, they're due to be released on the brand's website in the New Year. Failing that, we're sure one or two will pop up on Instagram. \ No newline at end of file diff --git a/tests/data/text/graziadaily.co.uk2.txt b/tests/data/text/graziadaily.co.uk2.txt new file mode 100644 index 00000000..3fd51900 --- /dev/null +++ b/tests/data/text/graziadaily.co.uk2.txt @@ -0,0 +1,9 @@ +'Tis the season to wrap up, get glam and ingest your body weight in food and drink. No better time, then, for ASMALLWORLD's utterly fabulous multi-day 10th birthday party in Gstaad, Switzerland. And so it was that our Alya Mooro joined a rolodex of some of the most alluring individuals, the likes of Dianna Agron, Caitlin FitzGerald, Pixie Geldof, Tali Lennox and the utterly charming Carey Mulligan - who hosted the Saturday night charity gala in aid of War Child - to descend upon the Palace Hotel in Gstaad for a weekend that positively oozed glamour. We round up some of our highlights of the trip below... + +The Palace Hotel acted as ground zero for the weekend, hosting a number of fabulous gatherings including welcome cocktails on the Friday night ahead of an epic dinner at in-house restaurant Gildo's where guests got to know each other over wine and three courses of superb cuisine before decamping to the basement club (more on which later). + +While some of the more ambitious of individuals spent their morning after the night before taking to the slopes in their ski gear, the rest of us took a more leisurely approach in the form of a chairlift to the top of Wasserngrat mountain, where we were greeted by a beaming sun, ridiculously Instagrammable views and cauldrons and cauldrons of bubbling cheese. And our lives were basically complete. Giddy from having ingested a lifetime supply of cheese and wine, the remainder of the afternoon was spent concoting the best hashtags to accompany our fairytale-perfect snaps. Our favourite? 'Gstaad'd from the bottom now we here.' Yup, we think Drake would be proud of that one, too. All credit to the altitude. And the wine. + +Snow boots and gigantic fluffy hats safely tucked away, guests descended upon the Palace ballroom in a dizzying array of black tie extravagance. With trailing dresses in any and all colours imaginable, glittering jewels and a collection of headdresses that frankly make Blair Waldorf's look preeeetty pitiful, guests took their seats for a five course meal that was overseen by none other than Carey Mulligan and ASW Chairman Patrick Liotard-Vogt. 'I am begging you to give recklessly and generously,' said Mulligan in her impassioned address. And who could turn her down? With help from raffle prizes from the likes of Hublot, Maiyet, Tinker Tailor, and even a peck on the cheek from Dianna Agron, $100,000 was raised for War Child - a charity that provides support to children in high conflict zones around the world. + +The Palace's basement nightclub - hailed as one of the most exclusive in the region - hosted the afterparties that concluded both festive evenings, and almost contributed to many a missed flight on the Sunday morning! Shoes in hand and fists pumping, attendees of the soirees quite literally let their hair down, dancing into the morning with a little help from the DJs who alternated between the latest in chart topping hits to never forgotten classics, pleasing just about everyone in the process. As attendees from literally all around the world revelled in the moment, we realised just how small a world it is, and how happy we are to be a part of it. \ No newline at end of file diff --git a/tests/data/text/gulflive.com1.txt b/tests/data/text/gulflive.com1.txt new file mode 100644 index 00000000..1b751448 --- /dev/null +++ b/tests/data/text/gulflive.com1.txt @@ -0,0 +1,73 @@ +For the serious genealogist and family historian there are some worthwhile learning opportunities coming up during the year 2015. + + + + The Federation of Genealogical Societies will meet Feb. 11-14 in Salt Lake City, Utah, in conjunction with RootsTech for a one-time special genealogy event at the Salt Palace Convention Center. This gives the genealogy community a unique opportunity to experience two different conferences under one roof or the option to attend only one. + + + + "Destined to be the largest family history event North America has ever seen, the FGS 2015 conference is the perfect destination for anyone interested in tracing their roots," said D. Joshua Taylor, FGS president. + + + + FGS sessions will focus on methodology, records, ethnic research and migration. RootsTech will offer a program of technology-based solutions for the genealogy needs of both individuals and societies. + + + + Those attending will have access to the famous Salt Lake City Family History Library, a dream destination of genealogists everywhere. It houses the largest genealogical collection in the world, which includes more than 2.4 million rolls of microfilmed records and 356,000 books. Visitors are welcome to bring a flash drive for digital copies. + + + + Special hotel rates are offered to attendees. For more information on the innumerable number of sessions and events and the expo, log on to www.fgsconference.org. + + + + Another stellar conference is that of the National Genealogical Society. Its 2015 Family History Conference themed "Crossroads of America" will be held in St. Charles, Mo., May 13-16. + + + +The program features numerous tracks each day covering a broad array of topics including records for Missouri and the surrounding states, migration into and out of the Midwest, methodology, analysis, problem solving, genetics, technology, military records, and lectures on locations ranging from Colonial America to Eastern Europe. + + + + Attendees will be offered a variety of conference tours from which to choose that include the historic Daniel Boone Home & Heritage Center, the Lewis & Clark Boathouse, historic St. Louis, Mo., and the National Archives Personnel Records Center. + + Throughout this conference the Board for Certification of Genealogists will again sponsor a skill-building track for intermediate to advanced researchers interested in improving their research skills. + + + + Multiple lectures, workshops, showcases, luncheons with noted speakers are on the agenda. For details, log on to www.ngsgenealogy.org. + + + + And for those who would like to learn in the comfort of their homes, the NGS is offering a new online study this year called "Researching Your Revolutionary War Ancestors" by author Craig Roberts Scott. This course introduces numerous U.S. records created for, during and after the war for those who fought for independence, and provides strategies to identify and locate information on an ancestor living at the time of the war. + + + + Scott is a nationally recognized lecturer, educator and genealogical and historical researcher with more than 30 years' experience. He specializes in the diverse military records at the National Archives. The eight-module course consists of lessons, examples, exercises and self-graded exams that can be viewed on a home computer. Students work independently. + + + +Specifically, students will learn how to: + +• identify an ancestor living at the time of the Revolutionary War, + +• locate information about the service of an ancestor in the RW, + +• understand pension law and locate pension application files, + +• locate pension ledgers, payment vouchers, last and final payments, + +• locate information about the unit that the ancestor served in during the war; + +• locate and understand the Compiled Military Service Record, and + +• compile a post-war record. + + + +A course syllabus is available online for review. For more details, log on to www.ngsgenealogy.org. + + + +Correspondent Joanne Anderson may be reached at joandy42@cableone.net. \ No newline at end of file diff --git a/tests/data/text/gulflive.com2.txt b/tests/data/text/gulflive.com2.txt new file mode 100644 index 00000000..c9367582 --- /dev/null +++ b/tests/data/text/gulflive.com2.txt @@ -0,0 +1,3 @@ +Crown Equity Holdings Could be Sitting on a Gold Mine with CRWETube + +Crown Equity Holdings Inc. (OTCMKTS: CRWE) goal is to accelerate the success of the company, by refocusing on prior endeavors to deliver value for its stockholders in both the near and long term. CRWE has refocused its efforts and direction to an endeavor launched earlier this year in reference to an online business-to-business (B2B) marketplace platform for manufacturers and small to large businesses on a global basis to sell and acquire various types of merchandise. CRWE has started updating its B2B business plan and strategies to move forward. Strategic plans are also being developed for CRWE’s online video-sharing and VoIP communication projects to improve the potential future growth for the company Online ad spending has rapidly become a main stay in many advertising budgets with online TV ad budgets being with one of the main dynamics driving product and service revenue across all industries. BIA/Kelsey has projected that social media advertising spending will hit close to $10 billion by year 2016 well up from $4.8 billion spent in 2012. According to Break Media, video ad spending will reach $5.4 Billion by 2016. CRWE is targeting that multi-billion dollar market with its video sharing website CRWETube (www.crwetube.com) General disclaimer statements on this website or newsletter may constitute forward-looking statements and are subject to numerous risks and uncertainties, including the failure to complete successfully the development of new or enhanced products, the Company’s future capital needs, the lack of market demand for any new or enhanced products the Company may develop, any actions by the Company’s partners that may be adverse to the Company, the success of competitive products, other economic factors affecting the Company and its markets, seasonal changes, and other risks detailed from time to time in the Company’s filings with the U.S. Securities and Exchange Commission. The actual results may differ materially from those contained on this website. The Company disclaims any obligation to update any statements in this website. These stock quotes and related data are provided for information purposes only and are not intended for trading purposes. Crown Equity Holdings, Inc. . will not be liable for any inaccuracies or delays in such data, or for any actions taken in reliance thereon. Potential investors should seek independent information and advice from qualified investment professionals prior to investment Crown Equity Holdings, Inc. provides links to websites operated by third parties. These links may be of interest or of use to you, and are provided for convenience only. You should be aware that in using these links, you are leaving Crown Equity Holdings, Inc.‘s website. Crown Equity Holdings, Inc. does not approve or endorse the content, information or materials available on such third party websites. In addition, Crown Equity Holdings, Inc. makes no representation regarding, and is not responsible for, the content, information or material available on such websites. If you decide to access such websites or newsletter you do this at your own risk, and Crown Equity Holdings, Inc. will not be liable for any loss or damage associated with your use of, or reliance on, the content, information or material available on such websites. \ No newline at end of file diff --git a/tests/data/text/huffingtonpost.de1.txt b/tests/data/text/huffingtonpost.de1.txt new file mode 100644 index 00000000..e1277780 --- /dev/null +++ b/tests/data/text/huffingtonpost.de1.txt @@ -0,0 +1 @@ +// results fragments requested via ajax should be an empty response if ($_GET['ajax'] != 'true'): ? = __("no entries found")?> endif; ? \ No newline at end of file diff --git a/tests/data/text/huffingtonpost.de2.txt b/tests/data/text/huffingtonpost.de2.txt new file mode 100644 index 00000000..bcf2d0a8 --- /dev/null +++ b/tests/data/text/huffingtonpost.de2.txt @@ -0,0 +1 @@ +Der erste Eindruck zählt - das gilt nicht nur fürs Kennenlernen, sondern auch dann, wenn man die Person zum ersten Mal zu Hause besucht.... \ No newline at end of file diff --git a/tests/data/text/lifebuzz.com1.txt b/tests/data/text/lifebuzz.com1.txt new file mode 100644 index 00000000..c752b862 --- /dev/null +++ b/tests/data/text/lifebuzz.com1.txt @@ -0,0 +1,5 @@ +She Was Tired Of Being Photoshopped, So Here’s What She Did About It. + +Grammy Award-winning, singer-songwriter Colbie Caillat released her new EP, Gypsy Heart Side A. Her album’s lead single “Try” makes a powerful statement about beauty ideals. Share this with all of your girlfriends by clicking the Share button below. + +Tell us what you think \ No newline at end of file diff --git a/tests/data/text/lifebuzz.com2.txt b/tests/data/text/lifebuzz.com2.txt new file mode 100644 index 00000000..b15966c4 --- /dev/null +++ b/tests/data/text/lifebuzz.com2.txt @@ -0,0 +1,5 @@ +What Do You Think This Guy Is Doing? You Will Never Guess…. And It’s Going To Break Your Heart + +Sylvie a 13 year old Husky went on her morning walk with her owner when she fell through the ice. After being trapped in the water for about 30 minutes, Firefighter Sean Coyle came to the rescue. The pictures below capture the event as it transpires. Exhausted Sylvie had been in the frigid water for more than 30 minutes Firefighter Sean Coyle uses a basket to slide out to Sylvie The husky clings to the ice as firefighter Sean Coyle inches out to the hole Fireman Coyle grabs Sylvie by the scuff of the neck as he attempts to lift her from the water + +Tell us what you think \ No newline at end of file diff --git a/tests/data/text/livescience.com1.txt b/tests/data/text/livescience.com1.txt new file mode 100644 index 00000000..6a90f25a --- /dev/null +++ b/tests/data/text/livescience.com1.txt @@ -0,0 +1,29 @@ +The active ingredient in the psychedelic drug, psilocybin, seems to completely disrupt the normal communication networks in the brain, by connecting "brain regions that don't normally talk together," said study co-author Paul Expert, a physicist at King's College London. + +The research, which was published today (Oct. 28) in the Journal of the Royal Society Interface, is part of a larger effort to understand how psychedelic drugs work, in the hopes that they could one day be used by psychiatrists — in carefully controlled settings — to treat conditions such as depression, Expert said. [Trippy Tales: The History of 8 Hallucinogens] + +Psilocybin, the active ingredient in magic mushrooms, is best known for triggering vivid hallucinations. It can make colors seem oversaturated and dissolve the boundaries between objects. + +But the drug also seems to have more long-lasting effects. Many people report intensely spiritual experiences while taking the drug, and some studies even suggest that one transcendent trip can alter people's personalities on a long-term basis, making those individuals more open to new experiences and more appreciative of art, curiosity and emotion. + +People who experiment with psilocybin "report it as one of the most profound experiences they've had in their lives, even comparing it to the birth of their children," Expert told Live Science. + +Scientists have long known that psilocybin binds to a receptor in the brain for serotonin, a brain chemical that plays a role in mood, appetite and sleep, but exactly how the drug transforms the whole brain's pattern of communication isn't clear. + +In past work, Expert's colleagues had found that psilocybin spurred the brain into a more dreamlike state, and that the drug decreased brain activity. + +In the current study, the team used functional magnetic resonance imaging (fMRI) to scan the brain activity of 15 healthy volunteers — once after they had taken a placebo, and once after they took the hallucinogen psilocybin. (The team chose only people who had reported past positive experiences with magic mushrooms to prevent them from panicking inside the claustrophobic MRI machines.) + +The team then compared the brain activity of the individuals on and off the drug, and created a map of connections between different brain regions. + +Psilocybin dramatically transformed the participants' brain organization, Expert said. With the drug, normally unconnected brain regions showed brain activity that was synchronized tightly in time. That suggested the drug was stimulating long-range connections the brain normally wouldn't make. After the drug wore off, brain activity went back to normal. + +Psilocybin may create a brain state akin to synesthesia, a sensory effect in which one sense stimulus (such as a number) always gets paired in the brain with another (such as a color or a sound), the researchers wrote in the paper. People with synesthesia may see certain colors when they hear music, or always see the number 3 in yellow, for instance, Expert said. + +The findings could help scientists who are studying the drug as a potential treatment for depression, Expert said. Past work has found that people tend to be happier even after using psilocybin just once, but scientists would need to get a much better picture of how the drug impacts the brain before using psilocybin to treat depression, Expert said. + +The research could ultimately also help answer bigger questions of the mind, like how people construct a sense of self. + +"Through studies such as these we can really begin to tackle the questions of how we achieve coherent experiences of ourselves in the world around us, and understand what makes this break down," said Mitul Mehta, a psychopharmacology researcher at King's College London, who was not involved in the study. + +Follow Tia Ghose on Twitter and Google+. Follow Live Science @livescience, Facebook & Google+. Originally published on Live Science. \ No newline at end of file diff --git a/tests/data/text/livescience.com2.txt b/tests/data/text/livescience.com2.txt new file mode 100644 index 00000000..39147386 --- /dev/null +++ b/tests/data/text/livescience.com2.txt @@ -0,0 +1,33 @@ +They moan. They bite. They shuffle. Or sometimes, they sprint, swarm and carry on surprisingly intelligent conversation. + +Zombies are something of an open-source pop-culture phenomenon. Unlike Dracula or Frankenstein, these Halloween monsters aren't based on a literary resource. In fact, the modern conception of a zombie dates back to 1968, in a movie that doesn't so much as use the word: George Romero's "Night of the Living Dead." + +"He didn't call them zombies, and he didn't think about them as zombies," said Ozzy Inguanzo, a screenwriter and author of "Zombies on Film: The Definitive Story of Undead Cinema" (Rizzoli, 2014). But the public did, Inguanzo told Live Science. + +"Audiences saw these lumbering dead people, and they called them zombies … therefore, they became zombies," he said. Since then, the walking undead have wormed their way into video games, comic books — and even the classics (witness 2009's novel "Pride and Prejudice and Zombies). [Zombies! Your Complete Guide to the Attack of the Dead (Infographic) ] + +The true zombie origin story dates back further than 1968, of course. The sad beginning of the myth harks back to Haiti during the 1600s and 1700s, when African slaves were worked to death on sugar plantations. As UC Irvine journalism instructor Amy Wilentz pointed out in the New York Times in 2012, it's not hard to see how the notion of a dead body, stripped of will and personality, forced to do the bidding of a sorcerer, would occur to an enslaved people. + +The notion of zombies is still part of Haitian folklore. The belief is that, through magic or poison, a sorcerer makes a person fall ill and appear to die. After the family buries the body, the sorcerer retrieves the person, who is alive, but held in thrall. In a 1997 article in the medical journal The Lancet, researchers studied three real-life cases of "zombification" and diagnosed the three sufferers with catatonic schizophrenia, epilepsy and mistaken identity. In the final case, a 31-year-old woman with possible fetal alcohol syndrome was mistaken for another woman who had died 13 years before. The cases suggest that zombification has often been used to explain mental illness or brain disorders in rural Haiti. + +Zombies made the leap from Haitian religion to American entertainment in 1932, Inguanzo said, with a film starring Bela Lugosi and Madge Bellamy called "White Zombie." That movie, in turn, was largely inspired by a 1929 travelogue by William Seabrook, a journalist who also happened to be the kind of guy who, after failing to get a good enough description of the taste of human flesh from a West African chieftain, manages to acquire a hunk of flesh from a corpse to cook up himself. (In his 1931 book "Jungle Ways," Seabrook helpfully describes the taste as being very similar to veal.) + +Zombies popped up in horror flicks over the next few decades, frequently in keeping with the Haitian voodoo theme, but sometimes branching out: The "Revenge of the Zombies" in 1943 took a sci-fi angle, with a Nazi scientist trying to create an army of the undead for Hitler. [The 10 Weirdest Ways We Deal with the Dead] + +But the quintessential zombie flick was actually inspired less by these films and more by vampires. Working off the 1954 post-apocalyptic novel "I Am Legend," by Richard Matheson, which tells the story of the last man standing in a world of vampirelike monsters, George Romero and John Russo told their own tale of a group of bickering humans threatened by the shuffling, moaning living dead. + +"He took the Haitian component out of the previous character storylines and brought them here, to suburbia," Inguanzo said. "They were our friends, our relatives, our neighbors who were coming back from the dead." + +Romero's zombies became the touchstone for those that would follow, with writers adding their own quirks to the genre. In 1985's "Return of the Living Dead," the zombies hungered not just for human flesh, but also for "braaaains." That idea stuck. The movie was also the first to introduce talking zombies, and perhaps more crucially, fast-moving zombies. + +These days, fast-walking zombies are overtaking their shambling counterparts, at least on the big screen. The trend is, in part, inspired by video games like "Resident Evil," which first came out in 1996. + +"Video games had a huge impact in bringing [zombies] back to the forefront," Inguanzo said. "These are easy bad guys to kill. There's no remorse there." + +The zombie virus has also spread to the comic-book world, like the 2005-2006 Marvel series in which all of Marvel's superheroes get infected by a zombie virus. The superheroes stay strong and smart, but crave human flesh. Even wholesome Archie Comics has been bitten by the zombie bug, with a 2013 series called "Afterlife with Archie." + +Zombie comics are, in turn, making the leap back to the screen, as with AMC's series "The Walking Dead." And Hollywood seems to be recognizing that zombies are moneymakers, turning out big-budget Brad Pitt spectacles like 2013's "World War Z." + +"Those types of movies are focusing on explosions and action and adrenaline," Inguanzo said. + +Follow Stephanie Pappas on Twitter and Google+. Follow us @livescience, Facebook & Google+. Original article on Live Science. \ No newline at end of file diff --git a/tests/data/text/mashable.com1.txt b/tests/data/text/mashable.com1.txt new file mode 100644 index 00000000..49d234ef --- /dev/null +++ b/tests/data/text/mashable.com1.txt @@ -0,0 +1,5 @@ +Google doesn't just have a new logo today: the creative web company has also hidden a surprise on its homepage. + +Go to Google.com today and click "I'm Feeling Lucky" (without entering a search term) to get an unexpected treat. + +Google is increasingly drawing attention to its homepage through more and more custom logos: some argue that these are a counter to Bing.com's daily photo, which makes the rival search engine a daily visit for many. Is today's hidden gem just for fun, or a subtle reminder to visit Google.com once in awhile? \ No newline at end of file diff --git a/tests/data/text/mashable.com2.txt b/tests/data/text/mashable.com2.txt new file mode 100644 index 00000000..eb371a26 --- /dev/null +++ b/tests/data/text/mashable.com2.txt @@ -0,0 +1,17 @@ +A power outage at Stansted International Airport in London held up holiday travel Monday afternoon, adding some extra misery to an already busy travel week. + +The power outage lasted about two hours. Photos uploaded by holiday travelers show massive queues, although the outage affected only parts of the airport. + +The power outage is affecting the transit system that gets travelers to their airplanes, according to the airport. + +Stansted is located about 40 miles north of London's center; the terminal most affected is Ryanair. + +Planes on the tarmac were also affected, as passengers had to wait because of the outage. The airport was dispatching buses to take passengers back and forth. + +The airport reported that crews had restored power at about 5:30 p.m. local time (12:30 p.m. EST). + +While much smaller than Heathrow or Gatwick airports in London, Stansted had its busiest November in eight years last month with more than 1.5 million passengers. + +Stansted Airport reported that there were some delays, but flights are operating. FlightAware was not reporting a large number of delays in the early afternoon. + +Have something to add to this story? Share it in the comments. \ No newline at end of file diff --git a/tests/data/text/mlive.com1.txt b/tests/data/text/mlive.com1.txt new file mode 100644 index 00000000..62be13a0 --- /dev/null +++ b/tests/data/text/mlive.com1.txt @@ -0,0 +1,15 @@ +Police have released surveillance photos in hopes of garnering help to track down a suspect in the University Bank robbery on Dec. 22. + +Although the photos do not show clear images of the suspect’s face, they show what appears to be a white male in dark colored clothing. + +The man is described as being in his 20s, about 5 feet 11 inches tall and 200 pounds, Ann Arbor Police Department Detective Lt. Robert Pfannes said. He is described as wearing dark rimmed glasses, a dark grey or khaki hooded sweater, black pants and white gym shoes. + +He was wearing an open-face knit ski hat that encircled his face. + +The robbery took place at about 3:15 p.m. Dec. 22 when a man entered University Bank at 2015 Washtenaw Avenue and handed a note to a bank teller, Pfannes previously said. + +The note implied the suspect, described as a white male, had a weapon and demanded money, he said. After taking money, the man then fled on foot. + +The man may have then entered a car in 2100 block of Washtenaw, Pfannes said Monday. + +Those with information on the bank robbery are asked to contact the Ann Arbor Police Department tip line at 734-794-6939 or e-mail TIPS@a2gov.org. \ No newline at end of file diff --git a/tests/data/text/mlive.com2.txt b/tests/data/text/mlive.com2.txt new file mode 100644 index 00000000..dd92586f --- /dev/null +++ b/tests/data/text/mlive.com2.txt @@ -0,0 +1,35 @@ +GREEN BAY, Wis. -- Here we go again. + +Star defensive tackle Ndamukong Suh drew the ire of the Green Bay Packers on Sunday after he stepped on quarterback Aaron Rodgers during the second half of the Detroit Lions' 30-20 loss at Lambeau Field. + +The NFL is expected to review the play Monday, according to MMQB's Peter King. Suh probably will not be suspended, but given his extensive track record for in-game safety violations, anything is possible. + +This comes just one week after Dominic Raiola drew a one-game suspension for stomping on Chicago's Ego Ferguson. And though Suh's play was less violent than Raiola's -- which actually forced Ferguson from the game -- it also looked like it could have been intentional. + +Suh is seen stepping on Rodgers' left leg once, then reapplying pressure again, lifting up with the off foot to apply his weight to Rodgers' leg. + +Rodgers, who already had left the game once with a calf injury, shoved Suh in the back of the leg after the play. + +"He'll probably say it was an accident -- he was getting blocked into (me)," Rodgers said after the game. "But we'll see." + +What did Suh say after the play? + +"He was running off the field. I was talking to (referee) Walt (Anderson), actually. My calf and my ankle were getting stepped on, so we'll see what happens." + +Green Bay coach Mike McCarthy was much less diplomatic, going so far as to call Suh's play "ridiculous." + +"There's no place for that," McCarthy said. + +Suh skipped his postgame obligations -- a violation of league rules -- but coach Jim Caldwell vouched for him after the game. + +"I didn't see it, and I don't think it was intentional, either," Caldwell said. + +And why would Caldwell say it wasn't intentional if he hasn't seen it yet? + +"I get briefed on it," he said. "Guys look at it and tell me what they thought, what they saw, so that's it. Don't think it's intentional. End of story." + +Caldwell said he's not concerned about the mind-set of his team, despite high-profile extracurriculars in back-to-back weeks that have left Detroit vulnerable to suspensions during the most critical stretch of the season. + +"I'm not worried at all," he said. + +Suh's play drew a ton of heat on social media, with all kinds of experts and analysts weighing in, including noted rapper Lil Wayne. \ No newline at end of file diff --git a/tests/data/text/newyorker.com1.txt b/tests/data/text/newyorker.com1.txt new file mode 100644 index 00000000..38129c7c --- /dev/null +++ b/tests/data/text/newyorker.com1.txt @@ -0,0 +1,23 @@ +outside is frightful, The heat wave brutal and spiteful. Our crops have no water to grow— Let it snow, let it snow, let it snow! + +Looks like the heat wave ain’t stoppin’, Our dust-storm coughs are a-whoppin’. All fish went extinct long ago— For the love of sweet Christ, let it snow! + +may he rest in peace, poor soul. With his melted nose and his melted mouth, And two eyes made out of coal. Frosty the Snowman, did you say his eyes were coal? Can you pass that coal? We could use some coal. Let us rob his grave for coal. + +oh, Christmas tree! Thy plastic branches don’t shed. Oh, Christmas tree, oh, Christmas tree! We wish real trees weren’t all dead. + +are you listening? In L.A., hail is glistening. In New Mexico, there is six feet of snow. The desert is a winter wonderland! + +Gone away is the West Side, Here to stay is the high tide. If you want to spelunk, San Francisco has sunk. California’s now an underwater land! + +In Mojave, we can build a snowman, And pretend that he is Mom or Dad. He’ll say, “Did your parents escape the horrific flash floods?,” and we’ll say, “No, man. But you can be our new dad now.” + +an ozone layer, Just like the one I used to know, Which kept out UV rays, Before aerosol sprays, And allowed for actual snow. + +of our mountain hideout, Fa la la la la, la la la la! Hope all humans have not died out, Fa la la la la, la la la la! Don our hazmat suits on tight now, Fa la la, la la la, la la la! Oxygen is leaking right now, Fa la—! + +on an open fire, Cars and houses roasting, too. Our whole city is an open pyre— Put those chestnuts away, we have to leave now. + +little dreidel, I made it out of clay. And when it’s dry and ready, Oh, dreidel, I will pray— That I can barter you for enough cans of soup to last me through the tornado. + +Endless night! All is dark, there’s no light. Cyclone clouds have blocked out the sky, We’re almost out of our dry-meat supply. Sleep in uneasy peace. We may have to eat Aunt Bernice. ♦ \ No newline at end of file diff --git a/tests/data/text/newyorker.com2.txt b/tests/data/text/newyorker.com2.txt new file mode 100644 index 00000000..742d5fe7 --- /dev/null +++ b/tests/data/text/newyorker.com2.txt @@ -0,0 +1,5 @@ +Subscribe to 'The New Yorker' + + on YouTube to keep up with all of + + our latest videos and shows. \ No newline at end of file diff --git a/tests/data/text/nj.com1.txt b/tests/data/text/nj.com1.txt new file mode 100644 index 00000000..b54259ba --- /dev/null +++ b/tests/data/text/nj.com1.txt @@ -0,0 +1 @@ +Register now for free, or sign in with any of these services: \ No newline at end of file diff --git a/tests/data/text/nj.com2.txt b/tests/data/text/nj.com2.txt new file mode 100644 index 00000000..b54259ba --- /dev/null +++ b/tests/data/text/nj.com2.txt @@ -0,0 +1 @@ +Register now for free, or sign in with any of these services: \ No newline at end of file diff --git a/tests/data/text/nola.com1.txt b/tests/data/text/nola.com1.txt new file mode 100644 index 00000000..c9a87cbf --- /dev/null +++ b/tests/data/text/nola.com1.txt @@ -0,0 +1,33 @@ +As the founder of the New Orleans Jazz & Heritage Festival, George Wein has already made one major contribution to the continuation of the city's culture. On Thursday morning, he was on hand for the unveiling of another. + +The 89-year-old festival impresario attended a ribbon-cutting ceremony Thursday (Dec. 11) for the George and Joyce Wein Jazz & Heritage Center. The new educational and community center at 1225 N. Rampart St. is named for Wein and his late wife and business partner, Joyce. Wein helped cut the ceremonial red ribbon Thursday morning alongside Mayor Mitch Landrieu. + +The center is owned by the New Orleans Jazz and Heritage Foundation, the nonprofit organization that owns Jazz Fest. The foundation's longtime offices are next door to the new Jazz & Heritage Center. + +"We expect this facility, located at the gateway to the Tremé neighborhood, to give a major boost to the cultural and economic development of not only Tremé, but to our entire city," said Don Marshall, the Jazz & Heritage Foundation's executive director. + +In 2008, the foundation bought the former Tharp-Sontheimer-Laudumiey Funeral Home. Two separate townhouses on the site, built in the 1870s, were combined into one Italianate-style building in the early 20th century. After acquiring the property, the foundation's board of directors and staff spent several years deciding what to do with it. + +Eventually a plan emerged. The space, after an extensive renovation, would become the permanent home for the foundation's Don "Moose" Jamison Heritage School of Music, a free program for young musicians. Since its founding by saxophonist and educator Kidd Jordan in 1990, the Heritage School of Music has been housed on university campuses. Now it will inhabit the seven classrooms and 200-seat performance space -- every space is wired to a central control room for audio and visual recording -- at the George and Joyce Wein Jazz & Heritage Center. + +The building also will host cultural programs presented by the Jazz & Heritage Foundation and other arts and community organizations. + +Most of the building's original façade was maintained in the renovation, along with the architectural outlines of the older, front part of the building. Much of the 12,500-square-foot structure's rear section, a more recent addition, was rebuilt from the ground up. + +The total bill for the project came in at around $9 million. The foundation, which generally nets around $3 million from the Jazz and Heritage Festival each spring, self-financed the bulk of the cost. + +Around $3 million was donated by various benefactors, including George and Joyce Wein, the Goldring Family Foundation, ArtPlace (a consortium of major national foundations), the Louis Prima and Gia Maione Prima Foundation, the Ella West Freeman Foundation, the Helis Foundation and the State of Louisiana. Other individuals and local and national foundations contributed to the foundation's capital campaign. + +Numerous manufacturers of musical instruments donated gear to the center, including Shure (microphones), Yahama (drums), Casio (keyboards), Zildjian (cymbals) and D'Addario (strings). + +His sizable donation to the project notwithstanding, Wein and his wife were the obvious choice for namesakes of the new center. Wein was already a well-known jazz club owner when, in the 1950s, he founded the Newport Jazz Festival, the model for all outdoor jazz festivals that would follow. + +In the early 1960s, city leaders invited him to consider founding a festival in New Orleans. But several obstacles, including segregation laws that prohibited interracial bandstands, stood in the way. + +In the late 1960s, the city staged two versions of the International Jazz Festival without Wein. In 1970, Wein's Festival Productions produced the first New Orleans Jazz & Heritage Festival and Louisiana Heritage Fair in what is now the Congo Square area of Armstrong Park, augmented by evening shows in the Municipal Auditorium. Wein is credited with instigating the festival's early, and ongoing, emphasis on indigenous food and crafts, as well as music. He also hired a Tulane University student named Quint Davis, who is now the festival's producer/director. + +"George and Joyce Wein have done so much to benefit our community and our culture," Demetric Mercadel, president of the Jazz & Heritage Foundation's board of directors, said in a statement. "It is only fitting that we recognize their many contributions by having their names grace this wonderful new facility. This is a true testament to their legacy." + +On Friday (Dec. 12) at 8 p.m., avant-jazz saxophonist and educator Kidd Jordan - who founded the foundation's Don "Moose" Jamison Heritage School of Music in 1990 -- and his accomplished offspring, Kent, Stephanie, Marlon and Rachel Jordan, will headline a grand opening concert; students from the Heritage School of Music will open the show. Admission is free, but all advance tickets are sold out; any remaining seats will be available on a first-come, first-serve basis. The concert will also be live streamed at wwoz.org, the web site of the foundation-owned on WWOZ-FM. + +On Saturday (Dec. 13), the center throws open its doors for a Treme Neighborhood and Community Open House. Students from the Heritage School of Music will also perform a free holiday concert on Saturday. \ No newline at end of file diff --git a/tests/data/text/nola.com2.txt b/tests/data/text/nola.com2.txt new file mode 100644 index 00000000..b14dca93 --- /dev/null +++ b/tests/data/text/nola.com2.txt @@ -0,0 +1 @@ +Holden is an associate in the firm's New Orleans office, practicing in the areas of business and energy litigation. She received her Juris Doctorate from Loyola University New Orleans College of Law, where she graduated summa cum laude. At Loyola, Ms. Holden served as the Articles Editor of the Loyola Law Review and as a brief writer and oralist for the 2011-2012 National Environmental Law Moot Court Team. Holden was recognized as a William L. Crowe, Sr., Scholar and received the Faculty Award, the LSBA Civil Law Award, the Dean's Award, the Joseph V. Bologna Award, the Warren E. Mouledoux Professional Responsibility Award, and the LSBA Corporate and Business Award. Prior to joining the firm, she served as a law clerk to the Honorable Carl J. Barbier in the United States District Court for the Eastern District of Louisiana. \ No newline at end of file diff --git a/tests/data/text/nydailynews.com1.txt b/tests/data/text/nydailynews.com1.txt new file mode 100644 index 00000000..546b91be --- /dev/null +++ b/tests/data/text/nydailynews.com1.txt @@ -0,0 +1,3 @@ +Thank you for using the New York Daily News to find your next new car. + +In order to Find Autos By Brand it is necessary to download a browser that can support it, such as Firefox, Chrome, or Safari. \ No newline at end of file diff --git a/tests/data/text/nydailynews.com2.txt b/tests/data/text/nydailynews.com2.txt new file mode 100644 index 00000000..6efe448b --- /dev/null +++ b/tests/data/text/nydailynews.com2.txt @@ -0,0 +1,27 @@ +Best New York Daily News Front Covers of 2014 From the death of Robin Williams and Philip Seymour Hoffman to the rise of ISIS, the Daily News has covered the biggest stories of 2014. + +As 2014 winds down, take a look back at some of the biggest news stories of the year ... + +The devastating 9.1 magnitude earthquake that caused the 2004 Boxing Day tsunami left more than 230,000 dead in 14 countries. Take a look back at... + +The best Daily News photos of 2014 Through the lens of New York Daily News photographers, take a look back at the years most striking moments + +An 18-year-old black teenager has been shot dead by police at a gas station in St. Louis, Missouri on Dec. 24, 2014, just 5 miles from where Mich... + +With the Sony World Photography Awards contest quickly coming to an end, the top 15 best entries in the 'open' category so far have been surfacin... + +After receiving more than 9,200 enteries from over 150 countries, the winners of the 2014 National Geographic Photo Contest have finally been ann... + +The decline and decay of Cuba In 1962, the United States set an embargo against Cuba effectively cutting off trade between the two countries. Since then, Cuba has been faced w... + +Most shocking red carpet looks of 2014 It was the year of more skin and less dress! From plunging necklines to cut-outs and slits, 2014 had no shortage of shocking dresses on the red c... + +Thousands of protesters gathered in New York City, Washington D.C., and elswehere for an anti-police violence demonstration in response to the po... + +There's another royal baby on the way! In celebration of Kate Middleton and Prince William's news that they're expecting their second child, take... + +He was America's favorite TV Dad but now people are changing their tune as more sexual assault allegations shed light on the actors past. + +Following the Ferguson verdict and protests, thousands gathered across the country after a New York grand jury decided not to indict the police o... + +A look back at the life and career of Paul Walker Paul Walker, best known for his role in the "Fast & Furious" franchise, died in a fiery car crash north of Los Angeles on Nov. 30, 2013. He was 4... \ No newline at end of file diff --git a/tests/data/text/nypost.com1.txt b/tests/data/text/nypost.com1.txt new file mode 100644 index 00000000..ffa54ea2 --- /dev/null +++ b/tests/data/text/nypost.com1.txt @@ -0,0 +1,29 @@ +Book publishing giant Macmillan announced a peaceful settlement with Amazon on Thursday on pricing for print and digital books ordered through the giant online retailer. + +Similar agreements were reached with Simon & Schuster in October and Hachette in November. + +Hachette had waged a bruising six-month public battle with Amazon — claiming the e-tailer was acting as a bully by forcing book prices to unacceptably low levels that hurt publishers’ profits and authors’ commissions. + +Amazon countered that it was trying to get the best deal for consumers — who would ultimately buy more books, which would ultimately generate more revenue for publishers. + +While the battle was fierce between Hachette and Amazon, the Macmillan-Amazon dust-up was more civil — as were the settlement talks. + +Macmillan, which owns Henry Holt, Picador and St. Martin’s, is part of the Germany-based von Holtzbrinck group. + +After the settlement was reached, Macmillan CEO John Sargent said in a posting to authors, illustrators and agents that, while he is pleased with the deal, there are still some unsettling elements. + +Amazon released a separate statement saying, “It allows us to grow our business together with Macmillan and their authors. Importantly, the agreement specifically creates a financial incentive for Macmillan to deliver lower prices for readers.” + +But the days of super-discounting appear to be ending. + +Amazon’s agreements with each of the three publishers embrace what has been called the “agency model” of book pricing — which allows each publisher to set the price that the books will be sold at. + +Macmillan’s deal with Apple allows the Cupertino, Calif., company to be the only retailer allowed unlimited discounting, Sargent said in his post. + +“Irony prospers in the digital age,” he added. The reference to “irony” refers to the long-running anti-trust case that the Department of Justice had leveled against five publishers and Apple, claiming that the publishers conspired to fix prices in a battle to gain leverage over Amazon. + +Eventually, all five publishers reached out-of-court settlements. Apple fought on and is currently appealing a ruling that said it, too, had engaged in price-fixing with the publishers and must pay a hefty $450 million fine. + +Sargent said a two-year consent agreement allowing Amazon to discount prices expired on Dec. 18. + +Under a separate ruling in that case, Sargent said Apple can discount prices until Oct. 5, 2017. He complained that this wrinkle will “ensure a muddled and inefficient market.” \ No newline at end of file diff --git a/tests/data/text/nypost.com2.txt b/tests/data/text/nypost.com2.txt new file mode 100644 index 00000000..49be3943 --- /dev/null +++ b/tests/data/text/nypost.com2.txt @@ -0,0 +1,17 @@ +A surfer survived being attacked and dragged underwater by a juvenile great white shark off the central California coast Sunday. + +California State Park Ranger Supervisor Robert Colligan said the attack by the 8-to-10 foot shark happened at around 11 a.m. local time at Montana de Oro State Park, approximately 200 miles northwest of Los Angeles. + +The San Luis Obispo Tribune, citing witnesses, identified the surfer as 50-year-old Kevin Swanson of Morro Bay, Calif. He was airlifted to a local hospital with non-life-threatening injuries to his right hip and thigh and a hospital spokesman said Swanson was in fair condition Sunday afternoon. + +Andrew Walsh, who was surfing with Swanson, told the paper that the shark swam up from underneath Swanson’s board and grabbed him with no warning. Walsh added that Swanson surfaced after several seconds, yelled “shark attack!”, and began paddling to shore. + +Before Swanson got out of the water, Walsh said, he fashioned a tourniquet from his surfboard’s leash cord. Two doctors who happened to be walking on the beach at the time examined him and determined that no arteries were hit. + +“We’re really blessed that he was still able to get himself to shore,” Walsh said. “I was a few feet behind him, and we grabbed him and got him … up on the sand, and very quickly these doctors were there, helping out and calling 911.” + +The beach remained open, but signs will be posted for three days warning the public of the attack, Colligan said. He noted that if there is another shark sighting, the signs will remain up for another three days. + +Sharks are native to the area, and Colligan said that they are spotted several times a year. He added that attacks like this are rare. + +A woman swimming with seals was killed by a shark in 2003 about 10 miles south of the most recent attack, Colligan said. \ No newline at end of file diff --git a/tests/data/text/ok.co.uk1.txt b/tests/data/text/ok.co.uk1.txt new file mode 100644 index 00000000..c92f5731 --- /dev/null +++ b/tests/data/text/ok.co.uk1.txt @@ -0,0 +1,19 @@ +WE can't keep up with all the engagements happening at the moment. + +And it seems that there could be another one to add to our list. + +Sean Penn has reportedly popped the question to Charlize Theron. + +The I Am Sam actor is said to have proposed while on a romantic trip to Paris in November. + +US sources claimed that the 54-year-old was keen to take his relationship with the mum-of-one to the next level. + +A source added that Charlize isn't yet wearing a ring to symbolise their union. + +They said: "There's no ring, but they are committed." + +The pair have been friends for a long time but only became romantically involved last year. + +It will be the third time Sean has walked down the aisle. He was previously married to Madonna and split from second wife Robin Wright in 2010. + +The couple had two children together, Hopper and Dylan. diff --git a/tests/data/text/ok.co.uk2.txt b/tests/data/text/ok.co.uk2.txt new file mode 100644 index 00000000..4add2eb9 --- /dev/null +++ b/tests/data/text/ok.co.uk2.txt @@ -0,0 +1,19 @@ +EastEnders are set to show a controversial plot line in the near future. + +Kat Moon will discover that her late uncle Harry is a serial sex offender. + +Alfie Moon's wife was groomed by the relative from a young age. + +As a result of rape, she gave birth to her daughter Zoe Slater at the age of 13. + +However, Kat pretended to be her sister until the secret was revealed years later after Zoe said she wanted to live with Harry in Spain. + +Police will turn up at Kat's house telling her that a number of victims have come forward with allegations against Harry, despite him passing away 10 years ago. + +Kat, played by Jessie Wallace, will be drawn to helping the investigation after discovering that she has been left money in his will. + +She then confides in her husband who believes that the heart-to-heart means a reunion is possible. Struggling to cope, Kat has a one-night stand in an attempt to take her mind off the situation. + +A show insider told The Sun: "EastEnders has a rich history of tackling difficult social issues and Kat's continued story is one of these." + +Played by the late Michael Elphick, Harry passed away off screen when he suffered a heart attack. diff --git a/tests/data/text/oregonlive.com1.txt b/tests/data/text/oregonlive.com1.txt new file mode 100644 index 00000000..ed4195cd --- /dev/null +++ b/tests/data/text/oregonlive.com1.txt @@ -0,0 +1,33 @@ +BEAVERTON -- The Comcast Foundation has awarded $355,760 in grants to 31 nonprofit organizations in Oregon and Southwest Washington. + +The grants support programs aimed at the Comcast Foundation's areas of focus -- expanding digital literacy, promoting community service, and building tomorrow's leaders. + +Local recipients also include Junior Achievement of Oregon & SW Washington, Boys and Girls Club of Albany and Hacienda Community Development Corporation of Portland. + +PORTLAND -- NW Natural's Corporate Philanthropy Fund is contributing nearly $35,000 to local nonprofits that help children and families in need. + +The fund, supported by shareholders, is making the following donations: + +YAMHILL COUNTY -- The Yamhill County Sheriff's Office Project Lifesaver Team is the recipient of a bi-annual grant from the Alzheimer's Foundation of America for $5,000. + +This grant will enhance the current Project Lifesaver Program by upgrading certain equipment and providing up to 15 new transmitters for new clients. + +The Yamhill County Sheriff's Office and the Yamhill County Sheriff's Search and Rescue started Project Lifesaver in Yamhill County in 2007 thanks to many donations from both private individuals and organizations. + +The primary mission of Project Lifesaver is to provide timely response to save lives and reduce potential injury for adults and children who wander due to Alzheimer's, autism, and other related conditions or disorders. The program consists of a bracelet transmitter, which is place on a client's wrist or ankle, and a specialized receiver that the Project Lifesaver team uses to track the specific radio frequency that the transmitter emits on a constant basis. + +The Project Lifesaver Team is made up of volunteers from the Search and Rescue Team along with deputies who are trained on locating the clients and how to deal with the clients once they are located. Once a month, a member of the team visits the clients to change batteries and bands that hold the transmitter in place. + +Project Lifesaver aids Search and Rescue in reducing time searching for a client that goes missing. To date they have been 100 percent in finding clients who left their homes. Generally the searches last under 10 minutes once the team is on scene. Yamhill County has had 19 searches since 2007 for clients on the Project Lifesaver program. Project Lifesaver is provided free of charge to Yamhill County residents. + +PORTLAND -- Pam Adkins and Mary Kay Plass have opened a new bottle shop and tap room at 4214 N. Mississippi Ave. + +The Beer City Bottle Shop sells bottled craft beer, hard cider and sodas as well as offering eight rotating draft selections all produced on the west coast. + +The tap room seats 18 and offers a small food menu of Panini's and snack plates. + +One of the most popular offerings so far, said the owners, is the Pick 6, where customers create their own six-pack of any 12 ounce bottles or cans for $10. + +The shop is open from 3 to 9 p.m. Monday through Thursday, from 1 to 10 p.m. Friday and Saturday and from 1 to 8 p.m. Sunday. + +PORTLAND -- John W. Houston, managing director of the Financial Institutions Division of Raymond James Financial Services, Inc., announced that OnPoint Community Credit Union, headquartered in Portland, has partnered with and will offer investment and wealth management services to its clients through Raymond James. \ No newline at end of file diff --git a/tests/data/text/oregonlive.com2.txt b/tests/data/text/oregonlive.com2.txt new file mode 100644 index 00000000..4f526533 --- /dev/null +++ b/tests/data/text/oregonlive.com2.txt @@ -0,0 +1,13 @@ +A 36-year-old part-time missionary who served a year in a Cambodian prison for sexually abusing boys in an orphanage pleaded not guilty on Monday in a Eugene courtroom to a rarely imposed federal charge of engaging in illicit sexual conduct in a foreign place. + +Daniel Stephen Johnson faces a potential 30-year prison term on the new charge, which accuses him of having sex with a boy in the Kingdom of Cambodia sometime between Nov. 28, 2005, and Oct. 12, 2006. + +A federal grand jury indicted Johnson on Dec. 10. He's awaiting trial in the Lane County Jail. + +Johnson served a one-year sentence in Cambodia for sexually abusing boys in his care at an orphanage, The Register-Guard newspaper reported. He worked as a Christian missionary in the Southeast Asian country for about a decade, according to the Cambodia-based anti-pedophile group Action pour les Enfants. + +A 2003 federal law aimed at preventing child abuse made it a crime for any U.S. citizen to have illegal sexual contact with a minor in a foreign country. + +More than a decade ago, Johnson was accused in Oregon of molesting three children in his sister's care. + +Lincoln County prosecutors dismissed charges after investigators began to doubt the alleged victims' statements, according to a 2003 article in the Yamhill Valley News-Register. \ No newline at end of file diff --git a/tests/data/text/parsely.com1.txt b/tests/data/text/parsely.com1.txt new file mode 100644 index 00000000..039c5216 --- /dev/null +++ b/tests/data/text/parsely.com1.txt @@ -0,0 +1,25 @@ +Yesterday, Facebook announced a number of new tools for publishers, as part of their on-going relationship to encourage a high volume of quality content that will keep us scrolling through our newsfeeds. + +At the bottom was a short mention: + +ICYMI – this is a fix for direct traffic (sometimes called “dark social”) for traffic coming through the mobile app that sometimes dropped the referral information. + +Anyone who has looked at analytics, has seen “direct” as a traffic source. There tends to be a fair amount of confusion around this section, as it encompasses anything that analytics can’t track. + +Traditionally, publishers assumed direct traffic was people bookmarking their home page or typing in the address directly in the search bar. However, since Parse.ly looks at analytics by a post-by-post basis, we can’t make that assumption, as most people are not typing in new article post URLs “directly” into their browser. + +In 2012, Alexis Madrigal wrote a story about these sources, terming them “dark social”. (This post is worth re-reading for some fun facts like “Only about four percent of total traffic is on mobile at all.”) In summary, he supposed that since these were links that were being shared from person-to-person, this kind of traffic should still be thought of as social sharing. + +Parse.ly’s CTO, Andrew Montalenti, explained some other areas that “dark” traffic can come from – he points out that it’s not all social; some traffic might also be explained by search and sites that block their referral information from being passed through or programs used by people protecting their browsing (especially in the wake of Snowden/NSA revelations). + +Though many of those sources will probably continue to be “dark”, a lot of us in the analytics community realized a while ago that Facebook was sending some of this direct traffic. The Guardian’s Ophan architect, Graham Tackley, showed off charts at ONA this year on how The Guardian saw its direct traffic spike in association with Facebook spikes. + +We saw a similar pattern, and called up Facebook a few months ago. As the massive social network makes a concerted effort to work with publishers, they were concerned at the differences we saw compared to their own data, and we shared some of our aggregate data with them as they worked to fix the referral issue. + +The announcement above acknowledged the first fix, and we’ll continue to monitor the impact that this is making in our system. + +What does this mean for your traffic? We checked the amount of traffic that currently comes into posts as direct (aka “dark social”) but is actually Facebook by looking at the user agents and found in one sample that 11% of total traffic coming was from Facebook. (And yes – that means that almost 50% of all external traffic referrers in this sample came from Facebook, wow.) + +Our analysis also checked out how much of that dark traffic came in from Android vs. iOS. We’ll be keeping track of this for the next few weeks, and we expect to see those numbers go down as the app fixes roll out and people update their apps. + +We know that people have a lot of questions about this, and we’re happy to provide additional information or answer questions about it – let us know in a comment, or shoot us a note. \ No newline at end of file diff --git a/tests/data/text/parsely.com2.txt b/tests/data/text/parsely.com2.txt new file mode 100644 index 00000000..29612a10 --- /dev/null +++ b/tests/data/text/parsely.com2.txt @@ -0,0 +1,49 @@ +Though we hear about pageviews going up at many of the large online publishers, digital media companies still struggle to see a correlating increase in revenue. What’s the disconnect? Methods to increase almost any metric, including pageviews and visitors, don’t always focus on long-term benefits to the people that matter most when it comes to generating revenue: the audience. + +Think of metrics like the Force. Sure, they have a dark-side: user-unfriendly slideshows to increase clicks, or enlarged images for more scrolling and time engaged. + +But you can use the “Force” of metrics for good: to understand your audience and create better experiences for them. Metrics can arm anyone in digital media, from editors to business executives with with vital information about their readers’ preferences and behaviors. When it comes to audience insights, aim to be Luke Skywalker, not Darth Vader. + +Data allows publishers today to see where readers come from and how they found an article or story. This tells you how readers already spend their time, what they care about, and can give you a good idea of who they might be based on some basic demographics about the referral networks. + +Analytics that tell you where your traffic comes from on a story by story or author by author basis will give you the clearest picture of your audience. For example, here’s the referral sources for articles written by our CTO on the Parse.ly blog next to the referral sources for articles written by our Director of Marketing: + +Looking at this, we know that readers coming to our blog for articles about technology (Apache Storm overviews, information on our open-source technology streamparse) tend to search for these technologies, or come from technology-based aggregation sites like Hacker News, Lobsters and tend to congregate on Twitter. + +Readers coming to articles written by our Director of Marketing tend to use more social networks, like LinkedIn and Facebook as well as Twitter, and are active in the content and journalism communities, as indicated by Contently and Poynter.org’s presence. + +We consider what people on these sites and networks want to learn or read about when we write new blog posts or create whitepapers and ebooks. We think about what kind of information we have that might be useful and interesting to someone that also reads about content marketing or regularly participates in tech conversations. It’s also a great gut-check to see if the audience we intended to reach is actually seeing our stories. + +Access to internal data allows you to focus on your own readers, but what about understanding your broader competitive set? Where are their readers coming from? + +Parse.ly publishes a quarterly Authority Report that includes a summary of refferal traffic based on billions of pageviews on our network of digital publishing sites. In the most recent edition, we compared referral traffic on a year-over-year basis (Aug. 2013-Aug. 2014). + +Overall, the data told the story of the rise of social and the constricting of all other categories. Since search was the largest, it had the most landshare to lose, but in general, social’s rise was at the expense of all other categories. + +See the full breakdown of top twenty five referrals and how they changed when you subscribe to the report. + +Most of social’s gains came directly from the increase in Facebook traffic to sites. + +Facebook made a huge jump, going from 12% of overall traffic referrals to over 20% (22.7% to be precise). This is similar to findings of other vendors and has recently spurred a torrent of concern over how much influence the social giant has over readers. + +For instance, Shareaholic found almost exactly the same percentage increase for Facebook referrals over a similar time (they tracked Sept 2013 through Sept 2014). + +However, our network saw Twitter and Stumbleupon as the two highest referrers after Facebook. Pinterest didn’t make a mark until four, though we did see the directional changes in growth match for all three networks. + +Even at this scale of billions of referrers, the types of audiences that the Parse.ly network sees versus the Shareaholic network likely accounts for these differences. + +Say what you will about the rise of social, search still eeks out more overall web visits. Though we did see the two come toe-to-toe back in January, search has climbed back up into the dominate position. + +Google didn’t even lose any ground to the Facebook growth – it gained almost two percentage points as a referral source to news sites. + +One thing to note in our data: the search traffic shown controls for the “brand search” effect. Brand search is typing a brand name i.e. “Business Insider” into Google Chrome’s browser bar or in a search field instead of typing the domain directly or clicking on a bookmark. In those cases, though the referrer tracks as “search,” but it doesn’t accurately describe the intention of the searcher; they already knew where they wanted to go. + +The data in the Authority Report only considers post URLs, not landing pages or homepages. This means that every view we account for came directly to an article, video or story. + +Our data saw almost all aggregators sites account for less of the overall referral pie for traffic. One theory why? Over the past year, digital news outlets have started to embrace more aggregation on their own sites. Huffington Post and Gawker have a strong reputation for aggregating other outlets stories; the New York Times now links out to others’ top stories on their homepages and in their apps. + +RSS readers may have also suffered from social’s gains. Feedly, which was poised to take over the RSS game after Google Reader was shuttered dipped in the past year. Mobile apps and social media feeds provide similar functionality combined with better social/mobile elements. + +Overall, this sample of sites saw a 30% increase of total external traffic, bolstered by strong social and search networks, but also ultimately aided by an increasingly larger set of long-tail sites. Does the large increase in this group point to more reliance on paid traffic distribution through networks like Outbrain, Taboola and their ilk? + +It certainly points to the fact that digital publishers have to be more vigilant than ever about where their audiences live online, how they’re finding the content they read and how the publishers can craft cohesive strategies across their entire organization that use data on those activities to their advantage. \ No newline at end of file diff --git a/tests/data/text/pe.com1.txt b/tests/data/text/pe.com1.txt new file mode 100644 index 00000000..d1d78806 --- /dev/null +++ b/tests/data/text/pe.com1.txt @@ -0,0 +1 @@ +The content you are looking for is unavailable.Please use the search box at the top of this page to find the information you were seeking. \ No newline at end of file diff --git a/tests/data/text/pe.com2.txt b/tests/data/text/pe.com2.txt new file mode 100644 index 00000000..50decff5 --- /dev/null +++ b/tests/data/text/pe.com2.txt @@ -0,0 +1,69 @@ +DEVELOPMENT IN WINE COUNTRY The story: After four years of meetings, debate and revision, the Riverside County Board of Supervisors in March adopted a comprehensive Temecula Valley Wine Country Community...... + + + +LAKE ELSINORE: Grants to help city improve two parks Two small Lake Elsinore parks in and near the city’s historical downtown district will be refurbished thanks to more than $1 million in grant money. Much of the $1.1 million will be spent on...... + + + +For many people, today is first and foremost a day of worship. Across Southwest Riverside County, a cursory Internet search found more than 160 houses of worship. I’m sure there are more. But the...... + + + +One person was in critical condition and four dogs were rescued after a head-on crash on the Ortega Highway west of Lake Elsinore, fire officials said.... + + + +Attendance boundaries for a number of campuses in the Lake Elsinore Unified School District would be altered in the coming months if proposed changes are ultimately accepted by board members.... + + + +Los Angeles prosecutors on Tuesday declined to file any charges against Bill Cosby after an Inland woman recently claimed the comedian molested her around 1974.... + + + +As his alleged mission to defame Costa Mesa councilmen unraveled in 2012, a private investigator tried to get his story straight with the law firm that hired him on behalf of the local police union,...... + + + +December storms have given a much-needed boost to some Inland lakes, but it will take much more rain to bring reservoirs back to pre-drought levels, water suppliers said.... + + + +LOS ANGELES – A lawyer for a Riverside County woman who says Bill Cosby sexually abused her when she was 15 stated in a recent court filing that he has interviewed two witnesses who corroborate her...... + + + +CORRECTION: A previous version of this story contained incorrect information concerning the value of the former Jean Hayman Elementary School campus. It was appraised at $2.5 million in 2011. ... + + + +The Ortega Highway above Lake Elsinore was closed for seven hours Sunday after a rock slide left 6- to 8-foot boulders strewn across the road.... + + + +When Lake Elsinore Police Chief Leonard Hollingsworth looked at his city’s line in recently released nationwide crime statistics compiled by the FBI for 2013, he received a pleasant surprise.... + + + +Lake Elsinore’s Police Department is beefing up patrols in shopping areas within the city in an effort to provide greater security during the holidays. Crime, especially property theft, tends to...... + + + +School boards in Southwest Riverside County named new leaders at their annual organizational meetings this week. Lake Elsinore Unified School District’s new board president is Heidi Matthies Dodd...... + + + +Two schools in Lake Elsinore -- Cottonwood Canyon Elementary School and Canyon Lake Middle School -- were locked down for a portion of the school day Thursday after school officials reported hearing...... + + + +A highly regarded skateboard park designer and builder will construct one in southern Lake Elsinore after the City Council hired the company this week. The council voted 5-0 to award a $402,000 contract...... + + + +In its annual reorganization of responsibilities, Lake Elsinore’s City Council named Steve Manos to serve as mayor for the next year. In selecting Manos on Tuesday to replace Natasha Johnson, the...... + + + +Canyon Lake residents interested in serving the last two years of a vacant seat on the City Council have until Jan. 5 to turn in their applications, officials announced. City Clerk Ariel Hall opened the...... \ No newline at end of file diff --git a/tests/data/text/pewresearch.org1.txt b/tests/data/text/pewresearch.org1.txt new file mode 100644 index 00000000..57086b91 --- /dev/null +++ b/tests/data/text/pewresearch.org1.txt @@ -0,0 +1,25 @@ +Unauthorized immigrants from Mexico account for two-thirds of those who will be eligible for deportation relief under President Obama’s executive action, even as they account for about half of the nation’s unauthorized population, according to a new Pew Research Center analysis. + +The new action, which mainly applies to unauthorized immigrant parents of U.S. citizen or legal permanent resident children, would benefit those born in Mexico more than any other country of origin group. According to the Pew Research analysis, 44% of unauthorized immigrants from Mexico could apply for deportation protection under the new programs, compared with 24% of those from other parts of the world. + +President Obama’s new programs could affect about 4 million total unauthorized immigrants who will be eligible for deportation protection and a three-year work permit. The largest group — at least 3.5 million, according to Pew Research estimates of 2012 data — consists of unauthorized immigrant parents who have lived in the U.S. for at least five years and have children who either were born in the U.S. or are legal permanent residents. Of these, about 700,000 have adult children and the remaining 2.8 million have children younger than 18. + +The new policy also expands eligibility for the president’s Deferred Action for Childhood Arrivals (DACA) program that benefits young adults brought to the U.S. illegally as children. The program would allow an additional 330,000 people, according to our estimates, to apply for and receive temporary deportation relief. Previously, the program was available only to those up to age 30, but the executive action would lift that age cap. In addition, the program would allow immigrants who arrived as children illegally before Jan. 1, 2010 to become eligible, expanding the program beyond the original June 15, 2007 cutoff date. + +In some cases, the Obama administration’s estimates of how many would be affected differ from those calculated by Pew Research. For example, the government estimates that more than 4 million parents of U.S. citizen children or legal permanent residents could apply for relief compared with our 3.5 million figure. One possible difference is that the data Pew Research uses only includes parents who live with their children. + +While work permits and deportation relief will be available, those covered by the programs will not be eligible for certain government benefits, including subsidies for health care under the Affordable Care Act. + +Among the other policy changes announced in the president’s action are an increased number of visas for skilled workers and spouses of green card holders. There are several changes, including immigration enforcement that will now focus on recent arrivals and serious and repeat criminal offenders. + +Even though about 4 million unauthorized immigrants may be eligible for deportation relief, it remains to be seen how many people will apply for and receive it. Response to the existing DACA program may offer a clue. The 2012 program allowed an estimated 1.1 million people ages 30 or younger to become eligible for deportation relief and receive a two-year work permit. But through June 30, 2014, only about 712,000 applied. + +The general public disapproves of Obama taking executive action on immigration, by a margin of 48% to 38%, according to a NBC News/Wall Street Journal survey conducted this month. While Americans may not generally support an executive action, the poll found 57% favor a pathway to citizenship for undocumented immigrants, with support increasing to 74% when respondents are given a scenario in which a pathway requires paying fines, back taxes and taking other steps. + +According to a recent Pew Research Center survey, Hispanics place a priority on the relief from deportation offered by the executive action. By 56% to 35%, Hispanics said it is more important that unauthorized immigrants be able to live and work in the U.S. without threat of deportation than have a pathway to citizenship. + +The estimated 4 million covered under the executive action are in addition to an estimated 1.5 million unauthorized immigrants who are eligible for temporary relief from deportation through either the Deferred Action for Childhood Arrivals program or having Temporary Protected Status. + +The president’s executive action offers deportation relief for the largest number of unauthorized immigrants in recent history. However, the current action does not allow unauthorized immigrants the opportunity to obtain permanent residency or citizenship. Only Congress has the authority to offer a path to legal status, which means those eligible for relief under the new policy remain unauthorized. If a future administration reverses course or if Congress passes an immigration law, the protection from deportation could be taken away. + +Correction: This posting has been updated with a revised total of 3.9 million unauthorized immigrants affected by the president’s action, and other related revised estimates. A previous version of this post referred to the president’s implementation of his new immigration policy as an executive order. He has taken executive action. \ No newline at end of file diff --git a/tests/data/text/pewresearch.org2.txt b/tests/data/text/pewresearch.org2.txt new file mode 100644 index 00000000..11df2983 --- /dev/null +++ b/tests/data/text/pewresearch.org2.txt @@ -0,0 +1,13 @@ +Gasoline prices have been dropping since midsummer, and consumers’ confidence about the economy has been on the rise. Could there be a connection? + +According to a new Pew Research Center report, 70% of Americans now report hearing mostly good news about gas prices, up from just 15% in August. In truth, gas prices have been falling for months: As of Monday, the national average price of a gallon of self-serve regular was $2.554 — $1.15 less than in late June (representing a nearly one-third drop), according to the U.S. Energy Information Administration. That’s the cheapest gas has been since October 2009. Also, Brent crude oil has fallen more than $45 a barrel since June and is now below $59 a barrel for the first time since May 2009. + +Meanwhile, the University of Michigan’s Index of Consumer Sentiment stood at 88.8 in November, up 7 points since July and its highest reading since mid-2007. The preliminary December reading is even higher, at 93.8, though that’s still subject to revision. + +We plotted the monthly consumer-sentiment index against the monthly average price of regular gas (adjusted for inflation) and found a moderately strong negative correlation — that is, consumer sentiment rose as pump prices fell. That aligns with previous research: For example, a 2012 paper from two researchers at Loyola University Maryland and the University of Maryland Baltimore County not only found an inverse correlation between gas prices and consumer sentiment, but used causality testing to conclude that price changes predicted sentiment changes and not the other way around. + +A 2009 paper in the Journal of Monetary Economics, which also found a cause-and-effect relationship, analyzed the University of Michigan survey results in more detail to try to get at the dynamics of how consumers respond to energy-price changes. After a major one-time price jump, the researchers found, “an increasing number of people expect general business conditions and their personal financial situation to deteriorate over the coming year in response to an unanticipated loss in purchasing power….[T]he index for buying conditions for large household goods falls by 1.9 points. An even larger decrease is observed for vehicles….Increased pessimism about buying conditions in response to purchasing power losses is associated with expectations of higher unemployment, higher interest rates, and lower real family income.” + +Those attitudinal changes have real-world consequences, though not large ones. The authors of the JME paper, Paul Edelstein and Lutz Kilian, concentrated mainly on unexpected increases, rather than drops, in energy prices; they estimated that a one-time, 1% increase in energy prices was associated with a 0.08% fall in total real consumption a year later, based on 1988-2006 data. But that was considerably less than the 0.30% decline seen in the 1970-1987 data; energy price shocks, they concluded, have less effect on the overall economy than they used to. + +Edelstein and Kilian also estimated that one year after an unexpected, permanent increase of 25 cents a gallon in the price of gas, a typical household would have reduced its expenditures by $17 a month, with most of the adjustment coming in the first six months after the price increase. Concluded the researchers: “It takes repeated surprise increases in gasoline prices to generate large effects on household consumption, but over time the effects will add up.” \ No newline at end of file diff --git a/tests/data/text/pixable.com1.txt b/tests/data/text/pixable.com1.txt new file mode 100644 index 00000000..497b6f25 --- /dev/null +++ b/tests/data/text/pixable.com1.txt @@ -0,0 +1 @@ +Bad news for ladies, gentlemen and those who planned on splitting up a healthy, attractive couple: Sofia Vergara and Joe Manganiello are engaged. THEY ENGAGED. They getting married. Hitched. Tying the knot. They like-like each other. They do not want to date you. They only want to date themselves, together, from now until eternity. They have no plans to marry you in the near future. \ No newline at end of file diff --git a/tests/data/text/pixable.com2.txt b/tests/data/text/pixable.com2.txt new file mode 100644 index 00000000..011681cc --- /dev/null +++ b/tests/data/text/pixable.com2.txt @@ -0,0 +1 @@ +It’s hard to believe, but 2014 has come to an end and for better or worse 2015 is right around the corner. In 2014 we had some highs and some lows, but all in all it was an okay year. That being said there are still some pop culture trends that should never see the light of day again once the clock strikes midnight on December 31st. Don’t find yourself getting caught up in these trends we hope will stay behind once we move into 2015. The first “Sharknado” was okay. The second “Sharknado” was absurd and undoubtedly any other movies that have “Sharknado” in the title will be downright obnoxious. Yes, for the most part they’re fun to watch and make us think, but enough is enough. Some of these so-called social experiments are clearly staged and use innocent people for the sake of going viral. Yaaaaaaaaas we really do need to stop saying bae. Remember when Justin Bieber thought it was a good idea to revisit the 90s boyband era? Hopefully he gets some hair dye for Christmas and dyes it any other color but bleach blonde. We thought with the release of a new single off of Taylor Swift’s new album that we were done with the massive amounts of “Shake It Off” parodies, but it looks like we were wrong. Maybe next year we can pick a different song? Yes, “Frozen” was an excellent movie, but it’s time to let it go. For those who don’t know what normcore is it’s a fashion trend where people wear average looking clothing or whatever they can find. This contributes greatly to the next pop culture trend we hope stays in 2014. Where did this term even come from and why are we using it to describe average people who just happen to like everything in a non-ironic way? Cats are absolutely adorable — well sometimes — but it’s time for a new animal to take over the Internet in 2015. Goats maybe? “Peter Pan Live!” received a 59 percent on Rotten Tomatoes, which surprisingly was better than the 44 percent that “The Sound of Music Live!” received after it’s failed attempt to win over a large television audience. There was Solange, Beyonce and Jay Z and every time that Selena Gomez unfriended anyone of her famous friends and most recently the ongoing feud between Azealia Banks and Iggy Azalea. These are prime examples of celebrity drama that should be kept behind closed doors and does not need to be broadcast over the Internet. Kale was all the rage this past year, but it’s time for everyone to put down the vegetable and try some new leafy greens. ‘Keeping Up With The Kardashians” was so last year. Ever since Bruce and Kris called it quits, Kylie and Kendall grew up and Kourtney had yet another baby we just can’t seem to care about what the Kardashians are up to. It seems like everywhere you turn there’s a hashtag. Every TV show has one, Facebook has tried to incorporate them into their platform, they appear on clothing and as home decor. Please stop abusing the hashtag. Why does Hollywood feel the need to remake classics like “Annie,” “Godzilla” and “Teenage Mutant Ninja Turtles”? They rarely do as well as the originals. \ No newline at end of file diff --git a/tests/data/text/pixelmonkey.org1.txt b/tests/data/text/pixelmonkey.org1.txt new file mode 100644 index 00000000..99f0b6f8 --- /dev/null +++ b/tests/data/text/pixelmonkey.org1.txt @@ -0,0 +1,35 @@ +My first mobile device was a Palm V. I understood the power of mobile really early. I was 15 in 1999, when the Palm V was released. I first came across the Palm devices in 1998, when the Palm III came onto the scene. I never owned one, but played with the one my Dad owned, but barely used. + +This device was comically under-powered in retrospect. It had 2 megabytes of RAM, which had to be used as not only the working memory of the device, but also the storage. It had a 16 Hz processor, a 4-greyscale screen, and a stylus-driven interface. + +The Palm V was an amazing device. In lieu of the plastic of the Palm III, it had a finished anodized aluminum finish, very similar to the kinds of sleek devices we would only begin to regularly see in the last couple of years. It was nearly half the weight of its predecessor, and as thin as the stylus you used to control it. It had a surprisingly well-designed docking station (imagine this: since USB hadn’t yet been developed, it had to sync over the low-bandwidth Serial Port available on PCs at the time). + +I came across a time capsule of my Palm V usage. Since I was a web designer at this time, but since it was a “pre-web” era, I often put together little static HTML websites to demonstrate features and ideas to my friends. I did one of these for my Palm V. + +In a world without cell data plans, wifi networks, & laptop computers, my Palm V provided much of the same utility that is now spread across these various pieces of everyday technology infrastructure, but in a single device. I kept a unified to do list of my life, and a specialized task manager specifically for coursework and classes. In it, I tracked not just homework assignments and due dates grouped by class, but also my grades — so that I could prioritize work to optimize my grades. + +I think I only got my first cell phone a couple of years later — and they were nowhere near ubiquitous yet, so my contact manager was actually being used to store landline phone numbers and addresses of my friends — and likely, of local take-out / delivery restaurants. + +I sometimes tell my friends that despite the lack of infrastructure in school for it, I was indeed a “child of the Internet”. This was mostly thanks to AvantGo. + +When I was at home on my personal computer, I was already an avid web content reader from early pioneers like Salon.com, CNET, Wired, and BBC. But I couldn’t easily read their content at school. AvantGo was an early, popular service for Palm devices that let you download and sync “content channels”. Of course, this is way before RSS/Atom. Channels were really mobile-optimized static HTML websites that each provider would put out daily with a snapshot of their latest content. In the screenshot above, you can see me reading an article from Salon.com dated March 30, 2002. + +In addition to officially supported channels that were put out by major publishers, you could also download and sync arbitrary web content. This was essentially an early version of Readability / Instapaper — in some ways, much clunkier, in other ways, more complete. The software would go to the website and try to mobile-optimize a site (stripping it down to basic HTML), while also crawling the relative links. For example, at the time I was very interested in participating in the Debian open source project, so I had AvantGo sync the Debian Policy Manual, which contained hundreds of HTML pages full of reference information. + +So, yes — when I was bored in Spanish or Math class, I’d pull up a political article from Salon.com, a tech piece about the 2000 tech bubble forming in Silicon Valley from Wired, or read over open source software development guidelines. I was a child of the Internet, and the Palm V kept me tethered in a pre-mobile world. + +What makes AvantGo so interesting, upon reflection, is that despite everything against it — under-powered mobile devices, no ubiquitous wireless Internet access, and a less standardized world-wide-web, it still satisfied a use case that has yet to be satisfied today. Namely, it provided a way for me to curate the “best sources of content” I found on the web, and sync and download all their content to provide a personalized, offline library of content. Instead, we have nothing but fragmentation today — individual native apps for each news publisher, RSS readers like Newsblur providing a way to get the latest updates from a group of them, and mobile web browsers for everything else. + +Obviously, devices running iOS and Android are finally starting to make ubiquitous computing a reality, and are providing less clunky ways to access web content. + +But I think it’s worth reflecting on what my Palm V had going for it nearly a decade ago that has yet to arrive in this new era. + +A great example of online/offline mastery that I witnessed in a mobile application is Spotify’s support for bringing an entire music playlist offline. Offers the best of both worlds. + +We’re certainly living in an age of ubiquitous, wireless Internet access, whether via cell towers or wifi. But that doesn’t mean that the smartest applications are those that exclusively rely on data access being present. In fact, making your application rely on data access will almost certainly make it slower than it could be. + +Due to our obsession with connectivity, I think many developers are forgetting about use cases where the mobile device is meant to be a smart accessory to our lives, rather than a portal to the web. These are no longer “thin clients” — instead, we should think of them as powerful handheld computers. Recent iPhone and Android devices feature dual-core processors at speed above 1Ghz and 1GB of RAM. In 1998, typical desktop computers were slower than this (e.g. a single-core P3 600 Mhz w/ 128MB of RAM and 10GB of disk storage). + +Yes, we’ve put the power of a desktop computer into the palm of a user’s hand. But, are we really tapping this opportunity? + +Andrew Montalenti (aka pixelmonkey, amontalenti) is the co-founder and CTO of Parse.ly, which provides data insights to the web's best publishers. You can follow him on Twitter or LinkedIn. If you liked this post, you should subscribe to his free e-mail newsletter, Fire and Motion, which discusses technology startups and business from the trenches. This entry was posted on Sunday, December 30th, 2012 at 3:00 pm and is filed under Personal, Technology. You can follow any responses to this entry through the RSS 2.0 feed. You can leave a response, or trackback from your own site. \ No newline at end of file diff --git a/tests/data/text/pixelmonkey.org2.txt b/tests/data/text/pixelmonkey.org2.txt new file mode 100644 index 00000000..87cfd43d --- /dev/null +++ b/tests/data/text/pixelmonkey.org2.txt @@ -0,0 +1,23 @@ +Parse.ly made its funding announcement — a $5M series A, led by Grotech Ventures and with participation from FundersClub, Blumberg Capital, and ff Venture Capital. Read on for the full list of links to our coverage. + +Here is some coverage from around the web: + +TechCrunch: Parse.ly Raises $5M For Predictive Analytics Platform That Helps Media Companies Decide What To Publish +MediaPost: Parse.ly Raises $5 Million, Invests In Core Tech, Data +The Daily Startup @ Wall Street Journal, VC Dispatch +Washington Business Journal: Grotech leads $5 million round in analytics startup Parse.ly +MarketingLand: Facebook Driving More Than 2x Twitter Traffic To Parse.ly’s News Clients +GigaOM: Feedly dominating the post-Reader world, and other web-publishing insights from Parse.ly +peHUB: Parse.ly raises $5M +Parse.ly Blog: Parse.ly Investors Back the New Content Performance Authority +The Next Web: A month after Google Reader vanishes, Feedly ranks as the top RSS traffic referrer +VentureBeat: Parse.ly nabs $5 million to continue getting you to click on Grumpy Cat +PSFK: Platform Tells Bloggers What To Write About Next [Video] +memeburn: Life after Google Reader: yes, Feedly is still winning +Brafton: Yahoo might be top web property, but Google still drives most traffic +eMediaVitals: Publishing analytics startup Parse.ly releases first report on traffic drivers +IPG Media Lab: Parse.ly Does Intelligent Tagging +ContentStandard: Parse.ly Delivers Content Optimization for Online Publishers +More coming soon! + +Andrew Montalenti (aka pixelmonkey, amontalenti) is the co-founder and CTO of Parse.ly, which provides data insights to the web's best publishers. You can follow him on Twitter or LinkedIn. If you liked this post, you should subscribe to his free e-mail newsletter, Fire and Motion, which discusses technology startups and business from the trenches. This entry was posted on Tuesday, September 3rd, 2013 at 12:53 pm and is filed under Startups, Technology, Uncategorized. You can follow any responses to this entry through the RSS 2.0 feed. You can leave a response, or trackback from your own site. diff --git a/tests/data/text/readwrite.com1.txt b/tests/data/text/readwrite.com1.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/text/readwrite.com2.txt b/tests/data/text/readwrite.com2.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/text/recipe.com1.txt b/tests/data/text/recipe.com1.txt new file mode 100644 index 00000000..3ce751f1 --- /dev/null +++ b/tests/data/text/recipe.com1.txt @@ -0,0 +1,17 @@ +1 medium onion, finely chopped +2 tablespoons butter +4 teaspoons all-purpose flour +1 1/2 cups milk +8 ounces sharp cheddar cheese, shredded (2 cups) +3 pounds russet potatoes, peeled and thinly sliced* +1 1/2 cups chopped fresh broccoli +1 tablespoon vegetable oil +8 eggs +2 tablespoons milk +6 slices bacon, crisp-cooked, drained, and crumbled +1 large tomato, chopped + +1. Preheat oven to 325 degrees F. In a medium saucepan cook onion in butter over medium heat for 4 minutes or until tender, stirring occasionally. Stir in flour, 1/2 tsp. salt, and 1/2 tsp. ground black pepper. Stir in milk; cook and stir until slightly thickened and bubbly. Stir in cheese until melted. +2. In a 3-quart baking dish, layer potatoes, then cheese sauce. Bake, covered, about 55 minutes, until potatoes are tender. +3. In a large skillet, cook broccoli in hot oil over medium heat for 5 minutes, until nearly tender, stirring frequently. In a large bowl, beat together eggs, water, 1/2 tsp. salt, and 1/4 tsp. ground black pepper. Pour over broccoli in skillet. Cook over medium heat, without stirring, until mixture begins to set on bottom and around edges. Using a spatula, lift and fold partially cooked egg so uncooked portion flows underneath. Cook 2 minutes more or until egg is cooked yet still moist. +Spoon over potatoes. Top with bacon and chopped tomato. Serve immediately. diff --git a/tests/data/text/recipe.com2.txt b/tests/data/text/recipe.com2.txt new file mode 100644 index 00000000..97043072 --- /dev/null +++ b/tests/data/text/recipe.com2.txt @@ -0,0 +1,29 @@ +With winter cooking in full swing, I’ve been making heartier meat dishes such as braises, stews, and roasts. One my favorite side dishes to accompany them is sweet potato mash. They’re less starchy than white ones and higher in nutrition, packed with loads of vitamin A. I liked the addition of parsnips in this recipe and added a bulb of celery root as well to give the dish another layer of earthy flavor. + +I began by scrubbing and peeling all of my root vegetables. Celery root can be a little tricky to work with since the skin is thick and bumpy, so I use the peeler first to remove as much of the exterior as possible. Then I trim off any missed spots with my knife. By cutting them all into 1-inch cubes, they’ll cook more evenly together in the pot. + +I transferred the vegetables to a large stock pot and covered them in water with a large pinch of salt. After about 15 minutes of cooking, they were fork-tender. I drained and transferred everything to a large bowl. + +Because I like mine with a little chunkier texture, I used my hand masher to mix them together. The butter and milk help to smooth out the mash, while the brown sugar and allspice round out the flavors. + +This dish can be made the day ahead and reheated right before serving. Be sure to add an extra pat of butter on top to finish. Try this root vegetable mash as a sweet, earthy alternative to classic mashed potatoes at your next meal. + +1/4 pound celery root, peeled and cut into 1-inch pieces + + 1 pound parsnips, peeled and cut into 1-inch pieces + + 2 pounds sweet potatoes, peeled and cut into 1-inch pieces + + 1/2 cup 2% milk + + 1/4 cup packed brown sugar + + 2 tablespoons unsalted butter + + 3/4 teaspoon salt + + 1/4 teaspoon ground allspice + +1. Combine parsnips and sweet potatoes in a medium pot; cover with 1 inch of cold water. Bring to a boil, then reduce heat to medium and simmer for 10 to 15 minutes, or until vegetables are tender and parsnips have lost their bitterness. + + 2. Drain cooked vegetables and immediately return to pot over low heat. Mash with milk, brown sugar, butter, salt, and allspice. \ No newline at end of file diff --git a/tests/data/text/reuters.com1.txt b/tests/data/text/reuters.com1.txt new file mode 100644 index 00000000..2a646f97 --- /dev/null +++ b/tests/data/text/reuters.com1.txt @@ -0,0 +1,17 @@ +People walk past the Bombay Stock Exchange (BSE) building in Mumbai May 13, 2014. + +(Reuters) - The BSE Sensex and Nifty were trading flat on Tuesday, as weaker regional shares offset optimism over additional reforms a day after the government passed an executive order to ease land-acquisition rules. + +Monday's announcement could kick-start hundreds of billions of dollars in stalled projects, and comes after separate orders were passed to implement coal and insurance reforms. + +Infrastructure stocks gained on the news with Larsen and Toubro (LART.NS) adding 1.1 percent. Lanco Infratech (LAIN.NS) rose 6.7 percent and GMR Infrastructure (GMRI.NS) was trading 1.5 percent higher. + +However, weak sentiment across the region on a sharp selloff in commodities overnight and political uncertainty in Greece left investors hesitant to take big bets. + +"The overall outlook is good. The government is moving in the right direction. Investors expect the reform process to continue, which will be a long-term positive for Indian equities," said Suresh Parmar, head, institutional equities at KJMC Capital Markets. + +The benchmark Sensex was up 0.03 percent, while the broader Nifty was trading 0.01 percent lower. + +Godrej Properties (GODR.NS) gained 5.4 percent after the company bought back shares held by private equity firm Sun-Apollo, and also announced a new project. + +Energy firms however fell, with Oil and Natural Gas Corporation (ONGC.NS) losing 1.4 percent and Reliance Industries (RELI.NS) falling 1.3 percent after crude prices fell to a five-year low. diff --git a/tests/data/text/reuters.com2.txt b/tests/data/text/reuters.com2.txt new file mode 100644 index 00000000..d8ae8f9d --- /dev/null +++ b/tests/data/text/reuters.com2.txt @@ -0,0 +1,61 @@ +1 of 4. Authorities monitor progress in the search for AirAsia Flight QZ8501 in the Mission Control Center inside the National Search and Rescue Agency in Jakarta December 29, 2014. + +(Reuters) - Countries around Asia on Tuesday stepped up the search for an AirAsia plane carrying 162 people that is presumed to have crashed in shallow waters off the Indonesian coast, with Washington also sending a warship to help find the missing jet. + +Soelistyo, head of Indonesia's search and rescue agency, told local television the search area between the islands of Sumatra and Borneo would be expanded. Authorities would also begin scouring nearby islands as well as coastal land on Indonesia's side of Borneo. + +So far the focus of the search has been the Java Sea. + +There have been no confirmed signs of wreckage from the Airbus A320-200 operated by Indonesia AirAsia, which disappeared in poor weather on Sunday morning during a flight from the Indonesian city of Surabaya to Singapore. + +The missing plane, which was carrying mainly Indonesians, could be at the bottom of the sea, Soelistyo said on Monday. + +The Java Sea is relatively shallow, making it easier to spot wreckage in the water, say oceanographers, but strong currents and winds in the area mean any debris would be drifting up to 50 km (31 miles) a day east, away from the impact zone. + +"The lesson that should be learned from MH370 is that you need to move quickly," said Charitha Pattiaratchi, an oceanographer at the University of Western Australia, referring to the Malaysia Airlines flight that went missing on March 8 during a trip from Kuala Lumpur to Beijing with 239 passengers and crew and which has not been found. + +Around 30 ships and 21 aircraft from Indonesia, Australia, Malaysia, Singapore and South Korea would search up to 10,000 square nautical miles on Tuesday, officials said. + +Indonesian Air Force spokesman Hadi Tjahjanto said authorities would investigate an oil spill sighted on Monday, although a separate possible slick turned out to be a reef. + +Searchers had investigated several areas where possible debris had been sighted in the water but had found nothing connected to the missing plane, Tjahjanto told Reuters. + +Authorities would also investigate reports by local fishermen of an explosion on Sunday morning off an island in the area, Tjahjanto added, although dynamite fishing is common in Indonesian waters. + +The U.S. military said the USS Sampson, a guided missile destroyer, would be on the scene later on Tuesday. + +"We stand ready to assist in any way possible," Pentagon spokesman Mark Wright said. + +What happened to Flight QZ8501, which had sought permission from Indonesian air traffic control to ascend to avoid clouds, is still a mystery. + +Online discussions among pilots have centred on unconfirmed secondary radar data from Malaysia that suggested the aircraft was climbing at a speed of 353 knots, about 100 knots too slow in poor weather, and that it might have stalled. + +While searchers had picked up an emergency locator signal off the south of Borneo, no subsequent signal was found, officials said. + +The plane, whose engines were made by CFM International, co-owned by General Electric and Safran of France, lacked real-time engine diagnostics or monitoring, a GE spokesman said. Such systems are mainly used on long-haul flights and can provide clues to airlines and investigators when things go wrong. + +Officials said the sea in the general search area was only 50 to 100 (150 to 300 feet) metres deep, which would be a help in finding the plane. + +"The Java Sea area where they are now searching isn't even an ocean, it's more of an inland sea," Erik van Sebille, a physical oceanographer at the University of New South Wales in Sydney told Reuters. + +"It's so shallow that they may just be able to spot the plane," said van Sebille, noting that sunlight travels through water up to about 100 metres. + +Oceanographer Pattiaratchi said debris would normally be expected to float on the surface for around 18 days before sinking. + +Three airline disasters involving Malaysian-affiliated carriers in less than a year have dented confidence in the country's aviation industry and spooked air travellers across the region. + +In the third incident, Malaysia Airlines Flight MH17 was shot down over Ukraine on July 17, killing all 298 people on board. + +On board Flight QZ8501 were 155 Indonesians, three South Koreans, and one person each from Singapore, Malaysia and Britain. The co-pilot was French. + +U.S. law enforcement and security officials said passenger and crew lists were being closely examined but so far nothing significant had turned up and that the incident was still regarded as an unexplained accident. + +The plane, which did not issue a distress signal, disappeared after its pilot failed to get permission to fly higher because of heavy air traffic, officials said. + +Pilots and aviation experts said thunderstorms, and requests to gain altitude to avoid them, were not unusual in that area. + +The Indonesian pilot was experienced and the plane last underwent maintenance in mid-November, the airline said. + +The AirAsia group, including affiliates in Thailand, the Philippines and India, had not suffered a crash since its Malaysian budget operations began in 2002. + +The plane's disappearance comes at a sensitive time for Jakarta's aviation authorities, as they strive to improve the country's safety reputation to match its status as one of the airline industry's fastest growing markets. diff --git a/tests/data/text/reuters.com3.txt b/tests/data/text/reuters.com3.txt new file mode 100644 index 00000000..0ddab58d --- /dev/null +++ b/tests/data/text/reuters.com3.txt @@ -0,0 +1,47 @@ +1 of 18. A member of an Indonesian Hercules C130 aircrew watches through a window while monitoring the Belitung Timur sea during search operations for AirAsia flight QZ8501 near Belitung island, December 29, 2014 in this photo taken by Antara Foto. + +(Reuters) - Indonesian rescuers saw bodies and luggage off the coast of Borneo island on Tuesday and officials said they were "95 percent sure" debris spotted in the sea was from a missing AirAsia plane with 162 people on board. + +Indonesia AirAsia's Flight QZ8501, an Airbus A320-200, lost contact with air traffic control early on Sunday during bad weather on a flight from the Indonesian city of Surabaya to Singapore. + +Pictures of floating bodies were broadcast on television and relatives of the missing gathered at the crisis centre in Surabaya were shown weeping, their heads in their hands. + +Media quoted an air force official earlier as saying one suspected body, luggage and a life vest were among the debris in the Java Sea. + +"As we approached, the body seemed bloated," said First Lieutenant Tri Wibowo, who was on board a Hercules aircraft, was quoted by the Kompas.com website as saying. + +Search and Rescue Agency chief Soelistyo told reporters he was "95 percent sure" the debris was from the missing plane. + +Djoko Murjatmodjo, acting director general of air transportation at the transportation ministry, told reporters some of the debris spotted was red and white, AirAsia's colours. + +"It's probably from the aircraft," he said. + +About 30 ships and 21 aircraft from Indonesia, Australia, Malaysia, Singapore, South Korea and the United States were searching up to 10,000 square nautical miles on Tuesday. + +The plane, which did not issue a distress signal, disappeared after its pilot failed to get permission to fly higher to avoid bad weather because of heavy air traffic, officials said. + +Pilots and aviation experts said thunderstorms, and requests to gain altitude to avoid them, were not unusual in that area. + +The Indonesian pilot was experienced and the plane last underwent maintenance in mid-November, the airline said. + +Online discussion among pilots has centred on unconfirmed secondary radar data from Malaysia that suggested the aircraft was climbing at a speed of 353 knots, about 100 knots too slow, and that it might have stalled. + +The plane, whose engines were made by CFM International, co-owned by General Electric and Safran of France, lacked real-time engine diagnostics or monitoring, a GE spokesman said. + +Such systems are mainly used on long-haul flights and can provide clues to airlines and investigators when things go wrong. + +Three airline disasters involving Malaysian-affiliated carriers in less than a year have dented confidence in the country's aviation industry and spooked travellers across the region. + +Malaysian Airlines Flight MH370 went missing on March 8 on a trip from Kuala Lumpur to Beijing with 239 passengers and crew on board and has not been found. On July 17, the same airline's Flight MH17 was shot down over Ukraine, killing all 298 people on board. + +On board Flight QZ8501 were 155 Indonesians, three South Koreans, and one person each from Singapore, Malaysia and Britain. The co-pilot was French. + +U.S. law enforcement and security officials said passenger and crew lists were being examined but nothing significant had turned up and the incident was regarded as an unexplained accident. + +Indonesia AirAsia is 49 percent owned by Malaysia-based budget carrier AirAsia. + +The AirAsia group, including affiliates in Thailand, the Philippines and India, had not suffered a crash since its Malaysian budget operations began in 2002. + +India is waiting to know what went wrong with the missing plane and will investigate if AirAsia India is following all safety procedures, a senior Indian aviation ministry official told Reuters. AirAsia India, a joint venture of the Malaysian carrier, started flying this year and is expanding operations. + +The plane's disappearance comes at a sensitive time for Indonesia's aviation authorities, as they strive to improve the country's safety reputation to match its status as one of the airline industry's fastest growing markets. diff --git a/tests/data/text/reuters.com4.txt b/tests/data/text/reuters.com4.txt new file mode 100644 index 00000000..49bc4497 --- /dev/null +++ b/tests/data/text/reuters.com4.txt @@ -0,0 +1,19 @@ +1 of 2. A general view of Gartnavel General Hospital is seen in Glasgow, Scotland December 29, 2014. + +(Reuters) - A healthcare worker has been diagnosed with Ebola a day after flying home to Glasgow from Sierra Leone, the Scottish government said on Monday. + +The patient is being treated in isolation at Glasgow's Gartnavel Hospital, having flown back to Scotland's largest city late on Sunday on a British Airways flight via Casablanca in Morocco and London's Heathrow. + +"All possible contacts with the patient are now being investigated and anyone deemed to be at risk will be contacted and closely monitored," the Scottish government said in a statement. + +"However, having been diagnosed in the very early stages of the illness, the risk to others is considered extremely low." + +The patient, whom BBC sources described as a female aid worker, will be transferred to a high-level isolation unit in the Royal Free hospital in London. + +British Prime Minister Cameron has been informed, the Scottish government added. + +In August, another British aid worker, William Pooley, contracted the disease after working Sierra Leone. He was discharged in September after treatment at the Royal Free hospital. + +With more than 9,000 cases, Sierra Leone now accounts for nearly half of the known cases of Ebola in this year's West African outbreak, the worst ever. Neighbouring Liberia and Guinea have also been badly hit. + +The World Health Organization on Monday said the number of people infected by Ebola in Liberia, Sierra Leone and Guinea -- the worst affected by the outbreak -- has passed 20,000, with more than 7,842 deaths in the epidemic so far. diff --git a/tests/data/text/reuters.com5.txt b/tests/data/text/reuters.com5.txt new file mode 100644 index 00000000..ce495f28 --- /dev/null +++ b/tests/data/text/reuters.com5.txt @@ -0,0 +1,31 @@ +People are silhouetted as they pose with laptops in front of a screen projected with a Google logo, in this picture illustration taken in Zenica October 29, 2014. + +Large numbers of Gmail Web addresses were cut off in China on Friday, said GreatFire.org, a China-based freedom of speech advocacy group. Users said the service was still down on Monday. + +"I think the government is just trying to further eliminate Google's presence in China and even weaken its market overseas," said a member of GreatFire.org, who uses a pseudonym. + +Google's own Transparency Report, which shows real-time traffic to Google services, displayed a sharp drop-off in traffic to Gmail from China on Friday. + +"We've checked and there's nothing wrong on our end," a Singapore-based spokesman for Google said in an email. + +In Washington, the U.S. State Department expressed concern over China's actions. + +"We encourage China to be transparent in its dealings with international companies and to consider the market signal it sends with such acts," State Department spokesman Jeff Rathke said. + +Almost all of Google's services have been heavily disrupted in China since June this year, but until last week Gmail users could still access emails downloaded via protocols like IMAP, SMTP and POP3. These had let people communicate using Gmail on apps like the Apple iPhone's Mail and Microsoft Outlook. + +China maintains tight control over the Internet, nipping in the bud any signs of dissent or challenges to the ruling Communist Party's leadership. + +The country is host to the world's most sophisticated internet censorship mechanism, known as the Great Firewall of China. Critics say China has stepped up its disruption of foreign online services like Google over the past year to create an Internet cut off from the rest of the world. + +The Google disruption began in the run-up to the 25th anniversary of the government's bloody crackdown on pro-democracy demonstrators around Beijing's Tiananmen Square on June 4, 1989. + +Gmail's setback could make email communication difficult for companies operating in China, said GreatFire. + +Chinese Foreign Ministry spokeswoman Hua Chunying said she did not know anything about Gmail being blocked, adding that the government was committed to providing a good business environment for foreign investors. + +"China has consistently had a welcoming and supportive attitude towards foreign investors doing legitimate business here," she said. "We will, as always, provide an open, transparent and good environment for foreign companies in China." + +One popular way to get around China's internet censorship is to use a Virtual Private Network (VPN), which allows unhindered access to blocked sites and services. + +"Using a VPN seems to be the only answer to doing anything these days online in China," said Zach Smith, a Beijing-based digital products manager at City Weekend magazine. \ No newline at end of file diff --git a/tests/data/text/reuters.com6.txt b/tests/data/text/reuters.com6.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/text/self.com1.txt b/tests/data/text/self.com1.txt new file mode 100644 index 00000000..eacfc496 --- /dev/null +++ b/tests/data/text/self.com1.txt @@ -0,0 +1,7 @@ +Blair Waldorf who? The headband has come a long way from its schoolgirl reputation, thanks to the recent efforts of the always-stylish Taylor Swift. When she recently stepped out in NYC wearing a sparkly hair accessory, she demonstrated three must-know rules to follow when pulling off the accessory: + +Keep Your Hair’s Movement: Unless you’re fighting with grown-out bangs, refrain from pulling your headband straight back. Instead, keep your hair’s natural movement by placing your style on top of the hair, not under and behind the ears. + +Go Easy on Styling: The simplest way to keep the accessory from looking stuffy is to skip elaborate updos and ringlet curls. Keep the hair more natural and use the headband to add structure. + +Make It the Focal Point: With a piece of bling like Taylor’s, keep the rest of the beauty look soft. Although she wore fuchsia lipstick, the singer opted for a matte variety to tone-down the vivid hue. diff --git a/tests/data/text/self.com2.txt b/tests/data/text/self.com2.txt new file mode 100644 index 00000000..13980a9e --- /dev/null +++ b/tests/data/text/self.com2.txt @@ -0,0 +1,7 @@ +Starting the New Year on a good note means skipping the stress—especially when it comes to your hair! But since many of us will be working in the hours leading up to the ball drop (often with little to no time to adjust between office and party) our suggestion is to prep your locks in the AM. With the right style and a few night-appropriate tricks, your hair won't need much upkeep to make it to midnight. + +Start with structured waves in the morning, wrapping small sections of hair (1-1 1/2-inch pieces) around a 1-inch curling iron for 20 to 30 seconds. Brush the ringlets out and throughout the day, they’ll smooth to soft waves. After work, apply a texturizing spray into the hair, spraying and scrunching each section to increase volume. + +The secret to a topknot is in the volume. In the morning apply a generous amount of dry shampoo from the scalp to the tip, scrunching the hair to add fullness. Then pull the hair up into a twisted topknot towards the top of the head. Hold with hairspray. At night, either pull a few front pieces out to soften the look or add a glitzy headband. + +Pull off a lived-in braid the natural way. Start with a clean fishtail braid in the morning and let it naturally transition to messy-chic—pieces will fall out of the plait over time. At the end of the day, use a dry shampoo at the roots to make the entire style look lived in. Add a bow or leather cuff at the tip of the braid for subtle structure. diff --git a/tests/data/text/sitepoint.com1.txt b/tests/data/text/sitepoint.com1.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/text/sitepoint.com2.txt b/tests/data/text/sitepoint.com2.txt new file mode 100644 index 00000000..febeaf39 --- /dev/null +++ b/tests/data/text/sitepoint.com2.txt @@ -0,0 +1,81 @@ +One great way of organically promoting your application is to provide “badges”; snippets of content that people can embed on their own websites. + +This can contain up-to-the-minute information from your application about a user, piece of content or another object, dynamically generated and inserted into other websites. This is probably best illustrated with some examples: + +In this article I’m going to take a look at some of the ways you can implement this. + +All the code from this tutorial is available on Github. There’s also an online demo. + +First, we’ll define our application’s dependencies using Composer: + +Now, in , let’s pull in the Composer-generated autoloader, add our statement, initialize our Silex application and setup Twig templating: + +Let’s create a data-store of users, with some information about them which will form the basis of our example “badges”. For simplicity, we’ll use a static array; in practice you’d use a database, but it ought to be pretty simple to swap this out for something more dynamic. We’ll inject it into the application container like so: + +Now that we’ve got a basic application set up along with some data, let’s go through three approaches to how you can provide an embeddable “badge” which displays this data for a given user. + +IFrames are arguably a dirty word in web circles, perhaps deservedly so. But they are a common and practical approach to embedding content from one site into another. + +Let’s start with this approach; later we can reuse some of the code when we try out the JavaScript method. + +Start with a simple Twig template, which creates some HTML – with inlined styles, to minimize HTTP requests – which contains our “badge” as a : + +This is all pretty straightforward. Notice how we’re incorporating a variable named which we’ll set server-side, which is going to take care of one very important aspect – any images we incorporate must be referenced using absolute URLs. + +Now the corresponding route: + +All pretty straightforward. The bit which populates is a bit quick-and-dirty, but it’ll do the job for now. + +Embedding this into a third-party site is really simple: + +Later we’ll look at some of the things you need to think about with the iframe approach; for now, let’s move onto method number two. + +One of the simplest ways to implement this is to provide a URL to an image, which gets created server-side. + +Here’s a screenshot of the sort of image we’re going to create: + +Okay, it won’t win any awards for design, but you can use the same approach to create much more visually appealing images than this. + +You’ll find the necessary resources – the background and trophy images, avatars and fonts – in the example application’s repository. + +Here’s some example code to dynamically generate an embeddable image: + +It’s pretty much self-documented, and should be pretty simple to adapt to your needs or with better images. Note that we’re taking the background image from a non-web accessible directory ( ), but the avatars and trophy icon are in the directory. + +Embedding this into a third-party website couldn’t be simpler: + +There are a couple of improvements we could make. For one thing, we’re generating a new image on each request. However you can use WideImage’s method to cache the results, like so: + +It might also be better to provide a default image when the requested user cannot be found, rather than issuing a 404 error. + +Now onto the third and final approach. + +Using JavaScript to dynamically create embedded content is amongst the most common, and perhaps the most flexible approach. + +Again we’re going to generate some HTML, but this time we’ll return a simple snippet of JavaScript that will write it to the host page. All that requires is that the host website insert a simple tag where they want our content to appear. + +We’ll re-use the Twig template from earlier, but this time the route looks slightly different: + +The first part is identical to the iframe approach. This time, though, we’re generating a simple . Before we can do that, we use a little magic to minify the resulting HTML – which also ensures it will all be on one line – then insert it into some very simple dynamically created JavaScript. + +Embedding this into a page is just as simple: + +Strictly speaking, we don’t even need that container DIV, but it can be used to apply styling on the host site. + +Now that we’ve examined three approaches, let’s look at some of the things you need to think about when deciding which of these approaches to take. + +When choosing an approach to this problem, there are a few things you need to weigh up. + +If you’re intending people to be able to embed content into the body of CMS-driven content or within blog posts, it’s worth bearing in mind that any CMS or blog software worth its salt will block certain types of content. Inline scripts are almost certainly out. IFrames are probably going to be stripped out. That probably just leaves image tags. + +There are a few ways to approach styling; perhaps you want to control everything, keeping your badges consistent across sites. Alternatively, you could provide default styles but allow site owners the flexibility to override them to better fit the design of their site. + +Obviously images cannot be customized; aside, perhaps, their sizing. If you use iframes, it’s worth noting that any styling applied to the parent page will not be inherited by your content. On the other hand if you use the JavaScript approach, it may well be possible to override the styling, depending on specificity and how you incorporate your styles. The demo page that comes with the example application shows this in action. + +Perhaps, like Stackoverflow and their “User Flair” badges, you want to provide a number of alternative styles – light and dark, for example. This is entirely possible with any of the approaches I’ve outlined, though it’s arguably slightly more difficult with the image tag approach. + +So far our embeddable content has been dynamically generated, but in no way interactive. A Facebook “like” button, for example, doesn’t just provide a count – it also allows people to perform the “like” action from within the page. That sort of interactivity will be covered in a later article. + +Embeddable content is great way to promote your site. It can be used not only to link back to your site but to provide “live” content, right there on a third-party “host” website. + +We’ve looked at three common approaches to this – images, iframes and JavaScript. We’ve looked at some of the things you need to think about when deciding which one to use, along with some pitfalls to be wary of. \ No newline at end of file diff --git a/tests/data/text/slate.com1.txt b/tests/data/text/slate.com1.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/text/slate.com2.txt b/tests/data/text/slate.com2.txt new file mode 100644 index 00000000..d2bed311 --- /dev/null +++ b/tests/data/text/slate.com2.txt @@ -0,0 +1,3 @@ +The biggest challenge to Brazil’s World Cup preparations over the past few weeks has been the chaos caused by striking public workers . While São Paulo subway workers voted to at least temporarily suspend their strikes on Monday night, protests by activists angry at the money being spent by Brazil to host the event are expected to continue throughout the World Cup . + +Though nowhere near the size of last year’s broad-based social movement, which brought millions to the streets, protests have been building back up in recent months. The tournament, which begins Thursday in São Paulo, has reportedly cost in excess of $11 billion, money protesters say could have been spent on public infrastructure. The movement has already produced its share of powerful images. Here are some of the most striking photos. \ No newline at end of file diff --git a/tests/data/text/space.com1.txt b/tests/data/text/space.com1.txt new file mode 100644 index 00000000..2303491f --- /dev/null +++ b/tests/data/text/space.com1.txt @@ -0,0 +1,83 @@ +Mars is the fourth planet from the sun. Befitting the red planet's bloody color, the Romans named it after their god of war. The Romans copied the ancient Greeks, who also named the planet after their god of war, Ares. Other civilizations also typically gave the planet names based on its color — for example, the Egyptians named it "Her Desher," meaning "the red one," while ancient Chinese astronomers dubbed it "the fire star." + +The bright rust color Mars is known for is due to iron-rich minerals in its regolith — the loose dust and rock covering its surface. The soil of Earth is a kind of regolith, albeit one loaded with organic content. According to NASA, the iron minerals oxidize, or rust, causing the soil to look red. + +The cold, thin atmosphere means liquid water currently cannot exist on the Martian surface for any length of time. This means that although this desert planet is just half the diameter of Earth, it has the same amount of dry land. + +The red planet is home to both the highest mountain and the deepest, longest valley in the solar system. Olympus Mons is roughly 17 miles (27 kilometers) high, about three times as tall as Mount Everest, while the Valles Marineris system of valleys — named after the Mariner 9 probe that discovered it in 1971 — can go as deep as 6 miles (10 km) and runs east-west for roughly 2,500 miles (4,000 km), about one-fifth of the distance around Mars and close to the width of Australia or the distance from Philadelphia to San Diego. + +Mars has the largest volcanoes in the solar system, including Olympus Mons, which is about 370 miles (600 km) in diameter, wide enough to cover the entire state of New Mexico. It is a shield volcano, with slopes that rise gradually like those of Hawaiian volcanoes, and was created by eruptions of lavas that flowed for long distances before solidifying. Mars also has many other kinds of volcanic landforms, from small, steep-sided cones to enormous plains coated in hardened lava. Some minor eruptions might still occur on the planet. + +Scientists think the Valles Marineris formed mostly by rifting of the crust as it got stretched. Individual canyons within the system are as much as 60 miles (100 km) wide. They merge in the central part of the Valles Marineris in a region as much as 370 miles (600 km) wide. Large channels emerging from the ends of some canyons and layered sediments within suggest the canyons might once have been filled with liquid water. + +Channels, valleys, and gullies are found all over Mars, and suggest that liquid water might have flowed across the planet's surface in recent times. Some channels can be 60 miles (100 km) wide and 1,200 miles (2,000 km) long. Water may still lie in cracks and pores in underground rock. + +Many regions of Mars are flat, low-lying plains. The lowest of the northern plains are among the flattest, smoothest places in the solar system, potentially created by water that once flowed across the Martian surface. The northern hemisphere mostly lies at a lower elevation than the southern hemisphere, suggesting the crust may be thinner in the north than in the south. This difference between the north and south might be due to a very large impact shortly after the birth of Mars. + +The number of craters on Mars varies dramatically from place to place, depending on how old the surface is. Much of the surface of the southern hemisphere is extremely old, and so has many craters — including the planet's largest, 1,400-mile-wide (2,300 km) Hellas Planitia — while that of northern hemisphere is younger and so has fewer craters. Some volcanoes have few craters, which suggests they erupted recently, with the resulting lava covering up any old craters. Some craters have unusual-looking deposits of debris around them resembling solidified mudflows, potentially indicating that impactor hit underground water or ice. + +Vast deposits of what appear to be finely layered stacks of water ice and dust extend from the poles to latitudes of about 80 degrees in both hemispheres. These were probably deposited by the atmosphere over long spans of time. On top of much of these layered deposits in both hemispheres are caps of water ice that remain frozen all year round. Additional seasonal caps of frost appear in the wintertime. These are made of solid carbon dioxide, also known as "dry ice," which has condensed from carbon dioxide gas in the atmosphere, and in the deepest part of the winter, this frost can extend from the poles to latitudes as low as 45 degrees, or halfway to the equator. The dry ice layer appears to have a fluffy texture, like freshly fallen snow, according to the report in the Journal of Geophysical Research-Planets. + +Mars is much colder than Earth, in large part due to its greater distance from the sun. The average temperature is about minus 80 degrees Fahrenheit (minus 60 degrees Celsius), although they can vary from minus 195 F (minus 125 C) near the poles during the winter to as much as 70 F (20 C) at midday near the equator. + +The carbon-dioxide-rich atmosphere of Mars is also roughly 100 times less dense than Earth's on average, but it is nevertheless thick enough to support weather, clouds and winds. The density of the atmosphere varies seasonally, as winter forces carbon dioxide to freeze out of the Martian air. + +NASA’s Mars Reconnaissance Orbiter found the first definitive detections of carbon-dioxide snow clouds, making Mars the only body in the solar system known to host the unusual winter weather. The red planet also causes water-ice snow to fall from the clouds. + +The dust storms of the Mars are the largest in the solar system, capable of blanketing the entire red planet and lasting for months. One theory as to why dust storms can grow so big on Mars starts with airborne dust particles absorbing sunlight, warming the Martian atmosphere in their vicinity. Warm pockets of air flow toward colder regions, generating winds. Strong winds lift more dust off the ground, which in turn heats the atmosphere, raising more wind and kicking up more dust. + +The axis of Mars, like Earth's, is tilted with relation to the sun. This means that like Earth, the amount of sunlight falling on certain parts of the planet can vary widely during the year, giving Mars seasons. + +However, the seasons that Mars experiences are more extreme than Earth's because the red planet's elliptical, oval-shaped orbit around the sun is more elongated than that of any of the other major planets. When Mars is closest to the sun, its southern hemisphere is tilted toward the sun, giving it a short, very hot summer, while the northern hemisphere experiences a short, cold winter. When Mars is farthest from the sun, the northern hemisphere is tilted toward the sun, giving it a long, mild summer, while the southern hemisphere experiences a long, cold winter. + +Magnetic field: Mars currently has no global magnetic field, but there are regions of its crust that can be at least 10 times more strongly magnetized than anything measured on Earth, remnants of an ancient global magnetic field. + +Chemical composition: Mars likely has a solid core composed of iron, nickel, and sulfur. The mantle of Mars is probably similar to Earth's in that it is composed mostly of peridotite, which is made up primarily of silicon, oxygen, iron and magnesium. The crust is probably largely made of the volcanic rock basalt, which is also common in the crusts of the Earth and the moon, although some crustal rocks, especially in the northern hemisphere, may be a form of andesite, a volcanic rock that contains more silica than basalt does. + +Internal structure: Scientists think that on average, the Martian core is about 1,800 and 2,400 miles in diameter (3,000 and 4,000 km), its mantle is about 900 to 1,200 miles (5,400 to 7,200 km) wide and its crust is about 30 miles (50 km) thick. + +Average distance from the sun: 141,633,260 miles (227,936,640 km). By comparison: 1.524 times that of Earth + +Perihelion (closest): 128,400,000 miles (206,600,000 km). By comparison: 1.404 times that of Earth + +Aphelion (farthest): 154,900,000 miles (249,200,000 km). By comparison: 1.638 times that of Earth + +The two moons of Mars, Phobos and Deimos, were discovered by American astronomer Asaph Hall over the course of a week in 1877. Hall had almost given up his search for a moon of Mars, but his wife, Angelina, urged him on — he discovered Deimos the next night, and Phobos six days after that. He named the moons after the sons of the Greek war god Ares — Phobos means "fear," while Deimos means "rout." + +Both Phobos and Deimos are apparently made of carbon-rich rock mixed with ice and are covered in dust and loose rocks. They are tiny next to Earth's moon, and are irregularly shaped, since they lack enough gravity to pull themselves into a more circular form. The widest Phobos gets is about 17 miles (27 km), and the widest Deimos gets is roughly nine miles (15 km). + +Both moons are pockmarked with craters from meteor impacts. The surface of Phobos also possesses an intricate pattern of grooves, which may be cracks that formed after the impact created the moon's largest crater — a hole about 6 miles (10 km) wide, or nearly half the width of Phobos. They always show the same face to Mars, just as our moon does to Earth. + +It remains uncertain how Phobos and Deimos were born. They may have been asteroids captured by Mars' gravitational pull, or they may have been formed in orbit around Mars the same time the planet came into existence. Ultraviolet light reflected from Phobos provides strong evidence for its capture origin, according to astronomers at the University of Padova in Italy. + +Phobos is gradually spiraling toward Mars, drawing about 6 feet (1.8 meters) closer to the red planet each century. Within 50 million years, Phobos will either smash into Mars or break up and form a ring of debris around the planet. + +Both moons are potential targets for exploration. One NASA plan envisions bombarding Phobos with small, spiky spherical rovers called hedgehogs. + +The first person to watch Mars with a telescope was Galileo Galilei, and in the century after him, astronomers discovered its polar ice caps. In the 19th and 20th centuries, researchers believed they saw a network of long, straight canals on Mars, hinting at civilization, although later these often proved to be mistaken interpretations of dark regions they saw. + +Robot spacecraft began observing Mars in the 1960s, with the United States launching Mariner 4 there in 1964 and Mariners 6 and 7 in 1969. They revealed Mars to be a barren world, without any signs of the life or civilizations people had imagined there. In 1971, Mariner 9 orbited Mars, mapping about 80 percent of the planet and discovering its volcanoes and canyons. + +NASA's Viking 1 lander touched down onto the surface of Mars in 1976, the first successful landing onto the Red Planet. It took the first close-up pictures of the Martian surface but found no strong evidence for life. + +The next two craft to successfully reach Mars were the Mars Pathfinder, a lander, and Mars Global Surveyor, an orbiter, both launched in 1996. A small robot onboard Pathfinder named Sojourner — the first wheeled rover to explore the surface of another planet — ventured over the planet's surface analyzing rocks. + +In 2001, the United States launched the Mars Odyssey probe, which discovered vast amount of water ice beneath the Martian surface, mostly in the upper three feet (one meter). It remains uncertain whether more water lies underneath, since the probe cannot see water any deeper. + +In 2003, the closest Mars had passed to Earth in nearly 60,000 years, NASA launched two rovers, nicknamed Spirit and Opportunity, which explored different regions of the Martian surface, and both found signs that water once flowed on the planet's surface. In 2008, NASA sent another mission, Phoenix, to land in the northern plains of Mars and search for water, + +Two orbiters — NASA's Mars Reconnaissance Orbiter and ESA's Mars Express — are keeping Mars Odyssey company over the planet. In 2011, NASA's Mars Science Laboratory mission, with its rover named Mars Curiosity, began to investigate Martian rocks to determine the geologic processes that created them and find out more about the present and past habitability of Mars. Among its findings is the first meteorite on the surface of the red planet. + +In September 2014, India’s Mars Orbiter Mission reached the red planet, making it the fourth nation to successfully enter orbit around Mars. + +Robots aren’t the only ones looking to buy a ticket to Mars. A workshop group of government, academic, and industry scientists have found that a NASA-led manned mission to Mars should be possible by the 2030s. But NASA isn’t the only one with Martian astronaut hopefuls.. The Mars One colony project is looking to send private citizens on a one-way trip to the red planet. + +Mars could have once harbored life. Some conjecture that life might still exist there even today. A number of researchers have even speculated that life on Earth may have seeded Mars, or that life on Mars seeded Earth. + +The most public scientific claim for life on Mars came in 1996. Geologist David McKay at NASA's Johnson Space Center in Houston and his colleagues focused on rocks blasted off the surface of Mars by cosmic impacts that landed on Earth. Within they found complex organic molecules, grains of a mineral called magnetite that can form within some kinds of bacteria, and tiny structures that resembled fossilized microbes. However, these claims have proven controversial, and there is no consensus as to whether they are signs of life. + +Mars may have possessed oceans on its surface in the past, providing an environment for life to develop. Although the red planet is a cold desert today, researchers suggest that liquid water may be present underground, providing a potential refuge for any life that might still exist there. The rover Curiosity has found evidence for a lake that could have once supported life on the red planet, after previously establishing that the planet had the key ingredients present for life to evolve. + +Enthusiasm and excitement on Earth over the possibility of life on the red planet are revealed by the flurry of excitement that greets interesting objects spotted by orbiters and landers. While the Face on Mars garnered attention for the past four decades after it was first spotted by Viking 1, rovers today show close-ups of objects — such as a weathered Martian rock claimed to be a ‘thigh bone’ and a likely shiny rock that raised furor on the internet as a UFO light. + +Learn more about each of the primary planets: \ No newline at end of file diff --git a/tests/data/text/space.com2.txt b/tests/data/text/space.com2.txt new file mode 100644 index 00000000..ed48d9ad --- /dev/null +++ b/tests/data/text/space.com2.txt @@ -0,0 +1,25 @@ +WASHINGTON — NASA will receive $18 billion for 2015, more than a half a billion dollars above the Obama administration’s original request, under the terms of an omnibus spending bill released late Dec. 9. + +The appropriations bill, which funds NASA and most of the rest of the federal government for the remainder of the fiscal year that began Oct. 1, gives the agency $17.99 billion, including increases for several major exploration and science programs. That total is $530 million above the administration's request of $17.46 billion for the agency, and about $100 million above separate House and Senate appropriations bills considered earlier this year. It's also nearly $350 million above NASA’s enacted 2014 budget of $17.65 billion. + +Two major elements of NASA's exploration strategy won funding boosts in the final bill. The Space Launch System heavy-lift rocket will receive $1.7 billion, an increase of $320 million over the administration’s request. The Orion spacecraft will get $1.194 billion, an increase of $141.2 million over the request. + +The bill also requires NASA to submit in its next budget proposal five-year funding projections for the SLS and Orion programs that match the budget and schedule estimates for those programs developed in recent or ongoing reviews. Those estimates, developed by NASA for a program milestone called Key Decision Point C (KDP-C), were completed for SLS in August and will be completed for Orion in the spring of 2015. + +The bill specifically requires those projections for the SLS to show completion by December 2017. The KDP-C review for SLS estimated that the vehicle would be ready for its first launch no later than November 2018. + +NASA's planetary science program won an increase of $157 million, to $1.438 billion for 2015. The bill sets aside $100 million of that funding for a proposed Europa mission, for which the White House had requested just $15 million. + +NASA's astrophysics program won a $77.5 million increase, to $684.8 million, in the omnibus bill. That includes $70 million for the Stratospheric Observatory for Infrared Astronomy, an airborne telescope for which NASA requested only $12 million. NASA had planned to mothball the flying observatory in 2015 if it could not find partners to fund the telescope's annual operating cost of about $85 million. + +NASA's commercial crew program received $805 million in the bill, less than the requested $848 million but more than the program received in previous years. The omnibus bill leaves out a controversial provision from the Senate's appropriations bill that would have required certified cost and pricing data from commercial crew companies. That language was opposed by many supporters of the commercial crew program, as well as by the White House. + +NASA's aeronautics program received $651 million, an increase of $100 million over the administration’s request. In a report accompanying the bill, appropriators directed NASA to apply the additional funds proportionally across the programs in that mission directorate. + +NASA's space technology program was cut by nearly $110 million from the administration's request, to $596 million. Appropriators offered no specific rationale for the cut in report language, nor direction on how to apply the reduced funding among its programs. + +Other parts of the agency's budget saw only minimal changes from the administration's request. The bill includes $3.828 billion for space operations; $2.759 billion for safety, security, and mission operations (formerly known as cross-agency support); $419.1 million for construction; $119 million for education; and $37 million for the agency’s inspector general. + +The overall bill, sometimes called a "cromnibus" since it serves as only a continuing resolution (CR) for the Department of Homeland Security through February 2015, was slated to be voted on by the House Dec. 11, and immediately thereafter by the Senate. The federal government is currently funded by a continuing resolution that expired at midnight of Dec. 11, and another short-term continuing resolution, lasting only a few days, was needed to keep the government operating until Congress passes the omnibus bill. + +This story was provided by SpaceNews, dedicated to covering all aspects of the space industry. \ No newline at end of file diff --git a/tests/data/text/syracuse.com1.txt b/tests/data/text/syracuse.com1.txt new file mode 100644 index 00000000..73f56880 --- /dev/null +++ b/tests/data/text/syracuse.com1.txt @@ -0,0 +1 @@ +Today on the first day of Kwanzaa, a week-long celebration which honors African heritage in African-American culture, children at the Beauchamp Branch Library read Kwanzaa stories and make related crafts in a celebration organized by Children's Librarian Anne Gregory. In addition a related exhibit, Kwanzaa and the African Diaspora by Vanessa Johnson is on display at the library. \ No newline at end of file diff --git a/tests/data/text/syracuse.com2.txt b/tests/data/text/syracuse.com2.txt new file mode 100644 index 00000000..fd41b6f8 --- /dev/null +++ b/tests/data/text/syracuse.com2.txt @@ -0,0 +1,9 @@ +The Onondaga Historical Association leads groups through the Hotel Syracuse today on Historic Ghostwalks from 1 - 3:30 p.m. In what's subtitled 'Suite Stories' people tour many of the public spaces at the hotel as actors play personalities from the hotel's past. The building opened in 1924 and many famous names have been guests. A feature of the event is a view of Carl Roters 40 foot mural above the registration desk which has been covered with mirrors for the last 37 years. + +Here's an update from the OHA : + + • Reservation can only be made online at www.hotelsyracuseghostwalk.eventbrite.com, not by calling OHA + + • There are no walk-ins allowed + + • There are a limited amount of tickets being added today, December 29th, at 2 P.M. for the January 3rd tours. \ No newline at end of file diff --git a/tests/data/text/talkingpointsmemo.com1.txt b/tests/data/text/talkingpointsmemo.com1.txt new file mode 100644 index 00000000..4b49e30d --- /dev/null +++ b/tests/data/text/talkingpointsmemo.com1.txt @@ -0,0 +1 @@ +You're seeing this error because you have enabled the setting. \ No newline at end of file diff --git a/tests/data/text/talkingpointsmemo.com2.txt b/tests/data/text/talkingpointsmemo.com2.txt new file mode 100644 index 00000000..82218109 --- /dev/null +++ b/tests/data/text/talkingpointsmemo.com2.txt @@ -0,0 +1,37 @@ +Shurtleff, a Republican, served as the state's top legal official from 2001 to 2013. He was succeeded by Swallow, also a Republican, who had been one of his top deputies. Swallow resigned less than a year after taking office, as federal and state investigations into his and Shurtleff's alleged improprieties intensified. Both men had also pursued failed bids for Congress (Shurtleff for a U.S. Senate seat in 2009; Swallow for a House seat in 2002 and 2004.) + +Former Utah attorneys general Mark Shurtleff and John Swallow were accused Tuesday of numerous bribery and obstruction of justice charges, most of them felonies. The charging documents from the Salt Lake City district attorney allege a decadent lifestyle of private jets, all-expense-paid vacations and veiled threats of violence for those who caused trouble. + +Now, the men who served as the top legal officials in Utah stand accused of soliciting bribes, accepting gifts, money laundering and interfering with the resulting criminal investigation. TPM parsed through the charging documents from the district attorney to find a few of the juiciest and most unbelievable details from the massive alleged corruption scheme. + +While one of the duo's alleged associates, Mark Jenson, was on probation for securities fraud, he allegedly paid for Shurtleff and Swallow to vacation at Pelican Hills Resort in Newport Coast, Calif., twice in 2009. The resort was rated by Conde Nast Traveler's readers as one of the best hotels in the world from 2011 to 2014. The trip included massages, golf rounds, dining and men's apparel -- all allegedly paid for by Jenson. Swallow and his wife allegedly stayed there in July 2009 for their wedding anniversary as well. + +They also traveled in style on the dime of their business friends, prosecutors allege. At the same time Swallow was allegedly working behind the scenes to help another associate, Jeremy Johnson, as he was being investigated by the Federal Trade Commission, Swallow allegedly spent two nights on Johnson's 80-foot luxury houseboat in the fall 2010. Shurtleff and Swallow also allegedly took Johnson's private aircraft on flights to and from California and St. George, Utah. + +2. Shurtleff Allegedly Had A Fixer Who Threatened To 'Bust People Up' + +Shurtleff allegedly had an associate who thought of himself as something of a bodyguard or fixer for the attorney general, according to the documents. + +In one instance outlined by prosecutors, the man, Timothy Lawson, allegedly used Shurtleff's name when trying to get someone to back off from trying to collect on a $100,000 business loan that had gone bad in a third-party dispute. Lawson compared himself to "Porter Rockwell" to explain his relationship with Shurtleff, the documents state. The reference was apparently to Orrin Porter Rockwell, the legendary bodyguard for Joseph Smith, the founder of the Mormon Church. + +In his quest to stymie the dispute, Lawson also allegedly told one of the parties that he possessed guns and "Polynesian friends" who would "bust people up." + +One of the primary focuses of the investigation was Shurtleff and Swallow's alleged efforts to help Johnson in securing approval for online poker in Utah and navigating the FTC probe. In September 2010, Swallow allegedly told Johnson that he might be able to help him with the FTC investigation by offering him access to Senate Majority Leader Harry Reid (D-NV) through another associate. The access, he allegedly warned, "won't be cheap." + +Shortly afterward, Johnson allegedly wired a total of $250,000 to Swallow's business associate, Richard Rawle. Rawle then allegedly paid Swallow $8,500. + +Reid's office has previously dismissed the connections to the Shurtleff-Swallow case and the resulting questions from critics, saying that Reid “has never been contacted in regards to this investigation." + +As all of these alleged indiscretions piled up, the defendants -- Swallow in particular -- seemed to realize that they were potentially at risk. Swallow allegedly met with Johnson at a Krispy Kreme in April 2012, to discuss his possible vulnerabilities. Johnson allegedly advised that the $250,000 paid for possible access to Reid be repaid and that all relevant emails deleted. + +Swallow also allegedly told a campaign staffer to purchase a "burner," or pre-paid, cell phone -- with cash -- a few weeks after the Krispy Kreme meeting. He allegedly asked that his government laptop be wiped of its data, telling his office's tech support staff that he had it backed up on an external hard drive. He later claimed he had lost the hard drive during a flight from Phoenix to Salt Lake City, according to the documents. + +Swallow also allegedly obtained death-bed testimony from Rawle, who died of cancer in December 2012, by providing notes to Rawle's attorney for “anyone who would be interested at some point, including the court.” + +Prior to joining the attorney general's office, according to prosecutors, Swallow had worked with Rawle as general counsel for his payday lending business. When he left the company to join Shurtleff's office in 2009, Swallow allegedly received a dozen gold one-ounce coins as payment. + +Then between June 2011 and February 2012, Swallow allegedly sold the coins back to Rawle over 10 transactions for a total sum of $17,000. Swallow allegedly asked that the money be put onto a pre-paid debit card. Swallow then allegedly gave false or inconsistent statements about the coin sales while being deposed by the Utah lieutenant governor's office in October 2013. + +Though not present in the court documents reviewed by TPM, the Deseret News, which broke the news of the arrests on Tuesday morning, reported that Shurtleff is also alleged to have flown to New York in Johnson's jet to meet with actor Vincent D'Onofrio of "Law & Order" and "Men in Black" fame. + +Photos: Former Utah Attorneys General Mark Shurtleff (left) and John Swallow are shown after being released from Salt Lake County jail on Tuesday. \ No newline at end of file diff --git a/tests/data/text/technologyreview.com1.txt b/tests/data/text/technologyreview.com1.txt new file mode 100644 index 00000000..42895540 --- /dev/null +++ b/tests/data/text/technologyreview.com1.txt @@ -0,0 +1,45 @@ +Last month in Silicon Valley, biologists Jennifer Doudna and Emmanuelle Charpentier showed up in black gowns to receive the $3 million Breakthrough Prize, a glitzy award put on by Internet billionaires including Mark Zuckerberg. They’d won for developing CRISPR-Cas9, a “powerful and general technology” for editing genomes that’s been hailed as a biotechnology breakthrough. + +Not dressing up that night was Feng Zhang (see 35 Innovators Under 35, 2013), a researcher in Cambridge at the MIT-Harvard Broad Institute. But earlier this year Zhang claimed his own reward. In April, he won a broad U.S. patent on CRISPR-Cas9 that could give him and his research center control over just about every important commercial use of the technology. + +How did the high-profile prize for CRISPR and the patent on it end up in different hands? That’s a question now at the center of a seething debate over who invented what, and when, that involves three heavily financed startup companies, a half-dozen universities, and thousands of pages of legal documents. + +“The intellectual property in this space is pretty complex, to put it nicely,” says Rodger Novak, a former pharmaceutical industry executive who is now CEO of CRISPR Therapeutics, a startup in Basel, Switzerland, that was cofounded by Charpentier. “Everyone knows there are conflicting claims.” + +At stake are rights to an invention that may be the most important new genetic engineering technique since the beginning of the biotechnology age in the 1970s. The CRISPR system, dubbed a “search and replace function” for DNA, lets scientists easily disable genes or change their function by replacing DNA letters. During the last few months, scientists have shown that it’s possible to use CRISPR to rid mice of muscular dystrophy, cure them of a rare liver disease, make human cells immune to HIV, and genetically modify monkeys (see “Genome Surgery” and “10 Breakthrough Technologies 2014: Genome Editing”). + +No CRISPR drug yet exists. But if CRISPR turns out to be as important as scientists hope, commercial control over the underlying technology could be worth billions. + +The control of the patents is crucial to several startups that together quickly raised more than $80 million to turn CRISPR into cures for devastating diseases. They include Editas Medicine and Intellia Therapeutics, both of Cambridge, Massachusetts. Companies expect that clinical trials could begin in as little as three years. + +Zhang cofounded Editas Medicine, and this week the startup announced that it had licensed his patent from the Broad Institute. But Editas doesn’t have CRISPR sewn up. That’s because Doudna, a structural biologist at the University of California, Berkeley, was a cofounder of Editas, too. And since Zhang’s patent came out, she’s broken off with the company, and her intellectual property—in the form of her own pending patent—has been licensed to Intellia, a competing startup unveiled only last month. Making matters still more complicated, Charpentier sold her own rights in the same patent application to CRISPR Therapeutics. + +In an e-mail, Doudna said she no longer has any involvement with Editas. “I am not part of the company’s team at this point,” she said. Doudna declined to answer further questions, citing the patent dispute. + +Few researchers are now willing to discuss the patent fight. Lawsuits are certain and they worry anything they say will be used against them. “The technology has brought a lot of excitement, and there is a lot of pressure, too. What are we going to do? What kind of company do we want?” Charpentier says. “It all sounds very confusing for an outsider, and it’s also quite confusing as an insider.” + +Academic labs aren’t waiting for the patent claims to get sorted out. Instead, they are racing to assemble very large engineering teams to perfect and improve the genome-editing technique. On the Boston campus of Harvard’s medical school, for instance, George Church, a specialist in genomics technology, says he now has 30 people in his lab working on it. + +Because of all the new research, Zhang says, the importance of any patent, including his own, isn’t entirely clear. “It’s one important piece, but I don’t really pay attention to patents,” he says. “What the final form of this technology is that changes people’s lives may be very different.” + +The new gene-editing system was unearthed in bacteria—organisms that use it as a way to identify, and then carve up, the DNA of invading viruses. That work stretched across a decade. Then, in June 2012, a small team led by Doudna and Charpentier published a key paper showing how to turn that natural machinery into a “programmable” editing tool, to cut any DNA strand, at least in a test tube. + +The next step was clear—scientists needed to see if the editing magic could work on the genomes of human cells, too. In January 2013, the laboratories of Harvard’s Church and Broad’s Zhang were first to publish papers showing that the answer was yes. Doudna published her own results a few weeks later. + +Everyone by then realized that CRISPR might become an immensely flexible way to rewrite DNA, and possibly to treat rare metabolic problems and genetic diseases as diverse as hemophilia and the neurodegenerative disease Huntington’s. + +Venture capital groups quickly began trying to recruit the key scientists behind CRISPR, tie up the patents, and form startups. Charpentier threw in with CRISPR Therapeutics in Europe. Doudna had already started a small company, Caribou Biosciences, but in 2013 she joined Zhang and Church as a cofounder of Editas. With $43 million from leading venture funds Third Rock Ventures (see “50 Smartest Companies: Third Rock Ventures”), Polaris Partners, and Flagship Ventures, Editas looked like the dream team of gene-editing startups. + +In April of this year, Zhang and the Broad won the first of several sweeping patents that cover using CRISPR in eukaryotes—or any species whose cells contain a nucleus (see “Broad Institute Gets Patent on Revolutionary Gene-Editing Method”). That meant that they’d won the rights to use CRISPR in mice, pigs, cattle, humans—in essence, in every creature other than bacteria. + +The patent came as a shock to some. That was because Broad had paid extra to get it reviewed very quickly, in less than six months, and few knew it was coming. Along with the patent came more than 1,000 pages of documents. According to Zhang, Doudna’s predictions in her own earlier patent application that her discovery would work in humans was “mere conjecture” and that, instead, he was the first to show it, in a separate and “surprising” act of invention. + +The patent documents have caused consternation. The scientific literature shows that several scientists managed to get CRISPR to work in human cells. In fact, its easy reproducibility in different organisms is the technology’s most exciting hallmark. That would suggest that, in patent terms, it was “obvious” that CRISPR would work in human cells, and that Zhang’s invention might not be worthy of its own patent. + +What’s more, there’s scientific credit at stake. In order to show he was “first to invent” the use of CRISPR-Cas in human cells, Zhang supplied snapshots of lab notebooks that he says show he had the system up and running in early 2012, even before Doudna and Charpentier published their results or filed their own patent application. That timeline would mean he hit on the CRISPR-Cas editing system independently. In an interview, Zhang affirmed he’d made the discoveries on his own. Asked what he’d learned from Doudna and Charpentier’s paper, he said “not much.” + +Not everyone is convinced. “All I can say is that we did it in my lab with Jennifer Doudna,” says Charpentier, now a professor at the Helmholtz Centre for Infection Research and Hannover Medical School in Germany. “Everything here is very exaggerated because this is one of those unique cases of a technology that people can really pick up easily, and it’s changing researchers’ lives. Things are happening fast, maybe a bit too fast.” + +This isn’t the end of the patent fight. Although Broad moved very swiftly, lawyers for Doudna and Charpentier are expected to mount an interference proceeding in the U.S.—that is, a winner-takes-all legal process in which one inventor can take over another’s patent. Who wins will depend on which scientist can produce lab notebooks, e-mails, or documents with the earliest dates. + +“I am very confident that the future will clarify the situation,” says Charpentier. “And I would like to believe the story is going to end up well.” \ No newline at end of file diff --git a/tests/data/text/technologyreview.com2.txt b/tests/data/text/technologyreview.com2.txt new file mode 100644 index 00000000..b243da7e --- /dev/null +++ b/tests/data/text/technologyreview.com2.txt @@ -0,0 +1,15 @@ +Many of the estimated 50 million lithium-ion laptop batteries discarded every year could provide electricity storage sufficient to light homes in poor countries, researchers at IBM say. + +In work being aired this week at a conference in San Jose, researchers at IBM Research India in Bangalore found that at least 70 percent of all discarded batteries have enough life left to power an LED light at least four hours a day for a year. + +While it’s possible to combine LED lights with solar panels and rechargeable batteries (see “Innovators Under 35: Evans Wadongo”), using discarded batteries could make the approach far cheaper. + +“The most costly component in these systems is often the battery,” says Vikas Chandan, a research scientist at the lab’s Smarter Energy Group, who led the project. “In this case, the most expensive part of your storage solution is coming from trash.” + +The IBM group, working with a hardware R&D firm called RadioStudio, tore open discarded laptop battery packaging and extracted individual storage units called cells, tested those individually to pick out the good ones, and recombined them to form refurbished battery packs. Then, after adding charging dongles as well as circuitry to prevent overheating, they gave them to five users in Bangalore who lived in slums or operated sidewalk carts. + +Three months later, the users said the battery packs had worked well; the main request was for rat-resistant wires and brighter bulbs, says Mohit Jain, a research engineer with the group. A revised setup is now being tested. + +Around 50 million laptop and desktop computers are discarded in the United States every year, according to the Environmental Protection Agency. Meanwhile, in India alone, about 400 million people lack grid-connected electricity. + +IBM is not considering this as a business but says the technology could be offered free to poor countries. \ No newline at end of file diff --git a/tests/data/text/teenvogue.com1.txt b/tests/data/text/teenvogue.com1.txt new file mode 100644 index 00000000..b1d0b45d --- /dev/null +++ b/tests/data/text/teenvogue.com1.txt @@ -0,0 +1,3 @@ +As you may remember from High School Musical or Camp Rock or The Cheetah Girls, Disney Channel Original Movies aren't just made-for-TV movies, but actual events. Like something you looked forward to forever because it was basically like being at the exciting opening night showing, only in your own home. + +Not that the recent DCOM selection isn't great and all, but we definitely suggest locating these movies over holiday break and watching them over and over again. Scrunchies and roll-on body glitter optional. \ No newline at end of file diff --git a/tests/data/text/teenvogue.com2.txt b/tests/data/text/teenvogue.com2.txt new file mode 100644 index 00000000..94588a89 --- /dev/null +++ b/tests/data/text/teenvogue.com2.txt @@ -0,0 +1,3 @@ +Every year, we love looking back and picking out our favorite red carpet looks from 52 weeks worth of awards shows, highfalutin' fashion events, movie premieres, and every kind of party you can imagine. And while we adore seeing all of our favorite familiar faces, the most exciting moments come from the rookies that are either just hitting the scene this year or just hitting their stride in the style department. + +Ahead, we've rounded up our top 10 break-out style star picks for 2014. Click through to see who we chose, and don't forget to nominate your favorites in the comments! \ No newline at end of file diff --git a/tests/data/text/telegraph.co.uk1.txt b/tests/data/text/telegraph.co.uk1.txt new file mode 100644 index 00000000..1b070d1a --- /dev/null +++ b/tests/data/text/telegraph.co.uk1.txt @@ -0,0 +1,62 @@ +09.06 Further confirmation from Reuters: +Debris sighted in Indonesia's Java Sea is from the AirAsia jet presumed to have crashed two days ago, Tatang Zaenudin, an official at the country's search and rescue agency, told Reuters on Tuesday. +09.03 REUTERS - INDONESIA RESCUE AND SEARCH AGENCY OFFICIAL SAYS JAVA SEA DEBRIS IS FROM MISSING AIRASIA JET +08.46 The transcript of the final communication between air-traffic control and the pilot of AirAsia Flight 8501 has been released. +According to The New York Post, It reveals a calm request to redirect the plane and then to climb to avoid a storm. +Air-traffic control couldn’t say yes immediately because six other planes were crowding the higher airspace, forcing Flight 8501 to remain at a lower altitude, the transcript reveals. +By the time the pilot was given the OK, a mere two minutes later, it was apparently too late - there was no response from the cockpit. +Indonesia’s state navigation operator said late Monday that the Airbus 320-200 pilot, Capt. Iriyanto, who goes by one name, had requested permission to turn left to avoid a storm. +The request was granted and the plane turned left for seven miles. +The captain then sent his fateful message at 6:12 a.m., saying, “Request to higher level,” according to AirNav Standards and Safety Director Wisnu Darjono, as quoted by the Jakarta Post. +“Intended to what level?” the controller responded. When Iriyanto said 38,000 feet, he was told to hold off because there were six other planes in that area. +“But when we informed the pilot of the approval at 6:14 a.m., we received no reply,” Wisnu said. +08.41 The Jakarta Post quotes a co-pilot of the Air Force Hercules C130, who flew on Tuesday over waters near Pangkalan Bun in Central Kalimantan in the search for missing AirAsia flight QZ8501. +Lt. Tri Wibowo said that he saw dozens of floating bodies as well as bags and aircraft debris. +QuoteWe thought that the passengers were still alive and waved at us for help. But when we approached closer [we saw] they were already dead. +Tri and the team scoured the area at around 11 a.m. +The Transportation Ministry earlier confirmed that white and red debris found near Pangkalan Bun belonged to the ill-fated flight, which vanished on Sunday en route to Singapore from Indonesia's second-biggest city of Surabaya in East Java. +08.38 Tony Fernandes, the boss of AirAsia said his heart was "filled with sadness" after several bodies were found floating in the sea near where a missing AirAsia flight was last seen. +Earlier, an Indonesia National Search and Rescue spokesman said several pieces of debris seen floating off Borneo might be linked to the missing Flight 8501. +Mr Fernandes, the AirAsia chief executive, said +QuoteMy heart is filled with sadness. Words cannot express how sorry I am. On behalf of AirAsia, my condolences to all. Words cannot express how sorry I am. +08.29 In other plane news, it's just breaking now that a Thai Airways plane TG916 flying from Bangkok to London is dumping fuel and returning to Bangkok, with problems. +08.15 Plane spots 'shadow' on seabed believed to be AirAsia jet: Indonesia search chief +08.10 The Telegraph will not be publishing the images of the victims. The Indonesian search agency chief says the six bodies are swollen but intact and have been transferred to a navy ship. +08.08 As if it couldn't get worse for the families: +08.00 What we know at the moment: +- Items resembling emergency slide, life jackets, plane door and luggage spotted six miles from last point of contact +- Bodies have been spotted by search planes +- 11 divers heading to the debris area +- the waters are warm and relatively shallow at 80ft so teams are holding out hope for surviors +- Officials say they are 95 per cent sure items are from the missing Airbus +- Distraught relatives have been broken the news +07.52 Jonathan Pearlman, our correspondent at the press conference in Jakarta, says Indonesia's search authority has confirmed a body has been found in the water along with debris confirmed to be from the missing Air Asia plane. Indonesia's TV One has shown a rather gruesome photo of one body floating in water. +Henry Bambang Sulistyo, head of Indonesia’s National Search and Rescue Agency, said a search team found the plane's emergency exit at 12.40pm, local time, and confirmed it came from the missing plane. At 1.25pm, further objects were found in the waters, which are apparently about 80 to 100 feet deep. +Mr Sulistyo said the debris is being recovered and taken back to shore at the ton of Pangkalan Bun to be examined. Twenty-one naval and rescue divers are being sent to the area. He said heavy weather is hampering the search. +07.45 Preparing for the grim reality: +07.40 Debris sighted is the emergency exit door, says Indonesia's National Search and Rescue Agency. The chief says he's 95 per cent sure the flotsam is from the Airbus. The waters where the objects have been found are not deep, the team estimate about 80-100ft. +07.38 Indonesia's director general of civil aviation Djoko Murjatmodjo says: "Based on the observation by search and rescue personnel, significant things have been found, such as a passenger door and cargo door. It's in the sea, 100 miles southwest of Pangkalan Bun," he said, referring to the town in Central Kalimantan, on the island of Borneo. +07.30 Distraught family members are being called in to meet officials +07.18 A navy chief spokesman says "crew had visual of people at sea surface, not far from debris". He says he is also checking whether anyone is still alive. The waters in the Java Sea are fairly warm, but it has been more than 40 hours since the crash. +07.12 It seems to have been confirmed. Debris is "from AirAsia plane", Indonesian civil aviation chief says. "For the time being it can be confirmed that it's the AirAsia plane and the transport minister will depart soon to Pangkalan Bun," Djoko Murjatmodjo said. +07.00 Distressing news, but Jakarta-based Kompas TV has said that the co-pilot of an Indonesian air force C130 Hercules is reporting floating objects resembling humans, suitcases, buoys and aircraft debris. A reminder - this is all siz miles from where AirAsia flight QZ8501 was last logged on radar. Airborne search teams are naturally concerned that the debris could be moved by the current in the time it takes one of the ships to reach it. +06.50 It is nearly 2pm in Indonesia, and the sun sets around 6pm, so they have a little time to try to get search ships to the area and gather the debris to analyse. +06.46 For reference, this is what an Airbus 320 emergency slide looks like: +06.45 "The debris is red and white," Djoko Murjatmodjo, acting director general of air transportation at the transportation ministry, told reporters. "We are checking if it's debris from the aircraft. It's probably from the body of the aircraft." +06.38 BREAKING: An Indonesian transport ministry official says debris found in waters near Borneo is likely from Flight QZ8501. The Acting Director General of Transport Murjatmodjo also says he "strongly suspects fragments are part of AirAsia plane." +06.15 The AFP photographer on board the search flight has the taken the clearest photos yet of the debris: +06.05 “These look like items that are not usually seen on sea surface,” Indonesian air force official Agus Dwi Putranto has said. +06.00 Items resembling an emergency slide, plane door and other objects were spotted during an aerial search Tuesday for missing AirAsia flight 8501, Indonesian officials said. +"We spotted about 10 big objects and many more small white-coloured objects which we could not photograph," Indonesian air force official Agus Dwi Putranto told a press conference. "The position is 10 kilometres (six miles) from the location the plane was last captured by radar," he said. +He displayed 10 photos of objects resembling a plane door, emergency slide, and a square box-like object. +An AFP photographer on the same flight that spotted the debris said he had seen objects in the sea resembling a life raft, life jackets and long orange tubes. +05.35 This just in: Items resembling an emergency slide and a plane door have been spotted in AirAsia search, according to Indonesia and as reported by AFP. The debris is said to be to be seen 105 nautical miles from Pangkalan Bun. Jonathan Pearlman, our correspondent in Jakarta, has this: +Quote The search authority has confirmed that an object believed to be from the plane has been spotted in the south-west area of the search zone. It was spotted at 10.15am, local time, apparently by a plane from the National Search and Rescue Agency. An agency official told me that a fisherman spotted the object and reported it to the agency, which sent a plane to investigate. Apparently, they saw numerous objects including what appeared to be a flashing light. They are attempting to retrieve the objects now.05.12 Our correspondent in Surabaya, Tom Phillips, has just been to a press conference with the families. He says they will be flown over the search area tomorrow on a specially chartered Airbus 320 - the same plane as the QZ8051. "They believe their presence, their praying, will help the search and rescue to quickly find the aircraft', the AirAsia Indonesia chief says. +04.20 Officials have said the sea in the general search area was only 150 to 300 feet deep, which would be a help in finding the plane. +"The Java Sea area where they are now searching isn't even an ocean, it's more of an inland sea," Erik van Sebille, a physical oceanographer at the University of New South Wales in Sydney told Reuters. +"It's so shallow that they may just be able to spot the plane," said van Sebille, noting that sunlight travels through water up to about 100 metres. +Oceanographer Pattiaratchi said debris would normally be expected to float on the surface for around 18 days before sinking. +USS Sampson, the American warship on its way to help with the search: +04.05 In answer to our earlier question: why are planes still not fitted with the necessary technology to help find them if they lose contact with ground control? Wall Street Journal is reporting that AirAsia had adopted dedicated tracking technology but the QZ8501 had not been upgraded yet: +04.00 Unfortunately the search area only seems to be getting bigger, not smaller, as the hours pass: +02.50 The cause of the smoke could be any number of factors so should be treated with caution. One aviation expert has an alternate theory as to what is causing it: diff --git a/tests/data/text/telegraph.co.uk2.txt b/tests/data/text/telegraph.co.uk2.txt new file mode 100644 index 00000000..4b41a9f5 --- /dev/null +++ b/tests/data/text/telegraph.co.uk2.txt @@ -0,0 +1,45 @@ +Margaret Thatcher attempted to put off the introduction of GCSEs because she feared the exams would lead to lower standards and a “can’t fail” mentality among pupils, newly released files show. + +In comments which are likely to be seen by Conservatives as a further vindication of their sweeping reforms of the exams, the then prime minister said the system would allow results to be distorted by “biased” teachers helping teenagers with coursework. + +Previously unseen papers show Mrs Thatcher warned six months before GCSEs replaced O-levels in 1986 that she did not “like the sound of the new exam”, and asked for its introduction to be delayed. + +However she eventually concluded that to intervene would amount to a public “contradiction” of Keith Joseph, the education secretary and a close friend, and appear as if she was taking the side of teaching unions, which wanted more time to prepare for the new system. She therefore had “no option but to go ahead”, she told aides. + +Her previously unknown concerns are revealed in official papers from 1986 released by the National Archive in Kew, west London. + +Her fears appear to chime with the views of Conservatives about the GCSE system today - almost 30 years later. + +Michael Gove, who was education secretary until the summer, is said to believe that the introduction of the exam was a “historic mistake” that has led to a dramatic fall in standards. Before stepping aside to become Chief Whip he set in train an overhaul of the curriculum which he said would address “the pernicious damage caused by grade inflation and dumbing down”. + +GCSEs were eventually introduced in September 1986 with Kenneth Baker, now Lord Baker of Dorking, as education secretary, after Mr Joseph stepped down in May - weeks after his clash with Mrs Thatcher over the exams. + +Mr Joseph had insisted that the new scheme was intended to better stretch teenagers, creating a tougher, “clearer and fairer” system. Under the old system, as part of which more academic teenagers took O-levels and others took CSEs, individual grades were awarded largely based on the relative performance of competing candidates. + +GCSEs were intended to ensure a focus on “how much or how little pupils understand, know and can do”. + +Mrs Thatcher, herself a former education secretary, raised her concerns about GCSEs with Mr Joseph in the spring of 1986. However on March 6 Mr Joseph wrote to her insisting she was “misleading herself” about the exams. + +He insisted that the new system would “inject more rigour” and “use-able learning” and would be “a key instrument for improving standards”. + +Mrs Thatcher marked his three-page briefing note with a series of hand-written annotations, complaining about his use of “an awful lot of high language” and questioning a number of his claims. + +Mr Joseph said that under the O-level and CSE system pupils were “simply ranked in merit order, with little regard to how much or how little pupils understand, know and can do”. + +But in a hand-written annotation to his letter Mrs Thatcher said: “This is not a correct judgement of the present examinations system. We were taught to think and apply 50 years ago.” + +Mrs Thatcher was advised by the No 10 policy unit to postpone the new system until it was clear that it was “workable”. In one briefing note she was warned that “GCSE is an exam nobody will fail” and “does little for the lowest 30 per cent of students”. Implementing the new system in September was a “hopelessly unrealistic” prospect, an official said. + +In a summary of Mrs Thatcher’s concerns, dated March 18 1986, Mark Addison, then her private secretary for home affairs, said the prime minister believed the new approach would lead to “lower standards; a shift away from the traditional approach to learning in favour of a ‘can’t fail’ mentality; assessment by the pupils’ own teachers with the consequent risk of introducing more bias.” + +Mrs Thatcher had not, Mr Addison added, been impressed “by the jargon-soaked justifications” of the exam produced by Mr Joseph’s department. + +She asked for implementation of the exam to be postponed for at least a year, in line with the demands of many teachers, to help ensure the syllabuses were “sufficiently rigorous” and the coursework “properly assessed”. + +However in early April she acquiesced with Mr Joseph’s insistence that the Government should hold its course, agreeing with Tim Flesher, another of her private secretaries, that to back down would “look like taking the side of the unions”. “I agree - no option but to go ahead,” she said. + +Asked about the disclosures, Lord Baker told the Telegraph: "She was concerned ... because she always felt that whenever you change anything in education it might be for the worse." + +However, he added: "In defence of Keith I don't think she fully appreciated that the great thing about GCSEs is it did away with CSE, which was virtually valueless." + +The standard of GCSEs was initially "high", Lord Baker added, saying there had been a "degeneration" in grades over time. \ No newline at end of file diff --git a/tests/data/text/theatlantic.com1.txt b/tests/data/text/theatlantic.com1.txt new file mode 100644 index 00000000..b0279ec4 --- /dev/null +++ b/tests/data/text/theatlantic.com1.txt @@ -0,0 +1,21 @@ +After signing up to write a script for Croatian television, I learned that virtually all TV comedies, from Seinfeld to South Park, follow a simple formula. + +As happens to so many of us, I was asked to write a sitcom for Croatian television. I’m an American ex-pat living in Slovenia, and I know next to nothing about Croatia, besides the fact that it’s Slovenia’s southern neighbor, a fellow ex-Yugoslav republic, and that the language resembles Slovene except with a lot more “js” in it. I am a writer of books and articles, and I used to write a lot of plays, but I’ve never written for television. So I immediately said, “Sure, of course I can do that,” before rushing off to Google “How to write a sitcom.” + +In addition to much Googling, I spent a good deal of time watching sitcoms. I was after tips on how they are constructed, and watched actively, looking to crack open their laugh-tracked shiny exterior to get at the goopy mechanism within, to see how they functioned. What I found out surprised me, and changed the way that I watch television. + +From The Simpsons to Seinfeld, from Everybody Loves Raymond to Everybody Hates Chris, from Taxi to Arrested Development to Parks & Recreation, there is a highly-specific, minute-by-minute recipe used to write the vast majority of sitcoms out there. And once you know the formula, it makes it much easier to write them, and much harder to watch them without seeing that formula—the “sitcom code”—everywhere you look. + +My giddy-panicked Googling actually produced fruitful results. With little idea as to where I should begin, I turned to the confidence-inspiring blog, Wise Sloth (whose author, like me, has no TV writing experience), which provided a 15-page breakdown of sitcom formats that I used as a point of departure for my own study. And by study, I mean hopping into my pajamas, cuddling up to my Peruvian Hairless, and watching TV with a notebook in hand. Talib Visram recently wrote in The Atlantic about his experience counting jokes per minute in popular TV shows. My approach was more deconstructionist, and directly applicable to my new gig. I had to figure out how such shows were built, and fast. + +Fortunately, the answer presented itself very quickly. + +First of all, word-processing programs often come with screenwriting templates. FinalDraft, the most popular software for those penning scripts, even has a Sitcom Template, which of course makes life much easier. But as for how to construct an episode, various bloggers, from the Wise Sloth to helpful folks at the BBC, noted a basic structure that I immediately recognized in every sitcom episode I tested. This structure is so formulaic that you’d think it would suck the fun out of writing and watching such shows, but it does nothing of the sort. While knowing the code it changes the way I watch TV, it only increases my admiration for the good writers who do so much within relatively strict confines. + +To demonstrate how this formula works, I’ve chosen an episode of a favorite show, somewhat at random, because it ideally exemplifies the template: episode 4 of season 1 of Parks & Recreation. + +The Sitcom Code breaks down what needs to happen in each episode, by the minute. As Dan Richter of Demand Media notes, “Sitcoms, minus commercials, are typically 22 minutes long [with] a script of 25-40 pages. Every sitcom episode has a main plot (story A), as well as one or two subplots (stories B and C).” There are three main acts, divided by two commercial breaks (in most American TV), with 3-5 scenes per act. One of the distinguishing characteristics of sitcoms, as opposed to other forms of television, is that the main protagonist(s) barely change from one episode to the next, let alone from season to season (Maggie Simpson has been sucking on a pacifier for nearly thirty years). Therefore whatever happens in the episode, the situation must end largely where it began. The Wise Sloth points out that 22 minutes is “not even really time enough to tell a full story. The whole story has to be on fast-forward,” so simplification is key. + +Poet Philip Larkin described all plots as “a beginning, a muddle, and an end,” which is as good a description as any. Each episode begins with the protagonist stating a goal or problem that must be solved, and which we understand will be solved by the end of the episode. If the problem is solved too quickly, then the episode won’t stretch out to 22 minutes, so the first attempt at reaching the goal or solving the problem must fail (“the muddle”), requiring a new approach, before the episode ends and the protagonist either does, or does not, achieve what they set out to do. The goal might be Homer trying to make a fortune by selling recycled grease in The Simpsons, or Job Bluth setting out to sabotage the family’s banana stand in Arrested Development, or the Seinfeld crew looking for where they parked in a vast lot. Another hallmark of sitcoms is that the protagonists frequently fail, and we often want them to, because we do not want our favorite characters to change too much. If Leslie Knope ever left Pawnee for a career as a DC politician, we would be distraught. If Kramer got married and moved to the suburbs—whoa, now! + +When writers sit around and prepare a new episode, many literally map out what will happen, minute-by-minute, in the main storyline and sub-storylines, filling in jokes later. Let’s see how this played out in the Parks & Recreation episode, “Boys' Club.” \ No newline at end of file diff --git a/tests/data/text/theatlantic.com2.txt b/tests/data/text/theatlantic.com2.txt new file mode 100644 index 00000000..788692b8 --- /dev/null +++ b/tests/data/text/theatlantic.com2.txt @@ -0,0 +1,25 @@ +Despite the controversy, the James Franco-Seth Rogen flick is surprisingly relevant, very funny, and deserving of praise. + +Let's assume you knew absolutely nothing about North Korea and walked into The Interview, which after a very public and protracted to-do, is now showing at a few hundred American cinemas and streaming through a number of outlets online. + +Given what's been said about the movie, would you expect it to tell you about North Korea's concentration camps? Or the complexity of the dictatorship's propaganda system in which its leaders are touted as deities? Would you expect to hear (twice) that 16 million of the country's 24 million people are malnourished? + +Okay, sure, The Interview might feature North Korean dictator Kim Jong Un wondering aloud about the relative "gayness" of drinking margaritas and affirming his love of Katy Perry. But caricature notwithstanding, the comedy has a surprisingly nuanced streak in its celluloid sea of swears and weiner jokes. + +The Interview centers around Dave Skylark (James Franco), a fratty, self-absorbed, and disconnected celebrity TV journalist, and his producer Aaron Rapoport (Seth Rogen), who are pushed by the CIA to turn a massive scoop (an interview with reclusive North Korean dictator Kim Jong Un) into an assassination mission. Rapoport, believe it or not, laments his station as the producer of tabloid garbage and sees the Kim interview as an opportunity to perform an admirable act of journalism. + +What happens next is some of what you might expect (sex, drugs, anatomy jokes) from Franco and Rogen, whose recent projects have included Pineapple Express and last year’s This Is the End. (Franco earned a Golden Globe nomination for the former and the latter still enjoys considerable sleeper praise.). + +But as Skylark preens as an impressionable outsider, his gullibility and ignorance allow him to be a useful mechanism for explaining North Korea's depravity. He falls dumbly under the sway of Kim and the North Korean rhetoric machine, which paints the regime as an honorable force against the oppressive West. A major plot point turns on the dim Skylark's realization that a lush-looking grocery store he spotted earlier is a total fake. During a live-tweeting of the film on Sunday, Seth Rogen noted, "They actually have fake grocery stores in Pyongyang." + +In a master stroke (spoilers ahead), the duo decides to flout the CIA's assassination order, which they determine could bring a possibly worse replacement than Kim to power. Instead, they use the live interview with Kim to destroy his credibility by stating the facts. It's not exactly Mike Wallace's showdown with Ayatollah Khomeini, but what is? + +At New York, David Edelstein says that critics of the film who reduce it to a silly and sophomoric bromance, "don't know what the hell they're talking about." He adds: + +To that Jay McInerney adds, "Forget what you heard. #TheInterview gets my vote for best picture. Sadly I'm not a member of the Academy." + +The film doesn't skate away without some problematic parts. In the end, Kim Jong Un's head does explode when a tank shell fired by Rogen and company hits the dictator's helicopter. As Uri Friedman pointed out, the decision to show Kim's death, in slow motion and while Katy Perry's "Firework" twinkles in the background, was worrisome given that "that leader's government, which presides over nuclear weapons...has described the movie as an 'act of war.'" It probably doesn't matter that Kim's death happens during a getaway battle rather than a result of an assassination. + +Nevertheless, on Saturday, as if to prove the movie's point, a spokesman for the National Defense Commission, North Korea's highest governing body had this to say about The Interview, the release of which it blames on President Obama: "Obama always goes reckless in words and deeds like a monkey in a tropical forest." + +After all the fuss, it's entirely understandable that many object to seeing the real, living leader of a rogue country killed onscreen. But what surprises is that The Interview also spotlights other truths that North Korea doesn't want people to hear about. Given the stakes involved, that's important too. \ No newline at end of file diff --git a/tests/data/text/theatlanticcities.com1.txt b/tests/data/text/theatlanticcities.com1.txt new file mode 100644 index 00000000..70930c02 --- /dev/null +++ b/tests/data/text/theatlanticcities.com1.txt @@ -0,0 +1,11 @@ +Antarctica remained largely untouched until roughly 200 years ago, and now, more than 10,000 people travel there every year. But tourists bring more than cameras. Scientists are warning that pathogens brought by visitors could threaten the continent’s most iconic inhabitant: the penguin. + +Isolation has left local wildlife populations particularly vulnerable to diseases commonplace elsewhere in the world. “The effects of both a growing tourism industry and research presence will not be without consequences,” Wray Grimaldi of the University of Otago in Dunedin, New Zealand, said to New Scientist. “Penguins are highly susceptible to infectious diseases.” + +Her team of Antarctic researchers found multiple infectious agents—bacteria such as salmonella and E. coli, viruses such as West Nile and the Avian pox virus—in captive penguins dating back to 1947. Outbreaks from those diseases have killed thousands of penguins over the years, the team reported in a paper published this month in the journal Polar Biology. + +Another theory is that migrating animals may have brought diseases to Antarctica, as the warming climate is attracting more species than ever before. But previous studies have identified tourist boots as vectors for disease transmission. One group of researchers tested 72 tourists' boots and found 20 different fecal pathogens on just 15 pairs of shoes. + +Norman Ratcliffe, an Antarctic ecologist from the Antarctic Survey in Cambridge, United Kingdom, told New Scientist that the evidence blaming tourists for sick penguins is lacking. He said that tourism companies are very strict on what they let visitors bring on their journey. “The tour companies are quite careful to make sure everyone cleans their boots before they go ashore,” he said. “They don't allow any animal products to be taken ashore.” + +This story originally appeared on The Atlantic. \ No newline at end of file diff --git a/tests/data/text/theatlanticcities.com2.txt b/tests/data/text/theatlanticcities.com2.txt new file mode 100644 index 00000000..d6432cc3 --- /dev/null +++ b/tests/data/text/theatlanticcities.com2.txt @@ -0,0 +1,17 @@ +When warp-speed Santa drones are shooting Christmas presents down the air holes of our Disastro-Bunkers in 2089, will there be snow on the ground? + +That's a ridiculous question to ask—and now it's been answered by David Taylor, a 44-year-old data scientist and writer in Montreal, who's made an animation of predicted white Christmases for each remaining year of the century. + +Taylor is aware that, by its nature, this simulation includes many wild misses. Today, a snowstorm's exact movements are difficult to nail down three days in advance; guessing whether there will be powder in six decades on December 25 is harder than keeping Jim Cantore silent during thundersnow. The humor of this impossible task was part of what made Taylor want to attempt it. + +"Basically, I was cruising around the Statistics Canada website looking for interesting data that hadn't already been mined out, and I came across this huge collection of climate models going to the year 2100, containing day-by-day predictions of atmospheric conditions including temperature and precipitation," he emails. "I found that amusing because even the seven-day weather forecasts are so often way off. But of course the accuracy of day-by-day predictions aren't the point of the model, they're just the product of the model." + +The model Taylor relied on, the CanRCM4, generated a few interesting trends over the century. The effects of climate change are obvious, with the snowpack receding evermore north like the white hairline of a balding Kris Kringle. Within this general pattern, though, are what Taylor calls "mini-cycles of a few years of cooling, then a few years of warming." There are also a couple of mystery zones, where snow or barrenness stubbornly reign. + +"There are... these persistent spots in and around Utah that are always snow-covered no matter what's going on in the rest of the continent, and there's this spot between Walla Walla and Spokane, Washington, that's always snow-free even though it's surrounded by snow," he says. "I'm just a data analyst and programmer, not a climatologist, but my first guess is that the Rocky Mountains have something to do with the fact that snow cover seems less variable in the West than the East." (It might also relate to how climate change is building a hot/cold divide between America's coasts.) + +So why did Taylor pick this treasured event for scrutiny? Is his first holiday memory waking up to ivory drifts pressing against the window, signaling school cancellations stretching as far as the mind's eye could see? + +Not really. "Well, the data is arranged day by day, so Christmas seemed the natural choice," he says. As for his own weather experiences, he's only seen one white Christmas in his life, way back in 1997. + +"It's ironic because most of the rest of Canada had snow on the ground, but Edmonton, which is the large city the furthest north, was experiencing a Chinook (warm air from the Rockies) and didn't," he says. "In Edmonton, there's usually snow on the ground to stay by Halloween." \ No newline at end of file diff --git a/tests/data/text/thedailybeast.com1.txt b/tests/data/text/thedailybeast.com1.txt new file mode 100644 index 00000000..e8cbf4e1 --- /dev/null +++ b/tests/data/text/thedailybeast.com1.txt @@ -0,0 +1,25 @@ +The Catholic Church released its final report on its investigation into ‘feminist’ American nuns—and it took a softer stance toward the sisters. + +VATICAN CITY—Six years ago Pope Benedict XVI aimed to curb “a certain feminist spirit” and “secularist mentality” that the Church fathers discerned, to their dismay, in the ranks of American nuns. On Tuesday, when the Vatican under Pope Francis delivered its final report on the subject, those concerns were nowhere to be found. + + + +The report was the fruit of an investigation into 341 American congregations guided by Mother Superior Marie Clare Millea, a matronly sister who became tearful at times, in front of a packed pressroom, while describing how she went about collecting her data. Only the cloistered convents were excluded, and not all of the religious orders complied with the visitors’ requests and questionnaires, though Millea was unable to recall just how many or what percentage of American nuns refused to cooperate. + +The focus of her investigation was on how nuns live, work and pray, ranging from issues of communal prayer and community service to whether nuns who chose not to wear a religious habit were hiding their faith. She wrote individual reports on each of the congregations she visited, and Tuesday’s report was the Vatican’s synthesis of those findings. + +The report concluded that while all American nuns need to recheck the rulebooks to make sure they are living their vows as they made them, they were largely doing what they were supposed to be doing. “Women religious have courageously been in the forefront, selflessly tending to the spiritual, moral, educational, physical and social needs of countless individuals,” the report said with no mention of secularism or feminism at all. + +The report will undoubtedly be welcomed by most American nuns, but few will forget the fighting words from a separate investigation called a doctrinal assessment that is not nearly as conciliatory. In 2009, the Vatican’s Congregation for the Doctrine of the Faith started a separate investigation into an umbrella group called the Leadership Conference of Women Religious (LCWR) that represents roughly 80 percent of American nuns. Then, the Church accused the LCWR of “pushing radical feminist themes incompatible with the Catholic faith.” + +The clampdown backfired, spawning a social media frenzy with more than a million tweets under the hashtag #whatsistersmeantome in support of the nuns and their role in holding Catholic communities together. At the time, Father James Martin, a Jesuit priest and author of several faith-based books, who was the first to tweet under #whatsistersmeantome, warned the Vatican not to mess with the nuns. + +“There is a danger of backlash because of the esteem [in which] so many Catholics hold nuns,” Martin told The Daily Beast at the height of the scandal. “For many Catholics, sisters are the glue that holds the church together.” + +There was much hope when Pope Francis was elected that he would show mercy to the American sisters. But in April, Gerhard Müller, the prefect for the Congregation for the Doctrine of the Faith, quashed that hope, insisting instead that Francis backed up the 2009 report. Muller criticized the nuns’ choice of speakers for their annual event accusing the sisters of being overtly provocative. “This is a decision that will be seen as a rather open provocation against the Holy See and the doctrinal assessment,” Mueller said in scathing remarks. “Not only that, but it further alienates the LCWR from the bishops, as well.” + +The Vatican is still working with the LCWR to find a middle ground and, despite what amounted to a lovefest at Tuesday’s press conference, there is still obvious concern. Sister Sharon Holland, the head of the LCWR, who incidentally was the only sister on the stage without a religious habit, told reporters that there were still hard feelings. “I’m concerned about those who may still be angry,” she said. “It’s a concern to me because it’s not healthy to remain angry.” + +Prior to the publication of report, several sisters who coauthored a book called The Power of Sisterhood that was borne of the apostolic visitation (and not the LCWR doctrinal assessment), voiced their concern for the damage the investigations had done. “The Apostolic Visitation broke open the heart of who we are,” said Mary Ann Zollman at an event to launch their book at Loyola University. “It was met with disbelief and confusion. We were hurt and angry.” + +But she said through the accusations from Rome and the hurt they caused, the American sisters have banded together. “This wasn’t about one religious congregation or religious organization. It was about all of us together,” she said. “It created a sense of solidarity and sisterhood. And the same was true with our relationship with the laity—we discovered a communal vision for the church and the world.” \ No newline at end of file diff --git a/tests/data/text/thedailybeast.com2.txt b/tests/data/text/thedailybeast.com2.txt new file mode 100644 index 00000000..42c9900d --- /dev/null +++ b/tests/data/text/thedailybeast.com2.txt @@ -0,0 +1,43 @@ +At a peaceful gathering of the Justice League, they condemned the killings of Officers Liu and Ramos, but also wondered why the slaying of citizens by cops doesn’t spark as much outrage. + +A modest crowd moved East on 110th Street in New York City on Sunday evening. They walked silently, some carrying Anthora cups illuminated by candlesticks, others holding plastic tea lights handed out by protest organizers. A few carried signs: "IMAGINE JUSTICE," "BLACK LIVES MATTER," "CLAIM HUMANITY." But mostly they just walked, their faces somber, their hands shaking as the snow began to fall. + +The peaceful vigil was somewhat unexpected, given the heightened tensions between protesters as the NYPD in recent weeks, culminating in the murder of two officers in Bed-Stuy on Saturday afternoon. + +Just over 24 hours before the silent march down 110th, Wenjian Liu, 32, and Rafael Ramos, 40, had been shot "execution-style," Mayor Bill DeBlasio said, while sitting in their patrol car. The suspect, Ismaaiyl Brinsley, had begun the day with the shooting of his ex-girlfriend in Maryland before taking the trip up to New York, armed with a silver semi-automatic handgun. After murdering the officers, Brinsley walked over to a subway platform where he turned the weapon on himself. + +In some ways, the tragedy seemed inevitable. + +As the nation continued to cope with Ferguson, a grand jury in Staten Island chose not to indict in the case of Eric Garner, a man choked to death –– as he pleaded "I can't breathe" –– by a police offer in what the coroner declared a homicide. #BlackLivesMatter has been the prevailing message as protesters have taken to the streets to express their outrage. They often chant: "No justice, no peace; No racist police." + +All the while, there have been critics. Why, they ask, do black lives matter? Why not all lives? They contend that the protests are not anti-police brutality, but anti-police, period. They believe that the protesters and those who enabled them—the media, or De Blasio, who spoke frankly about how he talks to his biracial son, whose mother is black, about the threats he might face due to his skin color—have been vying for NYPD scalps. With two now dead, the critics are pointing fingers. + +"The mayor’s hands are literally dripping with our blood because of his words, actions and policies and we have, for the first time in a number of years, become a 'wartime' police department," Pat Lynch, head of the police union, said at a press conference on Saturday night. "There's blood on many hands tonight…That blood on the hands starts at City Hall in the Office of the Mayor." + +Former mayor Rudy Giuliani echoed the sentiment, but laid blame on President Obama: "We've had four months of propaganda starting with the president that everybody should hate the police…The protests are being embraced, the protests are being encouraged. The protests, even the ones that don't lead to violence—a lot of them lead to violence, all of them lead to a conclusion: The police are bad, the police are racist. That is completely wrong." + +The march, organized by the Justice League NYC—effectively anointed by De Blasio as City Hall's preferred protest group with a 4-minute meeting to discuss theories of policing—continued slowly along 110th street, a police van humming alongside. The crowd snaked into Central Park, where it came to a stop at the boathouse, a lighted Christmas tree illuminating the water's surface. + +"Let me not say this wrong," a 21-year-old East Flatbush resident who would say only that his name was "Perry" told me as he looked around, seemingly searching for the right words. "Yesterday's situation that happened in Brooklyn, Bed-Stuy, was uncalled for. It was the wrong way about handling it. Because we're upset that on our end, they're killing us, so I don't think nobody should go take the anger out on, you know—nobody knows these two specific individuals. They could've been good people. They could've been, you know, legit officers who wasn't down with whatever's going on. My condolences do go to their families. But at the same time, since Zimmerman, you've been hearing cases about officers killing black children—black men, Hispanic men. + +"This is not about who did it more, but it's more: all these lives didn't matter, but now two cops are dead, and now it matters? If you look at the newspapers, it's all broadcast all over the place. The whole world's supposed to mourn because—don't get me wrong, it was wrong––but because two officers died, now the world's supposed to mourn? What happened to all the other people that died? Nobody mourned. They wasn't in the front page of newspapers. They was in back pages—some didn't even make the newspaper." + +Next to "Perry" stood Jonathan Alvarez, 18, of Queens, who in November made headlines when he announced plans to sue the city after allegedly being assaulted by police officers for being a "smart guy." + +"I got targeted. I'm a victim of police brutality," Alvarez told me. "They arrested me, they beat me up in the 102 precinct while I was handcuffed, no justice. So I feel for these people, I understand they're just brutes. They can get away with it, and they do…They don't give a fuck until one of them dies, that's all they care about." + +"Perry" chimed in: "They don't want to take blame," he said of the police union attacking De Blasio. "There has to be peace on all sides. + +"I think this all could've been prevented if they just listened to the 25,000 people who marched last week," Alvarez said. "No, not even," "Perry" countered: "If they would've just re-analyzed the Zimmerman case, or the Ferguson case." + +Then, "Perry" and Alvarez went silent. The marchers began to stream out of the park, where they walked West on 110th and then hung a right on 7th Avenue. NYPD officers lingered, politely asking participants to move in the line, and saying "thank you" when they obliged, which they all did. + +The march reached its conclusion at the First Corinthian Baptist Church on 116th street, where piano music befitting a funeral greeting guests who filed into the warm pews. + +Religious leaders—Pastor Willie Francois III, the Rev. Mike Walrond, and the Rev. Stephen Phelps—and over a dozen activists, led by Justice League NYC's Tamika Mallory, took turns addressing the crowd. + +There were prayers for 40 black men, beginning with Eric Garner, who had been killed by police; and prayers for De Blasio, who "is attacked on one side and the other," and repeated acknowledgement that violence is not the way to change what the protesters believe is a fundamentally broken system. + +"We're not anti-police, we're anti-police brutality," a member of the Justice League told the crowd, to cheers. "Every cop isn't bad…Every black man isn't a criminal." + +The crowd rose, embraced each other in groups of two and three, and prayed. \ No newline at end of file diff --git a/tests/data/text/thedebrief.co.uk1.txt b/tests/data/text/thedebrief.co.uk1.txt new file mode 100644 index 00000000..e1e85188 --- /dev/null +++ b/tests/data/text/thedebrief.co.uk1.txt @@ -0,0 +1,23 @@ +So in our eyes Ed Sheeran can do no wrong. He’s the most loveable, adorkable (that’s adorable crossed with dorky FYI) pop star EVER. And we love him, we really do. However when it comes to his matchmaking skills for our fave gal, and his BFF, Taylor Swift – well we think he’s a little bit off. + + + +This week Ed told Now magazine that he hoped he could set up Taylor with none other that Orlando Bloom. He said ‘He’s lovely, and they live in the same building. [I'm hoping that] the magic might present itself eventually.’ + + + +Er, we’re not so sure. + +Now don’t get us wrong we’ve had a soft spot for floppy haired Mr Bloom since his Pirate days, and when he took a swing at Bieber earlier this year well we know which side of the ring we were on. + +But is he worthy of all that is T-Swift? The jury is out. And let’s not forget Bloom's also already dated Taylor’s BFF Selena, we predict that might make this matchmake very unlikely as Taylor is 100% 'Sisters Before Misters'. + +Ultimately we think Tay can do better, but then we're not sure there is any man who is worthy! + +Like this? You might also be interested in: + +Lena Dunham Has Declared Today Taylor Swift Day. Here’s Why We All Need To Get Better At Celebrating Female Success + +South Park Does A Lorde Skit. But Who’s It Taking The Piss Out Of? + +Lorde Gave Taylor Swift The First Listen To Her New Song \ No newline at end of file diff --git a/tests/data/text/thedebrief.co.uk2.txt b/tests/data/text/thedebrief.co.uk2.txt new file mode 100644 index 00000000..329602b4 --- /dev/null +++ b/tests/data/text/thedebrief.co.uk2.txt @@ -0,0 +1,39 @@ +Some absolute genius of a woman has written a break-up letter that comprises entirely of Taylor Swift lyrics because, let’s face it, pretty much every single Taylor Swift song is about a break-up. Apart from... nope. I’ve got nothing. + +Anyway, while T-Swift can provide excellent firepower for a particular kind of breakup letter – what if you want to go a bit more ballsy? Like, Beyonce ballsy? Wonder no more because we’ve channelled Queen Bey into the perfect method for getting rid of a cheatin’ man. Or woman. Or someone who hasn’t cheated at all, but you’re just not feeling it anymore. + +READ MORE: Style And Design Tips From Taylor Swift's New Video For Blank Space + +Hey babe, + +There was a time I thought that you did everything right. No lies, no wrong. Boy I, must’ve been out of my mind, so when I think of the time that I almost loved you, you showed your self and I saw the real you. If I were a boy, I think I could understand how it feels to love a girl – I swear I’d be a better man. I wanted you bad – I'm so through with that, because honestly, you turned out to be the best thing I never had. I’m taking back the things I got from you, on top of you not calling me back – you see I bet you think it’s all on track, but what about my body? You would rather go and party. + +I give you everything you want everything you need. Even your friends say I’m a good woman, all I need to know is why? Why don’t you love me? I got beauty, I got class, I got style, and I got ass and you don’t even care to care. Bet it sucks to be you right now – don’t you ever for a second get to thinking you're irreplaceable; what goes around comes back around. + +Baby I won’t shed a tear for you, I won’t lose a wink of sleep, because the truth of the matter is replacing you is so easy + +[insert your name here] + +Sure, it makes you sound a bit all over the place (while Tay mainly does nostalgic, hurt songs – Bey goes all out raging, as well as hurt, and secure as well as insecure), but what great break-up letter doesn’t? I remember once running down the road in no shoes after an ex yelling, ‘I’M NOT MAD, COME BACK HERE.’ Breaking up is a paradox. + +And Beyonce’s lyrics don’t just go well with relationships, oh no – while Taylor tends to focus on matters of the (broken) heart, Beyonce’s back catalogue works pretty well in most scenarios. Look, there’s a Beyonce for all occasions: + +Quitting your job email: Fuck you, pay me. + +Drunk birthday speech: This goes out to all my girls that’s in the club rocking the latest! Who will buy it for themselves and get more money later. I think I need a barber. + +Letter to your best mate: And if you wasn’t for you, and if I didn’t know you, and if you never reached me, and if you didn’t teach me, I wouldn’t be, who I am right now. + +Discussing your monthly cycle with your GP: I been on, I been on, I been on – tell me who gone take me off, take me off, take me off, take me off, ’cause I been on + +Speaking with changing room attendants: Stop, I ain’t ready yet. Wait, let me fix my hair. I think I’m ready + +Giving sex advice from the 1950s: Ladies look here, when you been with your man for a long time, every now and then you gotta go back in the closet and pull out that freakum dress. + +Like this? You might also be interested in... + +Sold Out Moisturiser Is The Hero Of Beyonce's 7/11 Video + +Beyonce Behaves A Bit Weirdly, The Internet Gets Even Weirder + +A Few Signs We Might Be Drifting Apart From Beyonce diff --git a/tests/data/text/theglobeandmail.com1.txt b/tests/data/text/theglobeandmail.com1.txt new file mode 100644 index 00000000..9023d109 --- /dev/null +++ b/tests/data/text/theglobeandmail.com1.txt @@ -0,0 +1,55 @@ +A modern airliner’s abrupt disappearance from radar as it flew through a line of equatorial thunderstorms – with no distress call received – points to another in-flight “loss of control” aviation disaster. + +On Monday, as hopes of finding survivors of Indonesia AirAsia Flight 8501 faded, searchers in ships and aircraft scoured the Java Sea looking for debris to pinpoint where the Singapore-bound Airbus A320 crashed. + +With 33 million commercial airline departures and 3.2 billion passengers carried in 2014, the rate of crashes has never been lower, though the death rate of passengers is up slightly from its all time low of 0.09 last year. + +SOURCE: Aviation Safety Network; ICAO; WORLD BANK. + +* Accidents refers to the number of fatal airliner (14+ passengers) hull-loss accidents. Does not include corporate jet and military transport accidents/hijackings. + +** Includes 162 unconfirmed passengers from AirAsia Flight QZ8501 + +Nearly two days after the flight vanished, chances of finding anyone alive, even if some of the 162 passengers and crew survived a crash into the sea, were exceedingly remote, but officials were still calling the multinational effort a search-and-rescue operation. + +The fate of Flight 8501 remains perplexing and until the cockpit voice and flight-data recorders are recovered, investigators won’t be able to determine what exactly went wrong during the plane’s final few minutes. + +However, the loss of the airliner invites comparison with the 2009 crash of Air France’s Flight 447. That crash killed all 228 on board the Paris-bound flight from Rio de Janeiro as it flew directly into a line of tropical thunderstorms. + +In the Air France crash, all three pilots, including a veteran captain and two less-experienced co-pilots, were so spatially disoriented that they were still arguing over which way was up when the Airbus A330 slammed into the sea. In just over four minutes, the undamaged and perfectly flyable aircraft fell more than 11 kilometres. The flight’s autopilot had disengaged because of faulty airspeed readings, forcing the pilots to manually fly the aircraft. + +On Monday, two oil patches in the Java Sea east of Belitung island, close to Flight 8501’s last known position, were located by Indonesian air-force helicopters. Several other debris findings were discounted as being unrelated to the missing Airbus. + +Hadi Tjahnanto, a senior Indonesian air-force officer, told MetroTV that the slick samples were being analyzed to determine if they came from Flight QZ8501. + +The search seems certain to shift to a recovery effort soon, focused on finding the flight data recorders that should hold clues as to why a modern, sophisticated jetliner, piloted by an experienced crew working for a major regional carrier with an unblemished safety record, apparently flew straight into the sort of severe thunderstorms routinely found near the equator. + +Massive thunderstorms are common in the tropics and pose a routine, albeit serious, challenge to flight crews. Modern jet airliners can survive even the most severe turbulence and multiple lightning strikes without structural failure, but flight crews are trained to fly around severe storms. Deviation around, rather than over, is standard procedure and considered safer, since some tropical thunderstorms can reach far higher than commercial jetliners can fly. + +The disappearance of Flight 8501 over the weekend caps a catastrophic year for Malaysian aviation, with three major crashes in unrelated occurences. Malaysia Airlines Flight 370, a Boeing 777 en route to Beijing from Kuala Lumpur, disappreared on March 8 with 239 on board. It has never been found and may have been deliberately flown until it ran out of fuel and crashed in the remote south Indian Ocean. On July 17, Malaysia Airlines Flight 17, another Boeing 777, was shot down by a surface-to-air missile fired from pro-Russian rebel-held territory in eastern Ukraine, killing all 298 people on board. + +On Monday, the flamboyant Malaysian founder of the AirAsia group, the low-cost airline with affiliates in half-a-dozen countries, spoke about the loss of Flight 8501. “My heart bleeds for all the relatives of my crew and our passengers,” Tony Fernandes said. “Nothing is more important to us. Until today, we have never lost a life. But I think that any airline CEO who says he can guarantee that his airline is 100-per-cent safe is not accurate.” + +Indonesian President Joko “Jokowi” Widodo ordered an immediate review of all aviation procedures. + +Most of the passengers on Flight 8501 were Indonesians, headed for Singapore on vacation – a two-hour flight from Surabaya, the country’s second-largest city. + +At dawn on Sunday morning, about 50 minutes after takeoff, the twin-engined Airbus A320 was already at its planned cruising altitude and nearly halfway to Singapore, when one of its two pilots asked air-traffic control for permission to climb from 32,000 to 38,000 feet, perhaps in an attempt to climb over severe weather directly ahead. The request was denied because of other aircraft already occupying the higher flight altitudes. A few minutes later – at 6:17 a.m. local time – controllers offered Flight 8501 permission to climb to 34,000 feet, but there was no reply. The twin-engine, single-aisle plane was last seen on radar four minutes after the final communication. Some flight-tracking sites record the last data from its transponder – the device on board modern aircraft that broadcasts position, speed, altitude and flight number – at very low altitude. + +“Based on the co-ordinates that we know, the evaluation would be that any estimated crash position is in the sea,” Indonesia search-and-rescue director Henry Bambang Soelistyo said. + +Search aircraft and ships from several countries converged on a 100-square-kilometre area between the island of Belitung, off Sumatra, and Borneo. The Java Sea is less than 100 metres deep where the aircraft vanished, which should make recovery of the flight recorders easier. + +In Southeast Asia, the AirAsia group has redefined flying with a focus on low-cost, no-frills flights in a rapidly growing market. Its distinctive fleet of more than 160 red-and-white-painted Airbus A320s, with “Now Everyone Can Fly” emblazoned on the undersides, has become a familiar sight in Southeast Asian skies. + +Indonesia AirAsia, run by and 49-per-cent owned by the Malaysian AirAsia parent company, has another 28 Airbus A320s, each capable of seating 180 passengers. It was one of those aircraft that is missing and presumed lost at sea. + +Mr. Fernandes flew to Surabaya on Monday, saying that until the investigation was completed it was premature to speculate on whether procedures or crew training needed to be changed. + +On board the missing flight were 155 passengers, including 17 children and an infant. A crew of seven – two pilots, four flight attendants and, unusually, a flight engineer – were assigned to the flight. + +The captain, Iriyanto, an Indonesian who uses only one name, was a former Indonesian air-force fighter pilot with more than 20,537 flying hours, of which 6,100 were on Indonesia AirAsia Airbus A320s. “Papa, come home, I still need you,” Angela Anggi Ranastianis, his 22-year-old daughter, pleaded in social-media comments. + +The co-pilot was Rémi Emmanuel Plesel, a French citizen who gained his pilot’s licence at age 42 and had 2,275 hours on the Airbus A320. + +Usually, one pilot flies a flight segment while another operates the radios and communicates with air-traffic control. It’s not yet known which pilot was handling the flight. However, in case of difficulties or unforeseen problems, the captain can, and usually does, take control. \ No newline at end of file diff --git a/tests/data/text/theglobeandmail.com2.txt b/tests/data/text/theglobeandmail.com2.txt new file mode 100644 index 00000000..7b64d5d8 --- /dev/null +++ b/tests/data/text/theglobeandmail.com2.txt @@ -0,0 +1,33 @@ +Oil. Russia. Interest rates. The United States. Europe. Commodities. China. Trade. Japan. The global economy enters 2015 with a lot on its mind, and a long list of competing storylines set to develop as the year progresses. Here, four notable economists share their views on the key issue that will grab their attention in 2015 – and how it might unfold, for better or worse. + +Collapsing oil prices are poised to reverberate across 2015. If current pricing sticks, global producers will lose over a trillion dollars in annual profits, crimping their investment intentions and impairing their ability to pay dividends and debts. + +Oil-exporting nations – Canada among them – naturally suffer, with the greatest agony reserved for emerging-market members who must perversely raise interest rates to halt capital flight and combat ballooning U.S.-denominated debt loads. + +Desperate times lead to desperate measures, meaning Russia and the Middle East must be watched with an eagle eye next year. + +However, let us not lose sight of the bigger picture. Oil’s extreme dislocation should partially unwind in 2015. After all, oil supply outpaces demand by a mere 2 per cent. There is no need for extreme cutbacks, especially if demand grows as enthusiastically as our models indicate. + +For that matter, the global economy actually likes low oil prices. For all of the bellyaching, every dollar lost by oil producers is more than recovered across a diffuse set of oil consumers. Indeed, the combination of low oil prices, low bond yields and weakening exchange rates should help revive the moribund European and Japanese economies. This could prove to be the most lasting consequence of oil’s not-so-excellent adventure. + +The year 2014 closed with a focus on the oil price, but developments in other commodity markets will also be worth watching in 2015. The knock-on effects could work in either direction. + +We expect the price of oil to remain low. As energy is an important cost in the production and transportation of other commodities, this is one reason to expect price pressures more generally to be weak. + +Nonetheless, most countries – and the world as a whole – should be better off as a result of lower oil prices, notably China and India. Of the advanced economies, Japan is the biggest winner, but there should also be sizable boosts to the U.S., euro zone and Britain. The upshot is that cheaper oil should support a rebound in the prices of industrial metals as economic activity picks up. + +What’s more, China and India are the two largest markets for gold. While inflation will be lower as a result of the fall in oil prices, global monetary policy is also likely to be looser for longer. We, therefore, expect base and precious metals prices to do well in 2015, improving the prospects for miners and the more diversified commodity economies. + +What should we expect from the world of central banking in 2015? I see two major forces. + +First, given that real gross domestic product in the euro zone is still 2 per cent below the 2008 level, there’s a pretty clear economic case for more fiscal and monetary stimulus. On the other hand, the political forces in opposition to such actions continue to be massive. My best guess for 2015 is that the euro zone will continue to muddle through, without much new stimulus of any kind. + +Meanwhile, the United States is showing ever-more promising signs of an economic renaissance. The Federal Reserve’s balance sheet has stopped its expansion, and most observers think that policy interest rates will begin their ascent in 2015. While such rate increases may wreak havoc on financial markets for a while, their existence will be a positive sign for the United States and for the world. + +These different economic scenarios suggest that the U.S. dollar will be strengthening against the euro. How the Canadian dollar fares in such a situation is unclear, although the single best predictor for the Canadian dollar is still the path of world commodity prices – which is pretty much anybody’s guess. + +My biggest worry for 2015 is of a default by Russia on its foreign debt. Vladimir Putin could intentionally renege on his foreign debt obligations – an Argentine approach – or Russia could just run out of liquid foreign-exchange assets. Russia owes $670-billion (U.S.) to foreigners. No one is sure who owns it all. Most of it likely is not in the hands of banks. We should remember that LTCM [Long-Term Capital Management] in the U.S. was not a bank, but its failure did a lot of damage to the financial system. + +I am also waiting to see if an anti-austerity government comes to power in Greece and defaults on its debts to EFSF, the IMF and Euroland governments. EFSF has sold €180-billion ($254-billion Canadian) in bonds, backed largely by loans to Greece. In the worst case, banks and hedge funds owning these AAA bonds will find them downgraded. Some of them may fail. Euroland’s governments may have to ante up real money to recapitalize EFSF, triggering a fiscal crisis. + +Finally, I worry that Britain’s current account deficit – at a record nearly 6 per cent of GDP – is wide enough to trigger a sterling crisis. The government has no strategy for external deficit reduction. A sterling crash would blow out inflation; the interest rate response would kill the economy. \ No newline at end of file diff --git a/tests/data/text/thekitchn.com1.txt b/tests/data/text/thekitchn.com1.txt new file mode 100644 index 00000000..c88c0fcb --- /dev/null +++ b/tests/data/text/thekitchn.com1.txt @@ -0,0 +1 @@ +Kristin is a contributing editor for The Kitchn. A former editor at Real Simple, she is compulsively organized and loves solving people's problems. She has a weakness for desserts, especially ice cream. \ No newline at end of file diff --git a/tests/data/text/thekitchn.com2.txt b/tests/data/text/thekitchn.com2.txt new file mode 100644 index 00000000..c88c0fcb --- /dev/null +++ b/tests/data/text/thekitchn.com2.txt @@ -0,0 +1 @@ +Kristin is a contributing editor for The Kitchn. A former editor at Real Simple, she is compulsively organized and loves solving people's problems. She has a weakness for desserts, especially ice cream. \ No newline at end of file diff --git a/tests/data/text/thenextweb.com1.txt b/tests/data/text/thenextweb.com1.txt new file mode 100644 index 00000000..236c2954 --- /dev/null +++ b/tests/data/text/thenextweb.com1.txt @@ -0,0 +1,3 @@ +Crunchyroll is a leading destination and platform for Japanese anime, Korean drama, live-action titles, and tons of other Asian media. + +All videos are professionally translated into multiple languages, and are available for viewing within minutes of their original TV broadcasts through tons of different applications. \ No newline at end of file diff --git a/tests/data/text/thenextweb.com2.txt b/tests/data/text/thenextweb.com2.txt new file mode 100644 index 00000000..e06ab695 --- /dev/null +++ b/tests/data/text/thenextweb.com2.txt @@ -0,0 +1,3 @@ +TNW Weekly The best of the week, handpicked every Friday + +Daily Top stories The most shared stories of the day \ No newline at end of file diff --git a/tests/data/text/theonion.com1.txt b/tests/data/text/theonion.com1.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/text/theonion.com2.txt b/tests/data/text/theonion.com2.txt new file mode 100644 index 00000000..f9607fc1 --- /dev/null +++ b/tests/data/text/theonion.com2.txt @@ -0,0 +1 @@ +Explaining that her statements indicated a failure to understand and implement the district’s goal of providing a comprehensive education to all children, Southwest High School officials reportedly fired ninth-grade history teacher Jennifer Steenman today after she was heard saying she learns more from her students than they do from her. Full article. \ No newline at end of file diff --git a/tests/data/text/theroot.com1.txt b/tests/data/text/theroot.com1.txt new file mode 100644 index 00000000..22215427 --- /dev/null +++ b/tests/data/text/theroot.com1.txt @@ -0,0 +1,19 @@ +Within social-justice movements, tensions often surface between older and youngster activists—that’s hardly new. But the Rev. Al Sharpton says that recent criticism that he isn’t doing enough to groom younger, up-and-coming activists within his civil rights tradition is simply not true. + +“This is more about ideology than it is about generational differences,” Sharpton told The Root. He pointed to dozens of his protégés—all in their 20s and 30s—who have committed themselves to the principles of an interracial and nonviolent movement. + +“I have spent an inordinate amount of time trying to make sure that we can continue this movement and National Action Network for the next 30 to 40 years when I am gone,” he said. “Leadership cannot be willed. I can’t pass the torch. I can only keep the flame lit.” + +For his part, Sharpton is not sure who will replace him after he exits the national stage, but he said that it’s not up to him to handpick his successor. That person “has to put in the work and earn it,” he said. + +“I’m sure I was not Jesse’s [Jackson] choice. There were other guys who were probably more palpable to him,” said Sharpton, who as a teenager served as the youth director of Brooklyn’s Operation Breadbasket—the economic arm of the Southern Christian Leadership Conference that Jackson directed at the request of Martin Luther King Jr. + +But by the mid-2000s, Sharpton had gone on to surpass his mentor in national prominence, much to the chagrin of those who had hoped that Sharpton would fade away from the public spotlight. “I was too young to know for sure, but it’s possible that Jesse wasn’t Dr. King’s choice, either,” he said. + +Sharpton said that contrary to some reports, a diverse group of young people were invited to speak at the Dec. 13 Justice for All rally in Washington, D.C. The event drew more than 10,000 people to the nation’s capital to train a spotlight on the killing of unarmed black men in the wake of several high-profile shootings, as well as to urge Congress to take up legislation that would require closer monitoring of police departments across the country. + +“This was not a revolutionary march, and I don’t apologize for that,” said Sharpton, who added that he had not yet arrived at the rally when a group of protesters, whom he did not know, stormed the stage and demanded to speak. He said that when he arrived on the scene later, he granted the activists speaking time after they assured him that they were not going to call for violence or promote inflammatory rhetoric against nonblacks. + +Sharpton recalled that at one point during the protests after the Trayvon Martin shooting, a demonstrator publicly called for a $10,000 bounty to be placed on the life of George Zimmerman, the neighborhood watchman who killed Trayvon. The slain teen’s parents, however, as well as the parents of the other shooting victims who have worked with Sharpton, have called for peaceful demonstrations. + +Sharpton said he was unsure why activists who disagreed with his political tactics and strategy would want to attend the gathering in the first place. “I don’t go to marches spearheaded by people who I disagree with,” said Sharpton. “I just do what I’m doing.” \ No newline at end of file diff --git a/tests/data/text/theroot.com2.txt b/tests/data/text/theroot.com2.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/text/tnr.com1.txt b/tests/data/text/tnr.com1.txt new file mode 100644 index 00000000..f8a2b7a6 --- /dev/null +++ b/tests/data/text/tnr.com1.txt @@ -0,0 +1,33 @@ +I was touched that you asked for my advice about going into politics. Anyone whose career in politics was nasty, brutish, and short—as mine was—is grateful that anyone thinks their opinion is worth hearing. All I’d claim is that my thoughts come with what Scott Fitzgerald called “the authority of failure.” + +First of all, you need to know why you want it. You’d be amazed at how many people who go into politics can’t give you an honest answer to why they want it so badly. + +All the best reasons for going into politics never really change: the desire for glory and fame and the chance to do something that really matters, that will make life better for a lot of people. You have to be one of those people with outsized, even laughable ambition, who want their convictions to mean something more than smart conversation at dinner tables. You have to have a sense of vocation, a belief that something must be done and that you’re the person to do it. + +I had the vocation for politics. What I didn’t have was any aptitude for political combat. I took the attacks personally, which is a great mistake. It’s never personal: It’s just business. It was ever thus. You can prepare yourself for combat by going in as a staffer, watching it from the sidelines, as I did when I was in my twenties, but believe me, when you step in the ring yourself, the first punch always comes as a shock. That’s when you’ll know, as you snap your head back into place, whether your first instinct is fight or flight. + +I went into politics thinking that, if I made arguments in good faith, I’d get a hearing. It’s a reasonable assumption, but it’s wrong. In five and a half years in politics up north, no one really bothered to criticize my ideas, such as they were. It was never my message that was the issue. It was always the messenger. + +They will not attack what you say, so much as your right to say anything at all. In my case, they said I’d been out of the country too long, I wasn’t really “one of us,” but one of “them.” I was just visiting. + +The attacks that are hardest to deal with are not the ones that are false, but the ones that have a sliver of truth. Being out of the country was nothing to be ashamed of, but it didn’t exactly help me to establish the trust that any politician must establish with voters. + +Conjuring that trust requires authenticity. You can’t pretend to be somebody you’re not. People who say politics is acting get it wrong. You’re not playing a role. You’re on stage, true enough, but you’re playing yourself. People don’t have to identify with your life in order to vote for you, but they have to believe that you are who you say you are. + +You will now list for me all the duplicitous villains who attained power without being authentic. You misunderstand me. A man like Nixon had authenticity aplenty. Voters knew exactly who he was: suspicious, manipulative, duplicitous, and just like them. They saw through him to themselves. + +To be authentic, you have to own your life. All of it. John Kerry fell victim to the swift-boat attack because he couldn’t own the young lieutenant back from Vietnam who gave that damning testimony in Congress about the terrible things he witnessed up the Mekong Delta. He was unable, deep inside, to say, “Yes, I was that young lieutenant.” If you don’t want to vote for a man who criticized his country, go ahead. People, it turns out, will forgive candidates almost anything if they fight for their right to be themselves. + +The real battle in politics is this battle over standing, your right to get a hearing as the person you are. Once the swift-boat attacks hit their target, once he failed to reply, Kerry could talk, but no one was listening. He had lost his standing. Once my opponents said I was just visiting, I lost mine. I could speak, but I couldn’t be heard. + +So my advice is: Never let your opponents own your story. If you can’t do this truthfully, choose another business. And if you can’t defend your own life when people attack it, there are plenty of other lives you could choose that don’t require the same naked exposure. + +It doesn’t pay, either, to pretend to be better than the business you’re in. You can’t succeed in politics if you give too much appearance of despising the low arts by which we govern ourselves. Fastidious distaste for the roughness and meanness of political life may work in a seminar room, but it’s fatal on the campaign trail. + +This distaste is common among people who’ve enjoyed success outside of politics, in academia or journalism or business, and who go into politics with the reasonable assumption that the prestige they achieved in their former profession should automatically transfer into politics. It doesn’t. People who think they’re entitled to standing—because they are brainy, rich, or famous—almost always lose. They forget you earn your standing, you are not entitled to it. That’s the best thing about democracy, the single reason why we’re not yet entirely governed by wealthy oligarchs. + +I may have come into politics with an unacknowledged condescension toward the game and the people who played it, but I left with more respect for politicians than when I went in. The worst of them—the careerists and predators—you find in all professions. The best of them were a credit to democracy. They knew the difference between an adversary and an enemy, knew when to take half a loaf and when to insist on the whole bakery, knew when to trust their own judgment and when to listen to the people. + +As I learned while watching wiser colleagues than I in a democratic legislature, it is really something in life to be utterly disabused about human motive, venality, capacity for double-crossing, and yet still come to work every day, trying to get something done. + +Liberalism will become an enclave conviction of a shrinking minority unless those who call themselves liberal reconnect their faith in tolerance, equality, opportunity for all with the more difficult faith in the dirty, loud-mouthed, false, lying business of politics itself. This disdain is cynicism, masking as high principle. The ultimate allegiance of a democratic politician is not to party, not even to principle, but to the venal process called politics. So my final advice is this: Politics is not a vulgar means to a goal, it’s a noble life unto itself, and unless you love it, you can’t do it well. I didn’t get there, but I hope you will. \ No newline at end of file diff --git a/tests/data/text/tnr.com2.txt b/tests/data/text/tnr.com2.txt new file mode 100644 index 00000000..1f165acb --- /dev/null +++ b/tests/data/text/tnr.com2.txt @@ -0,0 +1,23 @@ +The de facto assumption of climate change policy is that the world must limit the increase in global temperatures to 3.6 degrees Fahrenheit (2 degrees Celsius) above pre-Industrial levels, or risk hitting a tipping point where the impact becomes irreversible. The figure dates back to 1975, when economist William Nordhaus suggested that more than 3.6 degrees of warming would “take the climate outside of the range of observations which have been made over the last several hundred thousand years.” By the 1990s, 3.6 degrees gained traction in the scientific community and then in politics, when the European Council argued in 1996 that 3.6 degrees should be the United Nations’ red line for global warming. It wasn’t until four years ago, at a climate conference in Cancun, Mexico, that countries finally committed to “hold the increase in global average temperatures below” 3.6 degrees. + +Despite being almost 40 years old, this temperature threshold remains controversial—and for good reason. One: It's rather arbitrary. Two: It's unrealistic. To stay within 3.6 degrees, global carbon pollution would have to begin coming down in the next decade, according to the United Nations Environmental Program Emissions Gap Report. The world would have to reach zero net greenhouse gas emissions before the end of the century. + +In other words, we're nowhere near where we need to be to stay under this target. Pollution continues to rise, and global temperatures are already locked in for warming that puts the planet two-thirds of the way there, currently around 1.5 degrees above pre-industrial level temperatures, counting the pollution we've emitted and will continue to emit in the short-term to medium-term. + +As it becomes clear that international talks will fall short of meeting this target, some are calling to abandon it altogether. + +A paper in Nature in October argued that we should instead measure various "vital signs,” like looking at extreme events around the planet. Negotiators at the December climate talks in Lima, Peru, have recognized that the proposed cuts mean accepting warming anywhere between 4 and 10 degrees. While many point out that 3.6 degrees is unrealistically low, scientists are meeting to reconsider whether this target is already too high. + +NASA Goddard Institute for Space Studies climatologist Gavin Schmidt likened the 3.6 threshold to a speed limit: Going over the speed limit doesn't ensure disaster, but it certainly raises the risk of it. “It means the faster you’re going around that curve, the more dangerous it is going to be,” he said. “What any one person might judge as a level that can be adapted to is going to be very dependent on where they’re coming from, what their culture is, how resilient they are, and how much money they have to adapt." + +Let's say we resign ourselves to a future reality where global temperatures are twice the current target. This future is not all that unlikely, since “present emission trends put the world plausibly on a path toward” 7.2 degrees Fahrenheit (4°C) warming by the end of the century, according to a 2014 report from the World Bank. + +At that level, what might the planet look like? + +In starker language than what scientists normally use, the report described the impacts at 7.2 degrees as "devastating," stating, "given that uncertainty remains about the full nature and scale of impacts, there is also no certainty that adaptation to a 4°C world is possible. This world means "communities, cities and countries would experience severe disruptions, damage, and dislocation, with many of these risks spread unequally." + +Scientists believe that 3.6 degrees is roughly the temperature at which the expansive ice sheets in Greenland and West Antarctica become unstable and melt at an unstoppable rate. If the West Antarctic ice sheet melted entirely, it would raise sea levels by more than 10 feet. If Greenland’s ice sheet melted away, add another 23 feet. It's still a matter of contention how many centuries this would take, but a number of studies this year show these ice sheets melting at an alarming rate. In the last month, four studies show these ice sheets may be more unstable than previous models, which would mean current projections of sea level rise, of up to 4 feet by the century's end, are too conservative. One of these analyses, from NASA and the University of California at Irvine, shows that western Antarctica lost water that's equivalent to the weight of Mt. Everest every two years for the past 21 years. + +This would change the world map as we know it. Some 12.3 million people live on U.S. land that would go underwater if the sea rises by 10 feet. Climate Central shows who in the U.S. would be affected: + +Uncontrolled climate change would also fuel greater extremes across the world. We’re already experiencing hotter average temperatures; the first 11 months of 2014 means this year is on track to be the hottest year recorded. Those effects grow even worse in a world that's 7.2 degrees warmer than pre-industrial times. The coolest months in tropical regions are likely to be "substantially warmer than the warmest months in the 20th century," according to the World Bank. The summer months in the Mediterranean, North Africa, and the Middle East will exceed today's heat extremes. Parts of Africa, the Middle East, and South Asia will face devastating droughts as temperatures rise. Food production will plummet and water supplies will be drained, even as the world population rises. No doubt, the frequency and severity of human conflict will rise right along with it. \ No newline at end of file diff --git a/tests/data/text/uproxx.com1.txt b/tests/data/text/uproxx.com1.txt new file mode 100644 index 00000000..900f44f7 --- /dev/null +++ b/tests/data/text/uproxx.com1.txt @@ -0,0 +1,13 @@ +It may seem old fashioned, but I sometimes view the internet and the real world as two separate places. Because of that, it seems weird to me when I see something as typically impact-free as a Reddit post have a very real effect on a small town, but that’s exactly what happened to the town of Mammoth, Arizona after a fictional post got taken literally. + +On the social media site Reddit, a user who goes by the name Lindsey posted a story about an Ebola-like illness taking over the small town. “Eleven days ago, on the 3rd, an older woman who runs a home daycare in my town was found dead in her bathroom by a parent who was picking up his daughter,” Lindsey wrote. “The kids were all really agitated and told police that Mrs. Booker (the deceased) had been yelling at them and bleeding from her eye before she went into the bathroom and collapsed in her bathtub.” Lindsey goes on to say children began dying, then it spread to others around Mammoth. She said everyone who passed away experienced similar symptoms. Adding to the panic, Lindsey wrote: “To the person calling businesses here: that is not us answering. Our downtown has been shut down all day. I called a couple places where I know the business owners and employees and the people who answered are not locals. I don’t know what they told you but they’re not from here.” She said it was a cover-up to keep the sickness contained. But, Lindsey was about to fall victim: “I noticed a small bruise on my arm that upon further f***ing inspection spreads all the way down one side of my back. I’m so f***ing scared. I had a breakdown where I just screamed at the wall and cried and there was blood in my tears and that was like an hour ago.” She said she needed help, and with the exposure of 1.4 million Reddit users, she got the attention she needed. + +Apparently Lindsey’s posts were so well constructed that people called residents in the town to see what was up and the 911 dispatcher was inundated. Folks in the comment section also began to corroborate and expound upon the claims. + +Curious about the inadvertent internet firestarter behind this? Here’s an excerpt from an email interview that the local ABC affiliate did with C.K. Walker aka “Lindsey”, the person who started the story. + +“We received mixed reactions. Some people thought it was provocative and interesting – others got upset and called us “terrorists”. This really only happened because the story went viral – those who are a part of the forum know that everything posted in it is fiction; it’s actually noted on the sidebar. When it made it outside of Reddit, people didn’t bother to research the website the stories were posted on and that’s when things went downhill. Also, suspension of disbelief is a requirement for the website and any comments debunking the story were deleted for breaking the rules. This just added to the immersion and eventually things got out of control. There was a sort of perfect storm that occurred when not only were people in the comments playing along quite convincingly but also that there were, for unknown reasons, actual roadblocks set up on the 77 (from what I was told).” + +So, what have we learned? F*ck if I know. These are either terrifying times or a time when the things that terrify us are constantly shown in a loop on a screen that is constantly in front of our faces. That means that people’s freak-out button is a bit more accessible than usual. It’s not good to hit that button, but it’s ultimately not the writer’s fault that people didn’t check to see what was at the heart of this minor panic. + +Really, the only defense against something like this is a more diligent inspection of things before we react to them (which would seem to defy our reflexive nature), or a heartier application of skepticism. The former is more time consuming than the latter but we also shouldn’t get to a point where we roll our eyes whenever someone says that they saw a wolf. Cause wolves, man. They out there. \ No newline at end of file diff --git a/tests/data/text/uproxx.com2.txt b/tests/data/text/uproxx.com2.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/text/upworthy.com1.txt b/tests/data/text/upworthy.com1.txt new file mode 100644 index 00000000..202d1722 --- /dev/null +++ b/tests/data/text/upworthy.com1.txt @@ -0,0 +1,35 @@ +Benjamin: Hello. My name is Benjamin, and I'm proud to be a service dog from America's Vet Dogs. Not just because I get to wear this cool vest, but because they brought me to Joe. That's Joe. He's my best friend and my hero and this is our story. + +Joe: A guy who's in a wheelchair, especially a loud mouth red headed guy like me, people are looking at you all the time and I usually don't mind, at least, I didn't think I minded until I got Benjamin and a lot of that attention shifted to him. And it made it so much easier for me to feel like I was comfortable in public, because people aren't looking at the jacked up one-legged guy, they're looking at the beautiful golden retriever. + +Benjamin: Thanks, Joe. You're not so bad looking yourself. + +Joe: My name's Joseph Worley. I was in the Navy as a hospital corpsman, third class, with 21 Marines out of Camp Pendleton and this is my dog Benji. I deployed to Iraq March 1st of 2004. I got hit on foot by an IED. I ended up losing my left leg and getting some pretty good damage to my right leg. Coming home to my family was incredible. I hadn't seen them in so long and I'd been through so much and I was worried about what kind of father I was going to be with my injury and what kind of husband I was going to be. I needed them and I hoped they still wanted me. Is that fun or what? + +Benjamin: Joe was joyfully reunited with his family and began his recovery. A couple of years later, they would welcome a fluffy, four legged addition. That's where I come in. + +Joe: The application process for America's Vet Dogs is really intricate, because they train each individual dog for each individual veteran. He spent a year and a half of his life training for me, just for me. + +Benjamin: Worth every minute. It's an honor to be by your side. + +Joe: One of the biggest things that changed after I got Benjamin was an improvement in my ability to walk. I can't really describe how having a dog on a leash can help someone feel safe and stable to walk but, for me, that was a big deal because, if anyone's ever walked on railroad tracks and if you can even get your hand on a leaf up above you or something, you just feel like okay, I got something and I guess that's what it was. He can brace and let me hold onto him. If I go to fall, I could grab him, and I just felt safer and more comfortable with him there, just knowing that I had help if I needed it. + +Joe: Good boy. I think, for a lot of guys, dogs like Benjamin could literally be the thing that makes them want to wake up in the morning and have a schedule and that cold nose on their elbow could be the reason why they get up and do the things that they need to do. Benjamin, where's my shoe? + +Benjamin: Right where you left it, Joe. + +Joe: A typical day with me and Benji usually starts with me trying to find my shoe and Benji can help me with that. Good job, buddy. Good job. Thank you. He always looks like he's thinking. His eyebrows are always going up and down and he always just seems like he's waiting on me to do something dumb so he can give me a look like what are you doing? When I go to school, he goes with me. I plan on going back in the fall and he's going to look forward to that because he likes going to school because all the girls flirt with him. + +Joe: Benjamin is a part of my family. I can't imagine my family dynamic without him in it. So, he means an awful lot to me. You wanna play tug a war? If I could ask Benjamin three questions, I would probably ask him who his favorite kid is, which would probably be Izzy, because she just dotes so much attention on him all the time. + +Benjamin: My favorite? Let me see. Yep, yep, it's Izzy. + +Joe: I'd probably ask him what his favorite thing to do is, because he's eight years old and I want him to have fun and enjoy things. + +Joe: I would probably ask him if I've made him happy, if I've done right by him, because he's my friend and he's improved my life and I hope that he feels safe and comfortable around me and that I've improved his life as well. + +Benjamin: Well, now, you're making this old dog get all misty eyed. + +Joe: I wish that I could express to him how much he means to me, because we always read those things that say, they're only there for a part of yours, but you're there for all of theirs and I'm his whole life and I hope that he is proud of me and that I've made him feel like he's done something incredible in my life, because he has. + +Benjamin: Am I proud of Joe? Joe told me once that a hero is someone who does something selfless and who functions even when they are afraid. He may not call himself a hero, but in dog years, I'm older and therefore wiser, and I say he is and I'm proud to be his dog. \ No newline at end of file diff --git a/tests/data/text/upworthy.com2.txt b/tests/data/text/upworthy.com2.txt new file mode 100644 index 00000000..fa7a3d36 --- /dev/null +++ b/tests/data/text/upworthy.com2.txt @@ -0,0 +1,17 @@ +Narrator: If you eat a strawberry, it probably came from California, where it was grown with some pretty toxic stuff called fumigants. Fumigants are pesticides used to grow practically all strawberries we eat. So how did we start using this poisons to help grow our food. The chemical that started it all was chloropicrin, a tear gas used in World War I. Chloropicrin made soldiers vomit. So they tore out their masks only to be gassed with other deadly fumes. When the war ended, the U.S. military was left with millions of pounds of surplus teargas until a pineapple crisis. + +Parasites were attacking Hawaii's pineapple crops. Maybe that surplus could come in handy after all. So in 1927, chloropicrin was shift to the islands and scientists did some test. Fungus, worms, chloropicrin obliterated them. Now the soil was ready for planting. The whole process was called fumigation and the pineapples thrived. + +Thirty years later, California strawberries were struggling with subterranean foes. The fumigants killed them too. But in 1970s, farmers were hooked. New fumigant cocktails were invented. More new strawberry drinks were introduced. Every season was now strawberry season. Harvest tripled. The new interstate was great percipient truckloads of fruit across America. The only problem was there was too much fruit and not enough demand. + +So the California Strawberry Advisory Board got creative, Jello, Bisquick, Cornflakes, Cheerios, and of course, Cool Whip. It gave companies tips on how and when to market foods with strawberries. The goal was simple, promote anything that feature the little red fruit. It worked. Americans now eat four times as many strawberries as they did 40 years ago, with California growing 90% of that fruit. + +Each year millions of pounds of different fumigants are pumped into California strawberry fields. The chemicals don't end up on the fruit, so the berries are safe to eat. The danger is when farmers fumigate, gases can drift to communities nearby. They've been linked to cancer, birth defects and even holes in the ozone layer. The problem is many people live where strawberries thrive and workers have to handle these toxins. Local laws and global treaties have tried dealing with these risks. The grower say that they need to use more. And California has sometimes let them despite health warnings from its own scientists. + +Without fumigants, farmer say, "Our crops can fail. There's no clear alternative to these chemicals. We need them," they say," to keep consumers happy." It's how you can buy cheap strawberries anytime of the year. And so these toxic pesticides called fumigants continue to be the foundation of our strawberry industry. + +ARCHIVAL IMAGES AND FOOTAGE COURTESTY OF U.S. Army The U.S. National Archives and Records Administration Sam Hodgson Library of Congress The New York Times + + © Woman's Day magazine 1975 + + © Family Circle magazine 1978 California Strawberry Commission \ No newline at end of file diff --git a/tests/data/text/usnews.com1.txt b/tests/data/text/usnews.com1.txt new file mode 100644 index 00000000..d91ec7af --- /dev/null +++ b/tests/data/text/usnews.com1.txt @@ -0,0 +1,21 @@ +Long before the 2014 midterm elections, the next potential candidates for president began scoping out their 2016 prospects with visits to the early nominating states of Iowa, New Hampshire and South Carolina. + +U.S. News is on the case, unveiling a set of graphs detailing the visits by the long slate of candidates to the first three primary states since January of 2013. + +The goal is to provide a useful tool in tracking who has gone where and how often, with updates to the charts every week or so. Dropping into these places is the first indication of a politician's likelihood to run for president. + +A special credit to data reporter Lindsey Cook and Andrew Soergel who put together the numbers and assembled the charts. + +We hope this is a useful tool for political junkies and the media at large. To the campaigns themselves, if our count doesn't match up with your own, we'd like to hear from you. Amendments to the charts will be part of the tracking process as some visits are more publicized than others. Email dcatanese@usnews.com. + +The first chart demonstrates total trips by presidential candidates to the first three nominating states of Iowa, New Hampshire and South Carolina. In this overall category, it's GOP Sen. Ted Cruz, R-Texas, who leads the pack. + +The second chart shows trips to the first-in-the-nation caucus state of Iowa, where Republican Gov. Rick Perry has completed the most visits of any aspirant. On the Democratic side, it's Maryland Gov. Martin O'Malley. + +On to New Hampshire, where it's neighboring Vermont Sen. Bernie Sanders who has logged the most visits, according to our totals. + +Finally, in South Carolina, it's GOP Sen. Rand Paul, R-Ky., who has dropped in most frequently to the Palmetto State. + +There are more charts and maps on Github. All charts include embed codes so that anyone can easily use our charts on their own websites. Data for these charts are current through Monday, Nov. 3. + +The Presidential Tracker was created for U.S. News by Lindsey Cook, Andrew Soergel and Dave Catanese using information sourced from www.p2016.org. \ No newline at end of file diff --git a/tests/data/text/usnews.com2.txt b/tests/data/text/usnews.com2.txt new file mode 100644 index 00000000..0f42c4ed --- /dev/null +++ b/tests/data/text/usnews.com2.txt @@ -0,0 +1 @@ +It's a formerly 'forbidden fruit' sure to be enjoyed by cigar aficionados on both sides of the Florida Straits. On the list of President Barack Obama's new Cuba policy are legalized Cuban cigars, \ No newline at end of file diff --git a/tests/data/text/vanityfair.com1.txt b/tests/data/text/vanityfair.com1.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/text/vanityfair.com2.txt b/tests/data/text/vanityfair.com2.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/text/vogue.com1.txt b/tests/data/text/vogue.com1.txt new file mode 100644 index 00000000..7ac6f9a0 --- /dev/null +++ b/tests/data/text/vogue.com1.txt @@ -0,0 +1,7 @@ +© 2014 Condé Nast. All rights reserved. + + Use of this site constitutes acceptance of our User Agreement (effective 1/2/2014) and Privacy Policy (Effective 1/2/2014). + + Your California Privacy Rights + + The material on this site may not be reproduced, distributed, transmitted, cached, or otherwise used, except with the prior written permission of Condé Nast. \ No newline at end of file diff --git a/tests/data/text/vogue.com2.txt b/tests/data/text/vogue.com2.txt new file mode 100644 index 00000000..7ac6f9a0 --- /dev/null +++ b/tests/data/text/vogue.com2.txt @@ -0,0 +1,7 @@ +© 2014 Condé Nast. All rights reserved. + + Use of this site constitutes acceptance of our User Agreement (effective 1/2/2014) and Privacy Policy (Effective 1/2/2014). + + Your California Privacy Rights + + The material on this site may not be reproduced, distributed, transmitted, cached, or otherwise used, except with the prior written permission of Condé Nast. \ No newline at end of file diff --git a/tests/data/text/vogue.de1.txt b/tests/data/text/vogue.de1.txt new file mode 100644 index 00000000..9ead80d2 --- /dev/null +++ b/tests/data/text/vogue.de1.txt @@ -0,0 +1,3 @@ +Im Design von Chanel-Nagellackfläschchen sind die iPhone-Cases von Iphoria der Hingucker bei jedem iPhone. Sie werden Ton in Ton mit der passenden Nagellackfarbe oder im Kontrastlook (oben) getragen. Die Hüllen gibt es in vier Farben: "Rouge Pur" (oben), "Sky Blue", "Sea Mint" und "Candy Pink". Unten sind die iPhone-Cases in Rot, Himmelblau und Mintgrün neben den Chanel-Nagellacken in den Farbtönen "Le Vernis Taboo", "Le Vernis Pirate" und "Le Vernis Lilis" abgebildet. + +Die Chanel-Nagellacke kosten je 24 Euro. Die Iphoria-iPhone-Cases gibt es über www.iphoria.com, ab 45 Euro. \ No newline at end of file diff --git a/tests/data/text/vogue.de2.txt b/tests/data/text/vogue.de2.txt new file mode 100644 index 00000000..3e3b2eb7 --- /dev/null +++ b/tests/data/text/vogue.de2.txt @@ -0,0 +1 @@ +Sie können ironisch-unschön, lustig und überraschend, aber auch klassisch-traditionell sein, doch das Wichtigste ist, dass sie gemütlich sind, Spaß machen und uns an die schönsten Tage des Jahres erinnern: Christmas Sweater! Die Tradition des Weihnachtspullovers ist eine recht junge, erst in den 80er-Jahren vor allem im angelsächsischen Sprachraum entstandene – und vielleicht ist sie gerade deswegen besonders individuell. Weil auch wir bei VOGUE finden, dass man gewisse Traditionen pflegen sollte, zeigen wir unsere liebsten Weihnachtspullover, in denen wir es uns an den Feiertagen gemütlich machen \ No newline at end of file diff --git a/tests/data/text/wetpaint.com1.txt b/tests/data/text/wetpaint.com1.txt new file mode 100644 index 00000000..d95c0ae9 --- /dev/null +++ b/tests/data/text/wetpaint.com1.txt @@ -0,0 +1,21 @@ +So how did Chris and Whitney sitting in a tree K-I-S-S-I-N-G turn into us imagining their babies? We first got whiff of their whirlwind romance when she got a 1-on-1 date in Des Moines, the final destination before Hometowns. Their romantic evening included a stroll along the Des Moines River Walk followed by dinner at a downtown hotspot. After picking the lobster from their teeth, the duo kissed awkwardly in front of a mural some production assistants who are rethinking their career choices painted of the couple. + + + +Apparently the alley makeout sesh wasn’t as awkward as it seemed, because Chris gifted Whit a rose, for which she gifted him a trip to her favorite Chicago Italian restaurant, Quartino, during their Hometown Date. All must have gone swimmingly, because she then (presumably) has all her fantasies come true in the Fantasy Suite in Bali. + + + +Prince Farming narrows it down to two, taking Whit and one other lucky lady back to… Iowa! Apparently he leads the two final gals to a barn where things go the way of Children of the Corn for the other gal but get all Field of Dreams happy for Whitney. + + + +Chris proposes, Whitney says yee haw, and the two ride off on a tractor into the sunset together. Way to get it, girl! + + + +Are you surprised Whitney is the last woman standing? Tell us below. + + + +Source: Reality Steve \ No newline at end of file diff --git a/tests/data/text/wetpaint.com2.txt b/tests/data/text/wetpaint.com2.txt new file mode 100644 index 00000000..abf6e4b0 --- /dev/null +++ b/tests/data/text/wetpaint.com2.txt @@ -0,0 +1 @@ +The raven-haired gal bares a striking resemblance to a former Bachelorette who may have broken Chris’s heart last season, with her shiny locks and sparkling eyes. So, what else makes her perfect for Prince Farming? Read on, gentle reader. The 24-year-old from Hamilton, New Jersey was likely the quickest packer of the 30 gals cast on The Bachelor 2015, as she works as a flight attendant. Unfortunately for her, producers sent the lucky ladies on a bizarre tour of ‘Merica rather than letting them dip their toes into foreign oceans. When Alissa isn’t flying the friendly skies, she keeps her head on her shoulders with “family, friends, laughter, hope, and faith,” according to her official ABC bio. Should you want to hop aboard her social media flight, you’re a bit outta luck. Miss Alissa’s Facebook is currently deactivated, and you’ll need a boarding pass to her Instagram, as it’s set to private. Her LinkedIn is also seemingly out of date, as it says she works for Gallo Winery. Needless to say, that job likely gave her the skills needed to hang tough for the hours of drinking before rose ceremonies. Girlfriend keeps it limber between flights with her love of yoga, and she even earned a certificate in being bendy. Let’s hope things work out with Herr Soules, because they might get a touch awkward if not — Alissa says her greatest fear is running into recent exes. She even admits that she once stalked a crush hardcore — and then texted him her findings on accident. Yikes. But just in case you’re wondering, Alissa made the Dean’s List at St. Joseph’s, where she studied business. To sum her up, let’s turn to the gal’s description of her spirit animal: “A wild mustang. Free to run and explore, they're unpredictable and beautiful, and are loyal to their herd.” If that’s not a winning Bachelor contestant, we don’t know who is. Do you think Alissa is Chris’ perfect match? Sound off in the comments below. Sources: Reality Steve, ABC, LinkedIn \ No newline at end of file diff --git a/tests/data/text/wired.com1.txt b/tests/data/text/wired.com1.txt new file mode 100644 index 00000000..80b0002f --- /dev/null +++ b/tests/data/text/wired.com1.txt @@ -0,0 +1,19 @@ +A shiny Morpho butterfly is a master of nanoscale light bending. That blue isn’t from a pigment — it’s light reflecting off scales built from the same strings of sugars that the rest of the insect’s skeleton is made of. A butterfly scale is basically a big, flat hair made of chitin. + +If it’s all made of the same stuff, why are some parts so shiny, and other parts not? This video by KQED explains: + +The shine of butterflies (and beetles) is created by incredibly detailed nanostructures of longitudinal ridges and crossribs. They reflect specific wavelengths of light, creating the insect’s sparkle. The photo at right shows some detail; the white scale bar is is 65 µm long, or 0.0650 mm. + +Butterflies extrude their scales during metamorphosis from cells on their epidermis, just like we do with hair. And, just like our hair, the final version of a butterfly’s scales are not living cells. + +The lab profiled in the video published a new paper this year where they closely examined developing wings from a pupa, the transitional stage between a caterpillar and a butterfly. Their stunning microphotographs document how you build a shiny butterfly from a caterpillar. + +Two kinds of cells are involved; scale cells and socket cells. In a caterpillar, they are just ordinary cells, and don’t have any distinguishing characteristics. But early on in pupal development (7% complete), they begin to organize in rows corresponding to where the future wing will grow. Each butterfly scale is the product of a single scale cell. + +The socket cell anchors the scale to the membrane of the insects’ wing; the scale cell pokes through the insect’s wing surface like a hernia. The scale cell forms strings of proteins called F-actins, which provides a framework on which the detailed nanostructures of the scales are built. At 28% complete metamorphosis, you can see both cells and ribbed bundles of actins forming. It’s a ghostly outline of the scale structure to come. + +Like a pasta maker, the scale cell squeezes out and assembles a lattice of actins. These actins form the template on which the rest of the scale’s cuticle will be laid down, and foreshadow the fine ribs of the finished scale. At around 64% of the way through metamorphosis, the actin bundles begin to disappear, and the finishing touches are added to the scale. Eventually, the scale cell dies, revealing the finished scale which hardens upon emergence. + +And the butterfly flies away, to the delight of everyone. + +Dinwiddie et al. 2014. Dynamics of F-actin prefigure the structure of butterfly wing scales. Developmental Biology 392(2): 404–418. doi:10.1016/j.ydbio.2014.06.005 \ No newline at end of file diff --git a/tests/data/text/wired.com2.txt b/tests/data/text/wired.com2.txt new file mode 100644 index 00000000..21dfdc31 --- /dev/null +++ b/tests/data/text/wired.com2.txt @@ -0,0 +1,25 @@ +The call came while Hal Finney was in the final stages of his five-year battle with Lou Gehrig’s disease. When the phone rang, his wife Fran was giving him a shower, with help from his nurse. Fran took the call, which came from a 911 emergency dispatch operator. “Are you OK?” the voice asked. “Is anyone being attacked in your house?” + +Fran didn’t quite know what to make of the bizarre call, and the operator kept talking, in rather pleasant tones. “I need to let you know that you are about to have a SWAT team come to your home,” the voice said, “and they’re going to ask you to leave.” + +When Fran poked her head out the door of her Santa Barbara home, she found the building surrounded by police, and a helicopter buzzing in the air above. It was just days after a disturbed young man named Elliot Rodger had killed six people near Santa Barbara’s University of California campus and the police were especially concerned. The cops yelled at her to drop her telephone and come out onto the lawn, and that’s what she did, leaving her disabled husband, her son Jason, and the nurse in house behind her. + +The police eventually cleared the building, and Hal Finney, a noted computer cryptography expert, waited on the lawn for a half hour, shivering in the morning air. Fran worried that Hal, who was unable to swallow, might choke on his own saliva. “I was just panicking that he was going to need suction or something,” she says. “He didn’t have anything with him except his ventilator.” + +The Finneys were the victims of a “swatting,” a nasty online hoax where the perpetrator calls up emergency dispatch using a spoofed telephone number and pretends to have committed a heinous crime in the hopes of provoking an armed police response to the victim’s home. In this case, the caller phoned 911, announced that he had just murdered two people, and said was going to kill himself too. + +For a year, the caller had been demanding that the Finneys pay an extortion fee of 1,000 bitcoin—worth more than $400,000 at the time—and according to Fran Finney, the FBI agents working the case believe that Hal was just one of several people extorted in this way by the caller. The incident further exposes the rather bizarre and often criminal element that continues to hover around bitcoin, a digital currency that grew out of the internet underworld but has since expanded into the mainstream. + +Previously, Fran Finney has not publicly spoken about this incident for fear of compromising the investigation, but she spoke with WIRED after the investigating agent gave her the go-ahead. The FBI did not have a comment for this story. + +What I’m angry about is it took away some of the peace that he could have had for the last few months of his life. + +When someone calling himself Satoshi Nakamoto first proposed the idea of bitcoin back in 2008, his ideas went largely unnoticed. But Hal Finney paid attention. He quickly became one of the world’s first bitcoin users. That early enthusiasm proved lucrative for Hal Finney, allowing him to join the digital currency’s network and “mine” many bitcoins during the early days. The stash helped the Finneys cover Hal’s medical expenses, but it also came at a price. + +Hal Finney died in August, and his wife Fran says he spent his final months being harassed by the online extortionist. He called the Finney’s home number nine times in the two months after the attack, threatening to assault family members and expose their personal information. “What I’m angry about is it took away some of the peace that he could have had for the last few months of his life,” she says. “This was taking up a lot of his emotional energy.” + +Roger Ver, another early bitcoin adopter, believes he was victimized by the same person the week before the Finney family was swatted. That’s when someone using the names Nitrous and Savaged hacked into Ver’s email accounts and demanded that he cough up 37 bitcoins—about $20,000 at the time—in order to prevent his private information from being published online. Ver refused, and the hacker apparently backed off after Ver put a 37 bitcoin bounty on his head. + +Ver, who was himself sentenced to 10 months in federal prison for illegally shipping explosive across state lines, believes that Savaged is not only the same person who swatted Hal Finney, but also the person who gained access to Satoshi Nakamoto’s email account earlier this year. And he’s mad that this extortionist hasn’t been caught. + +The “police have been devoting a huge amount of resources to track down peaceful people engaged in voluntary trade like Charlie Shrem and the operators of the Silk Road Market,” Ver says, “while evil hackers were busy terrorizing quadriplegic Hal Finney and his family.” \ No newline at end of file diff --git a/tests/data/text/wnet.org1.txt b/tests/data/text/wnet.org1.txt new file mode 100644 index 00000000..26885857 --- /dev/null +++ b/tests/data/text/wnet.org1.txt @@ -0,0 +1,9 @@ +AUDIT COMMITTEE (at Skadden, Arps, Slate, Meagher & Flom LLP 4 Times Square, 42nd St. between Bwy and Sixth Ave.) + +Tuesday, September 24, 2013 at 2:30 PM **Telephonic + + Monday, November 25, 2013 at 3:00 PM + + Tuesday, April 22, 2014 at 1:00 PM + +** To request the toll-free conference call-in number, please call (212) 560 – 6928 \ No newline at end of file diff --git a/tests/data/text/wnet.org2.txt b/tests/data/text/wnet.org2.txt new file mode 100644 index 00000000..e1bf0334 --- /dev/null +++ b/tests/data/text/wnet.org2.txt @@ -0,0 +1,5 @@ +Tuesday, October 15, 2013 at 9:00 AM (at Loews Corporation, 667 Madison Avenue at 61st Street, 7th Fl) + + Monday, January 27, 2014 at 9:00 AM (at Loews Corporation, 667 Madison Avenue at 61st Street, 7th Fl) + + Tuesday, April 8, 2014 at 9:00 AM (at WNET, 825 Eighth Avenue, 14th Fl. Kellen Board Room) \ No newline at end of file diff --git a/tests/data/text/youbeauty.com1.txt b/tests/data/text/youbeauty.com1.txt new file mode 100644 index 00000000..488f590e --- /dev/null +++ b/tests/data/text/youbeauty.com1.txt @@ -0,0 +1,27 @@ +The joint elevation of fan culture and nail art has left us with something magical: Drake nail decals. It doesn’t stop there – you can now wear your favorite '90s TV characters and even movie quotes on your nails with pride. Although its a personal style game changer, it all costs less than $10. + +Here are our top five picks and recommendations of totally appropriate places to wear them. But remember the Internet is a wonderland, so don’t stop until you find your own favorite pop culture moment: + +"Saved by the Bell" nail decals at Forever 21, $9 + +Zack Morris, Kelly Kapowski and the gang reunite right here on your hands. The decals are printed in color and black and white, but if you’re a "Saved by the Bell" fan, you’re obviously choosing the neon. + +"Harry Potter" decals by fingerprintsdecals on Etsy, $5.50 + +Remember when Emma Watson wasn’t an international feminist and was instead a small wizard? The Etsy shop fingerprints has zapped her and her fellow Gryffindors until they’re small enough to fit on a pinky. + +For that one friend… + +"Mean Girls" quotes by PaipurNails at Etsy, $4 + +Don’t we all know one person who has memorized every line from "Mean Girls"? For Etsy seller PaipurNails, she is that one person, and she has found her calling with decals that say “Burn Book” and “She doesn’t even go here.” + +"The Godfather" by NailSpin at Etsy, $5 + +What better way to show how cultured you are than adding Vito Corleone to your beauty routine? Your high school friends probably had a "Godfather" poster in their dorm rooms – one up them by putting him on your nails. + +You can stick a Post-It to your desk that says “‘Live In the Moment” or you can wear YOLO nail decals. We all sometimes need a little daily inspiration – personally, I’d love to get it from Drake’s face on my ring finger. + +Flower Nail Art That Works for Every Season + +3 Nail Art Designs That Last a Whole Month \ No newline at end of file diff --git a/tests/data/text/youbeauty.com2.txt b/tests/data/text/youbeauty.com2.txt new file mode 100644 index 00000000..0297e317 --- /dev/null +++ b/tests/data/text/youbeauty.com2.txt @@ -0,0 +1,9 @@ +I have confused skin. One moment it is dry and needs hydration and the next it is oily and greasy. So I need a powder that will cut the oil, but still let my skin breathe. Clinique Acne Solutions Powder Makeup gently covers blemishes, evens skin and even absorbs oil to leave skin looking fresh. It's made specifically for "dry combination to oily skin types," so it may be just what I have been looking for. + +How to Use it: The powder comes with a sponge applicator, but feel free to apply it with a powder brush as well. Apply to oily spots only or use the powder on your entire face for full-coverage. (It's also available in liquid form.) + +Results: The shade that I received as a trial, Golden, was a bit too dark for my complexion, but I'm expecting to get a tan soon on a trip to the tropics. That aside, my skin has been in its oily phase for the past few days, so I was super excited to see if Clinique's Acne Solutions Powder Makeup really delivered — and it definitely did! The powder immediately cut the shine and gave my skin an even, natural look. It is very light and even thought it builds on your skin, it doesn't look or feel "caked on" like some powders do. I will be purchasing it for myself in the future, although maybe in a lighter shade. + +Could Resveratrol Be the Secret to Clearer Skin? + +5 Stubborn Skin Issues and How to Fix Them \ No newline at end of file diff --git a/tests/generate_fulltext.py b/tests/generate_fulltext.py new file mode 100644 index 00000000..ee60405f --- /dev/null +++ b/tests/generate_fulltext.py @@ -0,0 +1,57 @@ +import os +import sys +import traceback + +TEST_DIR = os.path.abspath(os.path.dirname(__file__)) +PARENT_DIR = os.path.join(TEST_DIR, '..') +sys.path.insert(0, PARENT_DIR) + +from newspaper import Article +from newspaper.urls import get_domain + + +FULLTEXT_OUTPUT_PREFIX = os.path.join('data', 'text') +URLS_FILE = os.path.join('data', 'fulltext_url_list.txt') + + +def get_base_domain(url): + """For example, the base url of uk.reuters.com => reuters.com + """ + domain = get_domain(url) + tld = '.'.join(domain.split('.')[-2:]) + if tld in ['co.uk', 'com.au', 'al.com']: # edge cases + end_chunks = domain.split('.')[-3:] + else: + end_chunks = domain.split('.')[-2:] + base_domain = '.'.join(end_chunks) + return base_domain + + +with open(URLS_FILE, 'r') as f: + urls = [d.strip() for d in f.readlines() if d.strip()] + + +domain_counters = {} + +for url in urls: + domain = get_base_domain(url) + if domain in domain_counters: + domain_counters[domain] += 1 + else: + domain_counters[domain] = 1 + + print('URL:', url, 'Domain:', str(domain_counters[domain])) + filename = domain + str(domain_counters[domain]) + '.txt' + filename = os.path.join(FULLTEXT_OUTPUT_PREFIX, filename) + try: + a = Article(url) + a.download() + a.parse() + out_text = a.text + except Exception: + print('URL: %s has failed!' % url) + traceback.print_exc() + out_text = '' + + with open(filename, 'w') as f: + f.write(out_text) diff --git a/tests/generate_urls.py b/tests/generate_urls.py new file mode 100644 index 00000000..37161909 --- /dev/null +++ b/tests/generate_urls.py @@ -0,0 +1,55 @@ +import os +import sys +import traceback + +TEST_DIR = os.path.abspath(os.path.dirname(__file__)) +PARENT_DIR = os.path.join(TEST_DIR, '..') +sys.path.insert(0, PARENT_DIR) + +import newspaper +from newspaper.urls import get_domain + + +DOMAINS_FILE = os.path.join('data', 'fulltext_domain_list.txt') +URLS_FILE = os.path.join('data', 'fulltext_url_list.txt') + + +def get_base_domain(url): + """For example, the base url of uk.reuters.com => reuters.com + """ + domain = get_domain(url) + end_chunks = domain.split('.')[-2:] + base_domain = '.'.join(end_chunks) + return base_domain + + +with open(DOMAINS_FILE, 'r') as f: + domains = ['http://' + d.strip() for d in f.readlines() + if d.strip()] + + +with open(URLS_FILE, 'a') as f: + for domain in domains: + print('on domain', domain) + + try: + paper = newspaper.build(domain) + except Exception: + print('domain %s has failed!' % domain) + traceback.print_exc() + continue + + if paper.size() < 2: + print('domain %s has < 2 articles, skipping' % domain) + continue + + written = 0 + for article in paper.articles: + if get_base_domain(article.url) == domain[len('http://'):]: + f.write(article.url + '\n') + written += 1 + if written == 2: + break + + if written != 2: + print('domain %s does not have >= 2 valid urls' % domain) diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 389be4c0..1cc5b956 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -22,11 +22,13 @@ TEXT_FN = os.path.join(TEST_DIR, 'data/text') HTML_FN = os.path.join(TEST_DIR, 'data/html') +URLS_FILE = os.path.join(TEST_DIR, 'data/fulltext_url_list.txt') import newspaper from newspaper import ( Article, Source, ArticleException, news_pool) from newspaper.configuration import Configuration +from newspaper.urls import get_domain from newspaper.utils.encoding import smart_str, smart_unicode from newspaper.utils import encodeValue # from newspaper import Config @@ -62,6 +64,56 @@ def mock_response_with(url, response_file): return requests.get(url) +def get_base_domain(url): + """For example, the base url of uk.reuters.com => reuters.com + """ + domain = get_domain(url) + tld = '.'.join(domain.split('.')[-2:]) + if tld in ['co.uk', 'com.au', 'au.com']: # edge cases + end_chunks = domain.split('.')[-3:] + else: + end_chunks = domain.split('.')[-2:] + base_domain = '.'.join(end_chunks) + return base_domain + + +class ExhaustiveFullTextCase(unittest.TestCase): + + def runTest(self): + # The "correct" fulltext needs to be manually checked + # we have 50 so far + FULLTEXT_PREPARED = 50 + domain_counters = {} + + with open(URLS_FILE, 'r') as f: + urls = [d.strip() for d in f.readlines() if d.strip()] + + for url in urls[:FULLTEXT_PREPARED]: + domain = get_base_domain(url) + if domain in domain_counters: + domain_counters[domain] += 1 + else: + domain_counters[domain] = 1 + + try: + a = Article(url) + a.download() + a.parse() + except Exception: + print('<< URL: %s parse ERROR >>' % url) + continue + + out_fn = domain + str(domain_counters[domain]) + '.txt' + out_fn = os.path.join(TEXT_FN, out_fn) + with open(out_fn, 'r') as f: + correct_text = f.read() + + condensed_url = url[:30] + ' ...' + print('%s -- fulltext status: %s' % + (condensed_url, a.text == correct_text)) + # assert a.text == correct_text + + class ArticleTestCase(unittest.TestCase): def runTest(self): self.test_url() @@ -479,13 +531,15 @@ def test_spanish_fulltext_extract(self): suite = unittest.TestSuite() - suite.addTest(ConfigBuildTestCase()) - suite.addTest(MultiLanguageTestCase()) + # suite.addTest(ConfigBuildTestCase()) + # suite.addTest(MultiLanguageTestCase()) + + suite.addTest(ExhaustiveFullTextCase()) + # suite.addTest(EncodingTestCase()) + # suite.addTest(UrlTestCase()) + # suite.addTest(ArticleTestCase()) + # suite.addTest(APITestCase()) - suite.addTest(EncodingTestCase()) - suite.addTest(UrlTestCase()) - suite.addTest(ArticleTestCase()) - suite.addTest(APITestCase()) unittest.TextTestRunner().run(suite) # TODO: suite.addTest(SourceTestCase()) From 918a83b4107154d1909698bc23f5e3435ab0307e Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Wed, 31 Dec 2014 21:56:04 -0800 Subject: [PATCH 05/13] Integrate UnicodeDammit for smooth encoding detection --- newspaper/parsers.py | 40 ++++++++++++++++++++++------------------ tests/unit_tests.py | 15 +++++++-------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/newspaper/parsers.py b/newspaper/parsers.py index cbc86216..eb6c842c 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -13,10 +13,10 @@ import re import traceback +from bs4 import UnicodeDammit from copy import deepcopy from . import text -from . import utils log = logging.getLogger(__name__) @@ -41,29 +41,30 @@ def drop_tag(cls, nodes): def css_select(cls, node, selector): return node.cssselect(selector) + @classmethod + def get_unicode_html(cls, html): + converted = UnicodeDammit(html, is_html=True) + if not converted.unicode_markup: + raise Exception( + 'Failed to detect encoding of article HTML, tried: %s' % + ', '.join(converted.tried_encodings)) + html = converted.unicode_markup + return html + @classmethod def fromstring(cls, html): - html = utils.encodeValue(html) - # don't bring the entire library down because one article - # or article failed to parse + html = cls.get_unicode_html(html) + # Enclosed in a `try` to prevent bringing the entire library + # down due to one article (out of potentially many in a `Source`) try: - # remove encoding tag because lxml won't accept it for - # unicode objects (Issue #78) - if isinstance(html, bytes): - if html.startswith(b'', b'', html, flags=re.DOTALL) - else: - if html.startswith('', '', html, flags=re.DOTALL) + # lxml does not play well with encoding tags + if html.startswith('', '', html, flags=re.DOTALL) cls.doc = lxml.html.fromstring(html) return cls.doc except Exception: traceback.print_exc() - return None - - # @classmethod - # def set_doc(cls, html): - # cls.doc = cls.fromstring(html) + return @classmethod def node_to_string(cls, node): @@ -83,6 +84,9 @@ def clean_article_html(cls, node): @classmethod def nodeToString(cls, node): + """`decode` is needed at the end because `etree.tostring` + returns a python bytestring + """ return lxml.etree.tostring(node).decode() @classmethod @@ -266,6 +270,6 @@ def outerHtml(cls, node): class ParserSoup(Parser): @classmethod def fromstring(cls, html): - html = utils.encodeValue(html) + html = cls.get_unicode_html(html) cls.doc = lxml.html.soupparser.fromstring(html) return cls.doc diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 1cc5b956..2032f716 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -531,14 +531,13 @@ def test_spanish_fulltext_extract(self): suite = unittest.TestSuite() - # suite.addTest(ConfigBuildTestCase()) - # suite.addTest(MultiLanguageTestCase()) - - suite.addTest(ExhaustiveFullTextCase()) - # suite.addTest(EncodingTestCase()) - # suite.addTest(UrlTestCase()) - # suite.addTest(ArticleTestCase()) - # suite.addTest(APITestCase()) + # suite.addTest(ExhaustiveFullTextCase()) + suite.addTest(ConfigBuildTestCase()) + suite.addTest(MultiLanguageTestCase()) + suite.addTest(EncodingTestCase()) + suite.addTest(UrlTestCase()) + suite.addTest(ArticleTestCase()) + suite.addTest(APITestCase()) unittest.TextTestRunner().run(suite) From 1059ae7f711e69332b30db73a81b189a933bfa7b Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Wed, 31 Dec 2014 22:01:53 -0800 Subject: [PATCH 06/13] [bugfix] Issue #103, metatag to dict extraction --- newspaper/extractors.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 40421ce6..228d8d8a 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -352,7 +352,13 @@ def get_meta_data(self, doc): continue key = key.split(':') - ref = data[key.pop(0)] + key_head = key.pop(0) + ref = data[key_head] + + if isinstance(ref, str): + data[key_head] = {key_head: ref} + ref = data[key_head] + for idx, part in enumerate(key): if idx == len(key) - 1: ref[part] = value From e53134f2ce0e3bfc5ddab95ba0ad9e012861349b Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Wed, 31 Dec 2014 22:08:53 -0800 Subject: [PATCH 07/13] Deprecate parser_class config option After integrating UnicodeDammit there is no need to let users customize parsers to BeautifulSoup over lxml, lxml is faster and UnicodeDammit gives us the encoding recognition win from BeautifulSoup --- docs/user_guide/advanced.rst | 2 -- newspaper/configuration.py | 6 ++---- newspaper/parsers.py | 8 -------- 3 files changed, 2 insertions(+), 14 deletions(-) diff --git a/docs/user_guide/advanced.rst b/docs/user_guide/advanced.rst index b8a7df05..4aac8508 100644 --- a/docs/user_guide/advanced.rst +++ b/docs/user_guide/advanced.rst @@ -211,8 +211,6 @@ Here is a full list of the configuration options: ``MAX_FILE_MEMO``, default 20000, "python setup.py sdist bdist_wininst upload" -``parser_class``, default 'lxml', "lxml vs soup" - ``memoize_articles``, default True, "cache and save articles run after run" ``fetch_images``, default True, "set this to false if you don't care about getting images" diff --git a/newspaper/configuration.py b/newspaper/configuration.py index fa5d6d67..c0960a6b 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -12,7 +12,7 @@ import logging -from .parsers import Parser, ParserSoup +from .parsers import Parser from .text import (StopWords, StopWordsArabic, StopWordsChinese, StopWordsKorean) from .version import __version__ @@ -38,8 +38,6 @@ def __init__(self): # max number of urls we cache for each news source self.MAX_FILE_MEMO = 20000 - self.parser_class = 'lxml' # 'lxml' or 'soup' - # Cache and save articles run after run self.memoize_articles = True @@ -104,7 +102,7 @@ def get_stopwords_class(self, language): return StopWords def get_parser(self): - return Parser if self.parser_class == 'lxml' else ParserSoup + return Parser class ArticleConfiguration(Configuration): diff --git a/newspaper/parsers.py b/newspaper/parsers.py index eb6c842c..b2144a66 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -265,11 +265,3 @@ def outerHtml(cls, node): e0 = deepcopy(e0) e0.tail = None return cls.nodeToString(e0) - - -class ParserSoup(Parser): - @classmethod - def fromstring(cls, html): - html = cls.get_unicode_html(html) - cls.doc = lxml.html.soupparser.fromstring(html) - return cls.doc From d4b4e0ceb5322649f15e712f1d030cae0d614e33 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang Date: Wed, 31 Dec 2014 23:27:08 -0800 Subject: [PATCH 08/13] Deprecate encodeValue, in python3 str == unicode --- newspaper/article.py | 28 +++----- newspaper/source.py | 8 +-- newspaper/utils/__init__.py | 18 +---- newspaper/utils/encoding.py | 129 ------------------------------------ tests/unit_tests.py | 31 +-------- 5 files changed, 18 insertions(+), 196 deletions(-) delete mode 100644 newspaper/utils/encoding.py diff --git a/newspaper/article.py b/newspaper/article.py index a9ec169f..b58d905d 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -19,7 +19,7 @@ from .configuration import Configuration from .extractors import ContentExtractor from .outputformatters import OutputFormatter -from .utils import (URLHelper, encodeValue, RawHelper, extend_config, +from .utils import (URLHelper, RawHelper, extend_config, get_available_languages) from .videos.extractors import VideoExtractor @@ -49,12 +49,12 @@ def __init__(self, url, title='', source_url='', config=None, **kwargs): raise ArticleException('input url bad format') # URL to the main page of the news source which owns this article - self.source_url = encodeValue(source_url) + self.source_url = source_url - url = encodeValue(url) + url = url self.url = urls.prepare_url(url, self.source_url) - self.title = encodeValue(title) + self.title = title # URL of the "best image" to represent this article self.top_img = self.top_image = '' @@ -375,13 +375,12 @@ def set_title(self, title): # extraction failed return title = title[:self.config.MAX_TITLE] - title = encodeValue(title) + title = title if title: self.title = title def set_text(self, text): text = text[:self.config.MAX_TEXT] - text = encodeValue(text) if text: self.text = text @@ -390,16 +389,16 @@ def set_html(self, html): """ self.is_downloaded = True if html: - self.html = encodeValue(html) + self.html = html def set_article_html(self, article_html): """Sets the HTML of just the article's `top_node` """ if article_html: - self.article_html = encodeValue(article_html) + self.article_html = article_html def set_meta_img(self, src_url): - self.meta_img = encodeValue(src_url) + self.meta_img = src_url self.set_top_img_no_check(src_url) def set_top_img(self, src_url): @@ -412,7 +411,6 @@ def set_top_img_no_check(self, src_url): """Provide 2 APIs for images. One at "top_img", "imgs" and one at "top_image", "images" """ - src_url = encodeValue(src_url) self.top_img = src_url self.top_image = src_url @@ -420,7 +418,6 @@ def set_imgs(self, imgs): """The motive for this method is the same as above, provide APIs for both `article.imgs` and `article.images` """ - imgs = [encodeValue(i) for i in imgs] self.images = imgs self.imgs = imgs @@ -430,8 +427,7 @@ def set_keywords(self, keywords): if not isinstance(keywords, list): raise Exception("Keyword input must be list!") if keywords: - self.keywords = [encodeValue(k) - for k in keywords[:self.config.MAX_KEYWORDS]] + self.keywords = keywords[:self.config.MAX_KEYWORDS] def set_authors(self, authors): """Authors are in ["firstName lastName", "firstName lastName"] format @@ -439,15 +435,13 @@ def set_authors(self, authors): if not isinstance(authors, list): raise Exception("authors input must be list!") if authors: - authors = authors[:self.config.MAX_AUTHORS] - self.authors = [encodeValue(author) for author in authors] + self.authors = authors[:self.config.MAX_AUTHORS] def set_summary(self, summary): """Summary here refers to a paragraph of text from the title text and body text """ - summary = summary[:self.config.MAX_SUMMARY] - self.summary = encodeValue(summary) + self.summary = summary[:self.config.MAX_SUMMARY] def set_meta_language(self, meta_lang): """Save langauges in their ISO 2-character form diff --git a/newspaper/source.py b/newspaper/source.py index 0a0200ff..367e77aa 100644 --- a/newspaper/source.py +++ b/newspaper/source.py @@ -30,7 +30,7 @@ class Category(object): def __init__(self, url): - self.url = utils.encodeValue(url) + self.url = url self.html = None self.doc = None @@ -38,7 +38,7 @@ def __init__(self, url): class Feed(object): def __init__(self, url): - self.url = utils.encodeValue(url) + self.url = url self.rss = None # TODO self.dom = None, speed up Feedparser @@ -64,7 +64,7 @@ def __init__(self, url, config=None, **kwargs): self.extractor = ContentExtractor(self.config) - self.url = utils.encodeValue(url) + self.url = url self.url = urls.prepare_url(url) self.domain = urls.get_domain(self.url) @@ -141,7 +141,7 @@ def set_description(self): desc html attribute """ desc = self.extractor.get_meta_description(self.doc) - self.description = utils.encodeValue(desc) + self.description = desc def download(self, response=None): """Downloads html of source diff --git a/newspaper/utils/__init__.py b/newspaper/utils/__init__.py index 1bc7db86..7255f485 100644 --- a/newspaper/utils/__init__.py +++ b/newspaper/utils/__init__.py @@ -22,7 +22,6 @@ from hashlib import sha1 -from . import encoding from .. import settings log = logging.getLogger(__name__) @@ -255,19 +254,6 @@ def clear_memo_cache(source): print('memo file for', source.domain, 'has already been deleted!') -def encodeValue(value): - if value is None: - return '' - string_org = value - try: - value = encoding.smart_unicode(value) - except (UnicodeEncodeError, encoding.DjangoUnicodeDecodeError): - value = encoding.smart_str(value) - except: - value = string_org - return value.strip() - - def memoize_articles(source, articles): """When we parse the <a> links in an <html> page, on the 2nd run and later, check the <a> links of previous runs. If they match, @@ -299,11 +285,11 @@ def memoize_articles(source, articles): valid_urls = list(memo.keys()) + list(cur_articles.keys()) memo_text = '\r\n'.join( - [encodeValue(href.strip()) for href in (valid_urls)]) + [href.strip() for href in (valid_urls)]) # Our first run with memoization, save every url as valid else: memo_text = '\r\n'.join( - [encodeValue(href.strip()) for href in list(cur_articles.keys())]) + [href.strip() for href in list(cur_articles.keys())]) # new_length = len(cur_articles) if len(memo) > config.MAX_FILE_MEMO: diff --git a/newspaper/utils/encoding.py b/newspaper/utils/encoding.py deleted file mode 100644 index 5a6a6036..00000000 --- a/newspaper/utils/encoding.py +++ /dev/null @@ -1,129 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Byte string <---> unicode conversions take place -here, pretty much anything encoding related -""" -import datetime -import types - -from decimal import Decimal - - -class DjangoUnicodeDecodeError(UnicodeDecodeError): - def __init__(self, obj, *args): - self.obj = obj - UnicodeDecodeError.__init__(self, *args) - - def __str__(self): - original = UnicodeDecodeError.__str__(self) - return '%s. You passed in %r (%s)' % (original, self.obj, - type(self.obj)) - - -class StrAndUnicode(object): - """A class whose __str__ returns its __unicode__ as a UTF-8 bytestring. - Useful as a mix-in. - """ - def __str__(self): - return self.__unicode__().encode('utf-8') - - -def smart_unicode(s, encoding='utf-8', strings_only=False, errors='strict'): - """Returns a unicode object representing 's'. Treats bytestrings using the - 'encoding' codec. - If strings_only is True, don't convert (some) non-string-like objects. - """ - # if isinstance(s, Promise): - # # The input is the result of a gettext_lazy() call. - # return s - return force_unicode(s, encoding, strings_only, errors) - - -def is_protected_type(obj): - """Determine if the object instance is of a protected type. - - Objects of protected types are preserved as-is when passed to - force_unicode(strings_only=True). - """ - return isinstance(obj, ( - type(None), - int, - datetime.datetime, datetime.date, datetime.time, - float, Decimal) - ) - - -def force_unicode(s, encoding='utf-8', strings_only=False, errors='strict'): - """Similar to smart_unicode, except that lazy instances are resolved to - strings, rather than kept as lazy objects. - If strings_only is True, don't convert (some) non-string-like objects. - """ - # Handle the common case first, saves 30-40% in performance when s - # is an instance of unicode. This function gets called often in that - # setting. - if isinstance(s, str): - return s - if strings_only and is_protected_type(s): - return s - try: - if not isinstance(s, str,): - if hasattr(s, '__unicode__'): - s = str(s) - else: - try: - s = str(str(s), encoding, errors) - except UnicodeEncodeError: - if not isinstance(s, Exception): - raise - # If we get to here, the caller has passed in an Exception - # subclass populated with non-ASCII data without special - # handling to display as a string. We need to handle this - # without raising a further exception. We do an - # approximation to what the Exception's standard str() - # output should be. - s = ' '.join([force_unicode(arg, encoding, strings_only, - errors) for arg in s]) - elif not isinstance(s, str): - # Note: We use .decode() here, instead of unicode(s, encoding, - # errors), so that if s is a SafeString, it ends up being a - # SafeUnicode at the end. - s = s.decode(encoding, errors) - except UnicodeDecodeError as e: - if not isinstance(s, Exception): - raise DjangoUnicodeDecodeError(s, *e.args) - else: - # If we get to here, the caller has passed in an Exception - # subclass populated with non-ASCII bytestring data without a - # working unicode method. Try to handle this without raising a - # further exception by individually forcing the exception args - # to unicode. - s = ' '.join([force_unicode(arg, encoding, strings_only, - errors) for arg in s]) - return s - - -def smart_str(s, encoding='utf-8', strings_only=False, errors='strict'): - """Returns a bytestring version of 's', encoded as specified in 'encoding'. - If strings_only is True, don't convert (some) non-string-like objects. - """ - if strings_only and isinstance(s, (type(None), int)): - return s - # if isinstance(s, Promise): - # return unicode(s).encode(encoding, errors) - if not isinstance(s, str): - try: - return str(s) - except UnicodeEncodeError: - if isinstance(s, Exception): - # An Exception subclass containing non-ASCII data that doesn't - # know how to print itself properly. We shouldn't raise a - # further exception. - return ' '.join([smart_str(arg, encoding, strings_only, - errors) for arg in s]) - return str(s).encode(encoding, errors) - elif isinstance(s, str): - return s.encode(encoding, errors) - elif s and encoding != 'utf-8': - return s.decode('utf-8', errors).encode(encoding, errors) - else: - return s diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 2032f716..621c2f49 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -29,8 +29,6 @@ Article, Source, ArticleException, news_pool) from newspaper.configuration import Configuration from newspaper.urls import get_domain -from newspaper.utils.encoding import smart_str, smart_unicode -from newspaper.utils import encodeValue # from newspaper import Config # from newspaper.network import multithread_request # from newspaper.text import (StopWords, StopWordsArabic, @@ -148,7 +146,7 @@ def test_url(self): @print_test def test_download_html(self): self.canon_url = ('http://www.cnn.com/2013/11/27/travel/' - 'weather-thanksgiving/index.html') + 'weather-thanksgiving/index.html') resp = mock_response_with(self.canon_url, 'cnn_article') self.article.download(resp) assert len(self.article.html) == 75176 @@ -392,32 +390,6 @@ def test_popular_urls(self): newspaper.popular_urls() -class EncodingTestCase(unittest.TestCase): - def runTest(self): - self.test_encode_val() - self.test_smart_unicode() - self.test_smart_str() - - def setUp(self): - self.uni_string = "∆ˆˆø∆ßåßlucas yang˜" - self.normal_string = "∆ƒˆƒ´´lucas yang" - - @print_test - def test_encode_val(self): - assert encodeValue(self.uni_string) == self.uni_string - assert encodeValue(self.normal_string) == '∆ƒˆƒ´´lucas yang' - - @print_test - def test_smart_unicode(self): - assert smart_unicode(self.uni_string) == self.uni_string - assert smart_unicode(self.normal_string) == '∆ƒˆƒ´´lucas yang' - - @print_test - def test_smart_str(self): - assert smart_str(self.uni_string) == b'\xe2\x88\x86\xcb\x86\xcb\x86\xc3\xb8\xe2\x88\x86\xc3\x9f\xc3\xa5\xc3\x9flucas yang\xcb\x9c' - assert smart_str(self.normal_string) == b'\xe2\x88\x86\xc6\x92\xcb\x86\xc6\x92\xc2\xb4\xc2\xb4lucas yang' - - class MThreadingTestCase(unittest.TestCase): def runTest(self): self.test_download_works() @@ -534,7 +506,6 @@ def test_spanish_fulltext_extract(self): # suite.addTest(ExhaustiveFullTextCase()) suite.addTest(ConfigBuildTestCase()) suite.addTest(MultiLanguageTestCase()) - suite.addTest(EncodingTestCase()) suite.addTest(UrlTestCase()) suite.addTest(ArticleTestCase()) suite.addTest(APITestCase()) From f137922fdc57c46ac4b71c139518344e7cbb6da3 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang <lucasyangpersonal@gmail.com> Date: Wed, 31 Dec 2014 23:32:50 -0800 Subject: [PATCH 09/13] Move the utils module from a utils/__init__ into utils.py --- newspaper/{utils/__init__.py => utils.py} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename newspaper/{utils/__init__.py => utils.py} (99%) diff --git a/newspaper/utils/__init__.py b/newspaper/utils.py similarity index 99% rename from newspaper/utils/__init__.py rename to newspaper/utils.py index 7255f485..9373d90d 100644 --- a/newspaper/utils/__init__.py +++ b/newspaper/utils.py @@ -22,7 +22,7 @@ from hashlib import sha1 -from .. import settings +from . import settings log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) @@ -33,7 +33,7 @@ class FileHelper(object): def loadResourceFile(self, filename): if not os.path.isabs(filename): dirpath = os.path.abspath(os.path.dirname(__file__)) - path = os.path.join(dirpath, '../resources', filename) + path = os.path.join(dirpath, 'resources', filename) else: path = filename try: From 52e744c9eaa3a6156494d1bbe300dad4d2ef5fee Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang <lucasyangpersonal@gmail.com> Date: Wed, 31 Dec 2014 23:40:41 -0800 Subject: [PATCH 10/13] [refactor] Remove derpy code and format as per PEP8 --- newspaper/extractors.py | 8 +++++--- newspaper/images.py | 12 ++++++++---- newspaper/mthreading.py | 3 ++- newspaper/source.py | 4 ++-- newspaper/utils.py | 3 ++- 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 228d8d8a..b9e72316 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -396,7 +396,8 @@ def get_img_urls(self, article_url, doc): img_tags = self.parser.getElementsByTag(doc, **img_kwargs) urls = [img_tag.get('src') for img_tag in img_tags if img_tag.get('src')] - img_links = set([urllib.parse.urljoin(article_url, url) for url in urls]) + img_links = set([urllib.parse.urljoin(article_url, url) + for url in urls]) return img_links def get_first_img_url(self, article_url, top_node): @@ -442,7 +443,7 @@ def get_urls(self, doc_or_html, titles=False, regex=False): doc_or_html = [i.strip() for i in doc_or_html] return doc_or_html or [] # If the doc_or_html is html, parse it into a root - if isinstance(doc_or_html, str) or isinstance(doc_or_html, str): + if isinstance(doc_or_html, str): doc = self.parser.fromstring(doc_or_html) else: doc = doc_or_html @@ -463,7 +464,8 @@ def get_category_urls(self, source_url, doc): if not domain and not path: if self.config.verbose: - print('elim category url %s for no domain and path' % p_url) + print('elim category url %s for no domain and path' + % p_url) continue if path and path.startswith('#'): if self.config.verbose: diff --git a/newspaper/images.py b/newspaper/images.py index 1ae5a6ab..54e5b1e1 100644 --- a/newspaper/images.py +++ b/newspaper/images.py @@ -12,8 +12,9 @@ import math import io import traceback -import urllib.request, urllib.parse, urllib.error -import urllib.request, urllib.error, urllib.parse +import urllib.request +import urllib.parse +import urllib.error from http.client import InvalidURL from PIL import Image, ImageFile @@ -79,7 +80,8 @@ def clean_url(url): """Url quotes unicode data out of urls """ url = url.encode('utf8') - url = ''.join([urllib.parse.quote(c) if ord(c) >= 127 else c for c in url.decode('utf-8')]) + url = ''.join([urllib.parse.quote(c) + if ord(c) >= 127 else c for c in url.decode('utf-8')]) return url @@ -151,7 +153,9 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False): return content_type, content - except (urllib.error.URLError, urllib.error.HTTPError, InvalidURL) as e: + except (urllib.error.URLError, + urllib.error.HTTPError, + InvalidURL) as e: cur_try += 1 if cur_try >= retries: log.debug('error while fetching: %s refer: %s' % diff --git a/newspaper/mthreading.py b/newspaper/mthreading.py index c1cb355c..5164635a 100644 --- a/newspaper/mthreading.py +++ b/newspaper/mthreading.py @@ -101,7 +101,8 @@ def join(self): resets the task. """ if self.pool is None: - print('Call set(..) with a list of source objects before .join(..)') + print('Call set(..) with a list of source ' + 'objects before .join(..)') raise self.pool.wait_completion() self.papers = [] diff --git a/newspaper/source.py b/newspaper/source.py index 367e77aa..9bd39ac5 100644 --- a/newspaper/source.py +++ b/newspaper/source.py @@ -338,8 +338,8 @@ def download_articles(self, threads=1): self.is_downloaded = True if len(failed_articles) > 0: if self.config.verbose: - print('[ERROR], these article urls failed the download:', \ - [a.url for a in failed_articles]) + print('[ERROR], these article urls failed the download:', + [a.url for a in failed_articles]) def parse_articles(self): """Parse all articles, delete if too small diff --git a/newspaper/utils.py b/newspaper/utils.py index 9373d90d..aede89b5 100644 --- a/newspaper/utils.py +++ b/newspaper/utils.py @@ -195,7 +195,8 @@ def inner_function(*args, **kwargs): """Calculate a cache key based on the decorated method signature args[1] indicates the domain of the inputs, we hash on domain! """ - key = sha1((str(args[1]) + str(kwargs)).encode('utf-8')).hexdigest() + key = sha1((str(args[1]) + + str(kwargs)).encode('utf-8')).hexdigest() filepath = os.path.join(cache_folder, key) # verify that the cached object exists and is less than From c1918fae45b563df60f9a1e6da982424ab270f0f Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang <lucasyangpersonal@gmail.com> Date: Wed, 31 Dec 2014 23:55:00 -0800 Subject: [PATCH 11/13] Removing derp --- newspaper/article.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/newspaper/article.py b/newspaper/article.py index b58d905d..afa19d2c 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -51,7 +51,6 @@ def __init__(self, url, title='', source_url='', config=None, **kwargs): # URL to the main page of the news source which owns this article self.source_url = source_url - url = url self.url = urls.prepare_url(url, self.source_url) self.title = title @@ -375,7 +374,6 @@ def set_title(self, title): # <title> extraction failed return title = title[:self.config.MAX_TITLE] - title = title if title: self.title = title From bcc50ed74c2375e486c5363ef0cb50cb6985bef6 Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang <lucasyangpersonal@gmail.com> Date: Thu, 1 Jan 2015 00:03:30 -0800 Subject: [PATCH 12/13] Remove unused folder/file under /resources --- newspaper/resources/images/known-image-css.txt | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 newspaper/resources/images/known-image-css.txt diff --git a/newspaper/resources/images/known-image-css.txt b/newspaper/resources/images/known-image-css.txt deleted file mode 100644 index a7afb7ce..00000000 --- a/newspaper/resources/images/known-image-css.txt +++ /dev/null @@ -1,9 +0,0 @@ -latimes.com^thumbnail -cnn.com^storytext|cnn_strycntntlft -foxnews.com^entry-content -msn.com^articleText -go.com^mediaimage -lefigaro.fr^photo center -cadres.apec.fr^noFieldsTable -emploi.lesechos.fr^offerHeader -linkfinance.fr^offerHeader \ No newline at end of file From 3c72358e2dd37fd299efd5be5656d790c2812f4a Mon Sep 17 00:00:00 2001 From: Lucas Ou-Yang <lucasyangpersonal@gmail.com> Date: Thu, 1 Jan 2015 00:08:31 -0800 Subject: [PATCH 13/13] Bump version to 0.1.2 --- README.rst | 4 +--- docs/index.rst | 2 +- newspaper/version.py | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 1c5ed672..04aa9e5e 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,7 @@ Newspaper3k: Article scraping & curation ======================================== -.. image:: https://badge.fury.io/py/newspaper.png - :target: http://badge.fury.io/py/newspaper - :alt: Latest version +[![PyPI version](https://badge.fury.io/py/newspaper3k.svg)](http://badge.fury.io/py/newspaper3k) Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed: diff --git a/docs/index.rst b/docs/index.rst index 49577ca4..108471e7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ Newspaper: Article scraping & curation ====================================== -Release v0.0.7. :ref:`(Installation) <install>`. +Release v0.1.2. :ref:`(Installation) <install>`. Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed. diff --git a/newspaper/version.py b/newspaper/version.py index 558d6c6c..8b41029f 100644 --- a/newspaper/version.py +++ b/newspaper/version.py @@ -7,5 +7,5 @@ __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' -version_info = (0, 1, 1) +version_info = (0, 1, 2) __version__ = ".".join(map(str, version_info)) diff --git a/setup.py b/setup.py index eb18e300..6c5aae18 100755 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ setup( name='newspaper3k', - version='0.1.1', + version='0.1.2', description='Simplified python article discovery & extraction.', long_description=readme, author='Lucas Ou-Yang',