diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..f8c996b6 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +docs/* linguist-documentation +tests/** linguist-vendored diff --git a/.gitignore b/.gitignore index 7fbac9b4..3d20afb1 100644 --- a/.gitignore +++ b/.gitignore @@ -40,8 +40,3 @@ nosetests.xml .project .pydevproject venv - -# Utility files for generating resources for unit testing -generate_fulltext.py -generate_html.py -generate_urls.py diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..fb179079 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,9 @@ +version: 2 + +build: + os: ubuntu-24.04 + tools: + python: "3.12" + +sphinx: + configuration: docs/conf.py diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..8af415d8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,12 @@ +language: python +python: + - "3.5" + - "3.6" + - "3.7" +install: + - pip install -r requirements.txt coverage coveralls + - python download_corpora.py +script: + - coverage run --source newspaper tests/unit_tests.py +after_success: + coveralls diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..f88b3f62 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,263 @@ +# Change Log + +## [0.1.7](https://github.com/codelucas/newspaper/tree/0.1.7) (2016-01-30) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.1.6...0.1.7) + +**Closed issues:** + +- ImportError: cannot import name 'Image' [\#183](https://github.com/codelucas/newspaper/issues/183) +- Won't let me import [\#182](https://github.com/codelucas/newspaper/issues/182) +- Install on Mac - El Capitan Failed - "Operation not permitted" [\#181](https://github.com/codelucas/newspaper/issues/181) +- Downgrades to old versions of required packages upon installation [\#174](https://github.com/codelucas/newspaper/issues/174) +- Handling 404, 500, and other non-200 http response codes to prevent scraping error pages [\#142](https://github.com/codelucas/newspaper/issues/142) +- Libray downgrading in installation [\#138](https://github.com/codelucas/newspaper/issues/138) + +**Merged pull requests:** + +- Don't scrape error pages [\#190](https://github.com/codelucas/newspaper/pull/190) ([yprez](https://github.com/yprez)) +- Added Hebrew stop words for language support [\#188](https://github.com/codelucas/newspaper/pull/188) ([alon7](https://github.com/alon7)) +- Fix installation and build [\#187](https://github.com/codelucas/newspaper/pull/187) ([yprez](https://github.com/yprez)) +- Fix installation docs [\#184](https://github.com/codelucas/newspaper/pull/184) ([yprez](https://github.com/yprez)) +- Travis CI integration [\#180](https://github.com/codelucas/newspaper/pull/180) ([yprez](https://github.com/yprez)) +- requirements.txt - Use minimal instead of exact versions [\#179](https://github.com/codelucas/newspaper/pull/179) ([yprez](https://github.com/yprez)) +- Handle lxml raising ValueError on node.itertext\(\) - Python 3 [\#178](https://github.com/codelucas/newspaper/pull/178) ([yprez](https://github.com/yprez)) +- Handle lxml raising ValueError on node.itertext\(\) [\#144](https://github.com/codelucas/newspaper/pull/144) ([yprez](https://github.com/yprez)) +- Parse byline fix [\#132](https://github.com/codelucas/newspaper/pull/132) ([davecrumbacher](https://github.com/davecrumbacher)) + +## [0.1.6](https://github.com/codelucas/newspaper/tree/0.1.6) (2016-01-10) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.1.5...0.1.6) + +**Closed issues:** + +- Critical leak in newspaper.mthreading.Worker [\#177](https://github.com/codelucas/newspaper/issues/177) +- HTMLParseError [\#165](https://github.com/codelucas/newspaper/issues/165) +- Take local paths to .html files [\#153](https://github.com/codelucas/newspaper/issues/153) +- Wall Street Journal Full Text is not Correctly Scraped [\#150](https://github.com/codelucas/newspaper/issues/150) +- Article HTML Returning Null [\#131](https://github.com/codelucas/newspaper/issues/131) +- No articles [\#130](https://github.com/codelucas/newspaper/issues/130) +- Loading Pages that use heavy javascript [\#127](https://github.com/codelucas/newspaper/issues/127) +- Login handling for premium websites [\#126](https://github.com/codelucas/newspaper/issues/126) +- Installation of nltk is failing [\#121](https://github.com/codelucas/newspaper/issues/121) + +**Merged pull requests:** + +- Support urls with dots [\#176](https://github.com/codelucas/newspaper/pull/176) ([alexanderlukanin13](https://github.com/alexanderlukanin13)) +- upgrade beautifulsoup4 to 4.4.1 for python 3.5 [\#171](https://github.com/codelucas/newspaper/pull/171) ([AlJohri](https://github.com/AlJohri)) +- Updated requests version [\#170](https://github.com/codelucas/newspaper/pull/170) ([adrienthiery](https://github.com/adrienthiery)) +- Turkish Language added [\#169](https://github.com/codelucas/newspaper/pull/169) ([muratcorlu](https://github.com/muratcorlu)) +- Add macedonian stopwords [\#166](https://github.com/codelucas/newspaper/pull/166) ([dimitrovskif](https://github.com/dimitrovskif)) +- Issue\#95 added graceful string concatenation [\#157](https://github.com/codelucas/newspaper/pull/157) ([surajssd](https://github.com/surajssd)) +- fix for "jpeg error with PIL, Can't convert 'NoneType' object to str implicitly" [\#154](https://github.com/codelucas/newspaper/pull/154) ([hnykda](https://github.com/hnykda)) +- bugfix in article.py, is\_valid\_body [\#149](https://github.com/codelucas/newspaper/pull/149) ([ms8r](https://github.com/ms8r)) +- Fixed typo [\#139](https://github.com/codelucas/newspaper/pull/139) ([Eleonore9](https://github.com/Eleonore9)) +- Correct link for the Python 3 branch [\#136](https://github.com/codelucas/newspaper/pull/136) ([jtpio](https://github.com/jtpio)) +- Add python3-pip install step for Ubuntu [\#135](https://github.com/codelucas/newspaper/pull/135) ([irnc](https://github.com/irnc)) + +## [0.1.5](https://github.com/codelucas/newspaper/tree/0.1.5) (2015-03-04) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.1.4...0.1.5) + +**Closed issues:** + +- is there any kind of documentation on centos 7? [\#114](https://github.com/codelucas/newspaper/issues/114) +- Add extraction publishing date from article. [\#3](https://github.com/codelucas/newspaper/issues/3) + +**Merged pull requests:** + +- bumping nltk to 2.0.5 - see \#824 in nltk [\#125](https://github.com/codelucas/newspaper/pull/125) ([hexelon](https://github.com/hexelon)) + +## [0.1.4](https://github.com/codelucas/newspaper/tree/0.1.4) (2015-02-04) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.1.3...0.1.4) + +**Closed issues:** + +- Getting rate limiting issue? [\#116](https://github.com/codelucas/newspaper/issues/116) +- newspaper.build\( \) error [\#111](https://github.com/codelucas/newspaper/issues/111) +- Allow lists in Parser.clean\_article\_html\(\) [\#108](https://github.com/codelucas/newspaper/issues/108) + +**Merged pull requests:** + +- Fix incorrect log call while generating articles [\#115](https://github.com/codelucas/newspaper/pull/115) ([curita](https://github.com/curita)) +- Allow lists in clean\_article\_html\(\) - fixes \#108 [\#112](https://github.com/codelucas/newspaper/pull/112) ([ecesena](https://github.com/ecesena)) +- Fixed nodeToString\(\) to return valid HTML [\#110](https://github.com/codelucas/newspaper/pull/110) ([ecesena](https://github.com/ecesena)) +- Fixed empty return in top\_meta\_image [\#109](https://github.com/codelucas/newspaper/pull/109) ([ecesena](https://github.com/ecesena)) + +## [0.1.3](https://github.com/codelucas/newspaper/tree/0.1.3) (2015-01-15) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.1.2...0.1.3) + +**Implemented enhancements:** + +- Fulltext extraction improvement \#1 [\#105](https://github.com/codelucas/newspaper/issues/105) + +**Closed issues:** + +- Tags h1 in article\_html - indented behavior? [\#107](https://github.com/codelucas/newspaper/issues/107) + +**Merged pull requests:** + +- Fulltext extraction improvement \#1 [\#106](https://github.com/codelucas/newspaper/pull/106) ([codelucas](https://github.com/codelucas)) + +## [0.1.2](https://github.com/codelucas/newspaper/tree/0.1.2) (2015-01-01) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.1.1...0.1.2) + +**Closed issues:** + +- Metatags on Vice.com [\#103](https://github.com/codelucas/newspaper/issues/103) +- Can't extract images from german newspapers [\#96](https://github.com/codelucas/newspaper/issues/96) +- article\_html misses many of the images [\#89](https://github.com/codelucas/newspaper/issues/89) + +**Merged pull requests:** + +- Integrate UnicodeDammit, deprecate parser\_class, deprecate encodeValue, refactor, scaffolding for more unit tests [\#104](https://github.com/codelucas/newspaper/pull/104) ([codelucas](https://github.com/codelucas)) + +## [0.1.1](https://github.com/codelucas/newspaper/tree/0.1.1) (2014-12-27) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.1.0...0.1.1) + +**Closed issues:** + +- UnicodeDecodeError: 'utf8' codec can't decode byte 0xcc [\#99](https://github.com/codelucas/newspaper/issues/99) +- TypeError: Can't convert 'bytes' object to str implicitly [\#98](https://github.com/codelucas/newspaper/issues/98) +- \[Parse lxml ERR\] Unicode strings with encoding declaration are not supported. Please use bytes input or XML fragments without declaration. [\#78](https://github.com/codelucas/newspaper/issues/78) +- UnicodeDecodeError: 'ascii' codec can't decode byte 0xe2 in position 11: ordinal not in range\(128\) [\#77](https://github.com/codelucas/newspaper/issues/77) +- article.text and keywords error [\#47](https://github.com/codelucas/newspaper/issues/47) + +**Merged pull requests:** + +- Huge bugfix to aid lxml DOM parsing + remove unhelpful and excess exception messages and added tracebacks to exception logging [\#102](https://github.com/codelucas/newspaper/pull/102) ([codelucas](https://github.com/codelucas)) +- Decode bytestring returned from lxml's `toString` early on before sending it out to outer code [\#101](https://github.com/codelucas/newspaper/pull/101) ([codelucas](https://github.com/codelucas)) +- Fixed \#78: Remove encoding tag because lxml won't accept it for unicode [\#97](https://github.com/codelucas/newspaper/pull/97) ([mhall1](https://github.com/mhall1)) + +## [0.1.0](https://github.com/codelucas/newspaper/tree/0.1.0) (2014-12-17) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.0.9...0.1.0) + +## [0.0.9](https://github.com/codelucas/newspaper/tree/0.0.9) (2014-12-17) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.0.8...0.0.9) + +**Closed issues:** + +- object has no attribute clean Error when using parse method [\#90](https://github.com/codelucas/newspaper/issues/90) +- Questions [\#85](https://github.com/codelucas/newspaper/issues/85) +- \[nltk\_data\] Error loading brown: \ [\#84](https://github.com/codelucas/newspaper/issues/84) +- newspaper unable to find embeded youtube video [\#82](https://github.com/codelucas/newspaper/issues/82) +- Bound for memory usage [\#81](https://github.com/codelucas/newspaper/issues/81) +- Hosted demo [\#80](https://github.com/codelucas/newspaper/issues/80) +- Having issues installing due to lxml [\#79](https://github.com/codelucas/newspaper/issues/79) +- Add a BeautifulSoup4 parser. [\#44](https://github.com/codelucas/newspaper/issues/44) +- python 3 support request [\#36](https://github.com/codelucas/newspaper/issues/36) + +**Merged pull requests:** + +- update jieba to 0.35 [\#94](https://github.com/codelucas/newspaper/pull/94) ([WingGao](https://github.com/WingGao)) +- Parse was breaking in the method clean\_article\_html when keep\_article\_ht... [\#88](https://github.com/codelucas/newspaper/pull/88) ([phoenixwizard](https://github.com/phoenixwizard)) +- split title with \_ [\#87](https://github.com/codelucas/newspaper/pull/87) ([deweydu](https://github.com/deweydu)) +- Update to support python3 [\#86](https://github.com/codelucas/newspaper/pull/86) ([log0ymxm](https://github.com/log0ymxm)) +- Added link to basic demo [\#83](https://github.com/codelucas/newspaper/pull/83) ([iwasrobbed](https://github.com/iwasrobbed)) +- Add splitting of slash-separated titles [\#75](https://github.com/codelucas/newspaper/pull/75) ([igor-shevchenko](https://github.com/igor-shevchenko)) + +## [0.0.8](https://github.com/codelucas/newspaper/tree/0.0.8) (2014-10-13) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.0.7...0.0.8) + +**Closed issues:** + +- Parsing Raw HTML [\#74](https://github.com/codelucas/newspaper/issues/74) +- Can't install newspaper [\#72](https://github.com/codelucas/newspaper/issues/72) +- Refactor codebase so newspaper is actually pythonic [\#70](https://github.com/codelucas/newspaper/issues/70) +- Article.top\_node == Article.clean\_top\_node [\#65](https://github.com/codelucas/newspaper/issues/65) +- article.movies missing 'http:' [\#64](https://github.com/codelucas/newspaper/issues/64) +- KeyError when calling newspaper.languages\(\) [\#62](https://github.com/codelucas/newspaper/issues/62) +- Memoize Articles - Not Printing [\#61](https://github.com/codelucas/newspaper/issues/61) +- Add URL headers while building a "paper" [\#60](https://github.com/codelucas/newspaper/issues/60) +- AttributeError: 'module' object has no attribute 'build' [\#59](https://github.com/codelucas/newspaper/issues/59) +- Typo in newspaper.build argument "memoize\_articles" [\#58](https://github.com/codelucas/newspaper/issues/58) +- issue with stopwords-tr.txt [\#51](https://github.com/codelucas/newspaper/issues/51) +- Other language support. [\#34](https://github.com/codelucas/newspaper/issues/34) +- Character encoding detection [\#2](https://github.com/codelucas/newspaper/issues/2) + +**Merged pull requests:** + +- Huge refactor: entire codebase in PEP8, imports alphabetized, bugfixes, core changes [\#71](https://github.com/codelucas/newspaper/pull/71) ([codelucas](https://github.com/codelucas)) +- Meta tag extraction fixes [\#69](https://github.com/codelucas/newspaper/pull/69) ([karls](https://github.com/karls)) +- Test suite improvements [\#68](https://github.com/codelucas/newspaper/pull/68) ([karls](https://github.com/karls)) +- Test suite fixes [\#67](https://github.com/codelucas/newspaper/pull/67) ([karls](https://github.com/karls)) +- Revert "Added published date to the extractor+article" [\#66](https://github.com/codelucas/newspaper/pull/66) ([codelucas](https://github.com/codelucas)) +- Added published date to the extractor+article [\#63](https://github.com/codelucas/newspaper/pull/63) ([parhammmm](https://github.com/parhammmm)) + +## [0.0.7](https://github.com/codelucas/newspaper/tree/0.0.7) (2014-06-17) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.0.6...0.0.7) + +**Closed issues:** + +- no document on how to add language [\#57](https://github.com/codelucas/newspaper/issues/57) +- Retain \ tags in top article node? [\#56](https://github.com/codelucas/newspaper/issues/56) +- DocumentCleaner is missing clean\_body\_classes [\#55](https://github.com/codelucas/newspaper/issues/55) +- You must download and parse an article before parsing it [\#52](https://github.com/codelucas/newspaper/issues/52) +- Not extracting UL LI text [\#50](https://github.com/codelucas/newspaper/issues/50) +- article does not release\_resources\(\) [\#42](https://github.com/codelucas/newspaper/issues/42) +- Doesn't work on http://www.le360.ma/fr [\#40](https://github.com/codelucas/newspaper/issues/40) +- How to assign html content without downloading it? [\#37](https://github.com/codelucas/newspaper/issues/37) +- Python venv only? [\#32](https://github.com/codelucas/newspaper/issues/32) +- .nlp\(\) could not work [\#27](https://github.com/codelucas/newspaper/issues/27) +- Doesn't work with Arabic news sites [\#23](https://github.com/codelucas/newspaper/issues/23) +- SyntaxError: invalid syntax [\#19](https://github.com/codelucas/newspaper/issues/19) +- Retain HTML markup for extracted article [\#18](https://github.com/codelucas/newspaper/issues/18) +- Portuguese is misspelled [\#14](https://github.com/codelucas/newspaper/issues/14) +- Multi-threading article downloads not working [\#12](https://github.com/codelucas/newspaper/issues/12) +- Timegm error? [\#10](https://github.com/codelucas/newspaper/issues/10) +- Problem in Brazilian sites [\#9](https://github.com/codelucas/newspaper/issues/9) +- Brazilian portuguese support [\#6](https://github.com/codelucas/newspaper/issues/6) + +**Merged pull requests:** + +- Fix typo in code and documentation [\#54](https://github.com/codelucas/newspaper/pull/54) ([jacquerie](https://github.com/jacquerie)) +- removed quotes of 'filename' in utils\\_\_init\_\_.py [\#53](https://github.com/codelucas/newspaper/pull/53) ([jay8688](https://github.com/jay8688)) +- Fixed long-form article issue w/ calculate\_best\_node [\#49](https://github.com/codelucas/newspaper/pull/49) ([jeffnappi](https://github.com/jeffnappi)) +- Use first image from article top\_node [\#35](https://github.com/codelucas/newspaper/pull/35) ([otemnov](https://github.com/otemnov)) +- Add a section with links to related projects [\#33](https://github.com/codelucas/newspaper/pull/33) ([cantino](https://github.com/cantino)) +- Original [\#30](https://github.com/codelucas/newspaper/pull/30) ([otemnov](https://github.com/otemnov)) +- Fix reddit top image [\#29](https://github.com/codelucas/newspaper/pull/29) ([otemnov](https://github.com/otemnov)) +- Extract Meta Tags in structured way [\#28](https://github.com/codelucas/newspaper/pull/28) ([voidfiles](https://github.com/voidfiles)) +- Replace instances of 'Portugease' with 'Portuguese' [\#26](https://github.com/codelucas/newspaper/pull/26) ([WheresWardy](https://github.com/WheresWardy)) +- It's The Changelog not The ChangeLog :\) [\#24](https://github.com/codelucas/newspaper/pull/24) ([adamstac](https://github.com/adamstac)) +- syntax errors [\#22](https://github.com/codelucas/newspaper/pull/22) ([arjun024](https://github.com/arjun024)) +- Support for more HTML tags in parsers.py [\#21](https://github.com/codelucas/newspaper/pull/21) ([WheresWardy](https://github.com/WheresWardy)) +- Fixed syntax error [\#20](https://github.com/codelucas/newspaper/pull/20) ([damilare](https://github.com/damilare)) +- Minor Performance tweaks [\#17](https://github.com/codelucas/newspaper/pull/17) ([techaddict](https://github.com/techaddict)) +- Update README.rst [\#15](https://github.com/codelucas/newspaper/pull/15) ([girasquid](https://github.com/girasquid)) +- Minor Typo candiate\_words -\> candidate\_words [\#13](https://github.com/codelucas/newspaper/pull/13) ([techaddict](https://github.com/techaddict)) + +## [0.0.6](https://github.com/codelucas/newspaper/tree/0.0.6) (2014-01-18) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.0.5...0.0.6) + +**Closed issues:** + +- Port to Ruby [\#8](https://github.com/codelucas/newspaper/issues/8) +- Huge internationalization / API revamp underway! [\#7](https://github.com/codelucas/newspaper/issues/7) +- Multithread & gevent framework built into newspaper [\#4](https://github.com/codelucas/newspaper/issues/4) + +**Merged pull requests:** + +- Add article html extraction [\#11](https://github.com/codelucas/newspaper/pull/11) ([voidfiles](https://github.com/voidfiles)) + +## [0.0.5](https://github.com/codelucas/newspaper/tree/0.0.5) (2014-01-09) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.0.4...0.0.5) + +## [0.0.4](https://github.com/codelucas/newspaper/tree/0.0.4) (2013-12-31) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.0.3...0.0.4) + +**Closed issues:** + +- Calling nlp\(\) on an article causes 'tokenizers/punkt/english.pickle' Not Found Error [\#1](https://github.com/codelucas/newspaper/issues/1) + +**Merged pull requests:** + +- Fix for keyword arg usage in print\(\) on Python 2.7 [\#5](https://github.com/codelucas/newspaper/pull/5) ([michaelhood](https://github.com/michaelhood)) + +## [0.0.3](https://github.com/codelucas/newspaper/tree/0.0.3) (2013-12-22) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.0.2...0.0.3) + +## [0.0.2](https://github.com/codelucas/newspaper/tree/0.0.2) (2013-12-21) +[Full Changelog](https://github.com/codelucas/newspaper/compare/0.0.1...0.0.2) + +## [0.0.1](https://github.com/codelucas/newspaper/tree/0.0.1) (2013-12-21) + + +\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 7de2595d..edecd02a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include requirements.txt README.rst LICENSE recursive-include newspaper * \ No newline at end of file +include requirements.txt README.rst LICENSE recursive-include newspaper * recursive-exclude * __pycache__ recursive-exclude * *.py[co] \ No newline at end of file diff --git a/README.rst b/README.rst index 54b7a849..dcbee056 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,14 @@ Newspaper3k: Article scraping & curation :target: http://badge.fury.io/py/newspaper3k.svg :alt: Latest version +.. image:: https://travis-ci.org/codelucas/newspaper.svg + :target: http://travis-ci.org/codelucas/newspaper/ + :alt: Build status + +.. image:: https://coveralls.io/repos/github/codelucas/newspaper/badge.svg?branch=master + :target: https://coveralls.io/github/codelucas/newspaper + :alt: Coverage status + Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed: "Newspaper is an amazing python library for extracting & curating articles." @@ -15,66 +23,19 @@ Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed: .. _`tweeted by`: https://twitter.com/kennethreitz/status/419520678862548992 .. _`The Changelog`: http://thechangelog.com/newspaper-delivers-instapaper-style-article-extraction/ - -**Newspaper is a Python3 library**! Or, view the `Python2 branch`_ +**Newspaper is a Python3 library**! Or, view our **deprecated and buggy** `Python2 branch`_ .. _`Python2 branch`: https://github.com/codelucas/newspaper/tree/python-2-head -**We support 10+ languages and everything is in unicode!** - -.. code-block:: pycon - - >>> import newspaper - >>> newspaper.languages() - - Your available languages are: - input code full name - - ar Arabic - ru Russian - nl Dutch - de German - en English - es Spanish - fr French - it Italian - ko Korean - no Norwegian - pt Portuguese - sv Swedish - hu Hungarian - fi Finnish - da Danish - zh Chinese - id Indonesian - vi Vietnamese - A Glance: --------- .. code-block:: pycon - >>> import newspaper - - >>> cnn_paper = newspaper.build('http://cnn.com') - - >>> for article in cnn_paper.articles: - >>> print(article.url) - 'http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/' - 'http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html' - ... - - >>> for category in cnn_paper.category_urls(): - >>> print(category) - - 'http://lifestyle.cnn.com' - 'http://cnn.com/world' - 'http://tech.cnn.com' - ... - -.. code-block:: pycon + >>> from newspaper import Article - >>> article = cnn_paper.articles[0] + >>> url = 'http://fox13now.com/2013/12/30/new-year-new-laws-obamacare-pot-guns-and-drones/' + >>> article = Article(url) .. code-block:: pycon @@ -90,6 +51,9 @@ A Glance: >>> article.authors ['Leigh Ann Caldwell', 'John Honway'] + >>> article.publish_date + datetime.datetime(2013, 12, 30, 0, 0) + >>> article.text 'Washington (CNN) -- Not everyone subscribes to a New Year's resolution...' @@ -109,8 +73,41 @@ A Glance: >>> article.summary 'The study shows that 93% of people ...' +.. code-block:: pycon -Newspaper has *seamless* language extraction and detection. + >>> import newspaper + + >>> cnn_paper = newspaper.build('http://cnn.com') + + >>> for article in cnn_paper.articles: + >>> print(article.url) + http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/ + http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html + ... + + >>> for category in cnn_paper.category_urls(): + >>> print(category) + + http://lifestyle.cnn.com + http://cnn.com/world + http://tech.cnn.com + ... + + >>> cnn_article = cnn_paper.articles[0] + >>> cnn_article.download() + >>> cnn_article.parse() + >>> cnn_article.nlp() + ... + +.. code-block:: pycon + + >>> from newspaper import fulltext + + >>> html = requests.get(...).text + >>> text = fulltext(html) + + +Newspaper can extract and detect languages *seamlessly*. If no language is specified, Newspaper will attempt to auto detect a language. .. code-block:: pycon @@ -134,6 +131,8 @@ If no language is specified, Newspaper will attempt to auto detect a language. >>> print(a.title) 港特首梁振英就住宅违建事件道歉 +Multi-lingual +============= If you are certain that an *entire* news source is in one language, **go ahead and use the same api :)** @@ -144,9 +143,9 @@ If you are certain that an *entire* news source is in one language, **go ahead a >>> for category in sina_paper.category_urls(): >>> print(category) - 'http://health.sina.com.cn' - 'http://eladies.sina.com.cn' - 'http://english.sina.com' + http://health.sina.com.cn + http://eladies.sina.com.cn + http://english.sina.com ... >>> article = sina_paper.articles[0] @@ -165,18 +164,93 @@ If you are certain that an *entire* news source is in one language, **go ahead a 两年双免0手续0利率 科鲁兹掀背金融轻松购_武汉车市_武汉汽 车网_新浪汽车_新浪网 -Documentation -------------- -Check out `The Documentation`_ for full and detailed guides using newspaper. +Scraping by topic: where do the URLs come from? +=============================================== + +``newspaper.build()`` is perfect when you know *which sites* to crawl. But the other question I get constantly is: "I want every article about **X**, across all publications — where do I get the URLs?" The answer is to search Google News for your keyword first, then feed the result links straight into newspaper for extraction. + +The easiest way to query Google News programmatically is the `Google News API`_ from `SerpApi - Search API`_ (they also cover Google Search, Google Maps, and more). The two libraries snap together in a few lines: + +.. code-block:: python + + # pip3 install google-search-results + from serpapi import GoogleSearch + from newspaper import Article + + search = GoogleSearch({ + "engine": "google_news", + "q": "electric vehicles", + "api_key": "YOUR_SERPAPI_KEY", # free plan at serpapi.com + }) + + for result in search.get_dict()["news_results"]: + article = Article(result["link"]) + article.download() + article.parse() + article.nlp() + print(article.title, "--", article.summary[:120]) + +This pattern of SerpApi for *discovery*, newspaper3k for *extraction*, is how most production news-monitoring pipelines are built, and it sidesteps writing a crawler for every source you care about. -Interested in adding a new language for us? Refer to: `Docs - Adding new languages `_ +.. _`SerpApi - Search API`: https://serpapi.com?utm_source=newspaper3k_github +.. _`Google News API`: https://serpapi.com/google-news-api?utm_source=newspaper3k_github + + +Scraping at scale: avoiding IP blocks +===================================== + +Once you move past scraping a handful of articles, you'll hit the same wall every news scraper hits: 403s, captchas, rate limits, and silent shadow bans. Your code is fine — your IP is the problem. The fix is rotating residential proxies. + +I personally route my own newspaper3k pipelines through `Swiftproxy`_ — 80M+ residential IPs across 195+ countries, a 99.89% success rate, non-expiring traffic, and a free trial so you can pressure-test it before paying. Plugging it into newspaper3k takes about four lines: + +.. code-block:: python + + from newspaper import Article, Config + + config = Config() + config.proxies = { + 'http': 'http://USERNAME:PASSWORD@gate.swiftproxy.net:7777', + 'https': 'http://USERNAME:PASSWORD@gate.swiftproxy.net:7777', + } + # a real browser UA helps too + config.browser_user_agent = ( + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/124.0.0.0 Safari/537.36' + ) + config.request_timeout = 20 + + article = Article('https://example.com/some-news-story', config=config) + article.download() + article.parse() + print(article.title) + +The same ``config`` object works with ``newspaper.build()`` — every article fetched by the source will rotate through residential IPs automatically: + +.. code-block:: python + + import newspaper + paper = newspaper.build('http://cnn.com', config=config, memoize_articles=False) + for article in paper.articles: + article.download() + article.parse() + +Grab credentials and a free trial at `swiftproxy.net `_. Use code ``PROXY90`` for 10% off your first plan. + +.. _`Swiftproxy`: https://www.swiftproxy.net/?ref=codelucas + + +Docs +---- + +Check out `The Docs`_ for full and detailed guides using newspaper. + +Interested in adding a new language for us? Refer to: `Docs - Adding new languages `_ Features -------- -- Full Python3 and Python2 support -- Works in 10+ languages (English, Chinese, German, Arabic, ...) - Multi-threaded article download framework - News url identification - Text extraction from html @@ -186,18 +260,72 @@ Features - Summary extraction from text - Author extraction from text - Google trending terms extraction +- Works in 10+ languages (English, Chinese, German, Arabic, ...) + +.. code-block:: pycon + + >>> import newspaper + >>> newspaper.languages() + + Your available languages are: + input code full name + + ar Arabic + be Belarusian + bg Bulgarian + da Danish + de German + el Greek + en English + es Spanish + et Estonian + fa Persian + fi Finnish + fr French + he Hebrew + hi Hindi + hr Croatian + hu Hungarian + id Indonesian + it Italian + ja Japanese + ko Korean + lt Lithuanian + mk Macedonian + nb Norwegian (Bokmål) + nl Dutch + no Norwegian + pl Polish + pt Portuguese + ro Romanian + ru Russian + sl Slovenian + sr Serbian + sv Swedish + sw Swahili + th Thai + tr Turkish + uk Ukrainian + vi Vietnamese + zh Chinese Get it now ---------- -Installing newspaper is simple with `pip `_. -However, you will run into fixable issues if you are trying to install on ubuntu. +Run ✅ ``pip3 install newspaper3k`` ✅ -Note that our Python3 package name is ``newspaper3k`` while our Python2 -package name is ``newspaper``. +NOT ⛔ ``pip3 install newspaper`` ⛔ + +On python3 you must install ``newspaper3k``, **not** ``newspaper``. ``newspaper`` is our python2 library. +Although installing newspaper is simple with `pip `_, you will +run into fixable issues if you are trying to install on ubuntu. **If you are on Debian / Ubuntu**, install using the following: +- Install ``pip3`` command needed to install ``newspaper3k`` package:: + + $ sudo apt-get install python3-pip + - Python development version, needed for Python.h:: $ sudo apt-get install python-dev @@ -210,14 +338,15 @@ package name is ``newspaper``. $ sudo apt-get install libjpeg-dev zlib1g-dev libpng12-dev -- Install the distribution via pip:: - - $ pip3 install newspaper3k +NOTE: If you find problem installing ``libpng12-dev``, try installing ``libpng-dev``. - Download NLP related corpora:: $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3 +- Install the distribution via pip:: + + $ pip3 install newspaper3k **If you are on OSX**, install using the following, you may use both homebrew or macports: @@ -249,9 +378,6 @@ NOTE: You will still most likely need to install the following libraries via you Development ----------- -Newspaper has two branches up for development. *This* branch, the master, is our Python3 -codebase while our Python2 branch is located on *python-2-head*. - If you'd like to contribute and hack on the newspaper project, feel free to clone a development version of this repository locally:: @@ -267,17 +393,63 @@ Feel free to give our testing suite a shot, everything is mocked!:: $ python3 tests/unit_tests.py +Planning on tweaking our full-text algorithm? Add the ``fulltext`` parameter:: + + $ python3 tests/unit_tests.py fulltext Demo ---- View a working online demo here: http://newspaper-demo.herokuapp.com +This is another working online demo: http://newspaper.chinazt.cc/ + + +Interested in scraping APIs & proxies? +====================================== + +Unlock the Web — the Smart Way +------------------------------ +`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`_. +Scrape Google Search, Google News, Google Maps, and more! + +.. image:: https://github.com/user-attachments/assets/9a80eeb4-72a8-43f1-9413-93c7a47b2bf6 + :target: https://serpapi.com/google-news-api?utm_source=newspaper3k_github + :alt: Scrape search engines easily with SerpApi - Search API. + +.. _`Click here to see SerpApi, scrape search engines easily with SerpApi - Search API`: https://serpapi.com?utm_source=newspaper3k_github + + +Power your scraping and automation at real-world scale +------------------------------------------------------ +`Click here to try Swiftproxy`_ — built for developers running scraping, automation, and data collection workflows at scale. Access 80M+ residential IPs from $0.7/GB, fast ISP proxies from $6/IP, global coverage across 195+ countries, non-expiring traffic, and a 99.89% success rate. Free trial available — use code ``PROXY90`` for 10% off. + +.. image:: https://github.com/user-attachments/assets/913f1fd6-20e9-4f37-89b7-ba6b0bd0724a + :target: https://www.swiftproxy.net/?ref=codelucas + :alt: Swiftproxy — residential and ISP proxies built for scrapers and developers. + +.. _`Click here to try Swiftproxy`: https://www.swiftproxy.net/?ref=codelucas + + +Stay private, fast, and fully in control +---------------------------------------- +`Click here to explore BestProxy`_, your go-to solution for premium residential proxies. BestProxy's proxies ensure smooth browsing, fast speeds, and total anonymity. `Get Started`_ today and experience the difference! + +.. image:: https://github.com/user-attachments/assets/1c6ef38c-f0c0-4db0-aad2-3ed9d6adf0b5 + :target: https://bestproxy.com/?keyword=b2vgzl0r + :alt: Experience BestProxy, smooth browsing, fast speeds, and total anonymity. + +.. _`Click here to explore BestProxy`: https://bestproxy.com/?keyword=b2vgzl0r +.. _`Get Started`: https://bestproxy.com/?keyword=b2vgzl0r + LICENSE ------- Authored and maintained by `Lucas Ou-Yang`_. +`Parse.ly`_ sponsored some work on newspaper, specifically focused on +automatic extraction. + Newspaper uses a lot of `python-goose's`_ parsing code. View their license `here`_. Please feel free to `email & contact me`_ if you run into issues or just would like @@ -288,7 +460,12 @@ to talk about the future of this library and news extraction in general! .. _`python-goose's`: https://github.com/grangier/python-goose .. _`here`: https://github.com/codelucas/newspaper/blob/master/GOOSE-LICENSE.txt -.. _`Quickstart guide`: https://newspaper.readthedocs.org/en/latest/ -.. _`The Documentation`: http://newspaper.readthedocs.org +.. _`https://www.paypal.me/codelucas`: https://www.paypal.me/codelucas +.. _`Venmo`: https://www.venmo.com/Lucas-Ou-Yang + +.. _`Quickstart guide`: https://newspaper.readthedocs.io/en/latest/ +.. _`The Docs`: https://newspaper.readthedocs.io .. _`lxml`: http://lxml.de/ .. _`requests`: https://github.com/kennethreitz/requests +.. _`Parse.ly`: http://parse.ly +.. _`It takes only one click`: https://tracking.gitads.io/?campaign=gitads&repo=newspaper&redirect=gitads.io diff --git a/docs/_themes/kr/layout.html b/docs/_themes/kr/layout.html index b59a06fa..49a44ef4 100755 --- a/docs/_themes/kr/layout.html +++ b/docs/_themes/kr/layout.html @@ -6,7 +6,6 @@ {% endif %} - {% endblock %} {%- block relbar2 %}{% endblock %} @@ -15,60 +14,4 @@
© Copyright {{ copyright }}.
- - Fork me on GitHub - - - - - - - - - - - {%- endblock %} diff --git a/docs/index.rst b/docs/index.rst index 108471e7..b4e0bae7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,9 +1,20 @@ -Newspaper: Article scraping & curation -====================================== +Newspaper3k: Article scraping & curation +======================================== -Release v0.1.2. :ref:`(Installation) `. +.. image:: https://badge.fury.io/py/newspaper3k.svg + :target: http://badge.fury.io/py/newspaper3k.svg + :alt: Latest version -Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed. +.. image:: https://travis-ci.org/codelucas/newspaper.svg + :target: http://travis-ci.org/codelucas/newspaper/ + :alt: Build status + +.. image:: https://coveralls.io/repos/github/codelucas/newspaper/badge.svg?branch=master + :target: https://coveralls.io/github/codelucas/newspaper + :alt: Coverage status + + +Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed: "Newspaper is an amazing python library for extracting & curating articles." -- `tweeted by`_ Kenneth Reitz, Author of `requests`_ @@ -11,86 +22,48 @@ Inspired by `requests`_ for its simplicity and powered by `lxml`_ for its speed. "Newspaper delivers Instapaper style article extraction." -- `The Changelog`_ .. _`tweeted by`: https://twitter.com/kennethreitz/status/419520678862548992 -.. _`The Changelog`: http://thechangelog.com/ - -**We support 10+ languages and everything is in unicode!** - -.. code-block:: pycon +.. _`The Changelog`: http://thechangelog.com/newspaper-delivers-instapaper-style-article-extraction/ - >>> import newspaper - >>> newspaper.languages() - - Your available langauges are: - input code full name +**Newspaper is a Python3 library**! `View on Github here`_, or, view our **deprecated and buggy** `Python2 branch`_ - ar Arabic - ru Russian - nl Dutch - de German - en English - es Spanish - fr French - it Italian - ko Korean - no Norwegian - pt Portuguese - sv Swedish - hu Hungarian - fi Finnish - da Danish - zh Chinese - id Indonesian - vi Vietnamese +.. _`Python2 branch`: https://github.com/codelucas/newspaper/tree/python-2-head +.. _`View on Github here`: https://github.com/codelucas/newspaper A Glance: --------- .. code-block:: pycon - >>> import newspaper - - >>> cnn_paper = newspaper.build('http://cnn.com') - - >>> for article in cnn_paper.articles: - >>> print(article.url) - u'http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/' - u'http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html' - ... - - >>> for category in cnn_paper.category_urls(): - >>> print(category) - - u'http://lifestyle.cnn.com' - u'http://cnn.com/world' - u'http://tech.cnn.com' - ... - -.. code-block:: pycon + >>> from newspaper import Article - >>> article = cnn_paper.articles[0] + >>> url = 'http://fox13now.com/2013/12/30/new-year-new-laws-obamacare-pot-guns-and-drones/' + >>> article = Article(url) .. code-block:: pycon >>> article.download() >>> article.html - u'>> article.parse() >>> article.authors - [u'Leigh Ann Caldwell', 'John Honway'] + ['Leigh Ann Caldwell', 'John Honway'] + + >>> article.publish_date + datetime.datetime(2013, 12, 30, 0, 0) >>> article.text - u'Washington (CNN) -- Not everyone subscribes to a New Year's resolution...' + 'Washington (CNN) -- Not everyone subscribes to a New Year's resolution...' >>> article.top_image - u'http://someCDN.com/blah/blah/blah/file.png' + 'http://someCDN.com/blah/blah/blah/file.png' >>> article.movies - [u'http://youtube.com/path/to/link.com', ...] + ['http://youtube.com/path/to/link.com', ...] .. code-block:: pycon @@ -100,7 +73,40 @@ A Glance: ['New Years', 'resolution', ...] >>> article.summary - u'The study shows that 93% of people ...' + 'The study shows that 93% of people ...' + +.. code-block:: pycon + + >>> import newspaper + + >>> cnn_paper = newspaper.build('http://cnn.com') + + >>> for article in cnn_paper.articles: + >>> print(article.url) + http://www.cnn.com/2013/11/27/justice/tucson-arizona-captive-girls/ + http://www.cnn.com/2013/12/11/us/texas-teen-dwi-wreck/index.html + ... + + >>> for category in cnn_paper.category_urls(): + >>> print(category) + + http://lifestyle.cnn.com + http://cnn.com/world + http://tech.cnn.com + ... + + >>> cnn_article = cnn_paper.articles[0] + >>> cnn_article.download() + >>> cnn_article.parse() + >>> cnn_article.nlp() + ... + +.. code-block:: pycon + + >>> from newspaper import fulltext + + >>> html = requests.get(...).text + >>> text = fulltext(html) Newspaper has *seamless* language extraction and detection. @@ -119,9 +125,9 @@ If no language is specified, Newspaper will attempt to auto detect a language. >>> print(a.text[:150]) 香港行政长官梁振英在各方压力下就其大宅的违章建 筑（僭建）问题到立法会接受质询，并向香港民众道歉。 - 梁振英在星期二（12月10日）的答问大会开始之际在其 - 演说中道歉，但强调他在违章建筑问题上没有隐瞒的意 - 图和动机。 一些亲北京阵营议员欢迎梁振英道歉， + 梁振英在星期二（12月10日）的答问大会开始之际 + 在其演说中道歉，但强调他在违章建筑问题上没有隐瞒的 + 意图和动机。 一些亲北京阵营议员欢迎梁振英道歉， 且认为应能获得香港民众接受，但这些议员也质问梁振英有 >>> print(a.title) @@ -137,9 +143,9 @@ If you are certain that an *entire* news source is in one language, **go ahead a >>> for category in sina_paper.category_urls(): >>> print(category) - u'http://health.sina.com.cn' - u'http://eladies.sina.com.cn' - u'http://english.sina.com' + http://health.sina.com.cn + http://eladies.sina.com.cn + http://english.sina.com ... >>> article = sina_paper.articles[0] @@ -147,22 +153,27 @@ If you are certain that an *entire* news source is in one language, **go ahead a >>> article.parse() >>> print(article.text) - 新浪武汉汽车综合 随着汽车市场的日趋成熟，传统的“集 - 全家之力抱得爱车归”的全额购车模式已然过时，另一种轻 - 松的新兴 车模式――金融购车正逐步成为时下消费者购买 - 爱车最为时尚的消费理 念，他们认为，这种新颖的购车模 - 式既能在短期内 + 新浪武汉汽车综合 随着汽车市场的日趋成熟， + 传统的“集全家之力抱得爱车归”的全额购车模式已然过时， + 另一种轻松的新兴 车模式――金融购车正逐步成为时下消费者购 + 买爱车最为时尚的消费理念，他们认为，这种新颖的购车 + 模式既能在短期内 ... >>> print(article.title) - 两年双免0手续0利率 科鲁兹掀背金融轻松购_武汉车市_武汉 - 汽车网_新浪汽车_新浪网 + 两年双免0手续0利率 科鲁兹掀背金融轻松购_武汉车市_武汉汽 + 车网_新浪汽车_新浪网 + +Documentation +------------- + +Check out `The Documentation`_ for full and detailed guides using newspaper. +Interested in adding a new language for us? Refer to: `Docs - Adding new languages `_ Features -------- -- Works in 10+ languages (English, Chinese, German, Arabic, ...) - Multi-threaded article download framework - News url identification - Text extraction from html @@ -172,7 +183,146 @@ Features - Summary extraction from text - Author extraction from text - Google trending terms extraction +- Works in 10+ languages (English, Chinese, German, Arabic, ...) +.. code-block:: pycon + + >>> import newspaper + >>> newspaper.languages() + + Your available languages are: + input code full name + + ar Arabic + be Belarusian + bg Bulgarian + da Danish + de German + el Greek + en English + es Spanish + et Estonian + fa Persian + fi Finnish + fr French + he Hebrew + hi Hindi + hr Croatian + hu Hungarian + id Indonesian + it Italian + ja Japanese + ko Korean + lt Lithuanian + mk Macedonian + nb Norwegian (Bokmål) + nl Dutch + no Norwegian + pl Polish + pt Portuguese + ro Romanian + ru Russian + sl Slovenian + sr Serbian + sv Swedish + sw Swahili + th Thai + tr Turkish + uk Ukrainian + vi Vietnamese + zh Chinese + + +Get it now +---------- + +Run ✅ ``pip3 install newspaper3k`` ✅ + +NOT ⛔ ``pip3 install newspaper`` ⛔ + +On python3 you must install ``newspaper3k``, **not** ``newspaper``. ``newspaper`` is our python2 library. +Although installing newspaper is simple with `pip `_, you will +run into fixable issues if you are trying to install on ubuntu. + +**If you are on Debian / Ubuntu**, install using the following: + +- Install ``pip3`` command needed to install ``newspaper3k`` package:: + + $ sudo apt-get install python3-pip + +- Python development version, needed for Python.h:: + + $ sudo apt-get install python-dev + +- lxml requirements:: + + $ sudo apt-get install libxml2-dev libxslt-dev + +- For PIL to recognize .jpg images:: + + $ sudo apt-get install libjpeg-dev zlib1g-dev libpng12-dev + +NOTE: If you find problem installing ``libpng12-dev``, try installing ``libpng-dev``. + +- Download NLP related corpora:: + + $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3 + +- Install the distribution via pip:: + + $ pip3 install newspaper3k + +**If you are on OSX**, install using the following, you may use both homebrew or macports: + +:: + + $ brew install libxml2 libxslt + + $ brew install libtiff libjpeg webp little-cms2 + + $ pip3 install newspaper3k + + $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3 + + +**Otherwise**, install with the following: + +NOTE: You will still most likely need to install the following libraries via your package manager + +- PIL: ``libjpeg-dev`` ``zlib1g-dev`` ``libpng12-dev`` +- lxml: ``libxml2-dev`` ``libxslt-dev`` +- Python Development version: ``python-dev`` + +:: + + $ pip3 install newspaper3k + + $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python3 + +Using python 2.X? We support python 2, however development work has stopped on the 2.X branch for a +few years now so it is behind in features and is more buggy. `See python 2 installation instructions here `_ + +Development +----------- + +If you'd like to contribute and hack on the newspaper project, feel free to clone +a development version of this repository locally:: + + git clone git://github.com/codelucas/newspaper.git + +Once you have a copy of the source, you can embed it in your Python package, +or install it into your site-packages easily:: + + $ pip3 install -r requirements.txt + $ python3 setup.py install + +Feel free to give our testing suite a shot, everything is mocked!:: + + $ python3 tests/unit_tests.py + +Planning on tweaking our full-text algorithm? Add the ``fulltext`` parameter:: + + $ python3 tests/unit_tests.py fulltext User Guide ---------- @@ -180,33 +330,50 @@ User Guide .. toctree:: :maxdepth: 2 - user_guide/install user_guide/quickstart user_guide/advanced -.. toctree:: - :maxdepth: 1 - - user_guide/contributors - - -.. _`lxml`: http://lxml.de/ -.. _`nltk`: http://nltk.org/ -.. _`requests`: https://github.com/kennethreitz/requests -.. _`goose`: https://github.com/grangier/python-goose +Demo +---- +View a working online demo here: http://newspaper-demo.herokuapp.com +This is another working online demo: http://newspaper.chinazt.cc/ LICENSE ------- Authored and maintained by `Lucas Ou-Yang`_. +`Parse.ly`_ sponsored some work on newspaper, specifically focused on +automatic extraction. + Newspaper uses a lot of `python-goose's`_ parsing code. View their license `here`_. Please feel free to `email & contact me`_ if you run into issues or just would like to talk about the future of this library and news extraction in general! -.. _`Lucas Ou-Yang`: http://lucasou.com +Sponsored by SerpApi +-------------------- + +`Scrape search engines easily with SerpApi - Search API`_. +Scrape Google Search, Google News, Google Maps, and more! Their `Google News API`_ +pairs perfectly with newspaper: use it to discover article URLs by keyword, then +extract them with ``Article``. + +.. image:: https://github.com/user-attachments/assets/9a80eeb4-72a8-43f1-9413-93c7a47b2bf6 + :target: https://serpapi.com/google-news-api?utm_source=newspaper3k_docs + :alt: Scrape search engines easily with SerpApi - Search API. + +.. _`Scrape search engines easily with SerpApi - Search API`: https://serpapi.com?utm_source=newspaper3k_docs +.. _`Google News API`: https://serpapi.com/google-news-api?utm_source=newspaper3k_docs + +.. _`Lucas Ou-Yang`: http://codelucas.com .. _`email & contact me`: mailto:lucasyangpersonal@gmail.com .. _`python-goose's`: https://github.com/grangier/python-goose .. _`here`: https://github.com/codelucas/newspaper/blob/master/GOOSE-LICENSE.txt + +.. _`Quickstart guide`: https://newspaper.readthedocs.io/en/latest/ +.. _`The Documentation`: https://newspaper.readthedocs.io +.. _`lxml`: http://lxml.de/ +.. _`requests`: https://github.com/kennethreitz/requests +.. _`Parse.ly`: http://parse.ly diff --git a/docs/user_guide/advanced.rst b/docs/user_guide/advanced.rst index 4aac8508..f38d7327 100644 --- a/docs/user_guide/advanced.rst +++ b/docs/user_guide/advanced.rst @@ -112,6 +112,13 @@ in ``newspaper/resources/text/.`` 1. Push up a stopwords file in the format of ``stopwords-<2-char-language-code>.txt`` in ``newspaper/resources/text/.`` and we are done! +**Finally, add the new language to the list of available languages in the following files:** + +* README.rst +* docs/index.rst +* docs/user_guide/quickstart.rst +* newspaper/utils.py + Explicitly building a news source --------------------------------- @@ -195,6 +202,8 @@ Here is a full list of the configuration options: ``keep_article_html``, default False, "set to True if you want to preserve html of body text" +``http_success_only``, default True, "set to False to capture non 2XX responses as well" + ``MIN_WORD_COUNT``, default 300, "num of word tokens in article text" ``MIN_SENT_COUNT``, default 7, "num of sentence tokens" @@ -209,12 +218,16 @@ Here is a full list of the configuration options: ``MAX_SUMMARY``, default 5000, "num of chars of the summary" +``MAX_SUMMARY_SENT``, default 5, "num of sentences in summary" + ``MAX_FILE_MEMO``, default 20000, "python setup.py sdist bdist_wininst upload" ``memoize_articles``, default True, "cache and save articles run after run" ``fetch_images``, default True, "set this to false if you don't care about getting images" +``follow_meta_refresh``, default False, "follows a redirect url in a meta refresh html tag" + ``image_dimension_ration``, default 16/9.0, "max ratio for height/width, we ignore if greater" ``language``, default 'en', "run ``newspaper.languages()`` to see available options." diff --git a/docs/user_guide/install.rst b/docs/user_guide/install.rst deleted file mode 100644 index 1e85af86..00000000 --- a/docs/user_guide/install.rst +++ /dev/null @@ -1,82 +0,0 @@ -.. _install: - -Installation -============ - -This part of the documentation covers the installation of newspaper. -The first step to using any software package is getting it properly installed. - -Distribute & Pip ----------------- -Installing newspaper is simple with `pip `_. -However, you will run into fixable issues if you are trying to install on ubuntu. - -**If you are on Debian / Ubuntu**, install using the following: - -- Python development version, needed for Python.h:: - - $ sudo apt-get install python-dev - -- lxml requirements:: - - $ sudo apt-get install libxml2-dev libxslt-dev - -- For PIL to recognize .jpg images:: - - $ sudo apt-get install libjpeg-dev zlib1g-dev libpng12-dev - -- Install the distribution via pip:: - - $ pip install newspaper - -- Download NLP related corpora:: - - $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python2.7 - - -**If you are on OSX**, install using the following, you may use both homebrew or macports: - -:: - - $ brew install libxml2 libxslt - - $ brew install libtiff libjpeg webp little-cms2 - - $ pip install newspaper - - $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python2.7 - - -**Otherwise**, install with the following: - -NOTE: You will still most likely need to install the following libraries via your package manager - -- PIL: ``libjpeg-dev`` ``zlib1g-dev`` ``libpng12-dev`` -- lxml: ``libxml2-dev`` ``libxslt-dev`` -- Python Development version: ``python-dev`` - -:: - - $ pip install newspaper - - $ curl https://raw.githubusercontent.com/codelucas/newspaper/master/download_corpora.py | python2.7 - -Get the Code ------------- - -Newspaper is actively developed on GitHub, where the code is -`always available `_. - -You can clone the public repository:: - - git clone git://github.com/codelucas/newspaper.git - -Once you have a copy of the source, you can embed it in your Python package, -or install it into your site-packages easily:: - - $ pip install -r requirements.txt - $ python setup.py install - -Feel free to give our testing suite a shot:: - - $ python tests/unit_tests.py diff --git a/docs/user_guide/quickstart.rst b/docs/user_guide/quickstart.rst index d7a12c57..24fe07b6 100644 --- a/docs/user_guide/quickstart.rst +++ b/docs/user_guide/quickstart.rst @@ -148,6 +148,25 @@ Initializing an ``Article`` by itself. Note the similar ``language=`` named paramater above. All the config parameters as described for ``Source`` objects also apply for ``Article`` objects! **Source and Article objects have a very similar api**. +Initializing an ``Article`` with the particular content-type ignoring. + +There is option to skip loading of articles with particular content-type, +that can be useful if it is not desired to have delays because of long PDF resources. +The default html value for the particular content type can be provided and then used in order to define the actual content-type of the article + +.. code-block:: pycon + + >>> from newspaper import Article + >>> pdf_defaults = {"application/pdf": "%PDF-", + "application/x-pdf": "%PDF-", + "application/x-bzpdf": "%PDF-", + "application/x-gzpdf": "%PDF-"} + >>> pdf_article = Article(url='https://www.adobe.com/pdf/pdfs/ISO32000-1PublicPatentLicense.pdf', + ignored_content_types_defaults=pdf_defaults) + >>> pdf_article.download() + >>> print(pdf_article.html) + %PDF- + There are endless possibilities on how we can manipulate and build articles. Downloading an Article @@ -252,9 +271,23 @@ of popular news source urls.. In case you need help choosing a news source! en English es Spanish fr French + he Hebrew it Italian ko Korean no Norwegian + fa Persian + pl Polish pt Portuguese sv Swedish zh Chinese + uk Ukrainian + sw Swahili + bg Bulgarian + hr Croatian + ro Romanian + sl Slovenian + sr Serbian + et Estonian + ja Japanese + be Belarusian + lt Lithuanian diff --git a/newspaper/__init__.py b/newspaper/__init__.py index 53df549a..3ce5aa49 100644 --- a/newspaper/__init__.py +++ b/newspaper/__init__.py @@ -7,9 +7,10 @@ __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' +from .api import (build, build_article, fulltext, hot, languages, + popular_urls, Configuration as Config) from .article import Article, ArticleException -from .api import (build, build_article, hot, languages, popular_urls, - NewsPool, Configuration as Config) +from .mthreading import NewsPool from .source import Source from .version import __version__ diff --git a/newspaper/api.py b/newspaper/api.py index c308d1f4..fb98e81d 100644 --- a/newspaper/api.py +++ b/newspaper/api.py @@ -13,13 +13,12 @@ from .article import Article from .configuration import Configuration -from .mthreading import NewsPool from .settings import POPULAR_URLS, TRENDING_URL from .source import Source from .utils import extend_config, print_available_languages -def build(url='', dry=False, config=None, **kwargs): +def build(url='', dry=False, config=None, **kwargs) -> Source: """Returns a constructed source object without downloading or parsing the articles """ @@ -32,7 +31,7 @@ def build(url='', dry=False, config=None, **kwargs): return s -def build_article(url='', config=None, **kwargs): +def build_article(url='', config=None, **kwargs) -> Article: """Returns a constructed article object without downloading or parsing """ @@ -67,3 +66,28 @@ def hot(): except Exception as e: print('ERR hot terms failed!', str(e)) return None + + +def fulltext(html, language='en'): + """Takes article HTML string input and outputs the fulltext + Input string is decoded via UnicodeDammit if needed + """ + from .cleaners import DocumentCleaner + from .configuration import Configuration + from .extractors import ContentExtractor + from .outputformatters import OutputFormatter + + config = Configuration() + config.language = language + + extractor = ContentExtractor(config) + document_cleaner = DocumentCleaner(config) + output_formatter = OutputFormatter(config) + + doc = config.get_parser().fromstring(html) + doc = document_cleaner.clean(doc) + + top_node = extractor.calculate_best_node(doc) + top_node = extractor.post_cleanup(top_node) + text, article_html = output_formatter.get_formatted(top_node) + return text diff --git a/newspaper/article.py b/newspaper/article.py index 99df1473..df0d9c43 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -8,6 +8,9 @@ import copy import os import glob +from urllib.parse import urlparse + +import requests from . import images from . import network @@ -20,12 +23,18 @@ from .extractors import ContentExtractor from .outputformatters import OutputFormatter from .utils import (URLHelper, RawHelper, extend_config, - get_available_languages) + get_available_languages, extract_meta_refresh) from .videos.extractors import VideoExtractor log = logging.getLogger(__name__) +class ArticleDownloadState(object): + NOT_STARTED = 0 + FAILED_RESPONSE = 1 + SUCCESS = 2 + + class ArticleException(Exception): pass @@ -37,13 +46,22 @@ def __init__(self, url, title='', source_url='', config=None, **kwargs): """The **kwargs argument may be filled with config values, which is added into the config object """ + if isinstance(title, Configuration) or \ + isinstance(source_url, Configuration): + raise ArticleException( + 'Configuration object being passed incorrectly as title or ' + 'source_url! Please verify `Article`s __init__() fn.') + self.config = config or Configuration() self.config = extend_config(self.config, kwargs) self.extractor = ContentExtractor(self.config) if source_url == '': - source_url = urls.get_scheme(url) + '://' + urls.get_domain(url) + scheme = urls.get_scheme(url) + if scheme is None: + scheme = 'http' + source_url = scheme + '://' + urls.get_domain(url) if source_url is None or source_url == '': raise ArticleException('input url bad format') @@ -82,8 +100,7 @@ def __init__(self, url, title='', source_url='', config=None, **kwargs): # List of authors who have published the article, via parse() self.authors = [] - # TODO: Date of when this article was published - self.published_date = '' + self.publish_date = '' # Summary generated from the article's body txt self.summary = '' @@ -94,10 +111,10 @@ def __init__(self, url, title='', source_url='', config=None, **kwargs): # The HTML of this article's main node (most important part) self.article_html = '' - # Flags warning users in-case they forget to download() or parse() - # or if they call methods out of order + # Keep state for downloads and parsing self.is_parsed = False - self.is_downloaded = False + self.download_state = ArticleDownloadState.NOT_STARTED + self.download_exception_msg = None # Meta description field in the HTML source self.meta_description = "" @@ -108,6 +125,9 @@ def __init__(self, url, title='', source_url='', config=None, **kwargs): # Meta favicon field in HTML source self.meta_favicon = "" + # Meta site_name field in HTML source + self.meta_site_name = "" + # Meta tags contain a lot of structured data, e.g. OpenGraph self.meta_data = {} @@ -142,19 +162,55 @@ def build(self): self.parse() self.nlp() - def download(self, html=None): + def _parse_scheme_file(self, path): + try: + with open(path, "r") as fin: + return fin.read() + except OSError as e: + self.download_state = ArticleDownloadState.FAILED_RESPONSE + self.download_exception_msg = e.strerror + return None + + def _parse_scheme_http(self): + try: + return network.get_html_2XX_only(self.url, self.config) + except requests.exceptions.RequestException as e: + self.download_state = ArticleDownloadState.FAILED_RESPONSE + self.download_exception_msg = str(e) + return None + + def download(self, input_html=None, title=None, recursion_counter=0): """Downloads the link's HTML content, don't use if you are batch async downloading articles + + recursion_counter (currently 1) stops refreshes that are potentially + infinite """ - if html is None: - html = network.get_html(self.url, self.config) + if input_html is None: + parsed_url = urlparse(self.url) + if parsed_url.scheme == "file": + html = self._parse_scheme_file(parsed_url.path) + else: + html = self._parse_scheme_http() + if html is None: + log.debug('Download failed on URL %s because of %s' % + (self.url, self.download_exception_msg)) + return + else: + html = input_html + + if self.config.follow_meta_refresh: + meta_refresh_url = extract_meta_refresh(html) + if meta_refresh_url and recursion_counter < 1: + return self.download( + input_html=network.get_html(meta_refresh_url), + recursion_counter=recursion_counter + 1) + self.set_html(html) + self.set_title(title) def parse(self): - if not self.is_downloaded: - print('You must `download()` an article before ' - 'calling `parse()` on it!') - raise ArticleException() + self.throw_if_not_downloaded_verbose() self.doc = self.config.get_parser().fromstring(self.html) self.clean_doc = copy.deepcopy(self.doc) @@ -186,6 +242,9 @@ def parse(self): meta_favicon = self.extractor.get_favicon(self.clean_doc) self.set_meta_favicon(meta_favicon) + meta_site_name = self.extractor.get_meta_site_name(self.clean_doc) + self.set_meta_site_name(meta_site_name) + meta_description = \ self.extractor.get_meta_description(self.clean_doc) self.set_meta_description(meta_description) @@ -204,12 +263,13 @@ def parse(self): meta_data = self.extractor.get_meta_data(self.clean_doc) self.set_meta_data(meta_data) - # TODO self.publish_date = ... + self.publish_date = self.extractor.get_publishing_date( + self.url, + self.clean_doc) # Before any computations on the body, clean DOM object self.doc = document_cleaner.clean(self.doc) - text = '' self.top_node = self.extractor.calculate_best_node(self.doc) if self.top_node is not None: video_extractor = VideoExtractor(self.config, self.top_node) @@ -223,8 +283,7 @@ def parse(self): self.set_article_html(article_html) self.set_text(text) - if self.config.fetch_images: - self.fetch_images() + self.fetch_images() self.is_parsed = True self.release_resources() @@ -243,9 +302,12 @@ def fetch_images(self): if self.clean_top_node is not None and not self.has_top_image(): first_img = self.extractor.get_first_img_url( self.url, self.clean_top_node) - self.set_top_img(first_img) + if self.config.fetch_images: + self.set_top_img(first_img) + else: + self.set_top_img_no_check(first_img) - if not self.has_top_image(): + if not self.has_top_image() and self.config.fetch_images: self.set_reddit_top_img() def has_top_image(self): @@ -268,7 +330,8 @@ def is_valid_body(self): wordcount = self.text.split(' ') sentcount = self.text.split('.') - if meta_type == 'article' and wordcount > (self.config.MIN_WORD_COUNT): + if (meta_type == 'article' and len(wordcount) > + (self.config.MIN_WORD_COUNT)): log.debug('%s verified for article and wc' % self.url) return True @@ -309,17 +372,18 @@ def is_media_news(self): def nlp(self): """Keyword extraction wrapper """ - if not self.is_downloaded or not self.is_parsed: - print('You must `download()` and `parse()` an article ' - 'before calling `nlp()` on it!') - raise ArticleException() + self.throw_if_not_downloaded_verbose() + self.throw_if_not_parsed_verbose() + nlp.load_stopwords(self.config.get_language()) text_keyws = list(nlp.keywords(self.text).keys()) title_keyws = list(nlp.keywords(self.title).keys()) keyws = list(set(title_keyws + text_keyws)) self.set_keywords(keyws) - summary_sents = nlp.summarize(title=self.title, text=self.text) + max_sents = self.config.MAX_SUMMARY_SENT + + summary_sents = nlp.summarize(title=self.title, text=self.text, max_sents=max_sents) summary = '\n'.join(summary_sents) self.set_summary(summary) @@ -361,22 +425,27 @@ def release_resources(self): def set_reddit_top_img(self): """Wrapper for setting images. Queries known image attributes - first, then uses Reddit's imgage algorithm as a fallback. + first, then uses Reddit's image algorithm as a fallback. """ try: s = images.Scraper(self) self.set_top_img(s.largest_image_url()) + except TypeError as e: + if "Can't convert 'NoneType' object to str implicitly" in e.args[0]: + log.debug('No pictures found. Top image not set, %s' % e) + elif 'timed out' in e.args[0]: + log.debug('Download of picture timed out. Top image not set, %s' % e) + else: + log.critical('TypeError other than None type error. ' + 'Cannot set top image using the Reddit ' + 'algorithm. Possible error with PIL., %s' % e) except Exception as e: - log.critical('jpeg error with PIL, %s' % e) + log.critical('Other error with setting top image using the ' + 'Reddit algorithm. Possible error with PIL, %s' % e) - def set_title(self, title): - if self.title and not title: - # Title has already been set by an educated guess and - # extraction failed - return - title = title[:self.config.MAX_TITLE] - if title: - self.title = title + def set_title(self, input_title): + if input_title: + self.title = input_title[:self.config.MAX_TITLE] def set_text(self, text): text = text[:self.config.MAX_TEXT] @@ -390,7 +459,7 @@ def set_html(self, html): if isinstance(html, bytes): html = self.config.get_parser().get_unicode_html(html) self.html = html - self.is_downloaded = True + self.download_state = ArticleDownloadState.SUCCESS def set_article_html(self, article_html): """Sets the HTML of just the article's `top_node` @@ -459,6 +528,9 @@ def set_meta_keywords(self, meta_keywords): def set_meta_favicon(self, meta_favicon): self.meta_favicon = meta_favicon + def set_meta_site_name(self, meta_site_name): + self.meta_site_name = meta_site_name + def set_meta_description(self, meta_description): self.meta_description = meta_description @@ -476,3 +548,20 @@ def set_movies(self, movie_objects): """ movie_urls = [o.src for o in movie_objects if o and o.src] self.movies = movie_urls + + def throw_if_not_downloaded_verbose(self): + """Parse ArticleDownloadState -> log readable status + -> maybe throw ArticleException + """ + if self.download_state == ArticleDownloadState.NOT_STARTED: + raise ArticleException('You must `download()` an article first!') + elif self.download_state == ArticleDownloadState.FAILED_RESPONSE: + raise ArticleException('Article `download()` failed with %s on URL %s' % + (self.download_exception_msg, self.url)) + + def throw_if_not_parsed_verbose(self): + """Parse `is_parsed` status -> log readable status + -> maybe throw ArticleException + """ + if not self.is_parsed: + raise ArticleException('You must `parse()` an article first!') diff --git a/newspaper/cleaners.py b/newspaper/cleaners.py index 5de87b06..47b6f1a8 100644 --- a/newspaper/cleaners.py +++ b/newspaper/cleaners.py @@ -3,6 +3,7 @@ Holds the code for cleaning out unwanted tags from the lxml dom xpath. """ +import copy from .utils import ReplaceSequence @@ -40,12 +41,13 @@ def __init__(self, config): self.google_re = " google " self.entries_re = "^[^entry-]more.*$" self.facebook_re = "[^-]facebook" - self.facebook_braodcasting_re = "facebook-broadcasting" + self.facebook_broadcasting_re = "facebook-broadcasting" self.twitter_re = "[^-]twitter" self.tablines_replacements = ReplaceSequence()\ .create("\n", "\n\n")\ .append("\t")\ .append("^\\s+$") + self.contains_article = './/article|.//*[@id="article"]|.//*[@itemprop="articleBody"]' def clean(self, doc_to_clean): """Remove chunks of the DOM as specified @@ -61,11 +63,12 @@ def clean(self, doc_to_clean): doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.entries_re) doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.facebook_re) doc_to_clean = self.remove_nodes_regex(doc_to_clean, - self.facebook_braodcasting_re) + self.facebook_broadcasting_re) doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.twitter_re) doc_to_clean = self.clean_para_spans(doc_to_clean) doc_to_clean = self.div_to_para(doc_to_clean, 'div') doc_to_clean = self.div_to_para(doc_to_clean, 'span') + doc_to_clean = self.div_to_para(doc_to_clean, 'section') return doc_to_clean def clean_body_classes(self, doc): @@ -119,15 +122,18 @@ def clean_bad_tags(self, doc): # ids naughty_list = self.parser.xpath_re(doc, self.nauthy_ids_re) for node in naughty_list: - self.parser.remove(node) + if not node.xpath(self.contains_article): + self.parser.remove(node) # class naughty_classes = self.parser.xpath_re(doc, self.nauthy_classes_re) for node in naughty_classes: - self.parser.remove(node) + if not node.xpath(self.contains_article): + self.parser.remove(node) # name naughty_names = self.parser.xpath_re(doc, self.nauthy_names_re) for node in naughty_names: - self.parser.remove(node) + if not node.xpath(self.contains_article): + self.parser.remove(node) return doc def remove_nodes_regex(self, doc, pattern): @@ -224,9 +230,13 @@ def div_to_para(self, doc, dom_type): self.replace_with_para(doc, div) bad_divs += 1 elif div is not None: - replaceNodes = self.get_replacement_nodes(doc, div) + replace_nodes = self.get_replacement_nodes(doc, div) + replace_nodes = [n for n in replace_nodes if n is not None] + attrib = copy.deepcopy(div.attrib) div.clear() - for c, n in enumerate(replaceNodes): - div.insert(c, n) + for i, node in enumerate(replace_nodes): + div.insert(i, node) + for name, value in attrib.items(): + div.set(name, value) else_divs += 1 return doc diff --git a/newspaper/configuration.py b/newspaper/configuration.py index c0960a6b..94688e70 100644 --- a/newspaper/configuration.py +++ b/newspaper/configuration.py @@ -14,26 +14,26 @@ from .parsers import Parser from .text import (StopWords, StopWordsArabic, StopWordsChinese, - StopWordsKorean) + StopWordsKorean, StopWordsHindi, StopWordsJapanese, StopWordsThai) from .version import __version__ log = logging.getLogger(__name__) class Configuration(object): - def __init__(self): """ Modify any of these Article / Source properties - TODO: Have a seperate ArticleConfig and SourceConfig extend this! + TODO: Have a separate ArticleConfig and SourceConfig extend this! """ self.MIN_WORD_COUNT = 300 # num of word tokens in text - self.MIN_SENT_COUNT = 7 # num of sentence tokens - self.MAX_TITLE = 200 # num of chars - self.MAX_TEXT = 100000 # num of chars - self.MAX_KEYWORDS = 35 # num of strings in list - self.MAX_AUTHORS = 10 # num strings in list - self.MAX_SUMMARY = 5000 # num of chars + self.MIN_SENT_COUNT = 7 # num of sentence tokens + self.MAX_TITLE = 200 # num of chars + self.MAX_TEXT = 100000 # num of chars + self.MAX_KEYWORDS = 35 # num of strings in list + self.MAX_AUTHORS = 10 # num strings in list + self.MAX_SUMMARY = 5000 # num of chars + self.MAX_SUMMARY_SENT = 5 # num of sentences # max number of urls we cache for each news source self.MAX_FILE_MEMO = 20000 @@ -43,7 +43,10 @@ def __init__(self): # Set this to false if you don't care about getting images self.fetch_images = True - self.image_dimension_ration = 16/9.0 + self.image_dimension_ration = 16 / 9.0 + + # Follow meta refresh redirect when downloading + self.follow_meta_refresh = False # Don't toggle this variable, done internally self.use_meta_language = True @@ -51,6 +54,9 @@ def __init__(self): # You may keep the html of just the main article body self.keep_article_html = False + # Fail for error responses (e.g. 404 page) + self.http_success_only = True + # English is the fallback self._language = 'en' @@ -58,11 +64,15 @@ def __init__(self): self.stopwords_class = StopWords self.browser_user_agent = 'newspaper/%s' % __version__ + self.headers = {} self.request_timeout = 7 + self.proxies = {} self.number_threads = 10 self.verbose = False # for debugging + self.thread_timeout_seconds = 1 + self.ignored_content_types_defaults = {} # Set this to False if you want to recompute the categories # *every* time you build a `Source` object # TODO: Actually make this work @@ -76,13 +86,13 @@ def del_language(self): def set_language(self, language): """Language setting must be set in this method b/c non-occidental - (western) langauges require a seperate stopwords class. + (western) languages require a separate stopwords class. """ if not language or len(language) != 2: - raise Exception("Your input language must be a 2 char langauge code, \ + raise Exception("Your input language must be a 2 char language code, \ for example: english-->en \n and german-->de") - # If explicitly set langauge, don't use meta + # If explicitly set language, don't use meta self.use_meta_language = False # Set oriental language stopword class @@ -90,18 +100,28 @@ def set_language(self, language): self.stopwords_class = self.get_stopwords_class(language) language = property(get_language, set_language, - del_language, "langauge prop") + del_language, "language prop") - def get_stopwords_class(self, language): + @staticmethod + def get_stopwords_class(language): if language == 'ko': return StopWordsKorean + elif language == 'hi': + return StopWordsHindi elif language == 'zh': return StopWordsChinese - elif language == 'ar': + # Persian and Arabic Share an alphabet + # There is a persian parser https://github.com/sobhe/hazm, but nltk is likely sufficient + elif language == 'ar' or language == 'fa': return StopWordsArabic + elif language == 'ja': + return StopWordsJapanese + elif language == 'th': + return StopWordsThai return StopWords - def get_parser(self): + @staticmethod + def get_parser(): return Parser diff --git a/newspaper/extractors.py b/newspaper/extractors.py index b9e72316..96255401 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -11,29 +11,30 @@ __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' -from collections import defaultdict import copy import logging import re -import urllib.parse +import re +from collections import defaultdict +from dateutil.parser import parse as date_parser from tldextract import tldextract +from urllib.parse import urljoin, urlparse, urlunparse from . import urls - -from .utils import ReplaceSequence, StringReplacement, StringSplitter +from .utils import StringReplacement, StringSplitter log = logging.getLogger(__name__) MOTLEY_REPLACEMENT = StringReplacement("�", "") ESCAPED_FRAGMENT_REPLACEMENT = StringReplacement( "#!", "?_escaped_fragment_=") -TITLE_REPLACEMENTS = ReplaceSequence().create("»").append("»") +TITLE_REPLACEMENTS = StringReplacement("»", "»") PIPE_SPLITTER = StringSplitter("\\|") DASH_SPLITTER = StringSplitter(" - ") UNDERSCORE_SPLITTER = StringSplitter("_") SLASH_SPLITTER = StringSplitter("/") -ARROWS_SPLITTER = StringSplitter("»") +ARROWS_SPLITTER = StringSplitter(" » ") COLON_SPLITTER = StringSplitter(":") SPACE_SPLITTER = StringSplitter(' ') NO_STRINGS = set() @@ -52,7 +53,6 @@ class ContentExtractor(object): - def __init__(self, config): self.config = config self.parser = self.config.get_parser() @@ -60,10 +60,10 @@ def __init__(self, config): self.stopwords_class = config.stopwords_class def update_language(self, meta_lang): - '''Required to be called before the extraction process in some + """Required to be called before the extraction process in some cases because the stopwords_class has to set incase the lang is not latin based - ''' + """ if meta_lang: self.language = meta_lang self.stopwords_class = \ @@ -78,11 +78,24 @@ def get_authors(self, doc): def contains_digits(d): return bool(_digits.search(d)) + def uniqify_list(lst): + """Remove duplicates from provided list but maintain original order. + Derived from http://www.peterbe.com/plog/uniqifiers-benchmark + """ + seen = {} + result = [] + for item in lst: + if item.lower() in seen: + continue + seen[item.lower()] = 1 + result.append(item.title()) + return result + def parse_byline(search_str): - """Takes a candidate line of html or text and - extracts out the name(s) in list form - >>> search_str('<div>By: <strong>Lucas Ou-Yang</strong>, \ - <strong>Alex Smith</strong></div>') + """ + Takes a candidate line of html or text and + extracts out the name(s) in list form: + >>> parse_byline('<div>By: <strong>Lucas Ou-Yang</strong>,<strong>Alex Smith</strong></div>') ['Lucas Ou-Yang', 'Alex Smith'] """ # Remove HTML boilerplate @@ -94,21 +107,19 @@ def parse_byline(search_str): search_str = search_str.strip() # Chunk the line by non alphanumeric tokens (few name exceptions) - # >>> re.split("[^\w\'\-]", "Lucas Ou, Dean O'Brian and Ronald") - # ['Lucas Ou', '', 'Dean O'Brian', 'and', 'Ronald'] - name_tokens = re.split("[^\w\'\-]", search_str) + # >>> re.split("[^\w\'\-\.]", "Tyler G. Jones, Lucas Ou, Dean O'Brian and Ronald") + # ['Tyler', 'G.', 'Jones', '', 'Lucas', 'Ou', '', 'Dean', "O'Brian", 'and', 'Ronald'] + name_tokens = re.split("[^\w\'\-\.]", search_str) name_tokens = [s.strip() for s in name_tokens] _authors = [] # List of first, last name tokens curname = [] - DELIM = ['and', ''] + delimiters = ['and', ',', ''] for token in name_tokens: - if token in DELIM: - # should we allow middle names? - valid_name = (len(curname) == 2) - if valid_name: + if token in delimiters: + if len(curname) > 0: _authors.append(' '.join(curname)) curname = [] @@ -125,9 +136,9 @@ def parse_byline(search_str): # Try 1: Search popular author tags for authors ATTRS = ['name', 'rel', 'itemprop', 'class', 'id'] - VALS = ['author', 'byline'] + VALS = ['author', 'byline', 'dc.creator', 'byl'] matches = [] - _authors, authors = [], [] + authors = [] for attr in ATTRS: for val in VALS: @@ -142,15 +153,11 @@ def parse_byline(search_str): if len(mm) > 0: content = mm[0] else: - content = match.text or '' + content = match.text_content() or '' if len(content) > 0: - _authors.extend(parse_byline(content)) + authors.extend(parse_byline(content)) - uniq = list(set([s.lower() for s in _authors])) - for name in uniq: - names = [w.capitalize() for w in name.split(' ')] - authors.append(' '.join(names)) - return authors or [] + return uniqify_list(authors) # TODO Method 2: Search raw html for a by-line # match = re.search('By[\: ].*\\n|From[\: ].*\\n', html) @@ -162,8 +169,87 @@ def parse_byline(search_str): # return [] # Failed to find anything # return authors + def get_publishing_date(self, url, doc): + """3 strategies for publishing date extraction. The strategies + are descending in accuracy and the next strategy is only + attempted if a preferred one fails. + + 1. Pubdate from URL + 2. Pubdate from metadata + 3. Raw regex searches in the HTML + added heuristics + """ + + def parse_date_str(date_str): + if date_str: + try: + return date_parser(date_str) + except (ValueError, OverflowError, AttributeError, TypeError): + # near all parse failures are due to URL dates without a day + # specifier, e.g. /2014/04/ + return None + + date_match = re.search(urls.STRICT_DATE_REGEX, url) + if date_match: + date_str = date_match.group(0) + datetime_obj = parse_date_str(date_str) + if datetime_obj: + return datetime_obj + + PUBLISH_DATE_TAGS = [ + {'attribute': 'property', 'value': 'rnews:datePublished', + 'content': 'content'}, + {'attribute': 'property', 'value': 'article:published_time', + 'content': 'content'}, + {'attribute': 'name', 'value': 'OriginalPublicationDate', + 'content': 'content'}, + {'attribute': 'itemprop', 'value': 'datePublished', + 'content': 'datetime'}, + {'attribute': 'property', 'value': 'og:published_time', + 'content': 'content'}, + {'attribute': 'name', 'value': 'article_date_original', + 'content': 'content'}, + {'attribute': 'name', 'value': 'publication_date', + 'content': 'content'}, + {'attribute': 'name', 'value': 'sailthru.date', + 'content': 'content'}, + {'attribute': 'name', 'value': 'PublishDate', + 'content': 'content'}, + {'attribute': 'pubdate', 'value': 'pubdate', + 'content': 'datetime'}, + {'attribute': 'name', 'value': 'publish_date', + 'content': 'content'}, + ] + for known_meta_tag in PUBLISH_DATE_TAGS: + meta_tags = self.parser.getElementsByTag( + doc, + attr=known_meta_tag['attribute'], + value=known_meta_tag['value']) + if meta_tags: + date_str = self.parser.getAttribute( + meta_tags[0], + known_meta_tag['content']) + datetime_obj = parse_date_str(date_str) + if datetime_obj: + return datetime_obj + + return None + def get_title(self, doc): """Fetch the article title and analyze it + + Assumptions: + - title tag is the most reliable (inherited from Goose) + - h1, if properly detected, is the best (visible to users) + - og:title and h1 can help improve the title extraction + - python == is too strict, often we need to compare filtered + versions, i.e. lowercase and ignoring special chars + + Explicit rules: + 1. title == h1, no need to split + 2. h1 similar to og:title, use h1 + 3. title contains h1, title contains og:title, len(h1) > len(og:title), use h1 + 4. title starts with og:title, use og:title + 5. use title, after splitting """ title = '' title_element = self.parser.getElementsByTag(doc, tag='title') @@ -175,48 +261,111 @@ def get_title(self, doc): title_text = self.parser.getText(title_element[0]) used_delimeter = False + # title from h1 + # - extract the longest text from all h1 elements + # - too short texts (fewer than 2 words) are discarded + # - clean double spaces + title_text_h1 = '' + title_element_h1_list = self.parser.getElementsByTag(doc, + tag='h1') or [] + title_text_h1_list = [self.parser.getText(tag) for tag in + title_element_h1_list] + if title_text_h1_list: + # sort by len and set the longest + title_text_h1_list.sort(key=len, reverse=True) + title_text_h1 = title_text_h1_list[0] + # discard too short texts + if len(title_text_h1.split(' ')) <= 2: + title_text_h1 = '' + # clean double spaces + title_text_h1 = ' '.join([x for x in title_text_h1.split() if x]) + + # title from og:title + title_text_fb = ( + self.get_meta_content(doc, 'meta[property="og:title"]') or + self.get_meta_content(doc, 'meta[name="og:title"]') or '') + + # create filtered versions of title_text, title_text_h1, title_text_fb + # for finer comparison + filter_regex = re.compile(r'[^\u4e00-\u9fa5a-zA-Z0-9\ ]') + filter_title_text = filter_regex.sub('', title_text).lower() + filter_title_text_h1 = filter_regex.sub('', title_text_h1).lower() + filter_title_text_fb = filter_regex.sub('', title_text_fb).lower() + + # check for better alternatives for title_text and possibly skip splitting + if title_text_h1 == title_text: + used_delimeter = True + elif filter_title_text_h1 and filter_title_text_h1 == filter_title_text_fb: + title_text = title_text_h1 + used_delimeter = True + elif filter_title_text_h1 and filter_title_text_h1 in filter_title_text \ + and filter_title_text_fb and filter_title_text_fb in filter_title_text \ + and len(title_text_h1) > len(title_text_fb): + title_text = title_text_h1 + used_delimeter = True + elif filter_title_text_fb and filter_title_text_fb != filter_title_text \ + and filter_title_text.startswith(filter_title_text_fb): + title_text = title_text_fb + used_delimeter = True + # split title with | - if '|' in title_text: - title_text = self.split_title(title_text, PIPE_SPLITTER) + if not used_delimeter and '|' in title_text: + title_text = self.split_title(title_text, PIPE_SPLITTER, + title_text_h1) used_delimeter = True # split title with - if not used_delimeter and '-' in title_text: - title_text = self.split_title(title_text, DASH_SPLITTER) + title_text = self.split_title(title_text, DASH_SPLITTER, + title_text_h1) used_delimeter = True # split title with _ if not used_delimeter and '_' in title_text: - title_text = self.split_title(title_text, UNDERSCORE_SPLITTER) + title_text = self.split_title(title_text, UNDERSCORE_SPLITTER, + title_text_h1) + used_delimeter = True # split title with / if not used_delimeter and '/' in title_text: - title_text = self.split_title(title_text, SLASH_SPLITTER) + title_text = self.split_title(title_text, SLASH_SPLITTER, + title_text_h1) used_delimeter = True # split title with » - if not used_delimeter and '»' in title_text: - title_text = self.split_title(title_text, ARROWS_SPLITTER) - used_delimeter = True - - # split title with : - if not used_delimeter and ':' in title_text: - title_text = self.split_title(title_text, COLON_SPLITTER) + if not used_delimeter and ' » ' in title_text: + title_text = self.split_title(title_text, ARROWS_SPLITTER, + title_text_h1) used_delimeter = True title = MOTLEY_REPLACEMENT.replaceAll(title_text) + + # in some cases the final title is quite similar to title_text_h1 + # (either it differs for case, for special chars, or it's truncated) + # in these cases, we prefer the title_text_h1 + filter_title = filter_regex.sub('', title).lower() + if filter_title_text_h1 == filter_title: + title = title_text_h1 + return title - def split_title(self, title, splitter): + def split_title(self, title, splitter, hint=None): """Split the title to best part possible """ large_text_length = 0 large_text_index = 0 title_pieces = splitter.split(title) + if hint: + filter_regex = re.compile(r'[^a-zA-Z0-9\ ]') + hint = filter_regex.sub('', hint).lower() + # find the largest title piece - for i in range(len(title_pieces)): - current = title_pieces[i] + for i, title_piece in enumerate(title_pieces): + current = title_piece.strip() + if hint and hint in filter_regex.sub('', current).lower(): + large_text_index = i + break if len(current) > large_text_length: large_text_length = len(current) large_text_index = i @@ -264,7 +413,7 @@ def get_meta_lang(self, doc): # look up for a Content-Language in meta items = [ {'tag': 'meta', 'attr': 'http-equiv', - 'value': 'content-language'}, + 'value': 'content-language'}, {'tag': 'meta', 'attr': 'name', 'value': 'lang'} ] for item in items: @@ -280,14 +429,14 @@ def get_meta_lang(self, doc): return None - def get_meta_content(self, doc, metaName): + def get_meta_content(self, doc, metaname): """Extract a given meta content form document. Example metaNames: "meta[name=description]" "meta[name=keywords]" "meta[property=og:type]" """ - meta = self.parser.css_select(doc, metaName) + meta = self.parser.css_select(doc, metaname) content = None if meta is not None and len(meta) > 0: content = self.parser.getAttribute(meta[0], 'content') @@ -300,29 +449,36 @@ def get_meta_img_url(self, article_url, doc): """ top_meta_image, try_one, try_two, try_three, try_four = [None] * 5 try_one = self.get_meta_content(doc, 'meta[property="og:image"]') - if try_one is None: - link_icon_kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'icon'} - elems = self.parser.getElementsByTag(doc, **link_icon_kwargs) + if not try_one: + link_img_src_kwargs = \ + {'tag': 'link', 'attr': 'rel', 'value': 'img_src|image_src'} + elems = self.parser.getElementsByTag(doc, use_regex=True, **link_img_src_kwargs) try_two = elems[0].get('href') if elems else None - if try_two is None: - link_img_src_kwargs = \ - {'tag': 'link', 'attr': 'rel', 'value': 'img_src'} - elems = self.parser.getElementsByTag(doc, **link_img_src_kwargs) - try_three = elems[0].get('href') if elems else None + if not try_two: + try_three = self.get_meta_content(doc, 'meta[name="og:image"]') - if try_three is None: - try_four = self.get_meta_content(doc, 'meta[name="og:image"]') + if not try_three: + link_icon_kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'icon'} + elems = self.parser.getElementsByTag(doc, **link_icon_kwargs) + try_four = elems[0].get('href') if elems else None top_meta_image = try_one or try_two or try_three or try_four - return urllib.parse.urljoin(article_url, top_meta_image) + if top_meta_image: + return urljoin(article_url, top_meta_image) + return '' def get_meta_type(self, doc): """Returns meta type of article, open graph protocol """ return self.get_meta_content(doc, 'meta[property="og:type"]') + def get_meta_site_name(self, doc): + """Returns site name of article, open graph protocol + """ + return self.get_meta_content(doc, 'meta[property="og:site_name"]') + def get_meta_description(self, doc): """If the article has meta description set in the source, use that """ @@ -355,7 +511,7 @@ def get_meta_data(self, doc): key_head = key.pop(0) ref = data[key_head] - if isinstance(ref, str): + if isinstance(ref, str) or isinstance(ref, int): data[key_head] = {key_head: ref} ref = data[key_head] @@ -365,7 +521,7 @@ def get_meta_data(self, doc): break if not ref.get(part): ref[part] = dict() - elif isinstance(ref.get(part), str): + elif isinstance(ref.get(part), str) or isinstance(ref.get(part), int): # Not clear what to do in this scenario, # it's not always a URL, but an ID of some sort ref[part] = {'identifier': ref[part]} @@ -373,21 +529,42 @@ def get_meta_data(self, doc): return data def get_canonical_link(self, article_url, doc): - """If the article has meta canonical link set in the url """ - kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'canonical'} - meta = self.parser.getElementsByTag(doc, **kwargs) - if meta is not None and len(meta) > 0: - href = self.parser.getAttribute(meta[0], 'href') - if href: - href = href.strip() - o = urllib.parse.urlparse(href) - if not o.hostname: - z = urllib.parse.urlparse(article_url) - domain = '%s://%s' % (z.scheme, z.hostname) - href = urllib.parse.urljoin(domain, href) - return href - return '' + Return the article's canonical URL + + Gets the first available value of: + 1. The rel=canonical tag + 2. The og:url tag + """ + links = self.parser.getElementsByTag(doc, tag='link', attr='rel', + value='canonical') + + canonical = self.parser.getAttribute(links[0], 'href') if links else '' + og_url = self.get_meta_content(doc, 'meta[property="og:url"]') + meta_url = canonical or og_url or '' + if meta_url: + meta_url = meta_url.strip() + parsed_meta_url = urlparse(meta_url) + if not parsed_meta_url.hostname: + # MIGHT not have a hostname in meta_url + # parsed_url.path might be 'example.com/article.html' where + # clearly example.com is the hostname + parsed_article_url = urlparse(article_url) + strip_hostname_in_meta_path = re. \ + match(".*{}(?=/)/(.*)". + format(parsed_article_url.hostname), + parsed_meta_url.path) + try: + true_path = strip_hostname_in_meta_path.group(1) + except AttributeError: + true_path = parsed_meta_url.path + + # true_path may contain querystrings and fragments + meta_url = urlunparse((parsed_article_url.scheme, + parsed_article_url.hostname, true_path, + '', '', '')) + + return meta_url def get_img_urls(self, article_url, doc): """Return all of the images on an html page, lxml root @@ -396,8 +573,8 @@ def get_img_urls(self, article_url, doc): img_tags = self.parser.getElementsByTag(doc, **img_kwargs) urls = [img_tag.get('src') for img_tag in img_tags if img_tag.get('src')] - img_links = set([urllib.parse.urljoin(article_url, url) - for url in urls]) + img_links = set([urljoin(article_url, url) + for url in urls]) return img_links def get_first_img_url(self, article_url, top_node): @@ -408,7 +585,7 @@ def get_first_img_url(self, article_url, top_node): node_images = self.get_img_urls(article_url, top_node) node_images = list(node_images) if node_images: - return urllib.parse.urljoin(article_url, node_images[0]) + return urljoin(article_url, node_images[0]) return '' def _get_urls(self, doc, titles): @@ -421,7 +598,7 @@ def _get_urls(self, doc, titles): a_kwargs = {'tag': 'a'} a_tags = self.parser.getElementsByTag(doc, **a_kwargs) - # TODO: this should be refactored! We should have a seperate + # TODO: this should be refactored! We should have a separate # method which siphones the titles our of a list of <a> tags. if titles: return [(a.get('href'), a.text) for a in a_tags if a.get('href')] @@ -504,7 +681,7 @@ def get_category_urls(self, source_url, doc): 'subdomain' % p_url)) continue else: - valid_categories.append(scheme+'://'+domain) + valid_categories.append(scheme + '://' + domain) # TODO account for case where category is in form # http://subdomain.domain.tld/category/ <-- still legal! else: @@ -515,7 +692,7 @@ def get_category_urls(self, source_url, doc): path_chunks.remove('index.html') if len(path_chunks) == 1 and len(path_chunks[0]) < 14: - valid_categories.append(domain+path) + valid_categories.append(domain + path) else: if self.config.verbose: print(('elim category url %s for >1 path chunks ' @@ -604,7 +781,7 @@ def calculate_best_node(self, doc): for node in nodes_to_check: text_node = self.parser.getText(node) - word_stats = self.stopwords_class(language=self.language).\ + word_stats = self.stopwords_class(language=self.language). \ get_stopword_count(text_node) high_link_density = self.is_highlink_density(node) if word_stats.get_stopword_count() > 2 and not high_link_density: @@ -617,7 +794,7 @@ def calculate_best_node(self, doc): for node in nodes_with_text: boost_score = float(0) # boost - if(self.is_boostable(node)): + if self.is_boostable(node): if cnt >= 0: boost_score = float((1.0 / starting_boost) * 50) starting_boost += 1 @@ -632,7 +809,7 @@ def calculate_best_node(self, doc): boost_score = float(5) text_node = self.parser.getText(node) - word_stats = self.stopwords_class(language=self.language).\ + word_stats = self.stopwords_class(language=self.language). \ get_stopword_count(text_node) upscore = int(word_stats.get_stopword_count() + boost_score) @@ -666,7 +843,7 @@ def calculate_best_node(self, doc): return top_node def is_boostable(self, node): - """Alot of times the first paragraph might be the caption under an image + """A lot of times the first paragraph might be the caption under an image so we'll want to make sure if we're going to boost a parent node that it should be connected to other paragraphs, at least for the first n paragraphs so we'll want to make sure that the next sibling is a @@ -684,38 +861,33 @@ def is_boostable(self, node): if current_node_tag == para: if steps_away >= max_stepsaway_from_node: return False - paraText = self.parser.getText(current_node) - word_stats = self.stopwords_class(language=self.language).\ - get_stopword_count(paraText) + paragraph_text = self.parser.getText(current_node) + word_stats = self.stopwords_class(language=self.language). \ + get_stopword_count(paragraph_text) if word_stats.get_stopword_count() > minimum_stopword_count: return True steps_away += 1 return False def walk_siblings(self, node): - current_sibling = self.parser.previousSibling(node) - b = [] - while current_sibling is not None: - b.append(current_sibling) - current_sibling = self.parser.previousSibling(current_sibling) - return b + return self.parser.previousSiblings(node) def add_siblings(self, top_node): - baselinescore_siblings_para = self.get_siblings_score(top_node) + baseline_score_siblings_para = self.get_siblings_score(top_node) results = self.walk_siblings(top_node) for current_node in results: ps = self.get_siblings_content( - current_node, baselinescore_siblings_para) + current_node, baseline_score_siblings_para) for p in ps: top_node.insert(0, p) return top_node def get_siblings_content( - self, current_sibling, baselinescore_siblings_para): + self, current_sibling, baseline_score_siblings_para): """Adds any siblings that may have a decent score to this node """ if current_sibling.tag == 'p' and \ - len(self.parser.getText(current_sibling)) > 0: + len(self.parser.getText(current_sibling)) > 0: e0 = current_sibling if e0.tail: e0 = copy.deepcopy(e0) @@ -731,13 +903,14 @@ def get_siblings_content( for first_paragraph in potential_paragraphs: text = self.parser.getText(first_paragraph) if len(text) > 0: - word_stats = self.stopwords_class(language=self.language).\ + word_stats = self.stopwords_class( + language=self.language). \ get_stopword_count(text) paragraph_score = word_stats.get_stopword_count() sibling_baseline_score = float(.30) high_link_density = self.is_highlink_density( first_paragraph) - score = float(baselinescore_siblings_para * + score = float(baseline_score_siblings_para * sibling_baseline_score) if score < paragraph_score and not high_link_density: p = self.parser.createElement( @@ -761,7 +934,7 @@ def get_siblings_score(self, top_node): for node in nodes_to_check: text_node = self.parser.getText(node) - word_stats = self.stopwords_class(language=self.language).\ + word_stats = self.stopwords_class(language=self.language). \ get_stopword_count(text_node) high_link_density = self.is_highlink_density(node) if word_stats.get_stopword_count() > 2 and not high_link_density: @@ -773,7 +946,7 @@ def get_siblings_score(self, top_node): return base - def update_score(self, node, addToScore): + def update_score(self, node, add_to_score): """Adds a score to the gravityScore Attribute we put on divs we'll get the current score then add the score we're passing in to the current. @@ -783,7 +956,7 @@ def update_score(self, node, addToScore): if score_string: current_score = float(score_string) - new_score = current_score + addToScore + new_score = current_score + add_to_score self.parser.setAttribute(node, "gravityScore", str(new_score)) def update_node_count(self, node, add_to_count): @@ -802,22 +975,24 @@ def is_highlink_density(self, e): link to text ratio, then the text is less likely to be relevant """ links = self.parser.getElementsByTag(e, tag='a') - if links is None or len(links) == 0: + if not links: return False text = self.parser.getText(e) - words = text.split(' ') + words = [word for word in text.split() if word.isalnum()] + if not words: + return True words_number = float(len(words)) sb = [] for link in links: sb.append(self.parser.getText(link)) - linkText = ''.join(sb) - linkWords = linkText.split(' ') - numberOfLinkWords = float(len(linkWords)) - numberOfLinks = float(len(links)) - linkDivisor = float(numberOfLinkWords / words_number) - score = float(linkDivisor * numberOfLinks) + link_text = ''.join(sb) + link_words = link_text.split() + num_link_words = float(len(link_words)) + num_links = float(len(links)) + link_divisor = float(num_link_words / words_number) + score = float(link_divisor * num_links) if score >= 1.0: return True return False @@ -829,10 +1004,10 @@ def get_score(self, node): return self.get_node_gravity_score(node) or 0 def get_node_gravity_score(self, node): - grvScoreString = self.parser.getAttribute(node, 'gravityScore') - if not grvScoreString: + gravity_score = self.parser.getAttribute(node, 'gravityScore') + if not gravity_score: return None - return float(grvScoreString) + return float(gravity_score) def nodes_to_check(self, doc): """Returns a list of nodes we want to search @@ -845,36 +1020,34 @@ def nodes_to_check(self, doc): return nodes_to_check def is_table_and_no_para_exist(self, e): - subParagraphs = self.parser.getElementsByTag(e, tag='p') - for p in subParagraphs: + sub_paragraphs = self.parser.getElementsByTag(e, tag='p') + for p in sub_paragraphs: txt = self.parser.getText(p) if len(txt) < 25: self.parser.remove(p) - subParagraphs2 = self.parser.getElementsByTag(e, tag='p') - if len(subParagraphs2) == 0 and e.tag != "td": + sub_paragraphs_2 = self.parser.getElementsByTag(e, tag='p') + if len(sub_paragraphs_2) == 0 and e.tag != "td": return True return False def is_nodescore_threshold_met(self, node, e): top_node_score = self.get_score(node) - current_nodeScore = self.get_score(e) - thresholdScore = float(top_node_score * .08) + current_node_score = self.get_score(e) + threshold = float(top_node_score * .08) - if (current_nodeScore < thresholdScore) and e.tag != 'td': + if (current_node_score < threshold) and e.tag != 'td': return False return True def post_cleanup(self, top_node): - """Remove any divs that looks like non-content, - clusters of links, or paras with no gusto + """Remove any divs that looks like non-content, clusters of links, + or paras with no gusto; add adjacent nodes which look contenty """ node = self.add_siblings(top_node) for e in self.parser.getChildren(node): e_tag = self.parser.getTag(e) if e_tag != 'p': - if self.is_highlink_density(e) \ - or self.is_table_and_no_para_exist(e) \ - or not self.is_nodescore_threshold_met(node, e): + if self.is_highlink_density(e): self.parser.remove(e) return node diff --git a/newspaper/images.py b/newspaper/images.py index 54e5b1e1..77476625 100644 --- a/newspaper/images.py +++ b/newspaper/images.py @@ -12,11 +12,9 @@ import math import io import traceback -import urllib.request import urllib.parse -import urllib.error -from http.client import InvalidURL +import requests from PIL import Image, ImageFile from . import urls @@ -58,8 +56,8 @@ def image_entropy(img): def square_image(img): - """If the image is taller than it is wide, square it off. determine - which pieces to cut off based on the entropy pieces + """If the height of the image is greater than its width, then a square image is returned. + Pieces to cut is based on the entropy pieces. """ x, y = img.size while y > x: @@ -91,24 +89,23 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False): url = clean_url(url) if not url.startswith(('http://', 'https://')): return nothing + + response = None while True: try: - req = urllib.request.Request(url) - req.add_header('User-Agent', useragent) - if referer: - req.add_header('Referer', referer) - - opener = urllib.request.build_opener() - open_req = opener.open(req, timeout=5) + response = requests.get(url, stream=True, timeout=5, headers={ + 'User-Agent': useragent, + 'Referer': referer, + }) # if we only need the dimension of the image, we may not # need to download the entire thing if dimension: - content = open_req.read(chunk_size) + content = response.raw.read(chunk_size) else: - content = open_req.read() + content = response.raw.read() - content_type = open_req.headers.get('content-type') + content_type = response.headers.get('Content-Type') if not content_type: return nothing @@ -137,7 +134,7 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False): raise e p = None break - new_data = open_req.read(chunk_size) + new_data = response.raw.read(chunk_size) content += new_data if p is None: @@ -153,17 +150,17 @@ def fetch_url(url, useragent, referer=None, retries=1, dimension=False): return content_type, content - except (urllib.error.URLError, - urllib.error.HTTPError, - InvalidURL) as e: + except requests.exceptions.RequestException as e: cur_try += 1 if cur_try >= retries: log.debug('error while fetching: %s refer: %s' % (url, referer)) return nothing finally: - if 'open_req' in locals(): - open_req.close() + if response is not None: + response.raw.close() + if response.raw._connection: + response.raw._connection.close() def fetch_image_dimension(url, useragent, referer=None, retries=1): @@ -195,7 +192,7 @@ def largest_image_url(self): if area > max_area: max_area = area max_url = img_url - log.debug('using max img ' + max_url) + log.debug('using max img {}'.format(max_url)) return max_url def calculate_area(self, img_url, dimension): diff --git a/newspaper/mthreading.py b/newspaper/mthreading.py index 5164635a..51907bf7 100644 --- a/newspaper/mthreading.py +++ b/newspaper/mthreading.py @@ -9,27 +9,38 @@ __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' +import logging import queue import traceback + from threading import Thread +from .configuration import Configuration + +log = logging.getLogger(__name__) + + +class ConcurrencyException(Exception): + pass + class Worker(Thread): """ Thread executing tasks from a given tasks queue. """ - def __init__(self, tasks): + def __init__(self, tasks, timeout_seconds): Thread.__init__(self) self.tasks = tasks + self.timeout = timeout_seconds self.daemon = True self.start() def run(self): while True: try: - func, args, kargs = self.tasks.get() + func, args, kargs = self.tasks.get(timeout=self.timeout) except queue.Empty: - traceback.print_exc() + # Extra thread allocated, no job, exit gracefully break try: func(*args, **kargs) @@ -40,35 +51,21 @@ def run(self): class ThreadPool: - """ - Pool of threads consuming tasks from a queue. - """ - def __init__(self, num_threads): + def __init__(self, num_threads, timeout_seconds): self.tasks = queue.Queue(num_threads) for _ in range(num_threads): - Worker(self.tasks) + Worker(self.tasks, timeout_seconds) def add_task(self, func, *args, **kargs): - """ - Add a task to the queue. - """ self.tasks.put((func, args, kargs)) def wait_completion(self): - """ - Wait for completion of all the tasks in the queue. - """ self.tasks.join() - def clear_threads(self): - """ - """ - pass - class NewsPool(object): - def __init__(self): + def __init__(self, config=None): """ Abstraction of a threadpool. A newspool can accept any number of source OR article objects together in a list. It allocates one @@ -92,8 +89,8 @@ def __init__(self): >>> cnn_paper.articles[50].html u'<html>blahblah ... ' """ - self.papers = [] self.pool = None + self.config = config or Configuration() def join(self): """ @@ -101,20 +98,36 @@ def join(self): resets the task. """ if self.pool is None: - print('Call set(..) with a list of source ' - 'objects before .join(..)') - raise + raise ConcurrencyException('Call set(..) with a list of source objects ' + 'before calling .join(..)') self.pool.wait_completion() - self.papers = [] self.pool = None - def set(self, paper_list, threads_per_source=1): - """ - Sets the job batch. + def set(self, news_list, threads_per_source=1, override_threads=None): """ - self.papers = paper_list - num_threads = threads_per_source * len(self.papers) - self.pool = ThreadPool(num_threads) + news_list can be a list of `Article`, `Source`, or both. - for paper in self.papers: - self.pool.add_task(paper.download_articles) + If caller wants to decide how many threads to use, they can use + `override_threads` which takes precedence over all. Otherwise, + this api infers that if the input is all `Source` objects, to + allocate one thread per `Source` to not spam the host. + + If both of the above conditions are not true, default to 1 thread. + """ + from .source import Source + + if override_threads is not None: + num_threads = override_threads + elif all([isinstance(n, Source) for n in news_list]): + num_threads = threads_per_source * len(news_list) + else: + num_threads = 1 + + timeout = self.config.thread_timeout_seconds + self.pool = ThreadPool(num_threads, timeout) + + for news_object in news_list: + if isinstance(news_object, Source): + self.pool.add_task(news_object.download_articles) + else: + self.pool.add_task(news_object.download) diff --git a/newspaper/network.py b/newspaper/network.py index b4ee4c9e..29f0e699 100644 --- a/newspaper/network.py +++ b/newspaper/network.py @@ -18,50 +18,74 @@ log = logging.getLogger(__name__) -def get_request_kwargs(timeout, useragent): +FAIL_ENCODING = 'ISO-8859-1' + + +def get_request_kwargs(timeout, useragent, proxies, headers): """This Wrapper method exists b/c some values in req_kwargs dict are methods which need to be called every time we make a request """ return { - 'headers': {'User-Agent': useragent}, + 'headers': headers if headers else {'User-Agent': useragent}, 'cookies': cj(), 'timeout': timeout, - 'allow_redirects': True + 'allow_redirects': True, + 'proxies': proxies } def get_html(url, config=None, response=None): - """Retrieves the html for either a url or a response object. All html - extractions MUST come from this method due to some intricies in the - requests module. To get the encoding, requests only uses the HTTP header - encoding declaration requests.utils.get_encoding_from_headers() and reverts - to ISO-8859-1 if it doesn't find one. This results in incorrect character - encoding in a lot of cases. + """HTTP response code agnostic + """ + try: + return get_html_2XX_only(url, config, response) + except requests.exceptions.RequestException as e: + log.debug('get_html() error. %s on URL: %s' % (e, url)) + return '' + + +def get_html_2XX_only(url, config=None, response=None): + """Consolidated logic for http requests from newspaper. We handle error cases: + - Attempt to find encoding of the html by using HTTP header. Fallback to + 'ISO-8859-1' if not provided. + - Error out if a non 2XX HTTP response code is returned. """ - FAIL_ENCODING = 'ISO-8859-1' config = config or Configuration() useragent = config.browser_user_agent timeout = config.request_timeout + proxies = config.proxies + headers = config.headers if response is not None: - if response.encoding != FAIL_ENCODING: - return response.text - return response.content + return _get_html_from_response(response, config) - try: - html = None - response = requests.get( - url=url, **get_request_kwargs(timeout, useragent)) - if response.encoding != FAIL_ENCODING: - html = response.text - else: - html = response.content - if html is None: - html = '' - return html - except Exception as e: - log.debug('%s on %s' % (e, url)) - return '' + response = requests.get( + url=url, **get_request_kwargs(timeout, useragent, proxies, headers)) + + html = _get_html_from_response(response, config) + + if config.http_success_only: + # fail if HTTP sends a non 2XX response + response.raise_for_status() + + return html + + +def _get_html_from_response(response, config): + if response.headers.get('content-type') in config.ignored_content_types_defaults: + return config.ignored_content_types_defaults[response.headers.get('content-type')] + if response.encoding != FAIL_ENCODING: + # return response as a unicode string + html = response.text + else: + html = response.content + if 'charset' not in response.headers.get('content-type'): + encodings = requests.utils.get_encodings_from_content(response.text) + if len(encodings) > 0: + response.encoding = encodings[0] + html = response.text + + return html or '' class MRequest(object): @@ -72,17 +96,21 @@ class MRequest(object): """ def __init__(self, url, config=None): self.url = url + self.config = config config = config or Configuration() self.useragent = config.browser_user_agent self.timeout = config.request_timeout + self.proxies = config.proxies + self.headers = config.headers self.resp = None def send(self): try: self.resp = requests.get(self.url, **get_request_kwargs( - self.timeout, self.useragent)) - except Exception as e: - pass + self.timeout, self.useragent, self.proxies, self.headers)) + if self.config.http_success_only: + self.resp.raise_for_status() + except requests.exceptions.RequestException as e: log.critical('[REQUEST FAILED] ' + str(e)) @@ -92,8 +120,9 @@ def multithread_request(urls, config=None): """ config = config or Configuration() num_threads = config.number_threads + timeout = config.thread_timeout_seconds - pool = ThreadPool(num_threads) + pool = ThreadPool(num_threads, timeout) m_requests = [] for url in urls: @@ -105,28 +134,3 @@ def multithread_request(urls, config=None): pool.wait_completion() return m_requests -# def async_request(urls, timeout=7): -# """receives a list of requests and sends them all -# asynchronously at once""" -# -# rs = (grequests.request('GET', url, -# **get_request_kwargs(timeout)) for url in urls) -# responses = grequests.map(rs, size=10) -# -# return responses - - -# def sync_request(urls_or_url, config=None): -# """ -# Wrapper for a regular request, no asyn nor multithread. -# """ -# # TODO config = default_config if not config else config -# useragent = config.browser_user_agent -# timeout = config.request_timeout -# if isinstance(urls_or_url, list): -# resps = [requests.get(url, **get_request_kwargs(timeout, useragent)) -# for url in urls_or_url] -# return resps -# else: -# return requests.get(urls_or_url, -# **get_request_kwargs(timeout, useragent)) diff --git a/newspaper/nlp.py b/newspaper/nlp.py index aa859f38..23505255 100644 --- a/newspaper/nlp.py +++ b/newspaper/nlp.py @@ -9,19 +9,36 @@ import re import math +from os import path from collections import Counter from . import settings -with open(settings.NLP_STOPWORDS_EN, 'r') as f: - stopwords = set([w.strip() for w in f.readlines()]) - ideal = 20.0 +stopwords = set() -def summarize(url='', title='', text=''): - if (text == '' or title == ''): +def load_stopwords(language): + """ + Loads language-specific stopwords for keyword selection + """ + global stopwords + + # stopwords for nlp in English are not the regular stopwords + # to pass the tests + # can be changed with the tests + if language == 'en': + stopwordsFile = settings.NLP_STOPWORDS_EN + else: + stopwordsFile = path.join(settings.STOPWORDS_DIR,\ + 'stopwords-{}.txt'.format(language)) + with open(stopwordsFile, 'r', encoding='utf-8') as f: + stopwords.update(set([w.strip() for w in f.readlines()])) + + +def summarize(url='', title='', text='', max_sents=5): + if not text or not title or max_sents <= 0: return [] summaries = [] @@ -29,11 +46,12 @@ def summarize(url='', title='', text=''): keys = keywords(text) titleWords = split_words(title) - # Score setences, and use the top 5 sentences - ranks = score(sentences, titleWords, keys).most_common(5) + # Score sentences, and use the top 5 or max_sents sentences + ranks = score(sentences, titleWords, keys).most_common(max_sents) for rank in ranks: summaries.append(rank[0]) - return summaries + summaries.sort(key=lambda summary: summary[0]) + return [summary[1] for summary in summaries] def score(sentences, titleWords, keywords): @@ -45,14 +63,14 @@ def score(sentences, titleWords, keywords): sentence = split_words(s) titleFeature = title_score(titleWords, sentence) sentenceLength = length_score(len(sentence)) - sentencePosition = sentence_position(i+1, senSize) + sentencePosition = sentence_position(i + 1, senSize) sbsFeature = sbs(sentence, keywords) dbsFeature = dbs(sentence, keywords) frequency = (sbsFeature + dbsFeature) / 2.0 * 10.0 # Weighted average of scores from four categories totalScore = (titleFeature*1.5 + frequency*2.0 + sentenceLength*1.0 + sentencePosition*1.0)/4.0 - ranks[s] = totalScore + ranks[(i, s)] = totalScore return ranks @@ -63,7 +81,7 @@ def sbs(words, keywords): for word in words: if word in keywords: score += keywords[word] - return (1.0 / math.fabs(len(words)) * score)/10.0 + return (1.0 / math.fabs(len(words)) * score) / 10.0 def dbs(words, keywords): @@ -82,10 +100,10 @@ def dbs(words, keywords): second = first first = [i, score] dif = first[0] - second[0] - summ += (first[1]*second[1]) / (dif ** 2) + summ += (first[1] * second[1]) / (dif ** 2) # Number of intersections - k = len(set(keywords.keys()).intersection(set(words)))+1 - return (1/(k*(k+1.0))*summ) + k = len(set(keywords.keys()).intersection(set(words))) + 1 + return (1 / (k * (k + 1.0)) * summ) def split_words(text): @@ -125,7 +143,7 @@ def keywords(text): keywords = dict((x, y) for x, y in keywords) for k in keywords: - articleScore = keywords[k]*1.0 / max(num_words, 1) + articleScore = keywords[k] * 1.0 / max(num_words, 1) keywords[k] = articleScore * 1.5 + 1 return dict(keywords) else: diff --git a/newspaper/outputformatters.py b/newspaper/outputformatters.py index fb394ee5..47a76467 100644 --- a/newspaper/outputformatters.py +++ b/newspaper/outputformatters.py @@ -7,11 +7,15 @@ __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' -from html.parser import HTMLParser +from html import unescape +import logging from .text import innerTrim +log = logging.getLogger(__name__) + + class OutputFormatter(object): def __init__(self, config): @@ -48,30 +52,46 @@ def get_formatted(self, top_node): self.links_to_text() self.add_newline_to_br() + self.add_newline_to_li() self.replace_with_text() - self.remove_fewwords_paragraphs() - + self.remove_empty_tags() + self.remove_trailing_media_div() text = self.convert_to_text() + # print(self.parser.nodeToString(self.get_top_node())) return (text, html) def convert_to_text(self): txts = [] for node in list(self.get_top_node()): - txt = self.parser.getText(node) + try: + txt = self.parser.getText(node) + except ValueError as err: # lxml error + log.info('%s ignoring lxml node error: %s', __title__, err) + txt = None + if txt: - txt = HTMLParser().unescape(txt) + txt = unescape(txt) txt_lis = innerTrim(txt).split(r'\n') + txt_lis = [n.strip(' ') for n in txt_lis] txts.extend(txt_lis) return '\n\n'.join(txts) def convert_to_html(self): cleaned_node = self.parser.clean_article_html(self.get_top_node()) - return self.parser.node_to_string(cleaned_node) + return self.parser.nodeToString(cleaned_node) def add_newline_to_br(self): for e in self.parser.getElementsByTag(self.top_node, tag='br'): e.text = r'\n' + def add_newline_to_li(self): + for e in self.parser.getElementsByTag(self.top_node, tag='ul'): + li_list = self.parser.getElementsByTag(e, tag='li') + for li in li_list[:-1]: + li.text = self.parser.getText(li) + r'\n' + for c in self.parser.getChildren(li): + self.parser.remove(c) + def links_to_text(self): """Cleans up and converts any nodes that should be considered text into text. @@ -86,7 +106,7 @@ def remove_negativescores_nodes(self): self.top_node, "*[gravityScore]") for item in gravity_items: score = self.parser.getAttribute(item, 'gravityScore') - score = int(score, 0) + score = float(score) if score else 0 if score < 1: item.getparent().remove(item) @@ -100,28 +120,56 @@ def replace_with_text(self): self.parser.stripTags( self.get_top_node(), 'b', 'strong', 'i', 'br', 'sup') - def remove_fewwords_paragraphs(self): + def remove_empty_tags(self): + """It's common in top_node to exit tags that are filled with data + within properties but not within the tags themselves, delete them """ - Remove paragraphs that have less than x number of words, - would indicate that it's some sort of link. - """ - all_nodes = self.parser.getElementsByTags(self.get_top_node(), ['*']) + all_nodes = self.parser.getElementsByTags( + self.get_top_node(), ['*']) all_nodes.reverse() for el in all_nodes: tag = self.parser.getTag(el) text = self.parser.getText(el) - stop_words = self.stopwords_class(language=self.language).\ - get_stopword_count(text) if (tag != 'br' or text != '\\r') \ - and stop_words.get_stopword_count() < 3 \ + and not text \ and len(self.parser.getElementsByTag( el, tag='object')) == 0 \ and len(self.parser.getElementsByTag( el, tag='embed')) == 0: self.parser.remove(el) - # TODO - # check if it is in the right place - else: - trimmed = self.parser.getText(el) - if trimmed.startswith("(") and trimmed.endswith(")"): - self.parser.remove(el) + + def remove_trailing_media_div(self): + """Punish the *last top level* node in the top_node if it's + DOM depth is too deep. Many media non-content links are + eliminated: "related", "loading gallery", etc. It skips removal if + last top level node's class is one of NON_MEDIA_CLASSES. + """ + + NON_MEDIA_CLASSES = ('zn-body__read-all', ) + + def get_depth(node, depth=1): + """Computes depth of an lxml element via BFS, this would be + in parser if it were used anywhere else besides this method + """ + children = self.parser.getChildren(node) + if not children: + return depth + max_depth = 0 + for c in children: + e_depth = get_depth(c, depth + 1) + if e_depth > max_depth: + max_depth = e_depth + return max_depth + + top_level_nodes = self.parser.getChildren(self.get_top_node()) + if len(top_level_nodes) < 3: + return + + last_node = top_level_nodes[-1] + + last_node_class = self.parser.getAttribute(last_node, 'class') + if last_node_class in NON_MEDIA_CLASSES: + return + + if get_depth(last_node) >= 2: + self.parser.remove(last_node) diff --git a/newspaper/parsers.py b/newspaper/parsers.py index 227ea65e..a4d030c0 100644 --- a/newspaper/parsers.py +++ b/newspaper/parsers.py @@ -11,7 +11,8 @@ import lxml.html import lxml.html.clean import re -import traceback +from html import unescape +import string from bs4 import UnicodeDammit from copy import deepcopy @@ -45,6 +46,8 @@ def css_select(cls, node, selector): def get_unicode_html(cls, html): if isinstance(html, str): return html + if not html: + return html converted = UnicodeDammit(html, is_html=True) if not converted.unicode_markup: raise Exception( @@ -65,13 +68,9 @@ def fromstring(cls, html): cls.doc = lxml.html.fromstring(html) return cls.doc except Exception: - traceback.print_exc() + log.warn('fromstring() returned an invalid string: %s...', html[:20]) return - @classmethod - def node_to_string(cls, node): - return lxml.html.tostring(node) - @classmethod def clean_article_html(cls, node): article_cleaner = lxml.html.clean.Cleaner() @@ -80,7 +79,8 @@ def clean_article_html(cls, node): article_cleaner.allow_tags = [ 'a', 'span', 'p', 'br', 'strong', 'b', 'em', 'i', 'tt', 'code', 'pre', 'blockquote', 'img', 'h1', - 'h2', 'h3', 'h4', 'h5', 'h6'] + 'h2', 'h3', 'h4', 'h5', 'h6', + 'ul', 'ol', 'li', 'dl', 'dt', 'dd'] article_cleaner.remove_unknown_tags = False return article_cleaner.clean_html(node) @@ -89,7 +89,7 @@ def nodeToString(cls, node): """`decode` is needed at the end because `etree.tostring` returns a python bytestring """ - return lxml.etree.tostring(node).decode() + return lxml.etree.tostring(node, method='html').decode() @classmethod def replaceTag(cls, node, tag): @@ -109,13 +109,18 @@ def getElementById(cls, node, idd): @classmethod def getElementsByTag( - cls, node, tag=None, attr=None, value=None, childs=False): - NS = "http://exslt.org/regular-expressions" + cls, node, tag=None, attr=None, value=None, childs=False, use_regex=False) -> list: + NS = None # selector = tag or '*' selector = 'descendant-or-self::%s' % (tag or '*') if attr and value: - selector = '%s[re:test(@%s, "%s", "i")]' % (selector, attr, value) - elems = node.xpath(selector, namespaces={"re": NS}) + if use_regex: + NS = {"re": "http://exslt.org/regular-expressions"} + selector = '%s[re:test(@%s, "%s", "i")]' % (selector, attr, value) + else: + trans = 'translate(@%s, "%s", "%s")' % (attr, string.ascii_uppercase, string.ascii_lowercase) + selector = '%s[contains(%s, "%s")]' % (selector, trans, value.lower()) + elems = node.xpath(selector, namespaces=NS) # remove the root node # if we have a selection tag if node in elems and (tag or childs): @@ -163,12 +168,9 @@ def getChildren(cls, node): @classmethod def getElementsByTags(cls, node, tags): - selector = ','.join(tags) - elems = cls.css_select(node, selector) - # remove the root node - # if we have a selection tag - if node in elems: - elems.remove(node) + selector = 'descendant::*[%s]' % ( + ' or '.join('self::%s' % tag for tag in tags)) + elems = node.xpath(selector) return elems @classmethod @@ -215,28 +217,18 @@ def getText(cls, node): @classmethod def previousSiblings(cls, node): - nodes = [] - for c, n in enumerate(node.itersiblings(preceding=True)): - nodes.append(n) - return nodes + """ + returns preceding siblings in reverse order (nearest sibling is first) + """ + return [n for n in node.itersiblings(preceding=True)] @classmethod def previousSibling(cls, node): - nodes = [] - for c, n in enumerate(node.itersiblings(preceding=True)): - nodes.append(n) - if c == 0: - break - return nodes[0] if nodes else None + return node.getprevious() @classmethod def nextSibling(cls, node): - nodes = [] - for c, n in enumerate(node.itersiblings(preceding=False)): - nodes.append(n) - if c == 0: - break - return nodes[0] if nodes else None + return node.getnext() @classmethod def isTextNode(cls, node): @@ -245,7 +237,9 @@ def isTextNode(cls, node): @classmethod def getAttribute(cls, node, attr=None): if attr: - return node.attrib.get(attr, None) + attr = node.attrib.get(attr, None) + if attr: + attr = unescape(attr) return attr @classmethod diff --git a/newspaper/resources/text/stopwords-be.txt b/newspaper/resources/text/stopwords-be.txt new file mode 100644 index 00000000..16f019fd --- /dev/null +++ b/newspaper/resources/text/stopwords-be.txt @@ -0,0 +1,96 @@ +без +более +был +была +были +было +быть +вам +вас +весь +вот +все +всего +всех +где +гэтыя +даже +для +его +если +есть +еще +же +здесь +или +как +когда +кто +либо +мне +может +надо +наш +него +нее +нет +них +однако +она +они +оно +очень +под +при +так +также +такой +там +тем +того +тоже +той +только +том +ты +тыя +іх +у +уже +хотя +чего +чей +чем +что +чтобы +чье +чья +эта +эти +это +яго +яе +яму +яна +адзін +два +тры +чатыры +пяць +шэсць +сем +восем +дзевяць +адзінаццаць +дванаццаць +трынаццаць +чатырнаццаць +пятнаццаць +шаснаццаць +семнаццаць +васемнаццаць +дзевятнаццаць +дваццаць +сто +тысяча +мільён diff --git a/newspaper/resources/text/stopwords-bg.txt b/newspaper/resources/text/stopwords-bg.txt new file mode 100644 index 00000000..af9edf4d --- /dev/null +++ b/newspaper/resources/text/stopwords-bg.txt @@ -0,0 +1,259 @@ +а +автентичен +аз +ако +ала +бе +без +беше +би +бивш +бивша +бившо +бил +била +били +било +благодаря +близо +бъдат +бъде +бяха +в +вас +ваш +ваша +вероятно +вече +взема +ви +вие +винаги +внимава +време +все +всеки +всички +всичко +всяка +във +въпреки +върху +г +ги +главен +главна +главно +глас +го +година +години +годишен +д +да +дали +два +двама +двамата +две +двете +ден +днес +дни +до +добра +добре +добро +добър +докато +докога +дори +досега +доста +друг +друга +други +е +евтин +едва +един +една +еднаква +еднакви +еднакъв +едно +екип +ето +живот +за +забавям +зад +заедно +заради +засега +заспал +затова +защо +защото +и +из +или +им +има +имат +иска +й +каза +как +каква +какво +както +какъв +като +кога +когато +което +които +кой +който +колко +която +къде +където +към +лесен +лесно +ли +лош +м +май +малко +ме +между +мек +мен +месец +ми +много +мнозина +мога +могат +може +мокър +моля +момента +му +н +на +над +назад +най +направи +напред +например +нас +не +него +нещо +нея +ни +ние +никой +нито +нищо +но +нов +нова +нови +новина +някои +някой +няколко +няма +обаче +около +освен +особено +от +отгоре +отново +още +пак +по +повече +повечето +под +поне +поради +после +почти +прави +пред +преди +през +при +пък +първата +първи +първо +пъти +равен +равна +с +са +сам +само +се +сега +си +син +скоро +след +следващ +сме +смях +според +сред +срещу +сте +съм +със +също +т +тази +така +такива +такъв +там +твой +те +тези +ти +т.н. +то +това +тогава +този +той +толкова +точно +три +трябва +тук +тъй +тя +тях +у +утре +харесва +хиляди +ч +часа +че +често +чрез +ще +щом +юмрук +я +як diff --git a/newspaper/resources/text/stopwords-el.txt b/newspaper/resources/text/stopwords-el.txt new file mode 100644 index 00000000..7a81633b --- /dev/null +++ b/newspaper/resources/text/stopwords-el.txt @@ -0,0 +1,1079 @@ +αγάπη +αγιάζι +αγορά +αγώνα +αγώνας +αγώνες +αγωνιστική +αγωνιστικής +άδεια +αίμα +αίτημα +αίτηση +αιτία +αιώνα +ακόμα +ακόμη +ακριβώς +αλήθεια +άλλα +αλλά +αλλαγές +αλλαγή +αλλάξει +άλλες +άλλη +άλλο +άλλοι +άλλος +άλλους +άλλων +άλλωστε +άμεσα +άμεση +αμέσως +αν +ανά +ανάγκες +ανάγκη +ανάλογα +αναμένεται +ανάμεσα +ανάπτυξη +ανάπτυξης +αναφέρει +αναφέρεται +αναφέρονται +αναφορά +άνδρες +ανέφερε +ανήκει +άνθρωπο +άνθρωποι +άνθρωπος +ανθρώπου +ανθρώπους +ανθρώπων +αντί +αντίθεση +αντικείμενο +άνω +ανωτέρω +αξία +αξίας +απάντηση +απέναντι +απλά +απλώς +απο +από +απόδοση +αποκλειστικά +απόλυτα +απόλυτη +απολύτως +απόσταση +αποτελεί +αποτέλεσμα +αποτελέσματα +αποτελούν +αποφάσεις +απόφαση +απόψεις +άποψη +άρα +αργότερα +άρθρο +άρθρου +αριθμό +αριθμός +αρκετά +αρκετές +αρχαία +αρχές +αρχή +αρχίζει +αρχικά +άρχισαν +άρχισε +αρχίσει +ας +άσκηση +ασφάλεια +αὐτὸ +αὐτοῦ +άτομα +αύξηση +αυτά +αυτές +αυτή +αυτήν +αυτής +αυτό +αυτοί +αυτόν +αυτός +αυτού +αυτούς +αυτών +άφησε +αφορά +αφορμή +αφορούν +αφού +βαθμό +βάθος +βάλει +βάρος +βάσει +βάση +βασικό +βέβαια +βιβλία +βιβλίο +βλέπει +βλέπουμε +βοήθεια +βράδυ +βρεθεί +βρέθηκε +βρει +βρήκε +βρίσκεται +βρίσκονται +βρισκόταν +γεγονός +γεγονότα +γενικά +γενική +γεννήθηκε +γη +γιὰ +για +γιά +γιατί +γίνει +γίνεται +γίνονται +γινόταν +γίνουν +γλώσσα +γνώμη +γνωρίζει +γνωρίζουν +γνώση +γνωστή +γνωστό +γονείς +γὰρ +γραμμή +γράφει +γρήγορα +γυναίκα +γυναίκες +γύρω +δὲ +δε +δεδομένα +δει +δείτε +δείχνει +δείχνουν +δέκα +δεν +δέν +δεύτερη +δεύτερο +δήθεν +δηλαδή +δήλωσε +δηλώσεις +δήλωση +δημιουργεί +δημιουργήσει +δημιουργία +δημόσια +δημόσιο +δια +διά +διαδικασία +διάθεση +διαθέτει +διάρκεια +διάστημα +διατάξεις +διατάξεων +διάταξη +διάφορα +διαφορά +διάφορες +διαφορετικά +διαφορετικές +διεθνείς +διεθνή +δικά +δίκαιο +δικαίωμα +δικαιώματα +δικαιωμάτων +δικαστήριο +δικές +δική +δικό +δίνει +δίνουν +διοίκηση +διότι +δίπλα +δίχως +δὲν +δουλειά +δούμε +δράση +δραστηριότητα +δρόμο +δυνάμεις +δυνάμεων +δύναμη +δύναται +δυνατή +δυνατό +δυνατόν +δυνατότητα +δυο +δύο +δύσκολα +δύσκολο +δώσει +µε +εάν +εαυτό +έγιναν +έγινε +εγώ +έδρα +έδρας +εδώ +έδωσε +εθνική +εθνικής +εθνικό +είδε +ειδικά +είδος +είδους +εικόνα +είμαι +είμαστε +είναι +είπε +εις +είσαι +εισόδημα +εισοδήματος +εισφορά +είτε +είχα +είχαμε +είχαν +είχε +εκ +έκαναν +έκανε +εκατομμύρια +έκδοση +εκεί +εκείνα +εκείνη +εκείνο +εκείνος +έκθεση +εκλογές +εκτός +έλαβε +έλεγε +έλεγχο +ελευθερία +ελεύθερο +έλλειψη +ελληνικά +ελληνικές +ελληνική +ελληνικής +ελληνικό +ελληνικού +έμεινε +εμείς +εμπειρία +εμφανίζεται +εμφάνιση +εν +ένα +εἶναι +έναν +έναντι +ενάντια +εναντίον +έναρξη +ένας +ενδιαφέρον +ενέργεια +ενημέρωσης +ενημέρωσής +έννοια +ενός +εντελώς +εντολή +εντός +ενώ +ενώπιον +εξ +εξαιρετικά +εξαιτίας +εξέλιξη +εξής +έξι +εξουσία +εξουσίας +έξω +επανάσταση +επάνω +επειδή +έπειτα +επί +επίθεση +επικεφαλής +επιλογή +επίπεδα +επίπεδο +επιπλέον +επίσημη +επίσης +επιστροφή +επιτυχία +επιχειρήσεις +επόμενη +επόμενο +εποχή +εποχής +έπρεπε +έργα +εργασία +εργασίας +έργο +έρευνα +έρθει +έρχεται +έρχονται +εἰς +έστω +εσύ +εσωτερικό +εταιρεία +εταιρείας +εταιρείες +έτος +έτσι +ετών +ευθύνη +ευκαιρία +εύκολα +ευρώ +εφαρμογή +έφερε +εφημερίδα +εφόσον +έφτασε +έχασε +έχει +έχεις +έχετε +έχοντας +έχουμε +έχουν +έχω +έως +ζήτημα +ζητήματα +ζήτησε +ζουν +ζωή +ζωής +η +ή +ήδη +ήθελε +ηλικία +ηλικίας +ημέρα +ημέρες +ημών +ήρθε +ήσαν +ήταν +ήττα +θὰ +θα +θά +θάνατο +θέλει +θέλουν +θέλω +θέμα +θέματα +θέσεις +θέση +θεωρεί +θεωρείται +θεωρία +ιδέα +ίδια +ιδιαίτερα +ιδιαίτερη +ίδιο +ίδιοι +ίδιος +ίδιου +ιδίως +ιστορία +ιστορική +ισχύει +ίσως +καὶ +καθαρά +κάθε +καθεστώς +καθηγητής +καθόλου +καθώς +και +καί +καιρό +καλά +καλή +καλό +καλύτερα +καλύτερη +καλύτερο +καμία +καμιά +καν +κάνει +κάνεις +κανείς +κανένα +κανένας +κάνουμε +κάνουν +κάνω +κάποια +κάποιες +κάποιο +κάποιοι +κάποιον +κάποιος +κάποτε +καρδιά +κάρτα +κατὰ +κατά +κατασκευή +κατάσταση +καταστροφή +κατάφερε +κατηγορία +κάτι +κατόπιν +κάτω +κείμενο +κέντρο +κερδίσει +κεφάλαιο +κι +κίνδυνο +κίνηση +κοινή +κοινό +κοινωνία +κοινωνίας +κοινωνική +κοινωνικής +κοινωνικό +κόμμα +κομμάτι +κοντά +κόντρα +κόσμο +κόσμος +κόσμου +κόστος +κράτη +κράτος +κράτους +κρίση +κυβέρνηση +κυβέρνησης +κύρια +κύριο +κυρίως +λάβει +λάθος +λαό +λαός +λαού +λέει +λειτουργεί +λειτουργία +λειτουργίας +λένε +λέξεις +λέξη +λίγα +λίγες +λίγο +λιγότερο +ἀλλὰ +λογαριασμό +λόγια +λογική +λόγο +λόγος +λόγους +λόγω +λοιπόν +λύση +μὲ +μὴ +μα +μαζί +μακριά +μάλιστα +μάλλον +μας +μάτια +ματς +μάχη +με +μέ +μεγάλα +μεγάλες +μεγάλη +μεγάλης +μεγάλο +μεγάλος +μεγάλου +μεγαλύτερη +μεγαλύτερο +μεγάλων +μείνει +μείωση +μελέτη +μέλη +μέλλον +μέλος +μελών +μεν +μέρα +μέρες +μέρη +μερικά +μερικές +μέρος +μέρους +μέσα +μέσο +μέσω +μετὰ +μετά +μεταξύ +μέτρα +μέχρι +μη +μην +μήνα +μήνες +μήπως +μητέρα +μια +μία +μιας +μικρή +μικρό +μνήμη +μόλις +μόνη +μόνο +μόνον +μόνος +μορφή +μου +μπει +μπορεί +μπορείς +μπορούμε +μπορούν +μπορούσαν +μπορούσε +μπορώ +μπροστά +μυαλό +ὅμως +ἐν +νὰ +να +ἕνα +νά +νέα +νέας +νέες +νέο +νέος +νέου +νερό +νέων +νίκες +νίκη +νόμισμα +νόμο +νόμου +ξανά +ξεκίνησε +ξεκινήσει +ο +οἱ +ό +οδηγεί +οι +οικογένεια +οικονομία +οικονομίας +οικονομικά +οικονομική +οικονομικής +όλα +όλες +όλη +όλο +όλοι +ολόκληρη +ολόκληρο +όλους +όλων +ομάδα +ομάδας +ομάδες +όμως +όνομα +οποία +οποιαδήποτε +οποίας +οποίες +οποίο +οποίοι +οποίος +οποίου +οποίους +οποίων +οπότε +όπου +όπως +οργάνωση +όρια +όριο +όρος +όρους +όσα +όσο +όσοι +όσον +όσους +όταν +ότι +ουσία +ουσιαστικά +ούτε +οφείλεται +όχι +ἀπὸ +ἐπὶ +πάγκο +παγκόσμια +παγκόσμιο +πάει +παιδί +παιδιά +παίζει +παίρνει +παιχνίδι +παιχνίδια +πάλι +πάντα +πάντοτε +πάντως +πάνω +πάρα +παρά +παραγωγή +παραγωγής +παράδειγμα +παράδοση +παρακάτω +παράλληλα +παραμένει +παραπάνω +πάρει +παρελθόν +πάρουν +παρουσία +παρουσιάζει +πατέρα +πατέρας +πατρίδα +πεδίο +πει +πέντε +πέρα +περαιτέρω +πέρασε +περάσει +περί +περιβάλλον +περιεχόμενο +περιλαμβάνει +περίοδο +περιοχές +περιοχή +περιοχής +περίπου +περιπτώσεις +περίπτωση +περισσότερα +περισσότερες +περισσότερο +περισσότεροι +πήγε +πηγές +πηγή +πήρε +πια +πιο +πίστη +πίσω +πλαίσια +πλαίσιο +πλανήτη +πλειοψηφία +πλέον +πλευρά +πληθυσμού +πλήρη +πληροφορίες +πλήρως +πνεύμα +ποὺ +ἀπό +ποια +ποιο +ποιος +πόλεμο +πόλεμος +πολέμου +πόλη +πόλης +πολίτες +πολιτικά +πολιτικές +πολιτική +πολιτικής +πολιτικό +πολιτικών +πολιτών +πολλά +πολλές +πολλοί +πολλούς +πολλών +πολύ +πορεία +πόσο +ποσό +ποσοστό +ποσού +πότε +ποτέ +που +πού +πούμε +πράγμα +πράγματα +πράγματι +πραγματικά +πραγματική +πραγματικότητα +πράξεις +πράξη +πρέπει +πριν +προ +πρόβλημα +προβλήματα +πρόγραμμα +πρόεδρος +προέρχεται +προηγούμενη +προϊόντα +προϊόντων +προκαλεί +προκειμένου +πρόκειται +προκύπτει +προς +προσοχή +προσπαθεί +προσπάθεια +προσπάθειες +προστασία +προσφέρει +πρόσωπα +προσωπική +πρόσωπο +πρόταση +προφανώς +πρὸς +πρώην +πρωθυπουργός +πρωί +πρώτα +πρώτες +πρώτη +πρώτης +πρώτο +πρώτος +πρώτου +πτώση +πω +πως +ὅπως +πώς +ρόλο +ρύθμιση +σὲ +ὡς +σαν +σας +σε +σέ +σειρά +σερί +σημαίνει +σημαντικά +σημαντική +σημαντικό +σημασία +σημεία +σημείο +σήμερα +σημερινή +σίγουρα +σκέψη +σκοπό +σου +σπίτι +στὸ +στὴ +στὰ +στα +στάση +στη +στην +στήν +στιγμή +στις +στὴν +στὸν +στο +στό +στοιχεία +στοιχείο +στοιχείων +στοιχήματα +στον +στους +στόχο +στόχος +στρατιωτική +συγκεκριμένα +συγκεκριμένη +συγκεκριμένο +συζήτηση +συμβαίνει +συμμετοχή +συμπεριφορά +συμφέροντα +σύμφωνα +συμφωνία +συνάντηση +συνδυασμό +συνέβη +συνέπεια +συνέπειες +συνεργασία +συνέχεια +συνεχίζει +συνεχώς +συνήθως +συνθήκες +συνολικά +σύνολο +σύστημα +συστήματος +συχνά +σχέδια +σχέδιο +σχεδόν +σχέσεις +σχέση +σχετικά +σχετική +σώμα +τὸ +τὰ +τὴ +τῷ +τῇ +τα +τά +ἦταν +ὅταν +τάξη +τάξης +τας +ταυτόχρονα +τελευταία +τελευταίες +τελευταίο +τέλη +τελικά +τελικό +τέλος +τεράστια +τέσσερα +τέσσερις +τέτοια +τέτοιο +τη +την +τήν +της +τής +τι +ὅτι +τί +τιμές +τιμή +τίποτα +τίποτε +τις +τίς +τίτλο +τμήμα +τὴν +τῶν +τὸν +το +τοῦ +τό +τοις +τομέα +τον +τόν +τόπο +τοὺς +τοῖς +τόσο +τότε +του +τού +τουλάχιστον +τους +τούς +τούτο +τούτου +τράπεζα +τράπεζες +τρεις +τρία +τρίτη +τριών +τρόπο +τρόπος +τῆς +τὶς +τύπου +τυχόν +τω +των +τών +τώρα +υπάρξει +ύπαρξη +υπάρχει +υπάρχουν +υπέρ +υπηρεσίες +υπηρεσιών +υπήρξε +υπήρχαν +υπήρχε +υπό +υπόθεση +υπουργός +υποχρέωση +υπόψη +ύστερα +ύψος +ύψους +φαίνεται +φάση +φέρει +φορά +φορές +φτάσει +φύση +φυσικά +φυσικό +φως +χαρά +χαρακτήρα +χαρακτηριστικά +χάρη +χάσει +χέρι +χέρια +χθες +χιλιάδες +χρειάζεται +χρέος +χρέους +χρήματα +χρήση +χρησιμοποιούν +χρόνια +χρονιά +χρονικό +χρόνο +χρόνου +χώρα +χώρας +χώρες +χωριό +χωρίς +χώρο +χώρου +χωρών +ψυχή +ώρα +ώρες +ως +ώστε +ωστόσο diff --git a/newspaper/resources/text/stopwords-et.txt b/newspaper/resources/text/stopwords-et.txt new file mode 100644 index 00000000..412fda88 --- /dev/null +++ b/newspaper/resources/text/stopwords-et.txt @@ -0,0 +1,44 @@ +ei +oma +jah +nad +kes +mind +ning +nii +sa +olen +ka +mulle +või +te +ära +oled +sest +ole +mis +olema +see +seda +mida +oli +mul +on +kõik +minu +midagi +ma +pole +mu +kas +me +siis +kui +ega +ja +ta +et +selle +siin +ehk +aga diff --git a/newspaper/resources/text/stopwords-fa.txt b/newspaper/resources/text/stopwords-fa.txt new file mode 100644 index 00000000..b08ceeba --- /dev/null +++ b/newspaper/resources/text/stopwords-fa.txt @@ -0,0 +1,799 @@ +! +, +. +: +; +، +؛ +؟ +آباد +آره +آری +آمد +آمده +آن +آنان +آنجا +آنطور +آنقدر +آنكه +آنها +آنچه +آنکه +آورد +آورده +آيد +آی +آیا +آیند +اتفاقا +اثرِ +احتراما +احتمالا +اخیر +اری +از +ازجمله +اساسا +است +استفاد +استفاده +اش +اشکارا +اصلا +اصولا +اعلام +اغلب +اكنون +الان +البته +البتّه +ام +اما +امروز +امروزه +امسال +امشب +امور +ان +انجام +اند +انشاالله +انصافا +انطور +انقدر +انها +انچنان +انکه +انگار +او +اول +اولا +اي +ايشان +ايم +اين +اينكه +اکثرا +اکنون +اگر +ای +ایا +اید +ایشان +ایم +این +اینجا +ایند +اینطور +اینقدر +اینها +اینچنین +اینک +اینکه +اینگونه +با +بار +بارة +باره +بارها +باز +بازهم +باش +باشد +باشم +باشند +باشيم +باشی +باشید +باشیم +بالا +بالاخره +بالایِ +بالطبع +بايد +باید +بتوان +بتواند +بتوانی +بتوانیم +بخش +بخشی +بخواه +بخواهد +بخواهم +بخواهند +بخواهی +بخواهید +بخواهیم +بد +بدون +بر +برابر +برابرِ +براحتی +براساس +براستی +براي +برای +برایِ +برخوردار +برخي +برخی +برداري +برعکس +بروز +بزرگ +بزودی +بسا +بسيار +بسياري +بسیار +بسیاری +بطور +بعد +بعدا +بعدها +بعری +بعضا +بعضي +بلافاصله +بلكه +بله +بلکه +بلی +بنابراين +بنابراین +بندي +به +بهتر +بهترين +بود +بودم +بودن +بودند +بوده +بودی +بودید +بودیم +بویژه +بي +بيست +بيش +بيشتر +بيشتري +بين +بکن +بکند +بکنم +بکنند +بکنی +بکنید +بکنیم +بگو +بگوید +بگویم +بگویند +بگویی +بگویید +بگوییم +بگیر +بگیرد +بگیرم +بگیرند +بگیری +بگیرید +بگیریم +بی +بیا +بیاب +بیابد +بیابم +بیابند +بیابی +بیابید +بیابیم +بیاور +بیاورد +بیاورم +بیاورند +بیاوری +بیاورید +بیاوریم +بیاید +بیایم +بیایند +بیایی +بیایید +بیاییم +بیرون +بیرونِ +بیش +بیشتر +بیشتری +بین +ت +تا +تازه +تاكنون +تان +تاکنون +تحت +تر +تر براساس +ترين +تقریبا +تلویحا +تمام +تماما +تمامي +تنها +تو +تواند +توانست +توانستم +توانستن +توانستند +توانسته +توانستی +توانستیم +توانم +توانند +توانی +توانید +توانیم +توسط +تولِ +تویِ +ثانیا +جا +جاي +جايي +جای +جدا +جديد +جدید +جريان +جریان +جز +جلوگيري +جلویِ +جمعا +جناح +جهت +حاضر +حال +حالا +حتما +حتي +حتی +حداکثر +حدودا +حدودِ +حق +خارجِ +خب +خدمات +خصوصا +خلاصه +خواست +خواستم +خواستن +خواستند +خواسته +خواستی +خواستید +خواستیم +خواهد +خواهم +خواهند +خواهيم +خواهی +خواهید +خواهیم +خوب +خود +خودت +خودتان +خودش +خودشان +خودم +خودمان +خوشبختانه +خويش +خویش +خویشتن +خیاه +خیر +خیلی +داد +دادم +دادن +دادند +داده +دادی +دادید +دادیم +دار +دارد +دارم +دارند +داريم +داری +دارید +داریم +داشت +داشتم +داشتن +داشتند +داشته +داشتی +داشتید +داشتیم +دانست +دانند +دایم +دایما +در +درباره +درمجموع +درون +دریغ +دقیقا +دنبالِ +ده +دهد +دهم +دهند +دهی +دهید +دهیم +دو +دوباره +دوم +ديده +ديروز +ديگر +ديگران +ديگري +دیر +دیروز +دیگر +دیگران +دیگری +را +راحت +راسا +راستی +راه +رسما +رسید +رفت +رفته +رو +روب +روز +روزانه +روزهاي +روي +روی +رویِ +ريزي +زمان +زمانی +زمینه +زود +زياد +زير +زيرا +زیر +زیرِ +سابق +ساخته +سازي +سالانه +سالیانه +سایر +سراسر +سرانجام +سریعا +سریِ +سعي +سمتِ +سوم +سوي +سوی +سویِ +سپس +شان +شايد +شاید +شخصا +شد +شدم +شدن +شدند +شده +شدی +شدید +شدیدا +شدیم +شش +شش نداشته +شما +شناسي +شود +شوم +شوند +شونده +شوی +شوید +شویم +صرفا +صورت +ضدِّ +ضدِّ +ضمن +طبعا +طبقِ +طبیعتا +طرف +طريق +طریق +طور +طي +طی +ظاهرا +عدم +عقبِ +علّتِ +علیه +عمدا +عمدتا +عمل +عملا +عنوان +عنوانِ +غالبا +غير +غیر +فردا +فعلا +فقط +فكر +فوق +قابل +قبل +قبلا +قدری +قصدِ +قطعا +كرد +كردم +كردن +كردند +كرده +كسي +كل +كمتر +كند +كنم +كنند +كنيد +كنيم +كه +لااقل +لطفا +لطفاً +ما +مان +مانند +مانندِ +مبادا +متاسفانه +متعاقبا +مثل +مثلا +مثلِ +مجانی +مجددا +مجموعا +مختلف +مدام +مدت +مدّتی +مردم +مرسی +مستقیما +مسلما +مطمینا +معمولا +مقابل +ممکن +من +موارد +مورد +موقتا +مي +ميليارد +ميليون +مگر +می +می شود +میان +می‌رسد +می‌رود +می‌شود +می‌کنیم +ناشي +نام +ناگاه +ناگهان +ناگهانی +نبايد +نباید +نبود +نخست +نخستين +نخواهد +نخواهم +نخواهند +نخواهی +نخواهید +نخواهیم +ندارد +ندارم +ندارند +نداری +ندارید +نداریم +نداشت +نداشتم +نداشتند +نداشته +نداشتی +نداشتید +نداشتیم +نزديك +نزدِ +نزدیکِ +نسبتا +نشان +نشده +نظير +نظیر +نكرده +نمايد +نمي +نمی +نمی‌شود +نه +نهایتا +نوع +نوعي +نوعی +نيز +نيست +نگاه +نیز +نیست +ها +هاي +هايي +های +هایی +هبچ +هر +هرچه +هرگز +هزار +هست +هستم +هستند +هستيم +هستی +هستید +هستیم +هفت +هم +همان +همه +همواره +همين +همچنان +همچنين +همچنین +همچون +همیشه +همین +هنوز +هنگام +هنگامِ +هنگامی +هيچ +هیچ +هیچگاه +و +واقعا +واقعی +وجود +وسطِ +وضع +وقتي +وقتی +وقتیکه +ولی +وي +وگو +وی +ویژه +يا +يابد +يك +يكديگر +يكي +ّه +٪ +پارسال +پاعینِ +پس +پنج +پيش +پیدا +پیش +پیشاپیش +پیشتر +پیشِ +چرا +چطور +چقدر +چنان +چنانچه +چنانکه +چند +چندین +چنين +چنین +چه +چهار +چو +چون +چيزي +چگونه +چیز +چیزی +چیست +کاش +کامل +کاملا +کتبا +کجا +کجاست +کدام +کرد +کردم +کردن +کردند +کرده +کردی +کردید +کردیم +کس +کسانی +کسی +کل +کلا +کم +کماکان +کمتر +کمتری +کمی +کن +کنار +کنارِ +کند +کنم +کنند +کننده +کنون +کنونی +کنی +کنید +کنیم +که +کو +کَی +کی +گاه +گاهی +گذاري +گذاشته +گذشته +گردد +گرفت +گرفتم +گرفتن +گرفتند +گرفته +گرفتی +گرفتید +گرفتیم +گروهي +گفت +گفتم +گفتن +گفتند +گفته +گفتی +گفتید +گفتیم +گه +گهگاه +گو +گويد +گويند +گویا +گوید +گویم +گویند +گویی +گویید +گوییم +گيرد +گيري +گیرد +گیرم +گیرند +گیری +گیرید +گیریم +ی +یا +یابد +یابم +یابند +یابی +یابید +یابیم +یافت +یافتم +یافتن +یافته +یافتی +یافتید +یافتیم +یعنی +یقینا +یه +یک +یکی +۰ +۱ +۲ +۳ +۴ +۵ +۶ +۷ +۸ +۹ diff --git a/newspaper/resources/text/stopwords-he.txt b/newspaper/resources/text/stopwords-he.txt new file mode 100644 index 00000000..8ac77858 --- /dev/null +++ b/newspaper/resources/text/stopwords-he.txt @@ -0,0 +1,221 @@ +אני +את +אתה +אנחנו +אתן +אתם +הם +הן +היא +הוא +שלי +שלו +שלך +שלה +שלנו +שלכם +שלכן +שלהם +שלהן +לי +לו +לה +לנו +לכם +לכן +להם +להן +אותה +אותו +זה +זאת +אלה +אלו +תחת +מתחת +מעל +בין +עם +עד +נגר +על +אל +מול +של +אצל +כמו +אחר +אותו +בלי +לפני +אחרי +מאחורי +עלי +עליו +עליה +עליך +עלינו +עליכם +לעיכן +עליהם +עליהן +כל +כולם +כולן +כך +ככה +כזה +זה +זות +אותי +אותה +אותם +אותך +אותו +אותן +אותנו +ואת +את +אתכם +אתכן +איתי +איתו +איתך +איתה +איתם +איתן +איתנו +איתכם +איתכן +יהיה +תהיה +היתי +היתה +היה +להיות +עצמי +עצמו +עצמה +עצמם +עצמן +עצמנו +עצמהם +עצמהן +מי +מה +איפה +היכן +במקום שבו +אם +לאן +למקום שבו +מקום בו +איזה +מהיכן +איך +כיצד +באיזו מידה +מתי +בשעה ש +כאשר +כש +למרות +לפני +אחרי +מאיזו סיבה +הסיבה שבגללה +למה +מדוע +לאיזו תכלית +כי +יש +אין +אך +מנין +מאין +מאיפה +יכל +יכלה +יכלו +יכול +יכולה +יכולים +יכולות +יוכלו +יוכל +מסוגל +לא +רק +אולי +אין +לאו +אי +כלל +נגד +אם +עם +אל +אלה +אלו +אף +על +מעל +מתחת +מצד +בשביל +לבין +באמצע +בתוך +דרך +מבעד +באמצעות +למעלה +למטה +מחוץ +מן +לעבר +מכאן +כאן +הנה +הרי +פה +שם +אך +ברם +שוב +אבל +מבלי +בלי +מלבד +רק +בגלל +מכיוון +עד +אשר +ואילו +למרות +אס +כמו +כפי +אז +אחרי +כן +לכן +לפיכך +מאד +עז +מעט +מעטים +במידה +שוב +יותר +מדי +גם +כן +נו +אחר +אחרת +אחרים +אחרות +אשר +או \ No newline at end of file diff --git a/newspaper/resources/text/stopwords-hi.txt b/newspaper/resources/text/stopwords-hi.txt new file mode 100644 index 00000000..2cf52d70 --- /dev/null +++ b/newspaper/resources/text/stopwords-hi.txt @@ -0,0 +1,237 @@ +अंदर +अत +अदि +अप +अपना +अपनि +अपनी +अपने +अभि +अभी +आदि +इंहिं +इंहें +इंहों +इतयादि +इत्यादि +इन +इनका +इन्हीं +इन्हें +इन्हों +इस +इसका +इसकि +इसकी +इसके +इसमें +इसि +इसी +इसे +उंहिं +उंहें +उंहों +उन +उनका +उनकि +उनकी +उनको +उन्हीं +उन्हें +उन्हों +उस +उसके +उसि +उसी +एक +एवं +एस +एसे +ऐसे +ओर +कइ +कई +करता +करते +करना +करने +करें +कहते +कहा +का +काफि +काफ़ी +कि +किंहें +किंहों +कितना +किन्हें +किन्हों +किया +किर +किस +किसि +किसी +किसे +की +कुछ +कुल +के +कोइ +कोई +कोन +कोनसा +कौनसा +गया +घर +जब +जहाँ +जहां +जा +जिंहें +जिंहों +जितना +जिधर +जिन +जिन्हें +जिन्हों +जिस +जिसे +जीधर +जेसा +जेसे +जैसा +जो +तक +तब +तरह +तिंहें +तिंहों +तिन +तिन्हें +तिन्हों +तिस +तिसे +थि +थी +थे +दबारा +दवारा +दिया +दुसरा +दुसरे +दूसरे +दो +द्वारा +न +नहिं +नहीं +ना +निचे +निहायत +ने +पहले +पुरा +पूरा +पे +फिर +बनि +बनी +बहि +बही +बहुत +बाद +बाला +बिलकुल +भि +भितर +भी +भीतर +मगर +मानो +मे +यदि +यहाँ +यहां +यहि +यही +या +यिह +ये +रखें +रवासा +रहा +रहे +ऱ्वासा +लिए +लिये +लेकिन +व +वगेरह +वरग +वर्ग +वहां +वहिं +वहीं +वाले +वुह +वे +वग़ैरह +संग +सकता +सबसे +सभि +सभी +साथ +साबुत +साभ +सारा +से +सो +हि +ही +हुअ +हुआ +हुइ +हुई +हुए +हे +हें +हो +होता +होति +होती +होते +होना +होने +को +नीचे +सब +जैसे +पर +तो +कौन +यह +और +वहाँ +था +है +उसे +वह +कैसे +उनके +मैं +में +आप +कर +सकते +हैं +मुझे +अब +हमें +क्या +हम +करेगा +इतना +होगा + diff --git a/newspaper/resources/text/stopwords-hr.txt b/newspaper/resources/text/stopwords-hr.txt new file mode 100644 index 00000000..64388b0c --- /dev/null +++ b/newspaper/resources/text/stopwords-hr.txt @@ -0,0 +1,179 @@ +a +ako +ali +bi +bih +bila +bili +bilo +bio +bismo +biste +biti +bumo +da +do +duž +ga +hoće +hoćemo +hoćete +hoćeš +hoću +i +iako +ih +ili +iz +ja +je +jedna +jedne +jedno +jer +jesam +jesi +jesmo +jest +jeste +jesu +jim +joj +još +ju +kada +kako +kao +koja +koje +koji +kojima +koju +kroz +li +me +mene +meni +mi +mimo +moj +moja +moje +mu +na +nad +nakon +nam +nama +nas +naš +naša +naše +našeg +ne +nego +neka +neki +nekog +neku +nema +netko +neće +nećemo +nećete +nećeš +neću +nešto +ni +nije +nikoga +nikoje +nikoju +nisam +nisi +nismo +niste +nisu +njega +njegov +njegova +njegovo +njemu +njezin +njezina +njezino +njih +njihov +njihova +njihovo +njim +njima +njoj +nju +no +o +od +odmah +on +ona +oni +ono +ova +pa +pak +po +pod +pored +prije +s +sa +sam +samo +se +sebe +sebi +si +smo +ste +su +sve +svi +svog +svoj +svoja +svoje +svom +ta +tada +taj +tako +te +tebe +tebi +ti +to +toj +tome +tu +tvoj +tvoja +tvoje +u +uz +vam +vama +vas +vaš +vaša +vaše +već +vi +vrlo +za +zar +će +ćemo +ćete +ćeš +ću +što \ No newline at end of file diff --git a/newspaper/resources/text/stopwords-it.txt b/newspaper/resources/text/stopwords-it.txt index 98ffee1d..73a4182f 100644 --- a/newspaper/resources/text/stopwords-it.txt +++ b/newspaper/resources/text/stopwords-it.txt @@ -1,133 +1,133 @@ -ad -al -allo -ai -agli -all -agl -alla -alle -con -col -coi -da -dal -dallo -dai -dagli -dall -dagl -dalla -dalle -di -del -dello -dei -degli -dell -degl -della -delle -in -nel -nello -nei -negli -nell -negl -nella -nelle -su -sul -sullo -sui -sugli -sull -sugl -sulla -sulle -per -tra -contro -io -tu -lui -lei -noi -voi -loro -mio -mia -miei -mie -tuo -tua -tuoi -tue -suo -sua -suoi -sue -nostro -nostra -nostri -nostre -vostro -vostra -vostri -vostre -mi -ti -ci -vi -lo -la -li -le -gli -ne -il -un -uno -una -ma -ed -se -perchè +ad +al +allo +ai +agli +all +agl +alla +alle +con +col +coi +da +dal +dallo +dai +dagli +dall +dagl +dalla +dalle +di +del +dello +dei +degli +dell +degl +della +delle +in +nel +nello +nei +negli +nell +negl +nella +nelle +su +sul +sullo +sui +sugli +sull +sugl +sulla +sulle +per +tra +contro +io +tu +lui +lei +noi +voi +loro +mio +mia +miei +mie +tuo +tua +tuoi +tue +suo +sua +suoi +sue +nostro +nostra +nostri +nostre +vostro +vostra +vostri +vostre +mi +ti +ci +vi +lo +la +li +le +gli +ne +il +un +uno +una +ma +ed +se +perchè perché perche -anche -come -dov -dove -che -chi -cui -non -più +anche +come +dov +dove +che +chi +cui +non +più piu -quale -quanto -quanti -quanta -quante -quello -quelli -quella -quelle -questo -questi -questa -queste -si -tutto -tutti -a -c -e -i -l -o +quale +quanto +quanti +quanta +quante +quello +quelli +quella +quelle +questo +questi +questa +queste +si +tutto +tutti +a +c +e +i +l +o ho hai ha diff --git a/newspaper/resources/text/stopwords-ja.txt b/newspaper/resources/text/stopwords-ja.txt new file mode 100644 index 00000000..38735ee3 --- /dev/null +++ b/newspaper/resources/text/stopwords-ja.txt @@ -0,0 +1,134 @@ +あそこ +あっ +あの +あのかた +あの人 +あり +あります +ある +あれ +い +いう +います +いる +う +うち +え +お +および +おり +おります +か +かつて +から +が +き +ここ +こちら +こと +この +これ +これら +さ +さらに +し +しかし +する +ず +せ +せる +そこ +そして +その +その他 +その後 +それ +それぞれ +それで +た +ただし +たち +ため +たり +だ +だっ +だれ +つ +て +で +でき +できる +です +では +でも +と +という +といった +とき +ところ +として +とともに +とも +と共に +どこ +どの +な +ない +なお +なかっ +ながら +なく +なっ +など +なに +なら +なり +なる +なん +に +において +における +について +にて +によって +により +による +に対して +に対する +に関する +の +ので +のみ +は +ば +へ +ほか +ほとんど +ほど +ます +また +または +まで +も +もの +ものの +や +よう +より +ら +られ +られる +れ +れる +を +ん +何 +及び +彼 +彼女 +我々 +特に +私 +私達 +貴方 +貴方方 \ No newline at end of file diff --git a/newspaper/resources/text/stopwords-lt.txt b/newspaper/resources/text/stopwords-lt.txt new file mode 100644 index 00000000..a9fa60bd --- /dev/null +++ b/newspaper/resources/text/stopwords-lt.txt @@ -0,0 +1,165 @@ +taip +jas +be +kito +taps +juos +dvi +būti +jo +kita +juo +kokį +gero +tie +mes +bei +savo +mūsų +bus +rodo +jame +kam +prie +kada +itin +kuo +tiek +toks +ir +dar +nei +patį +visa +mums +kad +ko +arba +visą +pats +jis +pat +net +kurį +nėra +nė +pusė +pati +metu +ne +jam +jais +kuri +į +mano +mus +irgi +ta +šiuo +jų +vėl +vis +jei +su +šia +kiti +sau +na +virš +keli +būtų +jai +teks +o +namų +šalį +yra +šiol +šią +apie +kiek +daug +kai +gal +tų +jog +jos +gana +man +dėl +tą +jums +šis +gera +bet +per +negu +tarp +tokį +toli +liko +kaip +teko +nuo +tapo +būna +vos +jūsų +po +turi +ką +link +čia +šie +tai +jie +tuo +štai +ar +nors +tos +jau +ant +bent +esu +kol +pusę +vien +beje +nes +mane +koks +ši +iš +kitų +aš +juk +tu +kur +nori +tik +tuos +jūs +tas +tam +tada +jį +dalį +abu +pas +ten +ypač +šios +šio +tad +šių +iki +deja +viso +to +visų +ji +kas +lyg +save +šiek \ No newline at end of file diff --git a/newspaper/resources/text/stopwords-mk.txt b/newspaper/resources/text/stopwords-mk.txt new file mode 100644 index 00000000..c128a1f2 --- /dev/null +++ b/newspaper/resources/text/stopwords-mk.txt @@ -0,0 +1,173 @@ +а +е +од +до +без +со +за +на +ја +го +ги +низ +исто +истото +под +над +да +ќе +во +него +неа +тој +таа +тоа +ние +вие +тие +кој +која +кои +дали +се +не +беше +еден +едно +јас +рече +сите +сум +сме +овде +така +и +што +или +по +но +му +па +нив +ни +итн +бил +кај +ова +врз +a +две +би +она +си +кое +ако +два +има +в +како +само +дека +една +туку +кога +сега +ми +потоа +ти +кон +додека +веќе +нешто +уште +таму +ли +беа +ме +некој +ништо +тука +пред +им +каде +повторно +ниту +биде +толку +никогаш +мене +тогаш +своите +сето +нема +околу +многу +полека +секогаш +зошто +те +добро +можеби +колку +можеше +нивните +преку +миг +знам +малку +вратата +ох +навистина +оваа +покрај +повеќе +овој +сеуште +имаше +својата +неговите +неговата +друго +зашто +немаше +воопшто +понекогаш +ах +зад +еднаш +својот +дури +себе +ви +токму +зарем +сте +било +сосема +секој +неговото +друг +мошне +ајде +можел +може +при +пак +сè +други +треба +ама +после +некоја +нас +бе +никој +одма +сѐ +ај +нѐ +неколку +оние +мора +оди +еј diff --git a/newspaper/resources/text/stopwords-pl.txt b/newspaper/resources/text/stopwords-pl.txt new file mode 100644 index 00000000..eee0b22d --- /dev/null +++ b/newspaper/resources/text/stopwords-pl.txt @@ -0,0 +1,350 @@ +a +aby +ach +acz +aczkolwiek +aj +albo +ale +alez +ależ +ani +az +aż +bardziej +bardzo +beda +bedzie +bez +deda +będą +bede +będę +będzie +bo +bowiem +by +byc +być +byl +byla +byli +bylo +byly +był +była +było +były +bynajmniej +cala +cali +caly +cała +cały +ci +cie +ciebie +cię +co +cokolwiek +cos +coś +czasami +czasem +czemu +czy +czyli +daleko +dla +dlaczego +dlatego +do +dobrze +dokad +dokąd +dosc +dość +duzo +dużo +dwa +dwaj +dwie +dwoje +dzis +dzisiaj +dziś +gdy +gdyby +gdyz +gdyż +gdzie +gdziekolwiek +gdzies +gdzieś +go +i +ich +ile +im +inna +inne +inny +innych +iz +iż +ja +jak +jakas +jakaś +jakby +jaki +jakichs +jakichś +jakie +jakis +jakiś +jakiz +jakiż +jakkolwiek +jako +jakos +jakoś +ją +je +jeden +jedna +jednak +jednakze +jednakże +jedno +jego +jej +jemu +jesli +jest +jestem +jeszcze +jeśli +jezeli +jeżeli +juz +już +kazdy +każdy +kiedy +kilka +kims +kimś +kto +ktokolwiek +ktora +ktore +ktorego +ktorej +ktory +ktorych +ktorym +ktorzy +ktos +ktoś +która +które +którego +której +który +których +którym +którzy +ku +lat +lecz +lub +ma +mają +mało +mam +mi +miedzy +między +mimo +mna +mną +mnie +moga +mogą +moi +moim +moj +moja +moje +moze +mozliwe +mozna +może +możliwe +można +mój +mu +musi +my +na +nad +nam +nami +nas +nasi +nasz +nasza +nasze +naszego +naszych +natomiast +natychmiast +nawet +nia +nią +nic +nich +nie +niech +niego +niej +niemu +nigdy +nim +nimi +niz +niż +no +o +obok +od +około +on +ona +one +oni +ono +oraz +oto +owszem +pan +pana +pani +po +pod +podczas +pomimo +ponad +poniewaz +ponieważ +powinien +powinna +powinni +powinno +poza +prawie +przeciez +przecież +przed +przede +przedtem +przez +przy +roku +rowniez +również +sam +sama +są +sie +się +skad +skąd +soba +sobą +sobie +sposob +sposób +swoje +ta +tak +taka +taki +takie +takze +także +tam +te +tego +tej +ten +teraz +też +to +toba +tobą +tobie +totez +toteż +totobą +trzeba +tu +tutaj +twoi +twoim +twoj +twoja +twoje +twój +twym +ty +tych +tylko +tym +u +w +wam +wami +was +wasz +wasza +wasze +we +według +wiele +wielu +więc +więcej +wlasnie +właśnie +wszyscy +wszystkich +wszystkie +wszystkim +wszystko +wtedy +wy +z +za +zaden +zadna +zadne +zadnych +zapewne +zawsze +ze +zeby +zeznowu +zł +znow +znowu +znów +zostal +został +żaden +żadna +żadne +żadnych +że +żeby \ No newline at end of file diff --git a/newspaper/resources/text/stopwords-pt.txt b/newspaper/resources/text/stopwords-pt.txt index 66d557f9..9c8725d6 100644 --- a/newspaper/resources/text/stopwords-pt.txt +++ b/newspaper/resources/text/stopwords-pt.txt @@ -1,147 +1,560 @@ -último -é +a acerca +adeus agora +ainda +alem algmas +algo +algumas alguns ali +além +ambas ambos +ano +anos antes +ao +aonde +aos +apenas +apoio apontar +apos +após aquela aquelas aquele aqueles aqui +aquilo +as +assim +através atrás +até +aí +baixo +bastante bem +boa +boas bom +bons +breve cada caminho +catorze +cedo +cento +certamente +certeza cima +cinco +coisa com como comprido conhecido +conselho +contra +contudo corrente +cuja +cujas +cujo +cujos +custa +cá +da +daquela +daquelas +daquele +daqueles +dar das +de debaixo +dela +delas +dele +deles +demais dentro +depois desde desligado +dessa +dessas +desse +desses +desta +destas +deste +destes deve devem deverá +dez +dezanove +dezasseis +dezassete +dezoito +dia +diante direita +dispoe +dispoem +diversa +diversas +diversos diz +dizem dizer +do dois dos +doze +duas +durante +dá +dão +dúvida e ela +elas ele eles em +embora enquanto +entao +entre então -está -estão +era +eram +essa +essas +esse +esses +esta estado +estamos estar estará +estas +estava +estavam este +esteja +estejam +estejamos estes esteve estive estivemos +estiver +estivera estiveram +estiverem +estivermos +estivesse +estivessem +estiveste +estivestes +estivéramos +estivéssemos +estou +está +estás +estávamos +estão eu +exemplo +falta fará +favor faz +fazeis +fazem +fazemos fazer +fazes fazia +faço fez fim +final foi +fomos +for fora +foram +forem +forma +formos +fosse +fossem +foste +fostes +fui +fôramos +fôssemos +geral +grande +grandes +grupo +ha +haja +hajam +hajamos +havemos +havia +hei +hoje +hora horas +houve +houvemos +houver +houvera +houveram +houverei +houverem +houveremos +houveria +houveriam +houvermos +houverá +houverão +houveríamos +houvesse +houvessem +houvéramos +houvéssemos +há +hão iniciar inicio ir irá +isso ista iste isto +já +lado +lhe +lhes ligado +local +logo +longe +lugar +lá +maior maioria maiorias mais +mal mas +me +mediante +meio +menor +menos +meses +mesma +mesmas mesmo +mesmos meu +meus +mil +minha +minhas +momento muito muitos -nós -não +máximo +mês +na +nada +nao +naquela +naquelas +naquele +naqueles +nas +nem +nenhuma +nessa +nessas +nesse +nesses +nesta +nestas +neste +nestes +no +noite nome +nos +nossa +nossas nosso +nossos +nova +novas +nove novo +novos +num +numa +numas +nunca +nuns +não +nível +nós +número o +obra +obrigada +obrigado +oitava +oitavo +oito onde +ontem +onze os ou +outra +outras outro +outros para +parece parte +partir +paucas pegar +pela +pelas pelo +pelos +perante +perto pessoas pode +podem +poder poderá podia +pois +ponto +pontos por porque +porquê +portanto +posição +possivelmente +posso +possível +pouca +pouco +poucos povo +primeira +primeiras +primeiro +primeiros promeiro -quê +propios +proprio +própria +próprias +próprio +próprios +próxima +próximas +próximo +próximos +puderam +pôde +põe +põem +quais qual qualquer quando +quanto +quarta +quarto +quatro +que quem +quer +quereis +querem +queremas +queres +quero +questão quieto -são +quinta +quinto +quinze +quáis +quê +relação +sabe +sabem saber +se +segunda +segundo +sei +seis +seja +sejam +sejamos sem +sempre +sendo ser +serei +seremos +seria +seriam +será +serão +seríamos +sete seu +seus +sexta +sexto +sim +sistema +sob +sobre +sois somente -têm +somos +sou +sua +suas +são +sétima +sétimo +só tal +talvez +tambem também +tanta +tantas +tanto +tarde +te tem +temos tempo +tendes +tenha +tenham +tenhamos tenho +tens tentar tentaram tente tentei +ter +terceira +terceiro +terei +teremos +teria +teriam +terá +terão +teríamos teu +teus teve +tinha +tinham tipo tive +tivemos +tiver +tivera +tiveram +tiverem +tivermos +tivesse +tivessem +tiveste +tivestes +tivéramos +tivéssemos +toda +todas +todo todos trabalhar trabalho +treze +três tu +tua +tuas +tudo +tão +tém +têm +tínhamos um uma umas uns usa usar +vai +vais valor veja +vem +vens ver verdade verdadeiro +vez +vezes +viagem +vindo +vinte você +vocês +vos +vossa +vossas +vosso +vossos +vários +vão +vêm +vós +zero +à +às +área +é +éramos +és +último diff --git a/newspaper/resources/text/stopwords-ro.txt b/newspaper/resources/text/stopwords-ro.txt new file mode 100644 index 00000000..b84f7618 --- /dev/null +++ b/newspaper/resources/text/stopwords-ro.txt @@ -0,0 +1,281 @@ +acea +aceasta +această +aceea +acei +aceia +acel +acela +acele +acelea +acest +acesta +aceste +acestea +aceşti +aceştia +acolo +acord +acum +ai +aia +aibă +aici +al +ăla +ale +alea +ălea +altceva +altcineva +am +ar +are +aş +aşadar +asemenea +asta +ăsta +astăzi +astea +ăstea +ăştia +asupra +aţi +au +avea +avem +aveţi +azi +bine +bucur +bună +ca +că +căci +când +care +cărei +căror +cărui +cât +câte +câţi +către +câtva +caut +ce +cel +ceva +chiar +cinci +cînd +cine +cineva +cît +cîte +cîţi +cîtva +contra +cu +cum +cumva +curând +curînd +da +dă +dacă +dar +dată +datorită +dau +de +deci +deja +deoarece +departe +deşi +din +dinaintea +dintr- +dintre +doi +doilea +două +drept +după +ea +ei +el +ele +eram +este +eşti +eu +face +fără +fata +fi +fie +fiecare +fii +fim +fiţi +fiu +frumos +graţie +halbă +iar +ieri +îi +îl +îmi +împotriva +în +înainte +înaintea +încât +încît +încotro +între +întrucât +întrucît +îţi +la +lângă +le +li +lîngă +lor +lui +mă +mai +mâine +mea +mei +mele +mereu +meu +mi +mie +mîine +mine +mult +multă +mulţi +mulţumesc +ne +nevoie +nicăieri +nici +nimeni +nimeri +nimic +nişte +noastră +noastre +noi +noroc +noştri +nostru +nouă +nu +opt +ori +oricând +oricare +oricât +orice +oricînd +oricine +oricît +oricum +oriunde +până +patra +patru +patrulea +pe +pentru +peste +pic +pînă +poate +pot +prea +prima +primul +prin +puţin +puţina +puţină +rog +sa +să +săi +sale +şapte +şase +sau +său +se +şi +sînt +sîntem +sînteţi +spate +spre +ştiu +sub +sunt +suntem +sunteţi +sută +ta +tăi +tale +tău +te +ţi +ţie +timp +tine +toată +toate +tot +toţi +totuşi +trei +treia +treilea +tu +un +una +unde +undeva +unei +uneia +unele +uneori +unii +unor +unora +unu +unui +unuia +unul +vă +vi +voastră +voastre +voi +voştri +vostru +vouă +vreme +vreo +vreun +zece +zero +zi +zice \ No newline at end of file diff --git a/newspaper/resources/text/stopwords-ru.txt b/newspaper/resources/text/stopwords-ru.txt index 94984803..b0d2cf0d 100644 --- a/newspaper/resources/text/stopwords-ru.txt +++ b/newspaper/resources/text/stopwords-ru.txt @@ -1,4 +1,4 @@ -а +а е и ж diff --git a/newspaper/resources/text/stopwords-sl.txt b/newspaper/resources/text/stopwords-sl.txt new file mode 100644 index 00000000..7135ed3a --- /dev/null +++ b/newspaper/resources/text/stopwords-sl.txt @@ -0,0 +1,446 @@ +a +ali +april +avgust +b +bi +bil +bila +bile +bili +bilo +biti +blizu +bo +bodo +bojo +bolj +bom +bomo +boste +bova +boš +brez +c +cel +cela +celi +celo +d +da +daleč +dan +danes +datum +december +deset +deseta +deseti +deseto +devet +deveta +deveti +deveto +do +dober +dobra +dobri +dobro +dokler +dol +dolg +dolga +dolgi +dovolj +drug +druga +drugi +drugo +dva +dve +e +eden +en +ena +ene +eni +enkrat +eno +etc. +f +februar +g +g. +ga +ga. +gor +gospa +gospod +h +halo +i +idr. +ii +iii +in +iv +ix +iz +j +januar +jaz +je +ji +jih +jim +jo +julij +junij +jutri +k +kadarkoli +kaj +kajti +kako +kakor +kamor +kamorkoli +kar +karkoli +katerikoli +kdaj +kdo +kdorkoli +ker +ki +kje +kjer +kjerkoli +ko +koder +koderkoli +koga +komu +kot +kratek +kratka +kratke +kratki +l +lahka +lahke +lahki +lahko +le +lep +lepa +lepe +lepi +lepo +leto +m +maj +majhen +majhna +majhni +malce +malo +manj +marec +me +med +medtem +mene +mesec +mi +midva +midve +mnogo +moj +moja +moje +mora +morajo +moram +moramo +morate +moraš +morem +mu +n +na +nad +naj +najina +najino +najmanj +naju +največ +nam +narobe +nas +nato +nazaj +naš +naša +naše +ne +nedavno +nedelja +nek +neka +nekaj +nekatere +nekateri +nekatero +nekdo +neke +nekega +neki +nekje +neko +nekoga +nekoč +ni +nikamor +nikdar +nikjer +nikoli +nič +nje +njega +njegov +njegova +njegovo +njej +njemu +njen +njena +njeno +nji +njih +njihov +njihova +njihovo +njiju +njim +njo +njun +njuna +njuno +no +nocoj +november +npr. +o +ob +oba +obe +oboje +od +odprt +odprta +odprti +okoli +oktober +on +onadva +one +oni +onidve +osem +osma +osmi +osmo +oz. +p +pa +pet +peta +petek +peti +peto +po +pod +pogosto +poleg +poln +polna +polni +polno +ponavadi +ponedeljek +ponovno +potem +povsod +pozdravljen +pozdravljeni +prav +prava +prave +pravi +pravo +prazen +prazna +prazno +prbl. +precej +pred +prej +preko +pri +pribl. +približno +primer +pripravljen +pripravljena +pripravljeni +proti +prva +prvi +prvo +r +ravno +redko +res +reč +s +saj +sam +sama +same +sami +samo +se +sebe +sebi +sedaj +sedem +sedma +sedmi +sedmo +sem +september +seveda +si +sicer +skoraj +skozi +slab +smo +so +sobota +spet +sreda +srednja +srednji +sta +ste +stran +stvar +sva +t +ta +tak +taka +take +taki +tako +takoj +tam +te +tebe +tebi +tega +težak +težka +težki +težko +ti +tista +tiste +tisti +tisto +tj. +tja +to +toda +torek +tretja +tretje +tretji +tri +tu +tudi +tukaj +tvoj +tvoja +tvoje +u +v +vaju +vam +vas +vaš +vaša +vaše +ve +vedno +velik +velika +veliki +veliko +vendar +ves +več +vi +vidva +vii +viii +visok +visoka +visoke +visoki +vsa +vsaj +vsak +vsaka +vsakdo +vsake +vsaki +vsakomur +vse +vsega +vsi +vso +včasih +včeraj +x +z +za +zadaj +zadnji +zakaj +zaprta +zaprti +zaprto +zdaj +zelo +zunaj +č +če +često +četrta +četrtek +četrti +četrto +čez +čigav +š +šest +šesta +šesti +šesto +štiri +ž +že \ No newline at end of file diff --git a/newspaper/resources/text/stopwords-sr.txt b/newspaper/resources/text/stopwords-sr.txt new file mode 100644 index 00000000..3612f464 --- /dev/null +++ b/newspaper/resources/text/stopwords-sr.txt @@ -0,0 +1,134 @@ +baš +bez +biæe +bio +biti +blizu +broj +dana +danas +doæi +dobar +dobiti +dok +dole +došao +drugi +duž +dva +èesto +èiji +gde +gore +hvala +iæi +iako +ide +ima +imam +imao +ispod +izmeðu +iznad +izvan +izvoli +jedan +jedini +jednom +jeste +još +juèe +kad +kako +kao +koga +koja +koje +koji +kroz +mali +manji +misli +mnogo +moæi +mogu +mora +morao +naæi +naš +negde +nego +nekad +neki +nemam +nešto +nije +nijedan +nikada +nismo +ništa +njega +njegov +njen +njih +njihov +oko +okolo +ona +onaj +oni +ono +osim +ostali +otišao +ovako +ovamo +ovde +ove +ovo +pitati +poèetak +pojedini +posle +povodom +praviti +pre +preko +prema +prvi +put +radije +sada +smeti +šta +stvar +stvarno +sutra +svaki +sve +svim +svugde +taèno +tada +taj +takoðe +tamo +tim +uèinio +uèiniti +umalo +unutra +upotrebiti +uzeti +vaš +veæina +veoma +video +više +zahvaliti +zašto +zbog +želeo +želi +znati + diff --git a/newspaper/resources/text/stopwords-sw.txt b/newspaper/resources/text/stopwords-sw.txt new file mode 100644 index 00000000..3d74b845 --- /dev/null +++ b/newspaper/resources/text/stopwords-sw.txt @@ -0,0 +1,74 @@ +akasema +alikuwa +alisema +baada +basi +bila +cha +chini +hadi +hapo +hata +hivyo +hiyo +huku +huo +ili +ilikuwa +juu +kama +karibu +katika +kila +kima +kisha +kubwa +kutoka +kuwa +kwa +kwamba +kwenda +kwenye +la +lakini +mara +mdogo +mimi +mkubwa +mmoja +moja +muda +mwenye +na +naye +ndani +ng +ni +nini +nonkungu +pamoja +pia +sana +sasa +sauti +tafadhali +tena +tu +vile +wa +wakati +wake +walikuwa +wao +watu +wengine +wote +ya +yake +yangu +yao +yeye +yule +za +zaidi +zake diff --git a/newspaper/resources/text/stopwords-th.txt b/newspaper/resources/text/stopwords-th.txt new file mode 100644 index 00000000..34cc248f --- /dev/null +++ b/newspaper/resources/text/stopwords-th.txt @@ -0,0 +1,115 @@ +กล่าว +กว่า +กัน +กับ +การ +ก็ +ก่อน +ขณะ +ขอ +ของ +ขึ้น +คง +ครั้ง +ความ +คือ +จะ +จัด +จาก +จึง +ช่วง +ซึ่ง +ดัง +ด้วย +ด้าน +ตั้ง +ตั้งแต่ +ตาม +ต่อ +ต่าง +ต่างๆ +ต้อง +ถึง +ถูก +ถ้า +ทั้ง +ทั้งนี้ +ทาง +ที่ +ที่สุด +ทุก +ทํา +ทําให้ +นอกจาก +นัก +นั้น +นี้ +น่า +นํา +บาง +ผล +ผ่าน +พบ +พร้อม +มา +มาก +มี +ยัง +รวม +ระหว่าง +รับ +ราย +ร่วม +ลง +วัน +ว่า +สุด +ส่ง +ส่วน +สําหรับ +หนึ่ง +หรือ +หลัง +หลังจาก +หลาย +หาก +อยาก +อยู่ +อย่าง +ออก +อะไร +อาจ +อีก +เขา +เข้า +เคย +เฉพาะ +เช่น +เดียว +เดียวกัน +เนื่องจาก +เปิด +เปิดเผย +เป็น +เป็นการ +เพราะ +เพื่อ +เมื่อ +เรา +เริ่ม +เลย +เห็น +เอง +แต่ +แบบ +แรก +และ +แล้ว +แห่ง +โดย +ใน +ให้ +ได้ +ไป +ไม่ +ไว้ diff --git a/newspaper/resources/text/stopwords-tr.txt b/newspaper/resources/text/stopwords-tr.txt new file mode 100644 index 00000000..43b8a0ff --- /dev/null +++ b/newspaper/resources/text/stopwords-tr.txt @@ -0,0 +1,210 @@ +acaba +altmış +altı +ama +ancak +arada +aslında +ayrıca +bana +bazı +belki +ben +benden +beni +benim +beri +beş +bile +bin +bir +birçok +biri +birkaç +birkez +birşey +birşeyi +biz +bize +bizden +bizi +bizim +böyle +böylece +bu +buna +bunda +bundan +bunlar +bunları +bunların +bunu +bunun +burada +çok +çünkü +da +daha +dahi +de +defa +değil +diğer +diye +doksan +dokuz +dolayı +dolayısıyla +dört +edecek +eden +ederek +edilecek +ediliyor +edilmesi +ediyor +eğer +elli +en +etmesi +etti +ettiği +ettiğini +gibi +göre +halen +hangi +hatta +hem +henüz +hep +hepsi +her +herhangi +herkes +herkesin +hiç +hiçbir +için +iki +ile +ilgili +ise +işte +itibaren +itibariyle +kadar +karşın +katrilyon +kendi +kendilerine +kendini +kendisi +kendisine +kendisini +kez +ki +kim +kimden +kime +kimi +kimse +kırk +milyar +milyon +mu +mü +mı +nasıl +ne +neden +nedenle +nerde +nerede +nereye +niye +niçin +o +olan +olarak +oldu +olduğu +olduğunu +olduklarını +olmadı +olmadığı +olmak +olması +olmayan +olmaz +olsa +olsun +olup +olur +olursa +oluyor +on +ona +ondan +onlar +onlardan +onları +onların +onu +onun +otuz +oysa +öyle +pek +rağmen +sadece +sanki +sekiz +seksen +sen +senden +seni +senin +siz +sizden +sizi +sizin +şey +şeyden +şeyi +şeyler +şöyle +şu +şuna +şunda +şundan +şunları +şunu +tarafından +trilyon +tüm +üç +üzere +var +vardı +ve +veya +ya +yani +yapacak +yapılan +yapılması +yapıyor +yapmak +yaptı +yaptığı +yaptığını +yaptıkları +yedi +yerine +yetmiş +yine +yirmi +yoksa +yüz +zaten diff --git a/newspaper/resources/text/stopwords-uk.txt b/newspaper/resources/text/stopwords-uk.txt new file mode 100644 index 00000000..bab6599f --- /dev/null +++ b/newspaper/resources/text/stopwords-uk.txt @@ -0,0 +1,373 @@ +a +б +в +ж +з +у +я +є +і +аж +ви +де +до +за +зі +ми +на +не +ну +ні +по +та +ти +то +ту +ті +це +цю +ця +ці +чи +ще +що +як +їй +їм +їх +її +або +але +без +був +вам +вас +ваш +вже +все +всю +вся +від +він +два +дві +для +ким +моя +моє +мої +міг +між +мій +над +нам +нас +наш +нею +неї +них +ніж +ній +ось +при +про +під +раз +рік +сам +сих +сім +так +там +теж +тим +тих +той +тою +три +тут +хоч +хто +цей +цим +цих +час +щоб +яка +які +адже +буде +буду +будь +була +були +було +бути +вами +ваша +ваше +ваші +весь +вниз +вона +вони +воно +всею +всім +всіх +втім +геть +далі +двох +день +дуже +зате +його +йому +каже +кого +коли +кому +крім +куди +лише +люди +мало +мати +мене +мені +миру +мною +може +нами +наша +наше +наші +ними +ніби +один +поки +пора +рано +року +році +сама +саме +саму +самі +свою +своє +свої +себе +собі +став +суть +така +таке +такі +твоя +твоє +твій +тебе +тими +тобі +того +тоді +тому +туди +хоча +хіба +цими +цієї +часу +чого +чому +який +яких +якої +якщо +ім'я +інша +інше +інші +буває +будеш +більш +вгору +вміти +внизу +вісім +давно +даром +добре +довго +дякую +життя +зараз +знову +кожен +кожна +кожне +кожні +краще +ледве +майже +менше +могти +можна +назад +немає +нижче +нього +однак +п'ять +перед +поруч +потім +проти +після +років +самим +самих +самій +свого +своєї +своїх +собою +справ +такий +також +тепер +тисяч +тобою +треба +трохи +усюди +усіма +хочеш +цього +цьому +часто +через +шість +якого +іноді +інший +інших +багато +будемо +будете +будуть +більше +всього +всьому +далеко +десять +досить +другий +дійсно +завжди +звідси +зовсім +кругом +кілька +людина +можуть +навіть +навіщо +нагорі +небудь +низько +ніколи +нікуди +нічого +обидва +одного +однієї +п'ятий +перший +просто +раніше +раптом +самими +самого +самому +сказав +скрізь +сьомий +третій +тільки +хотіти +чотири +чудово +шостий +близько +важлива +важливе +важливі +вдалині +восьмий +говорив +дев'ять +десятий +зайнята +зайнято +зайняті +занадто +значить +навколо +нарешті +нерідко +повинно +посеред +початку +пізніше +сказала +сказати +скільки +спасибі +частіше +важливий +двадцять +дев'ятий +зазвичай +зайнятий +звичайно +здається +найбільш +не можна +недалеко +особливо +потрібно +спочатку +сьогодні +численна +численне +численні +відсотків +двадцятий +звідусіль +мільйонів +нещодавно +прекрасно +четвертий +численний +дванадцять +одинадцять +сімнадцять +тринадцять +безперервно +дванадцятий +одинадцятий +п'ятнадцять +сімнадцятий +тринадцятий +шістнадцять +вісімнадцять +п'ятнадцятий +чотирнадцять +шістнадцятий +вісімнадцятий +дев'ятнадцять +чотирнадцятий +дев'ятнадцятий \ No newline at end of file diff --git a/newspaper/settings.py b/newspaper/settings.py index 5add9eeb..df19733d 100644 --- a/newspaper/settings.py +++ b/newspaper/settings.py @@ -10,6 +10,7 @@ import logging import os +import tempfile from http.cookiejar import CookieJar as cj @@ -31,9 +32,7 @@ DATA_DIRECTORY = '.newspaper_scraper' -TOP_DIRECTORY = os.path.join(os.path.expanduser("~"), DATA_DIRECTORY) -if not os.path.exists(TOP_DIRECTORY): - os.mkdir(TOP_DIRECTORY) +TOP_DIRECTORY = os.path.join(tempfile.gettempdir(), DATA_DIRECTORY) # Error log LOGFILE = os.path.join(TOP_DIRECTORY, 'newspaper_errors_%s.log' % __version__) @@ -44,14 +43,14 @@ MEMO_FILE = 'memoized' MEMO_DIR = os.path.join(TOP_DIRECTORY, MEMO_FILE) -if not os.path.exists(MEMO_DIR): - os.mkdir(MEMO_DIR) - # category and feed cache CF_CACHE_DIRECTORY = 'feed_category_cache' ANCHOR_DIRECTORY = os.path.join(TOP_DIRECTORY, CF_CACHE_DIRECTORY) -if not os.path.exists(ANCHOR_DIRECTORY): - os.mkdir(ANCHOR_DIRECTORY) - TRENDING_URL = 'http://www.google.com/trends/hottrends/atom/feed?pn=p1' + +for path in (TOP_DIRECTORY, MEMO_DIR, ANCHOR_DIRECTORY): + try: + os.mkdir(path) + except FileExistsError: + pass diff --git a/newspaper/source.py b/newspaper/source.py index f4eed76b..e1f91138 100644 --- a/newspaper/source.py +++ b/newspaper/source.py @@ -9,26 +9,22 @@ __copyright__ = 'Copyright 2014, Lucas Ou-Yang' import logging - -import feedparser +from urllib.parse import urljoin, urlsplit, urlunsplit from tldextract import tldextract from . import network from . import urls from . import utils - from .article import Article -from .extractors import ContentExtractor from .configuration import Configuration +from .extractors import ContentExtractor from .settings import ANCHOR_DIRECTORY - log = logging.getLogger(__name__) class Category(object): - def __init__(self, url): self.url = url self.html = None @@ -36,13 +32,15 @@ def __init__(self, url): class Feed(object): - def __init__(self, url): self.url = url self.rss = None # TODO self.dom = None, speed up Feedparser +NUM_THREADS_PER_SOURCE_WARN_LIMIT = 5 + + class Source(object): """Sources are abstractions of online news vendors like huffpost or cnn. domain = 'www.cnn.com' @@ -52,6 +50,7 @@ class Source(object): articles = [<article obj>, <article obj>, ..] brand = 'cnn' """ + def __init__(self, url, config=None, **kwargs): """The config object for this source will be passed into all of this source's children articles unless specified otherwise or re-set. @@ -85,11 +84,11 @@ def __init__(self, url, config=None, **kwargs): self.is_parsed = False self.is_downloaded = False - def build(self, response=None): + def build(self): """Encapsulates download and basic parsing with lxml. May be a good idea to split this into download() and parse() methods. """ - self.download(response) + self.download() self.parse() self.set_categories() @@ -97,8 +96,8 @@ def build(self, response=None): self.parse_categories() self.set_feeds() - self.download_feeds() # mthread - # TODO: self.parse_feeds() # regex for now + self.download_feeds() # mthread + # self.parse_feeds() self.generate_articles() @@ -117,7 +116,7 @@ def purge_articles(self, reason, articles): articles[:] = [a for a in articles if a.is_valid_body()] return articles - @utils.cache_disk(seconds=(86400*1), cache_folder=ANCHOR_DIRECTORY) + @utils.cache_disk(seconds=(86400 * 1), cache_folder=ANCHOR_DIRECTORY) def _get_category_urls(self, domain): """The domain param is **necessary**, see .utils.cache_disk for reasons. the boilerplate method is so we can use this decorator right. @@ -133,7 +132,39 @@ def set_feeds(self): """Don't need to cache getting feed urls, it's almost instant with xpath """ - urls = self.extractor.get_feed_urls(self.url, self.categories) + common_feed_urls = ['/feed', '/feeds', '/rss'] + common_feed_urls = [urljoin(self.url, url) for url in common_feed_urls] + + split = urlsplit(self.url) + if split.netloc in ('medium.com', 'www.medium.com'): + # should handle URL to user or user's post + if split.path.startswith('/@'): + new_path = '/feed/' + split.path.split('/')[1] + new_parts = split.scheme, split.netloc, new_path, '', '' + common_feed_urls.append(urlunsplit(new_parts)) + + common_feed_urls_as_categories = [Category(url=url) for url in common_feed_urls] + + category_urls = [c.url for c in common_feed_urls_as_categories] + requests = network.multithread_request(category_urls, self.config) + + for index, _ in enumerate(common_feed_urls_as_categories): + response = requests[index].resp + if response and response.ok: + common_feed_urls_as_categories[index].html = network.get_html( + response.url, response=response) + + common_feed_urls_as_categories = [c for c in common_feed_urls_as_categories if c.html] + + for _ in common_feed_urls_as_categories: + doc = self.config.get_parser().fromstring(_.html) + _.doc = doc + + common_feed_urls_as_categories = [c for c in common_feed_urls_as_categories if + c.doc is not None] + + categories_and_common_feed_urls = self.categories + common_feed_urls_as_categories + urls = self.extractor.get_feed_urls(self.url, categories_and_common_feed_urls) self.feeds = [Feed(url=url) for url in urls] def set_description(self): @@ -160,9 +191,9 @@ def download_categories(self): self.categories[index].html = network.get_html( req.url, response=req.resp) else: - if self.config.verbose: - print(('deleting category', - self.categories[index].url, 'due to download err')) + log.warning(('Deleting category %s from source %s due to ' + 'download error') % + (self.categories[index].url, self.url)) self.categories = [c for c in self.categories if c.html] def download_feeds(self): @@ -177,9 +208,9 @@ def download_feeds(self): self.feeds[index].rss = network.get_html( req.url, response=req.resp) else: - if self.config.verbose: - print(('deleting feed', - self.categories[index].url, 'due to download err')) + log.warning(('Deleting feed %s from source %s due to ' + 'download error') % + (self.categories[index].url, self.url)) self.feeds = [f for f in self.feeds if f.rss] def parse(self): @@ -189,7 +220,7 @@ def parse(self): # TODO: This is a terrible idea, ill try to fix it when i'm more rested self.doc = self.config.get_parser().fromstring(self.html) if self.doc is None: - print('[Source parse ERR]', self.url) + log.warning('Source %s parse error.' % self.url) return self.set_description() @@ -204,21 +235,22 @@ def parse_categories(self): self.categories = [c for c in self.categories if c.doc is not None] + def _map_title_to_feed(self, feed): + doc = self.config.get_parser().fromstring(feed.rss) + if doc is None: + # http://stackoverflow.com/a/24893800 + return None + + elements = self.config.get_parser().getElementsByTag(doc, tag='title') + feed.title = next((element.text for element in elements if element.text), self.brand) + return feed + def parse_feeds(self): - """DEPRECATED - Due to the slow speed of feedparser, we won't be dom parsing - our .rss feeds, but rather regex searching for urls in the .rss - text and then relying on our article logic to detect false urls. + """Add titles to feeds """ - for feed in self.feeds: - try: - feed.dom = feedparser.parse(feed.html) - except Exception as e: - log.critical('feedparser failed %s' % e) - if self.config.verbose: - print('feed %s has failed parsing' % feed.url) - - self.feeds = [feed for feed in self.feeds if feed.dom is not None] + log.debug('We are parsing %d feeds' % + len(self.feeds)) + self.feeds = [self._map_title_to_feed(f) for f in self.feeds] def feeds_to_articles(self): """Returns articles given the url of a feed @@ -232,7 +264,7 @@ def feeds_to_articles(self): for url in urls: article = Article( url=url, - source_url=self.url, + source_url=feed.url, config=self.config) cur_articles.append(article) @@ -245,9 +277,6 @@ def feeds_to_articles(self): articles.extend(cur_articles) - if self.config.verbose: - print(('%d->%d->%d for %s' % - (before_purge, after_purge, after_memo, feed.url))) log.debug('%d->%d->%d for %s' % (before_purge, after_purge, after_memo, feed.url)) return articles @@ -268,7 +297,7 @@ def categories_to_articles(self): _article = Article( url=indiv_url, - source_url=self.url, + source_url=category.url, title=indiv_title, config=self.config ) @@ -283,9 +312,6 @@ def categories_to_articles(self): articles.extend(cur_articles) - if self.config.verbose: - print(('%d->%d->%d for %s' % - (before_purge, after_purge, after_memo, category.url))) log.debug('%d->%d->%d for %s' % (before_purge, after_purge, after_memo, category.url)) return articles @@ -305,7 +331,8 @@ def generate_articles(self, limit=5000): """ articles = self._generate_articles() self.articles = articles[:limit] - log.debug(len(articles), 'articles generated and cutoff at', limit) + log.debug('%d articles generated and cutoff at %d', + len(articles), limit) def download_articles(self, threads=1): """Downloads all articles attached to self @@ -323,9 +350,9 @@ def download_articles(self, threads=1): failed_articles.append(self.articles[index]) self.articles = [a for a in self.articles if a.html] else: - if threads > 5: - print(('Using 5+ threads on a single source ' - 'may get you rate limited!')) + if threads > NUM_THREADS_PER_SOURCE_WARN_LIMIT: + log.warning(('Using %s+ threads on a single source ' + 'may result in rate limiting!') % NUM_THREADS_PER_SOURCE_WARN_LIMIT) filled_requests = network.multithread_request(urls, self.config) # Note that the responses are returned in original order for index, req in enumerate(filled_requests): @@ -337,9 +364,8 @@ def download_articles(self, threads=1): self.is_downloaded = True if len(failed_articles) > 0: - if self.config.verbose: - print('[ERROR], these article urls failed the download:', - [a.url for a in failed_articles]) + log.warning('The following article urls failed the download: %s' % + ', '.join([a.url for a in failed_articles])) def parse_articles(self): """Parse all articles, delete if too small @@ -377,27 +403,13 @@ def article_urls(self): """ return [article.url for article in self.articles] - def get_key(self): - # TODO - pass - - def clear_anchor_directory(self): - """Clears out all files in our directory where we cache anchors - the key is sha1(self.domain).hexdigest() fn is ANCHOR_DIR/key. - """ - pass - # TODO: - # d_pth = os.path.join( - # settings.MEMO_DIR, domain_to_filename(source_domain)) - # os.path.remove(ANCHOR_DIRECTORY) - def print_summary(self): """Prints out a summary of the data in our source instance """ - print('[source url]:', self.url) - print('[source brand]:', self.brand) - print('[source domain]:', self.domain) - print('[source len(articles)]:', len(self.articles)) + print('[source url]:', self.url) + print('[source brand]:', self.brand) + print('[source domain]:', self.domain) + print('[source len(articles)]:', len(self.articles)) print('[source description[:50]]:', self.description[:50]) print('printing out 10 sample articles...') diff --git a/newspaper/text.py b/newspaper/text.py index d2bbc5af..23b4c6b1 100644 --- a/newspaper/text.py +++ b/newspaper/text.py @@ -58,8 +58,6 @@ def set_word_count(self, cnt): class StopWords(object): - PUNCTUATION = re.compile( - "[^\\p{Ll}\\p{Lu}\\p{Lt}\\p{Lo}\\p{Nd}\\p{Pc}\\s]") TRANS_TABLE = str.maketrans('', '') _cached_stop_words = {} @@ -89,13 +87,13 @@ def get_stopword_count(self, content): return WordStats() ws = WordStats() stripped_input = self.remove_punctuation(content) - candidate_words = self.candidate_words(stripped_input) + candidate_words = self.candidate_words(stripped_input.lower()) overlapping_stopwords = [] c = 0 for w in candidate_words: c += 1 - if w.lower() in self.STOP_WORDS: - overlapping_stopwords.append(w.lower()) + if w in self.STOP_WORDS: + overlapping_stopwords.append(w) ws.set_word_count(c) ws.set_stopword_count(len(overlapping_stopwords)) @@ -141,6 +139,32 @@ class StopWordsKorean(StopWords): def __init__(self, language='ko'): super(StopWordsKorean, self).__init__(language='ko') + def get_stopword_count(self, content): + if not content: + return WordStats() + ws = WordStats() + stripped_input = self.remove_punctuation(content) + candidate_words = self.candidate_words(stripped_input) + overlapping_stopwords = [] + c = 0 + for w in candidate_words: + c += 1 + for s in self.STOP_WORDS: + if w.endswith(s): + overlapping_stopwords.append(w) + + ws.set_word_count(c) + ws.set_stopword_count(len(overlapping_stopwords)) + ws.set_stop_words(overlapping_stopwords) + return ws + + +class StopWordsHindi(StopWords): + """Hindi segmentation + """ + def __init__(self, language='hi'): + super(StopWordsHindi, self).__init__(language='hi') + def get_stopword_count(self, content): if not content: return WordStats() @@ -158,3 +182,28 @@ def get_stopword_count(self, content): ws.set_stopword_count(len(overlapping_stopwords)) ws.set_stop_words(overlapping_stopwords) return ws + + +class StopWordsJapanese(StopWords): + """Japanese segmentation + """ + def __init__(self, language='ja'): + super(StopWordsJapanese, self).__init__(language='ja') + + def candidate_words(self, stripped_input): + import tinysegmenter + segmenter = tinysegmenter.TinySegmenter() + tokens = segmenter.tokenize(stripped_input) + return tokens + + +class StopWordsThai(StopWords): + """Thai segmentation + """ + def __init__(self, language='th'): + super(StopWordsThai, self).__init__(language='th') + + def candidate_words(self, stripped_input): + import pythainlp + tokens = pythainlp.word_tokenize(stripped_input) + return tokens diff --git a/newspaper/urls.py b/newspaper/urls.py index 2d414882..2126aaf3 100644 --- a/newspaper/urls.py +++ b/newspaper/urls.py @@ -20,10 +20,12 @@ MAX_FILE_MEMO = 20000 +_STRICT_DATE_REGEX_PREFIX = r'(?<=\W)' DATE_REGEX = r'([\./\-_]{0,1}(19|20)\d{2})[\./\-_]{0,1}(([0-3]{0,1}[0-9][\./\-_])|(\w{3,5}[\./\-_]))([0-3]{0,1}[0-9][\./\-]{0,1})?' +STRICT_DATE_REGEX = _STRICT_DATE_REGEX_PREFIX + DATE_REGEX ALLOWED_TYPES = ['html', 'htm', 'md', 'rst', 'aspx', 'jsp', 'rhtml', 'cgi', - 'xhtml', 'jhtml', 'asp'] + 'xhtml', 'jhtml', 'asp', 'shtml'] GOOD_PATHS = ['story', 'article', 'feature', 'featured', 'slides', 'slideshow', 'gallery', 'news', 'video', 'media', @@ -35,14 +37,15 @@ BAD_DOMAINS = ['amazon', 'doubleclick', 'twitter'] + def remove_args(url, keep_params=(), frags=False): """ Remove all param arguments from a url. """ parsed = urlsplit(url) - filtered_query= '&'.join( + filtered_query = '&'.join( qry_item for qry_item in parsed.query.split('&') - if qry_item.startswith(keep_params) + if qry_item.startswith(keep_params) ) if frags: frag = parsed[4:] @@ -51,6 +54,7 @@ def remove_args(url, keep_params=(), frags=False): return urlunsplit(parsed[:3] + (filtered_query,) + frag) + def redirect_back(url, source_domain): """ Some sites like Pinterest have api's that cause news @@ -73,6 +77,7 @@ def redirect_back(url, source_domain): return url + def prepare_url(url, source_url=None): """ Operations that purify a url, removes arguments, @@ -93,6 +98,7 @@ def prepare_url(url, source_url=None): return proper_url + def valid_url(url, verbose=False, test=False): """ Is this URL a valid news-article url? @@ -137,7 +143,7 @@ def valid_url(url, verbose=False, test=False): if verbose: print('\t%s rejected because len of url is less than 11' % url) return False - r1 = ('mailto:' in url) # TODO not sure if these rules are redundant + r1 = ('mailto:' in url) # TODO not sure if these rules are redundant r2 = ('http://' not in url) and ('https://' not in url) if r1 or r2: @@ -196,12 +202,12 @@ def valid_url(url, verbose=False, test=False): if url_slug and (dash_count > 4 or underscore_count > 4): if dash_count >= underscore_count: - if tld not in [ x.lower() for x in url_slug.split('-') ]: + if tld not in [x.lower() for x in url_slug.split('-')]: if verbose: print('%s verified for being a slug' % url) return True if underscore_count > dash_count: - if tld not in [ x.lower() for x in url_slug.split('_') ]: + if tld not in [x.lower() for x in url_slug.split('_')]: if verbose: print('%s verified for being a slug' % url) return True @@ -232,6 +238,7 @@ def valid_url(url, verbose=False, test=False): if verbose: print('%s caught for default false' % url) return False + def url_to_filetype(abs_url): """ Input a URL and output the filetype of the file @@ -245,8 +252,14 @@ def url_to_filetype(abs_url): path = path[:-1] path_chunks = [x for x in path.split('/') if len(x) > 0] last_chunk = path_chunks[-1].split('.') # last chunk == file usually - file_type = last_chunk[-1] if len(last_chunk) >= 2 else None - return file_type or None + if len(last_chunk) < 2: + return None + file_type = last_chunk[-1] + # Assume that file extension is maximum 5 characters long + if len(file_type) <= 5 or file_type.lower() in ALLOWED_TYPES: + return file_type.lower() + return None + def get_domain(abs_url, **kwargs): """ @@ -257,6 +270,7 @@ def get_domain(abs_url, **kwargs): return None return urlparse(abs_url, **kwargs).netloc + def get_scheme(abs_url, **kwargs): """ """ @@ -264,6 +278,7 @@ def get_scheme(abs_url, **kwargs): return None return urlparse(abs_url, **kwargs).scheme + def get_path(abs_url, **kwargs): """ """ @@ -271,6 +286,7 @@ def get_path(abs_url, **kwargs): return None return urlparse(abs_url, **kwargs).path + def is_abs_url(url): """ this regex was brought to you by django! @@ -285,4 +301,4 @@ def is_abs_url(url): r'(?:/?|[/?]\S+)$', re.IGNORECASE) c_regex = re.compile(regex) - return (c_regex.search(url) != None) + return (c_regex.search(url) is not None) diff --git a/newspaper/utils.py b/newspaper/utils.py index aede89b5..bfa44148 100644 --- a/newspaper/utils.py +++ b/newspaper/utils.py @@ -22,6 +22,8 @@ from hashlib import sha1 +from bs4 import BeautifulSoup + from . import settings log = logging.getLogger(__name__) @@ -29,8 +31,8 @@ class FileHelper(object): - @classmethod - def loadResourceFile(self, filename): + @staticmethod + def loadResourceFile(filename): if not os.path.isabs(filename): dirpath = os.path.abspath(os.path.dirname(__file__)) path = os.path.join(dirpath, 'resources', filename) @@ -47,23 +49,23 @@ def loadResourceFile(self, filename): class ParsingCandidate(object): - def __init__(self, urlString, link_hash): - self.urlString = self.url = urlString + def __init__(self, url, link_hash): + self.url = url self.link_hash = link_hash class RawHelper(object): - @classmethod - def get_parsing_candidate(self, url, raw_html): + @staticmethod + def get_parsing_candidate(url, raw_html): if isinstance(raw_html, str): - raw_html = raw_html.encode('utf-8') + raw_html = raw_html.encode('utf-8', 'replace') link_hash = '%s.%s' % (hashlib.md5(raw_html).hexdigest(), time.time()) return ParsingCandidate(url, link_hash) class URLHelper(object): - @classmethod - def get_parsing_candidate(self, url_to_crawl): + @staticmethod + def get_parsing_candidate(url_to_crawl): # Replace shebang in urls final_url = url_to_crawl.replace('#!', '?_escaped_fragment_=') \ if '#!' in url_to_crawl else url_to_crawl @@ -179,6 +181,29 @@ def onlyascii(char): return True +def extract_meta_refresh(html): + """ Parses html for a tag like: + <meta http-equiv="refresh" content="0;URL='http://sfbay.craigslist.org/eby/cto/5617800926.html'" /> + Example can be found at: https://www.google.com/url?rct=j&sa=t&url=http://sfbay.craigslist.org/eby/cto/ + 5617800926.html&ct=ga&cd=CAAYATIaYTc4ZTgzYjAwOTAwY2M4Yjpjb206ZW46VVM&usg=AFQjCNF7zAl6JPuEsV4PbEzBomJTUpX4Lg + """ + soup = BeautifulSoup(html, 'html.parser') + element = soup.find('meta', attrs={'http-equiv': 'refresh'}) + if element: + try: + wait_part, url_part = element['content'].split(";") + except ValueError: + # In case there are not enough values to unpack + # for instance: <meta http-equiv="refresh" content="600" /> + return None + else: + # Get rid of any " or ' inside the element + # for instance: + # <meta http-equiv="refresh" content="0;URL='http://sfbay.craigslist.org/eby/cto/5617800926.html'" /> + if url_part.lower().startswith("url="): + return url_part[4:].replace('"', '').replace("'", '') + + def to_valid_filename(s): """Converts arbitrary string (for us domain name) into a valid file name for caching @@ -187,7 +212,7 @@ def to_valid_filename(s): return ''.join(c for c in s if c in valid_chars) -def cache_disk(seconds=(86400*5), cache_folder="/tmp"): +def cache_disk(seconds=(86400 * 5), cache_folder="/tmp"): """Caching extracting category locations & rss feeds for 5 days """ def do_cache(function): @@ -223,7 +248,7 @@ def timed(*args, **kw): ts = time.time() result = method(*args, **kw) te = time.time() - print('%r %2.2f sec' % (method.__name__, te-ts)) + print('%r %2.2f sec' % (method.__name__, te - ts)) return result return timed @@ -232,9 +257,9 @@ def chunks(l, n): """Yield n successive chunks from l """ newn = int(len(l) / n) - for i in range(0, n-1): - yield l[i*newn:i*newn+newn] - yield l[n*newn-newn:] + for i in range(0, n - 1): + yield l[i * newn:i * newn + newn] + yield l[n * newn - newn:] def purge(fn, pattern): @@ -310,7 +335,7 @@ def get_useragent(): """ with open(settings.USERAGENTS, 'r') as f: agents = f.readlines() - selection = random.randint(0, len(agents)-1) + selection = random.randint(0, len(agents) - 1) agent = agents[selection] return agent.strip() @@ -322,6 +347,7 @@ def get_available_languages(): two_dig_codes = [f.split('-')[1].split('.')[0] for f in stopword_files] for d in two_dig_codes: assert len(d) == 2 + two_dig_codes.sort() return two_dig_codes @@ -329,25 +355,44 @@ def print_available_languages(): """Prints available languages with their full names """ language_dict = { - 'ar': 'Arabic', - 'ru': 'Russian', - 'nl': 'Dutch', - 'de': 'German', - 'en': 'English', - 'es': 'Spanish', - 'fr': 'French', - 'it': 'Italian', - 'ko': 'Korean', - 'no': 'Norwegian', - 'nb': 'Norwegian (Bokmål)', - 'pt': 'Portuguese', - 'sv': 'Swedish', - 'hu': 'Hungarian', - 'fi': 'Finnish', - 'da': 'Danish', - 'zh': 'Chinese', - 'id': 'Indonesian', - 'vi': 'Vietnamese', + 'ar': 'Arabic', + 'be': 'Belarusian', + 'bg': 'Bulgarian', + 'da': 'Danish', + 'de': 'German', + 'el': 'Greek', + 'en': 'English', + 'es': 'Spanish', + 'et': 'Estonian', + 'fa': 'Persian', + 'fi': 'Finnish', + 'fr': 'French', + 'he': 'Hebrew', + 'hi': 'Hindi', + 'hr': 'Croatian', + 'hu': 'Hungarian', + 'id': 'Indonesian', + 'it': 'Italian', + 'ja': 'Japanese', + 'ko': 'Korean', + 'lt': 'Lithuanian', + 'mk': 'Macedonian', + 'nb': 'Norwegian (Bokmål)', + 'nl': 'Dutch', + 'no': 'Norwegian', + 'pl': 'Polish', + 'pt': 'Portuguese', + 'ro': 'Romanian', + 'ru': 'Russian', + 'sl': 'Slovenian', + 'sr': 'Serbian', + 'sv': 'Swedish', + 'sw': 'Swahili', + 'th': 'Thai', + 'tr': 'Turkish', + 'uk': 'Ukrainian', + 'vi': 'Vietnamese', + 'zh': 'Chinese', } codes = get_available_languages() diff --git a/newspaper/version.py b/newspaper/version.py index 8b41029f..e2eab31a 100644 --- a/newspaper/version.py +++ b/newspaper/version.py @@ -7,5 +7,5 @@ __license__ = 'MIT' __copyright__ = 'Copyright 2014, Lucas Ou-Yang' -version_info = (0, 1, 2) +version_info = (0, 3, 0) __version__ = ".".join(map(str, version_info)) diff --git a/newspaper/videos/extractors.py b/newspaper/videos/extractors.py index ffbcbb54..22e0b048 100644 --- a/newspaper/videos/extractors.py +++ b/newspaper/videos/extractors.py @@ -79,7 +79,7 @@ def get_object_tag(self, node): self.candidates.remove(child_embed_tag[0]) # get the object source - # if wa don't have a src node don't coninue + # if we don't have a src node don't coninue src_node = self.parser.getElementsByTag( node, tag="param", attr="name", value="movie") if not src_node: diff --git a/requirements.txt b/requirements.txt index da49828d..61974601 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,14 @@ -beautifulsoup4==4.3.2 -Pillow==2.6.1 -PyYAML==3.11 -cssselect==0.9.1 -lxml==3.3.5 -nltk==3.0.0 -requests==2.3.0 -six==1.7.3 -feedparser==5.1.3 -tldextract==1.5.1 -feedfinder2==0.0.1 -jieba3k==0.35.1 +beautifulsoup4>=4.4.1 +cssselect>=0.9.2 +feedfinder2>=0.0.4 +feedparser>=5.2.1 +jieba3k>=0.35.1 +lxml>=3.6.0 +nltk>=3.2.1 +Pillow>=3.3.0 +pythainlp>=1.7.2 +python-dateutil>=2.5.3 +PyYAML>=3.11 +requests>=2.10.0 +tinysegmenter==0.3 # TODO(codelucas): Investigate making this >=0.3 +tldextract>=2.0.1 \ No newline at end of file diff --git a/setup.py b/setup.py index 6c5aae18..5569c7cb 100755 --- a/setup.py +++ b/setup.py @@ -1,13 +1,14 @@ #!/bin/python2.7 # -*- coding: utf-8 -*- """ -Lucas Ou 2014 -- http://lucasou.com +Lucas Ou-Yang 2014 -- http://codelucas.com """ import sys import os import codecs + try: from setuptools import setup except ImportError: @@ -20,10 +21,22 @@ if sys.argv[-1] == 'publish': - os.system('python3 setup.py sdist upload') # bdist_wininst + # PYPI now uses twine for package management. + # For this to work you must first `$ pip3 install twine` + os.system('python3 setup.py sdist bdist_wheel') + os.system('twine upload dist/*') sys.exit() +# This *must* run early. Please see this API limitation on our users: +# https://github.com/codelucas/newspaper/issues/155 +if sys.version_info[0] == 2 and sys.argv[-1] not in ['publish', 'upload']: + sys.exit('WARNING! You are attempting to install newspaper3k\'s ' + 'python3 repository on python2. PLEASE RUN ' + '`$ pip3 install newspaper3k` for python3 or ' + '`$ pip install newspaper` for python2') + + with open('requirements.txt') as f: required = f.read().splitlines() @@ -34,7 +47,7 @@ setup( name='newspaper3k', - version='0.1.2', + version='0.3.0', description='Simplified python article discovery & extraction.', long_description=readme, author='Lucas Ou-Yang', diff --git a/tests/data/fulltext_url_list.txt b/tests/data/fulltext_url_list.txt index b8db2777..442689e9 100644 --- a/tests/data/fulltext_url_list.txt +++ b/tests/data/fulltext_url_list.txt @@ -49,9 +49,8 @@ http://www.cleveland.com/books/index.ssf/2014/12/best_books_for_young_readers_e. http://cntraveler.com/galleries/2014-12-08/marrakech-morocco-best-shopping-guide http://cntraveler.com/galleries/2014-12-24/what-the-airlines-did-well-in-2014-american-jetblue-ryanair http://www.coolhunting.com/travel/monastero-santa-rosa-hotel-spa-on-the-almafi-coast -http://coolhunting.com/travel/monastero-santa-rosa-hotel-spa-on-the-almafi-coast http://letterstotheeditorblog.dallasnews.com/2014/12/kindhearted-police-with-holiday-spirit.html/' -http://dallasmorningviewsblog.dallasnews.com/2014/12/footballs-growing-caste-system-as-more-parents-say-no.html/feed/ +http://dallasmorningviewsblog.dallasnews.com/2014/12/footballs-growing-caste-system-as-more-parents-say-no.html http://details.com/watch/black-on-black-watches-for-men http://details.com/fashion-style/perfect-wardrobe/201408/gray-shirts-sweaters-pants-suits-coats-for-fall http://elle.com/runway/ready-to-wear/spring-2015-rtw/nicole-miller/collection/ @@ -60,30 +59,14 @@ http://fool.com/investing/general/2014/12/22/7-top-stocks-to-buy-for-2015.aspx#1 http://fool.com/../../personal-finance/2010/07/30/5-fixes-for-female-money-problems.aspx http://smallbusiness.foxbusiness.com/marketing-sales/2014/12/16/7-powerful-ways-to-convert-neutral-traffic-into-paying-customers/ http://smallbusiness.foxbusiness.com/marketing-sales/2014/12/23/how-to-deal-with-scrooge-client/ -http://foxnews.com/2014/12/23/he-cannot-run-city-new-york-oreilly-rips-incompetent-mayor-de-blasio -http://radio.foxnews.com/2014/12/25/afmw-michael-w-smith-michael-w-smith-friends-the-spirit-of-christmas-video/ -http://huffingtonpost.de/news/missing-airasia-plane/ -http://huffingtonpost.de/news/inspiration -http://reuters.com/article/2014/12/29/uk-google-china-idUKKBN0K70BJ20141229 -http://reuters.com/article/businessNews/idFRKBN0K70N920141229 http://mashable.com/2010/01/01/im-feeling-lucky-google-new-year/ http://mashable.com/2014/12/22/stansted-power-outage/ http://www.oregonlive.com/business/index.ssf/2014/12/alzheimers_foundation_comcast.html http://www.oregonlive.com/pacific-northwest-news/index.ssf/2014/12/missionary_from_oregon_pleads.html#comments -http://nydailynews.com/autos/news -http://www.nydailynews.com/photos/news http://www.nola.com/jazzfest/index.ssf/2014/12/multi-million-dollar_george_an.html -http://www.nola.com/business/index.ssf/2014/11/new_orleans_business_names_in_20.html -http://nj.com/news/article/3875334678772745563/bayonne-46-at-st-mary-eliz-77-in-season-tournament-boys-basketball/#comments -http://videos.nj.com/2013/09/raw_footage_seaside_boardwalk.html http://www.newyorker.com/magazine/2014/12/22/climate-change-christmas-carols -http://video.newyorker.com/watch/annals-of-sound-at-home-with-meredith-monk -http://readwrite.com/2014/12/29/microsoft-browser-spartan-internet-explorer-chrome -http://readwrite.com/2014/12/29/pinterest-opens-up-promoted-pins http://www.slate.com/articles/technology/the_back_end/2014/10/the_new_seven_wonders_defining_the_top_technological_marvels_of_the_contemporary.html http://www.slate.com/blogs/the_spot/2014/06/10/brazil_world_cup_protests_striking_images_of_the_country_s_anti_world_cup.html -https://deals.thenextweb.com/sales/free-anime-for-30-days-from-crunchyroll?utm_campaign=feed&utm_medium=RSS&utm_source=thenextweb -http://thenextweb.com/apps/2014/12/12/google-maps-finally-adds-lane-guidance-confused-european-drivers/feed/ http://www.thekitchn.com/before-after-the-details-make-a-difference-reader-kitchen-remodel-214061#comments http://www.thekitchn.com/drink-up-15-good-ideas-for-beer-and-wine-the-kitchns-best-of-2014-214146#comments http://www.thedailybeast.com/articles/2014/12/16/the-vatican-s-rare-nod-to-american-nuns.html @@ -92,8 +75,6 @@ http://theatlantic.com/entertainment/archive/2014/12/cracking-the-sitcom-code/38 http://www.theatlantic.com/entertainment/archive/2014/12/the-interview-is-a-pretty-smart-movie/384082/#disqus_thread http://theatlanticcities.com/crime/2014/12/tourism-to-antarctica-could-be-making-penguins-sick/383979/#disqus_thread http://theatlanticcities.com/weather/2014/11/a-climate-model-of-white-christmasesthrough-2100/383143/#disqus_thread -http://theonion.com/articles/i-dont-see-race-i-only-see-grayishbrown-vaguely-hu,37667/ -http://theonion.com/video/greyhound-now-offering-premium-upgrade-to-slightly,37261/ http://upworthy.com/what-a-service-dog-really-thinks-of-the-injured-vet-who-owns-him?c=tpstream http://www.upworthy.com/theyre-innocent-looking-enough-but-actually-quite-dangerous http://www.wired.com/2014/12/butterflies-get-shine/ @@ -102,10 +83,6 @@ http://www.wetpaint.com/the-bachelor/articles/2014-11-04-whitney-bischoff-elimin http://www.wetpaint.com/the-bachelor/articles/2014-12-29-alissa-giambrone-contestant-season-19 http://vogue.de/beauty/beauty-blog/lieblingsstueck-die-iphone-huelle-von-iphoria http://vogue.de/mode/mode-trends/weihnachten-die-redaktion-zeigt-ihre-christmas-sweater -http://www.vogue.com/slideshow/3619755/ -http://vogue.com/collection/fallwinter2014/ready-to-wear/ -http://vanityfair.com/show/cinema/14/12/18/baci-piu-belli-del-2014-film-serie-tv -http://vanityfair.com/show/tv/14/12/22/serie-tv-coppie-piu-belle-del-2014 http://www.pewresearch.org/fact-tank/2014/11/20/those-from-mexico-will-benefit-most-from-obamas-executive-action/ http://www.pewresearch.org/fact-tank/2014/12/16/gas-prices-consumer-sentiment/ http://backstage.com/casting/coney-island-55854/actors-208609/ @@ -115,31 +92,25 @@ http://www.syracuse.com/living/index.ssf/2014/12/echos_of_the_past_the_hotel_syr http://globalnews.ca/video/1747657/video-of-the-gardners-hours-before-birth-of-quadruplets http://globalnews.ca/video/1747659/quad-squad-born-in-utah/ http://uproxx.com/webculture/2014/12/a-reddit-post-about-a-fake-ebola-like-illness-sparked-a-panic-in-and-around-an-arizona-town/ -http://uproxx.com/filmdrunk/2014/12/simon-pegg-gave-his-blessing-for-a-touring-shaun-of-the-dead-stage-show/feed/ +http://uproxx.com/filmdrunk/2014/12/simon-pegg-gave-his-blessing-for-a-touring-shaun-of-the-dead-stage-show http://dailycaller.com/2014/12/25/happy-kwanzaa-the-holiday-brought-to-you-by-the-fbi/ http://dailycaller.com/2014/12/27/jennifer-lawrence-made-an-appearance-at-the-louisville-kentucky-flop-fest-video/ -http://pe.com/los-angeles-CA/legal/legal-services/The-Law-Offices-of-Jacob-Emrani-951-268-7555 -http://pe.com/sections/news/riverside-county/lake-elsinore/ http://avclub.com/article/best-film-scenes-2014-212903 http://avclub.com/article/finally-mail-kimp-remix-you-never-knew-you-wanted-212941 http://www.usnews.com/news/blogs/run-2016/2014/10/31/tracking-the-2016-presidential-candidates http://video.usnews.com/Cigar-rollers-in-Havana-thrilled-with-warming-USCuba-policy-28285271 -http://media.talkingpointsmemo.com/slideshow/astronomy-photographer-competition-2013 -http://talkingpointsmemo.com/muckraker/mark-shurtleff-john-swallow-wildest-details http://glamour.com/fashion/blogs/dressed/2014/09/rihanna-leaves-pants-at-home-f http://glamour.com/weddings/blogs/save-the-date/2014/12/simple-wedding-dresses http://blog.gulflive.com/mississippi-press-living/2014/12/sampling_history_new_year_offe.html http://videos.gulflive.com/2013/10/crown_equity_holdings_could_be.html -http://247wallst.com/investing/2014/12/29/the-4-stocks-that-slimmed-the-djia-monday-slide/feed/ -http://247wallst.com/investing/2014/12/29/the-52-week-low-club-for-monday-19/feed/ +http://247wallst.com/investing/2014/12/29/the-4-stocks-that-slimmed-the-djia-monday-slide +http://247wallst.com/investing/2014/12/29/the-52-week-low-club-for-monday-19 http://flavorwire.com/495982/13-media-moments-from-2014-we-could-have-lived-without#comments http://flavorwire.com/495418/2014-the-year-the-outrage-machine-started-eating-the-real-world http://www.businessweek.com/articles/2014-12-18/afghanistans-new-rich-navigate-u-dot-s-dot-pullout#r=nav-fst http://www.businessweek.com/articles/2014-12-22/cuomos-cap-on-hedge-funds-in-pensions-wont-solve-the-real-problem#r=nav-f-story http://www.bloomberg.com/news/2014-09-30/job-woes-linger-in-29-states-as-u-s-recovers-unevenly.html -http://bloomberg.com/infographics/2014-08-22/ukraine-russia-map.html http://www.theroot.com/articles/culture/2014/12/rev_sharpton_responds_to_his_critics.html -http://theroot.com/list/The-Root-100/video/The-Root-100-Panel-Discussion-2 http://livescience.com/48502-magic-mushrooms-change-brain-networks.html http://livescience.com/48543-how-zombies-evolved-in-pop-culture.html http://lifebuzz.com/colbie-caillat-got-tired-of-being-photoshopped-so-heres-what-she-did-about-it/ @@ -153,10 +124,9 @@ http://tnr.com/article/120578/global-warming-threshold-what-2-degrees-celsius-36 http://space.com/47-mars-the-red-planet-fourth-planet-from-the-sun.html http://space.com/27998-nasa-18-billion-omnibus-spending-bill.html http://adoption.com/what-not-to-say-to-a-birth-mom-or-adoptee/ -http://forums.adoption.com/search-birthfamily-adoptee/222937-if-you-looking-siblings-please-post-here.html http://youbeauty.com/skin/our-favorite-pop-culture-nail-art http://youbeauty.com/face/we-tried-it-clinique-acne-solutions-powder-makeup-in-golden -http://www.sitepoint.com/average-page-weight-increases-15-2014/feed/ +http://www.sitepoint.com/average-page-weight-increases-15-2014 http://www.sitepoint.com/3-ways-implement-embeddable-custom-badges/ http://www.technologyreview.com/featuredstory/532796/who-owns-the-biggest-biotech-discovery-of-the-century/ http://www.technologyreview.com/news/532896/discarded-laptop-batteries-keep-the-lights-on/ @@ -164,3 +134,12 @@ http://nypost.com/2014/12/18/amazon-macmillan-make-peace-over-book-pricing/ http://nypost.com/2014/12/29/surfer-survives-jaws-moment-with-great-white-shark/ http://bhg.com/thanksgiving/indoor-decorating/centerpiece-and-tabletop-decoration-ideas-fall/ http://www.bhg.com/videos/m/93269792/how-to-hang-a-christmas-wreath-two-no-fail-secrets.htm +http://www.huffingtonpost.com/2015/01/14/jamie-dimon-banks-assault_n_6470358.html +http://www.nydailynews.com/new-york/brooklyn/boyfriend-mob-wives-star-sue-brooklyn-club-article-1.2078107 +http://www.nj.com/politics/index.ssf/2015/01/conservative_nj_senator_rips_christie_over_state_o.html +http://readwrite.com/2015/01/13/apple-app-charlie-hebdo-mark-fiore-cartoons +http://thenextweb.com/insider/2015/01/14/blackberry-denies-samsung-buyout-rumors/ +http://www.theonion.com/articles/womans-parents-accepting-of-mixedattractiveness-re,37765/ +http://www.vanityfair.com/vf-hollywood/2015/01/gwyneth-paltrow-brad-pitt-ben-affleck-break-ups +http://www.pe.com/articles/perris-758171-police-exchange.html +http://talkingpointsmemo.com/ts/why-is-america-ignoring-kurdish-freedom-movement diff --git a/tests/data/html/247wallst.com1.html b/tests/data/html/247wallst.com1.html index 239b58fa..9dfc3cd5 100644 --- a/tests/data/html/247wallst.com1.html +++ b/tests/data/html/247wallst.com1.html @@ -1,19 +1,1246 @@ -<?xml version="1.0" encoding="UTF-8"?><rss version="2.0" - xmlns:content="http://purl.org/rss/1.0/modules/content/" - xmlns:dc="http://purl.org/dc/elements/1.1/" - xmlns:atom="http://www.w3.org/2005/Atom" - xmlns:sy="http://purl.org/rss/1.0/modules/syndication/" - xmlns:georss="http://www.georss.org/georss" xmlns:geo="http://www.w3.org/2003/01/geo/wgs84_pos#" xmlns:media="http://search.yahoo.com/mrss/" +<!DOCTYPE html> +<!--[if IE 6]> +<html id="ie6" class="no-js ie6" xmlns:fb="http://ogp.me/ns/fb#" + lang="en"> +<![endif]--> +<!--[if IE 7]> +<html id="ie7" class="no-js ie7" xmlns:fb="http://ogp.me/ns/fb#" + lang="en"> +<![endif]--> +<!--[if IE 8]> +<html id="ie8" class="no-js ie8" xmlns:fb="http://ogp.me/ns/fb#" + lang="en"> +<![endif]--> +<!--[if IE 9]> +<html id="ie9" class="no-js ie9" xmlns:fb="http://ogp.me/ns/fb#" + lang="en"> +<![endif]--> +<!--[if !(IE 6) | !(IE 7) | !(IE 8) | !(IE 9) ]><!--> +<html class="no-js" xmlns:fb="http://ogp.me/ns/fb#" + lang="en"> +<!--<![endif]--> +<head profile="http://gmpg.org/xfn/11"> +<title> The 4 Stocks That Slimmed the DJIA Monday Slide - Home Depot (NYSE:HD) - 24/7 Wall St. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+ +
+ +
+

The 4 Stocks That Slimmed the DJIA Monday Slide

+

+ By + +

+
+ + Print + + + + + + Email + + + + + +
+
+ + +
+
+ +
+
+ +
+ + + +
+
+
+ + +
+ + +

December 28, 2014: Markets opened lower on Monday as neither the bulls nor the bears can gin up a lot of support. Volumes are low and corporate news is nearly non-existent. Crude oil started the day higher, but has drifted to a loss of about 2% as the day wore on. Shortly before the closing bell the DJIA traded down 0.08% for the day, the S&P 500 traded up 0.09%, and the Nasdaq Composite traded flat. International Business Machines Corp. (NYSE: IBM) dropped more than 1% today and its high per share price pushed the DJIA lower just as the market closed.

+ +
+ +

The DJIA stock posting the largest daily gain ahead of the close Monday was The Home Depot Inc. (NYSE: HD) which traded up 0.87% at $104.64. The stock’s 52-week range is $73.96 to $104.80, and the high was posted early Monday. Trading volume was less than half the daily average of around 5.8 million shares. Reports of solid sales at home improvement stores during the holidays have helped push this stock higher today.

+

JPMorgan Chase & Co. (NYSE: JPM) traded up 0.77% at $63.03. The stock’s 52-week range is $52.97 to $63.34, and the high was set today. Trading volume was about 50% below the daily average of around 26 million shares. The big bank is one of the underwriters for the IPO of Shake Shack which was announced today.

+

The Boeing Co. (NYSE: BA) traded higher by 0.71% at $132.57. The stock’s 52-week range is $116.32 to $144.57. Volume was less than half the daily average of around 4.4 million shares. The company held its first test flight on Sunday of a new Air Force Tanker.

+

Cisco Systems Inc. (NASDAQ: CSCO) traded up 0.48% at $28.49. The stock’s 52-week range is $21.27 to $28.59. Trading volume about 60% below the daily average of around 29 million shares. The stock missed matching its 52-week high by just a couple of pennies and was touted as a Rocket Stock at thestreet.com today.

+

Of the Dow 30 stocks 13 are set to close higher today and 17 are on track to close lower.

+

ALSO READ: 5 Top Tech Stocks for 2015 With Potential Big Catalysts

+

By Paul Ausick

+ + +
+ +
+ + +
+ + RSS + + Facebook + + Twitter +
+ + +
+ + + +Yahoo BuzzBuzz up! + + +DiggDigg + + +farkFark + + +facebookFacebook + + + + tipdReddit + + + + + tumblrTumblr + + + +
+ + + + +
+ Read more: Investing, , , , , +
+ +
+ + + + +
+
+ +
+ +
+
+ +
+ + +
+
+ + + comments powered by Disqus +
+ +
+
+
+ +
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+ + + + + + + + + + +
+ +
+ + + + + +
+ +
+ + + + + + + + + + + + + diff --git a/tests/data/html/247wallst.com2.html b/tests/data/html/247wallst.com2.html index d9650433..58252398 100644 --- a/tests/data/html/247wallst.com2.html +++ b/tests/data/html/247wallst.com2.html @@ -1,19 +1,1251 @@ - + + + + + + + + + + The 52-Week Low Club for Monday - VIVUS, Inc. (NASDAQ:VVUS) - 24/7 Wall St. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+ +
+ +
+

The 52-Week Low Club for Monday

+

+ By + +

+
+ + Print + + + + + + Email + + + + + +
+
+ + +
+
+ +
+
+ +
+ + + +
+
+
+ + +
+ + +

December 28, 2014: Here are four stocks among the 66 equities making new 52-week lows today. Volumes continue to be low as we wait to ring in the new year.

+ +
+ +

Transocean Partners LLC (NYSE: RIGP) posted a new 52-week — and all-time — low on Monday of $13.18. Based on Friday night’s closing price of $14.32 that’s a drop of nearly 8%. The stock’s 52-week high is $29.43. Volume is about a third higher than the daily average of around 325,000 shares. The offshore drilling firm had no specific news today, but crude prices are down about 2%.

+

SandRidge Mississippian Trust II (NYSE: SDR) dropped about 5.3% on Monday to post a new 52-week — and all-time — low of $3.91 after closing at $4.13 on Wednesday. The stock’s 52-week high is $10.12. Share volume is nearly double the daily average of 400,000 shares traded. The royalty company had no specific news today but is feeling pressure from low crude prices.

+

Vivus Inc. (NASDAQ: VVUS) dropped about 6% Monday to post a new 52-week low of $2.78. The stock’s 52-week high is $9.80. Volume totaled 25% higher than the daily average of around 1.9 million shares. The U.S. FDA approved a competing diabetes drug last week to compete with Vivus’s Qsymia treatment.

+

Zulilly Inc. (NASDAQ: ZU) dropped about 3.8% Monday to establish a new 52-week — and all-time — low at $22.39 against a high of $73.50. Volume was 25% below the daily average of around 1.4 million shares. The online retailer had no specific news today.

+

ALSO READ: States Where People Live Longest

+

By Paul Ausick

+ + +
+ +
+ + +
+ + RSS + + Facebook + + Twitter +
+ + +
+ + + +Yahoo BuzzBuzz up! + + +DiggDigg + + +farkFark + + +facebookFacebook + + + + tipdReddit + + + + + tumblrTumblr + + + +
+ + + + +
+ Read more: Investing, , +
+ +
+ + + + +
+
+ +
+ +
+
+ +
+ + +
+
+ + + comments powered by Disqus +
+ +
+
+
+ +
+
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + +
+
+ + + + + + + + + + +
+ +
+ + + + + +
+ +
+ + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/data/html/about.com1.html b/tests/data/html/about.com1.html index 0fd9d9c4..075de2a3 100644 --- a/tests/data/html/about.com1.html +++ b/tests/data/html/about.com1.html @@ -7,9 +7,9 @@ data-gs="bandb" data-template-type="FLEXIBLEARTICLE" itemscope itemtype="http://schema.org/Article"> @@ -179,7 +179,7 @@ environment : "prod", application : "globe", dataCenter : "sj", -serverName : "sjglobe7" +serverName : "sjglobe5" }, server : { version : "1.17.4", @@ -188,8 +188,8 @@ }, resources : { version : "1.17.5", -loadStartTime : "1419289077663", -loadTimeTaken : "504" +loadStartTime : "1419289077537", +loadTimeTaken : "512" }, client : { browserUA : navigator.userAgent, @@ -1062,11 +1062,11 @@
Readers Recommend
diff --git a/tests/data/html/about.com2.html b/tests/data/html/about.com2.html index 9c515327..a44dffc8 100644 --- a/tests/data/html/about.com2.html +++ b/tests/data/html/about.com2.html @@ -1,15 +1,15 @@ @@ -154,12 +154,6 @@ bucketTrackingId:"99", bucketValue:2 },{ -testName:"internalMarketingTest", -bucketDescription:"control", -bucketName:"control", -bucketTrackingId:"28", -bucketValue:0 -},{ testName:"lock728Test", bucketDescription:"in test, lock 728 after fixed mast appears", bucketName:"lock728", @@ -185,7 +179,7 @@ environment : "prod", application : "globe", dataCenter : "sj", -serverName : "sjglobe6" +serverName : "sjglobe0" }, server : { version : "1.17.4", @@ -194,8 +188,8 @@ }, resources : { version : "1.17.5", -loadStartTime : "1419289075467", -loadTimeTaken : "344" +loadStartTime : "1419289074752", +loadTimeTaken : "456" }, client : { browserUA : navigator.userAgent, @@ -236,7 +230,6 @@ var ab_tracking_ids = []; ab_tracking_ids.push("99"); - ab_tracking_ids.push("28"); ab_tracking_ids.push("99"); ab_tracking_ids.push("99"); ab_tracking_ids.push("99"); diff --git a/tests/data/html/adoption.com1.html b/tests/data/html/adoption.com1.html index f219d9a3..d31ab249 100644 --- a/tests/data/html/adoption.com1.html +++ b/tests/data/html/adoption.com1.html @@ -148,101 +148,209 @@ -
  • How To Adopt +
  • Parent Profiles℠
    • -
  • Reunion +
  • How to Adopt +
    • +
  • Reunion +
  • Articles
  • Community
  • Photolisting +
  • Reviews +
  • Wiki +
    • @@ -285,7 +393,7 @@
      @@ -574,5 +682,5 @@

      Comments

      host: www6
      +
      host: www1
      diff --git a/tests/data/html/adoption.com2.html b/tests/data/html/adoption.com2.html deleted file mode 100644 index f5cf9ea9..00000000 --- a/tests/data/html/adoption.com2.html +++ /dev/null @@ -1,4092 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - If You Are Looking For Siblings Please Post Here!!!! - Adoption.com Forums - - - - - - - - - - - - - - - -
      - - - - - - - - -
      -
      -
      -
      -
      -
      - - - - -

      Adoption Forums®

      - - - - - - - - - - - - - - - - - - - - - - - -
      - - - - - - - - - - - - - - - - - - - - - - - - - - -
      Members ListPhotosEventsLocal Adoption SupportSearchArcadeReviewsMembership Upgrade
      -
      - - - - - - - - -
      -
      - -
      -
      - - - - - - - - - - - - - - - - - - - - - - - -
      Welcome to the Forums. - -
      - If this is your first visit, be sure to check out the FAQ. You may have to register before you can post or search: click here to proceed. To start viewing messages, select a forum below that you would like to view or click View All of Todays Posts. -
      - - - - - - - - - - - - - - - - - - - - - - - -
      -Forum Categories - - - -
      - -
      - -       		- -
      - -
      - - - - - - - - - - - - - - - - -
      Reply
      - - - - - - - - - - - -
      Page 1 of 57123451151>Last »
      -
      - - - - - - - - - - - - - - - - - - -
      -
      - -   -
      -       		- Thread Tools - - - Search this Thread - - - Display Modes - -
      - - - - - - - - - - - -
      - - -
      - - - - - - - - - - - - - - -
      -
      -   - #1   - - - - -
      - -
      - - Old - - 11-04-2005, 08:23 PM - - - -
      -
      - - - - - - - - -
      - - - - - southemissouri's Avatar - - - -
      - - - - - - - southemissouri - - southemissouri is offline - - - - -
      - - -
      Senior Member
      - - - -       		- - - -
      - -       		- -
      -
      Join Date: Dec 2004
      - - -
      - Posts: 175 - -
      - - Total Points: 160,518.32
      - Donate
      - - - -
      - -
      - - -
      - -
      - -
      - - - - -
      - - If You Are Looking For Siblings Please Post Here!!!! -
      -
      - - - - -
      Thought it might aid in someones search if there was a thread for siblings searching!
      - - - - - - - - -
      - - - - - Reply With Quote - - - - -
      - - - -
      - - - - - - - - - -
      - - -
      - - - - - - -
      -
      		 -
      -
      Adoption Reunion Information
      -
      Premium Membership
      -
      -
      - Become an adoption forums premium member to enjoy these Membership Benefits: -
        -
      • Remove Advertising -
      • Unlimited Arcade -
      • Unlimited Attachments -
      • Increased PM Storage -
      • Calendar Posting -
      • Larger Avatars -
      • Personal Page -
      • Just $19.95 / yr! -
      - -
      -
      -
      Reunion Websites
      -
      -Adoption Registry
      -Reunion Registries
      -Birthfamily Search Blogs
      -Adoption Reunion Encyclopedia
      -Adoption Search Articles
      -Search & Reunion
      -Reunite
      -Adoption Reunion
      -Reunion Registry Definition
      -
      -
      -
      -       		- - - -
      -
      Find Your Birth Family
      Washington
      -
      - - - - -
      - - -
      - Click here to visit Birth Family Finders -
      -
      -Our association with investigators and researchers nationwide allows us to find missing birth family members. We're fast, affordable and all investigations are handled by a personal search specialist with your privacy in mind.
      -
      -
      Birth Family Finders
       
      - - -
      - -
      - -
      - -
      - - -
      - - - - - - - - -
      -
      -

      - - -
      - - - - - - - - - - - - - - -
      -
      -   - #2   - - - - -
      - -
      - - Old - - 11-04-2005, 08:31 PM - - - -
      -
      - - - - - - - - -
      - - - - - doughton's Avatar - - - -
      - - - - - - - doughton - - doughton is offline - - - - -
      - - -
      Sister
      - - - -       		- - - -
      - -       		- -
      -
      Join Date: May 2005
      - - -
      - Posts: 158 - -
      - - Total Points: 12,317.28
      - Donate
      - - - -
      - -
      - - -
      - -
      - -
      - - - - - -
      This thread is a good idea! I am looking for bio sister born in VA, Norfolk. Mother went to unwed mothers home and named baby girl at birth, started with a "D". She was adopted through Chesapeake Dept. of social services. Mother was from Chesapeake area. If any of this sounds familiar please contact me. Year of birth: 1967-68. Jennie aka doughton
      - - - - - - - - -
      - - - - - Reply With Quote - - - - -
      - - - -
      - - - - - - - - - -
      - - - - - -
      - - - - - - - - - - - - - - -
      -
      -   - #3   - - - - -
      - -
      - - Old - - 11-04-2005, 08:55 PM - - - -
      -
      - - - - - - - - -
      - - - - - southemissouri's Avatar - - - -
      - - - - - - - southemissouri - - southemissouri is offline - - - - -
      - - -
      Senior Member
      - - - -       		- - - -
      - -       		- -
      -
      Join Date: Dec 2004
      - - -
      - Posts: 175 - -
      - - Total Points: 160,518.32
      - Donate
      - - - -
      - -
      - - -
      - -
      - -
      - - - - - -
      Thanks and good luck in your search. I have found that many mothers from an era do not actively search for various reasons but have made mention of their placed children to their other children or other family members. I am currently searching for a birth cousin (I know birth mother and father and half siblings names) and aiding a friend in the quest of her birth family. Without a name of either mother or father it is much like searching for a needle in a haystack! Thus the ideal of this thread was created.
      -
      -
      -My paternal male cousin was born May 24, 1962 in Hayti, Missouri. Possibly placed in Illinois, the Chicago area. His birth mother's name is Elain M. Strock or Strack. She had 2 older sons, Ricky and Garry Steigman.The birth father, my uncle, is Thelbert (Peck) Cole. He has 2 younger children.
      -
      -My female friend was born in St Louis, MO on August 4, 1963 at Bethseda Hospital. She was placed at the Children's home Society and the adoption was finalized within the City of St. Louis. She grew up in rural Missouri.
      - - - - - - - - -
      - - Last edited by southemissouri : 11-04-2005 at 08:58 PM. - - -
      - - - -
      - - - - - Reply With Quote - - - - -
      - - - -
      - - - - - - - - - -
      - - - - - -
      - - - - - - - - - - - - - - -
      -
      -   - #4   - - - - -
      - -
      - - Old - - 11-04-2005, 09:01 PM - - - -
      -
      - - - - - - - - -
      - - - -
      - - - - - - - Tim Wright - - Tim Wright is offline - - - - -
      - - -
      Junior Member
      - - - -       		- - - -
      - -       		- -
      -
      Join Date: Nov 2005
      - - -
      - Posts: 1 - -
      - - Total Points: 567.00
      - Donate
      - - - -
      - -
      - - -
      - -
      - -
      - - - - -
      - - Looking for adopted brother -
      -
      - - - - -
      -
      Quote:
      - - - - -
      - -
      Originally Posted by southemissouri
      -
      Thought it might aid in someones search if there was a thread for siblings searching!
      - -
      -
      Looking for our brother, born November 1967 in Southern New Jersey, believed to have been adopted by family in New York, born Michael Lamont Jones, mothers name Lynnette Jones, father, Theodore Muse, you have 2 siblings, (brothers) longing to meet you
      - - - - - - - - -
      - - - - - Reply With Quote - - - - -
      - - - -
      - - - - - - - - - -
      - - - - - -
      - - - - - - - - - - - - - - -
      -
      -   - #5   - - - - -
      - -
      - - Old - - 11-04-2005, 09:06 PM - - - -
      -
      - - - - - - - - -
      - - - - - ravynk's Avatar - - - -
      - - - - - - - ravynk - - ravynk is offline - - - - -
      - - -
      Junior Member
      - - - -       		- - - -
      - -       		- -
      -
      Join Date: Nov 2005
      - - -
      - Posts: 4 - -
      - - Total Points: 1,499.77
      - Donate
      - - - -
      - -
      - - -
      - -
      - -
      - - - - -
      - Talking - Looking for Timothy Burnnet -
      -
      - - - - -
      Hey all...I decided it's about time I restarted my search for my bio Brother.
      -Timothy Burnnet was born in June of 1957 to Shirley and Melbourne Burnnet. Mel was in the Navy..posted to the naval yard in Bremerton, Washington. In June of 1958 I was born..but it looks like mom and dad were divorced by then.
      -
      -If anyone has any information about any of the above people please reply.
      -
      -Marlaine (aka Kathy Burnnet)
      - - - - - - - - -
      - - - - - Reply With Quote - - - - -
      - - - -
      - - - - - - - - - -
      - - - - - -
      - - - - - - - - - - - - - - -
      -
      -   - #6   - - - - -
      - -
      - - Old - - 11-05-2005, 11:05 PM - - - -
      -
      - - - - - - - - -
      - - - -
      - - - - - - - lauralee3 - - lauralee3 is offline - - - - -
      - - -
      Member
      - - - -       		- - - -
      - -       		- -
      -
      Join Date: Jan 2005
      - - -
      - Posts: 65 - -
      - - Total Points: 6,000.29
      - Donate
      - - - -
      - -
      - - -
      - -
      - -
      - - - - - -
      Looking for birth sister named Lynn, born on 11.4.59 in Jersey City, NJ.
      - - - - - - - - -
      - - - - - Reply With Quote - - - - -
      - - - -
      - - - - - - - - - -
      - - - - - -
      - - - - - - - - - - - - - - -
      -
      -   - #7   - - - - -
      - -
      - - Old - - 11-06-2005, 06:21 AM - - - -
      -
      - - - - - - - - -
      - - - -
      - - - - - - - LisaD76 - - LisaD76 is offline - - - - -
      - - -
      Junior Member
      - - - -       		- - - -
      - -       		- -
      -
      Join Date: Apr 2005
      - - -
      - Posts: 46 - -
      - - Total Points: 9,218.14
      - Donate
      - - - -
      - -
      - - -
      - -
      - -
      - - - - - -
      I'm looking for my birth sister. At birth her name was Terry. She was born in either 1974 or early 1975 in upstate New York and placed for adoption in Galion, Ohio in late 1976. Birth mother's name was Prema Dizenberger (Moleski) and birth father's name is Joseph Moleski.
      - - - - - - - - -
      - - - - - Reply With Quote - - - - -
      - - - -
      - - - - - - - - - -
      - - - - - -
      - - - - - - - - - - - - - - -
      -
      -   - #8   - - - - -
      - -
      - - Old - - 11-06-2005, 08:43 AM - - - -
      -
      - - - - - - - - -
      - - - -
      - - - - - - - Nanners320 - - Nanners320 is offline - - - - -
      - - -
      Junior Member
      - - - -       		- - - -
      - -       		- -
      -
      Join Date: Jan 2005
      - - -
      - Posts: 40 - -
      - - Total Points: 4,731.14
      - Donate
      - - - -
      - -
      - - -
      - -
      - -
      - - - - -
      - - Looking for brother -
      -
      - - - - -
      Hi,
      -
      -I'm looking for my brother Nolan Morgan, born 8/5/82 in Grand Junction, Colorado. We lived in Delta, Colorado for a while. I'm using my new found birth aunts address. If anyone knows anything about Nolan please let me know.
      - - - - - - - - -
      - - - - - Reply With Quote - - - - -
      - - - -
      - - - - - - - - - -
      - - -
      -
      -
      -
      -

      - - -
      - - - - - - - - - - - - - - -
      -
      -   - #9   - - - - -
      - -
      - - Old - - 11-06-2005, 05:50 PM - - - -
      -
      - - - - - - - - -
      - - - - - mamaglo2's Avatar - - - -
      - - - - - - - mamaglo2 - - mamaglo2 is offline - - - - -
      - - -
      Junior Member
      - - - -       		- - - -
      - -       		- -
      -
      Join Date: Aug 2004
      - - -
      - Posts: 16 - -
      - - Total Points: 3,515.41
      - Donate
      - - - -
      - -
      - - -
      - -
      - -
      - - - - -
      - - looking for my brothers -
      -
      - - - - -
      iam looking for my brothers james franklin chapman born 04-03-1960 in catawba county nc.jimmy wayne chapman born 65 or 66 unsure of the state on jimmy.they were left in beaumont tx in 1970.found out that texas department of family and protective services handled the adoption.they were adopted into the same family but at different times.james franklin (franky) chapman birthcertificate was pulled 6-27-73 in catawba county nc.
      - - - - - - - - -
      - - - - - Reply With Quote - - - - -
      - - - -
      - - - - - - - - - -
      - - - - - -
      - - - - - - - - - - - - - - -
      -
      -   - #10   - - - - -
      - -
      - - Old - - 11-06-2005, 09:36 PM - - - -
      -
      - - - - - - - - -
      - - - -
      - - - - - - - GMMalone - - GMMalone is offline - - - - -
      - - -
      Junior Member
      - - - -       		- - - -
      - -       		- -
      -
      Join Date: Nov 2005
      - - -
      - Posts: 2 - -
      - - Total Points: 1,153.00
      - Donate
      - - - -
      - -
      - - -
      - -
      - -
      - - - - - -
      Looking for a brother my mom gave up in April between 1969 and 1971. He was born in Santa Fe, NM. That is all I know.
      - - - - - - - - -
      - - - - - Reply With Quote - - - - -
      - - - -
      - - - - - - - - - -
      - - - - - -
      - - - - - - - - - - - - - - -
      -
      -   - #11   - - - - -
      - -
      - - Old - - 11-07-2005, 08:36 AM - - - -
      -
      - - - - - - - - -
      - - - -
      - - - - - - - shadeinfla - - shadeinfla is offline - - - - -
      - - -
      Senior Member
      - - - -       		- - - -
      - -       		- -
      -
      Join Date: Jun 2004
      - - -
      - Posts: 448 - -
      - - Total Points: 35,977.40
      - Donate
      - - - -
      - -
      - - -
      - -
      - -
      - - - - -
      - - ISO 8 Half Sibilngs -
      -
      - - - - -
      Great idea ,, looking for my husband's eight half sibings , two on his b fathers side both sons born about 1968 and 1970, six on his b mom's side 4 sons 2 daughters born between approximatley 1968-1975(give or take a year),,,please read link for more information ... by the way , of the six children his b mom had after him , she herself has adopted some of them in ( not out like some have thought) ,,,, Good Luck to all searching
      -
      -
      -http://hometown.aol.com/shadechase/m...age/index.html
      -
      -XO
      -Shade
      - - - - - - -
      - __________________
      - "i hope you don't mind, I hope you don't mind, that I put down in words, how wonderful life is ,while your in this world" Bernie Taupin 1970 -
      - - - - - -
      - - Last edited by shadeinfla : 11-07-2005 at 08:41 AM. - - -
      - - - -
      - - - - - Reply With Quote - - - - -
      - - - -
      - - - - - - - - - -
      - - - - - -
      - - - - - - - - - - - - - - -
      -
      -   - #12   - - - - -
      - -
      - - Old - - 11-07-2005, 09:21 AM - - - -
      -
      - - - - - - - - -
      - - - - - Hawkwoman's Avatar - - - -
      - - - - - - - Hawkwoman - - Hawkwoman is offline - - - - -
      - - -
      Junior Member
      - - - -       		- - - -
      - -       		- -
      -
      Join Date: Dec 2004
      - - -
      - Posts: 17 - -
      - - Total Points: 2,544.00
      - Donate
      - - - -
      - -
      - - -
      - -
      - -
      - - - - -
      - - Looking for Casey Rae -
      -
      - - - - -
      I am looking for my bdaughter. She was born in 1988 Delaware. She was sent to a couple in Arkansas with another adopted daughter. Her bfather and I would like to find her.
      -
      -Angela Moore
      - - - - - - - - -
      - - - - - Reply With Quote - - - - -
      - - - -
      - - - - - - - - - -
      - - - - - -
      - - - - - - - - - - - - - - -
      -
      -   - #13   - - - - -
      - -
      - - Old - - 11-07-2005, 09:32 AM - - - -
      -
      - - - - - - - - -
      - - - -
      - - - - - - - TeresaKornmille - - TeresaKornmille is offline - - - - -
      - - -
      Baby Girl
      - - - -       		- - - -
      - -       		- -
      -
      Join Date: Nov 2005
      - - -
      - Posts: 1 - -
      - - Total Points: 526.00
      - Donate
      - - - -
      - -
      - - -
      - -
      - -
      - - - - -
      - Question - Girl born April 6, 1966 Cumberland County, Fayetteville NC. -
      -
      - - - - -
      Am looking for my 5 other siblings and my birth parents. German-Irish decent. B-mother was 26 years old when I was born. B-Father was in the military. They were married in 1955. Grandfather worked for the railroad. B-mother was from Pa, B-father was from Ky. Believes b-father now is in Fl.
      - - - - - - - - -
      - - - - - Reply With Quote - - - - -
      - - - -
      - - - - - - - - - -
      - - - - - -
      - - - - - - - - - - - - - - -
      -
      -   - #14   - - - - -
      - -
      - - Old - - 11-07-2005, 09:39 AM - - - -
      -
      - - - - - - - - -
      - - - - - KristinSullivan's Avatar - - - -
      - - - - - - - KristinSullivan - - KristinSullivan is offline - - - - -
      - - -
      Senior Member
      - - - -       		- - - -
      - -       		- -
      -
      Join Date: Oct 2005
      - - -
      - Posts: 185 - -
      - - Total Points: 10,018.52
      - Donate
      - - - -
      - -
      - - -
      - -
      - -
      - - - - - -
      Hi Teresa . . is Kornmille your birth name?
      -Also, have you tried to contact your birth father in Florida?
      - - - - - - - - -
      - - - - - Reply With Quote - - - - -
      - - - -
      - - - - - - - - - -
      - - - - - -
      - - - - - - - - - - - - - - -
      -
      -   - #15   - - - - -
      - -
      - - Old - - 11-07-2005, 10:23 AM - - - -
      -
      - - - - - - - - -
      - - - - - shirleyville's Avatar - - - -
      - - - - - - - shirleyville - - shirleyville is offline - - - - -
      - - -
      Senior Member
      - - - -       		- - - -
      - -       		- -
      -
      Join Date: Oct 2002
      - - -
      - Posts: 2,277 - -
      - - Total Points: 99,042.88
      - Donate
      - - - -
      - -
      - - -
      - -
      - -
      - - - - -
      - Talking - -
      -
      - - - - -
      I happen to know where my siblings are.....but I made a promise to my birthmother not to contact them. I thought I would post here, just in case they find out some other way, and start looking for me! At this point, they have no idea I exist.....but secrets of this nature rarely stay secrets forever, and if, by chance, they ever decide to search, they might come across this!
      -
      -Maternal Sister -- Debbie Carter
      -Maternal Brother -- Aaron Gray
      -Maternal Brother -- Calvin Gray
      -
      -I also know where my paternal sister, and am working on contacting her. Our father passed away many years ago, and I'm sure she has no clue I exist, either......
      -
      -Paternal Sister -- Roxanne Shaw Dupree.
      -
      -Great thread!!
      -Sally
      - - - - - - -
      - __________________
      - Pain is Inevitable -- Suffering is a Choice! -
      - - - - - -
      - - - - - Reply With Quote - - - - -
      - - - -
      - - - - - - - - - -
      - - -
      -
      -
      -
      -
      - - - - - - - - - - - - - - - - -
      Reply
      - - - - - - - - - - - -
      Page 1 of 57123451151>Last »
      -
      - -
      - - - - - -
      - -
      - « - Previous Thread - | - Next Thread - » -
      - - -
      - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      - Currently Active Users Viewing This Thread: 1 (0 members and 1 guests) -
      -   -
      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      Thread ToolsSearch this Thread
      - - -
      - - - - - - -
      - - -       		- - -
      -
      - - - - - - Search this Thread:
      -
      - Advanced Search -
      -
      - - -
      Display Modes
      - - - -
      -
      Linear Mode Linear Mode
      - - -
      - - - -
      -
      - - - - - - - - - - - -
      - - - - - - - - - - - -
      - - Posting Rules -
      - -
      You may not post new threads
      -
      You may not post replies
      -
      You may not post attachments
      -
      You may not edit your posts
      -
      - -
      vB code is On
      -
      Smilies are On
      -
      [IMG] code is Off
      -
      HTML code is Off
      Trackbacks are Off
      -
      Pingbacks are Off
      -
      Refbacks are Off
      - - - -
      -
      Points Per Thread View: 1.00
      -
      Points Per Thread: 15.00
      -
      Points Per Reply: 5.00
      - - - - -
      -       		- - - - - -
      - -
      -
      - - -
      - - - -
      - -
      All times are GMT -7. The time now is 04:01 AM.
      -
      - -
      -
      - -
      -
      -
      - - - - - -
      - - - - - - -
      -
      - - Contact Us - - Forum Rules - - Acronyms List - - Adoption Forums - - - - - - Top - -
      -
      - -
      - -
      -
      - -
      Adoption Zone
      -
      - -
      - - - - -
      -
      - -
      - - - - - - - - - - - - - - - - -
      - - - - - -
      - - - \ No newline at end of file diff --git a/tests/data/html/al.com1.html b/tests/data/html/al.com1.html index 0e8aba85..5327894b 100644 --- a/tests/data/html/al.com1.html +++ b/tests/data/html/al.com1.html @@ -1342,7 +1342,7 @@

      Follow Us

      Registration on or use of this site constitutes acceptance of our User Agreement and Privacy Policy

      -

      © 2014 Alabama Media Group All rights reserved (About Us).
      The material on this site may not be reproduced, distributed, transmitted, cached or otherwise used, except with the prior written permission of Alabama Media Group

      +

      © 2015 Alabama Media Group All rights reserved (About Us).
      The material on this site may not be reproduced, distributed, transmitted, cached or otherwise used, except with the prior written permission of Alabama Media Group

      Community Rules apply to all content you upload or otherwise submit to this site. Contact interactivity management.

      Ad Choices

      diff --git a/tests/data/html/al.com2.html b/tests/data/html/al.com2.html index d077fa3c..a63344e4 100644 --- a/tests/data/html/al.com2.html +++ b/tests/data/html/al.com2.html @@ -1773,7 +1773,7 @@

      Follow Us

      Registration on or use of this site constitutes acceptance of our User Agreement and Privacy Policy

      -

      © 2014 Alabama Media Group All rights reserved (About Us).
      The material on this site may not be reproduced, distributed, transmitted, cached or otherwise used, except with the prior written permission of Alabama Media Group

      +

      © 2015 Alabama Media Group All rights reserved (About Us).
      The material on this site may not be reproduced, distributed, transmitted, cached or otherwise used, except with the prior written permission of Alabama Media Group

      Community Rules apply to all content you upload or otherwise submit to this site. Contact interactivity management.

      Ad Choices

      diff --git a/tests/data/html/ap_meta_refresh.html b/tests/data/html/ap_meta_refresh.html new file mode 100644 index 00000000..48460bce --- /dev/null +++ b/tests/data/html/ap_meta_refresh.html @@ -0,0 +1,1162 @@ + + + + News from The Associated Press + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      +
      + +
      + +
      + + +
      +
      + + + + +

      + +AP Radio + +AP Radio News: + +

      +
      +

      + + + + + + + + +
      +
      +
      + + + + +
      + +
      + + +
      + +
      + +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      +
      +
      Jun 13, 4:03 PM EDT

      +

      Father of Orlando gunman known for rambling political videos +

      +

      +By LYNNE O'DONNELL
      +Associated Press

      + + + + + + + +
      + + + +
      AP Photo
      + AP Photo/Uncredited
      + + +
      + + +

      KABUL, Afghanistan (AP) -- The father of the Orlando nightclub gunman came to the U.S. from Afghanistan more than 30 years ago and has made a series of rambling political videos about his former homeland, even once describing himself as its "revolutionary president."

      Seddique Mir Mateen maintained a high profile on social media in the U.S., but is a mystery to Afghan authorities. Some government departments ran background and security checks Monday and found no trace of him, an official said.

      Mateen met with reporters Monday at his home in Port St. Lucie, Florida, and called the massacre by his son, Omar Mateen, "the act of a terrorist."

      The deadly weekend attack shocked the family, the father said, and went against what he had taught his son. The elder Mateen also said that if he had known what the 29-year-old was planning, he would have arrested him himself.

      In a Facebook video posted after the killings, Mateen said: "I don't know what caused him to shoot last night."

      "On the issue of homosexuality, it can be punished only by God, it is not the business of a person. But he (Omar) has killed those people, and I am so saddened," he added.

      The elder Mateen is a life insurance salesman who started a group in 2010 called Durand Jirga, Inc., according to Qasim Tarin, a businessman from California who was a Durand Jirga board member. The name refers to the Durand Line, the long disputed border established by the British between Afghanistan and Pakistan.

      He apparently left Afghanistan more than 30 years ago, after the 1979 invasion by the Soviet Union touched off a decade-long war of resistance. A civil war was followed by five years of radical Islamist rule by the Taliban until the U.S. invasion of 2001.

      Since then, the country has been trying to rebuild, with help from the U.S. and other countries, while the Taliban continue to fight the Kabul government.

      With constant turnover in government departments in the past 15 years, there appears to be no record of Mateen. Efforts were made to find out about his past, "but we haven't found a clue," according to an official in the Afghan intelligence agency, speaking on condition of anonymity because he was not authorized to talk to reporters.

      But Kabul-based political analyst Ahmad Saeedi said Mateen, who is about 70, is from the eastern province of Laghman and was living in the capital when he left 31 years ago for the U.S.

      A fierce anti-communist, Mateen was a captain in the ranks of the mujahedeen who fought the Soviet occupation, Saeedi said. Once in the U.S., he promoted himself to the rank of general.

      Mateen made a series of videos on social media in the Dari language in which he blamed Afghanistan's ills on a variety of people and organizations, notably Pakistan's intelligence agency, the ISI. The videos were called the "Durand Jirga Show."

      "For the past three or four years, he has been uploading these videos in which he claims to be the 'revolutionary president' of Afghanistan," Saeedi said. Many of the videos drew only a few hundred views.

      A former Afghan official said the "Durand Jirga Show" appears on Payam-e-Afghan, a California-based channel that supports ethnic solidarity with the Afghan Taliban, which are mostly Pashtun. The official spoke on condition of anonymity because he did not want to be linked to coverage of the shooting.

      But video reviewed by The Associated Press on Monday did not show support by Mateen for the Taliban. In an April 2015 video, Mateen said he and his supporters had called on the Taliban to join the peace initiative by current Afghan President Ashraf Ghani.

      Mateen once had "a brief and inconsequential meeting" with Rep. Ed Royce, the chairman of the House Foreign Affairs Committee, the California Republican said in a statement. Royce said he routinely meets Afghan-Americans and he vaguely recalls a discussion about relations between Afghanistan and Pakistan with Mateen.

      In a video dated May 23, 2015, Mateen declared himself a candidate for the Afghan presidency.

      "Afghanistan's sovereignty is in danger due to the free hand of foreigners on our territory, and the result is that 95 percent of the country is not under the government's control," he said, a reference to the influence of Pakistan.

      "Due to a lack of planning, and resultant economic crisis, I, Seddique Mateen, am declaring myself as a candidate for the presidency in order to rescue Afghanistan."

      A photo was posted of of a bright green T-shirt depicting him in a black jacket and red tie - the colors of the Afghan flag - and the words: "Seddique Mateen: the real leader of the Afghan people."

      His Facebook page includes a number of posts introducing "members of our government," showing young men but no biographical information.

      In another video, he accused President Barack Obama of supporting Pakistan's security agency and urged him to cut off U.S. funding.

      A YouTube video dated Jan. 11, 2014, showed Mateen interviewing Ghani during his presidential campaign. He deferentially asked Ghani about poverty, unemployment and corruption, and politely allowed the candidate to answer for 13 minutes.

      But in a video posted Monday on Facebook, Mateen listed senior Afghan officials, including Ghani and former President Hamid Karzai, and he twisted their names to show his disapproval of them. In a repetitive ramble, he called for Afghan unity.

      Mateen's views on Pakistan's role in Afghanistan reflect those of many Afghan people. Recently, Ghani has publicly blamed Pakistan and its secret service for using the Taliban to wage war on Afghanistan. Pakistan denies the allegations.

      Ghani's efforts to draw the Taliban into peace talks have failed, with the Taliban saying they have no intention of cooperating. Ghani has also pressured Pakistan to force the Taliban to the table, to no avail.

      ---

      Associated Press writers Rahim Faiez, Karim Sharifi and Mirwais Khan in Kabul, and Mitch Weiss in Greenville, South Carolina, contributed to this story.

      © 2016 The Associated Press. All rights reserved. This material may not be published, broadcast, rewritten or redistributed. Learn more about our Privacy Policy and Terms of Use.

      + +
      +
      +
      + + +
      + +
      + +
      + +
      + + +
      + +

      +Latest News +
      + + + + + +
      + + +

      + +
      + +

      + +
      + + +

      + +Advertisement +
      +
      + +
      +

      + + + + +
      + + + + + + + + + +
      +

      + ©2015 The Associated Press.
      + All rights reserved. Terms + under which this site is provided.
      + Learn more about our + Privacy Policy.

      +
      +
      +
      + + + diff --git a/tests/data/html/apartmenttherapy.com1.html b/tests/data/html/apartmenttherapy.com1.html index 93b7ac73..73c29dd4 100644 --- a/tests/data/html/apartmenttherapy.com1.html +++ b/tests/data/html/apartmenttherapy.com1.html @@ -9,9 +9,9 @@ - + - + @@ -401,7 +401,7 @@
      Holiday Gift Guides 2014
      Previous -5483c1c8697ab06af6007282. w.60 h.60 s.centercrop +5483c1c8697ab06af6007282. w.60 h.60 s.centercrop
      31 Kids Gift Ideas to Satisfy Your Inner Design Snob
      @@ -410,7 +410,7 @@
      Holiday Gift Guides 2014
      Next -5475181f2a099a3a32005314. w.60 h.60 s.centercrop +5475181f2a099a3a32005314. w.60 h.60 s.centercrop
      Great Gifts for Small-Space Dwellers
      @@ -453,9 +453,9 @@

      Categories

      - +
      @@ -657,7 +657,7 @@

      +

    @@ -605,9 +605,9 @@

    Aussies seal series win in MCG draw

    The opinions expressed here are the views of the writer and do not necessarily reflect the views and opinions of Cricket Australia.

      -
      • -
      • -
      • +
      • +
      • +
    @@ -657,8 +657,8 @@

    Top Stories

    - - Johnson Kohli + +
    @@ -667,17 +667,17 @@

    Top Stories

    CommBank Test Series v India

    -

    India a new side under Kohli: Mitch

    +

    Johnson an injury worry for Aussies

    AAP

      -
    • 3h ago
      • +
      • -
    • 0
      • +
    • 0
    @@ -709,7 +709,7 @@

    @@ -724,30 +724,30 @@

    -
    - - Hobart Hurricanes v Brisbane Heat - + + Shane Watson +
    -

    - - KFC T20 BBL|04 +

    + + CommBank Test Series v India -

    -

    LIVE: Brisbane Heat v Hobart Hurricanes

    +

    +

    Watson ready for emotional SCG return

    -

    cricket.com.au

    +

    Andrew Ramsey

      +
    • 2:02
      • -
    • 0
      • +
    • 0
    @@ -779,15 +779,13 @@

    -
    - - +
    @@ -796,15 +794,16 @@

    -

    Australia to 'bore' Kohli in Sydney Test

    +

    Stars soar, Renegades hit record low

    @@ -834,15 +833,16 @@

    Best World Cup XII of all time?

    +

    Harris backs Siddle return to Test cricket

    @@ -875,17 +875,16 @@

    -

    Mitch 'disappointed' with Test wickets

    +

    Hodge likely to miss rest of BBL|04

    @@ -915,7 +914,7 @@

    Southee to undergo fitness test

    +

    Sangakkara fastest to 12,000 Test runs

    @@ -981,7 +980,7 @@

    Most Popular

    1

    - Dhoni wants Hussey for India coach, report says + Reed debuts with big crash

    @@ -990,7 +989,7 @@

    2

    - Australia to 'bore' Kohli in Sydney Test + Ramdin leads Windies fightback in Cape Town

    @@ -999,7 +998,7 @@

    3

    - Reed debuts with big crash + Hurricanes send off Sammy in style

    @@ -1008,7 +1007,7 @@

    4

    - Best World Cup XII of all time? + Johnson an injury worry for Aussies

    @@ -1017,7 +1016,7 @@

    5

    - LIVE: Perth Scorchers v Sydney Thunder + Bailey sorry for swearing during BBL

    @@ -1256,12 +1255,12 @@

    if (response.errorCode == 0) { - document.getElementById("shareTwitterVertical7621").innerHTML = document.getElementById("shareTwitterVertical7621").innerHTML + response.shareCounts.twitter; - document.getElementById("shareFacebookVertical4055").innerHTML = document.getElementById("shareFacebookVertical4055").innerHTML + response.shareCounts.facebook; + document.getElementById("shareTwitterVertical1057").innerHTML = document.getElementById("shareTwitterVertical1057").innerHTML + response.shareCounts.twitter; + document.getElementById("shareFacebookVertical4086").innerHTML = document.getElementById("shareFacebookVertical4086").innerHTML + response.shareCounts.facebook; } else { // Had an error so we'll set counts to zero. - document.getElementById("shareTwitterVertical7621").innerHTML = ""; - document.getElementById("shareFacebookVertical4055").innerHTML = ""; + document.getElementById("shareTwitterVertical1057").innerHTML = ""; + document.getElementById("shareFacebookVertical4086").innerHTML = ""; //document.getElementById("googleplus-count").innerHTML = ""; } } diff --git a/tests/data/html/cricket.com.au2.html b/tests/data/html/cricket.com.au2.html index 64ae5042..0a6632d9 100644 --- a/tests/data/html/cricket.com.au2.html +++ b/tests/data/html/cricket.com.au2.html @@ -101,7 +101,7 @@ - + @@ -289,17 +289,17 @@

    Video

    • - +
    • - +
    • - +
      • @@ -317,8 +317,8 @@

      Video

      -
      - +
      +
      @@ -333,12 +333,12 @@

      Video

      Glenn Maxwell's unbelievable leave

    - + @@ -302,7 +302,7 @@
    - +
    @@ -315,6 +315,25 @@
    +
    + +
    +
    +

    My Watchlist

    +
    + + +
    + + Track the companies that matter to you. It's FREE! Click one of these fan favorites to get started: Apple; Google; Ford. + +
    + +
    + + + +
    @@ -459,11 +478,11 @@
    + title="46 members have recommended this article">

    Recs

    - 43 + 46

    @@ -664,11 +683,11 @@

    7 Top Stocks to Buy for 2015

    | - Recommend This Article (43) + Recommend This Article (46) @@ -1120,16 +1139,16 @@
    Add your comment.

    Compare Brokers

    - +
    - +
    - +
    - +
    Fool Disclosure
    @@ -1145,7 +1164,7 @@

    Sponsored Links

    -

    DocumentId: 3208784, ~/Articles/ArticleHandler.aspx, 1/2/2015 6:00:00 AM

    +

    DocumentId: 3208784, ~/Articles/ArticleHandler.aspx, 1/3/2015 6:47:11 AM

    @@ -1205,11 +1224,11 @@

    Today's Market

    - updated 1 day ago + updated 8 hours ago Sponsored by:
    @@ -1218,20 +1237,20 @@

    Today's Market

    - - + + DOW - 17,823.07 + 17,832.99 - - -160.00 + + 9.92 - - -0.89% + + 0.06% @@ -1243,31 +1262,31 @@

    Today's Market

    S&P 500 - 2,058.90 + 2,058.20 - -21.45 + -0.70 - -1.03% + -0.03% - - + + NASD - 4,736.05 + 4,726.81 - - 0.00 + + -9.24 - - 0.00% + + -0.20% @@ -1329,7 +1348,7 @@

    Related Tickers

    - 12/31/2014 4:00 PM + 1/2/2015 4:00 PM
    @@ -1344,19 +1363,19 @@

    AAPL - - $110.38 + + $109.33 Down - - -2.14 + + -1.05 - - -1.90% + + -0.95% @@ -1390,19 +1409,19 @@

    FB - - $78.02 + + $78.45 - Down + Up - - - -1.20 + + + +0.43 - - - -1.51% + + + +0.55% @@ -1436,19 +1455,19 @@

    GOOG - - $526.40 + + $524.81 Down - - -4.02 + + -1.59 - - -0.76% + + -0.30% @@ -1482,19 +1501,19 @@

    GOOGL - - $530.66 + + $529.55 Down - - -4.62 + + -1.11 - - -0.86% + + -0.21% @@ -1528,19 +1547,19 @@

    QCOM - - $74.33 + + $74.28 Down - - -0.69 + + -0.05 - - -0.92% + + -0.07% @@ -1574,19 +1593,19 @@

    SEAS - - $17.90 + + $17.97 - Down + Up - - - -0.03 + + + +0.07 - - - -0.17% + + + +0.39% @@ -1620,19 +1639,19 @@

    TSM - - $22.38 + + $22.28 Down - - -0.06 + + -0.10 - - -0.27% + + -0.45% @@ -1662,10 +1681,10 @@

    - +
    - +
    @@ -1741,8 +1760,8 @@

    SEC Filings and Insider Transactions provided by Edgar Online.
    Powered and implemented by Interactive Data Managed Solutions. Terms & Conditions

    -
    94ms USEQ\WEB04 -
    - + @@ -1917,7 +1936,7 @@

    - + diff --git a/tests/data/html/foxbusiness.com2.html b/tests/data/html/foxbusiness.com2.html index 31db602b..e76ec49d 100644 --- a/tests/data/html/foxbusiness.com2.html +++ b/tests/data/html/foxbusiness.com2.html @@ -1,7 +1,7 @@ - + diff --git a/tests/data/html/foxnews.com1.html b/tests/data/html/foxnews.com1.html index 7996324c..cedad1f1 100644 --- a/tests/data/html/foxnews.com1.html +++ b/tests/data/html/foxnews.com1.html @@ -1,7 +1,7 @@ - + +
    @@ -390,28 +390,28 @@

    More Stories

    @@ -674,29 +674,29 @@

    Also on the web

    @@ -800,8 +800,8 @@

    Networking

    diff --git a/tests/data/html/foxnews.com2.html b/tests/data/html/foxnews.com2.html index 905238e9..8fb8c723 100644 --- a/tests/data/html/foxnews.com2.html +++ b/tests/data/html/foxnews.com2.html @@ -1,7 +1,7 @@ - + +
    @@ -376,28 +376,28 @@

    More Stories

    @@ -660,29 +660,29 @@

    Also on the web

    @@ -786,8 +786,8 @@

    Networking

    diff --git a/tests/data/html/glamour.com1.html b/tests/data/html/glamour.com1.html index be50b350..4d44939b 100644 --- a/tests/data/html/glamour.com1.html +++ b/tests/data/html/glamour.com1.html @@ -69,7 +69,7 @@ var Courier = { config: {"previewSocial":false,"skipView":false,"site":"glamour","env":"production","hostName":"http://www.glamour.com","lipstickLaunchDate":"20140409","sansHtmlDate":"20140808","services":{"cnid":{"apiKey":"85111a62-d034-4b3c-b404-04a06da7b1fd","host":"https://cnid.condenastdigital.com"},"eventCollector":{"env":"PROD","host":"http://event.glamour.com"},"eventResult":{"host":"http://result.glamour.com"},"social":{"host":"http://social.glamour.com"},"stats":{"host":"http://api.glamour.com","root":"/content/stats"},"userServiceHost":"https://user-service.condenastdigital.com","userServiceKey":"0OobSjxeOBbB5CMWzh3sAB5Vsdk="},"version":"0.1.53","viewSuffix":"","type":"blog-entry","view":"blog-entry","livefyre":{"articleIdPrefix":"glamour_","commentCountSelector":".livefyre-commentcount","commentsEnabledTag":"comments","lfCookieName":"lt","mostCommentedEntries":4,"mostCommentedId":"commented","mostPopularAge":15,"mostPopularEntries":5,"previewSocial":false,"socialStreamSelector":".stream","streams":[],"strings":{"moderator":"glamour"},"userToken":"eyJhbGciOiJIUzI1NiJ9.eyJkaXNwbGF5X25hbWUiOiJUZXN0IFVzZXIgTGl2ZWZ5cmUgMSIsImV4cGlyZXMiOiIxNDUwODI1ODE4IiwiZG9tYWluIjoiZ2xhbW91ci0wLmZ5cmUuY28iLCJ1c2VyX2lkIjoiMTIzNDU2Nzg5MCJ9.oVkq4ULCBpu7vn9v7qLBFtrAu8z9ejcEvILKVCFIiTY","network":"condenast.fyre.co","scripts":{"livefyre":"http://zor.livefyre.com/wjs/v3.0/javascripts/livefyre.js","commentCount":"http://zor.livefyre.com/wjs/v1.0/javascripts/CommentCount.js"},"siteId":"314714","trendingServer":"http://bootstrap.condenast.fyre.co/api/v3.0/hottest/?site=","pageConfig":{"articleId":"glamour_2000000002863673","checksum":"8c90c5c816100bb9e446b0b3585dd66f","collectionMeta":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ0YWdzIjoiY29tbWVudHMiLCJ0aXRsZSI6IlJpaGFubmEgTGVhdmVzIFBhbnRzIGF0IEhvbWUgZm9yIEFsdHV6YXJyYSBTaG93LCBhbmQgV2UgTG92ZSBIZXIgZm9yIEl0IiwidXJsIjoiaHR0cDovL2dsYW1vdXIuY29tL2Zhc2hpb24vYmxvZ3MvZHJlc3NlZC8yMDE0LzA5L3JpaGFubmEtbGVhdmVzLXBhbnRzLWF0LWhvbWUtZj9pbWFnZVVybD0vaW1hZ2VzL2Zhc2hpb24vMjAxNC8wOS9yaWhhbm5hLWFsdHV6YXJyYTEtc3F1YXJlLXczNTIuanBnIiwiY2hlY2tzdW0iOiI4YzkwYzVjODE2MTAwYmI5ZTQ0NmIwYjM1ODVkZDY2ZiIsImFydGljbGVJZCI6ImdsYW1vdXJfMjAwMDAwMDAwMjg2MzY3MyJ9.lTcpwR51QLn-imMtCuBoGIX2ltHJTqwtuQmWIWaVU5s","el":"livefyre","siteId":"314714"}},"pageAdsPath":"//subscribe.glamour.com/ams/page-ads.js?ad_category_prefix=fashion&browser_path=%2Ffashion%2Fblogs%2Fdressed%2F2014%2F09%2Frihanna-leaves-pants-at-home-f&cat_prefixes=%2Cfashion%2Cblogs%2Cdressed%2C2014%2C09%2Crihanna-leaves-pants-at-home-f&keywords=fashion%2Cdesigners_and_fashion_week%2Cadamselman%2Crihanna%2Cblogs%2Cdressed%2C2014%2C09%2Crihanna_leaves_pants_at_home_f%2Cblog-entry&position_list=AMS_GLM_CONTENTPAGE_ZRAIL%2CAMS_GLM_GLOBAL_FLYIN%2CAMS_GLM_GLOBAL_FOOTER%2CAMS_GLM_GLOBAL_HEADERCALLOUT%2CAMS_GLM_GLOBAL_HEADERCALLOUT_COLLAPSE%2CAMS_GLM_GLOBAL_INTER%2CAMS_GLM_GLOBAL_LEFTNAV%2CAMS_GLM_LITEREG_SUBSCRIBE%2CAMS_GLM_MOBILE_CONTENTPAGE_ZRAIL%2CAMS_GLM_MOBILE_GLOBAL_FOOTER%2CAMS_GLM_TABLET_GLOBAL_HEADERCALLOUT%2CAMS_GLM_MOBILE_GLOBAL_NAVBAR%2CAMS_GLM_BLOG_RIGHTNAV&site_prefix=glamour&subdomain=www&url_name=rihanna-leaves-pants-at-home-f","dtm":{"pageValue":"all"}}, meta: {"contributor":"Sophia Chabbott","description":"Oh, RiRi. You truly are an icon. When Rihanna walks into a fashion show, even editors, publicists, other celebs gasp. Gasp they did on Friday and Saturday as the cooler-than-thou-crooner sashayed into her costume designer Adam Selman's show, as well as those of Joseph Altuzarra and Alexander Wang. And she did not disappoint. Let's take a look at what she wore: At Adam Selman, RiRi rocked an elegant and girly white A-line dress with pearls and my favorite Christian Louboutin strappy pumps. For Altuzarra, she glammed out in super sophistication with a plunging black jacket with fringe from the designer's collection—WITHOUT PANTS. I LOVE IT. At Alexander Wang, she brought out her inner '90s child with a matching midnight blue hoodie and skirt. Are you a Rihhana fan? I love her. Now where's my plunging black jacket? Not to fret—I will wear pants.","heds":{"display":"Rihanna Leaves Pants at Home for Altuzarra Show, and We Love Her for It","seo":"Rihanna at Alexander Wang, Adam Selman, and Altuzarra at New York Fashion Week: Glamour.com","social":"Rihanna Leaves Pants at Home for Altuzarra Show, and We Love Her for It"},"id":"2000000002863673","keywords":{"ads":["fashion","designers_and_fashion_week","adam selman","rihanna","blogs","dressed","2014","09","rihanna_leaves_pants_at_home_f","blog-entry"],"display":["adam selman","rihanna"],"hidden":[]},"pageType":"blogpost","pagination":{},"properties":"","publishDate":"2014-09-07T14:55:00.000Z","reactions":"","searchImage":"/images/fashion/2014/09/rihanna-altuzarra1-square-w352.jpg","section":"fashion","site":"glamour","source":"","subsection":"designers-and-fashion-week","type":"blog-entry","urls":{"canonical":"/fashion/blogs/dressed/2014/09/rihanna-leaves-pants-at-home-f","lookup":"/fashion/blogs/dressed/2014/09/rihanna-leaves-pants-at-home-f","social":"/fashion/blogs/dressed/2014/09/rihanna-leaves-pants-at-home-f"},"stitch":{}}, - stats: {} + stats: {"2000000003015754":{"likes":2}} }; Courier.config.cmPlacements = ["AMS_GLM_GLOBAL_LEFTNAV","AMS_GLM_MOBILE_GLOBAL_NAVBAR","AMS_GLM_GLOBAL_HEADERCALLOUT","AMS_GLM_GLOBAL_HEADERCALLOUT_COLLAPSE","AMS_GLM_BLOG_RIGHTNAV","AMS_GLM_CONTENTPAGE_ZRAIL","AMS_GLM_MOBILE_CONTENTPAGE_ZRAIL","AMS_GLM_MOBILE_GLOBAL_FOOTER","AMS_GLM_GLOBAL_FOOTER","AMS_GLM_GLOBAL_FLYIN"]; Courier.config.ads = { @@ -882,15 +882,15 @@

    More Stories Like This

    -
    +
    - + - 7 Pieces You Need to Get Noticed at Work in 2015 - 7 Pieces You Need to Get Noticed at Work in 2015 + The 5 New Ways To Style Pants That Will Rock Your World + The 5 New Ways To Style Pants That Will Rock Your World @@ -908,19 +908,19 @@

    More Stories Like This

    -     7 Pieces You Need to Get Noticed at Work in 2015 - 7 Pieces You Need to Get Noticed at Work in 2015 + The 5 New Ways To Style Pants That Will Rock Your World + The 5 New Ways To Style Pants That Will Rock Your World -
    By Leah Melby
    +
    By Avery Matera
    -
    20 hours ago
    +
    13 hours ago
    @@ -934,15 +934,15 @@

    More Stories Like This

    -
    +
    - + - Don't Buy Another Pair of Jeans Until You Read This: The Trick for Longer, Leaner Legs - Don't Buy Another Pair of Jeans Until You Read This: The Trick for Longer, Leaner Legs + Over Glitter? Never. Glitter Shoes That Are Year-Round Approved + Over Glitter? Never. Glitter Shoes That Are Year-Round Approved @@ -960,19 +960,19 @@

    More Stories Like This

    -     Don't Buy Another Pair of Jeans Until You Read This: The Trick for Longer, Leaner Legs - Don't Buy Another Pair of Jeans Until You Read This: The Trick for Longer, Leaner Legs + Over Glitter? Never. Glitter Shoes That Are Year-Round Approved + Over Glitter? Never. 17 Glitter Shoes That Are Year-Round Approved -
    By Leah Melby
    +
    By Avery Matera
    -
    1 day ago
    +
    14 hours ago
    @@ -1012,36 +1012,36 @@

    TRENDING

    - - Surprise! Joseph Gordon-Levitt Is Married + + Princess Mary Had the Dreamiest New Year's Outfit, Ever - - Surprise! Joseph Gordon-Levitt Is Married + + Princess Mary Had the Dreamiest New Year's Outfit, Ever
    - - 9 Love and Sex Resolutions You Should Make for 2015 + + Is This the Most Sexist Relationship Advice Ever? - - 9 Love and Sex Resolutions You Should Make for 2015 + + Is This the Most Sexist Relationship Advice Ever?
    - - 7 Bad Beauty Habits We're Kissing Goodbye to in 2015 + + Huh? The Bra You're Supposed to Wear at Night (I Tried It!) - - 7 Bad Beauty Habits We're Kissing Goodbye to in 2015 + + Huh? The Bra You're Supposed to Wear at Night (I Tried It!)
    @@ -1499,11 +1499,11 @@

    TRENDING

    - + - January is Veganuary. Will You Ditch Meat This Month? - January is Veganuary. Will You Ditch Meat This Month? + What Exactly Is a "Healthy" Sex Life, Anyway? + What Exactly Is a "Healthy" Sex Life, Anyway? @@ -1515,25 +1515,25 @@

    TRENDING

    -     health-fitness + sex-love-life - January is Veganuary. Will You Ditch Meat This Month? - January is Veganuary. Will You Ditch Meat This Month? + What Exactly Is a "Healthy" Sex Life, Anyway? + What Exactly Is a "Healthy" Sex Life, Anyway? -
    By Lexi Petronis
    +
    By Jillian Kramer
    -
    11 hours ago
    +
    12 hours ago
    @@ -1547,15 +1547,15 @@

    TRENDING

    -
    +
    - + - 7 Ways to Actually, Really Stick to Your 2015 Exercise Resolutions - 7 Ways to Actually, Really Stick to Your 2015 Exercise Resolutions + The 5 New Ways To Style Pants That Will Rock Your World + The 5 New Ways To Style Pants That Will Rock Your World @@ -1567,25 +1567,25 @@

    TRENDING

    -     health-fitness + fashion - 7 Ways to Actually, Really Stick to Your 2015 Exercise Resolutions - 7 Ways to Actually, Really Stick to Your 2015 Exercise Resolutions + The 5 New Ways To Style Pants That Will Rock Your World + The 5 New Ways To Style Pants That Will Rock Your World -
    By Lexi Petronis
    +
    By Avery Matera
    -
    15 hours ago
    +
    13 hours ago
    @@ -1599,15 +1599,15 @@

    TRENDING

    -
    +
    - + - 10 Cooking Resolutions to Make (and Keep!) This Year - 10 Cooking Resolutions to Make (and Keep!) This Year + Over Glitter? Never. Glitter Shoes That Are Year-Round Approved + Over Glitter? Never. Glitter Shoes That Are Year-Round Approved @@ -1619,25 +1619,25 @@

    TRENDING

    -     home + fashion - 10 Cooking Resolutions to Make (and Keep!) This Year - Here's to a Healthy and Happy 2015: 10 Cooking Resolutions to Make (and Keep!) + Over Glitter? Never. Glitter Shoes That Are Year-Round Approved + Over Glitter? Never. 17 Glitter Shoes That Are Year-Round Approved -
    By Danica Lo
    +
    By Avery Matera
    -
    17 hours ago
    +
    14 hours ago
    @@ -1655,11 +1655,11 @@

    TRENDING

    - + - 6 Sexual Ways to Heal That Holiday Hangover With Your Man - 6 Sexual Ways to Heal That Holiday Hangover With Your Man + Does Gwen Stefani Wear Makeup to Go Skiing? Here's Your Answer + Does Gwen Stefani Wear Makeup to Go Skiing? Here's Your Answer @@ -1671,25 +1671,25 @@

    TRENDING

    -     sex-love-life + beauty - 6 Sexual Ways to Heal That Holiday Hangover With Your Man - 6 Sexual Ways to Heal That Holiday Hangover With Your Man + Does Gwen Stefani Wear Makeup to Go Skiing? Here's Your Answer + Does Gwen Stefani Wear Makeup to Go Skiing? Here's Your Answer -
    By Jillian Kramer
    +
    By Beth Shapouri
    -
    17 hours ago
    +
    14 hours ago
    @@ -1707,11 +1707,11 @@

    TRENDING

    - + - Dudeoir Is the New Boudoir: Your Groom Deserves Some Sexy Wedding Portraits Too - Dudeoir Is the New Boudoir: Your Groom Deserves Some Sexy Wedding Portraits Too + What to Watch, Read, Stream, and Listen to This Weekend + What to Watch, Read, Stream, and Listen to This Weekend @@ -1723,25 +1723,25 @@

    TRENDING

    -     weddings + entertainment - Dudeoir Is the New Boudoir: Your Groom Deserves Some Sexy Wedding Portraits Too - Dudeoir Is the New Boudoir: Your Groom Deserves Some Sexy Wedding Portraits Too + What to Watch, Read, Stream, and Listen to This Weekend + What to Watch, Read, Stream, and Listen to This Weekend -
    By Kim Fusaro
    +
    By Lynsey Eidell
    -
    17 hours ago
    +
    14 hours ago
    @@ -1768,29 +1768,29 @@

    TRENDING

    - - Surprise! Joseph Gordon-Levitt Is Married + + Princess Mary Had the Dreamiest New Year's Outfit, Ever -
    - Surprise! Joseph Gordon-Levitt Is Married +
    + Princess Mary Had the Dreamiest New Year's Outfit, Ever
    - - 9 Love and Sex Resolutions You Should Make for 2015 + + Is This the Most Sexist Relationship Advice Ever? -
    - 9 Love and Sex Resolutions You Should Make for 2015 +
    + Is This the Most Sexist Relationship Advice Ever?
    - - 7 Bad Beauty Habits We're Kissing Goodbye to in 2015 + + Huh? The Bra You're Supposed to Wear at Night (I Tried It!) -
    - 7 Bad Beauty Habits We're Kissing Goodbye to in 2015 +
    + Huh? The Bra You're Supposed to Wear at Night (I Tried It!)
    @@ -1804,11 +1804,11 @@

    TRENDING

    - - Things That Always Happen in Movie Sex Scenes and Never in Real Life + + 7 Bad Beauty Habits We're Kissing Goodbye to in 2015 -
    - Things That Always Happen in Movie Sex Scenes and Never in Real Life +
    + 7 Bad Beauty Habits We're Kissing Goodbye to in 2015
    diff --git a/tests/data/html/glamour.com2.html b/tests/data/html/glamour.com2.html index 6e27d5fd..41e8d03b 100644 --- a/tests/data/html/glamour.com2.html +++ b/tests/data/html/glamour.com2.html @@ -69,7 +69,7 @@ var Courier = { config: {"previewSocial":false,"skipView":false,"site":"glamour","env":"production","hostName":"http://www.glamour.com","lipstickLaunchDate":"20140409","sansHtmlDate":"20140808","services":{"cnid":{"apiKey":"85111a62-d034-4b3c-b404-04a06da7b1fd","host":"https://cnid.condenastdigital.com"},"eventCollector":{"env":"PROD","host":"http://event.glamour.com"},"eventResult":{"host":"http://result.glamour.com"},"social":{"host":"http://social.glamour.com"},"stats":{"host":"http://api.glamour.com","root":"/content/stats"},"userServiceHost":"https://user-service.condenastdigital.com","userServiceKey":"0OobSjxeOBbB5CMWzh3sAB5Vsdk="},"version":"0.1.53","viewSuffix":"","type":"blog-entry","view":"blog-entry","livefyre":{"articleIdPrefix":"glamour_","commentCountSelector":".livefyre-commentcount","commentsEnabledTag":"comments","lfCookieName":"lt","mostCommentedEntries":4,"mostCommentedId":"commented","mostPopularAge":15,"mostPopularEntries":5,"previewSocial":false,"socialStreamSelector":".stream","streams":[],"strings":{"moderator":"glamour"},"userToken":"eyJhbGciOiJIUzI1NiJ9.eyJkaXNwbGF5X25hbWUiOiJUZXN0IFVzZXIgTGl2ZWZ5cmUgMSIsImV4cGlyZXMiOiIxNDUwODI1ODE4IiwiZG9tYWluIjoiZ2xhbW91ci0wLmZ5cmUuY28iLCJ1c2VyX2lkIjoiMTIzNDU2Nzg5MCJ9.oVkq4ULCBpu7vn9v7qLBFtrAu8z9ejcEvILKVCFIiTY","network":"condenast.fyre.co","scripts":{"livefyre":"http://zor.livefyre.com/wjs/v3.0/javascripts/livefyre.js","commentCount":"http://zor.livefyre.com/wjs/v1.0/javascripts/CommentCount.js"},"siteId":"314714","trendingServer":"http://bootstrap.condenast.fyre.co/api/v3.0/hottest/?site=","pageConfig":{"articleId":"glamour_2000000003009414","checksum":"10a3ce7e030d5c8c3fd4d7e41c04b042","collectionMeta":"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ0YWdzIjoiY29tbWVudHMiLCJ0aXRsZSI6IjYgV2VkZGluZyBEcmVzc2VzIFdpdGggWmVybyBGbHVmZiwgZm9yIEJyaWRlcyBXaG8gRG9uJiM4MjE3O3QgV2FudCB0byBMb29rIExpa2UgYSBEYW1uIEN1cGNha2UiLCJ1cmwiOiJodHRwOi8vZ2xhbW91ci5jb20vd2VkZGluZ3MvYmxvZ3Mvc2F2ZS10aGUtZGF0ZS8yMDE0LzEyL3NpbXBsZS13ZWRkaW5nLWRyZXNzZXM_aW1hZ2VVcmw9L2ltYWdlcy93ZWRkaW5ncy8yMDE0LzEyL3NpbXBsZS13ZWRkaW5nLWRyZXNzZXMtd2VkZGluZy1nb3ducy0xMjE3LXNxdWFyZS13MzUyLmpwZyIsImNoZWNrc3VtIjoiMTBhM2NlN2UwMzBkNWM4YzNmZDRkN2U0MWMwNGIwNDIiLCJhcnRpY2xlSWQiOiJnbGFtb3VyXzIwMDAwMDAwMDMwMDk0MTQifQ.05IEuUuxm9wzTZKZ9qJ6TC_-6AhvHgt3lKB94EuKJY0","el":"livefyre","siteId":"314714"}},"pageAdsPath":"//subscribe.glamour.com/ams/page-ads.js?ad_category_prefix=weddings&browser_path=%2Fweddings%2Fblogs%2Fsave-the-date%2F2014%2F12%2Fsimple-wedding-dresses&cat_prefixes=%2Cweddings%2Cblogs%2Csave-the-date%2C2014%2C12%2Csimple-wedding-dresses&keywords=weddings%2Cwedding_dresses%2Cweddingdresses%2Cweddinggowns%2Cblogs%2Csave_the_date%2C2014%2C12%2Csimple_wedding_dresses%2Cblog-entry&position_list=AMS_GLM_CONTENTPAGE_ZRAIL%2CAMS_GLM_GLOBAL_FLYIN%2CAMS_GLM_GLOBAL_FOOTER%2CAMS_GLM_GLOBAL_HEADERCALLOUT%2CAMS_GLM_GLOBAL_HEADERCALLOUT_COLLAPSE%2CAMS_GLM_GLOBAL_INTER%2CAMS_GLM_GLOBAL_LEFTNAV%2CAMS_GLM_LITEREG_SUBSCRIBE%2CAMS_GLM_MOBILE_CONTENTPAGE_ZRAIL%2CAMS_GLM_MOBILE_GLOBAL_FOOTER%2CAMS_GLM_TABLET_GLOBAL_HEADERCALLOUT%2CAMS_GLM_MOBILE_GLOBAL_NAVBAR%2CAMS_GLM_BLOG_RIGHTNAV&site_prefix=glamour&subdomain=www&url_name=simple-wedding-dresses","dtm":{"pageValue":"all"}}, meta: {"contributor":"Kim Fusaro","description":"A ton of brides are going for the pretty pretty princess vibe on their wedding day, so lots of wedding dress designers automatically reach for the tulle and the crinolines. {“Fluffier! Puffier! MORE! } If you’re a bride who isn’t feeling the cupcake vibe, I suggest checking out white evening dresses, which tend to be a lot more streamlined. And bonus: Evening gowns are generally less expensive than wedding gowns. (Four of the dresses below are under $1,000.) Here are six off-the-rack I like: !function(d,s,id){var e, p = /^http:/.test(d.location) ? 'http' : 'https';if(!d.getElementById(id)) {e = d.createElement(s);e.id = id;e.src = p + '://' + 'widgets.rewardstyle.com' + '/js/boutique.js';d.body.appendChild(e);}if(typeof window.__boutique === 'object') if(d.readyState === 'complete') {window.__boutique.init();}}(document, 'script', 'boutique-script');JavaScript is currently disabled in this browser. Reactivate it to view this content. Off-the-rack wedding dresses: Yes or no? Would you want to wear an evening gown on your wedding day? Or does it HAVE to be bridal?","heds":{"display":"6 Wedding Dresses With Zero Fluff, for Brides Who Don’t Want to Look Like a Damn Cupcake","seo":"Simple Wedding Dresses: Best Off-the-Rack Wedding Dresses, Wedding Gowns: Glamour.com","social":"Wedding Dresses With Zero Fluff, for Brides Who Don???t Want to Look Like a Damn Cupcake"},"id":"2000000003009414","keywords":{"ads":["weddings","wedding_dresses","wedding dresses","wedding gowns","blogs","save_the_date","2014","12","simple_wedding_dresses","blog-entry"],"display":["wedding dresses","wedding gowns"],"hidden":[]},"pageType":"blogpost","pagination":{},"properties":"","publishDate":"2014-12-17T16:06:00.000Z","reactions":"","searchImage":"/images/weddings/2014/12/simple-wedding-dresses-wedding-gowns-1217-square-w352.jpg","section":"weddings","site":"glamour","source":"","subsection":"wedding-dresses","type":"blog-entry","urls":{"canonical":"/weddings/blogs/save-the-date/2014/12/simple-wedding-dresses","lookup":"/weddings/blogs/save-the-date/2014/12/simple-wedding-dresses","social":"/weddings/blogs/save-the-date/2014/12/simple-wedding-dresses"},"stitch":{}}, - stats: {} + stats: {"2000000003015754":{"likes":2}} }; Courier.config.cmPlacements = ["AMS_GLM_GLOBAL_LEFTNAV","AMS_GLM_MOBILE_GLOBAL_NAVBAR","AMS_GLM_GLOBAL_HEADERCALLOUT","AMS_GLM_GLOBAL_HEADERCALLOUT_COLLAPSE","AMS_GLM_BLOG_RIGHTNAV","AMS_GLM_CONTENTPAGE_ZRAIL","AMS_GLM_MOBILE_CONTENTPAGE_ZRAIL","AMS_GLM_MOBILE_GLOBAL_FOOTER","AMS_GLM_GLOBAL_FOOTER","AMS_GLM_GLOBAL_FLYIN"]; Courier.config.ads = { @@ -1014,36 +1014,36 @@

    TRENDING

    - - Surprise! Joseph Gordon-Levitt Is Married + + Princess Mary Had the Dreamiest New Year's Outfit, Ever - - Surprise! Joseph Gordon-Levitt Is Married + + Princess Mary Had the Dreamiest New Year's Outfit, Ever
    - - 9 Love and Sex Resolutions You Should Make for 2015 + + Is This the Most Sexist Relationship Advice Ever? - - 9 Love and Sex Resolutions You Should Make for 2015 + + Is This the Most Sexist Relationship Advice Ever?
    - - 7 Bad Beauty Habits We're Kissing Goodbye to in 2015 + + Huh? The Bra You're Supposed to Wear at Night (I Tried It!) - - 7 Bad Beauty Habits We're Kissing Goodbye to in 2015 + + Huh? The Bra You're Supposed to Wear at Night (I Tried It!)
    @@ -1501,11 +1501,11 @@

    TRENDING

    - + - January is Veganuary. Will You Ditch Meat This Month? - January is Veganuary. Will You Ditch Meat This Month? + What Exactly Is a "Healthy" Sex Life, Anyway? + What Exactly Is a "Healthy" Sex Life, Anyway? @@ -1517,25 +1517,25 @@

    TRENDING

    -     health-fitness + sex-love-life - January is Veganuary. Will You Ditch Meat This Month? - January is Veganuary. Will You Ditch Meat This Month? + What Exactly Is a "Healthy" Sex Life, Anyway? + What Exactly Is a "Healthy" Sex Life, Anyway? -
    By Lexi Petronis
    +
    By Jillian Kramer
    -
    11 hours ago
    +
    12 hours ago
    @@ -1549,15 +1549,15 @@

    TRENDING

    -
    +
    - + - 7 Ways to Actually, Really Stick to Your 2015 Exercise Resolutions - 7 Ways to Actually, Really Stick to Your 2015 Exercise Resolutions + The 5 New Ways To Style Pants That Will Rock Your World + The 5 New Ways To Style Pants That Will Rock Your World @@ -1569,25 +1569,25 @@

    TRENDING

    -     health-fitness + fashion - 7 Ways to Actually, Really Stick to Your 2015 Exercise Resolutions - 7 Ways to Actually, Really Stick to Your 2015 Exercise Resolutions + The 5 New Ways To Style Pants That Will Rock Your World + The 5 New Ways To Style Pants That Will Rock Your World -
    By Lexi Petronis
    +
    By Avery Matera
    -
    15 hours ago
    +
    13 hours ago
    @@ -1601,15 +1601,15 @@

    TRENDING

    -
    +
    - + - 10 Cooking Resolutions to Make (and Keep!) This Year - 10 Cooking Resolutions to Make (and Keep!) This Year + Over Glitter? Never. Glitter Shoes That Are Year-Round Approved + Over Glitter? Never. Glitter Shoes That Are Year-Round Approved @@ -1621,25 +1621,25 @@

    TRENDING

    -     home + fashion - 10 Cooking Resolutions to Make (and Keep!) This Year - Here's to a Healthy and Happy 2015: 10 Cooking Resolutions to Make (and Keep!) + Over Glitter? Never. Glitter Shoes That Are Year-Round Approved + Over Glitter? Never. 17 Glitter Shoes That Are Year-Round Approved -
    By Danica Lo
    +
    By Avery Matera
    -
    17 hours ago
    +
    14 hours ago
    @@ -1657,11 +1657,11 @@

    TRENDING

    - + - 6 Sexual Ways to Heal That Holiday Hangover With Your Man - 6 Sexual Ways to Heal That Holiday Hangover With Your Man + Does Gwen Stefani Wear Makeup to Go Skiing? Here's Your Answer + Does Gwen Stefani Wear Makeup to Go Skiing? Here's Your Answer @@ -1673,25 +1673,25 @@

    TRENDING

    -     sex-love-life + beauty - 6 Sexual Ways to Heal That Holiday Hangover With Your Man - 6 Sexual Ways to Heal That Holiday Hangover With Your Man + Does Gwen Stefani Wear Makeup to Go Skiing? Here's Your Answer + Does Gwen Stefani Wear Makeup to Go Skiing? Here's Your Answer -
    By Jillian Kramer
    +
    By Beth Shapouri
    -
    17 hours ago
    +
    14 hours ago
    @@ -1709,11 +1709,11 @@

    TRENDING

    - + - Dudeoir Is the New Boudoir: Your Groom Deserves Some Sexy Wedding Portraits Too - Dudeoir Is the New Boudoir: Your Groom Deserves Some Sexy Wedding Portraits Too + What to Watch, Read, Stream, and Listen to This Weekend + What to Watch, Read, Stream, and Listen to This Weekend @@ -1725,25 +1725,25 @@

    TRENDING

    -     weddings + entertainment - Dudeoir Is the New Boudoir: Your Groom Deserves Some Sexy Wedding Portraits Too - Dudeoir Is the New Boudoir: Your Groom Deserves Some Sexy Wedding Portraits Too + What to Watch, Read, Stream, and Listen to This Weekend + What to Watch, Read, Stream, and Listen to This Weekend -
    By Kim Fusaro
    +
    By Lynsey Eidell
    -
    17 hours ago
    +
    14 hours ago
    @@ -1770,29 +1770,29 @@

    TRENDING

    - - Surprise! Joseph Gordon-Levitt Is Married + + Princess Mary Had the Dreamiest New Year's Outfit, Ever -
    - Surprise! Joseph Gordon-Levitt Is Married +
    + Princess Mary Had the Dreamiest New Year's Outfit, Ever
    - - 9 Love and Sex Resolutions You Should Make for 2015 + + Is This the Most Sexist Relationship Advice Ever? -
    - 9 Love and Sex Resolutions You Should Make for 2015 +
    + Is This the Most Sexist Relationship Advice Ever?
    - - 7 Bad Beauty Habits We're Kissing Goodbye to in 2015 + + Huh? The Bra You're Supposed to Wear at Night (I Tried It!) -
    - 7 Bad Beauty Habits We're Kissing Goodbye to in 2015 +
    + Huh? The Bra You're Supposed to Wear at Night (I Tried It!)
    @@ -1806,11 +1806,11 @@

    TRENDING

    - - Things That Always Happen in Movie Sex Scenes and Never in Real Life + + 7 Bad Beauty Habits We're Kissing Goodbye to in 2015 -
    - Things That Always Happen in Movie Sex Scenes and Never in Real Life +
    + 7 Bad Beauty Habits We're Kissing Goodbye to in 2015
    diff --git a/tests/data/html/globalnews.ca1.html b/tests/data/html/globalnews.ca1.html index 8cbf0508..cd8b5e2c 100644 --- a/tests/data/html/globalnews.ca1.html +++ b/tests/data/html/globalnews.ca1.html @@ -46,7 +46,7 @@ - + - + @@ -318,7 +318,7 @@ } } // Page targets - googletag.pubads().setTargeting("userregion","national").setTargeting("contentregion","national").setTargeting("page","1747657").setTargeting("template","single-video").setTargeting("category","world").setTargeting("tags",["gardner","infertility","ivf","quadruplets","twins"]).setTargeting("weathertemp","-3"); + googletag.pubads().setTargeting("userregion","national").setTargeting("contentregion","national").setTargeting("page","1747657").setTargeting("template","single-video").setTargeting("category","world").setTargeting("tags",["gardner","infertility","ivf","quadruplets","twins"]).setTargeting("weathertemp","-2"); //SRA not enabled //googletag.pubads().enableSingleRequest(); //googletag.pubads().enableSyncRendering(); @@ -351,7 +351,7 @@ }); googletag.enableServices(); } - var gnca_ads_page_ad_targets = "userregion=national<br/>contentregion=national<br/>page=1747657<br/>template=single-video<br/>category%5B0%5D=world<br/>tags%5B0%5D=gardner<br/>tags%5B1%5D=infertility<br/>tags%5B2%5D=ivf<br/>tags%5B3%5D=quadruplets<br/>tags%5B4%5D=twins<br/>weathertemp=-3"; + var gnca_ads_page_ad_targets = "userregion=national<br/>contentregion=national<br/>page=1747657<br/>template=single-video<br/>category%5B0%5D=world<br/>tags%5B0%5D=gardner<br/>tags%5B1%5D=infertility<br/>tags%5B2%5D=ivf<br/>tags%5B3%5D=quadruplets<br/>tags%5B4%5D=twins<br/>weathertemp=-2"; + + + +
    +
    + + + + + + +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + + + +
    + + +
    + + + + + + + + + + + + + + + + + + + + + +
    + + + +
    + + +
    + +
    + +
    +
    +
    + + +
    +
    + + + + Huffpost Business + +
    + + + + +
    + + Edition: U.S. + + + + + + +
    + + + + + +
    +
    +
    + +
    + +
    + +
    +
      +
    • + +
      • +
    +
    + +
    + +
    +
    + + + +
    +
    + +
    +
    + +
    +
    + + + +
    + +
    + + + + + + +
    +
    +
    + + + + + +
    + + + + + + +
    + + +
    + + +
    +
    + + + +
    +
    + Ben Walsh Headshot +
    +
    +
    + + Become a fan + + + +
    + + +
    +
    + +

    Jamie Dimon Says Banks Are Under Assault As He Announces $4.9 Billion Profit

    + +
    + +
    + + Posted: + + + Updated: + +
    +
    + + + +
    + JAMIE DIMON +
    Bloomberg via Getty Images
    + + + + + + + +
    +
    + + +
    +
    + + +
    +
    + +
    + +
    +
    + + + +

    We should all do so well while under assault.

    JPMorgan Chase earned $4.9 billion in the fourth quarter of 2014, the company announced on Wednesday, down from a year ago, but capping what CEO Jamie Dimon called a record year for the biggest U.S. bank by assets.

    Despite this success, Dimon warned that "banks are under assault," from government regulators.

    +
    +

    "In the old days," Dimon said, "you dealt with one regulator when you had an issue, maybe two. “Now it’s five or six. It makes it very difficult and very complicated.

    "You all should ask the question about how American that is. And how fair that is," he added. "And how complex that is for companies."

    The biggest U.S. bank by assets has had its fair share of trouble with regulators in recent years. In the fourth quarter, JPMorgan paid $1.1 billion to settle charges by U.S. and foreign regulators that its traders had manipulated currency markets. In those settlements, J.P. Morgan did not admit nor deny any wrongdoing.

    "Obviously, companies make mistakes," Dimon said on Wednesday. "We try to resolve it, we try to fix it, we admit it."

    In the past two years, JPMorgan has paid out $14 billion in settlements and fines related to the London Whale trading losses, manipulating the key interest rate benchmark Libor, and issuing bad mortgages that helped lead to the financial crisis.

    The bank earned $39.7 billion in net income over those same two years.

    Despite his complaints, Wall Street lobbyists, and Dimon himself, are orchestrating a "continuing assault" on the 2010 Dodd-Frank financial-reform law that "has achieved remarkable success," as the New York Times put it on Tuesday.

    Dimon personally lobbied for Congress to loosen derivatives rules put in place by Dodd-Frank financial. Late in December, the Federal Reserve gave banks another year to comply with certain aspects of that law's Volcker Rule, which prohibits proprietary trading. And now House Republicans are lining up a vote to extend that grace period until 2019, a full nine years after the bill was put into law.

    If record profits and regulatory rollbacks are Dimon's idea of an assault on banks, it's hard to imagine what he thinks victory looks like.
    +

    +
    + +
    + +
    + +
    +
    +
    + +
    + + + + + + +
    +

    Around the Web

    +

    JPMORGAN CHASE & CO (JPM:New York): Earnings Estimates ...

    JPMorgan Chase reports second-quarter 2014 earnings | J.P. Morgan

    JPMorgan Misses Fourth Quarter Earnings Amid Fixed Income Trading Declines

    Recap: JP Morgan's and Wells Fargo's Earnings

    JPMORGAN CHASE & CO (JPM:New York): Earnings Estimates ...

    What to watch for in JP Morgan earnings

    Tesla, JP Morgan drop; GameStop jumps on solid sales

    + + +
    + +
    +
    + +
     
    + +
    + + + + +
    +
    +

    Conversations

    +
    +
    + +
    +
    +
    + + + +
    + +
    + +
    + +
    +
    +
    + + +
    + +
    + + + + + + + + +
    +
    + + + + +
    + +
    +
    + + + + +
    +
    +
    +
    + + + +
    + +
    + + +
    + + + + +
    + +
    +
    + + +
    + + + + +
    + + + + + + + + + + + + +
    + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/data/html/huffingtonpost.de1.html b/tests/data/html/huffingtonpost.de1.html deleted file mode 100644 index b3f64dd6..00000000 --- a/tests/data/html/huffingtonpost.de1.html +++ /dev/null @@ -1,1749 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Missing Airasia Plane - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - - -
    - - -
    - - - - - - -
    - - - - -
    -
    - - - -
    - - - - - - - - - - - - - - - - - - - - - - - -
    - - - - - -
    - - - -
    - - - Mehr -
    - -
    -
    - -
    - - - - - - - -
    - - - -
    -
    - -
    - - - -
    - -
    - - - - - - - - - - -
    - - - - - - - - - - - - - - - -
    - -
    -
    - - -
    - - -
    -
    -
    - -
    - - -
    - - -
    - -
    - -
    - - - - - - - - - - -
    - -
    - - -
    -
    -
    - - - -
    -

    Missing Airasia Plane

    -
    -
    -
    - -
    - - -
    -
    - - - - - - - - -
    -
    - -
    - -
    - -
    -
    - - - -
    - -
    - - -
    - - - - - -
    -
    - -
    -
    - -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/data/html/huffingtonpost.de2.html b/tests/data/html/huffingtonpost.de2.html deleted file mode 100644 index f60be2eb..00000000 --- a/tests/data/html/huffingtonpost.de2.html +++ /dev/null @@ -1,2182 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #HuffPostInspiration - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    - - -
    - - -
    - - - - - - -
    - - - - -
    -
    - - - -
    - - - - - - - - - - - - - - - - - - - - - - - -
    - - - - - -
    - - - -
    - - - Mehr -
    - -
    -
    - -
    - - - - - - - -
    - - - -
    -
    - -
    - - - -
    - -
    - - - - - - - - - -
    - - - - - - -
    -
    - -
    - - - -
    - -
    - - - - - - - - - - - - - - - -
    - -
    -
    - - -
    - - -
    -
    -
    - -
    - - -
    - - -
    - -
    - -
    - - - - - - - -
    -
    -
    -
    - -
    - RSS-Feed dieser Seite -
    - - - -
    -
    -
    -
    - - - - - - - - -
    -
    -
    - -
    -
    - - -
    -
    -
    - Es sieht aus wie ein Schnappschuss, aber dahinter steckt eine unangenehme Botschaft -
    -
    -

    - - - Es sieht aus wie ein Schnappschuss, aber dahinter steckt eine unangenehme Botschaft -

    - Bianca Waldera  - 29.12.2014 -
    - Alles begann damit, dass sich Haley Morris-Cafiero am Time Square fotografieren ließ. Später entdeckte sie auf dem Bild einen Mann, der...
    - -
    -
    -
    - - -
    -
    -
    - Wunderbare Chalets -
    -
    -

    - - - Wunderbare Chalets -

    - 24.12.2014 -
    - Wie sollte man das Wohnzimmer gestalten, die Küche einrichten oder das Schlafzimmer dekorieren, damit es zu Hause genau so gemütlich...
    - -
    -
    -
    - - -
    -
    -
    - Diese Menschen haben Wünsche, die man mit Geld nicht kaufen kann -
    -
    -

    - - - Diese Menschen haben Wünsche, die man mit Geld nicht kaufen kann -

    - Andrea Maier  - 22.12.2014 -
    - Gehören Sie zu den Menschen, die Weihachten längst satt haben? Die wie Ottfried Fischer am liebsten den Adventskranz aus dem Fenster...
    - -
    -
    -
    - - -
    -
    -
    - Unglaubliche Vorher-Nachher-Bilder: Zusammen änderte dieses Paar sein Leben -
    -
    -

    - - - Unglaubliche Vorher-Nachher-Bilder: Zusammen änderte dieses Paar sein Leben -

    - Sarah Klein  - 19.12.2014 -
    - Der Dezember nähert sich seinem Ende und viele von uns versinken in einer Art Jahresrückblick. Sie denken nicht nur darüber nach, was...
    - -
    -
    -
    - - -
    -
    -
    - Wie ihr eure vier Wände mit Beton verschönern könnt -
    -
    -

    - - - Wie ihr eure vier Wände mit Beton verschönern könnt -

    - 18.12.2014 -
    - Im Gegensatz zu Materialien wie Milchglas oder Edelstahl gilt Beton in erster Linie als ein negativ besetztes Wort. Betonklotz, Betonwüste,...
    - -
    -
    -
    - - -
    -
    -
    - 17 Geschenkideen für Menschen, die schon alles haben -
    -
    -

    - - - 17 Geschenkideen für Menschen, die schon alles haben -

    - Andrea Maier  - 16.12.2014 -
    - Weihnachten rückt immer näher und selbst wenn wir es geschafft haben, Geschenke für den engsten Familienkreis zu besorgen, sind da immer...
    - -
    -
    -
    - - -
    -
    -
    - Meditation: 7 Gründe, warum Sie noch heute anfangen sollten -
    -
    -

    - - - Meditation: 7 Gründe, warum Sie noch heute anfangen sollten -

    - Gina Louisa Metzler   - 13.12.2014 -
    - Seit Tausenden von Jahren beziehen Menschen auf der ganzen Welt Kraft und Energie aus der Meditation. Die verschiedenen Techniken und Übungen...
    - -
    -
    -
    - - -
    -
    -
    - Atemberaubend: Diese 2 Minuten zeigen, was ein Mann in 90 Jahren sah -
    -
    -

    - - - Atemberaubend: Diese 2 Minuten zeigen, was ein Mann in 90 Jahren sah -

    - Lea Kosch  - 12.12.2014 -
    - Es gibt Menschen, die die Welt mit ihren Ideen voranbringen. Die mit ihrem Engagement versuchen, die Erde zu einem besseren Ort zu machen....
    - -
    -
    -
    - - -
    -
    -
    - Die richtigen Songs zum Start in den Tag -
    -
    -

    - - - Die richtigen Songs zum Start in den Tag -

    - Lissalina Marwig  - 10.12.2014 -
    - Morgens aus dem Bett zu kommen ist eine echte Herausforderung. Besonders wenn es morgens noch dunkel ist. Sie machen vielleicht sogar Fehler...
    - -
    -
    -
    - - -
    -
    -
    - Die nackte Wahrheit: Frau zeigt, wie sich ihr Körper in 7 Jahren verändert -
    -
    -

    - - - Die nackte Wahrheit: Frau zeigt, wie sich ihr Körper in 7 Jahren verändert -

    - Katherine Brooks  - 4.12.2014 -
    - Vor sieben Jahren begann die Fotografin Polly Penrose mit ihrem Projekt „A body of work“ (Körper der Arbeit). Seitdem macht sie Selbstportraits...
    - -
    -
    -
    - - -
    -
    -
    - Spektakuläre Häuser-Fassaden -
    -
    -

    - - - Spektakuläre Häuser-Fassaden -

    - 3.12.2014 -
    - Der erste Eindruck zählt - das gilt nicht nur fürs Kennenlernen, sondern auch dann, wenn man die Person zum ersten Mal zu Hause besucht....
    - -
    -
    -
    - - -
    -
    -
    - Kreative Menschen haben mehr Sex, nehmen häufiger Drogen und bleiben länger wach -
    -
    -

    - - - Kreative Menschen haben mehr Sex, nehmen häufiger Drogen und bleiben länger wach -

    - Gina Louisa Metzler  - 28.11.2014 -
    - Was macht kreative Menschen aus? Was unterscheidet sie von anderen? Anders als lange angenommen, liegt der Schlüssel zum kreativen Denken...
    - -
    -
    -
    - - -
    -
    -
    - Innovationskraft und Inspiration sind kein Privileg der Jugend -
    -
    -

    - - - Innovationskraft und Inspiration sind kein Privileg der Jugend -

    - 25.11.2014 -
    - Junge Arbeitnehmer gelten als innovativ, mutig, motiviert und kreativ. Die jungen Wilden sind es, die Innovation vorantreiben. Alte Arbeitnehmer...
    - -
    -
    -
    - - -
    -
    -
    - 25 Jahre Mauerfall: In diesen FOTOS verschmilzt die Zeit davor und danach -
    -
    -

    - - - 25 Jahre Mauerfall: In diesen FOTOS verschmilzt die Zeit davor und danach -

    - Chris McGonigal & Eline Gordts  - 7.11.2014 -
    - Es war ein Moment für die Ewigkeit: Am 9. November 1989 strömten Tausende Ostberliner zur Grenze, die sie vom Westen trennte und verlangten,...
    - -
    -
    -
    - -
    -Seiten:   -1 - 3 - 4 - 5 - 6 -
    -
    - - - - - - - -
    -
    - -
    - -
    - -
    -
    - - - -
    - -
    - - -
    - - - - - -
    -
    - -
    -
    - -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/data/html/japanese_article.html b/tests/data/html/japanese_article.html new file mode 100644 index 00000000..bb7e1bbf --- /dev/null +++ b/tests/data/html/japanese_article.html @@ -0,0 +1,1940 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +米朝首脳直通電話　習氏に挑むトランプ戦術の成否　（写真＝ロイター）　：日本経済新聞 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + +
    +
    +
    + +
    + +
    +
    +
    +
    +
    +
    +
    + 日本経済新聞
    +
    2018年6月20日（水）
    +
    +
    + + International + +
    +
    +
    + + + +
    お申し込み
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +
    +
    + +
    +
    + + +
    +
    +
    +
    +
    +
    + +
    + + + +
    +
    +
    +
    +
    +
    +
    +
    + +
    + + +
    + +
    + + +
    +
    + +
    +
    +
    + +
    + +
    + +
    +
    +
    +

    + 米朝首脳直通電話　習氏に挑むトランプ戦術の成否 + +
    + 編集委員　中沢克二 +

    +
    + +
    + 北朝鮮
    +
    + トランプ政権
    +
    + 南北首脳会談
    +
    + 習政権ウォッチ
    +
    + 習政権
    +
    + 経済
    +
    + 政治
    +
    + コラム（国際・アジア）
    +
    + 中国・台湾
    +
    + 朝鮮半島
    +
    + 北米
    +
    + +
    +
    （1/3ページ）
    +
    2018/6/19 5:50
    +
    情報元
    +
    日本経済新聞　電子版
    +
    +
    + +
    +
    + +
    +
    + +
    + + + + + +
    +
    +
    保存
    + 共有 + +
    +
    +
    + その他
    +
    +
    + + + +
    +
    +

    　６月12日の歴史的な米朝首脳会談を巡って戦っていたのは、米大統領のトランプと北朝鮮委員長の金正恩（キム・ジョンウン）ばかりではなかった。中国国家主席、習近平（シー・ジンピン）も重要な陰の参戦者だった。核放棄を巡って「検証可能な」の文字がなく、米韓軍事演習の中止にまで踏み込んだ米朝会談は、トランプによる金正恩への一方的譲歩であり、後ろ盾だった中国の習近平も「勝ち組」になったかに見える。

    　その判断は…

    + +
    +

    ［有料会員限定］ この記事は会員限定です。電子版に登録すると続きをお読みいただけます。

    +
    + + +
    + + + +
    +

    + 電子版トップ

    +
    +

    + 関連キーワード +

    + +
    + +
    + +
    + + +
    + +
    +
      +
      +
    +
      +
      +
      +
      +
      +
      + +
    +
    + +
    + +
    + +
    +
    +
      + +
      + +
      +
    +
    +
    +

    [PR]トレンドウオッチ

    +
    一覧
    +
    +
    +
    +
    +
      + +
      + +
      +
    +
    +
    +
    +
    +
      + +
      + +
      +
    +
    +
    +
    +
    +
      + +
      + +
      +
    +
    +
    +
    +
    +
      + +
      + +
      +
    +
    +
    +
    +
    +
    +
    + +
    +
    + +
    +
    +
    +
      +
    • + 日経平均(円)
      + 6/19 大引 +
      • +
    • 22,278.48
      • +
    • -401.85
      • +
    • -1.77%
      • +
      +
    • + NYダウ(ドル)
      + 6/19 11:06 +
      • +
    • 24,650.14
      • +
    • -337.33
      • +
    • -1.34%
      • +
      +
    • + 日経アジア300
      + 6/19 終値 +
      • +
    • 1,342.54
      • +
    • -32.61
      • +
    • -2.37%
      • +
      +
    • + ドル(円)
      + 6/20 0:16 +
      • +
    • 109.85-86
      • +
    • -0.62円高
      • +
    • -0.56%
      • +
      +
    • + ユーロ(円)
      + 6/20 0:16 +
      • +
    • 127.06-10
      • +
    • -0.78円高
      • +
    • -0.61%
      • +
      +
    • + 長期金利(％)
      + 6/19 16:56 +
      • +
    • 0.030
      • +
    • ±0.000
      • +
    • +
      +
    • + NY原油(ドル)
      + 6/19 11:11 +
      • +
    • 64.72
      • +
    • -1.13
      • +
    • -1.71%
      • +
    + +
    +
    日経平均について + （銘柄一覧）
    +
    QUICK
    +
    +
    +
    + +
    + +
    +
    + +
    +
    +
    +
    日経からのお知らせ
    +

    キャリア採用　記者募集

    +
    +
    +
    +
    日経電子版をもっと知りたい方はこちら
    +

    日経産業新聞をデジタルで読みませんか？

    +
    +
    +

    おすすめ情報

    + +
    + +
    +

    [PR]

    一覧はこちら
    + +
    +
    + +
    +
    +

    + アクセスランキング + + 一覧 +

    + +
    6/20 0:00 更新
    +
    +
    +
    +
    +
    + +
    +
    +
    +
    +
    +
    +

    + 日経ＢＰ社

    +
    + +
    +
    +

    日経 xTECH（クロステック）

    +

    + IT業界内での知名度を売り物にするエンジニアたち +

    +
    +

    日経クロストレンド

    +

    + ディープラーニングはヒット曲を作れるか +

    +
    +
    +
    +
    +

    + 日経ビジネスオンライン

    +
    + +
    +
    +

    「スジ」の日本、「量」の中国

    +

    + 「法律違反だが犯罪ではない」が成立する中国 +

    +
    +

    トレンド・ボックス

    +

    + 酒を断つには、人間関係もリセットするしかない +

    +
    +
    +
    +
    +

    + NIKKEI STYLE

    +
    + +
    +
    +

    MONO TRENDY

    +

    + 代表ユニホーム　ジャパンブルーは藍染めの青だった +

    +
    +

    出世ナビ　リーダーの母校

    +

    + クジラ研究で鍛えた交渉力　凄腕コンサルの筑波大時代 +

    +
    +

    ヘルスＵＰ

    +

    + 片足立ち、階段…　毎日やれば筋トレになる4つの習慣 +

    +
    +

    グルメクラブ　World Food Watch

    +

    + 韓国式お好み焼き、チヂミはなぜ雨の日に食べる？ +

    +
    +
    +
    +
    +

    + 日経BizGate +

    +
    + +
    +
    +

    東大卒棋士のＡＩ勝負脳

    +

    + 15歳の「藤井聡太」はなぜ将棋に勝てるのか +

    +
    +

    危機管理広報　エイレックス江良俊郎社長に聞く

    +

    + 謝罪会見「絶対ダメ」なあの言葉 +

    +
    +
    +
    +
    +

    + College Cafe

    +
    + +
    +
    +

    ハーバードと日本の架け橋

    +

    + 茶の世界を共有 +

    +
    +

    ホンネの就活ツッコミ論

    +

    + Ｗ内定をどうする？ +

    +
    +
    +
    +
    +

    +

    +
    + +
    +
    +

    ウーマノミクス・ストーリー

    +

    + 英語使える日本人を少しでも、長年の課題に新プログラムで挑む（ECC） +

    +
    +
    +
    +
    +

    + NIKKEI SmartWork

    +
    + +
    +
    +

    PICK UP

    +

    + 人も仕事も多様性を追求、イノベーション創り出す丸紅へ +

    +
    +
    +
    +
    +

    + Paravi

    +
    + +
    +
    +

    予定調和を壊す生放送への思い

    +

    + 青春高校３年Ｃ組～秋元康インタビュー (無料) +

    +
    +

    こだわる男は、かっこいい

    +

    + 俺のダンディズム（テレビ東京ドラマ） +

    +
    +
    +
    +

    + 日経のイベント・セミナー +

    + +
    +

    + + 天気 + プレスリリース検索

    +
    + + +
    +
    +

    訂正・おわび

    +
    +
    +
    + +
    + +
    +
    +
    +
    +
    +
    +
    +

    日本経済新聞社の関連サイト

    + +
    +
    +

    日経ＩＤの関連サイト

    + +
    +
    +
    + +
    +
    +

    日本経済新聞 関連情報

    +
    +

    イベント・セミナー

    + +
    +
    +

    教育・キャリア

    + +
    +
    +

    各種サービス

    + +
    +
    +
    +
    +
    +

    日本経済新聞 電子版について

    + +
    +
    +

    日本経済新聞社について

    + +
    +
    +
    +
    +
    + + Nikkei Inc. No reproduction without permission. +
    +
    + + + + + + + + + + + + + + + +
    + + + + diff --git a/tests/data/html/japanese_article2.html b/tests/data/html/japanese_article2.html new file mode 100644 index 00000000..539681af --- /dev/null +++ b/tests/data/html/japanese_article2.html @@ -0,0 +1,699 @@ + + + + + + + + + + +動画：ナイジェリアで少女らによる自爆攻撃、ロケット弾も撃ち込まれ31人死亡　写真1枚　国際ニュース：AFPBB News + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + +
    +
    +
    +
    + +
    +
    + + + + +ニュース + + +戦争・紛争 + + + + + + + + + + + + + + + + + + + + + + + + + +Video + + + + +
    + +

    動画：ナイジェリアで少女らによる自爆攻撃、ロケット弾も撃ち込まれ31人死亡

    + +
    + 2018年6月18日 10:01　発信地：カノ/ナイジェリア + + [ + ナイジェリア + アフリカ + ] + + +
    + + +
    + +
    +

    このニュースをシェア

    + + +
    + + + +
    +
    + + + +
    + +
    + + + + + + + +
    + + + + +
    + + + + + + + + +
    +

    【6月18日 AFP】ナイジェリア北東部ボルノ（Borno）州で16日夜、イスラム過激派組織「ボコ・ハラム（Boko Haram）」によるものと思われる少女を使った自爆攻撃が起き、31人が死亡した。地元当局者と民兵指導者が17日、AFPに明らかにした。

    +

    　攻撃はボルノ州の町ダンボア（Damboa）で、イスラム教の断食月「ラマダン（Ramadan）」明けの祭り「イード・アル・フィトル（Eid al-Fitr）」の祝いから帰る途中の人々を狙って起きたもので、ボコ・ハラムの犯行と思われる特徴があった。

    +

    　数回の自爆の後、襲撃者らは自爆攻撃の現場に集まった群集の中に携行式ロケット弾を撃ち込み、死傷者がさらに増えた。

    +

    　地元の民兵指導者は「昨夜ダンボアで2度の自爆攻撃とロケット弾による爆発があり、31人が死亡した。その他に数人が負傷した」と語り、ボコ・ハラムの犯行であることは明らかだと述べた。

    +

    　地元当局者は「死傷者の大半は町の外から発射されたロケット弾によるものだった。事件後に、自爆攻撃が6人の少女によって実行されたことが明らかになった。救急隊が現場で6人の頭部を発見した。顔つきからして7歳から10歳までの少女だった」と述べた。

    +

    　政府はボコ・ハラムが劣勢にあると繰り返し主張しているが、あるアナリストは「ボコ・ハラムはナイジェリア北東部で大量の死傷者を出す攻撃を行う意図と作戦能力を維持している」と指摘している。(c)AFP/Aminu ABUBAKAR

    +
    + + + + + + + +
    + + + + + + +戦争・紛争 一覧へ> + + + + + + + + + + + + + + + + + +
    +
    + メディア・報道関係・法人の方向け 写真・動画 購入のお問合せ> +
    + + +
    + + + + + + +
    + +
    +
    +
    +

    あわせて読みたい

    +
    +
    + + +
    + +
    + + +
    + +
    +
    +
    + + +
    + AFPBB NewsAFPBB News +

    AFPBB Newsに掲載している写真・見出し・記事の無断使用を禁じます。 © AFPBB News

    +
    + + + + + + + + + + + + + + + +
    + +
    + +
    + +
    + + + + + + + + diff --git a/tests/data/html/lifebuzz.com1.html b/tests/data/html/lifebuzz.com1.html index a2ae241f..97da95ba 100644 --- a/tests/data/html/lifebuzz.com1.html +++ b/tests/data/html/lifebuzz.com1.html @@ -159,6 +159,16 @@

    Live Inspired

    + + - -
    @@ -281,6 +281,21 @@

    Live Inspired

    + + + - - -
    @@ -500,7 +500,7 @@

    She Was Tired Of Being Photoshopped, So Here’s What She Did About It.

    - Next Post + Next Post
    @@ -640,47 +640,47 @@

    30 Facts That Are So Irresistibly Weird You’ll Want T @@ -693,58 +693,58 @@

    Featured Videos

    @@ -757,58 +757,58 @@

    Latest from LifeBuzz

    @@ -821,88 +821,88 @@

    Photo Galleries

    diff --git a/tests/data/html/lifebuzz.com2.html b/tests/data/html/lifebuzz.com2.html index f9728a4d..79d8ecf5 100644 --- a/tests/data/html/lifebuzz.com2.html +++ b/tests/data/html/lifebuzz.com2.html @@ -159,6 +159,16 @@

    Live Inspired

    + + - -
    @@ -281,6 +281,21 @@

    Live Inspired

    + + + - - -
    @@ -750,47 +750,47 @@

    30 Facts That Are So Irresistibly Weird You’ll Want T @@ -803,58 +803,58 @@

    Featured Videos

    @@ -867,58 +867,58 @@

    Latest from LifeBuzz

    @@ -931,88 +931,88 @@

    Photo Galleries

    diff --git a/tests/data/html/livescience.com1.html b/tests/data/html/livescience.com1.html index ff83f739..320dade8 100644 --- a/tests/data/html/livescience.com1.html +++ b/tests/data/html/livescience.com1.html @@ -341,7 +341,7 @@
    TRENDING: Geek Gifts - // + // Ebola Outbreak // Military & Spy Tech @@ -729,7 +729,7 @@

    Follow Us

    - Copyright © 2014 All Rights Reserved. + Copyright © 2015 All Rights Reserved.
    diff --git a/tests/data/html/livescience.com2.html b/tests/data/html/livescience.com2.html index 66a3ede2..56181864 100644 --- a/tests/data/html/livescience.com2.html +++ b/tests/data/html/livescience.com2.html @@ -341,7 +341,7 @@
    TRENDING: Geek Gifts - // + // Ebola Outbreak // Military & Spy Tech @@ -745,7 +745,7 @@

    Follow Us

    - Copyright © 2014 All Rights Reserved. + Copyright © 2015 All Rights Reserved.
    diff --git a/tests/data/html/mashable.com1.html b/tests/data/html/mashable.com1.html index 6fbc2c7b..46ff9678 100644 --- a/tests/data/html/mashable.com1.html +++ b/tests/data/html/mashable.com1.html @@ -18,14 +18,14 @@ I'm Feeling Lucky: Google's New Year Surprise - + @@ -45,7 +45,7 @@ - + @@ -252,7 +252,7 @@

    diff --git a/tests/data/html/mashable.com2.html b/tests/data/html/mashable.com2.html index 6814cf0a..4897cb54 100644 --- a/tests/data/html/mashable.com2.html +++ b/tests/data/html/mashable.com2.html @@ -18,7 +18,7 @@ - + @@ -229,7 +229,7 @@

    P
    Stansted
    Stansted Airport, Dec. 22, 2014.
    Image: Jiri Hosek
    -
    Avatar
    By Jessica Plautz
    +
    Avatar
    By Jessica Plautz
    @@ -305,8 +305,8 @@

    P
    diff --git a/tests/data/html/mlive.com1.html b/tests/data/html/mlive.com1.html index 1f298956..85e4242a 100644 --- a/tests/data/html/mlive.com1.html +++ b/tests/data/html/mlive.com1.html @@ -895,6 +895,44 @@

    Police release surveillance photos in Dec. 22 University Bank robbery

    Loading Photo Gallery

    + + + + + + + + + + + +
    @@ -1384,19 +1422,19 @@

    Best of MLive.com

    @@ -1404,19 +1442,19 @@

    Best of MLive.com

    @@ -1424,19 +1462,19 @@

    Best of MLive.com

    @@ -1600,7 +1638,7 @@

    Follow Us

    Registration on or use of this site constitutes acceptance of our User Agreement and Privacy Policy

    -

    © 2014 MLive Media Group All rights reserved (About Us).
    The material on this site may not be reproduced, distributed, transmitted, cached or otherwise used, except with the prior written permission of MLive Media Group

    +

    © 2015 MLive Media Group All rights reserved (About Us).
    The material on this site may not be reproduced, distributed, transmitted, cached or otherwise used, except with the prior written permission of MLive Media Group

    Community Rules apply to all content you upload or otherwise submit to this site. Contact interactivity management.

    Ad Choices

    diff --git a/tests/data/html/mlive.com2.html b/tests/data/html/mlive.com2.html index e596b1a5..fe8b198e 100644 --- a/tests/data/html/mlive.com2.html +++ b/tests/data/html/mlive.com2.html @@ -1226,19 +1226,19 @@

    Best of MLive.com

    @@ -1246,19 +1246,19 @@

    Best of MLive.com

    @@ -1266,19 +1266,19 @@

    Best of MLive.com

    @@ -1442,7 +1442,7 @@

    Follow Us

    Registration on or use of this site constitutes acceptance of our User Agreement and Privacy Policy

    -

    © 2014 MLive Media Group All rights reserved (About Us).
    The material on this site may not be reproduced, distributed, transmitted, cached or otherwise used, except with the prior written permission of MLive Media Group

    +

    © 2015 MLive Media Group All rights reserved (About Us).
    The material on this site may not be reproduced, distributed, transmitted, cached or otherwise used, except with the prior written permission of MLive Media Group

    Community Rules apply to all content you upload or otherwise submit to this site. Contact interactivity management.

    Ad Choices

    diff --git a/tests/data/html/newyorker.com1.html b/tests/data/html/newyorker.com1.html index 6e06a1f4..9b7d3336 100644 --- a/tests/data/html/newyorker.com1.html +++ b/tests/data/html/newyorker.com1.html @@ -53,7 +53,7 @@ - + @@ -159,7 +159,7 @@ if ( !result.allowAccess ) { window.location.reload(); } - }, null, "54a67a44e4b0947ff843c9da") + }, null, "54a7d6cce4b0947ff8551c57") @@ -432,10 +432,10 @@
  • Culture
  • Books & Fiction
  • Science & Tech
    • +
  • Business
  • Humor
  • Magazine
  • Archive
    • -
  • Video
  • + + + \ No newline at end of file diff --git a/tests/data/html/nydailynews.com2.html b/tests/data/html/nydailynews.com2.html deleted file mode 100644 index cf317dff..00000000 --- a/tests/data/html/nydailynews.com2.html +++ /dev/null @@ -1,511 +0,0 @@ - - - - - - - - - - - - - - - - - - - - News Photo Galleries - NY Daily News - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    • Friday, January 2, 2015
    • NYDailyNews.com /

      News Pics

    - - - - - - - -
      -
    • Subscriptions & Services - -
      • - - - -
    • -
      - -
      -
      - - -
      -
      • - -
    - - - - - -
    - - - - - - - -
    - - - -
    - - -
    - - - - - - - - - - - - - - - - - - - News Pics​​ - - - -
    New Year's Day Worldwide Plunge

    New Year's Day Worldwide Plunge

    Every New Year's Day, thousands of people from around the world take a plunge into water as a way to start the new year. While some are lucky eno...

    36 dead in New Year's stampede in Shanghai

    36 dead in New Year's stampede in Shanghai

    At least 36 people have been killed in a stampede during New Year's celebrations in Shanghai as thousands packed onto the Bund to watch the light...

    AirAsia relatives distraught as bodies found in sea

    AirAsia relatives distraught as bodies found in sea

    Friends and family were left completely overcome with grief as AirAsia confirms bodies recovered in the Java Sea are from flight 8501 -- carrying...

    Happy New Year! How the world celebrated the start of 2015

    Happy New Year! How the world celebrated the start of 2015

    2014 has come and gone with a bang. With a new year upon us, see how the rest of the world is celebrating the start of 2015.

    Happy New Year! The world rings in 2015

    Happy New Year! The world rings in 2015

    People around the world celebrate the start of 2015.

    Plane mysteries through history

    Plane mysteries through history

    A massive manhunt for the 162 victims of AirAsia Flight 8501 continues in the Java Sea, the next in a long string of planes that have disappeare...

    The funeral of slain NYPD officer Rafael Ramos

    The funeral of slain NYPD officer Rafael Ramos

    Thousands of police officers, family and friends from around the country attend the funeral of slain NYPD officer Rafael Ramos. Ramos, a 40-year-...

    Best New York Daily News Front Covers of 2014

    Best New York Daily News Front Covers of 2014

    From the death of Robin Williams and Philip Seymour Hoffman to the rise of ISIS, the Daily News has covered the biggest stories of 2014.

    N.Y. mourns slain NYPD officers

    N.Y. mourns slain NYPD officers

    NYPD officers Rafael Ramos and Wenjian Liu were assassinated by gunman Ismaaiyl Brinsley while sitting in marked patrol car in Brooklyn, New York...

    Top news stories of 2014

    Top news stories of 2014

    As 2014 winds down, take a look back at some of the biggest news stories of the year ...

    Asia remembers 2004 Indian Ocean Tsunami

    Asia remembers 2004 Indian Ocean Tsunami

    On Dec. 26, 2004, a 9.3-magnitude earthquake generated a series of massive waves that killed more than 220,000 people across 14 countries. Ten ye...

    Boxing Day tsunami: 10 years later

    Boxing Day tsunami: 10 years later

    The devastating 9.1 magnitude earthquake that caused the 2004 Boxing Day tsunami left more than 230,000 dead in 14 countries. Take a look back at...

    The best Daily News photos of 2014

    The best Daily News photos of 2014

    Through the lens of New York Daily News photographers, take a look back at the years most striking moments

    Police killing of black teenager triggers violent unrest

    Police killing of black teenager triggers violent unrest

    An 18-year-old black teenager has been shot dead by police at a gas station in St. Louis, Missouri on Dec. 24, 2014, just 5 miles from where Mich...

    2015 Sony World Photography Awards

    2015 Sony World Photography Awards

    With the Sony World Photography Awards contest quickly coming to an end, the top 15 best entries in the 'open' category so far have been surfacin...

    National Geographic Photo Contest 2014

    National Geographic Photo Contest 2014

    After receiving more than 9,200 enteries from over 150 countries, the winners of the 2014 National Geographic Photo Contest have finally been ann...

    The decline and decay of Cuba

    The decline and decay of Cuba

    In 1962, the United States set an embargo against Cuba effectively cutting off trade between the two countries. Since then, Cuba has been faced w...

    Fidel Castro does New York

    Fidel Castro does New York

    Fidel Castro came to New York, hired a public relations firm, ate hot dogs, kissed ladies like a rock star, and held babies like a politician. Lo...

    Sydney hostage crisis

    Sydney hostage crisis

    A gunman, identified by local media as 49-year-old Iranian refugee Man Haron Monis, held staff and customers at Lindt Chocolate Cafe in Sydney, A...

    Most shocking red carpet looks of 2014

    Most shocking red carpet looks of 2014

    It was the year of more skin and less dress! From plunging necklines to cut-outs and slits, 2014 had no shortage of shocking dresses on the red c...

    SantaCon 2014 in New York City

    SantaCon 2014 in New York City

    Thousands of revelers dress up as Santa in New York City for a day full of holiday fun.

    Anti-police violence protests spread across the country

    Anti-police violence protests spread across the country

    Thousands of protesters gathered in New York City, Washington D.C., and elswehere for an anti-police violence demonstration in response to the po...

    Royal baby photo album

    Royal baby photo album

    There's another royal baby on the way! In celebration of Kate Middleton and Prince William's news that they're expecting their second child, take...

    Mudslide devastates town in California

    Mudslide devastates town in California

    Homes in the Camarillo Springs area of Ventura County, California are left devastated after a heavy rain and wind storm triggered a mudslide on D...

    Daredevil surfers ride giant waves in Portugal

    Daredevil surfers ride giant waves in Portugal

    Thrill-seeking surfers descended on a small fishing village in Portugal to ride huge 'weather bomb' waves before aww-struck spectators on Dec. 11...

    Bill Cosby sexual assault allegations

    Bill Cosby sexual assault allegations

    He was America's favorite TV Dad but now people are changing their tune as more sexual assault allegations shed light on the actors past.

    Weather bomb batters Britain

    Weather bomb batters Britain

    High winds, huge waves and flash flooding hit parts of the U.K., as the so-called 'weather bomb' rolled over the country, leaving thousands witho...

    Protests erupt after Eric Garner grand jury decision

    Protests erupt after Eric Garner grand jury decision

    Following the Ferguson verdict and protests, thousands gathered across the country after a New York grand jury decided not to indict the police o...

    37th Annual Kennedy Center Honors

    37th Annual Kennedy Center Honors

    Sting, Tom Hanks, Al Green, Lily Tomlin and Patricia McBride receive Kennedy Center Honors at the 37th Annual Kennedy Center Honors gala hosted b...

    The life and death of John Lennon

    The life and death of John Lennon

    It has been 34 years since the death of John Lennon - English rock star, 'Beatles' heartthrob, and music legend. On Dec. 8, 1980, Mark David Chap...

    - - -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tests/data/html/nypost.com1.html b/tests/data/html/nypost.com1.html index 02f64de1..e1136215 100644 --- a/tests/data/html/nypost.com1.html +++ b/tests/data/html/nypost.com1.html @@ -12,8 +12,8 @@ @@ -47,7 +47,7 @@ - + @@ -502,7 +502,7 @@
  • -
    +
  • @@ -530,9 +530,7 @@ +
  • @@ -571,7 +569,7 @@

    Thanks for contacting us. We've received your submission.

    - -

    What’s New On Netflix Streaming in 2015: ‘Friends’, ‘Election’, Hunter S. Thompson Tales And More

    + +

    Girl Gets Caught Cheating On National Television … Or Does She?

    -

    The 14 Hottest, Weirdest, And Shocking Sex Scenes Of 2014 (Gallery)

    +

    2015 Midseason Premiere Dates: When Your Favorite Show Is Coming Back – And Where You Can Stream It Now

    -

    EXCLUSIVE VIDEO: Kevin Spacey Sings ‘Piano Man’ … Really!

    +

    The 14 Hottest, Weirdest, And Shocking Sex Scenes Of 2014 (Gallery)

    @@ -1025,13 +1023,13 @@

    Now On

    - -

    Met costume designer spends Christmas hand-beading for debut

    + +

    Johnny Football resolves to stop party antics — after NYE

    -

    Paltrow pouts about haters, says divorce going great

    +

    Coco Rocha reveals her stickiest fashion show situation

    -

    The real Rod Stewart spotted at Trump’s swanky NYE party

    +

    VIPs rally behind Amy Adams after axed ‘Today’ interview

    @@ -1095,11 +1093,11 @@

    SEE ALL

    @@ -1107,11 +1105,11 @@

    @@ -1129,12 +1127,12 @@

    -

    Revlon CEO rips blacks, Jews, hates ‘dirty’…

    - -
    -
    -
    +     @@ -1142,11 +1140,11 @@

    @@ -1154,11 +1152,11 @@

    @@ -1166,11 +1164,11 @@

    @@ -1178,11 +1176,11 @@

    @@ -1673,7 +1671,7 @@

    Sign in to complete account merge

    _tkq.push(['storeContext', {'blog_id':'56757169','blog_tz':'-5','user_lang':'en','blog_lang':'en','user_id':'0'}]); st_go({'blog':'56757169','v':'wpcom','tz':'-5','user_id':'0','post':'8344329','subd':'thenypost'}); function st_vt() {var x=document.createElement("img");x.src="http://pixel.wp.com/g.gif?blog=56757169&v=wpcomvt&tz=-5&user_id=0&post=8344329&subd=thenypost&rand="+Math.random();} -ex_go({'crypt':'UE40eW5QN0p8M2Y/RE1BNmNJfGhxNCVxUDExYmtib2E/SzdEJm0lUEtbdFU0fFk4cHZvcUFrZT90Tn4yQm1+R3NyNTdhJitIV3pwXVM1Smt6THRKS1srLiZtZ2FVYk5EaDB2SnYzWVV5TlFXaixdQ0J1NFswUG5IWT9wUkVGVkMrcmEvYzM1ekk3bSZVX1lqLy5DXXUtP3wwcGM4Zy5hNmd2Q1dFLUItWDNKZGhCTkxNdUFtcnBDP0MmOVJJWjg1OFQ5WXMxU2N5L1g5VkFZM3I3eWwvNU5jNT0sfnpsQVdnWjJ8eGttTm5XZ3FdT0RyX1pySFB6VVB6WQ=='}); +ex_go({'crypt':'UE40eW5QN0p8M2Y/RE1BNmNJfGhxNCVxUDExYmtib2E/SzdEJm0lUEtbdFU0fFk4cHZvcUFrZT90Tn4yQm1+R3NyNTdhJitIV3pwXVM1Smt6THRKS1srLiZtZ2FVYk5EaDB2SnYzWVV5Tj9KLTlRRzlFd340ZUVPSHFleWNQOXFNQXBaLmZ1UkctL0liVVBuREh6OVldfGQ0RVklcFdPZkg5ZEZhWjldZGZZQ0xWfD1zdGhdJVl6JktkaWxDVHgua01aVzlDLWVDbUlEWnguPTYtJmRWZ0NXRmtBZmRiM090VGZxPXJpJmV1T3pRdG9oQ3xVUFBzLDlnbQ=='}); addLoadEvent(function(){linktracker_init('56757169',8344329);}); diff --git a/tests/data/html/nypost.com2.html b/tests/data/html/nypost.com2.html index b71ead50..ebec87eb 100644 --- a/tests/data/html/nypost.com2.html +++ b/tests/data/html/nypost.com2.html @@ -11,6 +11,10 @@ + @@ -43,7 +47,7 @@ - + @@ -497,7 +501,7 @@
  • -
    +

  • @@ -525,9 +529,7 @@ +
  • @@ -566,7 +568,7 @@

    Thanks for contacting us. We've received your submission.

    - -

    Met costume designer spends Christmas hand-beading for debut

    + +

    Johnny Football resolves to stop party antics — after NYE

    -

    Paltrow pouts about haters, says divorce going great

    +

    Coco Rocha reveals her stickiest fashion show situation

    -

    The real Rod Stewart spotted at Trump’s swanky NYE party

    +

    VIPs rally behind Amy Adams after axed ‘Today’ interview

    @@ -989,13 +991,13 @@

    Now On

    - -

    What’s New On Netflix Streaming in 2015: ‘Friends’, ‘Election’, Hunter S. Thompson Tales And More

    + +

    Girl Gets Caught Cheating On National Television … Or Does She?

    -

    The 14 Hottest, Weirdest, And Shocking Sex Scenes Of 2014 (Gallery)

    +

    2015 Midseason Premiere Dates: When Your Favorite Show Is Coming Back – And Where You Can Stream It Now

    -

    EXCLUSIVE VIDEO: Kevin Spacey Sings ‘Piano Man’ … Really!

    +

    The 14 Hottest, Weirdest, And Shocking Sex Scenes Of 2014 (Gallery)

    @@ -1003,15 +1005,15 @@

    SEE ALL

    • Most Popular This Week

    -

    Passenger tossed after flipping out over staff’s ‘Merry Christmas’

    -
    -

    Arrests plummet 66% with NYPD in virtual work stoppage

    -
    -

    Astrophysicist angers Christians with Christmas tweet

    +

    Arrests plummet 66% with NYPD in virtual work stoppage

    Three cheers for the NYPD

    Cops turn backs on de Blasio at executed officer’s funeral

    +
    +

    ‘It was rough, man': Rex Ryan speaks to cap his...

    +
    +

    Astrophysicist angers Christians with Christmas tweet

    @@ -1040,11 +1042,11 @@

    @@ -1052,11 +1054,11 @@

    @@ -1074,12 +1076,12 @@

    -

    Revlon CEO rips blacks, Jews, hates ‘dirty’…

    - -
    -
    -
    +     @@ -1087,11 +1089,11 @@

    @@ -1099,11 +1101,11 @@

    @@ -1111,11 +1113,11 @@

    @@ -1123,11 +1125,11 @@

    @@ -1618,7 +1620,7 @@

    Sign in to complete account merge

    _tkq.push(['storeContext', {'blog_id':'56757169','blog_tz':'-5','user_lang':'en','blog_lang':'en','user_id':'0'}]); st_go({'blog':'56757169','v':'wpcom','tz':'-5','user_id':'0','post':'8430180','subd':'thenypost'}); function st_vt() {var x=document.createElement("img");x.src="http://pixel.wp.com/g.gif?blog=56757169&v=wpcomvt&tz=-5&user_id=0&post=8430180&subd=thenypost&rand="+Math.random();} -ex_go({'crypt':'UE40eW5QN0p8M2Y/RE1BNmNJfGhxNCVxUDExYmtib2E/SzdEJm0lUEtbdFU0fFk4cHZvcUFrZT90Tn4yQm1+R3NyNTdhJitIV3pwXVM1Smt6THRKS1srLiZtUUthZkgrPUxEblowT0NVWC9PcC54SCZDMyVrWWpKcE4ufHVPK0lUflBtbjBhP0NtVlJLeWdhLVlkRE9aeUNrV0gsZVBXJVc3VXVwOWlaSTBweXg3X1s5Zz1EOGpkZHp5U0tsbT1XeG1ddW1xN1dLc2xNUX5PZjBtd3d4YTVFfGFONWdMbnJdelp1M1BJR0Z+THBhNDU4YyY0cmE5LkxV'}); +ex_go({'crypt':'UE40eW5QN0p8M2Y/RE1BNmNJfGhxNCVxUDExYmtib2E/SzdEJm0lUEtbdFU0fFk4cHZvcUFrZT90Tn4yQm1+R3NyNTdhJitIV3pwXVM1Smt6THRKS1srLiZtUUthZkgrPUxEblowT0NVWDB8bkhbWXpYeDN3bmdnLmx0P2pwVF92ZmNzZyxKbGxTb3xSR0hIVmNVN2U5Uk53SC5ydGpuaHAvfFBjTSs5bCxPTiUtemZabnIrd0pVMWQxK2ImOTJqPSstdFpfQkZkQk43QmtFMFtTNjAwNW5PJnU/Y3U4Zz1tUituW2pLZFkyMllQW0o0NzdTc0FNOD9t'}); addLoadEvent(function(){linktracker_init('56757169',8430180);}); diff --git a/tests/data/html/ok.co.uk1.html b/tests/data/html/ok.co.uk1.html index 321dbc9b..6c67bb01 100644 --- a/tests/data/html/ok.co.uk1.html +++ b/tests/data/html/ok.co.uk1.html @@ -79,7 +79,7 @@ - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PERRIS: Woman ordered hit on father of her child, police say - Press Enterprise + + + + + + + + + + + + + + + - - - + + + + + + + + + + - - - - + - - - -
    - -
    - -
    + + + + + +
    + + +
    + +
    + +
    @@ -70,7 +151,7 @@
    - +
    -
    - + +
    + +
    +
    @@ -385,17 +385,17 @@

    Leave a Reply

    + Served from: pixelmonkey.org @ 2015-01-03 06:02:42 by W3 Total Cache --> +

    @@ -372,17 +372,17 @@

    Leave a Reply

    + Served from: pixelmonkey.org @ 2015-01-03 06:46:55 by W3 Total Cache --> - + - + @@ -151,7 +151,7 @@
    - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    -
    - -
    -
    -
    - - - - -
    -
    -
    -
    -
    - - - - - - - - - - - - diff --git a/tests/data/html/recipe.com1.html b/tests/data/html/recipe.com1.html index 53128457..5aba99d8 100644 --- a/tests/data/html/recipe.com1.html +++ b/tests/data/html/recipe.com1.html @@ -47,7 +47,7 @@ "activeSubs":[], "lastVisit":"" }, - "profile":{"id":"8654317849","recognized":false,"loggedIn":false,"pageViewNumber":0}, + "profile":{"id":"8659293364","recognized":false,"loggedIn":false,"pageViewNumber":0}, "util":{} }; @@ -267,7 +267,7 @@
    -
      +
      • 5
      • 4
      • 3
        • @@ -533,14 +533,14 @@ - +
    @@ -653,25 +653,25 @@ Fuller Brush Products
    - - Personalized Wine Bottle Pepper Mill (Ebony) + + NitroTap & Winekeeper Replacment Canisters (Set of 4)

    Wine Enthusiast

    - Personalized Wine Bottle Pepper Mill (Ebony) + NitroTap & Winekeeper Replacment Canisters (Set of 4)

    - $59.00 + $79.00

    Wine Enthusiast
    - - Personalized Wine Bottle Salt Mill + + Personalized Wine Bottle Pepper Mill (Ebony)

    Wine Enthusiast

    - Personalized Wine Bottle Salt Mill + Personalized Wine Bottle Pepper Mill (Ebony)

    $59.00 @@ -679,12 +679,12 @@ Wine Enthusiast

    - - Personalized Wine Bottle Pepper Mill + + Personalized Wine Bottle Salt Mill

    Wine Enthusiast

    - Personalized Wine Bottle Pepper Mill + Personalized Wine Bottle Salt Mill

    $59.00 @@ -731,31 +731,31 @@

    @@ -768,8 +768,8 @@ Your Account
    @@ -835,9 +835,9 @@
    © Copyright 2015, Meredith Corporation. All Rights Reserved | - Privacy Policy | - Data Policy | - Terms of Service | + Privacy Policy | + Data Policy | + Terms of Service | - - - - + + + + @@ -506,8 +506,8 @@ @@ -578,8 +578,8 @@ @@ -755,7 +755,7 @@

    Sensex ends flat; weaker Asian markets offset reform hopes

    var filter_arr = []; filter_arr = filter_str.split(","); - var parselyObjects= [{"data":[{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":2505,"pub_date":"2015-01-01T19:00:00","link":"http:\/\/in.reuters.com\/article\/2015\/01\/01\/health-cancer-luck-idINL1N0UE0VF20150101","video_link":"","section":"healthcareSector","url":"http:\/\/in.reuters.com\/article\/2015\/01\/01\/health-cancer-luck-idINL1N0UE0VF20150101","authors":["Will Dunham"],"author":"Will Dunham","title":"Biological bad luck blamed in two-thirds of cancer cases","image_url":"","thumb_url_medium":"","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":1324,"pub_date":"2015-01-01T15:38:27","link":"http:\/\/in.reuters.com\/article\/2015\/01\/01\/india-planningcommission-modi-idINKBN0KA1NA20150101","video_link":"","section":"businessNews","url":"http:\/\/in.reuters.com\/article\/2015\/01\/01\/india-planningcommission-modi-idINKBN0KA1NA20150101","authors":["Staff"],"author":"Staff","title":"Modi replaces Planning Commission, aiming to boost growth","image_url":"http:\/\/s4.reutersmedia.net\/resources\/r\/?m=02&d=20150101&t=2&i=1010416837&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEB000OW","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_a773efdc2feb5d4060a60378635aceb02f963ccd.jpg","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":822,"pub_date":"2015-01-02T04:40:38","link":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/india-energy-duty-idINKBN0KA1QL20150102","video_link":"","section":"businessNews","url":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/india-energy-duty-idINKBN0KA1QL20150102","authors":["Staff"],"author":"Staff","title":"India raises excise duties on petrol, diesel","image_url":"http:\/\/s3.reutersmedia.net\/resources\/r\/?m=02&d=20150102&t=2&i=1010771190&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEB000QL","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_531aa220d22fa2a9b8f5c38d750122053403b61d.jpg","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":776,"pub_date":"2015-01-02T00:09:59","link":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/markets-global-idINKBN0KB00520150102","video_link":"","section":"businessNews","url":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/markets-global-idINKBN0KB00520150102","authors":["Wayne Cole"],"author":"Wayne Cole","title":"Euro starts new year at new lows in Asia","image_url":"http:\/\/s1.reutersmedia.net\/resources\/r\/?m=02&d=20150102&t=2&i=1010649822&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEB01002","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_f0641c2f0e5b9c650e4b75c4fdc5d9ee2db14ad9.jpg","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":677,"pub_date":"2014-12-31T16:11:23","link":"http:\/\/in.reuters.com\/article\/2014\/12\/31\/indonesia-airplane-investigation-idINKBN0K90QA20141231","video_link":"","section":"topNews","url":"http:\/\/in.reuters.com\/article\/2014\/12\/31\/indonesia-airplane-investigation-idINKBN0K90QA20141231","authors":["Siva Govindasamy"],"author":"Siva Govindasamy","title":"Data point to \"unbelievably\" steep climb before AirAsia crash - source","image_url":"http:\/\/s3.reutersmedia.net\/resources\/r\/?m=02&d=20141231&t=2&i=1009684865&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEABU0B5","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_55430cf9b65e14d250ca208b9aed5b2d723bf384.jpg","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":496,"pub_date":"2015-01-01T22:01:24","link":"http:\/\/in.reuters.com\/article\/2015\/01\/01\/gm-recalls-reports-idINKBN0KA1VK20150101","video_link":"","section":"businessNews","url":"http:\/\/in.reuters.com\/article\/2015\/01\/01\/gm-recalls-reports-idINKBN0KA1VK20150101","authors":["Ryan Vlastelica"],"author":"Ryan Vlastelica","title":"General Motors issues three new recalls, cites ignition systems","image_url":"http:\/\/s3.reutersmedia.net\/resources\/r\/?m=02&d=20150101&t=2&i=1010589747&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEB000SJ","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_6867ef5b14aa1fd304dbcafd4ce370c2faebcfb9.jpg","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":398,"pub_date":"2015-01-02T01:36:49","link":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/indonesia-airplane-idINKBN0K900820150102","video_link":"http:\/\/in.reuters.com\/article\/video\/idINKBN0K900820150102?videoId=357636183","section":"globalCoverage2","url":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/indonesia-airplane-idINKBN0K900820150102","authors":["Fergus Jensen, Gayatri Suroyo and Charlotte Greenfield"],"author":"Fergus Jensen, Gayatri Suroyo and Charlotte Greenfield","title":"Investigators to use acoustic equipment to search for AirAsia black boxes","image_url":"http:\/\/s4.reutersmedia.net\/resources\/r\/?m=02&d=20150101&t=2&i=1010370199&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEB000NW","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_a7bd6b36a7745295bf11615034cfdbd5c0f0ac47.jpg","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":320,"pub_date":"2015-01-02T05:09:42","link":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/india-economy-pmi-idINKBN0KB03Y20150102","video_link":"","section":"businessNews","url":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/india-economy-pmi-idINKBN0KB03Y20150102","authors":["Staff"],"author":"Staff","title":"Dec factory growth at 2-year high as orders surge - PMI","image_url":"","thumb_url_medium":"","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":304,"pub_date":"2015-01-02T06:15:05","link":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/markets-india-stocks-idINKBN0KB05G20150102","video_link":"","section":"businessNews","url":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/markets-india-stocks-idINKBN0KB05G20150102","authors":["Staff"],"author":"Staff","title":"Sensex rises more than 1 pct; Bank Nifty hits record","image_url":"http:\/\/s3.reutersmedia.net\/resources\/r\/?m=02&d=20150102&t=2&i=1010812432&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEB01026","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_477269520aceecb0eb8964887abc67301a1732c0.jpg","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":277,"pub_date":"2015-01-02T06:05:31","link":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/travel-india-bylakuppe-idINKBN0KB03W20150102","video_link":"","section":"entertainmentNews","url":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/travel-india-bylakuppe-idINKBN0KB03W20150102","authors":["Debarati Dasgupta"],"author":"Debarati Dasgupta","title":"Trip Tips: A glimpse of little Tibet in southern India","image_url":"http:\/\/s4.reutersmedia.net\/resources\/r\/?m=02&d=20150102&t=2&i=1010796394&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEB0101R","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_b257125dd27438c698e9991d831a9eda45adde0d.jpg","metadata":""}]}]; + var parselyObjects= [{"data":[{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":5135,"pub_date":"2015-01-02T19:17:06","link":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/northkorea-cyberattack-sanctions-idINL1N0UH0YO20150102","video_link":"","section":"cyclicalConsumerGoodsSector","url":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/northkorea-cyberattack-sanctions-idINL1N0UH0YO20150102","authors":["Staff"],"author":"Staff","title":"UPDATE 1-Obama authorizes more sanctions against North Korea in Sony hack","image_url":"","thumb_url_medium":"","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":1244,"pub_date":"2015-01-02T23:53:48","link":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/people-donnadouglas-idINKBN0KB1GI20150102","video_link":"","section":"entertainmentNews","url":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/people-donnadouglas-idINKBN0KB1GI20150102","authors":["Will Dunham and Eric Kelsey"],"author":"Will Dunham and Eric Kelsey","title":"'Beverly Hillbillies' actress Donna Douglas dies at 81","image_url":"http:\/\/s2.reutersmedia.net\/resources\/r\/?m=02&d=20150102&t=2&i=1011303877&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEB010MB","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_fd0d3073fd1c576422327dffe40191fe37c13433.jpg","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":1085,"pub_date":"2015-01-03T02:58:18","link":"http:\/\/in.reuters.com\/article\/2015\/01\/03\/indonesia-airplane-objects-idINKBN0KC01J20150103","video_link":"","section":"globalCoverage2","url":"http:\/\/in.reuters.com\/article\/2015\/01\/03\/indonesia-airplane-objects-idINKBN0KC01J20150103","authors":["Staff"],"author":"Staff","title":"Indonesian search and rescue finds two large objects in hunt for AirAsia jet","image_url":"http:\/\/s2.reutersmedia.net\/resources\/r\/?m=02&d=20150103&t=2&i=1011388724&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEB0200B","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_3d9bac7e8a5e9468e7f79248db73e085ca59aa17.jpg","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":978,"pub_date":"2015-01-02T19:05:18","link":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/northkorea-cyberattack-sanctions-idINKBN0KB16Q20150102","video_link":"","section":"technologyNews","url":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/northkorea-cyberattack-sanctions-idINKBN0KB16Q20150102","authors":["Staff"],"author":"Staff","title":"Obama authorizes additional sanctions against North Korea in Sony hack","image_url":"http:\/\/s1.reutersmedia.net\/resources\/r\/?m=02&d=20150102&t=2&i=1011172028&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEB010K0","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_1917d19cc9e9a6ebbe3da1da28c6f8991fa1da2f.jpg","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":677,"pub_date":"2015-01-03T02:16:10","link":"http:\/\/in.reuters.com\/article\/slideshow\/idINKBN0K900820150103","video_link":"http:\/\/in.reuters.com\/article\/video\/idINKBN0K900820150103?videoId=358479000","section":"globalCoverage2","url":"http:\/\/in.reuters.com\/article\/slideshow\/idINKBN0K900820150103","authors":["Fergus Jensen"],"author":"Fergus Jensen","title":"Indonesia widens search area in hunt for AirAsia jet","image_url":"http:\/\/s3.reutersmedia.net\/resources\/r\/?m=02&d=20150102&t=2&i=1011105881&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEB010H7","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_15f61a7637af2d90311eec57bf5e1a051e75c542.jpg","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":587,"pub_date":"2015-01-02T16:21:56","link":"http:\/\/in.reuters.com\/article\/slideshow\/idINKBN0KB0YG20150102","video_link":"","section":"hollywood","url":"http:\/\/in.reuters.com\/article\/slideshow\/idINKBN0KB0YG20150102","authors":["Staff"],"author":"Staff","title":"Irish rock star Bono says he may never play guitar again","image_url":"http:\/\/s4.reutersmedia.net\/resources\/r\/?m=02&d=20150102&t=2&i=1011090896&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEB010FX","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_e7d1d3c55363ca00c2bd6b94b035e4caacf44466.jpg","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":579,"pub_date":"2015-01-02T16:31:33","link":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/palestinians-israel-idINKBN0KB0ZW20150102","video_link":"","section":"worldNews","url":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/palestinians-israel-idINKBN0KB0ZW20150102","authors":["Staff"],"author":"Staff","title":"Palestinians submit ICC documents to United Nations","image_url":"http:\/\/s3.reutersmedia.net\/resources\/r\/?m=02&d=20150102&t=2&i=1011095974&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEB010GK","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_04480a5b248afbb4213f88cc130574ee90a52f05.jpg","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":474,"pub_date":"2015-01-02T09:36:29","link":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/india-solar-idINKBN0KB0E920150102","video_link":"","section":"globalEnergyNews","url":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/india-solar-idINKBN0KB0E920150102","authors":["Krishna N. Das and Swetha Gopinath"],"author":"Krishna N. Das and Swetha Gopinath","title":"Govt raises solar investment target to $100 bln by 2022","image_url":"http:\/\/s1.reutersmedia.net\/resources\/r\/?m=02&d=20150102&t=2&i=1010902015&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEB01051","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_62ebe4ff51f21385ac200582d49fdd73cfd94139.jpg","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":424,"pub_date":"2015-01-02T22:17:01","link":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/usa-lawsuit-britain-royals-idINKBN0KB1E920150102","video_link":"","section":"entertainmentNews","url":"http:\/\/in.reuters.com\/article\/2015\/01\/02\/usa-lawsuit-britain-royals-idINKBN0KB1E920150102","authors":["Letitia Stein"],"author":"Letitia Stein","title":"Woman says forced as minor into sex with Prince Andrew; allegations denied","image_url":"http:\/\/s4.reutersmedia.net\/resources\/r\/?m=02&d=20150102&t=2&i=1011257990&w=130&fh=&fw=&ll=&pl=&r=LYNXMPEB010LT","thumb_url_medium":"http:\/\/c0001566.cdn1.cloudfiles.rackspacecloud.com\/medium_31a25cf8daacd73b2db24ef946814b265b6edf32.jpg","metadata":""},{"tags":["[rupee, dollar, sensex, nifty, India, shares, currencies, commodities]"],"_hits":418,"pub_date":"2015-01-03T03:30:25","link":"http:\/\/in.reuters.com\/article\/2015\/01\/03\/indonesia-airplane-idINL3N0UI01Q20150103","video_link":"","section":"industrialsSector","url":"http:\/\/in.reuters.com\/article\/2015\/01\/03\/indonesia-airplane-idINL3N0UI01Q20150103","authors":["Staff"],"author":"Staff","title":"UPDATE 1-Indonesian search and rescue finds two large objects in hunt for AirAsia jet","image_url":"","thumb_url_medium":"","metadata":""}]}]; if (typeof Reuters.parsely.parseParsely == 'function') { Reuters.parsely.parseParsely(parselyObjects); } @@ -785,8 +785,8 @@

    Sensex ends flat; weaker Asian markets offset reform hopes

    @@ -806,7 +806,7 @@

    Sensex ends flat; weaker Asian markets offset reform hopes

    -

    Economy

    A worker operates a lathe machine as he makes a steel cutter at a manufacturing unit in Noida, on the outskirts of New Delhi November 3, 2014. REUTERS/Anindito Mukherjee/Files

    December factory growth at two-year high as orders surge - PMI

    Indian manufacturing activity expanded at its fastest pace in two years in December as new orders flooded in and factories kept price increases to a minimum, a business survey showed on Friday.  Full Article 

    +

    Banking

    A cashier (L) counts currency notes as customers wait inside a bank in Hyderabad March 22, 2010. REUTERS/Krishnendu Halder/Files

    India needs reforms in banking sector, more autonomy for state banks: Jaitley

    India needs to "conceive" several reforms in the banking sector, Finance Minister Arun Jaitley told reporters on the sidelines of a two-day banking retreat.  Full Article 

    -
    -

    Economy

    A worker operates a lathe machine as he makes a steel cutter at a manufacturing unit in Noida, on the outskirts of New Delhi November 3, 2014. REUTERS/Anindito Mukherjee/Files

    December factory growth at two-year high as orders surge - PMI

    Indian manufacturing activity expanded at its fastest pace in two years in December as new orders flooded in and factories kept price increases to a minimum, a business survey showed on Friday.  Full Article 

    +
    +

    Banking

    A cashier (L) counts currency notes as customers wait inside a bank in Hyderabad March 22, 2010. REUTERS/Krishnendu Halder/Files

    India needs reforms in banking sector, more autonomy for state banks: Jaitley

    India needs to "conceive" several reforms in the banking sector, Finance Minister Arun Jaitley told reporters on the sidelines of a two-day banking retreat.  Full Article 

    + + + + + + +
    + + New Relic + + + + x +
    +
    + + New Relic + + + + x +
    +
    +
    + SitePoint Christmas Sale + Shop Now +
    +
    +
    + + + +
    +
    +
    +

    Average Page Weight Increases 15% in 2014

    + + +
    + +
    + + +
    + +
    +
    + Craig Buckler +
    + +
    + +
    + + +
    +

    Published

    +
    +
    + + + + + Tweet + + + + + +
    + +
    + +
    + +

    The HTTP Archive Report collates information from almost half a million of the web’s most popular websites. The latest figures indicate that average page weight has increased by 15% in one year to reach 1,953Kb — a little under 2Mb — and comprises 95 individual HTTP requests. While this is smaller than the 32% increase in 2013, it remains cause for concern.

    +

    The report analyzes publicly-accessible content and shopping web sites rather than complex web applications and provides a breakdown of the technologies used:

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    technologyend 2013end 2014increase
    HTML57Kb59Kb+4%
    CSS46Kb57Kb+24%
    JavaScript276Kb295Kb+7%
    Images1,030Kb1,243Kb+21%
    Flash87Kb76Kb-13%
    Other205Kb223Kb+9%
    Total1,701Kb1,953Kb+15%
    +

    These are average figures; a large proportion of pages will have greater file sizes.

    +

    A 2Kb rise for HTML seems reasonable although it’s a significant quantity of content given the trend for simpler, more concise text.

    +

    What surprises me most is CSS’s 11Kb rise. Responsive Web Design and CSS3 animations could account for some of this increase but there’s not been a drop in JavaScript. Despite the availability of CSS management and minification tools, the average site also makes six requests for CSS files.

    +

    JavaScript has risen by 19Kb. This is confusing; the need for shims is reducing, effects can be handed to CSS3 and monolithic libraries have fallen from favor. Sites make an average of 18 JavaScript file requests, which is unchanged from last year — although a quarter of sites make more than 30 requests. Perhaps some of the gain can be explained by increasingly sophisticated/bloated social networking scripts?

    +

    27% of sites continue to use Flash — a fall of 5% over the year. The majority is used for advertising, video, and games. Flash hasn’t dropped as fast as expected but its future is clear.

    +

    There’s been a 9% increase for “other” files. That figure doubled in 2013 but, back then, custom fonts and icon fonts were relatively new.

    +

    Finally, images are responsible for 85% of the weight gain. Using high-resolution (Retina) images could account for some of this hike, except:

    +
      +
    • Pages contain more than fifty images, which seems excessive.
      • +
    • Retina accounts for a relatively small proportion of devices.
      • +
    • SVG, icon fonts, and CSS3 effects can replace many images.
      • +
    • There are numerous tools to help reduce file sizes.
      • +
    +

    Additional Factors

    +

    The survey also reveals:

    +
      +
    • 95 HTTP requests are made per page — a drop of a single request from last year.
      • +
    • Pages contain 862 DOM elements.
      • +
    • Resources are loaded from sixteen domains with a maximum of 52 requests per domain.
      • +
    • The average PageSpeed score is 78 out of 100 — which is surprisingly good, given the bloat.
      • +
    • 46% of pages use Google libraries.
      • +
    • 47% of pages use custom fonts.
      • +
    • 79% of responses are compressed (gzip’d).
      • +
    • 14% of pages are loaded over HTTPS.
      • +
    • 20% of pages use localStorage.
      • +
    • 65% of pages use iframes (mostly videos and advertising).
      • +
    • 74% of pages use at least one redirect — which seems high.
      • +
    +

    The Primary Suspects

    +

    A 15% increase is less extravagant than the 32% rise in 2013 and the 30% rise in 2012, but it’s still too much. Has your bandwidth increased more than 15% in the past twelve months? A third of web users now use mobile devices — will they appreciate the additional weight?

    +

    Let’s put this into context for website owners. Bloated pages adversely affect your profitability:

    +
      +
    1. Users have a slower experience. It doesn’t matter how great your site looks — people will not wait.
      2. +
    2. There’s little point creating a site that works on mobile devices when your pages are 2Mb. Responsive Web Design != a responsive website. Are you losing up to a third of potential customers?
      3. +
    3. Google will downgrade your site and harm your search engine optimization efforts (though we’re never sure exactly how much this matters to Google’s algorithm).
      4. +
    4. Your hosting costs will increase.
      5. +
    5. The more code you use, the more likely it will break. Updates and maintenance are more difficult, take longer and cost more.
      6. +
    +

    It’s ironic that web developers praise the benefits of cross-device HTML5 apps when a single page is often larger to download and slower than an equivalent native app.

    +

    Overweight pages are unnecessary. My primary suspects remain bloated CMS templates and frameworks. They offer a cheaper and quicker development route at the expense of quality, efficiency and performance. Many are packed with features you’ll never use, but removing them can be laborious, tedious, and time-consuming.

    +

    We can summarize the problem in one simple word: laziness. Developers are at fault — that’s you and me. We have plenty of excuses:

    +
      +
    • there’s never enough time
      • +
    • the client insisted it should be done this way
      • +
    • the budget/schedule is too tight
      • +
    • I inherited a shoddy system
      • +
    • I don’t have the tools
      • +
    +

    Whether it’s technical boundaries or a failure to explain issues, it’s still laziness. We work at the coal face; the final decisions are ours alone. Why create a badly-optimized site when many bloat-blasting solutions are simple and take minutes to implement?

    +

    Clients rarely appreciate the efficiency gains we make but they don’t understand anything we do. We are the experts, and minimizing page weight is an essential part of the job. Do it. It’s easier to beg for forgiveness than to ask for permission.

    +

    </rant></soapbox>

    +

    Are you concerned by the web obesity problem? Are you pleased the scale of increases has dropped? Do you or fellow developers struggle to implement optimization techniques or to explain them to clients? Do you think there are other causes? Is Craig being too simplistic and shouty?!

    + +
    + +
    + +
    + +
    + + +
    + +
    + +
    + Craig Buckler +
    + + +

    Craig is a freelance UK web consultant who built his first page for IE2.0 in 1995. Since that time he's been advocating standards, accessibility, and best-practice HTML5 techniques. He's written almost 1,000 articles for SitePoint and you can find him @craigbuckler

    + +
      +
    +
    +
    + +
    + +
    +

    You might also like:

    + +
    + +
    +
    +
    +

    + Free book: Jump Start HTML5 Basics +

    +

    Grab a free copy of one our latest ebooks! Packed with hints and tips on HTML5's most powerful new features.

    +
    +
    + +
    +
    + +
    + +
    +
    + + Comments +
    +
    + + + Have Your Say + +
    + +
    + + isights + + +
    +
    + December 29, 2014 + isights: +
    + +

    Seems obvious to me. The increase in image weight isn't so much retina as it is the fact that we're increasingly building more single-page layouts utilizing ever larger background and parallax images.

    + +

    Further, the CSS size increase is (or should be) fairly obvious as well: we're building responsive layouts that contain more media queries and device specific breakpoints and layouts. Web sites now need the CSS to look good on the desktop, on tablets, and on mobile phones.

    +
    +
    + + RyanReese + + +
    +
    + December 29, 2014 + RyanReese: +
    + +

    I'd also think it's because more and more people are using frameworks - all that added code. That was my original thought when this was posted. Media queries are probably mixed in there also.

    +
    +
    + + zackw + + +
    +
    + December 30, 2014 + zackw: +
    + +

    I blame framework bloat and overuse of scaffolding as well.
    I cringe whenever I look at the CSS source of something cool, only to see something as simple as a box with a title drowning in 300 lines of CSS, with the same color and other attributes overridden 3, 4, 5 times.

    + +

    The typical workflow now is start with a framework, then another theme/style, then a master file, then an override file, etc etc. Don't touch the master, just customize in an override.

    + +

    I almost wish we could go back to the old days where we, shock, just style the site directly as it's supposed to be, and not have to constantly override CSS that isn't used in the first place.

    + +

    I believe a rise in simple, minimal CSS frameworks will come back, with technologies like SASS and SCSS allowing us to style sites and generate single-level CSS that doesn't not require overrides and bloat.
    Either that or we'll see a resurgence of designers wanting to build all custom CSS again, and a slew of tools to allow quick style development.

    +
    +
    + + chris22smith + + +
    +
    + December 30, 2014 + chris22smith: +
    + +

    That's all a bit depressing. I think the point about the laziness and dependency on ready-made, bloated solutions is absolutely spot on. At least now we all know what our resolutions should be for 2015!

    +
    +
    +
    + + + Have Your Say + +
    +
    +
    +
    + + + +
    + + + +
    + +
      +
      + + + +
      x
      + + + + + + + + + + + + + + + + \ No newline at end of file + Served from: www.sitepoint.com @ 2015-01-03 03:05:07 by W3 Total Cache --> \ No newline at end of file diff --git a/tests/data/html/sitepoint.com2.html b/tests/data/html/sitepoint.com2.html index 75ace6ab..7955da9f 100644 --- a/tests/data/html/sitepoint.com2.html +++ b/tests/data/html/sitepoint.com2.html @@ -1075,7 +1075,7 @@

      Connect

      \ No newline at end of file + Served from: www.sitepoint.com @ 2015-01-03 03:40:01 by W3 Total Cache --> \ No newline at end of file diff --git a/tests/data/html/slate.com1.html b/tests/data/html/slate.com1.html index 23f34406..f1111e62 100644 --- a/tests/data/html/slate.com1.html +++ b/tests/data/html/slate.com1.html @@ -342,12 +342,7 @@ - - - + @@ -2982,31 +2977,31 @@

      A quest to identify the top technological marvels of the contemp -
    • 1. Why the World Is Not Falling Apart
      • +
    • 1. Behind the Scenes of Australia’s Prostitution Boom
      • -
    • 2. My Favorite Graph of 2014: The Rise and Rise of the Top 0.1 Percent
      • +
    • 2. Why the World Is Not Falling Apart
      • -
    • 3. These C-Section Babies Were Photographed at Less Than 20 Seconds Old
      • +
    • 3. McDonald's Would Clearly Prefer If You Didn't Think About How Many Calories Are in Chicken McNuggets
      • -
    • 4. Orbitz and United Are Suing a 22-Year-Old Who Figured Out How to Game Airlines
      • +
    • 4. The Most Important Skill for Science or Self-Improvement
      • -
    • 5. Help! I’m Pregnant, so My Family Wants to Cancel Our Remote Vacation.
      • +
    • 5. Did the Jay Wilds Interview Change Our Perspective on Serial? A Bonus Spoiler Special.
      • @@ -3052,8 +3047,8 @@

      A quest to identify the top technological marvels of the contemp
    • - - 2. Orbitz and United Are Suing a 22-Year-Old Who Figured Out How to Game Airlines + + 2. The Rust Belt Theory of Low-Cost High Culture
      • @@ -3062,8 +3057,8 @@

      A quest to identify the top technological marvels of the contemp
    • - - 3. Transgender Teen Leelah Alcorn’s Final Words Need to Be Heard + + 3. Florida Clerks Cancel All Courthouse Weddings to Avoid Performing Gay Ceremonies
      • @@ -3072,8 +3067,8 @@

      A quest to identify the top technological marvels of the contemp
    • - - 4. Toddler Accidentally Shoots and Kills His Mother With Gun Taken From Her Purse + + 4. McDonald's Would Clearly Prefer If You Didn't Think About How Many Calories Are in Chicken McNuggets
      • @@ -3082,8 +3077,8 @@

      A quest to identify the top technological marvels of the contemp
    • - - 5. Add Norway’s Stunning Juvet Landscape Hotel to Your List of Dream Getaways + + 5. The Most Important Skill for Science or Self-Improvement
      • @@ -3116,10 +3111,10 @@

      A quest to identify the top technological marvels of the contemp

      @@ -1495,15 +1483,15 @@

        -
      1. Most cancers are caused by bad luck not genes or lifestyle, say scientists
        2. +
      2. Couple stranded in New York with newborn baby freed from £130,000 medical bill
        3. -
      3. Billionaire Elon Musk divorces wife for second time
        4. +
      4. Holiday windfall for Britons as euro slides
        5. -
      5. First Romanian to UK warns fellow countrymen ‘don’t come here’
        6. +
      6. Meet the Arab Christians who want to fight for Israel
        7. -
      7. Prince Harry was fathered by James Hewitt, new play claims
        8. +
      8. Gales and snow to divide the country
        9. -
      9. Company director lives it up in Mexico while staff go unpaid
        10. +
      10. Has Stanford University found a cure for Alzheimer's disease?
      @@ -1928,7 +1916,7 @@

      © Copyright of Telegraph Media Group Limited 2015

      Terms and Conditions

      -

      Today's News

      +

      Today's News

      Archive

      Style Book

      Weather Forecast

      @@ -2065,7 +2053,7 @@

      /* Read Next Widget Data */ - var readNextArticleList = [{"id":"11320497","title":"Most cancers are caused by bad luck not genes or lifestyle, say scientists ","url":"/news/science/science-news/11320497/Most-cancers-are-caused-by-bad-luck-not-genes-or-lifestyle-say-scientists.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02130/cancer1_2130923g.jpg"},{"id":"11320885","title":"Billionaire Elon Musk divorces wife for second time","url":"/news/celebritynews/11320885/Billionaire-Elon-Musk-divorces-wife-for-second-time.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/Elon-Must-and-Talu_3152442g.jpg"},{"id":"11310785","title":"First Romanian to UK warns fellow countrymen ‘don’t come here’ ","url":"/news/uknews/immigration/11310785/First-Romanian-to-UK-warns-fellow-countrymen-dont-come-here.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03147/victorspirescu_web_3147097g.jpg"},{"id":"11315109","title":"Prince Harry was fathered by James Hewitt, new play claims ","url":"/news/uknews/prince-harry/11315109/Prince-Harry-was-fathered-by-James-Hewitt-new-play-claims.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03149/harry-hewitt-charl_3149599g.jpg"},{"id":"11319519","title":"Angela Merkel attacks 'prejudice' and 'hatred' of German Islamisation marches","url":"/news/worldnews/europe/germany/angela-merkel/11319519/Angela-Merkel-attacks-prejudice-and-hatred-of-German-Islamisation-marches.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03151/angela-merkel-germ_3151842g.jpg"},{"id":"11320469","title":"Company director lives it up in Mexico while staff go unpaid","url":"/news/uknews/11320469/Company-director-lives-it-up-in-Mexico-while-staff-go-unpaid.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/temp-bos_3152225g.jpg"},{"id":"11320375","title":"MBE for man who represented remote South Atlantic island from his Birmingham bedroom","url":"/news/uknews/honours-list/11320375/MBE-for-man-who-represented-remote-South-Atlantic-island-from-his-Birmingham-bedroom.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/bates_3152205g.jpg"},{"id":"11320405","title":"Kim offers 'highest-level talks' with South Korea","url":"/news/worldnews/asia/northkorea/11320405/Kim-offers-highest-level-talks-with-South-Korea.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/Kim-Jong-Un-New-Ye_3152362g.jpg"},{"id":"11320797","title":"Public digs deep to help couple stranded in New York after baby arrives early ","url":"/news/uknews/11320797/Public-digs-deep-to-help-couple-stranded-in-New-York-after-baby-arrives-early.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/Stranded_3152349g.jpg"},{"id":"11319848","title":"Don't panic! How a rehearsal for a nuclear disaster descended into farce","url":"/news/uknews/defence/11319848/Dont-panic-How-a-rehearsal-for-a-nuclear-disaster-descended-into-farce.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/01822/astute_1822270g.jpg"},{"id":"11320815","title":"Fifty Shades of Grey actor: 'It's not sexist, it's a love story'","url":"/news/celebritynews/11320815/Fifty-Shades-of-Grey-actor-Its-not-sexist-its-a-love-story.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/Fifty-Shades-2_3152439g.jpg"},{"id":"11320503","title":"Amateur treasure hunter's £1m find of Anglo Saxon coins","url":"/news/earth/environment/archaeology/11320503/Amateur-treasure-hunters-1m-find-of-Anglo-Saxon-coins.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/coins_3152228g.jpg"},{"id":"11320690","title":"Nick Clegg: I won't use my wife to help me in the polls","url":"/news/politics/nick-clegg/11320690/Nick-Clegg-I-wont-use-my-wife-to-help-me-in-the-polls.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02816/miriam-clegg_2816435g.jpg"},{"id":"11320725","title":"US teenager refused permission to change sex by Christian parents commits suicide","url":"/news/worldnews/northamerica/usa/11320725/US-teenager-refused-permission-to-change-sex-by-Christian-parents-commits-suicide.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/Leelah-Alcorn_3152309g.jpg"},{"id":"11320715","title":"Rescue dog saved owner's life sniffing out aggressive tumour","url":"/news/telegraphchristmasappeal/11320715/Rescue-dog-saved-owners-life-sniffing-out-aggressive-tumour.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/Tumour_Dog_Sum_3152315g.jpg"},{"id":"11318824","title":"My £35 little red dress saved my life, says crash victim","url":"/news/uknews/11318824/My-35-little-red-dress-saved-my-life-says-crash-victim.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03151/Zoe-Turner_3151466g.jpg"},{"id":"11320629","title":"Tepid coffee anyone? Europe rules percolators must shut off after five minutes ","url":"/news/shopping-and-consumer-news/11320629/Tepid-coffee-anyone-Europe-rules-percolators-must-shut-off-after-five-minutes.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02628/coffee_2628160g.jpg"},{"id":"11320481","title":"Constipated goldfish undergoes £300 life-saving surgery","url":"/news/uknews/11320481/Constipated-goldfish-undergoes-300-life-saving-surgery.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/goldfish_3152233g.jpg"},{"id":"11313978","title":"North Korea's honey trap scheme revealed","url":"/news/worldnews/asia/northkorea/11313978/North-Koreas-honey-trap-scheme-revealed.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02870/kimjonil_2870610g.jpg"},{"id":"11320600","title":"Six stabbed in Belfry nightclub mayhem","url":"/news/uknews/crime/11320600/Six-stabbed-in-Belfry-nightclub-mayhem.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/belfry_3152252g.jpg"},{"id":"11319455","title":"F-35 stealth jet 'will not be able to fire its guns until 2019'","url":"/news/worldnews/northamerica/usa/11319455/F-35-stealth-jet-will-not-be-able-to-fire-its-guns-until-2019.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02973/F-35-Farnborough_2973406g.jpg"},{"id":"11320637","title":"This Act of folly has condemned us to five years of political torment","url":"/news/general-election-2015/11320637/This-Act-of-folly-has-condemned-us-to-five-years-of-political-torment.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02503/parliament_2503626g.jpg"},{"id":"11321114","title":"Second 'ghost ship' of migrants heading for Italian coast","url":"/news/worldnews/europe/italy/11321114/Uncrewed-ship-with-450-migrants-aboard-heading-for-Italian-coast.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/EZADEEN_3152514g.jpg"},{"id":"11315319","title":"Lying on Facebook profiles can implant false memories, experts warn","url":"/news/science/science-news/11315319/Lying-on-Facebook-profiles-can-implant-false-memories-experts-warn.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02704/facebook_2704219g.jpg"},{"id":"11294761","title":"Gordon Ramsay still fighting Tenerife restaurant trading on his name","url":"/news/uknews/law-and-order/11294761/Gordon-Ramsay-still-fighting-Tenerife-restaurant-trading-on-his-name.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/gordon-ransay_3152064g.jpg"},{"id":"11320996","title":"French "first sons" slug it out on Twitter","url":"/news/worldnews/europe/france/11320996/French-first-sons-slug-it-out-on-Twitter.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/Les-Boys_3152412g.jpg"},{"id":"11321193","title":"Man dies from electric shock as 100 people evacuated from block","url":"/news/uknews/11321193/Man-dies-from-electric-shock-as-100-people-evacuated-from-block.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02832/tape_2832196g.jpg"},] + var readNextArticleList = [{"id":"11322905","title":"Couple stranded in New York with newborn baby freed from £130,000 medical bill","url":"/news/uknews/11322905/Couple-stranded-in-New-York-with-newborn-baby-freed-from-130000-medical-bill.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/Stranded_3152349g.jpg"},{"id":"11322711","title":"Holiday windfall for Britons as euro slides","url":"/news/shopping-and-consumer-news/11322711/Holiday-windfall-for-Britons-as-euro-slides.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03132/tuscany_3132203g.jpg"},{"id":"11321651","title":"Meet the Arab Christians who want to fight for Israel","url":"/news/worldnews/middleeast/israel/11321651/Meet-the-Arab-Christians-who-want-to-fight-for-Israel.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/Shalayan_3152740g.jpg"},{"id":"11322787","title":"Gales and snow to divide the country ","url":"/news/weather/11322787/UK-weather-Gales-and-snow-to-divide-the-country.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02922/Storms_2922771g.jpg"},{"id":"11280504","title":"Has Stanford University found a cure for Alzheimer's disease? ","url":"/news/science/science-news/11280504/Has-Stanford-University-found-a-cure-for-Alzheimers-disease.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02902/dementia_2902785g.jpg"},{"id":"11322798","title":"Amal Clooney threatened with arrest by Egypt over criticism of justice system","url":"/news/worldnews/africaandindianocean/egypt/11322798/Amal-Clooney-threatened-with-arrest-by-Egypt-over-criticism-of-justice-system.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03153/Amal_Alamuddin_Clo_3153238g.jpg"},{"id":"11321947","title":"Prince Andrew 'categorically denies' claims he sexually abused teenager","url":"/news/uknews/theroyalfamily/11321947/Prince-Andrew-categorically-denies-claims-he-sexually-abused-teenager.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/andrew-epsteins-wa_3152973g.jpg"},{"id":"11321854","title":"Gwyneth Paltrow: I didn't invent 'conscious uncoupling' phrase","url":"/news/celebritynews/11321854/Gwyneth-Paltrow-I-didnt-invent-conscious-uncoupling-phrase.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02865/gwyneth_2865872g.jpg"},{"id":"11321710","title":"You're a keeper! Southampton FC steps in after fan asks club to help him meet girl of his dreams","url":"/news/newstopics/howaboutthat/11321710/Southampton-FC-acts-as-matchmaker-after-fan-asks-club-to-help-him-meet-girl.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/southampton-2_3152774g.jpg"},{"id":"11320497","title":"Most cancers are caused by bad luck not genes or lifestyle, say scientists ","url":"/news/science/science-news/11320497/Most-cancers-are-caused-by-bad-luck-not-genes-or-lifestyle-say-scientists.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02130/cancer1_2130923g.jpg"},{"id":"11322309","title":"The battle to keep our Union together has only just begun","url":"/news/uknews/scotland/11322309/The-battle-to-keep-our-Union-together-has-only-just-begun.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03110/salmond_3110243g.jpg"},{"id":"11314580","title":"Sir Winston Churchill 's family feared he might convert to Islam","url":"/news/religion/11314580/Sir-Winston-Churchill-s-family-feared-he-might-convert-to-Islam.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03149/churchill-family_3149302g.jpg"},{"id":"11321499","title":"Al Qaed Arr: Bristol bus timetable hacked by terrorists (who thought they would cause travel chaos in 'the West')","url":"/news/uknews/terrorism-in-the-uk/11321499/Bristol-bus-timetable-hacked-by-terrorists.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/travel-west-screen_3152659g.jpg"},{"id":"11322503","title":"Roman Holiday: Italian police 'skive off' New Year's Eve as 83 per cent call in sick ","url":"/news/worldnews/europe/italy/11322503/Roman-Holiday-Italian-police-skive-off-New-Years-Eve-as-83-per-cent-call-in-sick.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03153/carabinieri-roma_3153115g.jpg"},{"id":"11321846","title":"Elvis Presley's personal jets go on sale","url":"/news/celebritynews/11321846/Elvis-Presleys-personal-jets-go-on-sale.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/elvis-summ_3152815g.jpg"},{"id":"11321815","title":"Prince Andrew named in underage 'sex slave' case","url":"/news/uknews/theroyalfamily/11321815/Prince-Andrew-named-in-underage-sex-slave-case.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02681/Prince-Andrew_2681629g.jpg"},{"id":"11322583","title":"Conservatives call time on taxpayer-funded six-figure 'golden goodbyes'","url":"/news/politics/11322583/Conservatives-call-time-on-taxpayer-funded-six-figure-golden-goodbyes.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02394/Entwistle-Saturday_2394977g.jpg"},{"id":"11321890","title":"Benjamin Netanyahu 'repelled' by success of female MP","url":"/news/worldnews/middleeast/israel/11321890/Benjamin-Netanyahu-repelled-by-success-of-female-MP.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03153/Miri-Regev-netanya_3153123g.jpg"},{"id":"11322759","title":"Quadrantids meteor shower to light up your weekend","url":"/news/science/space/11322759/Quadrantids-meteor-shower-to-light-up-your-weekend.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02440/Quadrantid-Meteor_2440906g.jpg"},{"id":"11322956","title":"AirAsia search finds two 'big' parts of plane","url":"/news/worldnews/asia/indonesia/11322956/AirAsia-search-finds-two-big-parts-of-plane.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03153/airasia-bodies_3153336g.jpg"},{"id":"11322957","title":"Girl, 7, is sole survivor of plane crash in US","url":"/news/worldnews/northamerica/usa/11322957/Girl-7-is-only-survivor-of-plane-crash-that-kills-pilot-and-three-passengers-in-US.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02054/Plane_2054581g.jpg"},{"id":"11322251","title":"Opportunity for the Tories as Ukip swings Left","url":"/news/politics/ukip/11322251/Opportunity-for-the-Tories-as-Ukip-swings-Left.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03151/PX12568722_BRITAIN_3151952a.jpg"},{"id":"11321779","title":"I wished I'd been sent to state school, says Earl Spencer","url":"/news/uknews/theroyalfamily/11321779/I-wished-Id-been-sent-to-state-school-says-Earl-Spencer.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02554/earl-spence_2554688g.jpg"},{"id":"9041875","title":"116 people snubbed honours in past three years","url":"/news/uknews/honours-list/9041875/Revealed-116-people-have-turned-down-an-honour-in-three-years.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/02119/hon_2119931g.jpg"},{"id":"11322572","title":"Former British soldier becomes first man to walk the Nile","url":"/news/worldnews/africaandindianocean/11322572/Former-British-soldier-becomes-first-person-to-walk-the-Nile.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03153/Levison-Wood-Nile-_3153137g.jpg"},{"id":"11318824","title":"My £35 little red dress saved my life, says crash victim","url":"/news/uknews/11318824/My-35-little-red-dress-saved-my-life-says-crash-victim.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03151/Zoe-Turner_3151466g.jpg"},{"id":"11322615","title":"Cologne cathedral will go dark in protest at anti-Islam march","url":"/news/worldnews/europe/germany/11322615/Cologne-cathedral-will-go-dark-in-protest-at-anti-Islam-march.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03153/Cologne_Cathedral_3153175g.jpg"},{"id":"11321260","title":"Google thinks this is Kim Jong-un's younger sister","url":"/news/worldnews/asia/northkorea/11321260/Google-thinks-this-is-Kim-Jong-uns-younger-sister.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03152/Not_Kim_Yo-jong_3152624g.jpg"},{"id":"11322800","title":"Hostage rescue mission in Isil's capital Raqqa foiled, claims group","url":"/news/worldnews/islamic-state/11322800/Hostage-rescue-mission-in-Isils-capital-Raqqa-foiled-claims-group.html","imgUrl":"http://i.telegraph.co.uk/multimedia/archive/03153/isil-raqqa_3153245g.jpg"},] ; var readNextCurrentArticleId = "11313354"; diff --git a/tests/data/html/thai_article.html b/tests/data/html/thai_article.html new file mode 100644 index 00000000..72afa4c7 --- /dev/null +++ b/tests/data/html/thai_article.html @@ -0,0 +1,629 @@ + + + + + + + + + + + + + + + ผล DNA ยืนยัน ศพลอยแม่น้ำโขงเป็นคนสนิท อ.สุรชัย | ประชาไท Prachatai.com + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      +
      +
      +
      +
      +
      +
      +
      + +
      + + + +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      + + + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      + + + +
      +
      + + ประชาไท / + ข่าว
      +
      + + + + + + +
      +
      +
      +
      +
      +
      +
      +
      + +
      +
      + +

      ผล DNA ยืนยัน ศพลอยแม่น้ำโขงเป็นคนสนิท อ.สุรชัย

      +
      +
      + +
      +
      + + +
      + Submitted on Mon, 2019-01-21 13:49
      +

      +

      21 ม.ค.2561 เวลาประมาณ 12.00 น. ผู้สื่อข่าวได้รับแจ้งจากนาย ต.(ไม่ประสงค์เปิดเผยชื่อ) บุตรชายของ 'ภูชนะ' คนสนิทของนายสุรชัย แซ่ด่าน ที่หายไปว่า ผลการตรวจ DNA โดยใช้เนื้อเยื่อของศพที่ถูกสังหารด้วยการมัดแขน รัดคอ ทุบจนใบหน้าเละ และท้องถูกผ่ายัดเสาปูนที่ลอยมาติดที่ อ.ธาตุพนม จ.นครพนม นั้น เมื่อตรวจสอบแล้วมีความเกี่ยวพันทางสายเลือดกับนาย ต.จริง 

      +

      ภูชนะ (นามแฝง) เป็นคนใกล้ชิดของนายสุรชัย ด่านวัฒนานุสรณ์ หรือ สุรชัย แซ่ด่าน นักเคลื่อนไหวทางการเมืองที่ลี้ภัยออกจากประเทศไทยและได้หายตัวไปจากที่พักในประเทศเพื่อนบ้านในช่วงคืนวันที่ 12-13 ธ.ค.2561 พร้อมกันกับสุรชัยและคนสนิทอีกคนหนึ่ง

      +

      ปรานี ด่านวัฒนานุสรณ์ ภรรยาของสุรชัยกล่าวว่า ได้ทราบข่าวจากบุตรชายของภูชนะแล้ว แต่ยังไม่ขอพูดอะไร โดยเบื้องต้นได้ทำใจแต่แรกแล้วว่าเหตุการณ์ลักษณะนี้จะต้องเกิดขึ้นสักวันหนึ่ง

      +ลงโฆษณากับประชาไท
      ThisAble.me +

      +

      ขณะที่ นาย ว. พี่เขยของ 'กาสะลอง' อีกหนึ่งผู้ลี้ภัยที่ได้หายไปพร้อมกับนายสุรชัยกล่าวกับผู้สื่อข่าวว่า หลังจากได้ทราบผลการตรวจสอบ DNA จากลูกชายของภูชนะ ทางญาติของกาสะลองก็ได้ติดต่อไปที่พนักงานสอบสวนเจ้าของคดีและได้คำตอบว่า จะได้ทราบผลการตรวจสอบ DNA ภายในเวลา 2-3 วันนี้

      +

      สุรชัย (78 ปี) ภูชนะ (54 ปี) กาสะลอง (47 ปี) เป็นนักเคลื่อนไหวทางการเมือง และเป็นผู้ลี้ภัยจากเหตุการณ์รัฐประหาร 2557 ไปยังประเทศเพื่อนบ้านได้หายออกจากที่พักในประเทศเพื่อนบ้านโดยที่ไม่มีใครสามารถติดต่อได้จนปัจจุบันนับเป็นเวลานาน 1 เดือน กับอีก 10 วัน

      +

      เรื่องที่เกี่ยวข้อง
      ลืออุ้ม อ.สุรชัย ผู้ลี้ภัยหายจากที่พัก 12 วันแล้ว
      ตร.เร่งหาเบาะแสคดีฆ่ายัดเสาถ่วงน้ำโขง - เมียไม่เชื่อเป็น 'สุรชัย แซ่ด่าน'

      + +
      ข่าว
      การเมือง
      สิทธิมนุษยชน
      ผู้ลี้ภัย
      อุ้มหาย
      สุรชัย ด่านวัฒนานุสรณ์
      กาสะลอง
      ภูชนะ
      การทำให้สาบสูญ
      +
      + + +
      + เท่าไรก็ได้ การสนับสนุนจากคุณ คือการร่วมสร้างและรักษาสื่อเสรี ‘ประชาไท’ ... ร่วมสนับสนุนเรา
      โอนเงิน พร้อมเพย์ PromptPay "มูลนิธิสื่อเพื่อการศึกษาของชุมชน" 0993000060423
      โอนเงิน PayPal คลิกที่นี่ https://paypal.me/prachatai (รายงานยอดบริจาคสนับสนุน) +
      +
      + +
      ติดตามประชาไทอัพเดท ได้ที่:
      เฟซบุ๊ก https://fb.me/prachatai
      ทวิตเตอร์ https://twitter.com/prachatai
      LINE ไอดี = @prachatai
      + +
      + + +
      +
      +
      +
      +
      +

      แสดงความคิดเห็น

      + +
      + +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      +
      + +
      +

      +
      +
      +
      +
      +
      +
      +
      + +
      +

      +
      +
      +
      +
      +
      + +
      +
      +
      +
      + + + \ No newline at end of file diff --git a/tests/data/html/theatlantic.com1.html b/tests/data/html/theatlantic.com1.html index 22756a08..473a5ced 100644 --- a/tests/data/html/theatlantic.com1.html +++ b/tests/data/html/theatlantic.com1.html @@ -89,12 +89,12 @@ - + - + - +

      +
      @@ -682,12 +682,12 @@ } }) -
      +
      -
      @@ -1612,46 +1612,46 @@

      More in Entertainment

    • - - + +

      - - TV Shows to Look Forward to in 2015 + + The Imitation Game: A Smart Person's Fantasy

      -
      David Sims
      +
      Emma Green
    • - - + +

      - - Remembering Richard Gilmore + + Readability Is a Myth

      -
      Kevin O'Keeffe
      +
      Noah Berlatsky
    • - - + +

      - - What The Interview's Success Means for Streaming's Future + + A Design Magazine That's More Than Just Pretty

      -
      Zachary M. Seward
      +
      Steven Heller
      • @@ -1661,13 +1661,13 @@

      Atlantic.Ad.sizeListener('midboxright', Atlantic.Ad.detect.promoColumn);
      -
      +

      In Focus

       - -

      2014 Seen Through the Lens of Lucas Jackson

      + +

      2015: Ringing in the New Year

      @@ -1679,14 +1679,14 @@

    • -

      Mario Cuomo, a Thinker in Public

      - James Fallows
      • +

      Why Would Anybody Adopt the Euro in 2015?

      + Adam Chandler
    • -

      Chickenhawk Responses No. 7: Winning Battles, Losing Wars

      - James Fallows
      • +

      In Partial Defense of Lee Greenwood's Grammatically Challenged 'God Bless the U.S.A.'

      + Jeffrey Goldberg
    • -

      New York's Unfinished Tale of Two Cities

      - David Ludwig
      • +

      Human Traffickers Are Abandoning Boats Full of Migrants

      + Adam Chandler @@ -1694,7 +1694,7 @@

      Atlantic.Ad.sizeListener('bottomboxright', Atlantic.Ad.detect.promoColumn); - +
      @@ -1744,7 +1744,7 @@

      Most Popular

      @@ -1776,9 +1776,9 @@

      Quartz

      More from Quartz

      @@ -1787,9 +1787,9 @@

      National Journal

      More from National Journal

      @@ -2200,7 +2200,7 @@

      - - + + diff --git a/tests/data/html/theatlantic.com2.html b/tests/data/html/theatlantic.com2.html index cf1dd779..96c5de78 100644 --- a/tests/data/html/theatlantic.com2.html +++ b/tests/data/html/theatlantic.com2.html @@ -88,12 +88,12 @@ - + - + - +

      +
      @@ -685,12 +685,12 @@ } }) -
      +
      -
      @@ -1625,46 +1625,46 @@

      More in Entertainment

    • - - + +

      - - TV Shows to Look Forward to in 2015 + + The Imitation Game: A Smart Person's Fantasy

      -
      David Sims
      +
      Emma Green
    • - - + +

      - - Remembering Richard Gilmore + + Readability Is a Myth

      -
      Kevin O'Keeffe
      +
      Noah Berlatsky
    • - - + +

      - - What The Interview's Success Means for Streaming's Future + + A Design Magazine That's More Than Just Pretty

      -
      Zachary M. Seward
      +
      Steven Heller
      • @@ -1674,13 +1674,13 @@

      Atlantic.Ad.sizeListener('midboxright', Atlantic.Ad.detect.promoColumn);
      -
      +

      In Focus

       - -

      2014 Seen Through the Lens of Lucas Jackson

      + +

      2015: Ringing in the New Year

      @@ -1692,14 +1692,14 @@

    • -

      Mario Cuomo, a Thinker in Public

      - James Fallows
      • +

      Why Would Anybody Adopt the Euro in 2015?

      + Adam Chandler
    • -

      Chickenhawk Responses No. 7: Winning Battles, Losing Wars

      - James Fallows
      • +

      In Partial Defense of Lee Greenwood's Grammatically Challenged 'God Bless the U.S.A.'

      + Jeffrey Goldberg
    • -

      New York's Unfinished Tale of Two Cities

      - David Ludwig
      • +

      Human Traffickers Are Abandoning Boats Full of Migrants

      + Adam Chandler @@ -1707,7 +1707,7 @@

      Atlantic.Ad.sizeListener('bottomboxright', Atlantic.Ad.detect.promoColumn); - +
      @@ -1757,7 +1757,7 @@

      Most Popular

      @@ -1789,9 +1789,9 @@

      Quartz

      More from Quartz

      @@ -1800,9 +1800,9 @@

      National Journal

      More from National Journal

      @@ -2227,7 +2227,7 @@

      - - + + diff --git a/tests/data/html/theatlanticcities.com1.html b/tests/data/html/theatlanticcities.com1.html index 0d5eab83..f6bbd536 100644 --- a/tests/data/html/theatlanticcities.com1.html +++ b/tests/data/html/theatlanticcities.com1.html @@ -84,9 +84,9 @@ - + - + + @@ -1155,6 +1155,6 @@

      Newsletter sign up

      - + diff --git a/tests/data/html/theatlanticcities.com2.html b/tests/data/html/theatlanticcities.com2.html index 6b13a2f1..9e070c75 100644 --- a/tests/data/html/theatlanticcities.com2.html +++ b/tests/data/html/theatlanticcities.com2.html @@ -84,9 +84,9 @@ - + - + + @@ -1140,6 +1140,6 @@

      Newsletter sign up

      - + diff --git a/tests/data/html/thedailybeast.com1.html b/tests/data/html/thedailybeast.com1.html index 85707c7f..a5c7ac43 100644 --- a/tests/data/html/thedailybeast.com1.html +++ b/tests/data/html/thedailybeast.com1.html @@ -1,5 +1,5 @@ The Vatican’s Rare Nod to American Nuns - The Daily Beast

      Spencer Platt/Getty

      Sister Act

      12.16.14

      The Vatican’s Rare Nod to American Nuns

      The Catholic Church released its final report on its investigation into ‘feminist’ American nuns—and it took a softer stance toward the sisters.

      VATICAN CITY—Six years ago Pope Benedict XVI aimed to curb “a certain feminist spirit” and “secularist mentality” that the Church fathers discerned, to their dismay, in the ranks of American nuns. On Tuesday, when the Vatican under Pope Francis delivered its final report on the subject, those concerns were nowhere to be found.

      -

      The report was the fruit of an investigation into 341 American congregations guided by Mother Superior Marie Clare Millea, a matronly sister who became tearful at times, in front of a packed pressroom, while describing how she went about collecting her data.  Only the cloistered convents were excluded, and not all of the religious orders complied with the visitors’ requests and questionnaires, though Millea was unable to recall just how many or what percentage of American nuns refused to cooperate.  

      The focus of her investigation was on how nuns live, work and pray, ranging from issues of communal prayer and community service to whether nuns who chose not to wear a religious habit were hiding their faith.  She wrote individual reports on each of the congregations she visited, and Tuesday’s report was the Vatican’s synthesis of those findings.  

      The report concluded that while all American nuns need to recheck the rulebooks to make sure they are living their vows as they made them, they were largely doing what they were supposed to be doing. “Women religious have courageously been in the forefront, selflessly tending to the spiritual, moral, educational, physical and social needs of countless individuals,” the report said with no mention of secularism or feminism at all.

      The report will undoubtedly be welcomed by most American nuns, but few will forget the fighting words from a separate investigation called a doctrinal assessment that is not nearly as conciliatory. In 2009, the Vatican’s Congregation for the Doctrine of the Faith started a separate investigation into an umbrella group called the Leadership Conference of Women Religious (LCWR) that represents roughly 80 percent of American nuns. Then, the Church accused the LCWR of “pushing radical feminist themes incompatible with the Catholic faith.”

      The clampdown backfired, spawning a social media frenzy with more than a million tweets under the hashtag #whatsistersmeantome in support of the nuns and their role in holding Catholic communities together. At the time, Father James Martin, a Jesuit priest and author of several faith-based books, who was the first to tweet under #whatsistersmeantome, warned the Vatican not to mess with the nuns.

      “There is a danger of backlash because of the esteem [in which] so many Catholics hold nuns,” Martin told The Daily Beast at the height of the scandal.  “For many Catholics, sisters are the glue that holds the church together.”

      The focus of her investigation was on how nuns live, work and pray.

      There was much hope when Pope Francis was elected that he would show mercy to the American sisters.  But in April, Gerhard Müller, the prefect for the Congregation for the Doctrine of the Faith, quashed that hope, insisting instead that Francis backed up the 2009 report.  Muller  criticized the nuns’ choice of speakers for their annual event accusing the sisters of being overtly provocative. “This is a decision that will be seen as a rather open provocation against the Holy See and the doctrinal assessment,” Mueller said in scathing remarks. “Not only that, but it further alienates the LCWR from the bishops, as well.”

      The Vatican is still working with the LCWR to find a middle ground and, despite what amounted to a lovefest at Tuesday’s press conference, there is still obvious concern.  Sister Sharon Holland, the head of the LCWR, who incidentally was the only sister on the stage without a religious habit, told reporters that there were still hard feelings.  “I’m concerned about those who may still be angry,” she said.  “It’s a concern to me because it’s not healthy to remain angry.”

      Prior to the publication of report, several sisters who coauthored a book called The Power of Sisterhood that was borne of the apostolic visitation (and not the LCWR doctrinal assessment), voiced their concern for the damage the investigations had done.  “The Apostolic Visitation broke open the heart of who we are,” said Mary Ann Zollman at an event to launch their book at Loyola University. “It was met with disbelief and confusion. We were hurt and angry.”

      But she said through the accusations from Rome and the hurt they caused, the American sisters have banded together. “This wasn’t about one religious congregation or religious organization. It was about all of us together,” she said.  “It created a sense of solidarity and sisterhood. And the same was true with our relationship with the laity—we discovered a communal vision for the church and the world.”

      +

      The report was the fruit of an investigation into 341 American congregations guided by Mother Superior Marie Clare Millea, a matronly sister who became tearful at times, in front of a packed pressroom, while describing how she went about collecting her data.  Only the cloistered convents were excluded, and not all of the religious orders complied with the visitors’ requests and questionnaires, though Millea was unable to recall just how many or what percentage of American nuns refused to cooperate.  

      The focus of her investigation was on how nuns live, work and pray, ranging from issues of communal prayer and community service to whether nuns who chose not to wear a religious habit were hiding their faith.  She wrote individual reports on each of the congregations she visited, and Tuesday’s report was the Vatican’s synthesis of those findings.  

      The report concluded that while all American nuns need to recheck the rulebooks to make sure they are living their vows as they made them, they were largely doing what they were supposed to be doing. “Women religious have courageously been in the forefront, selflessly tending to the spiritual, moral, educational, physical and social needs of countless individuals,” the report said with no mention of secularism or feminism at all.

      The report will undoubtedly be welcomed by most American nuns, but few will forget the fighting words from a separate investigation called a doctrinal assessment that is not nearly as conciliatory. In 2009, the Vatican’s Congregation for the Doctrine of the Faith started a separate investigation into an umbrella group called the Leadership Conference of Women Religious (LCWR) that represents roughly 80 percent of American nuns. Then, the Church accused the LCWR of “pushing radical feminist themes incompatible with the Catholic faith.”

      The clampdown backfired, spawning a social media frenzy with more than a million tweets under the hashtag #whatsistersmeantome in support of the nuns and their role in holding Catholic communities together. At the time, Father James Martin, a Jesuit priest and author of several faith-based books, who was the first to tweet under #whatsistersmeantome, warned the Vatican not to mess with the nuns.

      “There is a danger of backlash because of the esteem [in which] so many Catholics hold nuns,” Martin told The Daily Beast at the height of the scandal.  “For many Catholics, sisters are the glue that holds the church together.”

      The focus of her investigation was on how nuns live, work and pray.

      There was much hope when Pope Francis was elected that he would show mercy to the American sisters.  But in April, Gerhard Müller, the prefect for the Congregation for the Doctrine of the Faith, quashed that hope, insisting instead that Francis backed up the 2009 report.  Muller  criticized the nuns’ choice of speakers for their annual event accusing the sisters of being overtly provocative. “This is a decision that will be seen as a rather open provocation against the Holy See and the doctrinal assessment,” Mueller said in scathing remarks. “Not only that, but it further alienates the LCWR from the bishops, as well.”

      The Vatican is still working with the LCWR to find a middle ground and, despite what amounted to a lovefest at Tuesday’s press conference, there is still obvious concern.  Sister Sharon Holland, the head of the LCWR, who incidentally was the only sister on the stage without a religious habit, told reporters that there were still hard feelings.  “I’m concerned about those who may still be angry,” she said.  “It’s a concern to me because it’s not healthy to remain angry.”

      Prior to the publication of report, several sisters who coauthored a book called The Power of Sisterhood that was borne of the apostolic visitation (and not the LCWR doctrinal assessment), voiced their concern for the damage the investigations had done.  “The Apostolic Visitation broke open the heart of who we are,” said Mary Ann Zollman at an event to launch their book at Loyola University. “It was met with disbelief and confusion. We were hurt and angry.”

      But she said through the accusations from Rome and the hurt they caused, the American sisters have banded together. “This wasn’t about one religious congregation or religious organization. It was about all of us together,” she said.  “It created a sense of solidarity and sisterhood. And the same was true with our relationship with the laity—we discovered a communal vision for the church and the world.”

      \ No newline at end of file diff --git a/tests/data/html/thedailybeast.com2.html b/tests/data/html/thedailybeast.com2.html index 45c9d746..2e8c4119 100644 --- a/tests/data/html/thedailybeast.com2.html +++ b/tests/data/html/thedailybeast.com2.html @@ -1,3 +1,3 @@ - Justice League Vigil for Slain NYPD Officers Asks Whose Life Matters - The Daily Beast

      Mike Segar/Reuters

      MIXED FEELINGS

      12.22.14

      Justice League Vigil for Slain NYPD Officers Asks Whose Life Matters

      At a peaceful gathering of the Justice League, they condemned the killings of Officers Liu and Ramos, but also wondered why the slaying of citizens by cops doesn’t spark as much outrage.

      A modest crowd moved East on 110th Street in New York City on Sunday evening. They walked silently, some carrying Anthora cups illuminated by candlesticks, others holding plastic tea lights handed out by protest organizers. A few carried signs: "IMAGINE JUSTICE," "BLACK LIVES MATTER," "CLAIM HUMANITY." But mostly they just walked, their faces somber, their hands shaking as the snow began to fall. 

      The peaceful vigil was somewhat unexpected, given the heightened tensions between protesters as the NYPD in recent weeks, culminating in the murder of two officers in Bed-Stuy on Saturday afternoon. 

      Just over 24 hours before the silent march down 110th, Wenjian Liu, 32, and Rafael Ramos, 40, had been shot "execution-style," Mayor Bill DeBlasio said, while sitting in their patrol car. The suspect, Ismaaiyl Brinsley, had begun the day with the shooting of his ex-girlfriend in Maryland before taking the trip up to New York, armed with a silver semi-automatic handgun. After murdering the officers, Brinsley walked over to a subway platform where he turned the weapon on himself. 

      In some ways, the tragedy seemed inevitable.

      As the nation continued to cope with Ferguson, a grand jury in Staten Island chose not to indict in the case of Eric Garner, a man choked to death –– as he pleaded "I can't breathe" –– by a police offer in what the coroner declared a homicide. #BlackLivesMatter has been the prevailing message as protesters have taken to the streets to express their outrage. They often chant: "No justice, no peace; No racist police." 

      All the while, there have been critics. Why, they ask, do black lives matter? Why not all lives? They contend that the protests are not anti-police brutality, but anti-police, period. They believe that the protesters and those who enabled them—the media, or De Blasio, who spoke frankly about how he talks to his biracial son, whose mother is black, about the threats he might face due to his skin color—have been vying for NYPD scalps. With two now dead, the critics are pointing fingers. 

      "The mayor’s hands are literally dripping with our blood because of his words, actions and policies and we have, for the first time in a number of years, become a 'wartime' police department," Pat Lynch, head of the police union, said at a press conference on Saturday night. "There's blood on many hands tonight…That blood on the hands starts at City Hall in the Office of the Mayor."

      Former mayor Rudy Giuliani echoed the sentiment, but laid blame on President Obama: "We've had four months of propaganda starting with the president that everybody should hate the police…The protests are being embraced, the protests are being encouraged. The protests, even the ones that don't lead to violence—a lot of them lead to violence, all of them lead to a conclusion: The police are bad, the police are racist. That is completely wrong."

      The march, organized by the Justice League NYC—effectively anointed by De Blasio as City Hall's preferred protest group with a 4-minute meeting to discuss theories of policing—continued slowly along 110th street, a police van humming alongside. The crowd snaked into Central Park, where it came to a stop at the boathouse, a lighted Christmas tree illuminating the water's surface. 

      "Let me not say this wrong," a 21-year-old East Flatbush resident who would say only that his name was "Perry" told me as he looked around, seemingly searching for the right words. "Yesterday's situation that happened in Brooklyn, Bed-Stuy, was uncalled for. It was the wrong way about handling it. Because we're upset that on our end, they're killing us, so I don't think nobody should go take the anger out on, you know—nobody knows these two specific individuals. They could've been good people. They could've been, you know, legit officers who wasn't down with whatever's going on. My condolences do go to their families. But at the same time, since Zimmerman, you've been hearing cases about officers killing black children—black men, Hispanic men.

      "This is not about who did it more, but it's more: all these lives didn't matter, but now two cops are dead, and now it matters? If you look at the newspapers, it's all broadcast all over the place. The whole world's supposed to mourn because—don't get me wrong, it was wrong––but because two officers died, now the world's supposed to mourn? What happened to all the other people that died? Nobody mourned. They wasn't in the front page of newspapers. They was in back pages—some didn't even make the newspaper."

      Next to "Perry" stood Jonathan Alvarez, 18, of Queens, who in November made headlines when he announced plans to sue the city after allegedly being assaulted by police officers for being a "smart guy." 

      "I got targeted. I'm a victim of police brutality," Alvarez told me. "They arrested me, they beat me up in the 102 precinct while I was handcuffed, no justice. So I feel for these people, I understand they're just brutes. They can get away with it, and they do…They don't give a fuck until one of them dies, that's all they care about." 

      "Perry" chimed in: "They don't want to take blame," he said of the police union attacking De Blasio. "There has to be peace on all sides.

      "I think this all could've been prevented if they just listened to the 25,000 people who marched last week," Alvarez said. "No, not even," "Perry" countered: "If they would've just re-analyzed the Zimmerman case, or the Ferguson case."

      Then, "Perry" and Alvarez went silent. The marchers began to stream out of the park, where they walked West on 110th and then hung a right on 7th Avenue. NYPD officers lingered, politely asking participants to move in the line, and saying "thank you" when they obliged, which they all did. 

      The march reached its conclusion at the First Corinthian Baptist Church on 116th street, where piano music befitting a funeral greeting guests who filed into the warm pews. 

      Religious leaders—Pastor Willie Francois III, the Rev. Mike Walrond, and the Rev. Stephen Phelps—and over a dozen activists, led by Justice League NYC's Tamika Mallory, took turns addressing the crowd. 

      There were prayers for 40 black men, beginning with Eric Garner, who had been killed by police; and prayers for De Blasio, who "is attacked on one side and the other," and repeated acknowledgement that violence is not the way to change what the protesters believe is a fundamentally broken system. 

      "We're not anti-police, we're anti-police brutality," a member of the Justice League told the crowd, to cheers. "Every cop isn't bad…Every black man isn't a criminal."

      The crowd rose, embraced each other in groups of two and three, and prayed.

      + Justice League Vigil for Slain NYPD Officers Asks Whose Life Matters - The Daily Beast

      Mike Segar/Reuters

      MIXED FEELINGS

      12.22.14

      Justice League Vigil for Slain NYPD Officers Asks Whose Life Matters

      At a peaceful gathering of the Justice League, they condemned the killings of Officers Liu and Ramos, but also wondered why the slaying of citizens by cops doesn’t spark as much outrage.

      A modest crowd moved East on 110th Street in New York City on Sunday evening. They walked silently, some carrying Anthora cups illuminated by candlesticks, others holding plastic tea lights handed out by protest organizers. A few carried signs: "IMAGINE JUSTICE," "BLACK LIVES MATTER," "CLAIM HUMANITY." But mostly they just walked, their faces somber, their hands shaking as the snow began to fall. 

      The peaceful vigil was somewhat unexpected, given the heightened tensions between protesters as the NYPD in recent weeks, culminating in the murder of two officers in Bed-Stuy on Saturday afternoon. 

      Just over 24 hours before the silent march down 110th, Wenjian Liu, 32, and Rafael Ramos, 40, had been shot "execution-style," Mayor Bill DeBlasio said, while sitting in their patrol car. The suspect, Ismaaiyl Brinsley, had begun the day with the shooting of his ex-girlfriend in Maryland before taking the trip up to New York, armed with a silver semi-automatic handgun. After murdering the officers, Brinsley walked over to a subway platform where he turned the weapon on himself. 

      In some ways, the tragedy seemed inevitable.

      As the nation continued to cope with Ferguson, a grand jury in Staten Island chose not to indict in the case of Eric Garner, a man choked to death –– as he pleaded "I can't breathe" –– by a police offer in what the coroner declared a homicide. #BlackLivesMatter has been the prevailing message as protesters have taken to the streets to express their outrage. They often chant: "No justice, no peace; No racist police." 

      All the while, there have been critics. Why, they ask, do black lives matter? Why not all lives? They contend that the protests are not anti-police brutality, but anti-police, period. They believe that the protesters and those who enabled them—the media, or De Blasio, who spoke frankly about how he talks to his biracial son, whose mother is black, about the threats he might face due to his skin color—have been vying for NYPD scalps. With two now dead, the critics are pointing fingers. 

      "The mayor’s hands are literally dripping with our blood because of his words, actions and policies and we have, for the first time in a number of years, become a 'wartime' police department," Pat Lynch, head of the police union, said at a press conference on Saturday night. "There's blood on many hands tonight…That blood on the hands starts at City Hall in the Office of the Mayor."

      Former mayor Rudy Giuliani echoed the sentiment, but laid blame on President Obama: "We've had four months of propaganda starting with the president that everybody should hate the police…The protests are being embraced, the protests are being encouraged. The protests, even the ones that don't lead to violence—a lot of them lead to violence, all of them lead to a conclusion: The police are bad, the police are racist. That is completely wrong."

      The march, organized by the Justice League NYC—effectively anointed by De Blasio as City Hall's preferred protest group with a 4-minute meeting to discuss theories of policing—continued slowly along 110th street, a police van humming alongside. The crowd snaked into Central Park, where it came to a stop at the boathouse, a lighted Christmas tree illuminating the water's surface. 

      "Let me not say this wrong," a 21-year-old East Flatbush resident who would say only that his name was "Perry" told me as he looked around, seemingly searching for the right words. "Yesterday's situation that happened in Brooklyn, Bed-Stuy, was uncalled for. It was the wrong way about handling it. Because we're upset that on our end, they're killing us, so I don't think nobody should go take the anger out on, you know—nobody knows these two specific individuals. They could've been good people. They could've been, you know, legit officers who wasn't down with whatever's going on. My condolences do go to their families. But at the same time, since Zimmerman, you've been hearing cases about officers killing black children—black men, Hispanic men.

      "This is not about who did it more, but it's more: all these lives didn't matter, but now two cops are dead, and now it matters? If you look at the newspapers, it's all broadcast all over the place. The whole world's supposed to mourn because—don't get me wrong, it was wrong––but because two officers died, now the world's supposed to mourn? What happened to all the other people that died? Nobody mourned. They wasn't in the front page of newspapers. They was in back pages—some didn't even make the newspaper."

      Next to "Perry" stood Jonathan Alvarez, 18, of Queens, who in November made headlines when he announced plans to sue the city after allegedly being assaulted by police officers for being a "smart guy." 

      "I got targeted. I'm a victim of police brutality," Alvarez told me. "They arrested me, they beat me up in the 102 precinct while I was handcuffed, no justice. So I feel for these people, I understand they're just brutes. They can get away with it, and they do…They don't give a fuck until one of them dies, that's all they care about." 

      "Perry" chimed in: "They don't want to take blame," he said of the police union attacking De Blasio. "There has to be peace on all sides.

      "I think this all could've been prevented if they just listened to the 25,000 people who marched last week," Alvarez said. "No, not even," "Perry" countered: "If they would've just re-analyzed the Zimmerman case, or the Ferguson case."

      Then, "Perry" and Alvarez went silent. The marchers began to stream out of the park, where they walked West on 110th and then hung a right on 7th Avenue. NYPD officers lingered, politely asking participants to move in the line, and saying "thank you" when they obliged, which they all did. 

      The march reached its conclusion at the First Corinthian Baptist Church on 116th street, where piano music befitting a funeral greeting guests who filed into the warm pews. 

      Religious leaders—Pastor Willie Francois III, the Rev. Mike Walrond, and the Rev. Stephen Phelps—and over a dozen activists, led by Justice League NYC's Tamika Mallory, took turns addressing the crowd. 

      There were prayers for 40 black men, beginning with Eric Garner, who had been killed by police; and prayers for De Blasio, who "is attacked on one side and the other," and repeated acknowledgement that violence is not the way to change what the protesters believe is a fundamentally broken system. 

      "We're not anti-police, we're anti-police brutality," a member of the Justice League told the crowd, to cheers. "Every cop isn't bad…Every black man isn't a criminal."

      The crowd rose, embraced each other in groups of two and three, and prayed.

      \ No newline at end of file diff --git a/tests/data/html/thedebrief.co.uk1.html b/tests/data/html/thedebrief.co.uk1.html index 4c07e59b..0bc82b97 100644 --- a/tests/data/html/thedebrief.co.uk1.html +++ b/tests/data/html/thedebrief.co.uk1.html @@ -205,7 +205,7 @@

      Lena deCasparis - | Contributing Writer | 4 days ago

      + | Contributing Writer | 5 days ago

      &nbsp diff --git a/tests/data/html/theglobeandmail.com1.html b/tests/data/html/theglobeandmail.com1.html index 27f88602..f972c2d0 100644 --- a/tests/data/html/theglobeandmail.com1.html +++ b/tests/data/html/theglobeandmail.com1.html @@ -37,7 +37,7 @@ - + @@ -46,11 +46,11 @@ - - - - - + + + + + @@ -372,7 +372,7 @@
      -A member of the Indonesian military looks out of the window during a search and rescue (SAR) operation for missing Malaysian air carrier AirAsia flight QZ8501. (Juni Kriswanto/AFP/Getty Images)
      A member of the Indonesian military looks out of the window during a search and rescue (SAR) operation for missing Malaysian air carrier AirAsia flight QZ8501. (Juni Kriswanto/AFP/Getty Images)In the know

      -
      +
      -
      +
      • @@ -731,138 +731,138 @@
      • @@ -887,32 +887,32 @@

        Highlights

        -
        +
        -
        +
        @@ -927,15 +927,15 @@

        More from The Globe and Mail

        -
        -
        - - + @@ -1006,8 +1006,8 @@

        Most popular

      • @@ -1015,8 +1015,8 @@
      • @@ -1024,8 +1024,8 @@
      • @@ -1033,8 +1033,8 @@
      • @@ -1042,8 +1042,8 @@
      • @@ -1061,14 +1061,14 @@

        More Top Stories

        @@ -1077,14 +1077,14 @@
        @@ -1093,62 +1093,63 @@
        -
        +
        -
        +
        -
        -
        - - +
        + @@ -1166,14 +1167,14 @@

        Video

        @@ -1182,14 +1183,14 @@
        @@ -1214,14 +1215,14 @@
        @@ -1237,16 +1238,16 @@

        Photos

        -
        +
        @@ -1255,14 +1256,14 @@
        @@ -1521,25 +1522,25 @@
        Other Sections
        s.prop55 = "Yes"; s.prop37 = "news"; s.prop36 = "art"; -s.prop54 = "4"; +s.prop54 = "5"; s.pageName = "news:world:article"; s.eVar33 = "D=c37"; s.prop1 = "Single Page View - No Pagination"; s.prop23 = "news:world"; s.eVar58 = "D=c58"; s.prop24 = "Indonesia"; -s.prop50 = "Weekday"; +s.prop50 = "Weekend"; s.eVar19 = "D=c24"; s.prop22 = "news:world"; s.prop44 = "news:world"; s.eVar61 = "+1"; s.prop43 = "news"; -s.prop48 = "Friday"; +s.prop48 = "Saturday"; s.prop47 = "news:world"; s.prop46 = "news:world"; s.prop45 = "news:world"; s.account = "bellgmpnewprod"; -s.prop49 = "05:59AM"; +s.prop49 = "06:46AM"; s.channel = "news"; s.prop8 = "Not Top 10/Other"; if (tlu.val == undefined) { @@ -1781,13 +1782,13 @@

        "name": "in-the-know", "ltname": "in-the-know", "view": "storyheadline", -"wId": ".lt20737382" +"wId": ".lt20737385" }, { "name": "in-the-know", "ltname": "in-the-know", "view": "storyheadline", -"wId": ".lt13299702" +"wId": ".lt13299736" }, { "name": "in-the-know", @@ -1805,13 +1806,13 @@

        "name": "highlights", "ltname": "highlights", "view": "storypicture", -"wId": ".lt6157900" +"wId": ".lt10037053" }, { "name": "highlights", "ltname": "highlights", "view": "storypicture", -"wId": ".lt10384885" +"wId": ".lt6159672" }, { "name": "more from the globe and mail", @@ -1853,19 +1854,19 @@

        "name": "more top stories", "ltname": "more_top_stories", "view": "storypicture", -"wId": ".lt4197915" +"wId": ".lt4197916" }, { "name": "more top stories", "ltname": "more_top_stories", "view": "storypicture", -"wId": ".lt4197919" +"wId": ".lt4197920" }, { "name": "more top stories", "ltname": "more_top_stories", "view": "storypicture", -"wId": ".lt4197924" +"wId": ".lt4197925" }, { "name": "video", @@ -1895,7 +1896,7 @@

        "name": "photos", "ltname": "photos", "view": "storypicture", -"wId": ".lt4178506" +"wId": ".lt4178523" }, { "name": "photos", @@ -2025,7 +2026,7 @@

        link: "http://www.theglobeandmail.com/news/world/airasia-search/article22224253/", desc: "Flight data show pilots attempted to fly over tropical storm instead of around, despite normal procedure", linktext: "Read this on The Globe and Mail", -image: "http://beta.images.theglobeandmail.com/4e8/multimedia/camera-club/in-photos/article22232497.ece/ALTERNATES/w220/web-dip29nw1.JPG" +image: "http://static.theglobeandmail.ca/4e8/multimedia/camera-club/in-photos/article22232497.ece/ALTERNATES/w220/web-dip29nw1.JPG" }); _gigya_sb.init(); }); diff --git a/tests/data/html/theglobeandmail.com2.html b/tests/data/html/theglobeandmail.com2.html index b8d1337d..bd744b09 100644 --- a/tests/data/html/theglobeandmail.com2.html +++ b/tests/data/html/theglobeandmail.com2.html @@ -654,10 +654,10 @@

        In the know

        -
        +
        -
        +
      • @@ -890,32 +890,32 @@

        Highlights

        -
        +
        -
        +
        @@ -930,15 +930,15 @@

        More from The Globe and Mail

        -
        -
        - - + @@ -1009,8 +1009,8 @@

        Most popular

      • @@ -1018,8 +1018,8 @@
      • @@ -1027,8 +1027,8 @@
      • @@ -1036,8 +1036,8 @@
      • @@ -1045,8 +1045,8 @@
      • @@ -1064,14 +1064,14 @@

        More Top Stories

        @@ -1080,14 +1080,14 @@
        @@ -1096,62 +1096,63 @@
        -
        +
        -
        +
        -
        -
        - - +
        + @@ -1169,14 +1170,14 @@

        Video

        @@ -1185,14 +1186,14 @@
        @@ -1217,14 +1218,14 @@
        @@ -1240,16 +1241,16 @@

        Photos

        -
        +
        @@ -1258,14 +1259,14 @@
        @@ -1542,25 +1543,25 @@
        Restrictions
        s.prop55 = "Yes"; s.prop37 = "column"; s.prop36 = "art"; -s.prop54 = "4"; +s.prop54 = "5"; s.pageName = "business:internationalbusiness:article"; s.eVar33 = "D=c37"; s.prop1 = "Single Page View - No Pagination"; s.prop23 = "business:internationalbusiness"; s.eVar58 = "D=c58"; s.prop24 = "United Kingdom;Japan;Europe;Russia;United States of America;China;Energy;Canada;India"; -s.prop50 = "Weekday"; +s.prop50 = "Weekend"; s.eVar19 = "D=c24"; s.prop22 = "business:internationalbusiness"; s.prop44 = "business:internationalbusiness"; s.eVar61 = "+1"; s.prop43 = "business"; -s.prop48 = "Friday"; +s.prop48 = "Saturday"; s.prop47 = "business:internationalbusiness"; s.prop46 = "business:internationalbusiness"; s.prop45 = "business:internationalbusiness"; s.account = "bellgmpnewprod"; -s.prop49 = "05:59AM"; +s.prop49 = "06:46AM"; s.channel = "business"; s.prop8 = "Not Top 10/Other"; if (tlu.val == undefined) { @@ -1802,13 +1803,13 @@

        "name": "in-the-know", "ltname": "in-the-know", "view": "storyheadline", -"wId": ".lt20737382" +"wId": ".lt20737385" }, { "name": "in-the-know", "ltname": "in-the-know", "view": "storyheadline", -"wId": ".lt13299702" +"wId": ".lt13299736" }, { "name": "in-the-know", @@ -1826,13 +1827,13 @@

        "name": "highlights", "ltname": "highlights", "view": "storypicture", -"wId": ".lt6157900" +"wId": ".lt10037053" }, { "name": "highlights", "ltname": "highlights", "view": "storypicture", -"wId": ".lt10384885" +"wId": ".lt6159672" }, { "name": "more from the globe and mail", @@ -1874,19 +1875,19 @@

        "name": "more top stories", "ltname": "more_top_stories", "view": "storypicture", -"wId": ".lt4197915" +"wId": ".lt4197916" }, { "name": "more top stories", "ltname": "more_top_stories", "view": "storypicture", -"wId": ".lt4197919" +"wId": ".lt4197920" }, { "name": "more top stories", "ltname": "more_top_stories", "view": "storypicture", -"wId": ".lt4197924" +"wId": ".lt4197925" }, { "name": "video", @@ -1916,7 +1917,7 @@

        "name": "photos", "ltname": "photos", "view": "storypicture", -"wId": ".lt4178506" +"wId": ".lt4178523" }, { "name": "photos", diff --git a/tests/data/html/thekitchn.com1.html b/tests/data/html/thekitchn.com1.html index 059ee792..d6f2bb50 100644 --- a/tests/data/html/thekitchn.com1.html +++ b/tests/data/html/thekitchn.com1.html @@ -9,9 +9,9 @@ - + - + @@ -307,12 +307,12 @@

        Before & After: Replacing Tiny Details in This Kitche
        -Pin it button big +Pin it button big

        @@ -327,7 +327,7 @@

        Before & After: Replacing Tiny Details in This Kitche
        - + @@ -399,7 +399,7 @@
        Before & After
        Previous -548ded5e697ab044fb008b68. w.60 h.60 s.centercrop +548ded5e697ab044fb008b68. w.60 h.60 s.centercrop
        Before & After: A Quirky Kitchen Gets a Farmhouse Makeover
        @@ -471,7 +471,7 @@

        Pin it button +Pin it button

      Show/Hide Navigation

      I Don’t See Race; I Only See Grayish-Brown, Vaguely Humanoid Shapes

      Commentary + data-viewports="mobile">

      Woman’s Parents Accepting Of Mixed-Attractiveness Relationship

      News in Brief • - Opinion - • race + parents + • dating + • Lifestyle • - - ISSUE 50•49 + + ISSUE 51•02 • - Dec 12, 2014 -
      - By Janice Ketchum - -

      I don’t understand why everyone seems to be so angry lately. Everywhere you look, there are marches, protests, riots—and all of it over so-called racism in our great country. I just don’t get it. I really don’t. But maybe that’s because, when I look at my fellow Americans, I don’t see a particular race or color. In fact, all I see is just a series of muted, roughly person-shaped silhouettes.

      And this world would be a much better place if everyone else did, too.

      If we want to go forward as a country, we have to move beyond race once and for all. It’s 2014! Why is it so hard for people to look past each other’s race, like I do, and see everyone as vaporous, beige-ish forms with limbs? All those blurry, vaguely human-shaped troublemakers shouting in the streets and the translucent bleeding-heart blobs moralizing on TV may feel the need to categorize everyone they see by their skin color, but I don’t give it a second thought. I wouldn’t even know how!

      Take, for example, the recent deaths of Michael Brown and Eric Garner, which sparked widespread outrage over “institutional racism.” These protesters are stuck in the past. They haven’t progressed enough to see that we’re all just grayish-brown smudges between approximately 4 and 6 and a half feet tall. They are part of the problem! Whereas they see “black” victims and “white” police, I just see amorphous clouds of the same basic hue, some of which have guns.

      Like Martin Luther King Jr., who spoke for nondescript blotches everywhere, I judge a man not based on the color of his skin, but on the content of his character. And I’ll be the first to admit that the civil rights era was important. It made sure that everyone, no matter what fuzzy, neutral tint their ill-defined, human-like shape happens to be, could enjoy the same liberties as anyone else. But it’s not the 1960s anymore, and the very idea of race is ridiculous now. We have a fuzzy, neutral-tinted president, for crying out loud! Get over it.

      I’m not saying this is an easy road to walk. In fact, in many respects, the way I see the world has made my life a living hell. I don’t know who anyone is. I have no way to tell where one person ends and another begins. I’ve been married for 13 years and I honestly do not know what race my husband is, let alone his gender. I assume he is alive, but there’s no actual proof of that. I have no clue what my children look like, or if I have children, or if they are dogs. The universe is a terrifying orchestra of matter and sound not tethered to any discernible reality. I am literally living in horror every moment of my life.

      But you don’t see me rioting in the streets about it, do you?

      People ask me all the time if I have any black friends, and my answer is, I have no idea. What is black? Are the globs who are good at basketball “black”? Or are the globs who are good at math “black”? Like I said, I don’t see race, because I am not capable of discerning any physical difference between the nebulous tangle of lifesmears that comprise all of humanity. People accuse me—me!—of being racist. I am the least racist person in the world! How can it be racism if all I can see are fleeting wisps of electromagnetic activity and a shimmering aura that represents each person’s individual essence?

      If everyone saw things from my perspective, we could finally put this issue of race in the past. Then every hazy, indistinct mass in this country could live in harmony. Except for those awful Spanish-accented humanoid shapes who are taking all our jobs. They need to go.

      CHICAGO—Admitting it took them some time to come around to the idea, the parents of local woman Laura Stevens said Wednesday that they had finally accepted their daughter’s mixed-attractiveness relationship with Kyle Baker, a man who is considerably worse-looking than she is. “To be honest, we were quite surprised when Laura brought Kyle to the house for the first time, but eventually we came around to it,” said Stevens’ mother, Janet, who noted that the pair were still met with uncomfortable stares and disapproval from other family members—especially Laura’s grandmother—at last year’s Thanksgiving dinner. “Her father was particularly upset at first, but now I think he’s learned to accept it, and he’s even grown to like Kyle. Besides, Kyle seems to make Laura happy, and that’s all that really matters.” Janet Stevens went on to say that, if the two ever got married, she would love their children no matter how average-looking they are.

      Popular Onion Video

      Watch more videos
        More Headlines -
      • High Cheekbones and Straight Black Hair?
      • How Do I Trace My Cape Verdean Ancestry?
      • In Goya’s Adoration of the Magi, a Black King Plays an Exalted Role
      • Who Was the Black Wise Man? 100 Amazing Facts About the Negro
      • How Do I Research My Guyanese Heritage?
        •  +
      • Did Trouble With Horse Rustling Do in My Texas Ancestor?
      • The Story of a Nubian King Etched in Stone
      • High Cheekbones and Straight Black Hair?
      • How Do I Trace My Cape Verdean Ancestry?
      • In Goya’s Adoration of the Magi, a Black King Plays an Exalted Role
        •
        @@ -835,24 +833,24 @@
        The Story of a Nubian King Etched in Stone
        - +
      • - -
        Good Parenting: Father Tells His Daughter How A Good Man Should Treat Her! (Video)
        + +
        Oregon Players Face Team Discipline after Chanting 'No Means No'
        •  - +
      • - -
        Commemoration for Oscar Grant Who Was Killed by Johannes Mehserle 6 years ago
        + +
        Waiter Fired for Racist Remarks Show Up on Receipts
        •
        More Headlines -
      • Kanye West - Only One (Feat. Paul McCartney)
      • One Million Ring in 2015 With NY Ball Drop
      • NFL 2014: A Year Of Controversy
      • "Thank You Jesus, Hallelujah!" Son & Daughter Surprise Their Mother With A Benz For Christmas
      • Purple Heart Returned to Rightful Owner
        •  +
      • Movie Trailer for 'That Daughter's Crazy'
      • Muriel Bowser Sworn in as D.C. Mayor
      • Reports: John Crawford's Girlfriend Killed in Crash
      • Make 2015 Awesome: Happy New Year from Kid President
      • Cleveland Baby Born on New Year's Day Shares Birthday with Mom and Dad
        •
        @@ -1271,42 +1269,42 @@

        See Also

        -
      • A ‘Good’ Friend Wouldn’t Do What You Just Did
        • +
      • Ala. Grandmother’s Home Spray-Painted: ‘Move N--ger Now’
        • -
      • New Year’s Resolutions for Olivia Pope and Co.? Handled
        • +
      • Texas Server Fired After Using N-Word on Receipt to Label Diners
        • -
      • Black Calif. Woman Denied Spot Almost 60 Years Ago on Rose Parade Float Gets Retribution
        • +
      • New Orleans Woman Loses Husband and Son to Gun Violence Nearly 10 Years After Youngest Son Was Shot to Death
        • -
      • A Black Man Says Good Riddance to 2014
        • +
      • Get THINspired This New Year and Stick to Your Weight-Loss Goals
        • -
      • Report: Baltimore Ravens’ Security Director Charged With Sex Offense
        • +
      • The NYPD Slowdown Is Proving That ‘Broken Windows’ Is a Failure. Isn’t It Time to Drop It?
        • -
      • Man Shoots and Robs Woman He Met on Online Dating Site 
        • +
      • John Crawford’s Girlfriend Killed in Car Crash
        • @@ -1654,24 +1652,24 @@

        Recent Videos

        @@ -2103,7 +2101,7 @@

        THE 2014 ROOT 100

        diff --git a/tests/data/html/theroot.com2.html b/tests/data/html/theroot.com2.html deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/data/html/tnr.com1.html b/tests/data/html/tnr.com1.html index b3c6361f..8977b545 100644 --- a/tests/data/html/tnr.com1.html +++ b/tests/data/html/tnr.com1.html @@ -278,7 +278,7 @@

        - +
        @@ -493,25 +493,25 @@

        More articles tagged as

        -
        +
        + + The Solar System Is Slowly Re-Arranging Itself +
        +
        Male Nerds Think They're Victims Because They Have No Clue What Female Nerds Go Through
        -
        - - At Last, Television Shows Have Figured Out What Sex Is Like for Women -
        -
        - - Why James Bond Isn't Black -
        Stop Trying to Save the World
        -
        - - The Best Fiction of 2014 +
        + + Oregon's Players Should Be Celebrated, Not Punished for Chanting "No Means No!" +
        +
        + + At Last, Television Shows Have Figured Out What Sex Is Like for Women
        @@ -525,7 +525,7 @@

        More articles tagged as

        - + +
        +

        + + Oregon's Players Should Be Celebrated, Not Punished for Chanting "No Means No!" +

        +
        +
        +
        + +
        +

        + + Personality Matters More Than Intelligence at School +

        +
        +
        +
        +

        - - The Solar System Is Slowly Re-Arranging Itself + + 'Selma' Ignores the Radical Grassroots Politics of the Civil Rights Movement

        @@ -649,7 +667,7 @@
        Sign Up For The New Republic Daily
        -
        +
        + + The Solar System Is Slowly Re-Arranging Itself +
        +
        Male Nerds Think They're Victims Because They Have No Clue What Female Nerds Go Through
        -
        - - At Last, Television Shows Have Figured Out What Sex Is Like for Women -
        -
        - - Why James Bond Isn't Black -
        Stop Trying to Save the World
        -
        - - The Best Fiction of 2014 +
        + + Oregon's Players Should Be Celebrated, Not Punished for Chanting "No Means No!" +
        +
        + + At Last, Television Shows Have Figured Out What Sex Is Like for Women
        More Articles in Politics
        -
        +
        + + The Solar System Is Slowly Re-Arranging Itself +
        +
        Male Nerds Think They're Victims Because They Have No Clue What Female Nerds Go Through
        -
        - - At Last, Television Shows Have Figured Out What Sex Is Like for Women -
        -
        - - Why James Bond Isn't Black -
        Stop Trying to Save the World
        -
        - - The Best Fiction of 2014 +
        + + Oregon's Players Should Be Celebrated, Not Punished for Chanting "No Means No!" +
        +
        + + At Last, Television Shows Have Figured Out What Sex Is Like for Women
        @@ -793,7 +811,7 @@
        Sign Up For The New Republic Daily
        - +
        @@ -1155,5 +1173,5 @@
        -
        +
        diff --git a/tests/data/html/tnr.com2.html b/tests/data/html/tnr.com2.html index 29c15639..943a932b 100644 --- a/tests/data/html/tnr.com2.html +++ b/tests/data/html/tnr.com2.html @@ -280,7 +280,7 @@

        -
        +
        @@ -309,7 +309,7 @@

        +
        diff --git a/tests/data/html/uproxx.com1.html b/tests/data/html/uproxx.com1.html index 7b1511c7..80836fe8 100644 --- a/tests/data/html/uproxx.com1.html +++ b/tests/data/html/uproxx.com1.html @@ -3,7 +3,7 @@ xmlns:og="http://opengraphprotocol.org/schema/" xmlns:fb="http://ogp.me/ns/fb#"> @@ -45,7 +45,7 @@ - + @@ -1020,13 +1020,13 @@

        Sign Up

        }catch(e){} })(); // ]]> - + diff --git a/tests/data/html/uproxx.com2.html b/tests/data/html/uproxx.com2.html index bc496bd8..cbb7546c 100644 --- a/tests/data/html/uproxx.com2.html +++ b/tests/data/html/uproxx.com2.html @@ -1,86 +1,1182 @@ - - - Comments on: Simon Pegg Gave His Blessing For A Touring ‘Shaun Of The Dead’ Stage Show - - http://uproxx.com/filmdrunk/2014/12/simon-pegg-gave-his-blessing-for-a-touring-shaun-of-the-dead-stage-show/ - The Culture Of What's Buzzing - Fri, 02 Jan 2015 10:55:37 +0000 - hourly - 1 - http://wordpress.com/ - - By: Phyllis Rivers - http://uproxx.com/filmdrunk/2014/12/simon-pegg-gave-his-blessing-for-a-touring-shaun-of-the-dead-stage-show/comment-page-1/#comment-2519455 - - Tue, 30 Dec 2014 17:04:05 +0000 - http://uproxx.com/?p=758539#comment-2519455 - - I loved that part. Ha ha!

        -

        Take that Baz Luhrmann!

        -]]>         -         - - By: AreWeFriendsJack? - http://uproxx.com/filmdrunk/2014/12/simon-pegg-gave-his-blessing-for-a-touring-shaun-of-the-dead-stage-show/comment-page-1/#comment-2519331 - - Tue, 30 Dec 2014 16:15:55 +0000 - http://uproxx.com/?p=758539#comment-2519331 - - Hopefully it’s as good as their take on Shakespeare’s Romeo + Juliet

        -

        -]]>         -         - - By: Ace Rimmer - http://uproxx.com/filmdrunk/2014/12/simon-pegg-gave-his-blessing-for-a-touring-shaun-of-the-dead-stage-show/comment-page-1/#comment-2519118 - - Tue, 30 Dec 2014 13:59:15 +0000 - http://uproxx.com/?p=758539#comment-2519118 - - Oh good, it’s not a musical, it’s a production with tongue-in-cheek songs. That won’t be obnoxious at all, then. Just … a musical with poor enunciation.

        -]]>         -         - - By: Darkest Timeline Zach Morris - http://uproxx.com/filmdrunk/2014/12/simon-pegg-gave-his-blessing-for-a-touring-shaun-of-the-dead-stage-show/comment-page-1/#comment-2517683 - - Tue, 30 Dec 2014 02:57:06 +0000 - http://uproxx.com/?p=758539#comment-2517683 - - Is he related to Charles in Charge’s alter ego?
        + + + + + + Simon Pegg Gave His Blessing For A Touring ‘Shaun Of The Dead’ Show + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
        + + + + +
        + + + + + + + + + + +
        +
        +
        + + + + + + + +
        +
        +
        + + +
        +
        +
        + + + + +
        + +
        + Paul Rudd Returns To Pawnee For The Final Season Of 'Parks And Recreat...
        +
        +         + + + +
        +
        + Ludacris And His Girlfriend Join The Mile-High Engagement Club
        + +
        + +         + +
        +
        + +
        +
        +
        + + +

        + Simon Pegg Gave His Blessing For A Touring ‘Shaun Of The Dead’ Stage Show +

        + + + By • 12.29.14 + + + +
        +
        + +
        +
        +
        +

        The people at the UK’s Almost Legal Productions want to turn Shaun of the Dead into the next Rocky Horror Picture Show, and thanks to Simon Pegg their dream may be on the fast track to reality. Last year, director Chas Burns* organized a stage show of the 2004 zombie comedy and the culmination was four sold-out shows in October and November, and the show certainly received a little marketing help from Pegg, who tweeted about the event. Burns said that after that glorious 140-character endorsement, people suddenly wanted his new show being performed all over the globe.

        +

        The great news is that those four shows have turned into 15 touring dates for 2015, as Burns told Chortle a simple phone call to Pegg’s assistant led to not only the actor giving his blessing for a Shaun of the Dead stage show, but Working Title and Universal Pictures have also signed off. If we all cross our fingers and toes, that could lead to Shaun and Ed eventually coming to performing arts centers and hipster theaters near you. But don’t worry, this isn’t your typical boring musical.

        +

        ‘It’s not a musical. Between you and I, I can’t abide musicals. It’s a production with tongue-in-cheek songs, with some extra stuff thrown in you won’t see in the film and deleted scenes that were left on the cutting room floor. I have always stated that if I ever see anyone yawning at one of our productions I will call it a day.’

        +

        ‘It’s incredible the interest we’ve had, people have been waiting for it to happen and we’ve been lucky enough to find ourselves with the rights.

        +

        ‘Yes we are a small company nobody has ever heard of, and some believe us to be amateur. We are often asked about our professional/amateur position and we merely answer with the same phrase every time: if you measure our professional character by the colour of money then it’s a sad day for theatre.’ (Via Chortle)

        +

        Now the bad news: Those 15 dates are only in Southwest England. So you can either start planning your trips across the pond now, or you can go sit on your front porch steps and pout. That is, unless you live in Gloucester or Worcester, in which case you can go and then brag about it. But then you’d be a jerk, so don’t do that (the bragging part, that is).

        +

        +

        *No relation, but if I had a British cousin, his name would totally be Chas.

        +

        Share This

        +
        +
        +
        +
        + +
        + +
        TAGSENGLANDMUSICALSSHAUN OF THE DEADsimon peggstage shows
        +
        + +
        + + + + + + + + + +
        +
        +
        + +
        + +
        + +
        + + + +
        +

        Join The Discussion

        +
          + + + +
        +
        + + + + + + + + + +
        +
        + + +
        +
        + [avatar]
        Profile +
        + +
        +
        + 6 comments
        +
        + Ashley Burns posted +
        +
        +

        "Simon Pegg Gave His Blessing For A Touring ‘Shaun Of The Dead’ Stage Show"

        +
        +
        + +
        +
        + +
        + +
          +
        1. +
          +
          + [avatar]
          Profile
          + +
          +
          + 12.29.14 at 8:09 pm
          +
          + Verbal Kunt
          + +
          + +

          …And people claim Hollywood is out of ideas…

          + + +
          + +
          +
          + +
          +
          2. +
        2. +
          +
          + [avatar]
          Profile
          + +
          +
          + 12.29.14 at 8:51 pm
          +
          + NobodyLikesMilhouse
          + +
          + +

          You’ve got red on you.

          + + +
          + +
          +
          + +
          +
          3. +
        3. +
          +
          + [avatar]
          Profile
          + +
          +
          + 12.29.14 at 9:57 pm
          +
          + Darkest Timeline Zach Morris
          + +
          + +

          Is he related to Charles in Charge’s alter ego?
          (I was going to put Charles’ full name here, but IT DOES NOT EXIST.)

          -]]> - - - By: NobodyLikesMilhouse - http://uproxx.com/filmdrunk/2014/12/simon-pegg-gave-his-blessing-for-a-touring-shaun-of-the-dead-stage-show/comment-page-1/#comment-2517037 - - Tue, 30 Dec 2014 01:51:29 +0000 - http://uproxx.com/?p=758539#comment-2517037 - - You’ve got red on you.

          -]]>           -           - - By: Verbal Kunt - http://uproxx.com/filmdrunk/2014/12/simon-pegg-gave-his-blessing-for-a-touring-shaun-of-the-dead-stage-show/comment-page-1/#comment-2516563 - - Tue, 30 Dec 2014 01:09:46 +0000 - http://uproxx.com/?p=758539#comment-2516563 - - …And people claim Hollywood is out of ideas…

          -]]>           -           - - + + +
          + +
          +
          + +
          +
          4. +
        4. +
          +
          + [avatar]
          Profile
          + +
          +
          + 12.30.14 at 8:59 am
          +
          + Ace Rimmer
          + +
          + +

          Oh good, it’s not a musical, it’s a production with tongue-in-cheek songs. That won’t be obnoxious at all, then. Just … a musical with poor enunciation.

          + + +
          + +
          +
          + +
          +
          5. +
        5. +
          +
          + [avatar]
          Profile
          + +
          +
          + 12.30.14 at 11:15 am
          +
          + AreWeFriendsJack?
          + +
          + +

          Hopefully it’s as good as their take on Shakespeare’s Romeo + Juliet

          +

          + + +
          + +
          +
          + +
          +
            +
          • +
            +
            + [avatar]
            Profile
            + +
            +
            + 12.30.14 at 12:04 pm
            +
            + Phyllis Rivers
            + +
            + +

            I loved that part. Ha ha!

            +

            Take that Baz Luhrmann!

            + + +
            + +
            +
            + +
            +
            • +
          +
          6. + +
        + +
        +
        + + +
        + + + +
        +
        + [avatar] +
        + +

        + Join the discussion. or Register +

        + +
        + + +
        +
        +
        + + +
        + +
        + + + + + + +
        + +
        +
        + + +
        +
        + + + + + + + +
        +
        + + +
        + +
        + +
        + +
        +
        + + + + + +
        + +
        + +
        +
        +
        +
        +
        + +
        +
        + Do you like UPROXX? +
        + +
        +
        +
        + RSS Feed +
        + Jobs at UPROXX | Contact | Advertising | The UPROXX Team | Terms of Use | Copyright Notices | Privacy Policy +
        +
        +
        +
        +
        +
        + +Powered by WordPress.com VIP + +
        + + +
        + + +
        + + + +
        +
        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/data/html/upworthy.com1.html b/tests/data/html/upworthy.com1.html index ddcf2e3d..ddce1417 100644 --- a/tests/data/html/upworthy.com1.html +++ b/tests/data/html/upworthy.com1.html @@ -18,7 +18,7 @@ - + @@ -135,7 +135,7 @@ diff --git a/tests/data/html/upworthy.com2.html b/tests/data/html/upworthy.com2.html index 26f42321..9f3b1c12 100644 --- a/tests/data/html/upworthy.com2.html +++ b/tests/data/html/upworthy.com2.html @@ -18,7 +18,7 @@ - + @@ -135,7 +135,7 @@ diff --git a/tests/data/html/usnews.com1.html b/tests/data/html/usnews.com1.html index fb28c15d..6beb8a46 100644 --- a/tests/data/html/usnews.com1.html +++ b/tests/data/html/usnews.com1.html @@ -41,7 +41,7 @@ - + @@ -212,7 +212,8 @@ - + + @@ -1177,28 +1178,28 @@

        You Might Also Like

        - - + +

        - - Workers’ Weakened Bargaining Power Fuels Income Inequality + + Obama Increases Sanctions Against North Korea After Sony Hack

        By - - Katherine Peralta + + Tom Risen | -

        @@ -1221,21 +1222,29 @@

        - - + +

        - - Mario Cuomo, Former New York Governor, Dead at 82 + + 'Price Is Right' Contestants Thinks A New iPhone Costs $7,500

        + By + + + Lauren Boyer + + | + + -

        @@ -1257,29 +1266,29 @@

        - - + +

        - - Google: Your Computer Is Stupid + + Mario Cuomo a Rare Voice Against the Death Penalty in Tough on Crime Era

        By - - Tom Risen + + Tierney Sneed | -

        @@ -1326,29 +1335,21 @@

        - - + +

        - - 2015 Means First-Time Health Insurance Coverage for Millions of Americans + + U.S. Sanctions North Korea Over Sony Cyberattack

        - By - - - Kimberly Leonard - - | - - -

        @@ -1370,29 +1371,29 @@

        - - + +

        - - Common Core, College Ratings and Campus Sexual Assault: 2015 Education Trends + + South Sudan Plans 2015 Elections Despite Continued Violence

        By - - Allie Bidwell + + Teresa Welsh | -

        @@ -1414,29 +1415,21 @@

        - - + +

        - - Palestinians Turn to International Criminal Court After Failed U.N. Vote + + Newlywed Man, 95, Dies After Wife Is Taken Away

        - By - - - Teresa Welsh - - | - - -

        @@ -1458,21 +1451,21 @@

        - - + +

        - - Maryland Governor Commutes Sentences For Remaining Death Row Inmates + + Harry Reid Suffers Broken Ribs, Bones in Accident

        -

        @@ -1494,29 +1487,21 @@

        - - + +

        - - The War With ISIS: 5 Things to Look for in 2015 + + Georgia Police Chief Shoots His Wife Early New Year's Day

        - By - - - Paul D. Shinkman - - | - - -

        @@ -1538,29 +1523,21 @@

        - - + +

        - - What the Battle Over Abortion Will Look Like in 2015 + + Florida Man Decapitates His Mother on New Year's Eve

        - By - - - Tierney Sneed - - | - - -

        @@ -1582,25 +1559,29 @@

        + + + +

        - - Most Inspiring News Stories of 2014 + + Workers’ Weakened Bargaining Power Fuels Income Inequality

        By - - Casey Leins + + Katherine Peralta | -

        @@ -1622,29 +1603,21 @@

        - - + +

        - - Libertarians Cheer Nosedive in NYPD Arrests, Citations + + Mario Cuomo, Former New York Governor, Dead at 82

        - By - - - Steven Nelson - - | - - -

        @@ -1666,29 +1639,29 @@

        - - + +

        - - Senate Committee Sets Hearing Date for Keystone XL + + Google: Your Computer Is Stupid

        By - - Alan Neuhauser + + Tom Risen | -

        @@ -1710,28 +1683,28 @@

        - - + +

        - - Photos: New Year's Around the World + + 2015 Means First-Time Health Insurance Coverage for Millions of Americans

        By - - Diana Soliwon + + Kimberly Leonard | -

        @@ -1754,20 +1727,28 @@

        - - + +

        - - Thieves Steal $300K Worth of Wine From French Laundry Restaurant + + Common Core, College Ratings and Campus Sexual Assault: 2015 Education Trends

        + By + + + Allie Bidwell + + | + + -

        @@ -1790,28 +1771,28 @@

        - - + +

        - - Photos: The Year in Extreme Weather + + Palestinians Turn to International Criminal Court After Failed U.N. Vote

        By - - Diana Soliwon + + Teresa Welsh | -

        @@ -1834,28 +1815,20 @@

        - - + +

        - - The Best and Worst of the Year in Politics + + Maryland Governor Commutes Sentences For Remaining Death Row Inmates

        - By - - - David Catanese - - | - - -

        @@ -1878,29 +1851,29 @@

        - - + +

        - - 2014: The U.S. Economy's Breakout Year + + The War With ISIS: 5 Things to Look for in 2015

        By - - Katherine Peralta + + Paul D. Shinkman | -

        @@ -1922,29 +1895,29 @@

        - - + +

        - - U.N. Security Council to Vote on Palestinian Resolution + + What the Battle Over Abortion Will Look Like in 2015

        By - - Teresa Welsh + + Tierney Sneed | -

        @@ -1966,29 +1939,25 @@

        - - - -

        - - Russian University Announces 'Noah's Ark' DNA Storage Project + + Most Inspiring News Stories of 2014

        By - - Andrew Soergel + + Casey Leins | -

        @@ -2010,21 +1979,29 @@

        - - + +

        - - North Carolina Town Won't Drop Live Opossum on New Year's Eve + + Libertarians Cheer Nosedive in NYPD Arrests, Citations

        + By + + + Steven Nelson + + | + + -

        @@ -9245,5 +9222,6 @@

        - + + diff --git a/tests/data/html/usnews.com2.html b/tests/data/html/usnews.com2.html index d7cd1932..4928c4f3 100644 --- a/tests/data/html/usnews.com2.html +++ b/tests/data/html/usnews.com2.html @@ -345,6 +345,27 @@

        Cigar rollers in Havana thrilled with wa

        Top Trending Videos

        @@ -742,6 +742,21 @@

        Viral

        Top Trending Videos

        @@ -1016,6 +1016,6 @@

        Top Trending Videos - + diff --git a/tests/data/html/vanityfair.com1.html b/tests/data/html/vanityfair.com1.html index 2da2ada5..ae1c1307 100644 --- a/tests/data/html/vanityfair.com1.html +++ b/tests/data/html/vanityfair.com1.html @@ -1,1965 +1,1288 @@ - - - - - + + + + + + + + - - - - + Gwyneth Paltrow Explains Her Breakups with Brad Pitt and Ben Affleck | Vanity Fair + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -404 | Vanity Fair - - - -
        -
        -
        - -
        -
        - - -
        -
        - -
        -
        - -
        -
        -
        -
        -

        -

        -
        -
        -
        -
        -
        -
        - - -
        -
        -
        -
        -
        - - - -
        -
        -
        -

        - Recent Features

        - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - Spotlight: Glenn Allen Sims and Linda Celeste Sims at Alvin Ailey American Dance Theater -

        - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - Profile: Tony Blair Defends the Choices He’s Made -

        - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - The de Blasio Diaries, Chapter 7: Fenced In -

        - - - - - - -
        -
        -
        -
        -
        -
        -

        - Recent Photo Galleries

        - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - Photos: Natalia Vodianova Poses for Mario Testino -

        - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - Photos: Natalia Vodianova's Top 10 Best Looks -

        - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - Photos: Celebrity Street-Style-to-Red-Carpet Transformations -

        - - - - - - -
        -
        -
        -
        -
        -
        -

        - Most Popular

        - -
        -
        -
        -
        -
        -
        -
        - - -
        -
        -
        - - -
        -
        - -
        -
        -
        -
        -
         •   
        - - -
        -

        - The Latest on VF.com

        - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - Spotlight: Glenn Allen Sims and Linda Celeste Sims at Alvin Ailey American Dance Theater -

        - - - - - - -
        -
        - - + + + + + + - - -
        -
        + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -

        - Profile: Tony Blair Defends the Choices He’s Made -

        + + -
        -
        - + + + + + + + + + + +
        + +
        +
        + +
        +
        +
        +
        + + Sign in +
        +
        + +
        + + +
        + +
        + + + + +
        Toggle
        +
        -
        -
        -

        - The Latest from VF Daily

        + -
        -
        - + +
      • + +
        • +
      • VF Agenda
        • +
      • + +
        • + + +
        + + +
        + +
        -
        -
        - +
        +
        +
        +

        Gwyneth Paltrow Explains Her Breakups with Brad Pitt and Ben Affleck

        + +
        + +
        + + + +
        +
        + + + + +
        + + +
        + +
        +
        + +
        + + +
        By Kevin Mazur/WireImage.
        + +
        +
        +
        + +

        Say what you will about Gwyneth Paltrow, the Oscar winner proved to be a pretty great hang when she stopped by Howard Stern’s Sirius XM radio show on Wednesday. Unlike other actors, Paltrow was game to discuss anything that the host asked about, whether it was her high-profile “uncoupling” from Chris Martin, the sexual advice she gave Chelsea Handler, or her past relationships with A-list actors.

        +

        On the last subject, Paltrow opened up about Brad Pitt, whom she started dating after co-starring with him in David Fincher’s 1995 thriller Seven. “I definitely fell in love with him,” she said of her former fiancé. “He was so gorgeous and sweet. I mean, he was Brad Pitt.”

        -

        - January Hot Type: The Books You Need to Read This Month -

        +

        Paltrow was not the only person to fall in love with him at the time—she revealed that her parents were equally smitten with the star. “My father was so devastated when we broke up,” she said. “My father loved him like a son.” Although she refused to confirm that she dumped Pitt, she did talk about why the relationship didn’t end up working out.

        +

        “I was such a kid,” she shared. “I was 22 when we met. It's taken me until 40 to get my head out of my ass. You can't make that decision when you're 22 years old . . . I wasn't ready, and he was too good for me . . . I didn't know what I was doing.” Although she doesn’t still “hang out with” Pitt, Paltrow said that they are “friendly” if they run into each other.

        +

        The Goop founder offered slightly less glowing remarks about Ben Affleck, whom she dated and co-starred with in the 2000 drama Bounce.

        +

        “He was not in a good place in his life to have a girlfriend,” the actress said of their time together, before suggesting that both her friends and her parents were supportive of her decision to break up with him. When Stern asked about a comment Paltrow previously made about Affleck—specifically, “He’d be better off dating a Scores stripper”—Paltrow laughed, “Well that was true at the time.”

        +

        Paltrow said that she approves of Affleck’s wife, Jennifer Garner, though, and has spent some time with her. “Not only is she beautiful [but] she is so warm and she cooks and is so into her kids. And she’s hilarious . . . I had a girl’s dinner before the summer and she came over. She’s my neighbor. I really like her a lot.”

        +

        Paltrow also opened up about her friends Jay Z and Beyoncé, saying, “They’re the greatest people. They are funny and sweet and they’re self-aware and they’re really intelligent.” When Stern asked whether Paltrow called them after the Solange elevator incident to check in, Paltrow revealed, “If any of my friends ever have anything like that, I call right away. If someone’s clearly going through something, I always go straight to it.”

        -
        -
        - +

        And on the subject of Johnny Depp, with whom she co-stars in the upcoming Mordecai, she said, “I just felt like I am not cool enough [for him]. He’s got all of his tattoos and his cool clothes . . You go into the room to rehearse and he is rolling his own cigarettes . . . He listens to music while he acts. He has a little [earbud] in his ear and he listens to music. It’s so cool. He’s creating the whole world in his head.”

        +

        To listen to the complete interview, click here.

        +

        Related: The Paltrow Affair

        +
        +

        +
        + +
        +
        + +
        /
        +
        +
        gwyneth-paltrow-vanity-fair-640-ss03.jpg
        +
        +
        +
        +
        + +
        + + +
        +
        +
        open
        +
        +
        + + + +
        + +
        + +
        +
        Filed under:
        + + +
        gwyneth paltrow
        + + + +
        brad pitt
        + + + +
        ben affleck
        + + +
        + + +
        +
        + +
        + Back to Main + + + + + + + +
        + +
        +
        +

        Latest Stories

        +
        + +
        +
        + +
        -

        - 10 -

        +

        Final Oscar Nominations Predictions

        +
        + +
        +
        + +
        +

        Front Runners: Krista Smith and Dave Karger Explain the Best Supporting Actress Race

        +
        + +
        +
        + +
        +

        How Shia LaBeouf Wound Up in a Cage Fight with a 12-Year-Old

        -
        -
        - Go to VF Daily -
        + + -
        -
        -

        + + + -

        -

        -
        -
        -
        - -
        - -
        -
        -
        -
        -

        - Featured on VF.com -

        - -
        -
        -
        + +
        + +
        -

        -
        Explore: The New Establishment Summit
         -

        -
        - -
        -
        -
        + + -
        -
        -
        -
        -
        + -
        -

        -
        The 2014 Vanity Fair Hollywood Portfolio
         -

        +
        -
        -
        -
        +
        • +
      • Instagram
        • +
      • Tumblr
        • +
      • Pinterest
        • +
        -

        -
        Audrey Hepburn, Frank Sinatra, Elizabeth Taylor, and More Stars of the Jet-Set Era
         -

        + -
        -
        -
        -
        -
        -
        +
        +
        +
        +
        Vanity Fair Worldwide:
        + +
        VF On:
        + +
        +
        +
        +
        Visit our sister sites
        + +
        +
        -
        - -
        + +
        +
        -
        - -

        -
        -
        -
        -

        Registration on or use of this site constitutes acceptance of our User Agreement (Effectve January 2, 2014) and Privacy Policy (Effectve January 2, 2014).
        -Vanity Fair © Condé Nast Digital. Your California Privacy Rights. The material on this site may not be reproduced, distributed, transmitted, cached or otherwise used,
        -except with the prior written permission of Condé Nast Digital.
        -Ad Choices

        + -
        -
        -
        -
         
        - + -
        -
        -

        - - +// VF.com +ga('create', 'UA-8293713-15', 'auto'); +ga('send', 'pageview'); -

        -
        +// VF Summit App +ga('create', 'UA-8293713-23', 'auto', {name: 'vfsummit'}); +ga('vfsummit.send', 'pageview'); -
        - -
        - - - if (div !== null) { - div.innerHTML = ""; + - if (div !== null) { - div.innerHTML = ""; + + - jQuery("#AMS_VYF_GLOBAL_NAVBAR").html(pageAds.AMS_VYF_GLOBAL_NAVBAR.replace(/document.write\(.*\)/gi,"/*filtered out because of document.write(...)*/")); - } - } - if ('AMS_VYF_GLOBAL_NAVBAR' === 'AMS_VYF_MAGAZINE_SUBNAV') { - if (jQuery('#AMS_VYF_GLOBAL_NAVBAR').length > 0) { - jQuery('#mag_subnav_subscribe').parent().html(pageAds.AMS_VYF_GLOBAL_NAVBAR); - jQuery('#AMS_VYF_GLOBAL_NAVBAR').hide(); - } - } -})(window.pageAds); -(function(pageAds){ - var - div = null; - if ((typeof pageAds !== 'undefined') && (typeof pageAds.AMS_VYF_GLOBAL_ZRAIL !== 'undefined')) { - div = document.getElementById("AMS_VYF_GLOBAL_ZRAIL"); + + + - if (div !== null) { - div.innerHTML = ""; - jQuery("#AMS_VYF_GLOBAL_ZRAIL").html(pageAds.AMS_VYF_GLOBAL_ZRAIL.replace(/document.write\(.*\)/gi,"/*filtered out because of document.write(...)*/")); - } - } - if ('AMS_VYF_GLOBAL_ZRAIL' === 'AMS_VYF_MAGAZINE_SUBNAV') { - if (jQuery('#AMS_VYF_GLOBAL_ZRAIL').length > 0) { - jQuery('#mag_subnav_subscribe').parent().html(pageAds.AMS_VYF_GLOBAL_ZRAIL); - jQuery('#AMS_VYF_GLOBAL_ZRAIL').hide(); - } - } -})(window.pageAds); -(function(pageAds){ - var - div = null; - if ((typeof pageAds !== 'undefined') && (typeof pageAds.AMS_VYF_SITE_FOOTER_SUBSCRIBE !== 'undefined')) { - div = document.getElementById("AMS_VYF_SITE_FOOTER_SUBSCRIBE"); - if (div !== null) { - div.innerHTML = ""; +
        + - - - + - -
        -
        - - - + - -CQ_Analytics.ClickstreamcloudMgr.addListener("storesloaded", function(e) { - // set page name - s.pageName="error:404"; - // collect component events - CQ_Analytics.Sitecatalyst.collect(); - // base mappings - CQ_Analytics.Sitecatalyst.getBaseMappings(); - // set basic variables - - // update - CQ_Analytics.Sitecatalyst.updateEvars(); - /************* DO NOT ALTER ANYTHING BELOW THIS LINE ! **************/ - var s_code=s.t();if(s_code)document.write(s_code) -}); +
        -//--> - - - -
        -
        - - + diff --git a/tests/data/html/vanityfair.com2.html b/tests/data/html/vanityfair.com2.html deleted file mode 100644 index fefa8456..00000000 --- a/tests/data/html/vanityfair.com2.html +++ /dev/null @@ -1,1965 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -404 | Vanity Fair - - - -
        -
        -
        - -
        -
        - - -
        -
        - -
        -
        - -
        -
        -
        -
        -

        -

        -
        -
        -
        -
        -
        -
        - - -
        -
        -
        -
        -
        - - - -
        -
        -
        -

        - Recent Features

        - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - Spotlight: Glenn Allen Sims and Linda Celeste Sims at Alvin Ailey American Dance Theater -

        - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - Profile: Tony Blair Defends the Choices He’s Made -

        - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - The de Blasio Diaries, Chapter 7: Fenced In -

        - - - - - - -
        -
        -
        -
        -
        -
        -

        - Recent Photo Galleries

        - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - Photos: Natalia Vodianova Poses for Mario Testino -

        - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - Photos: Natalia Vodianova's Top 10 Best Looks -

        - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - Photos: Celebrity Street-Style-to-Red-Carpet Transformations -

        - - - - - - -
        -
        -
        -
        -
        -
        -

        - Most Popular

        - -
        -
        -
        -
        -
        -
        -
        - - -
        -
        -
        - - -
        -
        - -
        -
        -
        -
        -
         •   
        - - -
        -

        - The Latest on VF.com

        - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - Spotlight: Glenn Allen Sims and Linda Celeste Sims at Alvin Ailey American Dance Theater -

        - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - Profile: Tony Blair Defends the Choices He’s Made -

        - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        - The de Blasio Diaries, Chapter 7: Fenced In -

        - - - - - - -
        -
        -
        -

        - The Latest from VF Daily

        - -
        -
        - - - - - - - - - - - - - - - - - - -

        - Remember Mario Cuomo with His Landmark Speech on Equality -

        - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - -

        - January Hot Type: The Books You Need to Read This Month -

        - - - - - - -
        -
        - - - - - - - - - - - - - - - - - - -

        - 10 -

        - - - - - - -
        -
        -
        - Go to VF Daily -
        - - -
        -
        -
        -
        -
        -
        - - -
        -
        -
        -
        - - -
        -
        -
        -
        -
        - -
        - -
        -
        -
        -
        -

        - Featured on VF.com -

        - -
        -
        -
        - - - -
        -

        -
        Explore: The New Establishment Summit
         -

        - -
        -
        -
        -
        -
        - - - -
        -

        -
        Madame Claude and Her High-Profile Client List
         -

        - -
        -
        -
        -
        -
        - - - -
        -

        -
        The International Best-Dressed List 2014
         -

        - -
        -
        -
        -
        -
        - - - -
        -

        -
        Photos: Royal Siblings from Around the World
         -

        - -
        -
        -
        -
        -
        - - - -
        -

        -
        The 2014 Vanity Fair Hollywood Portfolio
         -

        - -
        -
        -
        -
        -
        - - - -
        -

        -
        Audrey Hepburn, Frank Sinatra, Elizabeth Taylor, and More Stars of the Jet-Set Era
         -

        - -
        -
        -
        -
        -
        -
        -
        - - -
        -
        -
        -
        -
        -
        -
        -
        -
        - -
        -
        - -
        - -
        - - - - - - - - - -
        -
        - - - - - - -
        -
        - - - - - - - - - -
        -
        - - - diff --git a/tests/data/html/vogue.com1.html b/tests/data/html/vogue.com1.html deleted file mode 100644 index 74508b7f..00000000 --- a/tests/data/html/vogue.com1.html +++ /dev/null @@ -1,1099 +0,0 @@ - - - - - - - - Celebrate Your Year in Style - Vogue - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        - - - - - - -
        - -
        -
        - -
        - Fashion - Beauty - Culture - Fashion Shows -
        - -
        - -
        - - -
        - -
        -
        -
        - - - -
        - + get a free gift - - -
        - - Subscribe - -
        - Give A Gift - • - Renew - • - International -
        -
        - -
        -
        -
        - - -
        - - -
        - -
        - -

        Most Shared

        - - -
        - - - - - 1 -

        - 31 Gobsmacking Gorgeous Gifts

        - - -         - -
        - - -
        - - - - - 2 -

        - Supermodel Selfies in Spandex!

        - - -         - -
        - - -
        - - - - - 3 -

        - From Face-Morphing to Speed-Reading, the 10 Best Apps of 2014

        - - -         - -
        - - -
        - -
        -
        - - -
        -
        -
        -
        -
        - -
        - -
        - - - - - -
        - - - -
        -
        -
        -
        -
        - - Promotions - -
        - -

        Celebrate Your Year in Style

        - -
        -
        -
        - -
        -
        -

        Find the perfect winter shade in essie’s Winter 2014 Collection.

        - -
        - - -
        - -
        - - - - - - -
        - -
        -
        -
        -
        1 / 6
        -
        - -
        -
        - BACK IN THE LIMO -

        Take a spin in this fresh take on a sheer bright ripe melon–and roll up in style. - -

        SHOP NOW

        - -
        - -
        -
        -
        -
        - - - - - - -
        - -
        - - - - - - -
        - -
        -
        -
        -
        2 / 6
        -
        - -
        -
        - JIGGLE HI, JIGGLE LOW -

        Own the night and take center stage in shimmering style with this bold molten gold. -

        SHOP NOW

        - -
        - -
        -
        -
        -
        - - - - - - -
        - -
        - - - - - - -
        - -
        -
        -
        -
        3 / 6
        -
        - -
        -
        - BUMP UP THE PUMPS -

        Step out in this raucous coral mauve that will keep you warm all season. -

        SHOP NOW

        - -
        - -
        -
        -
        -
        - - - -
        -
        - -
        -
        -
        - -
        - - -
        -
        -
        -
        - -
        -
        -
        -
        -
        -
        -
        - - - - - -
        -
        -
        - -
        -
        - -
        - - -
        - -
        - - - - - - -
        - -
        -
        -
        -
        4 / 6
        -
        - -
        -
        - DOUBLE BREASTED JACKET -

        An unstoppable passion for fashion gives this tourmaline ruby a luxurious advantage. -

        SHOP NOW

        - -
        - -
        -
        -
        -
        - - - - - - -
        - -
        - - - - - - -
        - -
        -
        -
        -
        5 / 6
        -
        - -
        -
        - JUMP IN MY JUMPSUIT -

        Do the happy dance with this jubilant juicy red that's very of-the-moment. -

        SHOP NOW

        - -
        - -
        -
        -
        -
        - - - - - - -
        - -
        - - - - - - -
        - -
        -
        -
        -
        6 / 6
        -
        - -
        -
        - TUCK IT IN MY TUX -

        Tailored to perfection, this silken ivory créme is devastatingly elegant and in total control. -

        SHOP NOW

        - -
        - -
        -
        -
        -
        - - - - -
        - -
        - -
        - -
        - -
        - -
        - -
        - -
        - -
        - -
        - - - - -
        - - - - - - -
        - - -
        - - -
        - - -
        -
        - - - -
        - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tests/data/html/vogue.com2.html b/tests/data/html/vogue.com2.html deleted file mode 100644 index ce414fd0..00000000 --- a/tests/data/html/vogue.com2.html +++ /dev/null @@ -1,494 +0,0 @@ - - - - - - - - Page Not Found - Vogue - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        - - - - - - -
        - -
        -
        - -
        - Fashion - Beauty - Culture - Fashion Shows -
        - -
        - -
        - - -
        - -
        -
        -
        - - - -
        - + get a free gift - - -
        - - Subscribe - -
        - Give A Gift - • - Renew - • - International -
        -
        - -
        -
        -
        - - - -
        -
        - -
        - -
        -

        oops

        -
        - -
        -
        The page you're looking for cannot be found
        - Go to homepage -
        -
        - -
        -
        - -
        - - -
        - - -
        -
        - - - -
        - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/tests/data/html/vogue.de1.html b/tests/data/html/vogue.de1.html index ffe56e15..06dfeefe 100644 --- a/tests/data/html/vogue.de1.html +++ b/tests/data/html/vogue.de1.html @@ -22,23 +22,23 @@ - - - - - - - - - - + + + + + + + + + + - + @@ -357,7 +357,7 @@ } } - +

        @@ -429,29 +429,29 @@
        - +
        - beauty-vorschau-2015-hoch
        Beauty: Vorausschauend
        -

        30. Dezember 2014, 17:10 Die Beauty-Highlights des neuen Jahres -

        + jennifer-connelly-and-freja-beha-erichsen-for-louis-vuitton__2_
        Mode: Freja Beha Erichsen und Jennifer Connelly für Louis Vuitton
        +

        02. Januar 2015, 15:53 Im Mittelpunkt der neuen Kampagne von Louis Vuitton für Frühjahr/Sommer 2015 […]

         - +
        - taylor_swift_2014_getty_images_460395066_jamie_mccarthy__2_
        People & Kultur: Taylor Swift ist wohltätigster Star des Jahres 2014
        -

        30. Dezember 2014, 16:01 Taylor Swift wurde zum dritten Mal in Folge mit Platz eins der Liste "Celebs Gone Good" geehrt

        + mary_quant_2009_getty_images_93010806_mike_prior__2_
        Mode: Mary Quant auf der New Year's Honours List 2015
        +

        02. Januar 2015, 12:46 Mary Quant erhält den Verdienstorden Dame Commander of the Order of the British Empire

         - +
        - instagram_rosiehw__2_
        People & Kultur: Rosie Huntington-Whiteley trägt jetzt Bob
        -

        30. Dezember 2014, 15:55 Pünktlich zum Start ins neue Jahr hat Rosie Huntington-Whiteley ihre Frisur mit einem neuen, frischen Look versehen

        + gisele-bundchen-chanel-2015
        Mode: Gisele Bündchen für Chanel "Afterdark"
        +

        02. Januar 2015, 11:39 Karl Lagerfeld fotografierte die Frühjahr/Sommer-Kampagne 2015 für Chanel

         - +
        - instagram_victoriassecret__2_
        People & Kultur: Behati Prinsloo auf dem Cover des "Victoria's Secret Swim 2015"-Katalogs
        -

        30. Dezember 2014, 15:43 Behati Prinsloo ziert das Cover des aktuellen "Victoria's Secret Swim 2015"-Katalogs

        + beauty-vorschau-2015-hoch
        Beauty: Vorausschauend
        +

        30. Dezember 2014, 17:10 Die Beauty-Highlights des neuen Jahres +

         Alle News
        @@ -1010,7 +1010,7 @@

        Abo

        document.write(''); //--> - +
        diff --git a/tests/data/html/vogue.de2.html b/tests/data/html/vogue.de2.html index 77f10688..bbba0f44 100644 --- a/tests/data/html/vogue.de2.html +++ b/tests/data/html/vogue.de2.html @@ -22,36 +22,36 @@ - + - + - - - - - - - - - - - + + + + + + + + + + + - + - + - + - + - + @@ -387,7 +387,7 @@ } } - +

        @@ -459,29 +459,29 @@
        - +
        - beauty-vorschau-2015-hoch
        Beauty: Vorausschauend
        -

        30. Dezember 2014, 17:10 Die Beauty-Highlights des neuen Jahres -

        + jennifer-connelly-and-freja-beha-erichsen-for-louis-vuitton__2_
        Mode: Freja Beha Erichsen und Jennifer Connelly für Louis Vuitton
        +

        02. Januar 2015, 15:53 Im Mittelpunkt der neuen Kampagne von Louis Vuitton für Frühjahr/Sommer 2015 […]

         - +
        - taylor_swift_2014_getty_images_460395066_jamie_mccarthy__2_
        People & Kultur: Taylor Swift ist wohltätigster Star des Jahres 2014
        -

        30. Dezember 2014, 16:01 Taylor Swift wurde zum dritten Mal in Folge mit Platz eins der Liste "Celebs Gone Good" geehrt

        + mary_quant_2009_getty_images_93010806_mike_prior__2_
        Mode: Mary Quant auf der New Year's Honours List 2015
        +

        02. Januar 2015, 12:46 Mary Quant erhält den Verdienstorden Dame Commander of the Order of the British Empire

         - +
        - instagram_rosiehw__2_
        People & Kultur: Rosie Huntington-Whiteley trägt jetzt Bob
        -

        30. Dezember 2014, 15:55 Pünktlich zum Start ins neue Jahr hat Rosie Huntington-Whiteley ihre Frisur mit einem neuen, frischen Look versehen

        + gisele-bundchen-chanel-2015
        Mode: Gisele Bündchen für Chanel "Afterdark"
        +

        02. Januar 2015, 11:39 Karl Lagerfeld fotografierte die Frühjahr/Sommer-Kampagne 2015 für Chanel

         - +
        - instagram_victoriassecret__2_
        People & Kultur: Behati Prinsloo auf dem Cover des "Victoria's Secret Swim 2015"-Katalogs
        -

        30. Dezember 2014, 15:43 Behati Prinsloo ziert das Cover des aktuellen "Victoria's Secret Swim 2015"-Katalogs

        + beauty-vorschau-2015-hoch
        Beauty: Vorausschauend
        +

        30. Dezember 2014, 17:10 Die Beauty-Highlights des neuen Jahres +

         Alle News
        @@ -1582,7 +1582,7 @@

        Abo

        document.write(''); //--> - +
        diff --git a/tests/data/html/wetpaint.com1.html b/tests/data/html/wetpaint.com1.html index 3edf7d79..635d46ce 100644 --- a/tests/data/html/wetpaint.com1.html +++ b/tests/data/html/wetpaint.com1.html @@ -1,7 +1,7 @@ - + @@ -202,8 +202,8 @@ dogfood: false, client_ip: '127.0.0.1', - server_time_stamp: 1420195689745, - server_name: 'c3', + server_time_stamp: 1420282346872, + server_name: 'c6', ads: { placement: 'article', @@ -335,8 +335,8 @@

        - - Troian Bellisario + + Desiree Hartsock
        @@ -693,8 +693,40 @@

        +
        +
        + More +

        Popular

        +
        + +
        +

        @@ -792,7 +824,7 @@
        Register now to become a Wetpaint VIP

        - +
        @@ -880,7 +912,7 @@

        Register now to become a Wetpaint VIP
        })(); - + diff --git a/tests/data/html/wetpaint.com2.html b/tests/data/html/wetpaint.com2.html index 49b408ef..5eb89853 100644 --- a/tests/data/html/wetpaint.com2.html +++ b/tests/data/html/wetpaint.com2.html @@ -1,7 +1,7 @@ - + @@ -202,8 +202,8 @@ dogfood: false, client_ip: '127.0.0.1', - server_time_stamp: 1420196197270, - server_name: 'c3', + server_time_stamp: 1420263182471, + server_name: 'c5', ads: { placement: 'article', @@ -335,8 +335,8 @@
        - - Rihanna + + Adam Levine
        @@ -689,8 +689,40 @@

        +
        +
        + More +

        Popular

        +
        + +
        +

        @@ -788,7 +820,7 @@
        Register now to become a Wetpaint VIP

        - +
        @@ -876,7 +908,7 @@

        Register now to become a Wetpaint VIP
        })(); - + diff --git a/tests/data/html/wired.com1.html b/tests/data/html/wired.com1.html index 83eca14d..113905c7 100644 --- a/tests/data/html/wired.com1.html +++ b/tests/data/html/wired.com1.html @@ -323,31 +323,31 @@

        • - + - Our Absolute Favorite Gadgets of 2014 + From VR Gear to Quantum-Dot TVs: What to Expect at CES 2015 -

          Our Absolute Favorite Gadgets of 2014

          +

          From VR Gear to Quantum-Dot TVs: What to Expect at CES 2015
        • - + - These Egg-Shaped Speakers Sound So Good, They'll Scramble Your Brains + Our Absolute Favorite Gadgets of 2014 -

          These Egg-Shaped Speakers Sound So Good, They'll Scramble Your Brains

          +

          Our Absolute Favorite Gadgets of 2014
        • - + - 15 Essential Apps to Install on Your New iPad + These Egg-Shaped Speakers Sound So Good, They'll Scramble Your Brains -

          15 Essential Apps to Install on Your New iPad

          +

          These Egg-Shaped Speakers Sound So Good, They'll Scramble Your Brains
          • @@ -402,31 +402,31 @@

          • - + - Science Graphic of the Week: Scientists Discover the First Protein That Can Edit Other Proteins + The Space Missions and Events We're Most Looking Forward to in 2015 -

            Science Graphic of the Week: Scientists Discover the First Protein That Can Edit Other Proteins

            +

            The Space Missions and Events We're Most Looking Forward to in 2015
          • - + - Our Most Popular Science Image Galleries of 2014 + Science Graphic of the Week: Scientists Discover the First Protein That Can Edit Other Proteins -

            Our Most Popular Science Image Galleries of 2014

            +

            Science Graphic of the Week: Scientists Discover the First Protein That Can Edit Other Proteins
          • - + - WIRED's Top Science Stories of the Year + Our Most Popular Science Image Galleries of 2014 -

            WIRED's Top Science Stories of the Year

            +

            Our Most Popular Science Image Galleries of 2014
            • @@ -505,7 +505,7 @@

            Out-of-Print Games We Were Thrilled to Play Again in 2014< The Best Games of 2014, From Mario Kart to Sunset Overdrive -

            The Best Games of 2014, From Mario Kart to Sunset Overdrive

            +

            The Best Games of 2014, From Mario Kart to Sunset Overdrive

            @@ -521,7 +521,7 @@

            This Wristband Tracks Your B-Ball Skills and Suggests Shooting Drills -

            This Wristband Tracks Your B-Ball Skills and Suggests Shooting Drills

            +

            This Wristband Tracks Your B-Ball Skills and Suggests Shooting Drills

            @@ -541,7 +541,7 @@

            How This Guy Is Training to Do 50 Ultradistance Triathlons How the Man Who Wired Facebook Helped Build the NFL Stadium of the Future -

            How the Man Who Wired Facebook Helped Build the NFL Stadium of the Future

            +

            How the Man Who Wired Facebook Helped Build the NFL Stadium of the Future

            @@ -553,31 +553,31 @@

            • - + - Our 9 Favorite Memes of 2014, From Kermit to ¯\_(ツ)_/¯ + Our 10 Favorite Comics and Graphic Novels of the Year -

              Our 9 Favorite Memes of 2014, From Kermit to ¯\_(ツ)_/¯

              +

              Our 10 Favorite Comics and Graphic Novels of the Year
            • - + - Cape Watch: Here's What We Want From Our Superheroes in 2015 + Our 9 Favorite Memes of 2014, From Kermit to ¯\_(ツ)_/¯ -

              Cape Watch: Here's What We Want From Our Superheroes in 2015

              +

              Our 9 Favorite Memes of 2014, From Kermit to ¯\_(ツ)_/¯
            • - + - 2014 Kinda Sucked: A Look at Our Slow Descent Into Dystopia + Cape Watch: Here's What We Want From Our Superheroes in 2015 -

              2014 Kinda Sucked: A Look at Our Slow Descent Into Dystopia

              +

              Cape Watch: Here's What We Want From Our Superheroes in 2015
              • @@ -650,31 +650,31 @@

              • - + - An Extortionist Has Been Making Life Hell for Bitcoin's Earliest Adopters + Tech Time Warp of the Week: Watch John Cleese Compare a Compaq to a Dead Fish -

                An Extortionist Has Been Making Life Hell for Bitcoin's Earliest Adopters

                +

                Tech Time Warp of the Week: Watch John Cleese Compare a Compaq to a Dead Fish
              • - + - WIRED's 10 Most Hardcore Tech Stories of the Year + An Extortionist Has Been Making Life Hell for Bitcoin's Earliest Adopters -

                WIRED's 10 Most Hardcore Tech Stories of the Year

                +

                An Extortionist Has Been Making Life Hell for Bitcoin's Earliest Adopters
              • - + - Tech Time Warp of the Week: In the '90s, Apple Celebrated Christmas by Bashing Microsoft + WIRED's 10 Most Hardcore Tech Stories of the Year -

                Tech Time Warp of the Week: In the '90s, Apple Celebrated Christmas by Bashing Microsoft

                +

                WIRED's 10 Most Hardcore Tech Stories of the Year
                • @@ -686,31 +686,31 @@

                • - + - Startups and Crowd-First Tactics: The Minimum Run, and the Long Stretch + How Quantified-Self Will Redefine the Future of the Enterprise -

                  Startups and Crowd-First Tactics: The Minimum Run, and the Long Stretch

                  +

                  How Quantified-Self Will Redefine the Future of the Enterprise
                • - + - The Future for Consumer Goods in the Data Economy + Startups and Crowd-First Tactics: The Minimum Run, and the Long Stretch -

                  The Future for Consumer Goods in the Data Economy

                  +

                  Startups and Crowd-First Tactics: The Minimum Run, and the Long Stretch
                • - + - Yes, Entrepreneurship Can Be Taught + The Future for Consumer Goods in the Data Economy -

                  Yes, Entrepreneurship Can Be Taught

                  +

                  The Future for Consumer Goods in the Data Economy
                  • @@ -765,31 +765,31 @@

                  • - + - The Most Dangerous People on the Internet Right Now + What We Know About the New U.S. Sanctions Against North Korea In Response to Sony Hack -

                    The Most Dangerous People on the Internet Right Now

                    +

                    What We Know About the New U.S. Sanctions Against North Korea In Response to Sony Hack
                  • - + - Over 80 Percent of Dark-Web Visits Relate to Pedophilia, Study Finds + The Most Dangerous People on the Internet Right Now -

                    Over 80 Percent of Dark-Web Visits Relate to Pedophilia, Study Finds

                    +

                    The Most Dangerous People on the Internet Right Now
                  • - + - The Year's Biggest Winners and Losers in Privacy and Security + Over 80 Percent of Dark-Web Visits Relate to Pedophilia, Study Finds -

                    The Year's Biggest Winners and Losers in Privacy and Security

                    +

                    Over 80 Percent of Dark-Web Visits Relate to Pedophilia, Study Finds
                    • @@ -911,7 +911,7 @@

                    The Rapidly Disappearing Business of Design

                    The Best WIRED Stories of 2014 -

                    The Best WIRED Stories of 2014

                    +

                    The Best WIRED Stories of 2014

                    @@ -2100,6 +2100,6 @@

                    A Mini Farm That Produces Food From Plastic-Eating Mushrooms

                    })(document, "script", "nativeads-plugin"); - + \ No newline at end of file diff --git a/tests/data/html/wired.com2.html b/tests/data/html/wired.com2.html index d5cccc42..71951147 100644 --- a/tests/data/html/wired.com2.html +++ b/tests/data/html/wired.com2.html @@ -323,31 +323,31 @@

                    • - + - Our Absolute Favorite Gadgets of 2014 + From VR Gear to Quantum-Dot TVs: What to Expect at CES 2015 -

                      Our Absolute Favorite Gadgets of 2014

                      +

                      From VR Gear to Quantum-Dot TVs: What to Expect at CES 2015
                    • - + - These Egg-Shaped Speakers Sound So Good, They'll Scramble Your Brains + Our Absolute Favorite Gadgets of 2014 -

                      These Egg-Shaped Speakers Sound So Good, They'll Scramble Your Brains

                      +

                      Our Absolute Favorite Gadgets of 2014
                    • - + - 15 Essential Apps to Install on Your New iPad + These Egg-Shaped Speakers Sound So Good, They'll Scramble Your Brains -

                      15 Essential Apps to Install on Your New iPad

                      +

                      These Egg-Shaped Speakers Sound So Good, They'll Scramble Your Brains
                      • @@ -402,31 +402,31 @@

                      • - + - Science Graphic of the Week: Scientists Discover the First Protein That Can Edit Other Proteins + The Space Missions and Events We're Most Looking Forward to in 2015 -

                        Science Graphic of the Week: Scientists Discover the First Protein That Can Edit Other Proteins

                        +

                        The Space Missions and Events We're Most Looking Forward to in 2015
                      • - + - Our Most Popular Science Image Galleries of 2014 + Science Graphic of the Week: Scientists Discover the First Protein That Can Edit Other Proteins -

                        Our Most Popular Science Image Galleries of 2014

                        +

                        Science Graphic of the Week: Scientists Discover the First Protein That Can Edit Other Proteins
                      • - + - WIRED's Top Science Stories of the Year + Our Most Popular Science Image Galleries of 2014 -

                        WIRED's Top Science Stories of the Year

                        +

                        Our Most Popular Science Image Galleries of 2014
                        • @@ -505,7 +505,7 @@

                        Out-of-Print Games We Were Thrilled to Play Again in 2014< The Best Games of 2014, From Mario Kart to Sunset Overdrive -

                        The Best Games of 2014, From Mario Kart to Sunset Overdrive

                        +

                        The Best Games of 2014, From Mario Kart to Sunset Overdrive

                        @@ -521,7 +521,7 @@

                        This Wristband Tracks Your B-Ball Skills and Suggests Shooting Drills -

                        This Wristband Tracks Your B-Ball Skills and Suggests Shooting Drills

                        +

                        This Wristband Tracks Your B-Ball Skills and Suggests Shooting Drills

                        @@ -541,7 +541,7 @@

                        How This Guy Is Training to Do 50 Ultradistance Triathlons How the Man Who Wired Facebook Helped Build the NFL Stadium of the Future -

                        How the Man Who Wired Facebook Helped Build the NFL Stadium of the Future

                        +

                        How the Man Who Wired Facebook Helped Build the NFL Stadium of the Future

                        @@ -553,31 +553,31 @@

                        • - + - Our 9 Favorite Memes of 2014, From Kermit to ¯\_(ツ)_/¯ + Our 10 Favorite Comics and Graphic Novels of the Year -

                          Our 9 Favorite Memes of 2014, From Kermit to ¯\_(ツ)_/¯

                          +

                          Our 10 Favorite Comics and Graphic Novels of the Year
                        • - + - Cape Watch: Here's What We Want From Our Superheroes in 2015 + Our 9 Favorite Memes of 2014, From Kermit to ¯\_(ツ)_/¯ -

                          Cape Watch: Here's What We Want From Our Superheroes in 2015

                          +

                          Our 9 Favorite Memes of 2014, From Kermit to ¯\_(ツ)_/¯
                        • - + - 2014 Kinda Sucked: A Look at Our Slow Descent Into Dystopia + Cape Watch: Here's What We Want From Our Superheroes in 2015 -

                          2014 Kinda Sucked: A Look at Our Slow Descent Into Dystopia

                          +

                          Cape Watch: Here's What We Want From Our Superheroes in 2015
                          • @@ -650,31 +650,31 @@

                          • - + - An Extortionist Has Been Making Life Hell for Bitcoin's Earliest Adopters + Tech Time Warp of the Week: Watch John Cleese Compare a Compaq to a Dead Fish -

                            An Extortionist Has Been Making Life Hell for Bitcoin's Earliest Adopters

                            +

                            Tech Time Warp of the Week: Watch John Cleese Compare a Compaq to a Dead Fish
                          • - + - WIRED's 10 Most Hardcore Tech Stories of the Year + An Extortionist Has Been Making Life Hell for Bitcoin's Earliest Adopters -

                            WIRED's 10 Most Hardcore Tech Stories of the Year

                            +

                            An Extortionist Has Been Making Life Hell for Bitcoin's Earliest Adopters
                          • - + - Tech Time Warp of the Week: In the '90s, Apple Celebrated Christmas by Bashing Microsoft + WIRED's 10 Most Hardcore Tech Stories of the Year -

                            Tech Time Warp of the Week: In the '90s, Apple Celebrated Christmas by Bashing Microsoft

                            +

                            WIRED's 10 Most Hardcore Tech Stories of the Year
                            • @@ -686,31 +686,31 @@

                            • - + - Startups and Crowd-First Tactics: The Minimum Run, and the Long Stretch + How Quantified-Self Will Redefine the Future of the Enterprise -

                              Startups and Crowd-First Tactics: The Minimum Run, and the Long Stretch

                              +

                              How Quantified-Self Will Redefine the Future of the Enterprise
                            • - + - The Future for Consumer Goods in the Data Economy + Startups and Crowd-First Tactics: The Minimum Run, and the Long Stretch -

                              The Future for Consumer Goods in the Data Economy

                              +

                              Startups and Crowd-First Tactics: The Minimum Run, and the Long Stretch
                            • - + - Yes, Entrepreneurship Can Be Taught + The Future for Consumer Goods in the Data Economy -

                              Yes, Entrepreneurship Can Be Taught

                              +

                              The Future for Consumer Goods in the Data Economy
                              • @@ -765,31 +765,31 @@

                              • - + - The Most Dangerous People on the Internet Right Now + What We Know About the New U.S. Sanctions Against North Korea In Response to Sony Hack -

                                The Most Dangerous People on the Internet Right Now

                                +

                                What We Know About the New U.S. Sanctions Against North Korea In Response to Sony Hack
                              • - + - Over 80 Percent of Dark-Web Visits Relate to Pedophilia, Study Finds + The Most Dangerous People on the Internet Right Now -

                                Over 80 Percent of Dark-Web Visits Relate to Pedophilia, Study Finds

                                +

                                The Most Dangerous People on the Internet Right Now
                              • - + - The Year's Biggest Winners and Losers in Privacy and Security + Over 80 Percent of Dark-Web Visits Relate to Pedophilia, Study Finds -

                                The Year's Biggest Winners and Losers in Privacy and Security

                                +

                                Over 80 Percent of Dark-Web Visits Relate to Pedophilia, Study Finds
                                • @@ -911,7 +911,7 @@

                                The Rapidly Disappearing Business of Design

                                The Best WIRED Stories of 2014 -

                                The Best WIRED Stories of 2014

                                +

                                The Best WIRED Stories of 2014

                                @@ -1986,6 +1986,6 @@

                                The 5 Most Dangerous Software Bugs of 2014

                                })(document, "script", "nativeads-plugin"); - + \ No newline at end of file diff --git a/tests/data/html/wnet.org1.html b/tests/data/html/wnet.org1.html index 19f5d570..edd1c519 100644 --- a/tests/data/html/wnet.org1.html +++ b/tests/data/html/wnet.org1.html @@ -188,15 +188,15 @@

                                WNET Audit Committee Meetings

                                @@ -260,8 +260,8 @@

                                WNET Audit Committee Meetings

                                \ No newline at end of file +Served from: www.wnet.org @ 2015-01-03 06:48:28 --> \ No newline at end of file diff --git a/tests/data/html/wnet.org2.html b/tests/data/html/wnet.org2.html index 1cf89f64..e8cd31c1 100644 --- a/tests/data/html/wnet.org2.html +++ b/tests/data/html/wnet.org2.html @@ -187,15 +187,15 @@

                                WNET Interactive & Technology Committee Meetings

                                @@ -263,4 +263,4 @@

                                WNET Interactive & Technology Committee Meetings

                                Object Caching 1189/1199 objects using apc Content Delivery Network via cn1.wnet.org -Served from: www.wnet.org @ 2015-01-02 06:00:51 --> \ No newline at end of file +Served from: www.wnet.org @ 2015-01-03 06:48:30 --> \ No newline at end of file diff --git a/tests/data/html/youbeauty.com1.html b/tests/data/html/youbeauty.com1.html index ab9668d8..8f8142e2 100644 --- a/tests/data/html/youbeauty.com1.html +++ b/tests/data/html/youbeauty.com1.html @@ -24,7 +24,7 @@ - + @@ -32,7 +32,7 @@ - + @@ -50,8 +50,8 @@ - - + + - - - - - - + + + + + + - + +NREUMQ.push(["nrfj","beacon-6.newrelic.com","16de03fa57","191351","cVtXERBeWQ1SSx5SUFNaVwAOQhoARU1YUlRXRxYWCl5C",0,440,new Date().getTime(),"","","","",""]); diff --git a/tests/data/html/youbeauty.com2.html b/tests/data/html/youbeauty.com2.html index 6842bf16..bddd8a60 100644 --- a/tests/data/html/youbeauty.com2.html +++ b/tests/data/html/youbeauty.com2.html @@ -24,7 +24,7 @@ - + @@ -32,7 +32,7 @@ - + @@ -50,8 +50,8 @@ - - + + - - - - - - + + + + + + - + - + +NREUMQ.push(["nrfj","beacon-6.newrelic.com","16de03fa57","191351","cVtXERBeWQ1SSx5SUFNaVwAOQhoARU1YUlRXRxYWCl5C",0,659,new Date().getTime(),"","","","",""]); diff --git a/tests/data/test_prepare_urls.txt b/tests/data/test_prepare_urls.txt new file mode 100644 index 00000000..be593bf5 --- /dev/null +++ b/tests/data/test_prepare_urls.txt @@ -0,0 +1,4 @@ +http://example.com/cat/123/ ./123/ http://example.com/cat/ +http://example.com/cat/123/ 123/ http://example.com/cat/ +http://example.com/cat/123/ http://example.com/cat/123/ http://example.com/cat/ +http://example.com/123/ /123/ http://example.com/cat/ \ No newline at end of file diff --git a/tests/data/test_urls.txt b/tests/data/test_urls.txt index 8807b349..b851ed46 100644 --- a/tests/data/test_urls.txt +++ b/tests/data/test_urls.txt @@ -37,3 +37,4 @@ 1 http://www.alarabiya.net/ar/arab-and-world/syria/2014/02/02/%D8%A7%D9%84%D9%86%D8%A7%D8%AA%D9%88-%D9%8A%D8%B9%D9%84%D9%86-%D8%A7%D8%B3%D8%AA%D8%B9%D8%AF%D8%A7%D8%AF%D9%87-%D9%84%D8%AA%D8%AF%D9%85%D9%8A%D8%B1-%D8%A7%D9%84%D8%A3%D8%B3%D9%84%D8%AD%D8%A9-%D8%A7%D9%84%D9%83%D9%8A%D9%85%D9%8A%D8%A7%D9%88%D9%8A%D8%A9-%D8%A7%D9%84%D8%B3%D9%88%D8%B1%D9%8A%D8%A9.html 1 http://www.almasryalyoum.com/news/details/387323 1 http://tahrirnews.com/news/view.aspx?cdate=02022014&id=990c7733-9293-4441-982e-a44af000bb19 +1 http://www.enfieldindependent.co.uk/news/14140506.Review_of_the_year__July/ diff --git a/tests/data/test_urls_pubdate.txt b/tests/data/test_urls_pubdate.txt new file mode 100644 index 00000000..29a6fea9 --- /dev/null +++ b/tests/data/test_urls_pubdate.txt @@ -0,0 +1,4 @@ +0 http://www.sltrib.com/sltrib/mobile3/58201224-219/laws-animal-farm-gag.html.csp +1 http://clatl.com/cribnotes/archives/2013/10/11/rrest-plays-album-release-party-tonight-drops-goodnight-chanteal +1 http://www.miamiherald.com/2013/10/11/3683673/2-new-wind-farms-planned-in-oklahoma.html +0 http://eatocracy.cnn.com/category/make/recipes/ \ No newline at end of file diff --git a/tests/data/text/247wallst.com1.txt b/tests/data/text/247wallst.com1.txt index e69de29b..3e55b2aa 100644 --- a/tests/data/text/247wallst.com1.txt +++ b/tests/data/text/247wallst.com1.txt @@ -0,0 +1,13 @@ +December 28, 2014: Markets opened lower on Monday as neither the bulls nor the bears can gin up a lot of support. Volumes are low and corporate news is nearly non-existent. Crude oil started the day higher, but has drifted to a loss of about 2% as the day wore on. Shortly before the closing bell the DJIA traded down 0.08% for the day, the S&P 500 traded up 0.09%, and the Nasdaq Composite traded flat. International Business Machines Corp. (NYSE: IBM) dropped more than 1% today and its high per share price pushed the DJIA lower just as the market closed. + +The DJIA stock posting the largest daily gain ahead of the close Monday was The Home Depot Inc. (NYSE: HD) which traded up 0.87% at $104.64. The stock’s 52-week range is $73.96 to $104.80, and the high was posted early Monday. Trading volume was less than half the daily average of around 5.8 million shares. Reports of solid sales at home improvement stores during the holidays have helped push this stock higher today. + +JPMorgan Chase & Co. (NYSE: JPM) traded up 0.77% at $63.03. The stock’s 52-week range is $52.97 to $63.34, and the high was set today. Trading volume was about 50% below the daily average of around 26 million shares. The big bank is one of the underwriters for the IPO of Shake Shack which was announced today. + +The Boeing Co. (NYSE: BA) traded higher by 0.71% at $132.57. The stock’s 52-week range is $116.32 to $144.57. Volume was less than half the daily average of around 4.4 million shares. The company held its first test flight on Sunday of a new Air Force Tanker. + +Cisco Systems Inc. (NASDAQ: CSCO) traded up 0.48% at $28.49. The stock’s 52-week range is $21.27 to $28.59. Trading volume about 60% below the daily average of around 29 million shares. The stock missed matching its 52-week high by just a couple of pennies and was touted as a Rocket Stock at thestreet.com today. + +Of the Dow 30 stocks 13 are set to close higher today and 17 are on track to close lower. + +ALSO READ: 5 Top Tech Stocks for 2015 With Potential Big Catalysts \ No newline at end of file diff --git a/tests/data/text/247wallst.com2.txt b/tests/data/text/247wallst.com2.txt index e69de29b..036c30d3 100644 --- a/tests/data/text/247wallst.com2.txt +++ b/tests/data/text/247wallst.com2.txt @@ -0,0 +1,11 @@ +December 28, 2014: Here are four stocks among the 66 equities making new 52-week lows today. Volumes continue to be low as we wait to ring in the new year. + +Transocean Partners LLC (NYSE: RIGP) posted a new 52-week — and all-time — low on Monday of $13.18. Based on Friday night’s closing price of $14.32 that’s a drop of nearly 8%. The stock’s 52-week high is $29.43. Volume is about a third higher than the daily average of around 325,000 shares. The offshore drilling firm had no specific news today, but crude prices are down about 2%. + +SandRidge Mississippian Trust II (NYSE: SDR) dropped about 5.3% on Monday to post a new 52-week — and all-time — low of $3.91 after closing at $4.13 on Wednesday. The stock’s 52-week high is $10.12. Share volume is nearly double the daily average of 400,000 shares traded. The royalty company had no specific news today but is feeling pressure from low crude prices. + +Vivus Inc. (NASDAQ: VVUS) dropped about 6% Monday to post a new 52-week low of $2.78. The stock’s 52-week high is $9.80. Volume totaled 25% higher than the daily average of around 1.9 million shares. The U.S. FDA approved a competing diabetes drug last week to compete with Vivus’s Qsymia treatment. + +Zulilly Inc. (NASDAQ: ZU) dropped about 3.8% Monday to establish a new 52-week — and all-time — low at $22.39 against a high of $73.50. Volume was 25% below the daily average of around 1.4 million shares. The online retailer had no specific news today. + +ALSO READ: States Where People Live Longest \ No newline at end of file diff --git a/tests/data/text/about.com2.txt b/tests/data/text/about.com2.txt index 68418aaa..337fb699 100644 --- a/tests/data/text/about.com2.txt +++ b/tests/data/text/about.com2.txt @@ -1,43 +1,79 @@ -Updated August 02, 2014. +Updated August 02, 2014. Written or reviewed by a board-certified physician. See About.com's Medical Review Board. +One of the most exciting advances in the treatment of lung cancer has come from an understanding of genetic changes in lung cancer cells. Whereas in the past we broke lung cancers down into perhaps 5 types, we now know that no two lung cancers are the same. If there were 30 people in a room with lung cancer, they would have 30 different and unique types of lung cancer. + If you’ve been recently diagnosed with lung cancer, especially lung adenocarcinoma, your oncologist may have talked to you about genetic testing (otherwise known as molecular profiling or biomarker testing) of your tumor. It's now recommended that all lung cancer patients with advanced or metastatic lung adenocarcinoma (a type of non-small cell lung cancer) have biomarker testing to look for EGFR mutations and ALK rearrangements. In addition, patients with other forms of non-small cell lung cancer (for example, adenosquamous carcinoma in non-smokers) should also be considered for testing. What does this mean? +What is Genetic Testing (Molecular Profiling) of a Lung Cancer Tumor? + Genetic testing involves tests that a pathologist performs in the lab using a sample of your cancer tissue - tests that look at the cancer from a molecular level. This tissue may come from a biopsy of your tumor, or from tissue removed during surgery for lung cancer. The reason behind this is that cancers have gene mutations that "drive" or control the growth of the cancer. Simplistically, if these mutations can be identified, then treatments can be used which "target" these mutations, hence stopping the growth of the cancer. It is these mutations that lead to the development of a cancer in the first place. Before going further it's helpful to address something that is confusing for many people. There are two primary types of gene mutations. +Hereditary Mutations vs Aquired Mutations in Cancer + One type of mutations are hereditary mutations (also called germline mutations,) meaning you inherit genes with mutations from one or more parents. Common examples of these mutations include hemophilia associated as well as mutations that may predispose someone to developing breast cancer (BRCA1 and BRCA2 mutations.) The type of mutations that scientists look for in people with lung cancer are instead called acquired mutations (also called somatic mutations.) These mutations are not present at birth (they do not run in families) but rather develop in the process of cells becoming cancerous. +What Exactly are Gene Mutations? + Gene mutations are changes to a particular gene in a chromosome. All genes are made up of variable sequences of 4 amino acids (called bases); adenine, tyrosine, cytosine, and guanine. When a gene is exposed to toxins in the environment, or when an accident occurs in cell division, a mutation (change) may occur. In some cases it may mean that one base is substituted for another, say adenine instead of guanine. In other cases bases may be inserted, or deleted, or genes may be rearranged in some way. +Significance of Gene Mutations + Why are oncologists interested in acquired gene mutations in a tumor? First, we should talk about the two types of acquired mutations found in lung cancers. One type of mutation is termed a driver mutation. These mutations, via several mechanisms, “drive” the growth of a tumor. In lung cancer the number of driver mutations is variable. In one study, an average of 11 driver mutations per cancer were found. Another type of mutation is termed a passenger mutation. Just as someone may be a passenger in a car, these genes do not drive the cancer and are basically along for the ride. Again we don’t know exactly how many passenger mutations are present in a tumor (and the number varies from tumor to tumor) but some tumors may have more than 1,000 of these mutations. Driver mutations not only initiate the development of a cancer, but work to maintain the growth of a cancer as well. +Common Driver Mutations Found in Lung Cancer Tumors + There are many mutations that are being studied by scientists looking at lung tumors. So far driver mutations have been identified in approximately 60% of lung adenocarcinomas. Researchers are now finding driver mutations in squamous cell lung cancer as well. +Common driver mutations in lung cancer include (for starters): + +EGFR mutations + +KRAS mutations + +EML4-ALK Rearrangements + +MET mutation + These 4 mutations are in general mutually exclusive and are only rarely seen in the same tumor. +Personalized Treatments Available Based on Genetic Testing + The use of "targeted therapies" - that is medications that target particular genetic abnormalities in a tumor -- has been coined personalized medicine. What this means is that rather than a conventional chemotherapy drug that attacks all rapidly dividing cells, a targeted drug instead attacks a particular abnormality present only in your cancer cells. In general targeted treatments have fewer side effects than traditional chemotherapy. To date, targeted therapies that have been approved for people with lung cancer include: +Tarceva (erlotonib) has been approved for people whose tumor has an EGFR mutation (note: there are different types of EGFR mutations and not all of them are equally responsive to Tarceva) + +Tarceva (erlotonib) for people whose tumor has an EGFR mutation (note: there are different types of EGFR mutations and not all of them are equally responsive to Tarceva) Xalkori (crizotonib) was approved by the FDA in 2011 for people whose tumor has an ALK4-EML gene rearrangement + Other medications are being studied in clinical trials, including targeted therapies for those whose tumor becomes resistant to Tarceva or Xalkori.. +Resistance to Treatment + A challenging problem with current targeted treatments is that nearly everyone inevitably becomes resistant to treatments we have. There are many mechanisms by which this occurs making it difficult to find one solution. Research is ongoing in clinical trials; evaluating both the use of substituting a second drug to target the mutations, and drugs that use different targets or mechanisms to attack the cancer cell. +The Future + The ability to understand the molecular profile of lung tumors is an extremely exciting area of research, and it’s likely that new treatments for other mutations will soon be available. An example of how rapidly this area of medicine is advancing is the ALK4-EML gene rearrangment. This gene "mutation" (actually a rearranglement) was discovered as recently as 2007. Through a rapid process, the medication Xalkori (crizotinib) was approved in 2011 for general use by the FDA for those patients whose tumors have this rearrangment. There are clinical trials currently in progress evaluating the use of second generation drugs for those who have become resistant to Xalkori. +Next Step + If you have been diagnosed with non-small cell lung cancer, especially lung adenocarcinoma or squamous cell lung cancer, talk to your doctor about genetic testing. Although testing is now recommended for everyone with advanced non-small cell lung cancer, a recent study reported that only 60% of oncologists are currently ordering testing. You may also wish to talk to your doctor about clinical trials that may be an option for you. If you are interested in looking into trials evaluating these treatments worldwide, check out the article below on how to find clinical trials. It can be confusing as you check out these databases, but help is near. Recently a lung cancer clinical trial matching service backed by several lung cancer organizations has become available. With this free service a trained nurse navigator can help you locate any clinical trials that may be an option for you. +Sources: + Hensing, T., Chawla, A., Batra, R., and R. Salgia. A personalized treatment for lung cancer: molecular pathways, targeted therapies, and genomic characterization. Advances in Experimental Medicine and Biology. 2014. 799:85-117. Kim, H., Mitsudomi, T., Soo, R., and B. Cho. Personalized therapy on the horizon for squamous cell carcinoma of the lung. Lung Cancer. 2013. 80(3):249-55. Li, T., Kung, H., Mack, P., and D. Gandara. Genotyping and genomic profiling of non-small-cell lung cancer: implications for current and future therapies. Journal of Clinical Oncology. 2013. 31(8):1039-49. -Villaruz, L., Burns, T., Ramfidis, V., and M. Socinski. Personalizing therapy in advanced non-small cell lung cancer. Seminars in Respiratory and Critical Care Medicine. 2013. 34(6):822-36. \ No newline at end of file +Luo, S., and D. Lam. Oncogenic driver mutations in lung cancer. Translational Respiratory Medicine. 2013. 1:6. \ No newline at end of file diff --git a/tests/data/text/adoption.com1.txt b/tests/data/text/adoption.com1.txt index 0a9a9f7b..f4c1a5a8 100644 --- a/tests/data/text/adoption.com1.txt +++ b/tests/data/text/adoption.com1.txt @@ -1,7 +1,11 @@ This is solely my opinion. Every person is different, so the things that I have experienced in no way reflect EVERY birth mother or adoptee. +What Not to Ask or Say to a Birth Mom + First, the biggest mistake I often hear in adoption talk is “giving a baby up for adoption.” I know this a common mistake. I’ve said it too, and I’m a birth mom AND an adoptee! This phrase is such a stab in the heart to most birth moms because it presumes we just gave our baby away because we did not want him or her. Giving something away typically means you did not want whatever it was. In the adoption world, we use the term “place.” I placed my birth son for adoption. This phrase is much more loving and just sounds better. We placed our baby in the arms of their parents. We placed them in a good home. Placing something somewhere is usually done with care and caution because you have love and concern for what you are placing. In the adoption world, placing our children is done with love and concern. +“Birth moms are just lazy.” + I know someone who used to have this notion about birth moms. He said he thought birth mother simply did not want the responsibility of taking care of a child. Being a birth mother is much more difficult than one might think. Choosing to be a birth mom is choosing an emotionally difficult path, a path far from lazy. I hope not many people think this, but if you do, do not say it to a birth mom. Go talk to one so she can change your mind! “So, did you not want your baby?” @@ -18,12 +22,18 @@ This question is hurtful, but it allows a birth mother to share her testimony of I hate to think that people really do not understand what adoption is. Adoption is much different from foster care. In the foster care system, most children were taken out of the home due to the parents not doing what was best for their child. Yes, they can get their children back after they get things in order and clean up their life. However, adoption is different. An expectant parent chooses a family for their child and when that child is born, the birth mother then signs all parental rights away to the new parents. It varies by state, but usually the birth mother only has a limited amount of time to change her mind (in California it is 24 hours). I do not think this question is appropriate to ask any birth mother. The choice she made was not easy and a question like this may generate painful feelings. +What Not to Ask or Say to an Adoptee + “Do you know your real parents?” I love my birth mother and she is just that: my birth mother. My mom is the woman that raised me–my adoptive mother. So, to ask an adoptee if they know their real parents is thoughtless and can be hurtful. Many adoptees do not even know their birth parents. All they have ever known is their adoptive parents: their real parents. +“You know your birth mom didn’t love you.” + Someone once said this too me. It was someone who did not know me but knew I was adopted. I did not let this comment get to me because I knew without any doubt that my birth mom did love me. I knew because my parents had told me and my birth mother had written me a letter when I was just hours old. A comment like this is completely horrible and heartless. Yes, my birth mother loved me. That is why she chose life for me and gave me a wonderful family. +“All adopted kids are messed up.” + I have heard that my siblings and I are the exception to this rule because someone knows one family who has one adopted kid that has some problems. This must mean all adopted kids are messed up right? How many families do you know who have biological children who are “messed up”? I know plenty! Therefore, I do not think there is any merit to this comment. The child might have had problems if he or she was not adopted. Maybe their bad behavior is just their personality. If children are adopted at an older age and have experienced traumatic things, then yes, they may have some issues to be addressed. However, categorizing all adoptees into being “messed up” is wrong and hurtful. These comments and questions are just a few of the things not to say to a birth mom or adoptee. I picked these particular comments and questions because they have been directed at me at some point. Just remember: When speaking to an adoptee or birth mom, it is okay to ask questions. I am an open book! However, be sensitive. I know many people are curious, but maybe the adoptee or birth mom is not ready to answer certain questions. Let certain things stay personal. If the adoptee or birth mom chooses to share, they will. \ No newline at end of file diff --git a/tests/data/text/adoption.com2.txt b/tests/data/text/adoption.com2.txt deleted file mode 100644 index e5d097d4..00000000 --- a/tests/data/text/adoption.com2.txt +++ /dev/null @@ -1,11 +0,0 @@ -Thanks and good luck in your search. I have found that many mothers from an era do not actively search for various reasons but have made mention of their placed children to their other children or other family members. I am currently searching for a birth cousin (I know birth mother and father and half siblings names) and aiding a friend in the quest of her birth family. Without a name of either mother or father it is much like searching for a needle in a haystack! Thus the ideal of this thread was created. - - - - - - My paternal male cousin was born May 24, 1962 in Hayti, Missouri. Possibly placed in Illinois, the Chicago area. His birth mother's name is Elain M. Strock or Strack. She had 2 older sons, Ricky and Garry Steigman.The birth father, my uncle, is Thelbert (Peck) Cole. He has 2 younger children. - - - - My female friend was born in St Louis, MO on August 4, 1963 at Bethseda Hospital. She was placed at the Children's home Society and the adoption was finalized within the City of St. Louis. She grew up in rural Missouri. Last edited by southemissouri : 11-04-2005 at . \ No newline at end of file diff --git a/tests/data/text/al.com1.txt b/tests/data/text/al.com1.txt index 7272c2c7..006e5ebd 100644 --- a/tests/data/text/al.com1.txt +++ b/tests/data/text/al.com1.txt @@ -14,6 +14,8 @@ During a segment called "Final Verdict" on ESPN's College Football Final, Holtz In those segments, co-host Rece Davis, dressed like a judge, rules either in favor of Holtz or May. Both times, May, who said Ohio State and the Big Ten were out of the College Football Playoff hunt, got the ruling. +Holtz hasn't forgotten. + "I lost two 'Final Verdicts' and doggone it both of them turned out that Rece was wrong," Holtz said. "No. 1 the Big Ten would have somebody in (the playoff) and Ohio State had a chance. Both times he ruled against me." As for why May tends to have an anti-Big Ten opinion - something many Ohio State fans feel is a trend - Holtz decided to sidestep that question. diff --git a/tests/data/text/al.com2.txt b/tests/data/text/al.com2.txt index 8afc77c5..1af79536 100644 --- a/tests/data/text/al.com2.txt +++ b/tests/data/text/al.com2.txt @@ -1,3 +1,29 @@ HUNTSVILLE, Alabama - Hockey players, roller skating enthusiasts and Future City competitors are among groups expected to bring more than 5,900 people together next month in Huntsville/Madison County. -Here is a Huntsville/Madison County Convention & Visitors Bureau calendar of January events and conventions with host hotel if applicable and number of expected attendees: \ No newline at end of file +Here is a Huntsville/Madison County Convention & Visitors Bureau calendar of January events and conventions with host hotel if applicable and number of expected attendees: + +Jan. 1-3: UAH - Charger Hockey, UAH Chargers vs. Anchorage Alaska (January 2015), NHH, 200 + +Jan. 14-19: National Supreme Council Ancient & Accepted Scottish Rites Masons, National Supreme Council Mid Winter Meeting 2015, Marriott-Huntsville, 225 + +Jan. 15-18: BK Productions, 2015 Boat Show, NHH, 500 + +Jan. 15-17: Alabama Military Collectors' Association, 2015 Winter Military Collector's Show, Hampton Inn - Arsenal/South Parkway, 700 + +Jan. 15-19: Dogg Pound's Martin Luther King Jr Skate-A-Thon, 2015 Annual MLK Freestyle Skate Jam, Holiday Inn Research Park, 2000 + +Jan. 15-17: UAH - Charger Hockey, UAH Chargers vs. Northern Michigan (January 2015), NHH, 200 + +Jan. 16-19: North Alabama Hockey Association (NAHA), 2015 Freedom Tournament, Hilton Garden Inn-Space Center, 800 + +Jan. 17-18: Alabama Regional Future City Competition, 2015 Future City Competition, NHH, 200 + +Jan. 21-25: American Medical Association Alliance, Southern Regional Meeting, Embassy Suites Hotel & Spa, 100 + +Jan. 23-25: Huntsville Track Club, 21st Annual Mountain Mist Trail Run (2015), Embassy Suites Hotel & Spa, 300 + +Jan. 23-25: UAH - Charger Hockey, UAH Chargers vs. U.S. Under-18 (January 2015), NHH, 200 + +Jan. 27-28: UAH - Student Success Center (Career Fair), 2015 UAH Spring Career Fair, Courtyard by Marriott, 500 + +Jan. 27-30: The Aerospace Corporation, 2015 Executive Board Meeting, Embassy Suites Hotel & Spa, 15 \ No newline at end of file diff --git a/tests/data/text/apartmenttherapy.com1.txt b/tests/data/text/apartmenttherapy.com1.txt index 69696bd5..929af2c4 100644 --- a/tests/data/text/apartmenttherapy.com1.txt +++ b/tests/data/text/apartmenttherapy.com1.txt @@ -1 +1 @@ -I truly believe that all a children's book needs to be magical is a well illustrated wonderful story, but sometimes, as a gift,...a book can fall flat. These interactive books, on the other hand, have so many bells and whistles, they won't be cast aside on Christmas morning in favor of "real" toys. They are super creative, super fun and meant to be played with, over and over. 1. What's Inside by OKIDO. This special twist of this book is that when certain pages are held up to light, more pictures are revealed giving children a look at what's inside bodies, buildings, cars and more. A really unique, enjoyable experience. You can see a video here. 2. My First Keyboard Book by Sam Taplin. Part picture book, part sheet music, part working keyboard, this book is a wonderful introduction to an instrument or just plain fun for young children to experiment with. You can watch a video demo here. 3. I'm So Glad You're Here Giant Book by Michelle Romo and J. Betrue. Perhaps hard to tell from the photo collage, this soft book is actually child-sized. And by that I don't mean it's sized for a child; I mean that it is the size of a small child. Measuring in at 25" x 22", the book features beautiful appliqué and embroidery. 4. Is There a Dog in This Book? by Viviane Schwarz. A long-awaited follow-up to the hilarious "There are Cats in this Book" and "There are NO Cats in this Book", the cat trio of Tiny, Moonpie, and Andre are back for more interactive fun. Children will be amused by the story and delighted in the surprises awaiting them behind all the flaps. 5. Playbook Farm by Corina Fletcher and Britta Teckentrup. This is a fantastic pop-up book on its own, but in the (probably trademarked) words of Ron Popeil, "But wait, there's more!", the book unfolds to become a 3-D playmat. Really inventive format that has been continued in Playbook Pirates and Playbook Castle. You can see customer photos and a video here. 6. My Little Blue Robot by Stephen T. Johnson. Perfect for kids who love to build, or who love robots, this book includes sturdy cardboard parts to build your own robot (who even talks!). All the pieces use slots and tabs so you don't need glue or any extraneous materials. You can see a video of one being built here. 7. The Ultimate Book of Vehicles From Around the World by Anne-Sophie Baumann and Didier Balicevic. Not only is this a big book and filled with wonderful illustrations of all kinds of vehicles - cars, motorcycles, buses, cement trucks — but it has 60 moving parts: from simple flaps to open, tabs to pull and more complex parts a child can manipulate (see this video to get a better idea). This is truly a memorable book that a child can spend hours and hours with. There's also a follow-up: The Ultimate Construction Site Book. 8. Ocean: A Photicular Book created by Dan Kainen, written by Carol Kaufmann. If you haven't yet experienced a "photicular" book - you must. When this book arrived at the office everyone crowded around to marvel at the seemingly moving pictures. Kainen's first creation, Safari, is wonderful and his latest, Ocean, takes readers under the sea. This video gives you a peek behind the photicular technology. 9. My First Computer by Anne-Sophie Baumann and Marion Billet. This book won't keep your toddler from wanting to bang on your computer keyboard, but it will certainly keep them occupied and delighted, perhaps while you do your own work. With flaps to open and, of course, things to slide, ala a tablet, this book simulates the discovery of an app, but in a beautifully low tech/high creativity way. Watch a demo here. 10. Presto Change-O! A Book of Animal Magic by Édouard Manceau. This is one of the most creative interactive books I've ever seen. Children can literally move the pictures around to transform one scene into another. A teapot turns into an elephant, a hot air balloon turns into a rabbit - and back again, if you wish. Check out this video of the book in action. \ No newline at end of file +ONLY EXTRACTED AUTHOR BLURB AFTER THE ACTUAL ARTICLE \ No newline at end of file diff --git a/tests/data/text/apartmenttherapy.com2.txt b/tests/data/text/apartmenttherapy.com2.txt index 40816aef..368c336d 100644 --- a/tests/data/text/apartmenttherapy.com2.txt +++ b/tests/data/text/apartmenttherapy.com2.txt @@ -1,3 +1 @@ -Nothing stirs the DIY spirit like a good IKEA hack. IKEA is so accessible — anybody could stroll in there and buy a BEKVAM or two — but its Scandinavian design is deceptively simple. With a little ingenuity a box, or a table, or a stool from IKEA could become pretty much anything. Here's proof. - -Click on the slideshow to see the projects. To learn more about each project (and see more pictures), click the links in the captions. +EXTRACTED NOTHING diff --git a/tests/data/text/arabic.txt b/tests/data/text/arabic.txt index ece86433..27959316 100644 --- a/tests/data/text/arabic.txt +++ b/tests/data/text/arabic.txt @@ -10,6 +10,4 @@ أما الائتلاف الوطني السوري المعارض، فقد دعا في بيان له كافة الكتائب والفصائل المقاتلة في الشمال السوري إلى "ضرورة الوعي بأهمية المرحلة الراهنة، وبضبط النفس والتحلي بالحكمة لضمان سلامة المدنيين وإخلاء سبيل أي أشخاص موقوفين أو معتقلين." -وشدد الائتلاف على "ضرورة الابتعاد عن الأعمال الاستفزازية بكافة أشكالها، ويحذر كل من يستغل المرحلة الراهنة لتطبيق أجندات سياسية، وترك القرار للشعب السوري الحر ليختار مصيره بملء إرادته" في بيان يأتي بالترافق مع الحديث عن كون تلك المواجهات مقدمة لولادة حكومة تدير المناطق التي يقطنها الأكراد في سوريا بشكل مستقل. - -ترحب شبكة CNN بالنقاش الحيوي والمفيد، وكي لا نضطر في موقع CNN بالعربية إلى مراجعة التعليقات قبل نشرها. ننصحك بمراجعة إرشادات الاستخدام للتعرف إليها جيداً. وللعلم فان جميع مشاركاتك يمكن استخدامها، مع اسمك وصورتك، استنادا إلى سياسة الخصوصية بما يتوافق مع شروط استخدام الموقع. الآراء الواردة أدناه لا تعبر عن رأي موقع CNN بالعربية، بل تعكس وجهات نظر أصحابها فقط. \ No newline at end of file +وشدد الائتلاف على "ضرورة الابتعاد عن الأعمال الاستفزازية بكافة أشكالها، ويحذر كل من يستغل المرحلة الراهنة لتطبيق أجندات سياسية، وترك القرار للشعب السوري الحر ليختار مصيره بملء إرادته" في بيان يأتي بالترافق مع الحديث عن كون تلك المواجهات مقدمة لولادة حكومة تدير المناطق التي يقطنها الأكراد في سوريا بشكل مستقل. \ No newline at end of file diff --git a/tests/data/text/architecturaldigest.com1.txt b/tests/data/text/architecturaldigest.com1.txt index 34899de4..8549ac08 100644 --- a/tests/data/text/architecturaldigest.com1.txt +++ b/tests/data/text/architecturaldigest.com1.txt @@ -8,4 +8,8 @@ Any such notions quickly vanish, however, as the entry procession leads through Flanking the pool are two loggias—one off the master suite and the other off the guest quarters—delineated by limestone brise-soleils that orchestrate an ever-changing dance of reflected light. "I used limestone for both the courtyard floor and most of the walls to underscore the idea of the house as a configuration of interconnected pavilions with varying degrees of exposure," the architect says, noting the stone's luminous yet earthy quality. "Using one material throughout has a calming effect—it gives you the luxury of tuning out the cacophony of the outside world." -For the full story and more photos, subscribe now and get the digital edition immediately. \ No newline at end of file +For the full story and more photos, subscribe now and get the digital edition immediately. + + + +Tour Joann and Paul Delaney's sleek home. \ No newline at end of file diff --git a/tests/data/text/architecturaldigest.com2.txt b/tests/data/text/architecturaldigest.com2.txt index e4b6f627..364cff24 100644 --- a/tests/data/text/architecturaldigest.com2.txt +++ b/tests/data/text/architecturaldigest.com2.txt @@ -1 +1,3 @@ +AD Monthly News + Sign up for our newsletter to get the latest in design and decorating, celebrity style, shopping, and more. \ No newline at end of file diff --git a/tests/data/text/avclub.com1.txt b/tests/data/text/avclub.com1.txt index f6ae988e..6f12d098 100644 --- a/tests/data/text/avclub.com1.txt +++ b/tests/data/text/avclub.com1.txt @@ -1,67 +1,122 @@ Great scenes, like great movies, adhere to no single formula. Some of them work only in relation to the moments that occur before and/or after them. Others play like miniature movies themselves, their appeal independent of the feature-length films that house them. What all great scenes have in common, though, is their ability to imprint themselves on a viewer’s brain. In anticipation of our best films of 2014 list, which drops on Thursday, we’ve singled out 22 of our favorite scenes from the year in cinema. They’re in no particular order—save for the first one, which several contributors cited and we’ve hence decided is the year’s best. Fair warning: Some of these entries, including the first one below, disclose major plot points. Proceed with caution. +Scene of the year + +The ending, Whiplash + Those who haven’t seen Whiplash yet should probably skip right past this glowing appraisal of its final minutes, and rush to the nearest theater showing it. Those who have seen the film should finally catch their breath and read on. In the closing scene of the movie, aspiring jazz drummer Andrew Neyman (Miles Teller) suffers one last humiliation at the hands of his abusive mentor, Terence Fletcher (J.K. Simmons), who takes revenge on his star pupil by providing him with the wrong sheet music for a climactic concert at Carnegie Hall. Rather than accept defeat, however, the musician returns to his kit, answering this cruel betrayal the only way he knows how: by drumming his ass off—this time to his own beat, guiding the ensemble with an improvised burst of virtuosic playing. Nearly all of Whiplash operates on a level of pure anxiety, entwining the nerves of its audience and protagonist. And so this parting display of showboating talent, which director Damien Chazelle stages with all the kinetic verve of a car chase or a battle sequence, feels downright liberating in its sense of cathartic release. A true marvel of editing, composition, and performance, the scene would be a contender for the year’s finest even if seen completely out of context. What clinches its victory, though, is the troubling ambivalence lurking beneath the awe-inspiring spectacle: We’re watching not just the birth of a future jazz legend, but also the consummation of a truly toxic relationship—the moment, in other words, when one obsessive sociopath finally rebuilds another to his exact specifications. It’s equal parts disturbing and rousing, a thunderous cymbal crack on the beating hearts of its audience. [A.A. Dowd] +The deformed man, Under The Skin + The first half of Jonathan Glazer’s Under The Skin establishes a hypnotic pattern: An alien in human guise (Scarlett Johansson) picks up unsuspecting men, seduces them into accompanying her into a dilapidated-looking house, and, once inside, does away with them, in a fashion disturbing and visually striking in equal measure. But just around the film’s halfway mark, the alien picks up a man with a facial deformity. Their conversation, in a moving car, first proceeds in simple alternating one-shots as she asks him questions and repeatedly compliments his hands. But when she asks him to touch her face, they share the frame, and later one-shots are closer, more intimate. At first, she seems to simply assume a gentler seduction tactic. While she does bring the man back to her lair, she eventually allows him to leave without suffering the same fate as the others. When she stares at her own shadowy reflection that night, she may be catching a glimpse of empathy. The sequence is fascinating in its own right, but even more so for the way it reverberates through the rest of the movie, knocking the Johansson character off her axis and setting the second half of her quiet story in motion. [Jesse Hassenger] +First race, Need For Speed + The year’s best, purest chase scene—technically a street racing scene—is one part old-school style, one part newfangled tech. Shot on Canon C500s—beefier, 4K variants of the low-light-friendly cameras used to shoot micro-budgeted projects like Blue Ruin and Blue Is The Warmest Color—it follows five gleaming cars as they rip through a vivid small-town nightscape, saturated in sodium-vapor orange. The camera style is self-consciously 1980: slow zoom-ins at the starting line; passenger-side handheld shots of the drivers; the kind of natural, unaffected shaking that happens when a cameraman is trying to keep a speeding car in the frame. And, for the bulk of the scene, there’s no music, only the snarl and squeal of the cars. Five engines, half-a-dozen vicious turns, and a train barreling into the distance—nothing more is necessary. [Ignatiy Vishnevetsky] +Inflating and deflating the male ego, Force Majeure + Ruben Östlund’s Force Majeure is a stinging critique of the male ego, and it reaches a crescendo during a late scene in which a pitiful husband and father (Johannes Kuhnke)—having already proven his spinelessness by ditching his clan in order to save himself during an apparent avalanche at their ski-vacation resort—enjoys a beer with his best mate (Kristofer Hivju) at the bottom of the mountain. There, the two men are approached by a woman who tells them that her friend thinks they’re the best-looking guys at the place, news that naturally boosts their self-esteem. That high doesn’t last long, however, since just as they’re enjoying the praise, the woman returns to apologize for having relayed the compliment to them; it was really intended for two other nearby gentlemen. Left deflated beyond repair, they vacillate between anger and embarrassment, which Östlund depicts in a protracted single shot that reveals the clownish emptiness of their macho pretenses. [Nick Schager] +The safe wager, Dom Hemingway + Just out of prison following a 12-year stretch, expert safecracker Dom Hemingway (Jude Law) is looking for work, among other things, until a former associate named Lestor (Jumayn Hunter) tells Dom his skills are now useless, as newfangled electronic safes are impossible to crack via methods of a dozen years past. Since Lestor despises Dom, he offers him a deal: If Dom can open Lestor’s personal safe in less than 10 minutes, he’ll give him a highly lucrative job. But if Dom fails, Lestor gets to cut off his dick, right on the spot. Writer-director Richard Shepard plays this ludicrous ticking-clock scenario to the hilt, devising both a wholly unexpected safecracking method—no gentle taps and slowly twisted dials here—and a diabolical punchline. Mostly, though, it’s just a hoot to watch Law’s high-octane performance dovetail with a rare moment of concentrated focus for his character. Dom is ostensibly working feverishly to save his penis, but the expression on Law’s face throughout is pure, uncut fun. [Mike D’Angelo] +Creating the universe, Noah + Darren Aronofsky’s Noah is a work of conflicting aspirations—a Biblical epic that wants to be intimate and spectacular, “realistic” and fantastical, accommodating of both serious spiritual inquiry and talking, CGI rock monsters. The film’s most fruitful attempt at reconciling seemingly contradictory positions arrives during the centerpiece sequence, when Noah (Russell Crowe) recounts to his family the story of God creating the universe. Billions of years pass through simulated time-lapse photography, and as single-cell organisms transform into fish, which soon slither onto dry land and turn into something else entirely, it becomes clear that Aronofsky has incorporated evolutionary theory into his retelling of Genesis 1. Fundamentalists might balk at this revisionist take on scripture, but it’s hard to imagine anyone shrugging off the grandeur of the scene—a montage of blooming nebulas, scampering species, throbbing forbidden fruit, a menacing serpent, a glowing Adam and Eve, and Cain murdering Abel in striking silhouette. Noah has no shortage of grand imagery, but only this flashback to the beginning of time inspires a religious (or at least near-religious) awe. Cinephiles both devout and secular should give thanks. [A.A. Dowd] +The whoring bed, Nymphomaniac, Vol. I + Lars Von Trier’s epic study in carnal (and non-carnal) knowledge features plenty of explicit sex, but it’s a fully dressed, gatecrashing Uma Thurman who provides its most blistering, passionate sequence. After more than an hour spent exploring heroine Joe’s sexual self-indulgence, Nymphomaniac abruptly serves up a comically ghastly reminder that her actions have consequences. Into Joe’s apartment strides Thurman’s Mrs. H, her three young sons in tow, in order to confront their philandering father. “Confront” isn’t quite the word, though, since Mrs. H opts to channel her anger through relentless bitter sarcasm, congratulating her husband and Joe on their shared happiness and taking the kids on a tour of the premises, so that they can see what Dad will be up to from now on. “Would it be all right if I show the children the whoring bed?” she politely asks Joe, in the scene’s signature moment. Thurman plays this minor role (it’s the character’s sole appearance) without an ounce of vanity, digging so deep into Mrs. H’s feelings of debasement that all she can finally do, at the end, is emit a truly bloodcurdling shriek. [Mike D’Angelo] +End credits montage, 22 Jump Street + Phil Lord and Christopher Miller’s 22 Jump Street is the rare comedy sequel to equal its predecessor. Even more impressive still, it manages to surpass the original at that least likely of moments, the end credits. Having already spent its entire feature-length runtime making self-referential jokes about the derivativeness of sequels, 22 Jump Street goes that extra step—and then a few more—by delivering a montage of phony upcoming follow-ups, replete with clips and poster art. From “Culinary School” and “Foreign Exchange Students” to “A Semester At Sea” and “Traffic School,” it’s a sequence that pushes the material’s auto-critique craziness into outright absurdity, especially when Seth Rogen momentarily appears in “Sunday School” as a replacement for Jonah Hill (apparently due to a “contract dispute”). At once dim-bulb silly and cannily critical of its own inherent existence, it’s perhaps the best end-credits sequence in movie comedy history—as well as a capper that mocks (to the point of negating) the need for any further franchise installments. [Nick Schager] +Terror in the tent, Willow Creek + For much of its running time, comedian Bobcat Goldthwait’s first stab at found-footage horror seems content to operate like an affectionate goof on The Blair Witch Project. The film does, however, make a late attempt at eliciting goosebumps instead of giggles, and the results are quite effective. Having retreated to the woods in search of the elusive Bigfoot, amateur filmmaker Jim (Bryce Johnson) and his patient girlfriend Kelly (Alexie Gilmore) are awoken in the middle of the night by strange sounds coming from outside of their tent. Goldthwait captures the subsequent gauntlet of terror in a single, 20-minute take, locking his camera on the faces of his increasingly alarmed characters, who sit paralyzed with fear as the noises get louder, closer, and weirder. It’s a marvelously suspenseful sequence, relying on not just the credible distress of the actors, but also the claustrophobia of the tent—a structure that limits our view of the surroundings, allowing the imagination to run wild with thoughts of the beasts surely lurking on the other side of its flimsy, nylon walls. As in Blair Witch, what we envision is much scarier than anything we could be shown. [A.A. Dowd] +“I’ll Have To Dance With Cassie,” God Help The Girl + + + God Help The Girl has half a dozen lovely little production numbers in a variety of moods, but Stuart Murdoch’s musical is never more delightful than when it engages with the pure joy of making and experiencing music, crystallized in the scene scored to the song “I’ll Have To Dance With Cassie.” James (Olly Alexander) brings troubled singer-songwriter Eve (Emily Browning) to some kind of civic center, where the supporting members of their band are playing an afternoon dance attended mostly by the elderly. When Eve joins them onstage to sing and her song kicks into full-band ebullience, minor miracles abound: Dancers who look ready for a community-college production of Grease appear from nowhere; bandmate Cassie (Hannah Murray) arrives at just the right time; and Eve seems to forget her problems, however briefly. Murdoch cuts around madly; it’s exactly the kind of dance number that makes purists complain about not being able to see the dancing. But intricate choreography isn’t the point here; Murdoch stages a dance party halfway between homemade reality and music-video dreams. “Hell do I care what I look like when I feel this good?” the song asks—though as it happens, everyone looks pretty great. [Jesse Hassenger] +The classroom, Snowpiercer + Bong-Joon Ho’s Snowpiercer was the summer’s best action film, and it peaks when Chris Evans’ rebel leader, guiding his insurgency through a train that houses the last survivors of a global environmental apocalypse, arrives at the car where young children are being lectured by their teacher (Alison Pill) on the myth of their train’s creator. After the bleakness of their prior environments, Evans and company’s arrival in this brightly colored elementary-school compartment is jarring, and made more so by Pill’s overly cheery demeanor, which carries with it more than a whiff of madness. Basing its art-deco designs (both in terms of the classroom, and the historical video that the kids watch) on those found in the popular Bioshock first-person-shooter videogame series, the sequence has an unsettling strangeness that eventually erupts in a paroxysm of violence. Evil is rarely more chilling than when it comes in the form of a sunshiny mentor. [Nick Schager] +“The Step You Can’t Take Back,” Begin Again + A man (Mark Ruffalo) slumps over a bar, his left shoulder jutted toward the camera. He hears a guitar and a woman’s voice; slowly, he begins to peek over his arm, as though he were a sun reluctant to rise. The camera stays on him for almost 45 seconds, only revealing the singer—a songstress (Keira Knightley) with an acoustic guitar, awkwardly perched on a stool—once the chorus starts. This is actually the second time this scene has played out in Begin Again; the first time was at the very beginning of the movie, from her perspective, with the camera landing on his big goofy grin as she got off the stage. Now, the viewer is seeing it—and hearing it—from his angle, as he eyes the next act’s instruments and starts imaging a full-blown, Starbucks-playlist arrangement. Begin Again—a music-business musical that serves as a kind of spiritual sequel to director John Carney’s earlier Once—is largely bogus, but this sequence, its only lapse into overt fantasy, feels completely authentic. That’s thanks in so small part to Ruffalo’s performance, which turns what should be a simple reaction shot into a glimpse into a character’s soul. [Ignatiy Vishnevetsky] +Dave Schultz’s interview, Foxcatcher + For all the praise being heaped on Steve Carell’s transformation into a beaked, aristocratic sociopath, the best performance in Foxcatcher required no prosthetic noses. As Dave, the older and more decorated of the two Olympic-champion Schultz brothers, a bulked-up Mark Ruffalo finds layers of feeling his co-stars aren’t quite afforded. His showcase scene—and arguably the film’s single strongest moment—is the one in which Dave is forced to speak about the influence his sponsor and coach, John Du Pont (Carell), has had on his athletic career. At the behest of a filmmaker, who’s making a presumably propagandistic documentary on Du Pont, Dave struggles to explain the man’s coaching strategies (which are basically useless) and to muster up a single word of praise. When the director requests that he describe Du Pont as a “mentor,” Dave valiantly attempts to swallow his pride—and Ruffalo makes his thought process palpable, a small storm of emotions passing across his face. It’s a miniature master class in acting, and one of the brief, promising instances in which Foxcatcher threatens to burst its bubble of oppressive melancholia and become a stealth comedy of discomfort. [A.A. Dowd] +The wedding dance, In Bloom + + + Movie musicals may have gone out of style, but a dancing interlude can still elevate the pulse of just about any film. In Bloom, a Georgian drama about two 14-year-old girls and a gun, reaches its emotional apex during the mid-film scene in which one of them, Natia (Mariam Bokeria), marries a man she barely knows, at her family’s behest. Her best friend, Eka (Lika Babluani), is visibly upset during the reception, and even calls Natia into the bathroom at one point to ask whether she loves her new husband. (The answer: “I guess I do.”) Upon emerging, however, Eka suddenly takes the center of the room and performs a lengthy solo dance as the other guests stand around her in a circle, clapping and cheering. What’s remarkable about this routine is the wealth of contradictory feelings it somehow conveys—Eka’s decision to elbow past everyone and take command of the dance floor is at once a defiant, fuck-you gesture; a burst of self-liberation; and an ardent declaration of love for Natia. Plus, Georgians really know how to dance—when a standing Eka picks up a napkin from the floor with her teeth, you’ll feel the urge to whoop and holler, too. [Mike D’Angelo] +Destruction of the city, Pompeii + Paul W.S. Anderson—the widely maligned English director behind Resident Evil, Alien Vs. Predator, and The Three Musketeers 3D—has a real knack for organizing and diagramming space, and though Pompeii, his first blockbuster-budget production, is slow-going at first, it hits its stride once it comes time to depict the destruction of the titular city street by street, neighborhood by neighborhood. Obsessed with bunkers, mazes, and caverns, Anderson can’t help but turn the city grid into yet another of his deathtrap tunnel systems. He envisions the destruction of Pompeii as one long set piece, with characters scampering over bodies and ruins, trying to outrun a ship that’s been forced inland while fiery debris rains down in the foreground. It’s brisk and breathtaking. [Ignatiy Vishnevetsky] +“Time In A Bottle,” X-Men: Days Of Future Past + + + Recently, every year has had at least three or four superhero movies, and all of those superhero movies have at least three or four action sequences, many of which involve gigantic airships and/or collapsing buildings. It makes sense, then, that the most memorable superhero action of the year would be a bit more granular, focusing on the use of a single superpower. In Bryan Singer’s mutant-packed X-Men sequel, Professor X, Beast, and Wolverine enlist super-fast mutant Quicksilver (Evan Peters) to help break Magneto out of his metal-free holding cell in the Pentagon. When the group is discovered and security guards open fire, the movie switches to Quicksilver’s point of view as he flips on his Walkman and springs into action, running up and down the walls with the nonchalance of a morning jogger, scored to the quiet strains of Jim Croce’s “Time In A Bottle.” With a few gentle tweaks, he knocks out cops and sets bullets off course; when the film returns to normal speed, the conflict is over in a matter of seconds. This interlude is a sustained moment of playfulness in an otherwise fairly serious superhero epic, and a reminder that the X-Men movies distinguish themselves by toying with the idiosyncrasies and possibilities of their universe’s vast array of mutants, not by getting into a building-destruction contest. [Jesse Hassenger] +Sex and violence, Gone Girl + Gone Girl’s first half is a slow-drip mystery, but it’s the second half of David Fincher’s adaptation—after the big twist is revealed—that taps a truly sinister vein. The story’s malevolence reaches a fever pitch during the scene when Amy (Rosamund Pike), now a captive of her creepy ex-boyfriend (Neil Patrick Harris), finally “gives in” to his carnal wishes and takes him to bed. It’s an encounter of sexual aggression and manipulation that Fincher stages with mounting unease, until the moment the white lingerie-clad Amy, beneath her lover on the bed, suddenly and swiftly slices his throat, coating herself in blood at the very moment that he climaxes inside her. More chilling still: After letting him bleed out, Pike straddles him and then flips her blood-soaked hair out of her face—an offhand gesture of aggravation that speaks volumes about the depths of her mercilessness. [Nick Schager] +Opening credits, Godzilla + “Even with nuclear weapons there is no guarantee that the creatures will succumb,” declares the witty opening credits sequence for Hollywood’s latest iteration of Japan’s favorite monster. “Evidence show [sic] that it is likely the creatures will come back with David Strathairn’s head.” That’s what the text reads for a split second, anyway—only by freeze-framing the DVD can one catch more than a few words—before everything other than DAVID STRATHAIRN is redacted. Superimposed over stills ranging from Darwin’s Origin Of Species to news coverage of the Bikini Atoll tests, all of the credits are accompanied by alarmist phrases that are swiftly blacked out, creating a deliciously paranoid mood (augmented by Alexandre Desplat’s urgent score) before the story has even begun. There are even some in-jokes among the barely visible text: Bryan Cranston’s credit includes the phrase “Walter Malcolm has claimed that government men dressed in white lab coats routinely appear at site,” which is redacted in a way that leaves the words “Walter” and “white” visible by themselves for a fraction of a second. With “Malcolm” in the middle. [Mike D’Angelo] +Two decades of messages, Interstellar + Christopher Nolan has developed an inaccurate reputation as a chilly filmmaker with a Kubrickian detachment from human emotion. Despite superficial 2001 influences, Interstellar seems almost designed to correct that assumption, never more effectively than in a single scene that essentially involves space pilot Cooper (Matthew McConaughey) sitting down to check his messages. Cooper has been on a planet that experiences time far more slowly than Earth, and upon his return to his ship, he watches a series of video messages from his children, received in the hour or so that he was gone. In a few moments, he watches them age over two decades. Because it’s a Nolan movie, this scene also contains plenty of information: about what’s happened on Earth while Cooper has been gone and where his plot-crucial daughter Murph is now. But the exposition feels secondary to the emotional wallop of Cooper’s family life accelerating without him. In a movie of amazing sights, it’s the simple passage of time that hits the hardest, culminating in the final video message, from now-grown Murph (Jessica Chastain). When she finishes a goodbye to her father, thought to be lost in space, Nolan cuts to the other side of her camera and follows her as she returns to her job—bringing the movie to this decades-forward Earth for the first time. The transition doesn’t span anywhere near the amount of time traversed in the most famous cut of 2001, but it’s a similar technique, hurtling the audience ahead into the next phase of the story. [Jesse Hassenger] +“Pretty Girl Rock,” The Rover + Set in a depressed (and depressing) post-apocalyptic Australia, The Rover adopts a mood of consistent despair, rarely allowing for even the faintest trace of levity. Maybe that’s why the belated arrival of a bona fide pop anthem on the soundtrack—briefly replacing the ambient, dread-infused hum of Antony Partos’ original score—qualifies as downright triumphant. As the film’s mismatched road warriors (played by Guy Pearce and Robert Pattinson) wander into the Outback during a protracted wide shot, director David Michôd cues up Keri Hilson’s narcissistic earworm single “Pretty Girl Rock.” It seems like a bitterly ironic song selection, until the scene cuts to a nighttime image of Pattinson’s tragic simpleton sitting alone in a jeep, softly singing along to the suddenly diegetic music. From here, the tune takes on a melancholy quality, with Michôd employing it as a creature comfort from another era—a bittersweet blast of nostalgia, a sonic relic of the more hopeful world that now exists only in the rearview mirror of these characters’ lives. As needle drops go, it’s eccentric and weirdly, powerfully affecting. Also, good luck getting that damn song out of your head. [A.A. Dowd] +The Red Circle, John Wick + Chad Stahelski and David Leitch are both veteran stuntmen, which explains why their debut feature, the superbly entertaining John Wick, happens to contain some of the most energetic, best-directed action sequences in recent memory, with the standout being the movie’s centerpiece, in which Keanu Reeves’ eponymous ex-hitman shoots his way through a nightclub/bathhouse called the Red Circle. Starting in the back of the club—where silhouetted henchmen are dispatched one by one while Kaleida’s “Think” slinks on the soundtrack—and then bursting into the main floor, Collateral-style, it’s as much a dance piece as a gunfight. Reeves and an ensemble of stunt performers roll, tumble, shoot, and reload, over and over, moving through pools of blue and magenta light, always oriented around the camera and the geometry of the space; this is amped-up action as kinetic art. [Ignatiy Vishnevetsky] -Bill Hader and Kristen Wiig prove themselves adept serious actors in The Skeleton Twins, starring as siblings reunited after a 10-year estrangement. The movie itself doesn’t often transcend its indie-dramedy roots, but it generates more feeling than it might have otherwise due to its stars’ chemistry, and the SNL-honed comic timing they bring to their depressive characters. The movie’s strongest juxtaposition of angst and crowd-pleasing comedy comes when Milo (Hader) comes home to a frustrated Maggie (Wiig) demanding him to get his shit together. He attempts to cheer her up by silently putting Jefferson Starship’s “Nothing’s Gonna Stop Us Now” on the stereo and beginning to lip-sync, imploring her to join him. Maggie resists for well over a minute before finally, hilariously giving in to the emotive cheesiness—the mugging equivalent of an expertly delayed chorus. It’s the kind of goofy bit Wiig and Hader could’ve sold on SNL, and the movie draws on that history to make Maggie and Milo especially convincing, and touching, as family. [Jesse Hassenger] \ No newline at end of file +Lip-sync duet, The Skeleton Twins + + + +Bill Hader and Kristen Wiig prove themselves adept serious actors in The Skeleton Twins, starring as siblings reunited after a 10-year estrangement. The movie itself doesn’t often transcend its indie-dramedy roots, but it generates more feeling than it might have otherwise due to its stars’ chemistry, and the SNL-honed comic timing they bring to their depressive characters. The movie’s strongest juxtaposition of angst and crowd-pleasing comedy comes when Milo (Hader) comes home to a frustrated Maggie (Wiig) demanding him to get his shit together. He attempts to cheer her up by silently putting Jefferson Starship’s “Nothing’s Gonna Stop Us Now” on the stereo and beginning to lip-sync, imploring her to join him. Maggie resists for well over a minute before finally, hilariously giving in to the emotive cheesiness—the mugging equivalent of an expertly delayed chorus. It’s the kind of goofy bit Wiig and Hader could’ve sold on SNL, and the movie draws on that history to make Maggie and Milo especially convincing, and touching, as family. [Jesse Hassenger] + diff --git a/tests/data/text/avclub.com2.txt b/tests/data/text/avclub.com2.txt index f2cabae0..ba91fe92 100644 --- a/tests/data/text/avclub.com2.txt +++ b/tests/data/text/avclub.com2.txt @@ -1 +1,4 @@ -By now the Serial theme song has been chopped a screwed a million different ways by a million different people, including us for our Serial Serial podcast. But that little kid that says “mail kimp?” That’s a different story. Soundcloud user kpffkl has created the “Mail Kimp Remix” to give that little kid some glory, and it’s both annoying and mesmerizing, as most remixes of a single sound are, but it’s still new and potentially interesting Serial-related content, something the Internet is certainly hungry for this holiday season. \ No newline at end of file +By now the Serial theme song has been chopped a screwed a million different ways by a million different people, including us for our Serial Serial podcast. But that little kid that says “mail kimp?” That’s a different story. Soundcloud user kpffkl has created the “Mail Kimp Remix” to give that little kid some glory, and it’s both annoying and mesmerizing, as most remixes of a single sound are, but it’s still new and potentially interesting Serial-related content, something the Internet is certainly hungry for this holiday season. + +[via Buzzfeed] + diff --git a/tests/data/text/backstage.com1.txt b/tests/data/text/backstage.com1.txt index 0d11d7b2..90720812 100644 --- a/tests/data/text/backstage.com1.txt +++ b/tests/data/text/backstage.com1.txt @@ -1 +1,3 @@ +Instant Access + This casting/job notice is still under review by Backstage's editors. However, you can apply right away! Scams and inappropriate content will be promptly removed. \ No newline at end of file diff --git a/tests/data/text/bhg.com1.txt b/tests/data/text/bhg.com1.txt index e67f22db..c8dfdbd5 100644 --- a/tests/data/text/bhg.com1.txt +++ b/tests/data/text/bhg.com1.txt @@ -1,21 +1,45 @@ -Going simple is the best decorating solution for fall centerpieces and table toppers. Here, deeply hued roses and dahlias play off the more muted collection of faux tomatoes and fall foliage. Editor's Tip: Choose a vase color that recedes and doesn't compete with the blooms. +Eclectic Fall Bowl Centerpiece This decoration can be fashioned easily with a fall nature hunt and a halved globe used as a bowl. There's no need to plan the arrangement -- simply mix the treasures together: pinecones, bittersweet, acorns, and branches make the perfect fall centerpiece. -Without carved faces, pumpkins make a dramatic tabletop. Start with lengths of grapevine wrapped around and up and down pumpkins in different patterns. Display smaller gourds under glass and scatter a base of leaves underneath, and include a tall vase filled with wheat or dried lavender for subtle color complement. +Fall Branches in Vase Bring fall's burst of colors indoors. A simple glass vase transforms a foliage display into naturally beautiful art. Choose a bouquet of maple, oak, sweetgum, dogwood, or persimmon leaves, which typically develop the brightest hues. -It's easy to craft a no-sew table runner in the shapes and colors of fall. Find and enlarge a fallen leaf shape and cut it out from a piece of felt. Glue ribbon or single-fold bias tape to the edge and place on a contrasting piece of felt or complementary table runner. +Bouquet of Fall Flowers and Vegetables Going simple is the best decorating solution for fall centerpieces and table toppers. Here, deeply hued roses and dahlias play off the more muted collection of faux tomatoes and fall foliage. Editor's Tip: Choose a vase color that recedes and doesn't compete with the blooms. -These colorful vegetables are the perfect fall tabletop decorations when stuffed with flowers and foliage. Cut each pepper lengthwise an inch from the top to make an opening (don't cut the top completely off), hollow out the pepper and fill with different types of flowers. Set the peppers on small plates to finish the decoration. +Video: Fall Centerpiece Ideas Fall Centerpiece Ideas -Inexpensive tools make this uniquely carved pumpkin easy to recreate. Use a large leaf to trace onto a pumpkin and use a scraping tool to cut away only the outermost layer. Set it on a bed of moss and a pedestal for a centerpiece that will last all season. +Fall Flowers in Gourd Vases Display these pretty fall flowers in hollowed-out gourds for a unique twist on a fall centerpiece. Select flowers in a variety of autumn hues, such as bright purple, orange, and yellow, and accent the small bouquets with sprigs of orange berries. -The vibrant, diverse colors of Indian corn perfectly represent the color palette of fall, and you can put them to use in a no-fuss candleholder. Place a candle at the center and hot-glue corncobs together in a circle around it to make this pretty tabletop decoration. +White Pumpkins Tabletop Decoration Color-coordinate your tabletop decoration with your serving dishes to create a classy, monochromatic fall display. Place white gourds at different levels in white ceramic dishes, and accent the table with sprigs of red fall berries in a white ceramic vase. -Pumpkins typically are front and center at doorways and on tables, but this elegant collection of corn and wheat stalks fits in perfectly with the season. Include a variety of colors of corn and a few different sizes and include an individual cob at each place setting. +Decorative Tabletop Pumpkins Without carved faces, pumpkins make a dramatic tabletop. Start with lengths of grapevine wrapped around and up and down pumpkins in different patterns. Display smaller gourds under glass and scatter a base of leaves underneath, and include a tall vase filled with wheat or dried lavender for subtle color complement. -Small gourds and pumpkins are nearly sculptural, and a cluster of two or three on a pretty plain plate resembles a still life. Tuck in a pinecone and a few leaves, and wrap a bit of raffia or twine around one of the stems. +Fall Leaf Table Runner It's easy to craft a no-sew table runner in the shapes and colors of fall. Find and enlarge a fallen leaf shape and cut it out from a piece of felt. Glue ribbon or single-fold bias tape to the edge and place on a contrasting piece of felt or complementary table runner. -This decorative arrangement of cattails, pheasant feathers (found at crafts stores), and Purple Majesty ornamental millet doesn't need water, so it will last all season long. Editor's Tip: Place chicken wire or floral netting inside an urn to create structure within the vase. Mold the wire into a ball to fit the bottom of the vase and stick the stems into the holes to complete the arrangement. +Tiered Centerpiece of China and Roses This simple addition to your holiday table requires only household items. Using a large china plate as the base, create a layered decoration with stacked teacups and smaller plates. Adorn with orange flowers on each level, and tuck a small amount of foliage in the top teacup. -Pumpkins like the ones pictured can be used as decorations all season long. Elevate a few -- chosen for the beauty of their contrasting colors -- on a pedestal. Underneath, use a table runner or cloth that picks up on the hues, and tuck in few bittersweet branches for texture. +Vegetable Vases with Flowers Tabletop Decoration These colorful vegetables are the perfect fall tabletop decorations when stuffed with flowers and foliage. Cut each pepper lengthwise an inch from the top to make an opening (don't cut the top completely off), hollow out the pepper and fill with different types of flowers. Set the peppers on small plates to finish the decoration. -Repetition of color -- whether with pumpkins, foliage, or flowers -- creates a unified fall decorating scheme that accents all of fall's bounty. Dominant tones of orange and yellow are present in a vase of blooms and a cluster of branches on a sideboard, in addition to the main pumpkin display. \ No newline at end of file +Traced-Leaf Pumpkin Centerpiece Inexpensive tools make this uniquely carved pumpkin easy to recreate. Use a large leaf to trace onto a pumpkin and use a scraping tool to cut away only the outermost layer. Set it on a bed of moss and a pedestal for a centerpiece that will last all season. + +Indian Corn Candleholder The vibrant, diverse colors of Indian corn perfectly represent the color palette of fall, and you can put them to use in a no-fuss candleholder. Place a candle at the center and hot-glue corncobs together in a circle around it to make this pretty tabletop decoration. + +Indian Corn-and-Wheat Centerpiece Pumpkins typically are front and center at doorways and on tables, but this elegant collection of corn and wheat stalks fits in perfectly with the season. Include a variety of colors of corn and a few different sizes and include an individual cob at each place setting. + +Small Pumpkin Plate Centerpiece Small gourds and pumpkins are nearly sculptural, and a cluster of two or three on a pretty plain plate resembles a still life. Tuck in a pinecone and a few leaves, and wrap a bit of raffia or twine around one of the stems. + +Tall Grass-and-Feathers Tabletop Decoration This decorative arrangement of cattails, pheasant feathers (found at crafts stores), and Purple Majesty ornamental millet doesn't need water, so it will last all season long. Editor's Tip: Place chicken wire or floral netting inside an urn to create structure within the vase. Mold the wire into a ball to fit the bottom of the vase and stick the stems into the holes to complete the arrangement. + +Pumpkin Flower Arrangement Pumpkins are plentiful and inexpensive in fall. For a pretty centerpiece, use a large pumpkin to display richly colored blooms. Cut the top off of a pumpkin or squash, scoop out the flesh, and tuck in a vase. Display a variety of flowers including dahlias, sunflowers, and zinnias. + +Flowers-and-Candles Tabletop Decoration A rustic rectangular container disguises a few small flower vases, making a great way to create an eclectic collection of blooms, fall foliage, and candles. Tie long grass or fall berries around the container to make the decoration pop. + +Pretty Pumpkins Centerpiece Pumpkins like the ones pictured can be used as decorations all season long. Elevate a few -- chosen for the beauty of their contrasting colors -- on a pedestal. Underneath, use a table runner or cloth that picks up on the hues, and tuck in few bittersweet branches for texture. + +Tabletop Pumpkin Display Repetition of color -- whether with pumpkins, foliage, or flowers -- creates a unified fall decorating scheme that accents all of fall's bounty. Dominant tones of orange and yellow are present in a vase of blooms and a cluster of branches on a sideboard, in addition to the main pumpkin display. + +Seasonal Bounty Basket Centerpiece A wicker basket contains a lush arrangement of the growth of autumn. Put a square of floral foam into the container and add a collection of kale, dill, tomatoes, broccoli, and other harvest finds for a colorful, edible centerpiece. + +Colorful Fall Leaf Vase and Garland This easy decoration can brighten a living or dining room in just a few minutes, where a vase filled with colorful branches brings the beauty of the outdoors in. + +Pumpkin Tree Tabletop Decoration Simple decorations can make a dressed-up tabletop for fall. A few cylindrical vases filled with birdseed provide a sturdy base for long branches. Small pumpkins -- suspended with bits of twine -- hover above a beautiful assortment of branches, berries, and pinecones. To craft the candle-displaying pumpkins, hollow out a few gourds and tuck in candles and berries. + +Fall Foliage Centerpiece Brightly colored foliage gathered at its prime brings the perfect fall color accent to this room. Snip a few long branches with a variety of colored leaves still attached and tuck them in a cluster of tall, clear vases on a tabletop. \ No newline at end of file diff --git a/tests/data/text/bhg.com2.txt b/tests/data/text/bhg.com2.txt index 574eae5d..b5d3658b 100644 --- a/tests/data/text/bhg.com2.txt +++ b/tests/data/text/bhg.com2.txt @@ -1,3 +1,7 @@ +BHG.com / Videos / / / How to Hang a Christmas Wreath (Two No-Fail Secrets!) + +How to Hang a Christmas Wreath (Two No-Fail Secrets!) + It's not a holiday home until you've hung a Christmas wreath. This quick video offers nifty tricks for damage-free wreath hanging on doors and windows. Get our secrets to success! A pretty wreath, is just the trick to take your Christmas decorations from basic to bold. Here are the no fail secrets, for hanging window, and door wreaths. For outdoor wreaths, choose a ribbon that's strong, durable, and will make a high impact statement. Your best bet, is a two and half inch satin ribbon. To avoid a droopy look, hang your wreath in the top half of the window. On a door, center the wreath at about eye level. To hang a wreath in a window, lower the top window sash and place the wreath outside of the window while holding the ends of the length of the hanging ribbon. [MUSIC] A piece of paint-friendly tape provides stability. Tape just the very end of the ribbon before sliding your window securely close. When hanging on a door, use a staple gun, to staple the ends of the hanging ribbon to the center top of the door. That way, the staples will never be seen. To keep a lightweight wreath in place, use double-sided foam tape on the back. It's that simple. Use this method every Christmas, to hang your wreath without damaging your home. [MUSIC] \ No newline at end of file diff --git a/tests/data/text/bloomberg.com1.txt b/tests/data/text/bloomberg.com1.txt index b015a297..b19724a8 100644 --- a/tests/data/text/bloomberg.com1.txt +++ b/tests/data/text/bloomberg.com1.txt @@ -16,12 +16,16 @@ The weakest jobs rebound has been in the states central to the 2002-2006 housing Nevada, Arizona and Florida are among those furthest from their peak employment during the December 2007-June 2009 downturn. Those states, along with five others -- Alabama, Illinois, Michigan, New Jersey and Ohio -- remain more than 50,000 positions short of that level. +Regional Woes + Yet the regional woes are far broader: New Jersey has been hurt by the loss of casino and pharmaceutical industry positions, New Mexico by U.S. government defense cutbacks, Alabama by weakness in manufacturing, and Michigan by a loss in auto jobs. The energy industry is driving the economic expansion in 12 of the 13 states leading growth since the recession ended, Herrmann said. Leaders include Texas, North Dakota, Oklahoma and Louisiana, with Oregon the only non-energy state among the standouts. Oregon has been boosted by technology manufacturing and fast growth in exports. Excluding the 1.1 million jobs created in Texas, which has led the expansion, the nation would be 350,000 below the prerecession peak, according to Federal Reserve Bank of Dallas research cited by its president, Richard Fisher. +Spotty Recovery + The spotty regional recovery meshes with the Federal Open Market Committee’s view there is “significant underutilization of labor resources,” according to the statement after its September meeting, and continued room to keep interest rates near zero long after the central bank ends its bond buying as planned in October. “There is no pressure to raise interest rates this year and into next given that many parts of the country have a surfeit of unemployed and underemployed workers,” said Mark Zandi, chief economist at Moody’s Analytics Inc. in West Chester, Pennsylvania. @@ -30,6 +34,8 @@ IHS Global Insight economists in Lexington, Massachusetts, project that every st While the auto industry has recovered and sales in August reached the highest level since January 2006, employment levels in Michigan haven’t. +Moving Out + “The auto industry has been moving out of Michigan,” said James Diffley, IHS Global Insight chief U.S. regional economist, who is based in Philadelphia. “Job totals have been declining.” Michigan regained 34,185 automotive manufacturing jobs between 2010 and 2013. Yet that doesn’t come close to replacing the 174,429 industry positions lost from 2001 to 2010, a 59 percent decline, according to data calculated for Bloomberg by the Ann Arbor-based Center for Automotive Research. @@ -38,6 +44,8 @@ While the auto industry remains dominant in Michigan, “education, tourism, gre Weak wage growth outside a few areas, such as energy jobs in North Dakota and Texas, supports the view that most of the country remains far from fully recovered, said Gary Burtless, a senior fellow at the Brookings Institution in Washington and a former Labor Department official. +Hourly Earnings + Hourly earnings nationwide were up 2.1 percent over the past 12 months, compared to 3.9 percent in June 2007, six months prior to the past recession, Labor Department data show. Young adults 25 to 34 years old, who are most likely to relocate for jobs, have been moving at decreasing rates even after the recession ended with migration at historically low levels in 2013, according to a Brookings analysis. @@ -46,6 +54,8 @@ That mobility remains low suggests “that local job markets have been basically While housing prices have regained part of their losses, Nevada was 5.9 percent below its peak employment, Arizona 4.1 percent, Florida 1.5 percent and Georgia 1.1 percent. The data compiled by Bloomberg compares the top employment level for any month during 2014 with the maximum during the downturn. +Nevada Sluggish + “The economy is still sluggish in Nevada,” said Jim Mason, president and co-owner of Taylor International Corp. in Las Vegas. ‘We still have a lot of excess capacity in the employment market.’’ In 2007, Taylor employed 500 people managing $1 billion in construction primarily of hotels and casinos. Today, the company employs 30 in the state and gets 20 applications for each job opening. @@ -56,12 +66,16 @@ Casinos there have faced new competition from Pennsylvania, Connecticut and New “We have a lot of white elephants -- large empty casinos,” he said. “This is a significant problem for the state going forward.” +Construction Weakness + Alabama’s job level remains 4.6 percent below the peak during the downturn, according to data compiled by Bloomberg. Job creation has been hurt by weakness in construction, manufacturing and government spending, said Ahmad Ijaz, an economist at the University of Alabama in Tuscaloosa. “Commercial construction is still relatively weak due to sluggish consumer and business spending,” he said. “Nondurables, particularly the textile and apparel industry, will not be adding any jobs anyway because of off-shoring and automation. Government spending is weak because of cutbacks in federal spending which in turn also impacts state and local government spending.” Eddie Foreman, 40, of Opelika, Alabama, who has a part-time job at a fast-food restaurant, said better-paying and permanent industrial positions aren’t coming back. +‘No Jobs’ + “The economy is tough and there are no jobs for us here,” said Foreman, who has put out 20 applications in the past year and had no interviews. He worked in a variety of manufacturing and construction jobs before the recession. “Eventually I think it has to get better. It can’t get any worse.” New Mexico’s economy was slowed as reverberations from last year’s federal government shutdown rippled through the state, according to a Moody’s Analytics report in April. The state, home of the Los Alamos National Laboratory, has government as its largest employer, with 24 percent of all workers compared to 16 percent in the U.S., according to Moody’s Analytics. There were 30,000 federal workers among the 193,000 public sector employees in the state last year, according to the Moody’s report. diff --git a/tests/data/text/bloomberg.com2.txt b/tests/data/text/bloomberg.com2.txt deleted file mode 100644 index 26bd9102..00000000 --- a/tests/data/text/bloomberg.com2.txt +++ /dev/null @@ -1 +0,0 @@ -This site uses cookies. By continuing to browse the site you are agreeing to our use of cookies. X \ No newline at end of file diff --git a/tests/data/text/bostonherald.com1.txt b/tests/data/text/bostonherald.com1.txt index a98ad9e6..c434f97e 100644 --- a/tests/data/text/bostonherald.com1.txt +++ b/tests/data/text/bostonherald.com1.txt @@ -1,49 +1 @@ -, a 54-year-old motel maid, simply wanted to vote. But in 1965, as one of 100-plus black locals in line to register in , she confronted a segregationist sheriff - and ended up on the ground, beaten viciously with a billy club. When asked to portray the unlikely freedom fighter in the big-screen drama , hesitated: "I didn't want to do it because in every movie I'm hitting somebody!" she tells Us. (Previous wallops: The Color Purple and The Butler.) But director , 42, persuaded her (more on that below), and Winfrey, 60, joined the release - about ( ) and his -to­Montgomery marches for civil rights - as a star and producer. Shortly after DuVernay became the first black female director nominated for a Golden Globe (and the film's Oscar buzz began building), the two sat down with Us to talk, laugh and, in Winfrey's case, get a little misty-eyed. - -Ava, how did Oprah do when she had to punch the sheriff? - -AD She was all in. She had no problem working with the stunt guys to be taken down. I kept saying to them, "Please, can you just be a little more careful?" And they're like, "She's falling on her own!" - -OW I did it every time. - -With in the headlines, is particularly meaningful now? - -OW It's a jaw-dropping thing that this piece of art can meet this cultural moment that's so rich, so robust, so bursting with the energy of people finding their voices. This film's about being heard. I feel like this film, not to overstate it, but it is here for a reason in this moment. - -Will you be screening this for your close friend ? - -OW For sure. Our desire is to go to the and show it to him. - -With such intense scenes - marchers were teargassed - was it tough keeping your emotions in check? - -OW The very first time I saw it with the other producers, they're all taking notes. And I'm, like, sobbing. And I thought, OK, I guess as a producer you're not supposed to sob (continuing in a choked voice) so let me just take some notes too. I'm gonna blow my nose and stop crying. - -Oprah, how did Ava try to woo you for ? - -OW I was just gonna be in the background going, "Yay!" But she sent me the link to a story about when she turned 100, and it said every day she watched the Oprah show eating a tuna-fish sandwich! And Ava says, "Don't you think it would make her proud to know that you played her?" And I go, "Yeeesss, yes, it would." - -It probably also helped that as a girl you ­wanted to be Dr. King! - -OW What made me think I was gonna do that? I was 12, 13. I remember being at our yellow Formica kitchen table filling out one of those "What are you gonna be when you grow up?" forms, and my father was saying, "You can't be Dr. King because Dr. King is a man!" "Well, I'm gonna have me a church," I said. - -What about you, Ava? Who was your idol? - -AD (Pointing at Winfrey.) My mom had magazines with you on the cover on our coffee table since as long as I can remember, to the point where I thought you were my family member. She would tape your show at work and watch it at night. My mom would just say, "Look at her. Don't be like me, be like her," and get emotional. - -OW Oh, God, I'm emo­tional about that (tearing up). You never told me that! (She playfully smacks DuVernay.) Don't tell me that in an interview for the first time! - -Let's talk about something lighter. Ava, you worked on Scandal! - -AD Imagine a geek fan getting a chance to direct her favorite show! I can make be Fitz, and I can do whatever I want: "Walk over there. Now walk back. Turn. Oh, that looks good." You know what I mean? (Laughs.) So that was fun-fun. - -So you're Team Fitz, then - not Team Jake? - -AD Oh, Fitz. - -OW Fitz. - -AD Sorry, Scott Foley! - -The Real-Life Heroes - -Oyelowo says he gained insight from King friend , who revealed "the prankster … the man who was at times unsure." As for Ejogo, Winfrey says and sister Bernice found she "depicted their mother beautifully." +MISSED TINY PARAGRAPH AT THE VERY END diff --git a/tests/data/text/bostonherald.com2.txt b/tests/data/text/bostonherald.com2.txt index 7b946e19..9348da54 100644 --- a/tests/data/text/bostonherald.com2.txt +++ b/tests/data/text/bostonherald.com2.txt @@ -1,5 +1,3 @@ Chrome Industries' Mini Buran laptop messenger bag ($140) is a good investment for anyone who needs a sturdy bag that is built to last and can contain all of your stuff, whether it be a thin laptop or iPad. There are also small pockets for lighter objects. -There are four pockets in the front under the flap, a large pocket inside and a hidden lined pocket where a thin laptop or iPad and rest safe and snug​. This bag is large enough to carry a lot of stuff, but not bulky. - -Seemingly indestructible, the bag features a seatbelt-buckle strap that grips your body nice and tight, but is ridiculously easy to get on and off. \ No newline at end of file +The article you requested has been archived All coverage within bostonherald.com from the last 14 days remains free of charge. Articles do not always include original photos, charts or graphics. » Click here to search for this article within the archive. \ No newline at end of file diff --git a/tests/data/text/businessinsider.com1.txt b/tests/data/text/businessinsider.com1.txt index e188e70c..0bd65304 100644 --- a/tests/data/text/businessinsider.com1.txt +++ b/tests/data/text/businessinsider.com1.txt @@ -10,4 +10,4 @@ Even though the Typhoons are modern enough to deal with a dozen Su-24s, the Sovi We don’t know whether the potential deal includes armament; still the possible delivery of Su-24s to Argentina makes the Falkland Islands a bit more vulnerable to an attack by the Fuerza Aérea Argentina. -This article originally appeared at The Aviationist. Copyright 2014. Follow The Aviationist on Twitter. \ No newline at end of file +This article originally appeared at The Aviationist. Copyright 2015. Follow The Aviationist on Twitter. \ No newline at end of file diff --git a/tests/data/text/businessinsider.com2.txt b/tests/data/text/businessinsider.com2.txt index 9d85d88a..7b505253 100644 --- a/tests/data/text/businessinsider.com2.txt +++ b/tests/data/text/businessinsider.com2.txt @@ -1,4 +1,6 @@ -FA Insights is a daily newsletter from Business Insider that delivers the top news and commentary for financial advisors. +FA Insights is a daily newsletter from Business Insider that delivers the top news and commentary forfinancial advisors. + +Russia's Problems Are A Huge Warning Sign About Emerging Markets (Morningstar) Russia's seen a volatile December: hiked interest rates, increased inflation, and a plunging ruble. Morningstar's Karin Anderson suggests that Russia's problems reflect the larger risks associated with emerging markets. @@ -6,6 +8,8 @@ Russia's seen a volatile December: hiked interest rates, increased inflation, an "Because of those risks, Russia is a reminder that emerging markets remain subject to swift and meaningful changes in capital flows," Anderson adds. +F-Squared Paid $35 Million In A Settlement (The Wall Street Journal) + "F-Squared Investments Inc., which builds investment portfolios out of exchange-traded funds, admitted it misled clients about its track record and agreed to pay $35 million in a settlement with regulators," reports Corrie Driebusch. The SEC filed separate civil charges against the former CEO. He stepped down from his position last month, but his lawyers maintain that the SEC's allegations "are misdirected and meritless." @@ -20,7 +24,7 @@ Additionally, analysts have mixed feelings about agri-commodities in the next ye An SEC Investor Advocate Will Push For Spending More On RIA Exams (Financial Advisor Magazine) -"Securities and Exchange Commission Investor Advocate Rick Fleming said Tuesday he will push SEC chairman Mary Jo White to use a 'significant' portion of its extra $150 million for 2015 to go to increase financial advisor oversight," reports Ted Knutson. +"Securities and Exchange Commission Investor Advocate Rick Fleming said Tuesday he will push SEC chairman Mary Jo White to use a 'significant' portion of its extra $150 million for 2015 to go to increase financial advisoroversight," reports Ted Knutson. "Fraudulent or abusive practices by an investment advisor," said Fleming, "can be very difficult for an individual investor to detect, particularly if the advisor goes so far as to falsify account statements or other records." diff --git a/tests/data/text/businessweek.com1.txt b/tests/data/text/businessweek.com1.txt index c067a21b..504db0bf 100644 --- a/tests/data/text/businessweek.com1.txt +++ b/tests/data/text/businessweek.com1.txt @@ -1,3 +1,5 @@ +Features + “Do you want to listen to Taliban cassette?” Matiullah Matie asks as he steers his white Toyota Corolla along a narrow road surrounded by cornfields and mud huts. He keeps the tapes in the car for long drives, Matie explains, just in case he picks up a hitchhiker who looks like a Talib. “They think I am such a pious mujahid man,” the round, bearded businessman laughs. “They don’t know I am screwing them all.” We are driving to the Nawa district, just 30 minutes outside Lashkar Gah, the capital of Helmand province in the southwest corner of Afghanistan. Matie is going to show us how he first became a millionaire. @@ -22,9 +24,9 @@ In the five years that Mullah Omar and his Taliban regime dominated Afghanistan, About 36 percent of Afghanistan’s 30 million people live below the poverty line. “If you ask people on the streets whether we have a billionaire, they will shrug and say no,” says a senior Afghan economic official, who asked to remain anonymous because he is privy to sensitive information. “But I can tell you with confidence we have many. If the top 10 wealthy men in Afghanistan—I would say 9 of them products of the past 10 years—came together, they could buy this government, the bank, this whole system.” - - + + Afghanistan’s megarich are not shy about their wealth. Many are driven around in $150,000 armored vehicles, trailed by convoys of cars and pickup trucks full of security guards. Several live in Wazir Akbar Khan, Kabul’s diplomatic enclave, but others have illegally carved up Sherpur, an historic hill district in the capital. Some have second homes in Dubai, Istanbul, or various European cities. Just like Russia’s oligarchs, many members of this wealthy class owe their fortunes to politics. Some are warlords who helped the U.S. topple the Taliban; others are technocrats who returned from abroad to work in the new government. Both groups enriched themselves through the country’s system of patronage and influence—and by drawing on the immense sums of American cash flowing into it. Afghanistan runs on connections, and many of the biggest dealmakers operate with impunity. A clan can have one brother in the administration, another in parliament, and yet another running a huge company or state enterprise. The family of former President Hamid Karzai was the object of much criticism for that reason. @@ -34,9 +36,9 @@ There’s still money to be made from the American military—though it’s a mu The new president, Ashraf Ghani, is promising to bring order to procurement and contracting, but transparency may be difficult to achieve. Among Ghani’s first appointments was Hazrat Omar Zakhilwal, a finance minister under Karzai who was entangled in the country’s biggest banking scandal, among other controversies. Zakhilwal, who denies any wrongdoing, now has oversight over the entire financial portfolio of the country. - - + + In 2009, Matie, then in his late 20s, trundled up to the Marine compound on a donkey, after treading slowly through a heavily mined field. “Out of control mines,” he recalls. The son of a religious studies teacher, he had already tried many jobs, including joining the Taliban, twice. He had been working as a customer-care representative—making a lucrative $300 a month—with one of the new telecom companies when he realized he wanted to start his own company. He was sitting through a business development training seminar conducted by Malaysians when he thought to himself, “I want to be my own boss.” He quit and got a license to start a construction company. He wrote up a company profile and fact sheet—as the Malaysians had taught him—and, two weeks before Ramadan, put the papers in a saddle, mounted his donkey, and headed for the Marines in Nawa. “Hi, sir! Is there anybody to talk to me?” he’d shouted in his elementary English at a Marine manning a watchtower. They were happy to let him in. @@ -44,6 +46,8 @@ The Marines were part of Obama’s surge to push back a Taliban onslaught that t The Marines had cash and lots of it. Congress has appropriated about $3.7 billion over the past 10 years for the Commanders Emergency Response Program (CERP), a fund that officers in Afghanistan and Iraq could draw on for “urgent humanitarian relief and reconstruction requirements in their areas of responsibility.” In Helmand province—one of the areas fiercely contested with the Taliban—the U.S. military would spend $153 million on 2,164 CERP projects. USAID also poured money into the area through foreign contractors who implemented so-called stabilization projects—such as rebuilding bazaars and supplying technology to district offices. For example, according to the Washington Post, USAID spent $30 million on agriculture in Nawa over the course of nine months in 2010. All of this created opportunities for enterprising Afghans such as Matie. +“My only wish is to prove to the American public that in my case your tax money has not been wasted” + His first project was the reconstruction of the district governor’s office. It needed new doors, windows, fresh plaster—and walls. When the Marine captain proposed the project to him, Matie calculated an estimate on the spot: $10,109. He asked for five days to get things going. Only later did he realize what he’d agreed to do. The area between Lashkar Gah and Nawa was strictly Taliban country—and Matie had to transport gravel, shovels, and barrels from the provincial capital to the Marine compound. So when Matie recruited laborers, he didn’t tell them he was sending them to Nawa but rather to another, safer district nearby. “I went ahead on a motorcycle,” he recalls. “When they got here, I said, ‘Don’t you worry. I will give you more money than you want. As much as you want.’ ” The workers stayed for the entire 15 days of the project. @@ -60,11 +64,11 @@ Because the prosperity of Matie’s newly rich class often stems from loose Amer Corruption is pervasive and visible: the flashy car belonging to a tax clerk whose monthly salary is $200; the fancy bungalow of a precinct police chief. Money purchases status, buys protection—from the law and perhaps even from God, judging by the number of mosques built with ill-gotten funds and the many hajj pilgrimages financed by dirty money. “Has corruption become what holds everything together in Afghanistan?” one Western official asks. “Maybe.” -Unlike some of the extremely rich—who put their profits into foreign bank accounts—the entrepreneurial class tends to keep much of its cash within the country’s borders. “There’s everything-to-myself corruption,” the same official explains, “and then there’s this Tammany Hall kind of corruption, a sort of Robin Hood style, where you are generous to the community.” +Unlike some of the extremely rich—who put their profits into foreign bank accounts—the entrepreneurial class tends to keep much of its cash within the country’s borders. “There’s everything-to-myself corruption,” the same official explains, “and then there’s this Tammany Hall kind of corruption, a sort of Robin Hood style, where you are generous to the community.” + + - - Hikmatullah Shadman doesn’t look like Robin Hood, though the American investigators have grave suspicions about him. He works in what used to be the Kabul home of Ahmad Zahir, a legendary entertainer known as the Afghan Elvis. Shadman, 29, likes flowers. His pastel-colored compound in Wazir Akbar Khan looks like a dollhouse, with plastic flowers strung across the ceiling and framing paintings, mirrors, and photographs. “Flowers make me happy,” Shadman says as he sits down for an interview about his businesses and philanthropies—which are mainly in Kandahar, almost 300 miles to the south. He wears a black sports jacket over a black tunic embroidered in silver. On the table in front of us are platters of dried fruit and bottles of Gatorade, Starbucks frappuccino, and Ocean Spray cranberry juice. A cleanshaven elderly man—whom Shadman refers to as mama, or maternal uncle—is thumbing his prayer beads while lounging on a couch to the businessman’s right. @@ -74,13 +78,15 @@ Shadman also traces his riches back to U.S. military money. The son of a literat In a couple of years, he’d purchased hundreds of vehicles, renting them to the U.S. military and foreign contractors who came to Afghanistan. He also started doing construction projects for Canadian units that were part of the International Security Assistance Force, as the U.S. and its allied troops were called. His Special Forces bosses helped him with connections that got him contracts to supply propane to NATO bases in the south. Shadman’s main line of business, however, became trucking. Working first as a middleman for a Hungarian firm that provided the exuberantly decorated “jingle trucks” for ferrying goods throughout the country, Shadman quickly built a fleet of his own vehicles. Profits escalated with the U.S. surge. According to court documents, he carried out 5,421 transport missions for the ISAF. +“Someone threw us a rope to pull us up. But we somehow got all strangled up in that rope” + Shadman is accused of defrauding the U.S. government of $77 million. According to court documents, SIGAR alleges that Shadman managed to expand his trucking empire only because he “bribed and paid kickbacks” to managers of the Hungarian contractor, who then allegedly inflated prices for Shadman so he could charge the ISAF even more. In October 2012, at 4:30 a.m., the U.S. military raided his compound in Kandahar. He says they flashed a light in his eyes, blindfolded him, tied his hands, and flew him to the prison at the American military base at Bagram. He was held there for 74 days and accused of funding the enemy and supplying women to the Taliban and alcohol to U.S. soldiers. In a civil forfeiture lawsuit, SIGAR and the U.S. Department of Justice asked for Shadman’s accounts in an Afghan bank to be frozen. However, they were quickly unfrozen by Afghan authorities and some of the money has made its way to Dubai, where he has three homes. Shadman says he is heartbroken by the way the Americans turned against him. “I grew up with them, with their soldiers.” He insists he’s not afraid of litigation, because the evidence against him is flimsy. “My only wish is to prove to the American public that … in my case your tax money has not been wasted.” Then he turns from being politic to blunt. “My money is clean. I don’t hide it. It’s there in the open, for America to see it, for London to see it. I have no fear.” - - + + Shadman has been able, so far, to withstand the legal assault on his reputation. While he no longer has any contracts with the U.S. military, he imports German energy drinks, which are extremely popular among young Afghans. He is planning to build a pomegranate juice factory in Kandahar. diff --git a/tests/data/text/businessweek.com2.txt b/tests/data/text/businessweek.com2.txt index 91c72154..91e751bf 100644 --- a/tests/data/text/businessweek.com2.txt +++ b/tests/data/text/businessweek.com2.txt @@ -1,5 +1,9 @@ +Investing + New York Governor Andrew Cuomo vetoed a bill last week that would have allowed the New York state, city, and teachers’ pension funds to increase their investment in hedge funds from 25 percent to 30 percent of fund assets. He cited the high fees and risk associated with hedge funds. The now-dead bill contained a memo justifying its proposed increase: +“The portion of the NYCRS [New York City Retirement System] portfolios allocated to public equities is much more volatile than the investments allocated to the basket. As a result, a swing in public markets can push NYCRS dangerously close to the investment cap with no new investments.” + That’s a terrible reason to invest in hedge funds. Stocks are supposed to be riskier; that’s why they normally provide higher returns. If the investment board can’t handle volatility, it should invest in safer, lower-yielding assets—and while that may describe some hedge funds, cheaper options are out there. Hedge funds typically charge 2 percent of assets, plus 20 percent of gains, and rarely outperform the stock market. The figure below, from an article last year, plots an index of hedge fund strategy performance compared with the S&P 500 index. Underperformance is why such states as California are pulling out of hedge funds (though they are piling into risky alternatives, such as private equity). The sentiment is not uniform, though, and other states, including New Jersey, Ohio, New Mexico, and Illinois, are sticking with or increasing their hedge fund investment. diff --git a/tests/data/text/chinese.txt b/tests/data/text/chinese.txt index 42e12013..0c59916b 100644 --- a/tests/data/text/chinese.txt +++ b/tests/data/text/chinese.txt @@ -1,11 +1,17 @@ 香港行政长官梁振英在各方压力下就其大宅的违章建筑（僭建）问题到立法会接受质询，并向香港民众道歉。 +梁振英此前承认早在去年参选行政长官之前就已知悉其住宅的违建问题，引发诚信危机。 + 梁振英在星期二（12月10日）的答问大会开始之际在其演说中道歉，但强调他在违章建筑问题上没有隐瞒的意图和动机。 +不过泛民主派议员们普遍指责梁振英“撒谎”，要求他马上辞职下台。 + 一些亲北京阵营议员欢迎梁振英道歉，且认为应能获得香港民众接受，但这些议员也质问梁振英有否向执法部门施压。 梁振英强调承诺将在两周内解决其住宅的违建问题。 +“郑重道歉” + 香港媒体于6月份曝光梁振英大宅的首批违建部分后，于3月的选举中被击败的民主党参选人何俊仁向法院提出选举呈请，至11月中旬被终审法院驳回。 梁振英说，其位于太平山山顶的住宅内的违建部分大都不是由他所建，此前没有马上公开交待和处理，是因为律师意见认为司法程序仍在进行，他不应评论。 @@ -18,4 +24,6 @@ 现为间选议员的何俊仁说，梁振英至今仍不坦诚以对，让他“震惊”；亲北京政团工联会直选议员黄国健也批评梁振英抱着“不服输”的态度接受质询。 -历时1.5小时的答问大会在进入中段之际，泛民主派人民力量的直选议员黄毓民、陈伟业和社会民主连线的梁国雄先后因播放录音、叫嚣，和向梁振英扔掷文件而被议长驱逐。 \ No newline at end of file +历时1.5小时的答问大会在进入中段之际，泛民主派人民力量的直选议员黄毓民、陈伟业和社会民主连线的梁国雄先后因播放录音、叫嚣，和向梁振英扔掷文件而被议长驱逐。 + +民主党此前计划在星期三（11日）对梁振英提出不信任动议。 \ No newline at end of file diff --git a/tests/data/text/cleveland.com1.txt b/tests/data/text/cleveland.com1.txt index bfbf0b4f..9e00f027 100644 --- a/tests/data/text/cleveland.com1.txt +++ b/tests/data/text/cleveland.com1.txt @@ -3,14 +3,25 @@ Chia seeds and goji berries are the new kale and quinoa, according to Google's r Some of the other more interesting food-related data points include: Pizza was searched more than the World Cup. + The Cronut rose to 17th on the global recipe list after its arrival last year. + Our favorite ways to eat eggs are: 1) Deviled, 2) Scotch, 3) Scrambled, 4) Pickled, 5) Boiled. + This year we searched for 'recipes' less and 'restaurant' significantly more. + Our top slimming questions were 'how many calories should i eat in a day' and 'how to lose weight,' and the Paleo diet was the top searched way to trim down. + Foodies in Japan searched French food more than France. -Hungry folk in Australia searched Argentine food more than Argentina. + +Hungry folk in Australia searched Argentine food more than Argentina. + Spice-loving Brits searched Indian food more than India. + In 2014 'i am hungry' was searched a button-popping 7x more than 'i am thirsty.' + Oh, and nine million people watched a tiny hampster eating a tiny burrito. The lesson, as always: you are what you search. + +-- By Michael Russell Follow @tdmrussell \ No newline at end of file diff --git a/tests/data/text/cleveland.com2.txt b/tests/data/text/cleveland.com2.txt index 5a9e9e39..6f6864ba 100644 --- a/tests/data/text/cleveland.com2.txt +++ b/tests/data/text/cleveland.com2.txt @@ -1,13 +1,14 @@ -The Pilot and the Little Prince +PICTURE BOOKS -Peter Sís - Foster Books, $18.99; ages 5-8 -This exquisitely illustrated biography recounts the life of writer and aviator Antoine de Saint-Exupéry. Lovers of his classic tale "The Little Prince" will find that the masterful Sís has captured the wonder and dreaminess evoked in that book. -- Karen Sandstrom +The Pilot and the Little Prince -Goodrich finds hilarity and drama in "Mister Bud Wears the Cone," a story of one dog's humility in the confines of a plastic head cone. -- Sandstrom +Peter Sís +Foster Books, $18.99; ages 5-8 + +This exquisitely illustrated biography recounts the life of writer and aviator Antoine de Saint-Exupéry. Lovers of his classic tale "The Little Prince" will find that the masterful Sís has captured the wonder and dreaminess evoked in that book. -- Karen Sandstrom Mister Bud Wears the Cone @@ -18,17 +19,15 @@ Simon & Schuster, $16.99; ages 4-8 Goodrich finds hilarity and drama in "Mister Bud Wears the Cone," a story of one dog's humility in the confines of a plastic head cone. -- Sandstrom - Abuelo + +Abuelo Arthur Dorros; illustrated by Raúl Colón - Harper, $17.99; ages 6-8 +Harper, $17.99; ages 6-8 A young boy learns a lot from his grandfather in the country as they camp and ride horses under the big sky. The abuelo also teaches his grandson the Spanish words for important ideas. Colón's rich illustrations make each page dazzle. -- Sandstrom -A girl and her mouse keep their friendship a secret from their parents in this gorgeously illustrated look at parallel lives. You'll marvel over the details McClintock gets just right as she lays out the two worlds in a soft, golden palette. -- Sandstrom - - Where's My Mommy? Beverly Donofrio; illustrated by Barbara McClintock @@ -38,38 +37,115 @@ Schwartz & Wade, $17.99; ages 3-7 A girl and her mouse keep their friendship a secret from their parents in this gorgeously illustrated look at parallel lives. You'll marvel over the details McClintock gets just right as she lays out the two worlds in a soft, golden palette. -- Sandstrom - President Taft is Stuck in the Bath + +President Taft is Stuck in the Bath Mac Barnett; illustrated by Chris Van Dusen - Candlewick Press, $16.99; ages 4-8 +Candlewick Press, $16.99; ages 4-8 The story goes that our horizontally challenged 27th president once became trapped by in a bathtub. Branett's words and Van Dusen's silky smooth illustrations have the most possible fun with the idea, yet remain respectful. Historical notes at the end separate fact from fiction. -- Sandstrom +MIDDLE-GRADE BOOKS + + + +Brown Girl Dreaming + +Jacqueline Woodson + +Nancy Paulsen Books, $16.99; ages 10 and up + Woodson won the National Book Award for this childhood memoir in verse. It begins with Woodson's birth in 1963. "I am born on a Tuesday at University Hospital / Columbus, Ohio, / USA – / a country caught / between Black and White." A wonderful model for young readers wanting to write their own family stories. -- Tricia Springstubb +West of the Moon + +Margi Preus + +Amulet Books, $16.95; ages 10-14 + Weaving Norwegian folklore with actual hardships faced by 19th-century immigrants, Preus spins a riveting tale of love, sin, remorse, forgiveness and how stories give us courage. -- Springstubb +The Crossover + +Kwame Alexander + +Houghton Mifflin Harcourt, $16.99; ages 9-12 + Sports and life intertwine in this electric celebration of the game and of the power of brotherhood and family. The language sizzles and pops, ranging from free verse to hip-hop to poems for two voices. -- Springstubb +Ophelia and the Marvelous Boy + +Karen Foxlee + +Knopf, $16.99; ages 8-12 + Behind a museum door, Ophelia discovers a small boy with an unbelievable tale: Wizards have chosen him to defeat the wicked Snow Queen, and he needs help. Ophelia finds herself bravely battling evil, not just to save him, but her beloved father and sister, too, in this melancholy and beautiful book. -- Springstubb +The Meaning of Maggie + +Megan J. Sovern + +Chronicle Books, 220 pp., $16.99; ages 8-12 + Maggie is smart, self-centered and friendless – not that she minds one bit. Her own brain and ambition are company enough. Resolving to fix her father's illness, she discovers multiple sclerosis has no cure. Maggie's family copes with courage, humor and love, and readers will not only root for them, but believe in them. -- Springstubb +Revolution + +Deborah Wiles + +Scholastic, $19.99; ages 8-12 + Mississippi, 1964. Sunny and her stepbrother sneak into their town's segregated pool one hot night. To their surprise, a "colored" boy is there, too. Wiles chronicles the Freedom Summer through a mixture of fiction and documentary, including photos, excerpts of speeches and song lyrics. -- Springstubb The Family Romanov: Murder, Rebellion, and the Fall of Imperial Russia +Candace Fleming + +Schwartz and Wade, $ 18.99, ages 9-12 + No fiction is more fantastic than the true story of foolish Nicholas, fanatic Anastasia, and their five beautiful children. Geared toward middle-school readers, this superb account will also satisfy adults looking for an overview of this period of Russian history, when impoverished workers revolted, soldiers died by the millions, Lenin was on the rise, and the tsar, secluded in his opulent country palace, trusted in God to save his throne. -- Springstubb +YOUNG ADULT BOOKS + + + Why We Took the Car +Wolfgang Herrndorf + +Arthur A. Levine Books, 256 pp., $17.99 + Boys (and girls, too) will love this joy ride of a story, featuring German teens Mike and Tschick, who "borrow" a car to escape the boredom of summer break. Adventures – and misadventures – ensue, until they end up in police custody. Fasten your seatbelt for a rollicking read. -- Susan Glaser +We Were Liars + +E. Lockhart + +Delacorte, 240 pp., $20.99 + The protagonist of this story is Cady Sinclair, a member of an upper-crust New England family that spends its summers on a private island off the coast of Massachusetts. Cady, together with same-age cousins Johnny and Mirren and love interest Gat Patil, make up "the Liars," tightknit teens who share adventures – and one big surprise. At age 17, Cady returns to Beechwood Island to piece together the details of an accident two years prior. The result: An ending you won't see coming. -- Glaser +I'll Give You the Sun + +Jandy Nelson + +Dial Books, 384 pp., $17.99 + Twins Noah and Jude are the narrators of this tale, which alternates between their 13th and 16th years. In between: Their mother dies, their lives are upended and the two struggle to make peace with each other and the people around them. Their paths come back together at the end – but the real story is in the journey. -- Glaser +The Impossible Knife of Memory + +Laurie Halse Anderson + +Viking, 391 pp., $18.99 + Anderson, who has tackled date rape and eating disorders in previous works, turns her attention to post-traumatic stress disorder in this story about Hayley Kincain and her Iraq War vet father, Andy. Hoping for stability, the two return to Andy's hometown in upstate New York for Hayley's senior year of high school. Instead, the two are forced to deal with more chaos, including Andy's increasingly erratic behavior. A budding romance between Hayley and fellow high schooler Finn helps lighten the mood. Even so, Anderson maintains her status as one of the most serious storytellers in the young adult genre. -- Glaser -Imagine that a second girl from Kansas was swept away to Oz after Dorothy left. But when Amy arrives, Oz is falling apart. The color is dull, and a huge hole in the middle of the land is getting bigger. What has happened to Oz? Short answer: Dorothy. Now it is up to Amy to fix things, and decide which of her unlikely companions are the most trustworthy. Is it the order of the wicked witches or the wingless flying monkeys, or both? Or are there other players in this game that we don't know about yet? I loved this book. -- Ami Bray, Loganberry Books +Dorothy Must Die + +Danielle Paige + +HarperCollins, 464 pp., $17.99 + +Imagine that a second girl from Kansas was swept away to Oz after Dorothy left. But when Amy arrives, Oz is falling apart. The color is dull, and a huge hole in the middle of the land is getting bigger. What has happened to Oz? Short answer: Dorothy. Now it is up to Amy to fix things, and decide which of her unlikely companions are the most trustworthy. Is it the order of the wicked witches or the wingless flying monkeys, or both? Or are there other players in this game that we don't know about yet? I loved this book. -- Ami Bray, Loganberry Books \ No newline at end of file diff --git a/tests/data/text/cnn.txt b/tests/data/text/cnn.txt index 922b4680..819fdb05 100644 --- a/tests/data/text/cnn.txt +++ b/tests/data/text/cnn.txt @@ -22,6 +22,8 @@ Brian M. Good said he, too, was expecting a horrible trip when he departed New Y "Instead the roads were dead," he said in an e-mail. "It's warm outside and it stopped raining. No lines at the airport and flight is on time. Wish the forecasters were wrong all the time :)" +Will winds whip parade balloons? + Though the worst of the storm has passed, winds could still pose a problem. The forecast has left up in the air the fate of the balloons in Macy's Thanksgiving Day Parade. They are to be grounded if sustained winds reach 23 mph or gusts exceed 34 mph -- both slightly above predicted strength. @@ -42,8 +44,12 @@ Officials say that improved weather monitoring devices en route and a police ser Either way, the parade -- with or without the balloons -- will go on, organizers say. +Planes, trains ... + Delays were reported at several airports in the Northeast on Wednesday. +Don't get trapped by bad Thanksgiving weather: Top tips + Heavy winds at all three New York-area airports -- LaGuardia, John F. Kennedy and Newark-Liberty -- resulted in delays of 30 minutes to an hour, according to the Federal Aviation Administration. Departure delays from Philadelphia International Airport averaged nearly two hours. American Airlines' Kent Powell reported three cancellations, none of them related to the weather. @@ -60,6 +66,12 @@ Amtrak reported no major delays systemwide. Using the weather as a marketing too There may be something to that. +"QUIET CAR. Window seat. Polite seatmate. I have hit the Amtrak travel trifecta. #blessed," Ellie Hall tweeted early Wednesday. + +5 healthy eating tips for holiday travelers + +... and automobiles + Snow blanketed parts of the Midwest, where crews scrambled to clear roads. The storm was blamed for scores of accidents. Up to a foot of snow fell in parts of Pennsylvania, and it was falling from upstate New York into Canada, where more than a foot was possible. Snow also continued to fly in the central Appalachians and around the Great Lakes as cold air moved in and produced lake-effect snows. @@ -84,4 +96,6 @@ The Peterson family had initially planned to drive from Northern Virginia to Mas "It was a small fortune," Jennifer Peterson told CNN affiliate WUSA. "We could've gone to the Bahamas for what we paid!" +Storm prompts airlines to relax travel policies + CNN's Jason Carroll, Ben Brumfield, Dave Hennen, Aaron Cooper, Alexandra Field, Shannon Travis and Greg Botelho contributed to this report. \ No newline at end of file diff --git a/tests/data/text/cnn_summary.txt b/tests/data/text/cnn_summary.txt index fd2e9dc0..1723fdd6 100644 --- a/tests/data/text/cnn_summary.txt +++ b/tests/data/text/cnn_summary.txt @@ -1,5 +1,5 @@ -Wish the forecasters were wrong all the time :)"Though the worst of the storm has passed, winds could still pose a problem. -Forecasters see mostly smooth sailing into Thanksgiving. -The forecast has left up in the air the fate of the balloons in Macy's Thanksgiving Day Parade. -That's good news for people like Latasha Abney, who joined the more than 43 million Americans expected by AAA to travel over the Thanksgiving holiday weekend. -"I was second in line checking my bag with Delta (checked into my flight last night) and security was a breeze. \ No newline at end of file +The storm caused some complications and inconveniences, but no major delays or breakdowns. +Forecasters see mostly smooth sailing into Thanksgiving. +That's good news for people like Latasha Abney, who joined the more than 43 million Americans expected by AAA to travel over the Thanksgiving holiday weekend. +Though the worst of the storm has passed, winds could still pose a problem. +The forecast has left up in the air the fate of the balloons in Macy's Thanksgiving Day Parade. \ No newline at end of file diff --git a/tests/data/text/coolhunting.com1.txt b/tests/data/text/coolhunting.com1.txt index 5bc05dc5..5eb35e2e 100644 --- a/tests/data/text/coolhunting.com1.txt +++ b/tests/data/text/coolhunting.com1.txt @@ -1,3 +1,5 @@ +by Kelly Phillips Badal + Vacationing in a former convent may sound a bit austere—that is, until you experience Monastero Santa Rosa on Italy’s famous Amalfi Coast. This monastery-turned-hotel offers all the peaceful seclusion the sisters enjoyed, with none of the personal sacrifices. And though the popular coastline is studded with hotels, this distinctive destination—with its unmatched combination of five-star luxuries, natural beauty and centuries of character—is a serious standout. Getting there might have you pressing your palms together in supplication. The main route through the craggy Amalfi coastline is Strada Statale 163—also known as "the road of 1,000 bends." The ancient Roman-built drive is full of hairpin twists and turns, careening over steep precipices past sun-bleached villages and lemon groves for 50 miles, and is just barely wide enough for two cars to pass at some points. About 18 miles in sits Monastero Santa Rosa, perched on a cliff above the small fishing village of Conca dei Marini. The venue's American owner Bianca Sharma spotted the nunnery’s ruins from a boat in 1999 and promptly bought the property. After a lengthy decade-long restoration, it opened as a luxury hotel for the 2012 season. diff --git a/tests/data/text/coolhunting.com2.txt b/tests/data/text/coolhunting.com2.txt deleted file mode 100644 index 5bc05dc5..00000000 --- a/tests/data/text/coolhunting.com2.txt +++ /dev/null @@ -1,15 +0,0 @@ -Vacationing in a former convent may sound a bit austere—that is, until you experience Monastero Santa Rosa on Italy’s famous Amalfi Coast. This monastery-turned-hotel offers all the peaceful seclusion the sisters enjoyed, with none of the personal sacrifices. And though the popular coastline is studded with hotels, this distinctive destination—with its unmatched combination of five-star luxuries, natural beauty and centuries of character—is a serious standout. - -Getting there might have you pressing your palms together in supplication. The main route through the craggy Amalfi coastline is Strada Statale 163—also known as "the road of 1,000 bends." The ancient Roman-built drive is full of hairpin twists and turns, careening over steep precipices past sun-bleached villages and lemon groves for 50 miles, and is just barely wide enough for two cars to pass at some points. About 18 miles in sits Monastero Santa Rosa, perched on a cliff above the small fishing village of Conca dei Marini. The venue's American owner Bianca Sharma spotted the nunnery’s ruins from a boat in 1999 and promptly bought the property. After a lengthy decade-long restoration, it opened as a luxury hotel for the 2012 season. - -Much of the convent’s original 17th century architecture remains intact, thanks to Sharma’s careful conservation. Inside, reconfigured spaces pay thoughtful homage to the hotel’s heritage. Nuns' quarters are cleverly combined into 20 unique rooms, some with private terraces or multiple levels. One of the largest is located in what was the sisters’ refectory. All feature fine Italian linens, an array of tasteful period furniture, and deep soaking tubs made from fine Jerusalem stone, in addition to everyday contemporary amenities like minibars, television, and WiFi. Some of the rooms have vaulted ceilings, private alfresco dining areas, and seafront balconies; others have housed famous names like Prince Albert and Princess Charlene of Monaco. The guest rooms exude comfort, and every conceivable wish is quickly taken care of by the attentive staff. - -The tasteful restoration continues into Monastero Santa Rosa’s massive 750-foot state-of-the-art spa. Cavernous treatment rooms are carved from spaces once devoted to silent prayers, while the spa’s centerpiece, a vaulted tepidarium (warm relaxation room), is where the sisters made wine. There’s also a sauna, steam room, hydrotherapy pool, whirlpool footbaths, ice fountain and even programmable showers. - -The real star of Monastero Santa Rosa is the knockout coastal view, and there’s a countless number of magical places for taking it in. From the convent’s highest point, an airy sunset terrace, guests can gaze as far as the buzzy nearby towns of Amalfi or Positano. There’s the cascade of tiered, expertly manicured semi-tropical gardens, beset with ocean-facing sun loungers, daybeds and cabanas. There’s the breezy dining terrace of the hotel Ristorante, where guests can enjoy the scenery while tasting exclusively sourced extra virgin olive oil produced right in Conca dei Marini. The most breathtaking spot is floating within the curved cliff’s edge infinity pool while surveying the Gulf of Salerno some 660-feet below. - -After whiling away the hours on these sacred grounds, be sure to unburden your soul at the large antique wooden confessional resting in the main hallway. It cheekily invites guests “make a confession” by way of a written feedback for the hotel. - -Monastero Santa Rosa is located in the town of Conca dei Marini on Italy’s Amalfi Coast. Nightly room and suite rates range from $500 to $2,400. The resort will open for the 2015 season on 17 April. - -See more photos in the gallery, images by Tanveer Badal \ No newline at end of file diff --git a/tests/data/text/cricket.com.au1.txt b/tests/data/text/cricket.com.au1.txt index 8e6e776b..52e24bf4 100644 --- a/tests/data/text/cricket.com.au1.txt +++ b/tests/data/text/cricket.com.au1.txt @@ -1,3 +1,5 @@ +CommBank Test Series v India + Heroics from Johnson, Harris not enough but Australia still regain Border-Gavaskar Trophy The series win that brings the Border-Gavaskar Trophy back to Australia after almost two years in India arrived courtesy of a mutually agreed draw rather than the emphatic statement the home team seemed primed to deliver. diff --git a/tests/data/text/cricket.com.au2.txt b/tests/data/text/cricket.com.au2.txt index b6bf9a86..dae11166 100644 --- a/tests/data/text/cricket.com.au2.txt +++ b/tests/data/text/cricket.com.au2.txt @@ -1 +1 @@ -The Brisbane Heat's Ryan Duffield had his second wicket in as many balls when Glenn Maxwell left a ball swinging back into the stumps \ No newline at end of file +Is there nothing Andrew Flintoff can't do? Here he commentates the final ball of the Brisbane Heat's innings while batting \ No newline at end of file diff --git a/tests/data/text/dailycaller.com1.txt b/tests/data/text/dailycaller.com1.txt index 0b5e2101..7176786d 100644 --- a/tests/data/text/dailycaller.com1.txt +++ b/tests/data/text/dailycaller.com1.txt @@ -1,3 +1,7 @@ +Ann Coulter + +4419111 + I will not be shooting any Black Panthers this week because I am Kwanza-reform, and we are not that observant. Kwanzaa, celebrated exclusively by white liberals, is a fake holiday invented in 1966 by black radical/FBI stooge, Ron Karenga — aka Dr. Maulana Karenga, founder of United Slaves, the violent nationalist rival to the Black Panthers. In the annals of the American ’60s, Karenga was the Father Gapon, pawn of the czarist police. In what was ultimately a foolish gambit, during the madness of the ’60s, the FBI encouraged the most extreme black nationalist organizations in order to discredit and split the left. The more preposterous the group, the better. By that criterion, Karenga’s United Slaves was perfect. @@ -18,6 +22,8 @@ In a 1995 interview with Ethnic NewsWatch, Karenga matter-of-factly explained th Now we know: The FBI fueled the bloody rivalry between the Panthers and United Slaves. In one barbarous outburst, Karenga’s United Slaves shot two Black Panthers to death on the UCLA campus: Al “Bunchy” Carter and John Huggins. Karenga himself served time, a useful stepping-stone for his current position as a black studies professor at California State University at Long Beach. +(Speaking of which, Rep. Paul Ryan certainly is right about what a fantastic job his mentor Jack Kemp did reaching out to all those “socially conservative” minorities. Look at how California has swung decisively to the right since Kemp started all that outreach stuff. Good luck winning California now, Democrats!) + Back to the esteemed Cal State professor: Karenga’s invented holiday is a nutty blend of schmaltzy ’60s rhetoric, black racism and Marxism. The seven principles of Kwanzaa are the very same seven principles of the Symbionese Liberation Army, another invention of the Worst Generation. In 1974, Patty Hearst, kidnap victim-cum-SLA revolutionary, posed next to the banner of her alleged captors, a seven-headed cobra. Each snakehead stood for one of the SLA’s revolutionary principles: Umoja, Kujichagulia, Ujima, Ujamaa, Nia, Kuumba and Imani. These are the exact same seven “principles” of Kwanzaa. And here’s something interesting: Kawaida, Kwanzaa and Kuumba are also the only three Kardashian sisters not to have their own shows on the E! Network. diff --git a/tests/data/text/dailycaller.com2.txt b/tests/data/text/dailycaller.com2.txt index f1d1bc9a..17ecfcd7 100644 --- a/tests/data/text/dailycaller.com2.txt +++ b/tests/data/text/dailycaller.com2.txt @@ -1,13 +1,31 @@ +Sports + Dec 27, 2014; Louisville, KY, USA; Louisville Cardinals forward Montrezl Harrell (24) posts up against Kentucky Wildcats guard Aaron Harrison (2) during the first half at KFC Yum! Center. Mandatory Credit: Jamie Rhodes-USA TODAY Sports - RTR4JDW4 +4419915 + The Louisville Cardinals hosted the Kentucky Wildcats on Saturday for one of the 2014 season’s most highly anticipated college basketball games. Rick Pitino’s 4th ranked Cardinals have had a great start to the season, but they were still huge underdogs against John Calipari’s unbeaten powerhouse. (RELATED: Kentucky Basketball’s Christmas Card Must Be Pretty Intimidating For Opposing Teams) Louisville trailed by 4 points at halftime, but they couldn’t close the gap thanks to plays like this. +WATCH: + Slow it down one time. +What a joke. + In games like these, I usually cheer for the underdog, but that flop had me wanting Kentucky to win by fifty. (RELATED: Forget ISIS, China And Immigrants; Flopping Is Threatening To Destroy America) -Like the rest of their games this season, the Wildcats eventually pulled away from their cross-state rival. UK won by a final score of 58-50. \ No newline at end of file +Like the rest of their games this season, the Wildcats eventually pulled away from their cross-state rival. UK won by a final score of 58-50. + +On a side note, apparently Jennifer Lawrence is a Cardinals fan. + +NEW: Jennifer Lawrence at the Louisville Cardinals basketball game today (Dec. 27) pic.twitter.com/soxnfJGWwd — Jennifer Lawrence (@JenniferUpdates) December 27, 2014 + +"May the odds be ever in Louisville's favor!" – Jennifer Lawrence http://t.co/rE6Li6QWGp pic.twitter.com/zK8QMCzPDf — Sporting News (@sportingnews) December 27, 2014 + +I love you too, J. Law. + +Follow Datoc on Twitter \ No newline at end of file diff --git a/tests/data/text/dailystar.co.uk1.txt b/tests/data/text/dailystar.co.uk1.txt index 0b765474..2f8a2449 100644 --- a/tests/data/text/dailystar.co.uk1.txt +++ b/tests/data/text/dailystar.co.uk1.txt @@ -1,57 +1 @@ -The Scottish nurse arrived at the Royal Free Hospital in Hampstead, north London, for specialist treatment at the infectious diseases unit, surrounded by a convoy of ambulances and police cars. - -The woman returned to Scotland after working in the west African country and arrived on a British Airways flight at Glasgow airport at around 11.30pm on Sunday. - -The patient – who is understood to have been volunteering for Save The Children – was said to be displaying no symptoms on the plane and the risk to other patients is said to be "low". - -Public health officials are said to be looking to contact passengers who sat adjacent to the nurse as a precautionary approach. - -Meanwhile, another person is being tested in north west Scotland for the deadly virus but is said to be "low risk". - -Another patient, in Truro, Cornwall, has been placed in isolation and is also being tested for Ebola. - -The Scottish nurse was admitted to hospital early on Monday after feeling unwell. She is in a stable condition. - -She was accompanied to hospital by a convoy of ambulances and police cars as workers wore full protective suits. - -A tent has been set up around her bed so the infection can be contained while she is treated. - -The unit in London successfully treated British nurse WIlliam Pooley who was flown home from Sierra Leone in August after contracting the virus. - -Scotland's First Minister Nicola Sturgeon said: "Our first thoughts at this time must be with the patient diagnosed with Ebola and their friends and family. I wish them a speedy recovery. - -"Scotland has been preparing for this possibility from the beginning of the outbreak in West Africa and I am confident that we are well prepared. - -"We have the robust procedures in place to identify cases rapidly. Our health service also has the expertise and facilities to ensure that confirmed Ebola cases such as this are contained and isolated effectively minimising any potential spread of the disease. - -"Scotland’s NHS has proved it is well able to cope with infectious diseases in the past, such as swine flu, and I am confident we will be able to respond effectively again.” - -She has also spoken to Prime Minister David Cameron. - -Health Secretary Jeremy Hunt, who chaired an emergency Cobra meeting, vowed the Government was doing "absolutely everything" to keep the public safe. - -"We are reviewing our procedures and protocols for the other NHS workers who are working in Sierra Leone alongside colleagues from the Department for International Development and the Foreign Office," he said. - -"They are doing a very, very brave job, under very challenging circumstances. - -"We want to make absolutely sure that we are doing everything we can to keep them safe. - -"The clinical advice is that the risk is very low to other passengers. She wasn't exhibiting feverish symptoms. - -"The process has worked well because the moment she did exhibit those symptoms we were able to take her into isolation." - -A British Airways spokesman said: "We are working closely with the health authorities in England and Scotland and will offer assistance with any information they require. - -"Customers who flew from London Heathrow to Glasgow on BA1478 which departed at 2100 on Sunday December 28 and have concerns should contact the special number 08000 858531 set up by the Scottish Government. - -"The safety and security of our customers and crew is always our top priority and the risk to people on board that individual flight is extremely low." - -The current Ebola outbreak, which began in West Africa, is the largest epidemic of the virus in history. - -It began in Guinea in December 2013 and quickly spread to Liberia and Sierra Leone – the two worst-affected nations. - -The World Health Organisation (WHO) has reported a total of 19,980 suspected cases and 7,793 deaths. - -But the WHO believes these figures understate the true size of the deadly outbreak. - -Earlier this month Time magazine named those healthcare workers who have travelled to help with the crisis as their "Person of the Year". +ONLY EXTRACTS ARTICLE UP UNTIL THE FIRST IMAGE diff --git a/tests/data/text/dailystar.co.uk2.txt b/tests/data/text/dailystar.co.uk2.txt index 690016ea..59c71245 100644 --- a/tests/data/text/dailystar.co.uk2.txt +++ b/tests/data/text/dailystar.co.uk2.txt @@ -1,11 +1 @@ -Merry belated Christmas, 'tis the season to be jolly 'nd all that. And 'tis the season for all the dirty dawgs to get up to no good. - -The holidays come with many positives – families, no work, copious amounts of cheese (literally, eating it and the corny movies variety) – but they also come with a lot of naughtiness. When I say naughtiness, I don’t mean kids unwrapping their pressies early. I mean the parties where everyone is obliterated and mauling one another (with a wedding ring on their finger). Last week, I was sat in the local as an office party was going on beside me and my mates, just minding our own biz. Then a lad, about 25, sits down on the stool next to us before a rather trollied lass plonked her unsteady behind down on his knee. They were both slurring their words when we overheard the girl say, "Weeeell, you've got a girlfriend and I'm not single, but so what?!" Lovely. I especially loved her discretion, slap bang in the middle of the pub. She's basically got a wide-on over another guy while her other half was probably at home stuffing the turkey. Stats show that cheating increases at this time of year, instead of secret Santa we have the secret s***gers too. Jokes aside, I think it's disgusting. The classic excuse, 'it meant nothing'. Well, that's more of a f***ing insult! If someone is going to cheat, there must be some feelings or strong attraction, surely. Another lame backtrack is blaming dropping your chinos or lifting your skirt on being drunk. - -I don’t understand how anyone in their right mind gets THAT wasted that they don’t know what they're doing. Coming from someone who every time they go out they get sloshed, I still don’t wake up the next morning thinking, 'Oh my God, who have I slept with?' I've also never woken up after a night out thinking, 'S**t, did I s**g someone behind my partner's back last night?' I think if I ever did wake up after doing that, I'd give up alcohol altogether and reassess my morals. Alcohol is just used as an excuse, b*****ks do people only get these urges to cheat when under the influence. If you're going to cheat, you'll do it sober or drunk. Despite the trolls (who really should make getting out more their primary new year resolution) blasting me with 'You can't talk, homewrecker/s**g/blah blah' – I CAN talk. My views on cheating are extremely strong and always have been. Yeah, I've done my fair share of messing around and one-night stands, but I was never hurting anyone. When I was younger, it didn’t really bother me if I knew a guy had a girlfriend, if I didn’t know the girl, it didn’t effect me. That was my opinion at the time. - -Since getting older, my feelings have changed. I think if I gave my number out knowing that the lad has a chick, it'll bite me back on the a**e at some point in life. Although I've been the girl to cheat with men that are occupied, I'd never be able to do that myself. Having found cheap eyelashes, texts and lip-glosses in and around my ex’s things, it didn’t take a genius to work out what he’d been up to. Let's just say I've put that one to bed now. - -If I felt in any way sexual towards another guy, and I don’t mean looking at a fittie in a bar, it's time to leave. Wanting to bed another man clearly means my current partner is not the one I want to have sex with for the rest of my life – SO WHAT IS THE POINT IN BEING THERE?! One thing that actually grates on me, is when a couple 'move on' from cheating: how the f**k do you move on? I don’t care if it's texting, kissing, s***ging whatever, it's all still cheating. A text usually leads to sex, otherwise why else do people do this? You don’t start the ball rolling for a hot chocolate in Starbucks. - -I pity girls that stick with guys that mess around, I mean, I can never ever imagine getting back into bed with a bloke who had his penis in another girl. What do you talk about? 'So, erm, did she have a Brazilian or a Hollywood?' 'Does she spit or swallow?' 'Who went on top?' It's just bizarre, get a grip! I couldn’t even speak to a guy who had cheated on me, who had stripped me of dignity, trampled on my respect and took me for a fool. There is no moving on from there. With so many guys and girls in the world who are loyal, why waste your time on one who can't even keep an eyeball on you without the other wandering. We’re approaching a new year, thank God I'm approaching it single, unlike some of the desperadoes I know who will celebrate being a mug through to 2016 and onwards, not me thanks. Have a fab new year and keep it in your pants if you're otherwise engaged. Or if you're the victim of someone doing the dirty, you're worth more than that. Time to skedaddle. \ No newline at end of file +EXTRACTS ENTIRE ARTICLE UP UNTIL THE LAST IMAGE, CONCLUSION OF ARTICLE MISSING diff --git a/tests/data/text/dallasnews.com1.txt b/tests/data/text/dallasnews.com1.txt index b8215d3c..7df285d0 100644 --- a/tests/data/text/dallasnews.com1.txt +++ b/tests/data/text/dallasnews.com1.txt @@ -1,3 +1,9 @@ -It seems we can’t find what you’re looking for. Perhaps searching, or one of the links below, can help. +On Saturday, I went to volunteer at the Operation Care International’s Christmas Party For Jesus at the Kay Bailey Hutchison Convention Center, and being directionally impaired, I got lost and had no idea where to park. -Try looking in the monthly archives. :) \ No newline at end of file +I saw two female Dallas police officers and rolled down my window to ask about parking. Upon rolling down my window, the antler on my window fell off. Not wanting to backtrack and tie up traffic, I drove off leaving the antler on the street. I found the parking meter lot that the officer had given me directions to, but then realized I had no change. + +I was standing by the meter figuring out how to pay when I hear someone saying, “Hey, you lost your antler.” I turned around, and the female officer was in her squad car holding out the antler that had fallen off my window. I went over and got my antler, explaining that I was figuring out how to pay for parking because I had no change. The officer said they had just announced that they would not be giving out parking-meter ticket violations that day during the event hours. I’m thankful to our Dallas Police Department for all they do. + +Kim Gump, Dallas + +Top Picks \ No newline at end of file diff --git a/tests/data/text/dallasnews.com2.txt b/tests/data/text/dallasnews.com2.txt index a4b1a393..ef012047 100644 --- a/tests/data/text/dallasnews.com2.txt +++ b/tests/data/text/dallasnews.com2.txt @@ -1,5 +1,17 @@ -There is a caste system already in football with potential performance measured up front by the Wonderlic test. It ranges from quarterbacks in the 30 range (Romo 37 and Stafford 38 which is right up there with engineers and chemists) to single digits in other positions which rank below janitors at 14. There was even a study which indicated that the higher the score was for some positions, the less desirable they may be for draft purposes. +(File 2013/The Associated Press) Brenham's Ryan Nunn (22) tackles Aledo quarterback Luke Bishop (4) during the 2013 UIL Class 4A Division II state championship game. Resarch shows that parents are increasingly opposed to their children playing football. Brenham's Ryan Nunn (22) tackles Aledo quarterback Luke Bishop (4) during the 2013 UIL Class 4A Division II state championship game. Resarch shows that parents are increasingly opposed to their children playing football. -Football is just one of the ways we are no longer the country we once were. We just lost an orthopedic surgeon named Jimmy Swink who was a TCU All American in the days getting a college scholarship meant getting an education a lot of other people wouldn’t get. College football has since evolved into an NFL farm team model where attending class is an unfortunate, mind numbing byproduct you and your mentor just have to put up with to stay on the team. +Twenty years from now, will we look back on photos such as the one above and wonder why in the world we considered that to be “sport”? -We have achieved Rome’s greatness with the military machine, highways, architecture, government, and of course the bread and circuses. TANF, EBT, Section 8, Free Cell Phones, Obamacare, et al gives us the Bread and Football and MMA the gladiators. You don’t have to read up on Gibbon to see the parallels between the Decline phase of the first superpower and the process currently underway within the former superpower. \ No newline at end of file +A new poll by Bloomberg News indicates Americans are continuing to inch in that direction. + +It’s more comfortable to look past the head injury issue, especially this time of year. We have high school playoffs to cheer. And a packed college bowl schedule, exciting for everyone even if your school didn’t get into the game you thought it deserved. (Baylor and TCU fans, hope you saw our editorial on that topic.) And the NFL continues to score big by most any measure, regardless of its many missteps. (Hey, we wrote about that too.) + +But new research, commissioned by Bloomberg Politics, is the latest indicator that parents are increasingly refusing to let their sons play the sport. Just last month, our own sports department produced a revealing special report on former Dallas Cowboy stars and their discomfort with their sons following in their footsteps. + +Overall, the Bloomberg data shows that half of the respondents don’t want their sons playing football. What’s most eye-opening — and troubling — is that the percentage who don’t want their sons participating goes up a lot among those who are college-educated (62 percent) and who make over $100,00 (also 62 percent). + +While I’ve read lots of arguments about sports being a decent vehicle for young lower-income teens to ride into a better life, I hate to think of us all enjoying a game that likely will become increasingly populated by those whose parents aren’t educated about the risks. Polls such as Bloomberg’s leave me feeling a bit like we’ve got a growing caste system in this sport. + +One bright note in the survey: 58 percent of women (as opposed to 41 percent of men) oppose their sons playing football. That number includes moms of all educational and financial backgrounds. + +We’re a long way from becoming a nation of flag football supporters, but I hope the momentum grows for finding ways to create a sport that is a lot less violent. \ No newline at end of file diff --git a/tests/data/text/details.com2.txt b/tests/data/text/details.com2.txt index 18fefce6..312effc6 100644 --- a/tests/data/text/details.com2.txt +++ b/tests/data/text/details.com2.txt @@ -1 +1 @@ -This season, designers updated a classic color by way of modern silhouettes both tailored and casual. The verdict? The most effortless way to transition into fall is all in the shading. \ No newline at end of file +Going Gray This season, designers updated a classic color by way of modern silhouettes both tailored and casual. The verdict? The most effortless way to transition into fall is all in the shading. \ No newline at end of file diff --git a/tests/data/text/elle.com1.txt b/tests/data/text/elle.com1.txt index 707bf5ea..eb5d0dfb 100644 --- a/tests/data/text/elle.com1.txt +++ b/tests/data/text/elle.com1.txt @@ -1 +1,3 @@ -From our editors to your inbox. Sign up for our newsletter today. \ No newline at end of file +From our editors to your inbox. Sign up for our newsletter today. + +Your Email Address Sign Up Now! \ No newline at end of file diff --git a/tests/data/text/elle.com2.txt b/tests/data/text/elle.com2.txt index 03c563f4..72d54f55 100644 --- a/tests/data/text/elle.com2.txt +++ b/tests/data/text/elle.com2.txt @@ -1,65 +1,65 @@ Ah, the holidays: A time for gift giving, family gathering, and passive aggressive rage flames. Here are some well-meaning but totally backhanded digs we heard over the holidays. Add your own in the comments. It feels good to vent. -From: Grandma +From: Grandma -What she said: "Wouldn't it be nice to be in a relationship so you don't have to have roommates anymore?" +What she said: "Wouldn't it be nice to be in a relationship so you don't have to have roommates anymore?" -What we said: "I like my apartment; did Mom show you the new bar cart I found?" +What we said: "I like my apartment; did Mom show you the new bar cart I found?" What we wanted to say: "Literally the only thing harder than finding an affordable apartment in New York City is meeting a guy good enough to make you move out of it. If you know of a site that lists cute 1-bedrooms stocked with single guys, send it the hell over." -From Mom: +From Mom: -What she said: "It would be so nice to have some babies around here soon to open presents!" +What she said: "It would be so nice to have some babies around here soon to open presents!" -What we said: HAHAHAHAHAHA. +What we said: HAHAHAHAHAHA. What we wanted to say: HAHAHAHAHAHA. -From Aunt: +From Aunt: -What she said: "Have you heard of this thing called Tinder?" +What she said: "Have you heard of this thing called Tinder?" -What we said: "Um, yes, kind of." +What we said: "Um, yes, kind of." What we wanted to say: "If I hadn't heard of Tinder, I wouldn't have touched a male in over a year. So. Yes, yes I have." -From: Mother in Law: +From: Mother in Law: -What she said: "Did you REALLY cook all this yourself?" +What she said: "Did you REALLY cook all this yourself?" -What we said:" I sure did, hope you enjoy!" +What we said:" I sure did, hope you enjoy!" What we wanted to say: "Just because I’m not giving up my career to cook three meals a day for your precious angel of a son doesn’t mean I can’t figure out which end of the turkey is which, lady." -From: Mom +From: Mom -What she said: "Maybe it’s time to see a therapist." +What she said: "Maybe it’s time to see a therapist." -What we said: "Yeah, maybe that could help." +What we said: "Yeah, maybe that could help." What we wanted to say: "I am already seeing a therapist to try and resolve my relationship issues before I’m doomed to die alone, THANKS!" -From: Mom: +From: Mom: -What she said: "Did you go outside at all today?" +What she said: "Did you go outside at all today?" -What we said: "Oh, haha, I guess I haven't!" +What we said: "Oh, haha, I guess I haven't!" What we wanted to say: *crawls back under covers, fighting back shame waves* -From: Brother +From: Brother -What he said:"If she can wear that, then I don’t have to get dressed." +What he said:"If she can wear that, then I don’t have to get dressed." -What we said: "Then don’t get dressed." +What we said: "Then don’t get dressed." What we wanted to say: "Then don’t get dressed." -From a STRANGER at a Christmas party: +From a STRANGER at a Christmas party: -What she said: "I lived in New York and would have worked forever, like you, if my husband hadn’t moved us to Virginia. But really, you can’t work forever and wait to have kids. You should do Match.com. My sister did Match.com and she’s 42 and just got married and had a baby." +What she said: "I lived in New York and would have worked forever, like you, if my husband hadn’t moved us to Virginia. But really, you can’t work forever and wait to have kids. You should do Match.com. My sister did Match.com and she’s 42 and just got married and had a baby." -What we said: *Awkward polite laughter* +What we said: *Awkward polite laughter* What we wanted to say: "Save some of that white wine for the rest of us " \ No newline at end of file diff --git a/tests/data/text/flavorwire.com1.txt b/tests/data/text/flavorwire.com1.txt index 937ba521..956ef386 100644 --- a/tests/data/text/flavorwire.com1.txt +++ b/tests/data/text/flavorwire.com1.txt @@ -1,5 +1,7 @@ Presenting; a mix of the morally execrable and the culturally annoying moments in 2014 that we sort wish had never happened, to be honest, although we dutifully thought and wrote about them (and to be even more honest, these ten moments are merely the tip-top of the Christmas tree of head-smacking moments from 2014.) +Adieu, 2014! + 1. Celebrity nude photo leaks. It was a bummer when actresses were targeted by hackers who released their nude photos. And when everyone slut-shamed them as a result. Sidenote: same went for celebrity plastic surgery gawking. 2. Donald Sterling-gate. One of many examples of 2014 scandals that made us aware of how awfully racist people in power were, but this one was just the most epically depressing on a human level. @@ -22,6 +24,8 @@ Presenting; a mix of the morally execrable and the culturally annoying moments i 11. Ariel Pink. Was his misogyny a joke? A nasty slice of his real personality? Trolling because he realized he got more attention for being a jerk than for his music? To be real, much of this whole conversation was unnecessary. +12. This godawful, treacle-coated cover of Leonard Cohen’s “Hallelujah”. + 13. The disrespectful, racist right wing media circus around Ferguson and the death of Michael Brown. Bonus: Sarah Palin’s Christmas Pie-baking instruction and garbled speech special that’s eerily reminiscent of SNL’s “Martha Stewart’s Topless Christmas.” Just kidding, we’re so glad this happened! Never leave us, ex-governor Palin. \ No newline at end of file diff --git a/tests/data/text/foxbusiness.com1.txt b/tests/data/text/foxbusiness.com1.txt index b4b22f43..a77a7f9b 100644 --- a/tests/data/text/foxbusiness.com1.txt +++ b/tests/data/text/foxbusiness.com1.txt @@ -2,6 +2,8 @@ No matter how much traffic your site gets, what really matters is conversions. C Your site must grab visitors attention immediately, then lead them from where they land to checkout. Is it obvious how landing on your site will benefit your visitors? If they can’t immediately tell they will leave. Use analytics to check bounce rates and improve any pages that show high exit rates. +1. Upsell and Cross Sell + Offering related products on product pages to sell more than one product is a common strategy known as cross-selling. Upselling is offering more expensive related products to increase the amount spent. New SaaS provider Fanplayr.com helps to make this process seamless for businesses. Their service analyzes your existing traffic, assessing the behavior and profiles of each visitor in real-time and automatically upsells or offer specifically tailored deals. This is executed based on defined rules and customized offers and incentives set by you for your target customers. This customized approached helps you to increase your average order value and conversions. @@ -10,42 +12,80 @@ Fanplayr analyzes traffic you drive so you can target different offers to segmen +user type + +visit count + +past purchases + +time on site + +page views + +cart size + +device type + +browser + +source + +search term + +location + New visitors who place items with high profit margins in their carts can be offered a higher dollar discount. Reward the loyalty of your repeat buyers with percent off their current purchase. Be sure to let them know they received the discount because they have purchased from you before. Just as some sites offer free shipping for orders over $50, $75 or $100, increase visitor’s total spend by offering a higher discount based on the dollar amount they have in their carts or how much they have spent with you previously. Read more in Proven Online Cross-Selling and Upselling Techniques. +2. Improve Site Search + Make sure your customers can find what they want quickly. Test your site search to ensure that all your products show up as expected and just as importantly confirm that irrelevant products are not appearing in searches. Consumers will not trudge through page after page of search results. Standard site search often returns all results for any word in the search. What this means is that if someone searches for ‘pink widgets’ the search will return every pink item on the site. This is unacceptable as not all users will realize the problem and change their search phrases. There are many easily installed third party site search solutions to resolve these issues. +3. Conversion Test eCommerce Incentives + Traffic from AdWords can be offered multiple types of incentives which can be measured to see which increases sales more. For example, test whether your visitors would rather have: Free shipping if they spend $50 or more - 10% off sales of $100 or more +10% off sales of $100 or more - $20 off a $200 purchase +$20 off a $200 purchase Fanplayr not only analyzes, segments, and allows you to add incentives and offers; it integrates A/B testing into ecommerce software. Changes can be made to text campaigns on the fly. Conversion testing can increase conversion rates up to 25% ~ Up to 50% higher average order values ~ Targeted offers can generate up to 20% margin improvement. +4. Conversion Tests for SaaS + Solutions providers can use analysis and segmenting to decrease the time it takes to progress through their sales funnels. For example, Neil Patel at QuickSprout suggested in 7 Simple A/B Tests to Increase Conversions 10% or More to test: +Buy now vs free trial + +Credit card vs no credit card + Improve your sales funnel by comparing the actions of visitors who have converted with those who haven’t. Check page views to determine whether visiting particular pages increases conversions. If it does, change site navigation to add that page to the sales funnel. Provide different incentives to repeat visitors or visitors who have downloaded specific white papers or watched specific videos. Analysis can provide your sales team insights into what a particular lead is most interested in. +5. Strengthen Trust Signals + Unless you’re a household name, your site must convey how trustworthy your site is. Adding photos of the business location and people behind the brand to your about page can increase conversions. Your about page is usually visited just before a person checks out to make sure you are a “real” business. Optimize this page. Make it convey why they should buy from you. Online buyers are more sophisticated today. Many know to verify that your site is secure by looking for the https:// in your url and a Secure Shopping 256 bit icon. Display Trust Signals such as Better Business Bureau, TrustE, Norton Secured, TrustWave, Google Trusted Store. +6. Reviews and Testimonials + Real testimonials are another way to increase trust. If you sell a service, offer it to the leading influencers in your industry and ask for their endorsement. Use their photo, logo and text or even a video. Video testimonials are doable for any size business. Do your fellow consumers a favor and review the products and services you buy. Even though testimonials and reviews can be manipulated, they are still valuable during the decision-making process. Savvy buyers know to look at a reviewers other reviews or to search for the testimonial giver online to determine whether they can be trusted. Enable the ability for your buyers to review your products and services. What they write is feedback for you and your customers. No product or service is a perfect fit for everyone. Details left by buyers ensure buyers choose wisely. +7. Be Easy to Contact + People don’t buy unless they’re confident they are making the right decision. Even if you have excellent product descriptions and multiple photos, they may still have a question. At a minimum be sure you have a contact page that is easy to find. Test it to make sure it works. Better yet, add live chat to your site. Businesses that provide live interactions at least during business hours save money and experience increased sales and conversions. @@ -54,4 +94,6 @@ Better yet, add live chat to your site. Businesses that provide live interaction This is especially true if you are a very small brand. People want to know there is a real person they can contact behind the site. Answering them in real time increases confidence they can reach you should their package not arrive or they have problems with their purchase. -Gail Gardner is the Small Business Marketing Strategist who founded and provides consulting at GrowMap.com. She also answers questions in live chat as Community Manager at SocialImplications.com. \ No newline at end of file +Gail Gardner is the Small Business Marketing Strategist who founded and provides consulting at GrowMap.com. She also answers questions in live chat as Community Manager at SocialImplications.com. + +More from AllBusiness.com: \ No newline at end of file diff --git a/tests/data/text/foxbusiness.com2.txt b/tests/data/text/foxbusiness.com2.txt index e54cd313..5379224e 100644 --- a/tests/data/text/foxbusiness.com2.txt +++ b/tests/data/text/foxbusiness.com2.txt @@ -1,13 +1,19 @@ If a Scrooge client is scaring the dickens out of you, don’t let it spoil your holiday season. A visit from the Spirit of Business Past, the Spirit of Business Present, and the Spirit of Business Future will make even the sourest customer as sweet as Tiny Tim. +The Spirit of Business Past + Scrooge clients have short memories. Pay them a holiday visit and review all of the ways you have helped them save money, become more efficient, increase their sales, avert a crisis, and whatever else had a meaningful impact. It takes more than a casual visit. The more visual and vivid your presentation is, the greater its effect — recall how the cold heart of Ebenezer Scrooge began to thaw when he saw and felt and heard his former self. Do you think the Spirit of Christmas Past would have elicited the same response by showing Ebenezer a PowerPoint crammed with wordy bullet points? +The Spirit of Business Present + The holiday season is a time of joy — do you remember what a merry old soul the Spirit of Christmas Present was? A small thing such as taking your Scrooge client to lunch or meeting him for an after-work cocktail can change everything. Simply getting your Scrooge away from that damp, cold, and dreary office will make his mood more festive and his business more secure. Along with a dollop of holiday cheer, offer your Scrooge a business-present present: a gift, a December discount, or extra rewards points — something that answers the question, “What have you done for me lately?” Nothing makes a Scrooge client more Scrooge-like than the feeling of being taken for granted. +The Spirit of Business Future + The outlook for a Scrooge client is grim indeed; you cannot help but feel that it’s only a matter of time before his business is buried in the graveyard of lost opportunities. But as the Spirit of Christmas Future revealed to Ebenezer, the future can be changed! Give your Scrooge scenario a hopeful future by introducing incentives that reward his future business. Annual rebates, volume discounts, and loyalty rewards programs make it easier — far easier — for your Scrooge to overlook the day-to-day hiccups that occur in any business relationship. Just as the goal of curing Tiny Tim gave Ebenezer a reason to live well, a goal of getting more “something” gives clients a reason to do business. @@ -20,4 +26,8 @@ By demonstrating past value, present value, and future value, your Scrooge clien “I am as light as a feather, I am as happy as an angel, I am as merry as a school-boy. I am as giddy as a drunken man. A merry Christmas to every-body! A happy New Year to all the world! Hallo here! Whoop! Hallo!” – Charles Dickens, A Christmas Carol -Brad Shorr is Director of B2B Marketing for Straight North, an Internet marketing agency in the Chicago area. With in-house, freelance and agency experience, he writes frequently about content marketing, SEO, social media and small business strategy. \ No newline at end of file +-- + +Brad Shorr is Director of B2B Marketing for Straight North, an Internet marketing agency in the Chicago area. With in-house, freelance and agency experience, he writes frequently about content marketing, SEO, social media and small business strategy. + +More from AllBusiness.com: \ No newline at end of file diff --git a/tests/data/text/foxnews.com1.txt b/tests/data/text/foxnews.com1.txt index db75a005..3536cc51 100644 --- a/tests/data/text/foxnews.com1.txt +++ b/tests/data/text/foxnews.com1.txt @@ -28,7 +28,7 @@ Chastain said that the director "really left (the script) free for us to explore Chandor attributed the two actors’ long off-screen friendship for making Abel and Anna’s onscreen relationship feel so real. -“I’m not sure if I knew it at the time, but, looking back at it now, it was just amazing cause there’s this shared history. The fact that known each other almost as long as the characters had – it was just meant to be,” he said. +“I’m not sure if I knew it at the time, but, looking back at it now, it was just amazing cause there’s this shared history. The fact that they had known each other almost as long as the characters had – it was just meant to be,” he said. “A Most Violent Year” hits select theaters on Dec. 31, followed by a wider release in January. @@ -42,4 +42,6 @@ Although he said it was “wild” to see himself in the film’s teaser, Isaac Like us on Facebook - Follow us on Twitter & Instagram \ No newline at end of file +Follow us on Twitter & Instagram + +Follow Lucia I. Suarez on Twitter @luciasuarezsang \ No newline at end of file diff --git a/tests/data/text/foxnews.com2.txt b/tests/data/text/foxnews.com2.txt index 4fdf4e30..22dc9f99 100644 --- a/tests/data/text/foxnews.com2.txt +++ b/tests/data/text/foxnews.com2.txt @@ -16,4 +16,4 @@ Reynosa is a border city in the northern part of Tamaulipas, Mexico. It is locat Like us on Facebook - Follow us on Twitter & Instagram \ No newline at end of file +Follow us on Twitter & Instagram \ No newline at end of file diff --git a/tests/data/text/glamour.com1.txt b/tests/data/text/glamour.com1.txt index faba877b..605539a3 100644 --- a/tests/data/text/glamour.com1.txt +++ b/tests/data/text/glamour.com1.txt @@ -4,14 +4,16 @@ When Rihanna walks into a fashion show, even editors, publicists, other celebs g - At Adam Selman, RiRi rocked an elegant and girly white A-line dress with pearls and my favorite Christian Louboutin strappy pumps. +At Adam Selman, RiRi rocked an elegant and girly white A-line dress with pearls and my favorite Christian Louboutin strappy pumps. - For Altuzarra, she glammed out in super sophistication with a plunging black jacket with fringe from the designer's collection—WITHOUT PANTS. I LOVE IT. +For Altuzarra, she glammed out in super sophistication with a plunging black jacket with fringe from the designer's collection—WITHOUT PANTS. I LOVE IT. - At Alexander Wang, she brought out her inner '90s child with a matching midnight blue hoodie and skirt. +At Alexander Wang, she brought out her inner '90s child with a matching midnight blue hoodie and skirt. -Are you a Rihhana fan? I love her. Now where's my plunging black jacket? Not to fret—I will wear pants. \ No newline at end of file +Are you a Rihhana fan? I love her. Now where's my plunging black jacket? Not to fret—I will wear pants. + +Photos: Getty Images \ No newline at end of file diff --git a/tests/data/text/glamour.com2.txt b/tests/data/text/glamour.com2.txt index cf0f4a9a..9c20ae3b 100644 --- a/tests/data/text/glamour.com2.txt +++ b/tests/data/text/glamour.com2.txt @@ -1,5 +1,9 @@ A ton of brides are going for the pretty pretty princess vibe on their wedding day, so lots of wedding dress designers automatically reach for the tulle and the crinolines. {“Fluffier! Puffier! MORE! } If you’re a bride who isn’t feeling the cupcake vibe, I suggest checking out white evening dresses, which tend to be a lot more streamlined. And bonus: Evening gowns are generally less expensive than wedding gowns. (Four of the dresses below are under $1,000.) Here are six off-the-rack I like: +JavaScript is currently disabled in this browser. Reactivate it to view this content. - Off-the-rack wedding dresses: Yes or no? Would you want to wear an evening gown on your wedding day? Or does it HAVE to be bridal? \ No newline at end of file + +Off-the-rack wedding dresses: Yes or no? Would you want to wear an evening gown on your wedding day? Or does it HAVE to be bridal? + +Photos: Courtesy of the vendors. \ No newline at end of file diff --git a/tests/data/text/gq.com1.txt b/tests/data/text/gq.com1.txt index 2eeb7adf..b9184d45 100644 --- a/tests/data/text/gq.com1.txt +++ b/tests/data/text/gq.com1.txt @@ -1,3 +1,7 @@ +By + + + Ahhh, at last, Memorial Day weekend. The sweet sweet reminder that summer has officially arrived. It's just crab cakes and football straight through Labor Day, right? Yeah. Well, if you find yourself celebrating the return of the season with a road trip, but have yet to pack, we've got your back. You'll be traveling with everything you need in no time. All you have to do is pay close attention to the following. diff --git a/tests/data/text/gq.com2.txt b/tests/data/text/gq.com2.txt index e108f82c..a6709345 100644 --- a/tests/data/text/gq.com2.txt +++ b/tests/data/text/gq.com2.txt @@ -1,34 +1,36 @@ +Photo: Silver Screen Collection/Getty Images + With summer finally upon us, it's hard not look wistfully from the inside of a car and long for the wind on your face—barreling down country roads, freed from the tethers of smartphones and chatty passengers. For a lot of folks, this time of year is when they finally take the plunge, shirk that four-wheeled cage, and buy a motorcycle. Bikes are simple: two wheels, a motor, and miles and miles of outlaw-style freedom. But, as with everything in life, that simplicity comes with a price. So, if heading off into the sunset like Brando in The Wild One is your new life's goal, then here's everything you need to know about it before you do: The Real Cost of Ownership - It's not a surprise that when gas prices spike during the warmer months, so do motorcycle sales. And while it's true that some motorcycles do get better gas mileage than cars, and that they're often cheaper to buy, the fact is that the cost of bike ownership goes way beyond the MSRP and price at the pump: +It's not a surprise that when gas prices spike during the warmer months, so do motorcycle sales. And while it's true that some motorcycles do get better gas mileage than cars, and that they're often cheaper to buy, the fact is that the cost of bike ownership goes way beyond the MSRP and price at the pump: The Bike - Motorcycle prices can vary wildly, but on average, if you're buying a new motorcycle fit for a beginner, you're probably spending anywhere between $5,000 and $10,000. +Motorcycle prices can vary wildly, but on average, if you're buying a new motorcycle fit for a beginner, you're probably spending anywhere between $5,000 and $10,000. Insurance - If you are over 25 and have a spotless driving record, you can get a pretty decent rate on insurance, possibly under $500 a year. Unfortunately, there's a lot more involved than simply your age and driving record—the population density of where you live, the theft rate of the bike model, whether Christmas falls on a Tuesday...when it comes to insurance, it's Thunderdome. Shop around, obviously, but just know that you're definitely going to shell out some cash. +If you are over 25 and have a spotless driving record, you can get a pretty decent rate on insurance, possibly under $500 a year. Unfortunately, there's a lot more involved than simply your age and driving record—the population density of where you live, the theft rate of the bike model, whether Christmas falls on a Tuesday...when it comes to insurance, it's Thunderdome. Shop around, obviously, but just know that you're definitely going to shell out some cash. Equipment and Maintenance - This is where things can add up. Cars go a lot longer between service intervals, not to mention things like tire, spark plug, and belt replacement. Tires can be especially expensive on motorcycles, running between $400 and $600 for a set. And depending on how hard you ride, you may have to change at least the rear tire every 3,000 miles or so. Chains and drive belts need occasional replacing, and those can cost between $140 and $250. Maintenance intervals can run anywhere between 5,000 and 20,000 miles, depending on the motorcycle, but if there's a valve adjustment involved, expect to pay anywhere between $800 and $1,500. Add in regular oil changes, chain maintenance, and various other odds and ends and, if you ride often, you can expect to drop at least $1,000 per year just on maintenance. +This is where things can add up. Cars go a lot longer between service intervals, not to mention things like tire, spark plug, and belt replacement. Tires can be especially expensive on motorcycles, running between $400 and $600 for a set. And depending on how hard you ride, you may have to change at least the rear tire every 3,000 miles or so. Chains and drive belts need occasional replacing, and those can cost between $140 and $250. Maintenance intervals can run anywhere between 5,000 and 20,000 miles, depending on the motorcycle, but if there's a valve adjustment involved, expect to pay anywhere between $800 and $1,500. Add in regular oil changes, chain maintenance, and various other odds and ends and, if you ride often, you can expect to drop at least $1,000 per year just on maintenance. Gear - At the very least you will need a helmet, which can run anywhere from $150 to $900. But if a helmet is all you think you need, you should stick to four wheels. The smart rider who values his skin will also wear a motor jacket, preferably high-abrasion grade leather, gloves, and boots at all times. And while most people ride in jeans, the truth is, if you go down at any speed above 15 mph, jeans will come off like a wet paper towel; protective pants are highly recommended. Conservatively, you should plan to initially spend at least $800 to $1,200 on new gear, which, of course, will eventually have to be replaced as items wear out. +At the very least you will need a helmet, which can run anywhere from $150 to $900. But if a helmet is all you think you need, you should stick to four wheels. The smart rider who values his skin will also wear a motor jacket, preferably high-abrasion grade leather, gloves, and boots at all times. And while most people ride in jeans, the truth is, if you go down at any speed above 15 mph, jeans will come off like a wet paper towel; protective pants are highly recommended. Conservatively, you should plan to initially spend at least $800 to $1,200 on new gear, which, of course, will eventually have to be replaced as items wear out. Getting Started - All right, so you're still undaunted and have decided to take the plunge. So where to start? The best thing you can do for yourself, as well as everyone else on the road, is to sign up for the Basic RiderCourse at the Motorcycle Safety Foundation. It usually costs around $275-$350, depending on where you live, but it's a lot cheaper than a trip to the hospital because you had no idea what you were doing. The course consists of 10 hours of riding instruction, usually stretched over a weekend, and both motorcycles and helmets are provided (although if bowling shoe rentals give you the creeps, then you'll definitely want to bring your own helmet. Just make sure it's DOT or Snell approved). The class is generally taught in a big parking lot or other open space, so you have the benefit of making mistakes without cars barreling down on you. And in a lot of states, passing the class will count as your DMV riding test, which is worth the price of admission alone. +All right, so you're still undaunted and have decided to take the plunge. So where to start? The best thing you can do for yourself, as well as everyone else on the road, is to sign up for the Basic RiderCourse at the Motorcycle Safety Foundation. It usually costs around $275-$350, depending on where you live, but it's a lot cheaper than a trip to the hospital because you had no idea what you were doing. The course consists of 10 hours of riding instruction, usually stretched over a weekend, and both motorcycles and helmets are provided (although if bowling shoe rentals give you the creeps, then you'll definitely want to bring your own helmet. Just make sure it's DOT or Snell approved). The class is generally taught in a big parking lot or other open space, so you have the benefit of making mistakes without cars barreling down on you. And in a lot of states, passing the class will count as your DMV riding test, which is worth the price of admission alone. And if you think classes are for sissies, consider that in 2011, there were 4,323 deaths on motorcycles. Then consider that there were 81,000 recorded injuries as well, and that per vehicle mile, motorcyclists were 30 times more likely to die in a crash than passenger car occupants. Long story short? Take the class. Getting a Bike - The best buyer is an informed one—and there's this thing called the Internet with a lot of information on it, so you really have no excuse. Generally speaking, motorcycle salesmen are pretty knowledgeable, but they aren't shamans, so you need to have an idea of what you're looking for. And in the off chance that you get a clueless salesman, it's even more important to be informed. If you know what type of riding you plan on doing—track days, commuting, cruising, touring, or just tooling around town—you can usually narrow your options before even hitting the motorcycle shops. +The best buyer is an informed one—and there's this thing called the Internet with a lot of information on it, so you really have no excuse. Generally speaking, motorcycle salesmen are pretty knowledgeable, but they aren't shamans, so you need to have an idea of what you're looking for. And in the off chance that you get a clueless salesman, it's even more important to be informed. If you know what type of riding you plan on doing—track days, commuting, cruising, touring, or just tooling around town—you can usually narrow your options before even hitting the motorcycle shops. The first question is what size bike you should buy. As someone who's worked in a motorcycle shop, my suggestion is to aim for the middle. You can start with a 250cc bike, which will definitely be the easiest to learn on, but you'll probably outgrow it within a few months. Which means you'll just turn around and want to buy another bike, ending up spending far more money than you planned to. Something in the 500-600cc range, on the other hand, will stay fun for a long time, even as your skills improve. Engine size isn't the only factor, either: A Yamaha YZF-R6 is a 600cc, but it's essentially a race bike capable of serious acceleration and high top end speeds. This is not a bike for beginners. A 865cc Triumph Bonneville, on the other hand, is a fantastic bike to start on. @@ -36,8 +38,10 @@ The best thing to do when picking a motorcycle is to consider the seating positi Fatalities by engine size: - Up to 500cc: 6% +Up to 500cc: 6% + +500 to 1000cc: 40% - 500 to 1000cc: 40% +1001 to 1500cc: 30% - 1001 to 1500cc: 30% \ No newline at end of file +Got it? Good. \ No newline at end of file diff --git a/tests/data/text/graziadaily.co.uk2.txt b/tests/data/text/graziadaily.co.uk2.txt index 3fd51900..9bbdbbd2 100644 --- a/tests/data/text/graziadaily.co.uk2.txt +++ b/tests/data/text/graziadaily.co.uk2.txt @@ -1,9 +1,25 @@ 'Tis the season to wrap up, get glam and ingest your body weight in food and drink. No better time, then, for ASMALLWORLD's utterly fabulous multi-day 10th birthday party in Gstaad, Switzerland. And so it was that our Alya Mooro joined a rolodex of some of the most alluring individuals, the likes of Dianna Agron, Caitlin FitzGerald, Pixie Geldof, Tali Lennox and the utterly charming Carey Mulligan - who hosted the Saturday night charity gala in aid of War Child - to descend upon the Palace Hotel in Gstaad for a weekend that positively oozed glamour. We round up some of our highlights of the trip below... +Eating & Drinking... Lots + +Our Alya Mooro with ASMALLWORLD guest at the cocktail dinner + The Palace Hotel acted as ground zero for the weekend, hosting a number of fabulous gatherings including welcome cocktails on the Friday night ahead of an epic dinner at in-house restaurant Gildo's where guests got to know each other over wine and three courses of superb cuisine before decamping to the basement club (more on which later). +Gstaad'd From The Bottom... + +The view from the top of Wasserngrat + While some of the more ambitious of individuals spent their morning after the night before taking to the slopes in their ski gear, the rest of us took a more leisurely approach in the form of a chairlift to the top of Wasserngrat mountain, where we were greeted by a beaming sun, ridiculously Instagrammable views and cauldrons and cauldrons of bubbling cheese. And our lives were basically complete. Giddy from having ingested a lifetime supply of cheese and wine, the remainder of the afternoon was spent concoting the best hashtags to accompany our fairytale-perfect snaps. Our favourite? 'Gstaad'd from the bottom now we here.' Yup, we think Drake would be proud of that one, too. All credit to the altitude. And the wine. +Reckless & Generous... + +Paula Goldstein, Cleo Wade, Caitlin Fitzgerald rocking the headgear at ASW Winter Weekend [Rex] + Snow boots and gigantic fluffy hats safely tucked away, guests descended upon the Palace ballroom in a dizzying array of black tie extravagance. With trailing dresses in any and all colours imaginable, glittering jewels and a collection of headdresses that frankly make Blair Waldorf's look preeeetty pitiful, guests took their seats for a five course meal that was overseen by none other than Carey Mulligan and ASW Chairman Patrick Liotard-Vogt. 'I am begging you to give recklessly and generously,' said Mulligan in her impassioned address. And who could turn her down? With help from raffle prizes from the likes of Hublot, Maiyet, Tinker Tailor, and even a peck on the cheek from Dianna Agron, $100,000 was raised for War Child - a charity that provides support to children in high conflict zones around the world. +Take It To The Basement... + +Our Alya Mooro with Dianna Agron + The Palace's basement nightclub - hailed as one of the most exclusive in the region - hosted the afterparties that concluded both festive evenings, and almost contributed to many a missed flight on the Sunday morning! Shoes in hand and fists pumping, attendees of the soirees quite literally let their hair down, dancing into the morning with a little help from the DJs who alternated between the latest in chart topping hits to never forgotten classics, pleasing just about everyone in the process. As attendees from literally all around the world revelled in the moment, we realised just how small a world it is, and how happy we are to be a part of it. \ No newline at end of file diff --git a/tests/data/text/gulflive.com1.txt b/tests/data/text/gulflive.com1.txt index 1b751448..ca9d4750 100644 --- a/tests/data/text/gulflive.com1.txt +++ b/tests/data/text/gulflive.com1.txt @@ -2,27 +2,27 @@ For the serious genealogist and family historian there are some worthwhile learn - The Federation of Genealogical Societies will meet Feb. 11-14 in Salt Lake City, Utah, in conjunction with RootsTech for a one-time special genealogy event at the Salt Palace Convention Center. This gives the genealogy community a unique opportunity to experience two different conferences under one roof or the option to attend only one. +The Federation of Genealogical Societies will meet Feb. 11-14 in Salt Lake City, Utah, in conjunction with RootsTech for a one-time special genealogy event at the Salt Palace Convention Center. This gives the genealogy community a unique opportunity to experience two different conferences under one roof or the option to attend only one. - "Destined to be the largest family history event North America has ever seen, the FGS 2015 conference is the perfect destination for anyone interested in tracing their roots," said D. Joshua Taylor, FGS president. +"Destined to be the largest family history event North America has ever seen, the FGS 2015 conference is the perfect destination for anyone interested in tracing their roots," said D. Joshua Taylor, FGS president. - FGS sessions will focus on methodology, records, ethnic research and migration. RootsTech will offer a program of technology-based solutions for the genealogy needs of both individuals and societies. +FGS sessions will focus on methodology, records, ethnic research and migration. RootsTech will offer a program of technology-based solutions for the genealogy needs of both individuals and societies. - Those attending will have access to the famous Salt Lake City Family History Library, a dream destination of genealogists everywhere. It houses the largest genealogical collection in the world, which includes more than 2.4 million rolls of microfilmed records and 356,000 books. Visitors are welcome to bring a flash drive for digital copies. +Those attending will have access to the famous Salt Lake City Family History Library, a dream destination of genealogists everywhere. It houses the largest genealogical collection in the world, which includes more than 2.4 million rolls of microfilmed records and 356,000 books. Visitors are welcome to bring a flash drive for digital copies. - Special hotel rates are offered to attendees. For more information on the innumerable number of sessions and events and the expo, log on to www.fgsconference.org. +Special hotel rates are offered to attendees. For more information on the innumerable number of sessions and events and the expo, log on to www.fgsconference.org. - Another stellar conference is that of the National Genealogical Society. Its 2015 Family History Conference themed "Crossroads of America" will be held in St. Charles, Mo., May 13-16. +Another stellar conference is that of the National Genealogical Society. Its 2015 Family History Conference themed "Crossroads of America" will be held in St. Charles, Mo., May 13-16. @@ -30,33 +30,33 @@ The program features numerous tracks each day covering a broad array of topics i - Attendees will be offered a variety of conference tours from which to choose that include the historic Daniel Boone Home & Heritage Center, the Lewis & Clark Boathouse, historic St. Louis, Mo., and the National Archives Personnel Records Center. +Attendees will be offered a variety of conference tours from which to choose that include the historic Daniel Boone Home & Heritage Center, the Lewis & Clark Boathouse, historic St. Louis, Mo., and the National Archives Personnel Records Center. - Throughout this conference the Board for Certification of Genealogists will again sponsor a skill-building track for intermediate to advanced researchers interested in improving their research skills. +Throughout this conference the Board for Certification of Genealogists will again sponsor a skill-building track for intermediate to advanced researchers interested in improving their research skills. - Multiple lectures, workshops, showcases, luncheons with noted speakers are on the agenda. For details, log on to www.ngsgenealogy.org. +Multiple lectures, workshops, showcases, luncheons with noted speakers are on the agenda. For details, log on to www.ngsgenealogy.org. - And for those who would like to learn in the comfort of their homes, the NGS is offering a new online study this year called "Researching Your Revolutionary War Ancestors" by author Craig Roberts Scott. This course introduces numerous U.S. records created for, during and after the war for those who fought for independence, and provides strategies to identify and locate information on an ancestor living at the time of the war. +And for those who would like to learn in the comfort of their homes, the NGS is offering a new online study this year called "Researching Your Revolutionary War Ancestors" by author Craig Roberts Scott. This course introduces numerous U.S. records created for, during and after the war for those who fought for independence, and provides strategies to identify and locate information on an ancestor living at the time of the war. - Scott is a nationally recognized lecturer, educator and genealogical and historical researcher with more than 30 years' experience. He specializes in the diverse military records at the National Archives. The eight-module course consists of lessons, examples, exercises and self-graded exams that can be viewed on a home computer. Students work independently. +Scott is a nationally recognized lecturer, educator and genealogical and historical researcher with more than 30 years' experience. He specializes in the diverse military records at the National Archives. The eight-module course consists of lessons, examples, exercises and self-graded exams that can be viewed on a home computer. Students work independently. Specifically, students will learn how to: -• identify an ancestor living at the time of the Revolutionary War, +• identify an ancestor living at the time of the Revolutionary War, -• locate information about the service of an ancestor in the RW, +• locate information about the service of an ancestor in the RW, -• understand pension law and locate pension application files, +• understand pension law and locate pension application files, -• locate pension ledgers, payment vouchers, last and final payments, +• locate pension ledgers, payment vouchers, last and final payments, • locate information about the unit that the ancestor served in during the war; diff --git a/tests/data/text/gulflive.com2.txt b/tests/data/text/gulflive.com2.txt index c9367582..3fd7275e 100644 --- a/tests/data/text/gulflive.com2.txt +++ b/tests/data/text/gulflive.com2.txt @@ -1,3 +1,5 @@ Crown Equity Holdings Could be Sitting on a Gold Mine with CRWETube +Posted: Thursday, October 17, 2013, 8:09 PM Inappropriate video? Alert us. + Crown Equity Holdings Inc. (OTCMKTS: CRWE) goal is to accelerate the success of the company, by refocusing on prior endeavors to deliver value for its stockholders in both the near and long term. CRWE has refocused its efforts and direction to an endeavor launched earlier this year in reference to an online business-to-business (B2B) marketplace platform for manufacturers and small to large businesses on a global basis to sell and acquire various types of merchandise. CRWE has started updating its B2B business plan and strategies to move forward. Strategic plans are also being developed for CRWE’s online video-sharing and VoIP communication projects to improve the potential future growth for the company Online ad spending has rapidly become a main stay in many advertising budgets with online TV ad budgets being with one of the main dynamics driving product and service revenue across all industries. BIA/Kelsey has projected that social media advertising spending will hit close to $10 billion by year 2016 well up from $4.8 billion spent in 2012. According to Break Media, video ad spending will reach $5.4 Billion by 2016. CRWE is targeting that multi-billion dollar market with its video sharing website CRWETube (www.crwetube.com) General disclaimer statements on this website or newsletter may constitute forward-looking statements and are subject to numerous risks and uncertainties, including the failure to complete successfully the development of new or enhanced products, the Company’s future capital needs, the lack of market demand for any new or enhanced products the Company may develop, any actions by the Company’s partners that may be adverse to the Company, the success of competitive products, other economic factors affecting the Company and its markets, seasonal changes, and other risks detailed from time to time in the Company’s filings with the U.S. Securities and Exchange Commission. The actual results may differ materially from those contained on this website. The Company disclaims any obligation to update any statements in this website. These stock quotes and related data are provided for information purposes only and are not intended for trading purposes. Crown Equity Holdings, Inc. . will not be liable for any inaccuracies or delays in such data, or for any actions taken in reliance thereon. Potential investors should seek independent information and advice from qualified investment professionals prior to investment Crown Equity Holdings, Inc. provides links to websites operated by third parties. These links may be of interest or of use to you, and are provided for convenience only. You should be aware that in using these links, you are leaving Crown Equity Holdings, Inc.‘s website. Crown Equity Holdings, Inc. does not approve or endorse the content, information or materials available on such third party websites. In addition, Crown Equity Holdings, Inc. makes no representation regarding, and is not responsible for, the content, information or material available on such websites. If you decide to access such websites or newsletter you do this at your own risk, and Crown Equity Holdings, Inc. will not be liable for any loss or damage associated with your use of, or reliance on, the content, information or material available on such websites. \ No newline at end of file diff --git a/tests/data/text/huffingtonpost.com1.txt b/tests/data/text/huffingtonpost.com1.txt new file mode 100644 index 00000000..086eff41 --- /dev/null +++ b/tests/data/text/huffingtonpost.com1.txt @@ -0,0 +1,24 @@ +We should all do so well while under assault. + +JPMorgan Chase earned $4.9 billion in the fourth quarter of 2014, the company announced on Wednesday, down from a year ago, but capping what CEO Jamie Dimon called a record year for the biggest U.S. bank by assets. + +Despite this success, Dimon warned that "banks are under assault," from government regulators. + +"In the old days," Dimon said, "you dealt with one regulator when you had an issue, maybe two. “Now it’s five or six. It makes it very difficult and very complicated. + +"You all should ask the question about how American that is. And how fair that is," he added. "And how complex that is for companies." + +The biggest U.S. bank by assets has had its fair share of trouble with regulators in recent years. In the fourth quarter, JPMorgan paid $1.1 billion to settle charges by U.S. and foreign regulators that its traders had manipulated currency markets. In those settlements, J.P. Morgan did not admit nor deny any wrongdoing. + +"Obviously, companies make mistakes," Dimon said on Wednesday. "We try to resolve it, we try to fix it, we admit it." + +In the past two years, JPMorgan has paid out $14 billion in settlements and fines related to the London Whale trading losses, manipulating the key interest rate benchmark Libor, and issuing bad mortgages that helped lead to the financial crisis. + +The bank earned $39.7 billion in net income over those same two years. + +Despite his complaints, Wall Street lobbyists, and Dimon himself, are orchestrating a "continuing assault" on the 2010 Dodd-Frank financial-reform law that "has achieved remarkable success," as the New York Times put it on Tuesday. + +Dimon personally lobbied for Congress to loosen derivatives rules put in place by Dodd-Frank financial. Late in December, the Federal Reserve gave banks another year to comply with certain aspects of that law's Volcker Rule, which prohibits proprietary trading. And now House Republicans are lining up a vote to extend that grace period until 2019, a full nine years after the bill was put into law. + +If record profits and regulatory rollbacks are Dimon's idea of an assault on banks, it's hard to imagine what he thinks victory looks like. + diff --git a/tests/data/text/huffingtonpost.de1.txt b/tests/data/text/huffingtonpost.de1.txt deleted file mode 100644 index e1277780..00000000 --- a/tests/data/text/huffingtonpost.de1.txt +++ /dev/null @@ -1 +0,0 @@ -// results fragments requested via ajax should be an empty response if ($_GET['ajax'] != 'true'): ? = __("no entries found")?> endif; ? \ No newline at end of file diff --git a/tests/data/text/huffingtonpost.de2.txt b/tests/data/text/huffingtonpost.de2.txt deleted file mode 100644 index bcf2d0a8..00000000 --- a/tests/data/text/huffingtonpost.de2.txt +++ /dev/null @@ -1 +0,0 @@ -Der erste Eindruck zählt - das gilt nicht nur fürs Kennenlernen, sondern auch dann, wenn man die Person zum ersten Mal zu Hause besucht.... \ No newline at end of file diff --git a/tests/data/text/japanese.txt b/tests/data/text/japanese.txt new file mode 100644 index 00000000..f877b85e --- /dev/null +++ b/tests/data/text/japanese.txt @@ -0,0 +1,3 @@ +６月12日の歴史的な米朝首脳会談を巡って戦っていたのは、米大統領のトランプと北朝鮮委員長の金正恩（キム・ジョンウン）ばかりではなかった。中国国家主席、習近平（シー・ジンピン）も重要な陰の参戦者だった。核放棄を巡って「検証可能な」の文字がなく、米韓軍事演習の中止にまで踏み込んだ米朝会談は、トランプによる金正恩への一方的譲歩であり、後ろ盾だった中国の習近平も「勝ち組」になったかに見える。 + +その判断は… \ No newline at end of file diff --git a/tests/data/text/japanese2.txt b/tests/data/text/japanese2.txt new file mode 100644 index 00000000..0feca4fc --- /dev/null +++ b/tests/data/text/japanese2.txt @@ -0,0 +1,11 @@ +【6月18日 AFP】ナイジェリア北東部ボルノ（Borno）州で16日夜、イスラム過激派組織「ボコ・ハラム（Boko Haram）」によるものと思われる少女を使った自爆攻撃が起き、31人が死亡した。地元当局者と民兵指導者が17日、AFPに明らかにした。 + +攻撃はボルノ州の町ダンボア（Damboa）で、イスラム教の断食月「ラマダン（Ramadan）」明けの祭り「イード・アル・フィトル（Eid al-Fitr）」の祝いから帰る途中の人々を狙って起きたもので、ボコ・ハラムの犯行と思われる特徴があった。 + +数回の自爆の後、襲撃者らは自爆攻撃の現場に集まった群集の中に携行式ロケット弾を撃ち込み、死傷者がさらに増えた。 + +地元の民兵指導者は「昨夜ダンボアで2度の自爆攻撃とロケット弾による爆発があり、31人が死亡した。その他に数人が負傷した」と語り、ボコ・ハラムの犯行であることは明らかだと述べた。 + +地元当局者は「死傷者の大半は町の外から発射されたロケット弾によるものだった。事件後に、自爆攻撃が6人の少女によって実行されたことが明らかになった。救急隊が現場で6人の頭部を発見した。顔つきからして7歳から10歳までの少女だった」と述べた。 + +政府はボコ・ハラムが劣勢にあると繰り返し主張しているが、あるアナリストは「ボコ・ハラムはナイジェリア北東部で大量の死傷者を出す攻撃を行う意図と作戦能力を維持している」と指摘している。(c)AFP/Aminu ABUBAKAR \ No newline at end of file diff --git a/tests/data/text/lifebuzz.com1.txt b/tests/data/text/lifebuzz.com1.txt index c752b862..83b558a9 100644 --- a/tests/data/text/lifebuzz.com1.txt +++ b/tests/data/text/lifebuzz.com1.txt @@ -1,5 +1,34 @@ +By Lisa Be LifeBuzz Staff 1.7Mshares Share on Facebook + + + + + + + + + +× 1594x906 × Want More Positive Stories Like This One? Step 1: Like Us! Step 2: Join Our Newsletter I've completed the above steps, don't show me again. + She Was Tired Of Being Photoshopped, So Here’s What She Did About It. -Grammy Award-winning, singer-songwriter Colbie Caillat released her new EP, Gypsy Heart Side A. Her album’s lead single “Try” makes a powerful statement about beauty ideals. Share this with all of your girlfriends by clicking the Share button below. +Grammy Award-winning, singer-songwriter Colbie Caillat released her new EP, Gypsy Heart Side A. Her album’s lead single “Try” makes a powerful statement about beauty ideals. + + + + + + + +Like on Facebook Share this with all of your girlfriends by clicking the Share button below. Source: ColbieCaillatVEVO + +1.7Mshares Share on Facebook + + + + + + + + -Tell us what you think \ No newline at end of file diff --git a/tests/data/text/lifebuzz.com2.txt b/tests/data/text/lifebuzz.com2.txt index b15966c4..e7dea486 100644 --- a/tests/data/text/lifebuzz.com2.txt +++ b/tests/data/text/lifebuzz.com2.txt @@ -1,5 +1,48 @@ What Do You Think This Guy Is Doing? You Will Never Guess…. And It’s Going To Break Your Heart -Sylvie a 13 year old Husky went on her morning walk with her owner when she fell through the ice. After being trapped in the water for about 30 minutes, Firefighter Sean Coyle came to the rescue. The pictures below capture the event as it transpires. Exhausted Sylvie had been in the frigid water for more than 30 minutes Firefighter Sean Coyle uses a basket to slide out to Sylvie The husky clings to the ice as firefighter Sean Coyle inches out to the hole Fireman Coyle grabs Sylvie by the scuff of the neck as he attempts to lift her from the water +Sylvie a 13 year old Husky went on her morning walk with her owner when she fell through the ice. After being trapped in the water for about 30 minutes, Firefighter Sean Coyle came to the rescue. The pictures below capture the event as it transpires. Exhausted Sylvie had been in the frigid water for more than 30 minutes + + + + + + + +Boston Herald / Polaris Firefighter Sean Coyle uses a basket to slide out to Sylvie + + + + + + + +Boston Herald / Polaris The husky clings to the ice as firefighter Sean Coyle inches out to the hole + + + + + + + +Boston Herald / Polaris Fireman Coyle grabs Sylvie by the scuff of the neck as he attempts to lift her from the water + + + + + + + +Boston Herald / Polaris + +Next › Page 1 of 2 + +16shares Share on Facebook + + + + + + + + -Tell us what you think \ No newline at end of file diff --git a/tests/data/text/livescience.com1.txt b/tests/data/text/livescience.com1.txt index 6a90f25a..33f48cdb 100644 --- a/tests/data/text/livescience.com1.txt +++ b/tests/data/text/livescience.com1.txt @@ -1,13 +1,23 @@ +Magic mushrooms can create vivid hallucinations of strange and fantastical things, such as abstract geometric shapes. + +Credit: WhiteHaven/Shutterstock.com View full size image + +Magic mushrooms may give users trippy experiences by creating a hyperconnected brain. + The active ingredient in the psychedelic drug, psilocybin, seems to completely disrupt the normal communication networks in the brain, by connecting "brain regions that don't normally talk together," said study co-author Paul Expert, a physicist at King's College London. The research, which was published today (Oct. 28) in the Journal of the Royal Society Interface, is part of a larger effort to understand how psychedelic drugs work, in the hopes that they could one day be used by psychiatrists — in carefully controlled settings — to treat conditions such as depression, Expert said. [Trippy Tales: The History of 8 Hallucinogens] +Magic mushrooms + Psilocybin, the active ingredient in magic mushrooms, is best known for triggering vivid hallucinations. It can make colors seem oversaturated and dissolve the boundaries between objects. But the drug also seems to have more long-lasting effects. Many people report intensely spiritual experiences while taking the drug, and some studies even suggest that one transcendent trip can alter people's personalities on a long-term basis, making those individuals more open to new experiences and more appreciative of art, curiosity and emotion. People who experiment with psilocybin "report it as one of the most profound experiences they've had in their lives, even comparing it to the birth of their children," Expert told Live Science. +Making connections + Scientists have long known that psilocybin binds to a receptor in the brain for serotonin, a brain chemical that plays a role in mood, appetite and sleep, but exactly how the drug transforms the whole brain's pattern of communication isn't clear. In past work, Expert's colleagues had found that psilocybin spurred the brain into a more dreamlike state, and that the drug decreased brain activity. @@ -18,6 +28,8 @@ The team then compared the brain activity of the individuals on and off the drug Psilocybin dramatically transformed the participants' brain organization, Expert said. With the drug, normally unconnected brain regions showed brain activity that was synchronized tightly in time. That suggested the drug was stimulating long-range connections the brain normally wouldn't make. After the drug wore off, brain activity went back to normal. +Drug's effect + Psilocybin may create a brain state akin to synesthesia, a sensory effect in which one sense stimulus (such as a number) always gets paired in the brain with another (such as a color or a sound), the researchers wrote in the paper. People with synesthesia may see certain colors when they hear music, or always see the number 3 in yellow, for instance, Expert said. The findings could help scientists who are studying the drug as a potential treatment for depression, Expert said. Past work has found that people tend to be happier even after using psilocybin just once, but scientists would need to get a much better picture of how the drug impacts the brain before using psilocybin to treat depression, Expert said. diff --git a/tests/data/text/livescience.com2.txt b/tests/data/text/livescience.com2.txt index 39147386..a3ec91df 100644 --- a/tests/data/text/livescience.com2.txt +++ b/tests/data/text/livescience.com2.txt @@ -1,3 +1,7 @@ +The zombies in the 1968 film "Night of the Living Dead" had a taste for flesh, but had not yet broadened their horizons to braaaiins. + +Credit: Public Domain View full size image + They moan. They bite. They shuffle. Or sometimes, they sprint, swarm and carry on surprisingly intelligent conversation. Zombies are something of an open-source pop-culture phenomenon. Unlike Dracula or Frankenstein, these Halloween monsters aren't based on a literary resource. In fact, the modern conception of a zombie dates back to 1968, in a movie that doesn't so much as use the word: George Romero's "Night of the Living Dead." @@ -6,6 +10,12 @@ Zombies are something of an open-source pop-culture phenomenon. Unlike Dracula o "Audiences saw these lumbering dead people, and they called them zombies … therefore, they became zombies," he said. Since then, the walking undead have wormed their way into video games, comic books — and even the classics (witness 2009's novel "Pride and Prejudice and Zombies). [Zombies! Your Complete Guide to the Attack of the Dead (Infographic) ] +From Haiti to Hollywood + +Facts about the zombie phenomenon in pop culture, movies and gaming. [ + +Credit: By Karl Tate, Infographics Artist View full size image Facts about the zombie phenomenon in pop culture, movies and gaming. [ See full infographic + The true zombie origin story dates back further than 1968, of course. The sad beginning of the myth harks back to Haiti during the 1600s and 1700s, when African slaves were worked to death on sugar plantations. As UC Irvine journalism instructor Amy Wilentz pointed out in the New York Times in 2012, it's not hard to see how the notion of a dead body, stripped of will and personality, forced to do the bidding of a sorcerer, would occur to an enslaved people. The notion of zombies is still part of Haitian folklore. The belief is that, through magic or poison, a sorcerer makes a person fall ill and appear to die. After the family buries the body, the sorcerer retrieves the person, who is alive, but held in thrall. In a 1997 article in the medical journal The Lancet, researchers studied three real-life cases of "zombification" and diagnosed the three sufferers with catatonic schizophrenia, epilepsy and mistaken identity. In the final case, a 31-year-old woman with possible fetal alcohol syndrome was mistaken for another woman who had died 13 years before. The cases suggest that zombification has often been used to explain mental illness or brain disorders in rural Haiti. @@ -14,10 +24,16 @@ Zombies made the leap from Haitian religion to American entertainment in 1932, I Zombies popped up in horror flicks over the next few decades, frequently in keeping with the Haitian voodoo theme, but sometimes branching out: The "Revenge of the Zombies" in 1943 took a sci-fi angle, with a Nazi scientist trying to create an army of the undead for Hitler. [The 10 Weirdest Ways We Deal with the Dead] +The modern zombie + But the quintessential zombie flick was actually inspired less by these films and more by vampires. Working off the 1954 post-apocalyptic novel "I Am Legend," by Richard Matheson, which tells the story of the last man standing in a world of vampirelike monsters, George Romero and John Russo told their own tale of a group of bickering humans threatened by the shuffling, moaning living dead. "He took the Haitian component out of the previous character storylines and brought them here, to suburbia," Inguanzo said. "They were our friends, our relatives, our neighbors who were coming back from the dead." +The fast-paced zombies of "World War Z." + +Credit: Paramount Pictures View full size image The fast-paced zombies of "World War Z." + Romero's zombies became the touchstone for those that would follow, with writers adding their own quirks to the genre. In 1985's "Return of the Living Dead," the zombies hungered not just for human flesh, but also for "braaaains." That idea stuck. The movie was also the first to introduce talking zombies, and perhaps more crucially, fast-moving zombies. These days, fast-walking zombies are overtaking their shambling counterparts, at least on the big screen. The trend is, in part, inspired by video games like "Resident Evil," which first came out in 1996. diff --git a/tests/data/text/mashable.com2.txt b/tests/data/text/mashable.com2.txt index eb371a26..83d87321 100644 --- a/tests/data/text/mashable.com2.txt +++ b/tests/data/text/mashable.com2.txt @@ -2,14 +2,36 @@ A power outage at Stansted International Airport in London held up holiday trave The power outage lasted about two hours. Photos uploaded by holiday travelers show massive queues, although the outage affected only parts of the airport. +See also: The worst things people do on airplanes, ranked + +A power failure is currently affecting certain parts of @STN_Airport. The main terminal building and runway are unaffected (1/2) — Stansted Airport (@STN_Airport) December 22, 2014 + +Satellite failure at stansted airport - no planes landing or departing... Things surprisingly calm so far... — Fiona Kelly (@fizzlekelly) December 22, 2014 + The power outage is affecting the transit system that gets travelers to their airplanes, according to the airport. Stansted is located about 40 miles north of London's center; the terminal most affected is Ryanair. +Power outage at London Stansted, great day for it :/ — Kenneth Barrett (@kennbarr) December 22, 2014 + +power failure at stansted airport no planes going in or out :( #stansted pic.twitter.com/Q8FFsT2ITu — Lena (@lenanix) December 22, 2014 + +Lights still on in main #Stansted terminal but #RyanAir wing dark. Staff trying to get people out but connecting doors now appear jammed — Leigh Holmwood (@LeighHolmwood) December 22, 2014 + +Udajne velky vypadek proudu na londynskem letisti #Stansted. Nefunguje, zda se, vubec nic. Minimum informaci. Zmatek. pic.twitter.com/Bf2l5KcxJQ — Jiří Hošek (@hosekj) December 22, 2014 + +The scene at Stansted Airport with no flights currently going in or out due to a power failure. Merry Xmas everyone! pic.twitter.com/1LFnD6Onzr — Jennie Thomas (@localgovlove) December 22, 2014 + Planes on the tarmac were also affected, as passengers had to wait because of the outage. The airport was dispatching buses to take passengers back and forth. +Can't get off the plane. Actually it's not Ryanair's fault. It's Stansted's. Apparently there's a power cut.... — Jackzelia Banks (@JackieRue) December 22, 2014 + +Standing stranded in Stansted #punny #sorrynotsorry — Johnathan Mei (@jonomei) December 22, 2014 + The airport reported that crews had restored power at about 5:30 p.m. local time (12:30 p.m. EST). +Power has now been restored. Flights continue to operate but are subject to delay. We apologise to all passengers delayed this afternoon — Stansted Airport (@STN_Airport) December 22, 2014 + While much smaller than Heathrow or Gatwick airports in London, Stansted had its busiest November in eight years last month with more than 1.5 million passengers. Stansted Airport reported that there were some delays, but flights are operating. FlightAware was not reporting a large number of delays in the early afternoon. diff --git a/tests/data/text/newyorker.com1.txt b/tests/data/text/newyorker.com1.txt index 38129c7c..521ca014 100644 --- a/tests/data/text/newyorker.com1.txt +++ b/tests/data/text/newyorker.com1.txt @@ -1,23 +1 @@ -outside is frightful, The heat wave brutal and spiteful. Our crops have no water to grow— Let it snow, let it snow, let it snow! - -Looks like the heat wave ain’t stoppin’, Our dust-storm coughs are a-whoppin’. All fish went extinct long ago— For the love of sweet Christ, let it snow! - -may he rest in peace, poor soul. With his melted nose and his melted mouth, And two eyes made out of coal. Frosty the Snowman, did you say his eyes were coal? Can you pass that coal? We could use some coal. Let us rob his grave for coal. - -oh, Christmas tree! Thy plastic branches don’t shed. Oh, Christmas tree, oh, Christmas tree! We wish real trees weren’t all dead. - -are you listening? In L.A., hail is glistening. In New Mexico, there is six feet of snow. The desert is a winter wonderland! - -Gone away is the West Side, Here to stay is the high tide. If you want to spelunk, San Francisco has sunk. California’s now an underwater land! - -In Mojave, we can build a snowman, And pretend that he is Mom or Dad. He’ll say, “Did your parents escape the horrific flash floods?,” and we’ll say, “No, man. But you can be our new dad now.” - -an ozone layer, Just like the one I used to know, Which kept out UV rays, Before aerosol sprays, And allowed for actual snow. - -of our mountain hideout, Fa la la la la, la la la la! Hope all humans have not died out, Fa la la la la, la la la la! Don our hazmat suits on tight now, Fa la la, la la la, la la la! Oxygen is leaking right now, Fa la—! - -on an open fire, Cars and houses roasting, too. Our whole city is an open pyre— Put those chestnuts away, we have to leave now. - -little dreidel, I made it out of clay. And when it’s dry and ready, Oh, dreidel, I will pray— That I can barter you for enough cans of soup to last me through the tornado. - -Endless night! All is dark, there’s no light. Cyclone clouds have blocked out the sky, We’re almost out of our dry-meat supply. Sleep in uneasy peace. We may have to eat Aunt Bernice. ♦ \ No newline at end of file +VERY LAST LINE IN THE POEM IS MISSED diff --git a/tests/data/text/newyorker.com2.txt b/tests/data/text/newyorker.com2.txt deleted file mode 100644 index 742d5fe7..00000000 --- a/tests/data/text/newyorker.com2.txt +++ /dev/null @@ -1,5 +0,0 @@ -Subscribe to 'The New Yorker' - - on YouTube to keep up with all of - - our latest videos and shows. \ No newline at end of file diff --git a/tests/data/text/nj.com1.txt b/tests/data/text/nj.com1.txt index b54259ba..3b76a05b 100644 --- a/tests/data/text/nj.com1.txt +++ b/tests/data/text/nj.com1.txt @@ -1 +1,83 @@ -Register now for free, or sign in with any of these services: \ No newline at end of file +TRENTON — Gov. Chris Christie’s State of the State speech on Tuesday pushed one of New Jersey’s most conservative state senators over the edge, leading to the harshest criticism of the governor ever from a Republican state lawmaker. + + + +State Sen. Michael Doherty (R-Warren) said Christie when he first ran in 2009 “sold himself to the people of New Jersey that he was going to change it.” + + + +“And he didn’t,” Doherty said in a phone interview. + + + +“The governor’s possible national ambitions have distracted him from doing the job he was hired to do in the State of New Jersey,” he said. + + + +In an interview with PolitickerNJ.com published this morning, Doherty was quoted calling Christie a “failed” governor. Asked about it by NJ Advance Media, Doherty declined to repeat that exact word. But he nonetheless offered a scorching review of Christie’s speech — in particular his focus on Camden. + + + +“I don’t know that I particularly said the word ‘failed,’” Doherty said. “What I said was that Gov. Christie has missed his opportunity to make a great mark on the State of New Jersey.” + + + +Most New Jersey Republicans have been loath to publicly criticize Christie, and many on Tuesday issued statements praising his speech and record. State Sen. Dawn Addiego (R-Burlington), for example, said Christie "has dedicated his efforts to make New Jersey affordable and to provide opportunity for the families who live here." + +And Doherty has been the most critical of Christie among Republican lawmakers in the past. He is the only Republican who has so far agreed to vote to override Christie’s veto of a bill to overhaul the Port Authority. And Doherty in 2009 also supported Christie's primary rival for the GOP nomination for governor, Steve Lonegan. + +Still, Doherty’s past comments were restrained compared to the torrent he released today. + + + +Doherty, who represents a largely rural district, says his main frustration is that New Jersey’s poor, urban school districts are still getting a disproportionate amount of state aid thanks to a series of Supreme Court decisions dating back to the 1970s known as Abbott v. Burke. + + + +"For him yesterday to hold up Camden County government as an example of what we should all be? Give me a break. It’s one of the most wasteful and corrupt governments in the State of New Jersey," Doherty said. "How long can the suburban and rural towns be told keep funneling your money into the cities, we’re just around the corner, everything’s going to recover and everything’s going to come back to normal?" + + + +Doherty was particularly angry that Hoboken— once impoverished but now a wealthy New York City bedroom community — is still considered an “Abbott district” and has state-funded universal pre-K. (One town in Doherty’s district, Phillipsburg, is also an Abbott district). + + + +“We’re struggling tremendously. Houses in my district are selling for less than they did a quarter century ago. There has been no economic recovery. We’re getting our clocks cleaned by Pennslyvania. They’re so close,” Doherty said. “We need help.” + + + +Christie was critical of those decisions during his first run for governor in 2009. And while that led to some battles with Democrats who control the state Senate over Supreme Court nominees, Doherty said Christie “didn’t fight hard enough” to change the court’s makeup. + + + +For instance, Doherty noted that Christie declined to re-nominate former Justice Helen Hoens to the bench. + + + +“The one justice that wrote an opinion that all of us in the suburbs and rural areas agree with regarding school funding and challenging the urban areas to do better than they had was Justice Helen Hoens,” Doherty said. + + + +Doherty said Christie’s appeal to New Jersey’s urban mayors to follow his example of working with Camden to overhaul its police force and attract businesses with economic incentives underscored his feeling that he was ignoring the state’s suburban and rural areas. + + + +“I was very alarmed when Gov. Christie said to be ‘Big cities, come to me and we’re rebuild you,” Doherty said. “When does it ever end? These suburban areas of New Jersey are struggling tremendously. They never recovered from the Great Recession. And we’re told keep sending the money in.” + + + +Doherty said Christie has “did not follow through his promise to fight the corruption and break up the status quo.” + + + +There are some political undercurrents in Doherty's criticism. Doherty was a supporter of U.S. Rep. Ron Paul’s campaigns for the Republican presidential nomination in 2008 and 2012. Paul’s son, U.S. Sen. Rand Paul of Kentucky, is considered a likely presidential candidate in 2016. But Doherty said that had nothing to do with his criticism. + + + +“I’m not supporting anybody at this point,” he said. + +A spokesman for Christie declined to comment. + +Loading Photo Gallery + +Matt Friedman may be reached at mfriedman@njadvancemedia.com. Follow him on Twitter @MattFriedmanSL. Find NJ.com on Facebook. \ No newline at end of file diff --git a/tests/data/text/nj.com2.txt b/tests/data/text/nj.com2.txt deleted file mode 100644 index b54259ba..00000000 --- a/tests/data/text/nj.com2.txt +++ /dev/null @@ -1 +0,0 @@ -Register now for free, or sign in with any of these services: \ No newline at end of file diff --git a/tests/data/text/nola.com1.txt b/tests/data/text/nola.com1.txt index c9a87cbf..e3fcf3d6 100644 --- a/tests/data/text/nola.com1.txt +++ b/tests/data/text/nola.com1.txt @@ -28,6 +28,8 @@ In the late 1960s, the city staged two versions of the International Jazz Festiv "George and Joyce Wein have done so much to benefit our community and our culture," Demetric Mercadel, president of the Jazz & Heritage Foundation's board of directors, said in a statement. "It is only fitting that we recognize their many contributions by having their names grace this wonderful new facility. This is a true testament to their legacy." +Grand-opening festivities continue throughout the weekend. + On Friday (Dec. 12) at 8 p.m., avant-jazz saxophonist and educator Kidd Jordan - who founded the foundation's Don "Moose" Jamison Heritage School of Music in 1990 -- and his accomplished offspring, Kent, Stephanie, Marlon and Rachel Jordan, will headline a grand opening concert; students from the Heritage School of Music will open the show. Admission is free, but all advance tickets are sold out; any remaining seats will be available on a first-come, first-serve basis. The concert will also be live streamed at wwoz.org, the web site of the foundation-owned on WWOZ-FM. On Saturday (Dec. 13), the center throws open its doors for a Treme Neighborhood and Community Open House. Students from the Heritage School of Music will also perform a free holiday concert on Saturday. \ No newline at end of file diff --git a/tests/data/text/nola.com2.txt b/tests/data/text/nola.com2.txt deleted file mode 100644 index b14dca93..00000000 --- a/tests/data/text/nola.com2.txt +++ /dev/null @@ -1 +0,0 @@ -Holden is an associate in the firm's New Orleans office, practicing in the areas of business and energy litigation. She received her Juris Doctorate from Loyola University New Orleans College of Law, where she graduated summa cum laude. At Loyola, Ms. Holden served as the Articles Editor of the Loyola Law Review and as a brief writer and oralist for the 2011-2012 National Environmental Law Moot Court Team. Holden was recognized as a William L. Crowe, Sr., Scholar and received the Faculty Award, the LSBA Civil Law Award, the Dean's Award, the Joseph V. Bologna Award, the Warren E. Mouledoux Professional Responsibility Award, and the LSBA Corporate and Business Award. Prior to joining the firm, she served as a law clerk to the Honorable Carl J. Barbier in the United States District Court for the Eastern District of Louisiana. \ No newline at end of file diff --git a/tests/data/text/nydailynews.com1.txt b/tests/data/text/nydailynews.com1.txt index 546b91be..1ece56e0 100644 --- a/tests/data/text/nydailynews.com1.txt +++ b/tests/data/text/nydailynews.com1.txt @@ -1,3 +1,25 @@ -Thank you for using the New York Daily News to find your next new car. +Kevin C. Downs/for New York Daily News London Rene and 'Mob Wives' star Natalie Guercio pose in the office of their lawyer, Scott Rynecki. The couple was outside a Brooklyn club on Dec. 28 when a man with a box cutter attacked Rene. -In order to Find Autos By Brand it is necessary to download a browser that can support it, such as Firefox, Chrome, or Safari. \ No newline at end of file +The boyfriend of sexy “Mob Wives” star Natalie Guercio is suing the Brooklyn club where he was brutally slashed by a box cutter-wielding assailant, the Daily News has learned. + +London Rene is alleging Club Output's "inept" and "incompetent" security was negligent in failing to provide proper safety for its patrons on Dec. 28 when he was attacked outside the Williamsburg nightspot. + +Rene's lawyer Scott Rynecki said there was "little or no security" watching over the club, which has a capacity of 600-persons. + +The suit, which will be filed Thursday in Brooklyn Supreme Court, seeks unspecified monetary damages for Rene's physical and emotional scars. The court papers also name the security guard firm hired by the club. + +"When I look in the mirror I'm angry," Rene, 37, said Wednesday. "I'm scarred for life; the scars are going to be there until the day I die." + +Kevin C. Downs/for New York Daily News Rene seeks unspecified monetary damages for his physical and emotional scars. He plans to file the lawsuit Thursday. Kevin C. Downs/for New York Daily News London Rene suffered scars from the attack. Previous Next + +Enlarge + +The alleged slasher, Rodolfo Lopez, fled the scene and later surrendered to the NYPD. + +Rene's face and torso were cut in the attack. Guercio, whose family owns a funeral home in Philadelphia where mobsters and their families were laid out, witnessed the slashing but she was not hurt. + +"Any one of us could have been a victim," she said. + +There was no listed phone number for the club. + +jmarzulli@nydailynews.com \ No newline at end of file diff --git a/tests/data/text/nydailynews.com2.txt b/tests/data/text/nydailynews.com2.txt deleted file mode 100644 index 6efe448b..00000000 --- a/tests/data/text/nydailynews.com2.txt +++ /dev/null @@ -1,27 +0,0 @@ -Best New York Daily News Front Covers of 2014 From the death of Robin Williams and Philip Seymour Hoffman to the rise of ISIS, the Daily News has covered the biggest stories of 2014. - -As 2014 winds down, take a look back at some of the biggest news stories of the year ... - -The devastating 9.1 magnitude earthquake that caused the 2004 Boxing Day tsunami left more than 230,000 dead in 14 countries. Take a look back at... - -The best Daily News photos of 2014 Through the lens of New York Daily News photographers, take a look back at the years most striking moments - -An 18-year-old black teenager has been shot dead by police at a gas station in St. Louis, Missouri on Dec. 24, 2014, just 5 miles from where Mich... - -With the Sony World Photography Awards contest quickly coming to an end, the top 15 best entries in the 'open' category so far have been surfacin... - -After receiving more than 9,200 enteries from over 150 countries, the winners of the 2014 National Geographic Photo Contest have finally been ann... - -The decline and decay of Cuba In 1962, the United States set an embargo against Cuba effectively cutting off trade between the two countries. Since then, Cuba has been faced w... - -Most shocking red carpet looks of 2014 It was the year of more skin and less dress! From plunging necklines to cut-outs and slits, 2014 had no shortage of shocking dresses on the red c... - -Thousands of protesters gathered in New York City, Washington D.C., and elswehere for an anti-police violence demonstration in response to the po... - -There's another royal baby on the way! In celebration of Kate Middleton and Prince William's news that they're expecting their second child, take... - -He was America's favorite TV Dad but now people are changing their tune as more sexual assault allegations shed light on the actors past. - -Following the Ferguson verdict and protests, thousands gathered across the country after a New York grand jury decided not to indict the police o... - -A look back at the life and career of Paul Walker Paul Walker, best known for his role in the "Fast & Furious" franchise, died in a fiery car crash north of Los Angeles on Nov. 30, 2013. He was 4... \ No newline at end of file diff --git a/tests/data/text/nypost.com1.txt b/tests/data/text/nypost.com1.txt index ffa54ea2..c76eaff2 100644 --- a/tests/data/text/nypost.com1.txt +++ b/tests/data/text/nypost.com1.txt @@ -1,3 +1,5 @@ +Barely a shot was fired. + Book publishing giant Macmillan announced a peaceful settlement with Amazon on Thursday on pricing for print and digital books ordered through the giant online retailer. Similar agreements were reached with Simon & Schuster in October and Hachette in November. diff --git a/tests/data/text/nypost.com2.txt b/tests/data/text/nypost.com2.txt index 49be3943..a91ba55c 100644 --- a/tests/data/text/nypost.com2.txt +++ b/tests/data/text/nypost.com2.txt @@ -14,4 +14,24 @@ The beach remained open, but signs will be posted for three days warning the pub Sharks are native to the area, and Colligan said that they are spotted several times a year. He added that attacks like this are rare. -A woman swimming with seals was killed by a shark in 2003 about 10 miles south of the most recent attack, Colligan said. \ No newline at end of file +A woman swimming with seals was killed by a shark in 2003 about 10 miles south of the most recent attack, Colligan said. + +Share this: Share + +Facebook + +Twitter + + + +Google + +WhatsApp + + + +Email + + + +This article originally appeared on Fox News. \ No newline at end of file diff --git a/tests/data/text/ok.co.uk1.txt b/tests/data/text/ok.co.uk1.txt index c92f5731..6e3e93e2 100644 --- a/tests/data/text/ok.co.uk1.txt +++ b/tests/data/text/ok.co.uk1.txt @@ -2,6 +2,8 @@ WE can't keep up with all the engagements happening at the moment. And it seems that there could be another one to add to our list. +Has Sean Penn popped the question? [Wenn] + Sean Penn has reportedly popped the question to Charlize Theron. The I Am Sam actor is said to have proposed while on a romantic trip to Paris in November. @@ -10,6 +12,8 @@ US sources claimed that the 54-year-old was keen to take his relationship with t A source added that Charlize isn't yet wearing a ring to symbolise their union. +The couple have been dating for the last year [Splash] + They said: "There's no ring, but they are committed." The pair have been friends for a long time but only became romantically involved last year. @@ -17,3 +21,13 @@ The pair have been friends for a long time but only became romantically involved It will be the third time Sean has walked down the aisle. He was previously married to Madonna and split from second wife Robin Wright in 2010. The couple had two children together, Hopper and Dylan. + +Will it be third time lucky for Madonna's ex-husband? [Splash] + +Related article: Charlize Theron and Sean Penn plan to marry and adopt + +Related article: Charlize Theron causes outrage as she compares 'Googling herself to being raped' + +Related article: Josie Cunningham is engaged + +Related article: Hollyoaks co-stars Emmett Scanlan and Claire Cooper get engaged \ No newline at end of file diff --git a/tests/data/text/ok.co.uk2.txt b/tests/data/text/ok.co.uk2.txt index 4add2eb9..a6d9db0e 100644 --- a/tests/data/text/ok.co.uk2.txt +++ b/tests/data/text/ok.co.uk2.txt @@ -2,6 +2,8 @@ EastEnders are set to show a controversial plot line in the near future. Kat Moon will discover that her late uncle Harry is a serial sex offender. +Kat Slater will discover her uncle was a serial sex offender + Alfie Moon's wife was groomed by the relative from a young age. As a result of rape, she gave birth to her daughter Zoe Slater at the age of 13. @@ -10,6 +12,8 @@ However, Kat pretended to be her sister until the secret was revealed years late Police will turn up at Kat's house telling her that a number of victims have come forward with allegations against Harry, despite him passing away 10 years ago. +Previous victims of Harry Slater will come forward + Kat, played by Jessie Wallace, will be drawn to helping the investigation after discovering that she has been left money in his will. She then confides in her husband who believes that the heart-to-heart means a reunion is possible. Struggling to cope, Kat has a one-night stand in an attempt to take her mind off the situation. @@ -17,3 +21,13 @@ She then confides in her husband who believes that the heart-to-heart means a re A show insider told The Sun: "EastEnders has a rich history of tackling difficult social issues and Kat's continued story is one of these." Played by the late Michael Elphick, Harry passed away off screen when he suffered a heart attack. + +She gave birth to daughter Zoe Slater at 13 after being raped + +Related article: Laurie Brett splits from husband John Milroy + +Related article: EastEnders spoiler - Character to be killed off on New Year's Day + +Related article: Could Kat and Alfie Moon's marriage go up in smoke? + +Related article: Lucy Beale killer discovered by Emma Summerhayes \ No newline at end of file diff --git a/tests/data/text/oregonlive.com1.txt b/tests/data/text/oregonlive.com1.txt index ed4195cd..e288a6de 100644 --- a/tests/data/text/oregonlive.com1.txt +++ b/tests/data/text/oregonlive.com1.txt @@ -1,13 +1,35 @@ +GRANTS/DONATIONS + BEAVERTON -- The Comcast Foundation has awarded $355,760 in grants to 31 nonprofit organizations in Oregon and Southwest Washington. The grants support programs aimed at the Comcast Foundation's areas of focus -- expanding digital literacy, promoting community service, and building tomorrow's leaders. +Recipients include: + +The Native American Youth & Family grant will fund an on-site component of the College and Career Center program, primarily built around students accessing a computer lab with printing and mailing options to help the students apply for Colleges and seek employment in STEM related fields. + +grant will fund an on-site component of the College and Career Center program, primarily built around students accessing a computer lab with printing and mailing options to help the students apply for Colleges and seek employment in STEM related fields. The Comcast Foundation's grant awarded to Big Brothers Big Sisters Columbia Northwest funded the Beyond School Walls, a workplace mentoring program. The mission of Big Brothers Big Sisters is to provide children facing adversity with strong and enduring, professionally supported one-to-one relationships that change their lives for the better. + +funded the Beyond School Walls, a workplace mentoring program. The mission of Big Brothers Big Sisters is to provide children facing adversity with strong and enduring, professionally supported one-to-one relationships that change their lives for the better. To support the needs of BGCP's My.Future technology initiative, the Comcast Foundation provided funds for program software, robotics kits, iPads, laptops and other needed materials and supplies through a multi-year grant beginning in 2014. The grant positions Comcast as a lead partner in funding the technology needs for BGCP's Rockwood Club, its seventh and newest facility set to open in the fall of 2016. + Local recipients also include Junior Achievement of Oregon & SW Washington, Boys and Girls Club of Albany and Hacienda Community Development Corporation of Portland. +Visit the website for a complete list. + PORTLAND -- NW Natural's Corporate Philanthropy Fund is contributing nearly $35,000 to local nonprofits that help children and families in need. The fund, supported by shareholders, is making the following donations: +$7,500 to Stand For Children + +$6,500 to Oregon Food Bank + +$5,000 each to Central City Concern, Clackamas Women's Shelter and Self Enhancement, Inc. + +and $2,500 to Saturday Academy + +$1,500 each to Start Making a Reader Today and Food Share of Lincoln County + YAMHILL COUNTY -- The Yamhill County Sheriff's Office Project Lifesaver Team is the recipient of a bi-annual grant from the Alzheimer's Foundation of America for $5,000. This grant will enhance the current Project Lifesaver Program by upgrading certain equipment and providing up to 15 new transmitters for new clients. @@ -16,10 +38,14 @@ The Yamhill County Sheriff's Office and the Yamhill County Sheriff's Search and The primary mission of Project Lifesaver is to provide timely response to save lives and reduce potential injury for adults and children who wander due to Alzheimer's, autism, and other related conditions or disorders. The program consists of a bracelet transmitter, which is place on a client's wrist or ankle, and a specialized receiver that the Project Lifesaver team uses to track the specific radio frequency that the transmitter emits on a constant basis. +More Real estate transactions, awards, mergers and acquisitions, tech notes and grants + The Project Lifesaver Team is made up of volunteers from the Search and Rescue Team along with deputies who are trained on locating the clients and how to deal with the clients once they are located. Once a month, a member of the team visits the clients to change batteries and bands that hold the transmitter in place. Project Lifesaver aids Search and Rescue in reducing time searching for a client that goes missing. To date they have been 100 percent in finding clients who left their homes. Generally the searches last under 10 minutes once the team is on scene. Yamhill County has had 19 searches since 2007 for clients on the Project Lifesaver program. Project Lifesaver is provided free of charge to Yamhill County residents. +NEW BUSINESS + PORTLAND -- Pam Adkins and Mary Kay Plass have opened a new bottle shop and tap room at 4214 N. Mississippi Ave. The Beer City Bottle Shop sells bottled craft beer, hard cider and sodas as well as offering eight rotating draft selections all produced on the west coast. @@ -30,4 +56,8 @@ One of the most popular offerings so far, said the owners, is the Pick 6, where The shop is open from 3 to 9 p.m. Monday through Thursday, from 1 to 10 p.m. Friday and Saturday and from 1 to 8 p.m. Sunday. -PORTLAND -- John W. Houston, managing director of the Financial Institutions Division of Raymond James Financial Services, Inc., announced that OnPoint Community Credit Union, headquartered in Portland, has partnered with and will offer investment and wealth management services to its clients through Raymond James. \ No newline at end of file +NEW PARTNERSHIP + +PORTLAND -- John W. Houston, managing director of the Financial Institutions Division of Raymond James Financial Services, Inc., announced that OnPoint Community Credit Union, headquartered in Portland, has partnered with and will offer investment and wealth management services to its clients through Raymond James. + +-- Compiled by Vickie Kavanagh \ No newline at end of file diff --git a/tests/data/text/oregonlive.com2.txt b/tests/data/text/oregonlive.com2.txt index 4f526533..4de68c74 100644 --- a/tests/data/text/oregonlive.com2.txt +++ b/tests/data/text/oregonlive.com2.txt @@ -2,6 +2,8 @@ A 36-year-old part-time missionary who served a year in a Cambodian prison for s Daniel Stephen Johnson faces a potential 30-year prison term on the new charge, which accuses him of having sex with a boy in the Kingdom of Cambodia sometime between Nov. 28, 2005, and Oct. 12, 2006. +U.S. Magistrate Judge Thomas Coffin set Johnson's trial for Feb. 25. + A federal grand jury indicted Johnson on Dec. 10. He's awaiting trial in the Lane County Jail. Johnson served a one-year sentence in Cambodia for sexually abusing boys in his care at an orphanage, The Register-Guard newspaper reported. He worked as a Christian missionary in the Southeast Asian country for about a decade, according to the Cambodia-based anti-pedophile group Action pour les Enfants. @@ -10,4 +12,6 @@ A 2003 federal law aimed at preventing child abuse made it a crime for any U.S. More than a decade ago, Johnson was accused in Oregon of molesting three children in his sister's care. -Lincoln County prosecutors dismissed charges after investigators began to doubt the alleged victims' statements, according to a 2003 article in the Yamhill Valley News-Register. \ No newline at end of file +Lincoln County prosecutors dismissed charges after investigators began to doubt the alleged victims' statements, according to a 2003 article in the Yamhill Valley News-Register. + +-- Bryan Denson \ No newline at end of file diff --git a/tests/data/text/parsely.com1.txt b/tests/data/text/parsely.com1.txt index 039c5216..963010ce 100644 --- a/tests/data/text/parsely.com1.txt +++ b/tests/data/text/parsely.com1.txt @@ -2,12 +2,26 @@ Yesterday, Facebook announced a number of new tools for publishers, as part of t At the bottom was a short mention: +We also recently fixed a bug that caused many third-party analytics tools to undercount the percentage of their organic traffic from Facebook. In the past, referrer data was missing from some of our outbound mobile clicks. This issue has been resolved for iOS, with a fix for Android coming soon. Publishers may see an increase in iOS mobile traffic from Facebook and a decrease in undefined sources or direct traffic. + ICYMI – this is a fix for direct traffic (sometimes called “dark social”) for traffic coming through the mobile app that sometimes dropped the referral information. +Here’s a little background: + Anyone who has looked at analytics, has seen “direct” as a traffic source. There tends to be a fair amount of confusion around this section, as it encompasses anything that analytics can’t track. Traditionally, publishers assumed direct traffic was people bookmarking their home page or typing in the address directly in the search bar. However, since Parse.ly looks at analytics by a post-by-post basis, we can’t make that assumption, as most people are not typing in new article post URLs “directly” into their browser. +Sources for this traffic include: + +Email/IMs + +Copying and pasting a link + +Switching from a https:// site to an http:// site + +Chat apps (Whats App, etc.) + In 2012, Alexis Madrigal wrote a story about these sources, terming them “dark social”. (This post is worth re-reading for some fun facts like “Only about four percent of total traffic is on mobile at all.”) In summary, he supposed that since these were links that were being shared from person-to-person, this kind of traffic should still be thought of as social sharing. Parse.ly’s CTO, Andrew Montalenti, explained some other areas that “dark” traffic can come from – he points out that it’s not all social; some traffic might also be explained by search and sites that block their referral information from being passed through or programs used by people protecting their browsing (especially in the wake of Snowden/NSA revelations). @@ -16,10 +30,16 @@ Though many of those sources will probably continue to be “dark”, a lot of u We saw a similar pattern, and called up Facebook a few months ago. As the massive social network makes a concerted effort to work with publishers, they were concerned at the differences we saw compared to their own data, and we shared some of our aggregate data with them as they worked to fix the referral issue. +What to expect from the fix: + The announcement above acknowledged the first fix, and we’ll continue to monitor the impact that this is making in our system. What does this mean for your traffic? We checked the amount of traffic that currently comes into posts as direct (aka “dark social”) but is actually Facebook by looking at the user agents and found in one sample that 11% of total traffic coming was from Facebook. (And yes – that means that almost 50% of all external traffic referrers in this sample came from Facebook, wow.) Our analysis also checked out how much of that dark traffic came in from Android vs. iOS. We’ll be keeping track of this for the next few weeks, and we expect to see those numbers go down as the app fixes roll out and people update their apps. -We know that people have a lot of questions about this, and we’re happy to provide additional information or answer questions about it – let us know in a comment, or shoot us a note. \ No newline at end of file +We know that people have a lot of questions about this, and we’re happy to provide additional information or answer questions about it – let us know in a comment, or shoot us a note. + +Related + +Parse.ly, December 11, 2014 \ No newline at end of file diff --git a/tests/data/text/parsely.com2.txt b/tests/data/text/parsely.com2.txt index 29612a10..8e3844be 100644 --- a/tests/data/text/parsely.com2.txt +++ b/tests/data/text/parsely.com2.txt @@ -4,6 +4,8 @@ Think of metrics like the Force. Sure, they have a dark-side: user-unfriendly sl But you can use the “Force” of metrics for good: to understand your audience and create better experiences for them. Metrics can arm anyone in digital media, from editors to business executives with with vital information about their readers’ preferences and behaviors. When it comes to audience insights, aim to be Luke Skywalker, not Darth Vader. +Why Referrals Matter + Data allows publishers today to see where readers come from and how they found an article or story. This tells you how readers already spend their time, what they care about, and can give you a good idea of who they might be based on some basic demographics about the referral networks. Analytics that tell you where your traffic comes from on a story by story or author by author basis will give you the clearest picture of your audience. For example, here’s the referral sources for articles written by our CTO on the Parse.ly blog next to the referral sources for articles written by our Director of Marketing: @@ -14,6 +16,8 @@ Readers coming to articles written by our Director of Marketing tend to use more We consider what people on these sites and networks want to learn or read about when we write new blog posts or create whitepapers and ebooks. We think about what kind of information we have that might be useful and interesting to someone that also reads about content marketing or regularly participates in tech conversations. It’s also a great gut-check to see if the audience we intended to reach is actually seeing our stories. +Referral Trends and Benchmarking Against Your Peers + Access to internal data allows you to focus on your own readers, but what about understanding your broader competitive set? Where are their readers coming from? Parse.ly publishes a quarterly Authority Report that includes a summary of refferal traffic based on billions of pageviews on our network of digital publishing sites. In the most recent edition, we compared referral traffic on a year-over-year basis (Aug. 2013-Aug. 2014). @@ -22,6 +26,8 @@ Overall, the data told the story of the rise of social and the constricting of a See the full breakdown of top twenty five referrals and how they changed when you subscribe to the report. +Social: Facebook the disruptor + Most of social’s gains came directly from the increase in Facebook traffic to sites. Facebook made a huge jump, going from 12% of overall traffic referrals to over 20% (22.7% to be precise). This is similar to findings of other vendors and has recently spurred a torrent of concern over how much influence the social giant has over readers. @@ -32,6 +38,8 @@ However, our network saw Twitter and Stumbleupon as the two highest referrers af Even at this scale of billions of referrers, the types of audiences that the Parse.ly network sees versus the Shareaholic network likely accounts for these differences. +Search: The dominator + Say what you will about the rise of social, search still eeks out more overall web visits. Though we did see the two come toe-to-toe back in January, search has climbed back up into the dominate position. Google didn’t even lose any ground to the Facebook growth – it gained almost two percentage points as a referral source to news sites. @@ -40,10 +48,18 @@ One thing to note in our data: the search traffic shown controls for the “bran The data in the Authority Report only considers post URLs, not landing pages or homepages. This means that every view we account for came directly to an article, video or story. +Aggregators and RSS: The displaced + Our data saw almost all aggregators sites account for less of the overall referral pie for traffic. One theory why? Over the past year, digital news outlets have started to embrace more aggregation on their own sites. Huffington Post and Gawker have a strong reputation for aggregating other outlets stories; the New York Times now links out to others’ top stories on their homepages and in their apps. RSS readers may have also suffered from social’s gains. Feedly, which was poised to take over the RSS game after Google Reader was shuttered dipped in the past year. Mobile apps and social media feeds provide similar functionality combined with better social/mobile elements. +The Last Word: “OTHER” + Overall, this sample of sites saw a 30% increase of total external traffic, bolstered by strong social and search networks, but also ultimately aided by an increasingly larger set of long-tail sites. Does the large increase in this group point to more reliance on paid traffic distribution through networks like Outbrain, Taboola and their ilk? -It certainly points to the fact that digital publishers have to be more vigilant than ever about where their audiences live online, how they’re finding the content they read and how the publishers can craft cohesive strategies across their entire organization that use data on those activities to their advantage. \ No newline at end of file +It certainly points to the fact that digital publishers have to be more vigilant than ever about where their audiences live online, how they’re finding the content they read and how the publishers can craft cohesive strategies across their entire organization that use data on those activities to their advantage. + +Related + +Parse.ly, November 17, 2014 \ No newline at end of file diff --git a/tests/data/text/pe.com1.txt b/tests/data/text/pe.com1.txt index d1d78806..28a724f5 100644 --- a/tests/data/text/pe.com1.txt +++ b/tests/data/text/pe.com1.txt @@ -1 +1,37 @@ -The content you are looking for is unavailable.Please use the search box at the top of this page to find the information you were seeking. \ No newline at end of file +PERRIS: Woman ordered hit on father of her child, police say Share Pin It More Galleries ADVERTISEMENT Judy Avalos, 20, was charged with conspiracy to commit murder after a shooting near Perris that injured her boyfriend. RIVERSIDE COUNTY SHERIFF'S DEPT./CONTRIBUTED IMAGE Related article » + +After a man was shot four times in an exchange of gunfire near Perris, police at first thought it might have been a drug deal gone wrong. + +But their focus quickly turned to the 20-year-old mother of the man’s child. She is now accused of orchestrating an attempt to kill him. The man survived his injuries. + +Judy Avalos, of Perris, was arrested at the hospital where she was visiting her injured boyfriend the day after the Dec. 30 shooting. She has pleaded not guilty to conspiracy to commit murder and is being held in lieu of $1 million bail. + +Neither the gunman, nor a man believed to have accompanied him, have been identified, Lt. David Lelevier said. The gunman’s compensation for the shooting appears to have been not a cash payment but a romantic relationship with Avalos, he said. + +Defense attorney Nic Cocis said Avalos did not seek to have her boyfriend killed. Avalos believed he was going to be beaten up, not shot, Cocis said. + +The shooting was reported to the Riverside County Sheriff’s Department about 9:30 p.m. in the 23300 block of Edmond Street in the unincorporated area known as Good Hope. + +Avalos told investigators that she and her 24-year-old boyfriend had gone to Bakersfield together to retrieve a Jeep and she followed him back to Riverside County in her pickup, search warrant documents say. The boyfriend waited outside in the Jeep while Avalos went into her mother’s house to pick up their 4-year-old daughter. Avalos said she had been inside about 10 minutes when she heard a hail of gunfire and dropped to the floor. + +After the gunfire stopped, Avalos said she went outside to find her boyfriend, with four gunshot wounds, walking up the driveway and holding a gun, Investigator Robert Cornett wrote in search warrant documents. + +Deputies later discovered a car, which had been reported stolen, abandoned a few blocks away. It was riddled with bullet holes. + +The boyfriend told investigators that the shooter and another man were in a white car that pulled up next to him while he was parked on Edmond Street, search warrant documents say. He said the car had driven past him, stopped for a few minutes at the dead end of the street, then pulled up next to him with a window down. + +The men in the car told him they were lost and asked for directions to Highway 74. A minute or two later, he said, one of the men pulled a gun and started shooting at him. He said he grabbed the gun under his seat and fired back at them as the car drove off, search warrant documents say. + +“(He) went on to say that Avalos had a strange conversation with him a few days prior,” Cornett wrote. “He said she called him up and asked him if he had been shot at.” + +When investigators questioned Avalos about the conversation, she said she had had a dream about it, adding that “when she has a dream about things, they come true,” Cornett wrote. + +She denied having anything to do with the shooting and let deputies download data from her cell phone. + +Investigators said they found several suspicious texts, written in Spanish, that had been deleted from the phone. Avalos wrote to someone saying she wanted them to take out only her boyfriend and that it had to be tonight – Dec. 30, search warrant papers say. + +She wrote that he would be in a Jeep parked in front of her mother’s house and the person replied that he was set with two people for tonight. At 9:24 p.m., Avalos texted to say they were there. The other person replied, telling Avalos to delete the text messages before “they do their job,” Cornett wrote. + +After she was arrested, Avalos told investigators her boyfriend had been mistreating her and not letting her leave the house to go out with her friends, Cornett wrote. Avalos said he would point a gun at her and tell her he was going to kill her, but that she had never reported it to police, search warrant papers say. + +“She said they had been arguing all week and she was just ‘mad’ at him,” Cornett wrote. “She said she was wrong for what she did but did not know any other way to get out of the relationship.” \ No newline at end of file diff --git a/tests/data/text/pe.com2.txt b/tests/data/text/pe.com2.txt deleted file mode 100644 index 50decff5..00000000 --- a/tests/data/text/pe.com2.txt +++ /dev/null @@ -1,69 +0,0 @@ -DEVELOPMENT IN WINE COUNTRY The story: After four years of meetings, debate and revision, the Riverside County Board of Supervisors in March adopted a comprehensive Temecula Valley Wine Country Community...... - - - -LAKE ELSINORE: Grants to help city improve two parks Two small Lake Elsinore parks in and near the city’s historical downtown district will be refurbished thanks to more than $1 million in grant money. Much of the $1.1 million will be spent on...... - - - -For many people, today is first and foremost a day of worship. Across Southwest Riverside County, a cursory Internet search found more than 160 houses of worship. I’m sure there are more. But the...... - - - -One person was in critical condition and four dogs were rescued after a head-on crash on the Ortega Highway west of Lake Elsinore, fire officials said.... - - - -Attendance boundaries for a number of campuses in the Lake Elsinore Unified School District would be altered in the coming months if proposed changes are ultimately accepted by board members.... - - - -Los Angeles prosecutors on Tuesday declined to file any charges against Bill Cosby after an Inland woman recently claimed the comedian molested her around 1974.... - - - -As his alleged mission to defame Costa Mesa councilmen unraveled in 2012, a private investigator tried to get his story straight with the law firm that hired him on behalf of the local police union,...... - - - -December storms have given a much-needed boost to some Inland lakes, but it will take much more rain to bring reservoirs back to pre-drought levels, water suppliers said.... - - - -LOS ANGELES – A lawyer for a Riverside County woman who says Bill Cosby sexually abused her when she was 15 stated in a recent court filing that he has interviewed two witnesses who corroborate her...... - - - -CORRECTION: A previous version of this story contained incorrect information concerning the value of the former Jean Hayman Elementary School campus. It was appraised at $2.5 million in 2011. ... - - - -The Ortega Highway above Lake Elsinore was closed for seven hours Sunday after a rock slide left 6- to 8-foot boulders strewn across the road.... - - - -When Lake Elsinore Police Chief Leonard Hollingsworth looked at his city’s line in recently released nationwide crime statistics compiled by the FBI for 2013, he received a pleasant surprise.... - - - -Lake Elsinore’s Police Department is beefing up patrols in shopping areas within the city in an effort to provide greater security during the holidays. Crime, especially property theft, tends to...... - - - -School boards in Southwest Riverside County named new leaders at their annual organizational meetings this week. Lake Elsinore Unified School District’s new board president is Heidi Matthies Dodd...... - - - -Two schools in Lake Elsinore -- Cottonwood Canyon Elementary School and Canyon Lake Middle School -- were locked down for a portion of the school day Thursday after school officials reported hearing...... - - - -A highly regarded skateboard park designer and builder will construct one in southern Lake Elsinore after the City Council hired the company this week. The council voted 5-0 to award a $402,000 contract...... - - - -In its annual reorganization of responsibilities, Lake Elsinore’s City Council named Steve Manos to serve as mayor for the next year. In selecting Manos on Tuesday to replace Natasha Johnson, the...... - - - -Canyon Lake residents interested in serving the last two years of a vacant seat on the City Council have until Jan. 5 to turn in their applications, officials announced. City Clerk Ariel Hall opened the...... \ No newline at end of file diff --git a/tests/data/text/pewresearch.org1.txt b/tests/data/text/pewresearch.org1.txt index 57086b91..a3b21d79 100644 --- a/tests/data/text/pewresearch.org1.txt +++ b/tests/data/text/pewresearch.org1.txt @@ -22,4 +22,6 @@ The estimated 4 million covered under the executive action are in addition to an The president’s executive action offers deportation relief for the largest number of unauthorized immigrants in recent history. However, the current action does not allow unauthorized immigrants the opportunity to obtain permanent residency or citizenship. Only Congress has the authority to offer a path to legal status, which means those eligible for relief under the new policy remain unauthorized. If a future administration reverses course or if Congress passes an immigration law, the protection from deportation could be taken away. -Correction: This posting has been updated with a revised total of 3.9 million unauthorized immigrants affected by the president’s action, and other related revised estimates. A previous version of this post referred to the president’s implementation of his new immigration policy as an executive order. He has taken executive action. \ No newline at end of file +Correction: This posting has been updated with a revised total of 3.9 million unauthorized immigrants affected by the president’s action, and other related revised estimates. A previous version of this post referred to the president’s implementation of his new immigration policy as an executive order. He has taken executive action. + +Topics: Immigration, Unauthorized Immigration \ No newline at end of file diff --git a/tests/data/text/pewresearch.org2.txt b/tests/data/text/pewresearch.org2.txt index 11df2983..6e6ae4b6 100644 --- a/tests/data/text/pewresearch.org2.txt +++ b/tests/data/text/pewresearch.org2.txt @@ -10,4 +10,6 @@ A 2009 paper in the Journal of Monetary Economics, which also found a cause-and- Those attitudinal changes have real-world consequences, though not large ones. The authors of the JME paper, Paul Edelstein and Lutz Kilian, concentrated mainly on unexpected increases, rather than drops, in energy prices; they estimated that a one-time, 1% increase in energy prices was associated with a 0.08% fall in total real consumption a year later, based on 1988-2006 data. But that was considerably less than the 0.30% decline seen in the 1970-1987 data; energy price shocks, they concluded, have less effect on the overall economy than they used to. -Edelstein and Kilian also estimated that one year after an unexpected, permanent increase of 25 cents a gallon in the price of gas, a typical household would have reduced its expenditures by $17 a month, with most of the adjustment coming in the first six months after the price increase. Concluded the researchers: “It takes repeated surprise increases in gasoline prices to generate large effects on household consumption, but over time the effects will add up.” \ No newline at end of file +Edelstein and Kilian also estimated that one year after an unexpected, permanent increase of 25 cents a gallon in the price of gas, a typical household would have reduced its expenditures by $17 a month, with most of the adjustment coming in the first six months after the price increase. Concluded the researchers: “It takes repeated surprise increases in gasoline prices to generate large effects on household consumption, but over time the effects will add up.” + +Topics: National Economy \ No newline at end of file diff --git a/tests/data/text/pixable.com1.txt b/tests/data/text/pixable.com1.txt index 497b6f25..778fbdb0 100644 --- a/tests/data/text/pixable.com1.txt +++ b/tests/data/text/pixable.com1.txt @@ -1 +1 @@ -Bad news for ladies, gentlemen and those who planned on splitting up a healthy, attractive couple: Sofia Vergara and Joe Manganiello are engaged. THEY ENGAGED. They getting married. Hitched. Tying the knot. They like-like each other. They do not want to date you. They only want to date themselves, together, from now until eternity. They have no plans to marry you in the near future. \ No newline at end of file +MISSED ENTIRE ARTICLE BECAUSE IT'S LOADED DYNAMICALLY BY JAVASCRIPT \ No newline at end of file diff --git a/tests/data/text/pixable.com2.txt b/tests/data/text/pixable.com2.txt index 011681cc..778fbdb0 100644 --- a/tests/data/text/pixable.com2.txt +++ b/tests/data/text/pixable.com2.txt @@ -1 +1 @@ -It’s hard to believe, but 2014 has come to an end and for better or worse 2015 is right around the corner. In 2014 we had some highs and some lows, but all in all it was an okay year. That being said there are still some pop culture trends that should never see the light of day again once the clock strikes midnight on December 31st. Don’t find yourself getting caught up in these trends we hope will stay behind once we move into 2015. The first “Sharknado” was okay. The second “Sharknado” was absurd and undoubtedly any other movies that have “Sharknado” in the title will be downright obnoxious. Yes, for the most part they’re fun to watch and make us think, but enough is enough. Some of these so-called social experiments are clearly staged and use innocent people for the sake of going viral. Yaaaaaaaaas we really do need to stop saying bae. Remember when Justin Bieber thought it was a good idea to revisit the 90s boyband era? Hopefully he gets some hair dye for Christmas and dyes it any other color but bleach blonde. We thought with the release of a new single off of Taylor Swift’s new album that we were done with the massive amounts of “Shake It Off” parodies, but it looks like we were wrong. Maybe next year we can pick a different song? Yes, “Frozen” was an excellent movie, but it’s time to let it go. For those who don’t know what normcore is it’s a fashion trend where people wear average looking clothing or whatever they can find. This contributes greatly to the next pop culture trend we hope stays in 2014. Where did this term even come from and why are we using it to describe average people who just happen to like everything in a non-ironic way? Cats are absolutely adorable — well sometimes — but it’s time for a new animal to take over the Internet in 2015. Goats maybe? “Peter Pan Live!” received a 59 percent on Rotten Tomatoes, which surprisingly was better than the 44 percent that “The Sound of Music Live!” received after it’s failed attempt to win over a large television audience. There was Solange, Beyonce and Jay Z and every time that Selena Gomez unfriended anyone of her famous friends and most recently the ongoing feud between Azealia Banks and Iggy Azalea. These are prime examples of celebrity drama that should be kept behind closed doors and does not need to be broadcast over the Internet. Kale was all the rage this past year, but it’s time for everyone to put down the vegetable and try some new leafy greens. ‘Keeping Up With The Kardashians” was so last year. Ever since Bruce and Kris called it quits, Kylie and Kendall grew up and Kourtney had yet another baby we just can’t seem to care about what the Kardashians are up to. It seems like everywhere you turn there’s a hashtag. Every TV show has one, Facebook has tried to incorporate them into their platform, they appear on clothing and as home decor. Please stop abusing the hashtag. Why does Hollywood feel the need to remake classics like “Annie,” “Godzilla” and “Teenage Mutant Ninja Turtles”? They rarely do as well as the originals. \ No newline at end of file +MISSED ENTIRE ARTICLE BECAUSE IT'S LOADED DYNAMICALLY BY JAVASCRIPT \ No newline at end of file diff --git a/tests/data/text/pixelmonkey.org1.txt b/tests/data/text/pixelmonkey.org1.txt index 99f0b6f8..e9bbcad1 100644 --- a/tests/data/text/pixelmonkey.org1.txt +++ b/tests/data/text/pixelmonkey.org1.txt @@ -1,35 +1 @@ -My first mobile device was a Palm V. I understood the power of mobile really early. I was 15 in 1999, when the Palm V was released. I first came across the Palm devices in 1998, when the Palm III came onto the scene. I never owned one, but played with the one my Dad owned, but barely used. - -This device was comically under-powered in retrospect. It had 2 megabytes of RAM, which had to be used as not only the working memory of the device, but also the storage. It had a 16 Hz processor, a 4-greyscale screen, and a stylus-driven interface. - -The Palm V was an amazing device. In lieu of the plastic of the Palm III, it had a finished anodized aluminum finish, very similar to the kinds of sleek devices we would only begin to regularly see in the last couple of years. It was nearly half the weight of its predecessor, and as thin as the stylus you used to control it. It had a surprisingly well-designed docking station (imagine this: since USB hadn’t yet been developed, it had to sync over the low-bandwidth Serial Port available on PCs at the time). - -I came across a time capsule of my Palm V usage. Since I was a web designer at this time, but since it was a “pre-web” era, I often put together little static HTML websites to demonstrate features and ideas to my friends. I did one of these for my Palm V. - -In a world without cell data plans, wifi networks, & laptop computers, my Palm V provided much of the same utility that is now spread across these various pieces of everyday technology infrastructure, but in a single device. I kept a unified to do list of my life, and a specialized task manager specifically for coursework and classes. In it, I tracked not just homework assignments and due dates grouped by class, but also my grades — so that I could prioritize work to optimize my grades. - -I think I only got my first cell phone a couple of years later — and they were nowhere near ubiquitous yet, so my contact manager was actually being used to store landline phone numbers and addresses of my friends — and likely, of local take-out / delivery restaurants. - -I sometimes tell my friends that despite the lack of infrastructure in school for it, I was indeed a “child of the Internet”. This was mostly thanks to AvantGo. - -When I was at home on my personal computer, I was already an avid web content reader from early pioneers like Salon.com, CNET, Wired, and BBC. But I couldn’t easily read their content at school. AvantGo was an early, popular service for Palm devices that let you download and sync “content channels”. Of course, this is way before RSS/Atom. Channels were really mobile-optimized static HTML websites that each provider would put out daily with a snapshot of their latest content. In the screenshot above, you can see me reading an article from Salon.com dated March 30, 2002. - -In addition to officially supported channels that were put out by major publishers, you could also download and sync arbitrary web content. This was essentially an early version of Readability / Instapaper — in some ways, much clunkier, in other ways, more complete. The software would go to the website and try to mobile-optimize a site (stripping it down to basic HTML), while also crawling the relative links. For example, at the time I was very interested in participating in the Debian open source project, so I had AvantGo sync the Debian Policy Manual, which contained hundreds of HTML pages full of reference information. - -So, yes — when I was bored in Spanish or Math class, I’d pull up a political article from Salon.com, a tech piece about the 2000 tech bubble forming in Silicon Valley from Wired, or read over open source software development guidelines. I was a child of the Internet, and the Palm V kept me tethered in a pre-mobile world. - -What makes AvantGo so interesting, upon reflection, is that despite everything against it — under-powered mobile devices, no ubiquitous wireless Internet access, and a less standardized world-wide-web, it still satisfied a use case that has yet to be satisfied today. Namely, it provided a way for me to curate the “best sources of content” I found on the web, and sync and download all their content to provide a personalized, offline library of content. Instead, we have nothing but fragmentation today — individual native apps for each news publisher, RSS readers like Newsblur providing a way to get the latest updates from a group of them, and mobile web browsers for everything else. - -Obviously, devices running iOS and Android are finally starting to make ubiquitous computing a reality, and are providing less clunky ways to access web content. - -But I think it’s worth reflecting on what my Palm V had going for it nearly a decade ago that has yet to arrive in this new era. - -A great example of online/offline mastery that I witnessed in a mobile application is Spotify’s support for bringing an entire music playlist offline. Offers the best of both worlds. - -We’re certainly living in an age of ubiquitous, wireless Internet access, whether via cell towers or wifi. But that doesn’t mean that the smartest applications are those that exclusively rely on data access being present. In fact, making your application rely on data access will almost certainly make it slower than it could be. - -Due to our obsession with connectivity, I think many developers are forgetting about use cases where the mobile device is meant to be a smart accessory to our lives, rather than a portal to the web. These are no longer “thin clients” — instead, we should think of them as powerful handheld computers. Recent iPhone and Android devices feature dual-core processors at speed above 1Ghz and 1GB of RAM. In 1998, typical desktop computers were slower than this (e.g. a single-core P3 600 Mhz w/ 128MB of RAM and 10GB of disk storage). - -Yes, we’ve put the power of a desktop computer into the palm of a user’s hand. But, are we really tapping this opportunity? - -Andrew Montalenti (aka pixelmonkey, amontalenti) is the co-founder and CTO of Parse.ly, which provides data insights to the web's best publishers. You can follow him on Twitter or LinkedIn. If you liked this post, you should subscribe to his free e-mail newsletter, Fire and Motion, which discusses technology startups and business from the trenches. This entry was posted on Sunday, December 30th, 2012 at 3:00 pm and is filed under Personal, Technology. You can follow any responses to this entry through the RSS 2.0 feed. You can leave a response, or trackback from your own site. \ No newline at end of file +MISSED LAST SENTENCE IN THE ARTICLE diff --git a/tests/data/text/pixelmonkey.org2.txt b/tests/data/text/pixelmonkey.org2.txt index 87cfd43d..ddad5286 100644 --- a/tests/data/text/pixelmonkey.org2.txt +++ b/tests/data/text/pixelmonkey.org2.txt @@ -1,23 +1 @@ -Parse.ly made its funding announcement — a $5M series A, led by Grotech Ventures and with participation from FundersClub, Blumberg Capital, and ff Venture Capital. Read on for the full list of links to our coverage. - -Here is some coverage from around the web: - -TechCrunch: Parse.ly Raises $5M For Predictive Analytics Platform That Helps Media Companies Decide What To Publish -MediaPost: Parse.ly Raises $5 Million, Invests In Core Tech, Data -The Daily Startup @ Wall Street Journal, VC Dispatch -Washington Business Journal: Grotech leads $5 million round in analytics startup Parse.ly -MarketingLand: Facebook Driving More Than 2x Twitter Traffic To Parse.ly’s News Clients -GigaOM: Feedly dominating the post-Reader world, and other web-publishing insights from Parse.ly -peHUB: Parse.ly raises $5M -Parse.ly Blog: Parse.ly Investors Back the New Content Performance Authority -The Next Web: A month after Google Reader vanishes, Feedly ranks as the top RSS traffic referrer -VentureBeat: Parse.ly nabs $5 million to continue getting you to click on Grumpy Cat -PSFK: Platform Tells Bloggers What To Write About Next [Video] -memeburn: Life after Google Reader: yes, Feedly is still winning -Brafton: Yahoo might be top web property, but Google still drives most traffic -eMediaVitals: Publishing analytics startup Parse.ly releases first report on traffic drivers -IPG Media Lab: Parse.ly Does Intelligent Tagging -ContentStandard: Parse.ly Delivers Content Optimization for Online Publishers -More coming soon! - -Andrew Montalenti (aka pixelmonkey, amontalenti) is the co-founder and CTO of Parse.ly, which provides data insights to the web's best publishers. You can follow him on Twitter or LinkedIn. If you liked this post, you should subscribe to his free e-mail newsletter, Fire and Motion, which discusses technology startups and business from the trenches. This entry was posted on Tuesday, September 3rd, 2013 at 12:53 pm and is filed under Startups, Technology, Uncategorized. You can follow any responses to this entry through the RSS 2.0 feed. You can leave a response, or trackback from your own site. +ONLY "More coming soon!" AT END OF ARTICLE WAS EXTRACTED diff --git a/tests/data/text/readwrite.com1.txt b/tests/data/text/readwrite.com1.txt index e69de29b..539db220 100644 --- a/tests/data/text/readwrite.com1.txt +++ b/tests/data/text/readwrite.com1.txt @@ -0,0 +1 @@ +FAILED, ENTIRE ARTICLE LOADED VIA JS diff --git a/tests/data/text/readwrite.com2.txt b/tests/data/text/readwrite.com2.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/data/text/recipe.com1.txt b/tests/data/text/recipe.com1.txt index 3ce751f1..6b81e8e3 100644 --- a/tests/data/text/recipe.com1.txt +++ b/tests/data/text/recipe.com1.txt @@ -1,17 +1 @@ -1 medium onion, finely chopped -2 tablespoons butter -4 teaspoons all-purpose flour -1 1/2 cups milk -8 ounces sharp cheddar cheese, shredded (2 cups) -3 pounds russet potatoes, peeled and thinly sliced* -1 1/2 cups chopped fresh broccoli -1 tablespoon vegetable oil -8 eggs -2 tablespoons milk -6 slices bacon, crisp-cooked, drained, and crumbled -1 large tomato, chopped - -1. Preheat oven to 325 degrees F. In a medium saucepan cook onion in butter over medium heat for 4 minutes or until tender, stirring occasionally. Stir in flour, 1/2 tsp. salt, and 1/2 tsp. ground black pepper. Stir in milk; cook and stir until slightly thickened and bubbly. Stir in cheese until melted. -2. In a 3-quart baking dish, layer potatoes, then cheese sauce. Bake, covered, about 55 minutes, until potatoes are tender. -3. In a large skillet, cook broccoli in hot oil over medium heat for 5 minutes, until nearly tender, stirring frequently. In a large bowl, beat together eggs, water, 1/2 tsp. salt, and 1/4 tsp. ground black pepper. Pour over broccoli in skillet. Cook over medium heat, without stirring, until mixture begins to set on bottom and around edges. Using a spatula, lift and fold partially cooked egg so uncooked portion flows underneath. Cook 2 minutes more or until egg is cooked yet still moist. -Spoon over potatoes. Top with bacon and chopped tomato. Serve immediately. +ONLY EXTRACTED DIRECTION BULLETPOINT #3, REST OF ARTICLE FAILED TO BE EXTRACTED diff --git a/tests/data/text/recipe.com2.txt b/tests/data/text/recipe.com2.txt index 97043072..6ff9de3a 100644 --- a/tests/data/text/recipe.com2.txt +++ b/tests/data/text/recipe.com2.txt @@ -1,3 +1,5 @@ +Make-Ahead Side: Sweet Potato, Parsnip, and Celery Root Mash + With winter cooking in full swing, I’ve been making heartier meat dishes such as braises, stews, and roasts. One my favorite side dishes to accompany them is sweet potato mash. They’re less starchy than white ones and higher in nutrition, packed with loads of vitamin A. I liked the addition of parsnips in this recipe and added a bulb of celery root as well to give the dish another layer of earthy flavor. I began by scrubbing and peeling all of my root vegetables. Celery root can be a little tricky to work with since the skin is thick and bumpy, so I use the peeler first to remove as much of the exterior as possible. Then I trim off any missed spots with my knife. By cutting them all into 1-inch cubes, they’ll cook more evenly together in the pot. @@ -8,22 +10,30 @@ Because I like mine with a little chunkier texture, I used my hand masher to mix This dish can be made the day ahead and reheated right before serving. Be sure to add an extra pat of butter on top to finish. Try this root vegetable mash as a sweet, earthy alternative to classic mashed potatoes at your next meal. +Sweet Potato, Parsnip, and Celery Root Mash + +Serves: 8 + +Prep Time: 15 minutes + 1/4 pound celery root, peeled and cut into 1-inch pieces - 1 pound parsnips, peeled and cut into 1-inch pieces +1 pound parsnips, peeled and cut into 1-inch pieces - 2 pounds sweet potatoes, peeled and cut into 1-inch pieces +2 pounds sweet potatoes, peeled and cut into 1-inch pieces - 1/2 cup 2% milk +1/2 cup 2% milk - 1/4 cup packed brown sugar +1/4 cup packed brown sugar - 2 tablespoons unsalted butter +2 tablespoons unsalted butter - 3/4 teaspoon salt +3/4 teaspoon salt - 1/4 teaspoon ground allspice +1/4 teaspoon ground allspice 1. Combine parsnips and sweet potatoes in a medium pot; cover with 1 inch of cold water. Bring to a boil, then reduce heat to medium and simmer for 10 to 15 minutes, or until vegetables are tender and parsnips have lost their bitterness. - 2. Drain cooked vegetables and immediately return to pot over low heat. Mash with milk, brown sugar, butter, salt, and allspice. \ No newline at end of file +2. Drain cooked vegetables and immediately return to pot over low heat. Mash with milk, brown sugar, butter, salt, and allspice. + +Get the original recipe here. \ No newline at end of file diff --git a/tests/data/text/reuters.com1.txt b/tests/data/text/reuters.com1.txt index 2a646f97..dc749b97 100644 --- a/tests/data/text/reuters.com1.txt +++ b/tests/data/text/reuters.com1.txt @@ -1,17 +1,17 @@ People walk past the Bombay Stock Exchange (BSE) building in Mumbai May 13, 2014. -(Reuters) - The BSE Sensex and Nifty were trading flat on Tuesday, as weaker regional shares offset optimism over additional reforms a day after the government passed an executive order to ease land-acquisition rules. +The best and worst of Bollywood in 2014. -Monday's announcement could kick-start hundreds of billions of dollars in stalled projects, and comes after separate orders were passed to implement coal and insurance reforms. +MUMBAI (Reuters) - The BSE Sensex and Nifty ended flat on Tuesday as weaker regional shares offset optimism over additional reforms a day after the government passed an executive order to ease land-acquisition rules. -Infrastructure stocks gained on the news with Larsen and Toubro (LART.NS) adding 1.1 percent. Lanco Infratech (LAIN.NS) rose 6.7 percent and GMR Infrastructure (GMRI.NS) was trading 1.5 percent higher. +Monday's announcement could kick-start hundreds of billions of dollars worth of stalled projects, and comes after separate orders were passed to implement coal and insurance reforms. However, weak sentiment across the region on a sharp selloff in commodities overnight and political uncertainty in Greece left investors hesitant to take big bets. -"The overall outlook is good. The government is moving in the right direction. Investors expect the reform process to continue, which will be a long-term positive for Indian equities," said Suresh Parmar, head, institutional equities at KJMC Capital Markets. +The BSE Sensex gained 0.03 percent at 27,403.54 points, while the broader Nifty ended 0.02 higher percent at 8,248.25 points. -The benchmark Sensex was up 0.03 percent, while the broader Nifty was trading 0.01 percent lower. +Infrastructure stocks gained with Larsen and Toubro (LART.NS) adding 0.5 percent while Lanco Infratech (LAIN.NS) rose 4.3 percent. -Godrej Properties (GODR.NS) gained 5.4 percent after the company bought back shares held by private equity firm Sun-Apollo, and also announced a new project. +However, energy firms fell as Brent crude fell to a 5-1/2 year low below $57 a barrel. Reliance Industries (RELI.NS) closed down 1.9 percent and Oil and Natural Gas Corp (ONGC.NS) ended down 1.5 percent. -Energy firms however fell, with Oil and Natural Gas Corporation (ONGC.NS) losing 1.4 percent and Reliance Industries (RELI.NS) falling 1.3 percent after crude prices fell to a five-year low. +(Reporting by Indulal PM; Editing by Sunil Nair) \ No newline at end of file diff --git a/tests/data/text/reuters.com2.txt b/tests/data/text/reuters.com2.txt index d8ae8f9d..fead096c 100644 --- a/tests/data/text/reuters.com2.txt +++ b/tests/data/text/reuters.com2.txt @@ -1,6 +1,8 @@ 1 of 4. Authorities monitor progress in the search for AirAsia Flight QZ8501 in the Mission Control Center inside the National Search and Rescue Agency in Jakarta December 29, 2014. -(Reuters) - Countries around Asia on Tuesday stepped up the search for an AirAsia plane carrying 162 people that is presumed to have crashed in shallow waters off the Indonesian coast, with Washington also sending a warship to help find the missing jet. +The best and worst of Bollywood in 2014. + +JAKARTA (Reuters) - Countries around Asia on Tuesday stepped up the search for an AirAsia plane carrying 162 people that is presumed to have crashed in shallow waters off the Indonesian coast, with Washington also sending a warship to help find the missing jet. Soelistyo, head of Indonesia's search and rescue agency, told local television the search area between the islands of Sumatra and Borneo would be expanded. Authorities would also begin scouring nearby islands as well as coastal land on Indonesia's side of Borneo. @@ -26,6 +28,8 @@ The U.S. military said the USS Sampson, a guided missile destroyer, would be on "We stand ready to assist in any way possible," Pentagon spokesman Mark Wright said. +COULD PLANE HAVE STALLED? + What happened to Flight QZ8501, which had sought permission from Indonesian air traffic control to ascend to avoid clouds, is still a mystery. Online discussions among pilots have centred on unconfirmed secondary radar data from Malaysia that suggested the aircraft was climbing at a speed of 353 knots, about 100 knots too slow in poor weather, and that it might have stalled. @@ -46,6 +50,8 @@ Three airline disasters involving Malaysian-affiliated carriers in less than a y In the third incident, Malaysia Airlines Flight MH17 was shot down over Ukraine on July 17, killing all 298 people on board. +NO FOUL PLAY SEEN + On board Flight QZ8501 were 155 Indonesians, three South Koreans, and one person each from Singapore, Malaysia and Britain. The co-pilot was French. U.S. law enforcement and security officials said passenger and crew lists were being closely examined but so far nothing significant had turned up and that the incident was still regarded as an unexplained accident. @@ -56,6 +62,10 @@ Pilots and aviation experts said thunderstorms, and requests to gain altitude to The Indonesian pilot was experienced and the plane last underwent maintenance in mid-November, the airline said. +Indonesia AirAsia is 49 percent owned by Malaysia-based budget carrier AirAsia. + The AirAsia group, including affiliates in Thailand, the Philippines and India, had not suffered a crash since its Malaysian budget operations began in 2002. The plane's disappearance comes at a sensitive time for Jakarta's aviation authorities, as they strive to improve the country's safety reputation to match its status as one of the airline industry's fastest growing markets. + +(Additional reporting by Wilda Asmarini, Fransiska Nangoy, Cindy Silviana, Kanupriya Kapoor, Michael Taylor, Nilufar Rizki and Siva Govindasamy in JAKARTA, Al-Zaquan Amer Hamzah and Praveen Menon in KUALA LUMPUR, Saeed Azhar, Rujun Shen and Anshuman Daga in SINGAPORE, Jane Wardell in SYDNEY, Ben Blanchard in BEIJING, Tim Hepher in PARIS and Mark Hosenball, David Brunnstrom and Lesley Wroughton in WASHINGTON; Writing by Dean Yates; Editing by Michael Perry) \ No newline at end of file diff --git a/tests/data/text/reuters.com3.txt b/tests/data/text/reuters.com3.txt index 0ddab58d..7f4ffd95 100644 --- a/tests/data/text/reuters.com3.txt +++ b/tests/data/text/reuters.com3.txt @@ -1,31 +1,51 @@ -1 of 18. A member of an Indonesian Hercules C130 aircrew watches through a window while monitoring the Belitung Timur sea during search operations for AirAsia flight QZ8501 near Belitung island, December 29, 2014 in this photo taken by Antara Foto. +1 of 18. A Republic of Singapore Air Force personnel surveys the waters during a search and locate operation for the missing AirAsia flight QZ8501 plane at an undisclosed search area December 30, 2014. -(Reuters) - Indonesian rescuers saw bodies and luggage off the coast of Borneo island on Tuesday and officials said they were "95 percent sure" debris spotted in the sea was from a missing AirAsia plane with 162 people on board. +SURABAYA, Indonesia/JAKARTA (Reuters) - Indonesian rescuers searching for an AirAsia plane carrying 162 people pulled bodies and wreckage from the sea off the coast of Borneo on Tuesday, prompting relatives of those on board watching TV footage to break down in tears. Indonesia AirAsia's Flight QZ8501, an Airbus A320-200, lost contact with air traffic control early on Sunday during bad weather on a flight from the Indonesian city of Surabaya to Singapore. -Pictures of floating bodies were broadcast on television and relatives of the missing gathered at the crisis centre in Surabaya were shown weeping, their heads in their hands. +The navy initially said 40 bodies had been recovered, although other media later quoted the head of the search and rescue agency, Fransiskus Bambang Soelistyo, as saying only three bodies had been retrieved. The plane has yet to be found. -Media quoted an air force official earlier as saying one suspected body, luggage and a life vest were among the debris in the Java Sea. +"My heart is filled with sadness for all the families involved in QZ8501," airline boss Tony Fernandes tweeted. "On behalf of AirAsia, my condolences to all. Words cannot express how sorry I am." -"As we approached, the body seemed bloated," said First Lieutenant Tri Wibowo, who was on board a Hercules aircraft, was quoted by the Kompas.com website as saying. +The airline said in a statement that it was inviting family members to Surabaya, "where a dedicated team of care providers will be assigned to each family to ensure that all of their needs are met". -Search and Rescue Agency chief Soelistyo told reporters he was "95 percent sure" the debris was from the missing plane. +Pictures of floating bodies were broadcast on television and relatives of the missing already gathered at a crisis centre in Surabaya wept with heads in their hands. Several people collapsed in grief and were helped away. -Djoko Murjatmodjo, acting director general of air transportation at the transportation ministry, told reporters some of the debris spotted was red and white, AirAsia's colours. +Yohannes and his wife were at the centre awaiting news of her brother, Herumanto Tanus, and two of his children who were on board the doomed flight. -"It's probably from the aircraft," he said. +The Tanus family had been on their way to visit Herumanto's son, who studies in Singapore and who travelled to Surabaya on Monday after the plane went missing. -About 30 ships and 21 aircraft from Indonesia, Australia, Malaysia, Singapore, South Korea and the United States were searching up to 10,000 square nautical miles on Tuesday. +"He cries every time he watches the news," Yohannes said. + +The mayor of Surabaya, Tri Rismaharini, comforted relatives and urged them to be strong. + +"They are not ours, they belong to God," she said. + +SEARCHING THROUGH THE NIGHT + +A navy spokesman said a plane door, oxygen tanks and one body had been recovered and taken away by helicopter for tests. + +"The challenge is waves up to three metres high," Soelistyo told reporters, adding that the search operation would go on all night. He declined to answer questions on whether any survivors had been found. + +About 30 ships and 21 aircraft from Indonesia, Australia, Malaysia, Singapore, South Korea and the United States have been involved in the search. The plane, which did not issue a distress signal, disappeared after its pilot failed to get permission to fly higher to avoid bad weather because of heavy air traffic, officials said. +It was travelling at 32,000 feet (9,753 metres) and had asked to fly at 38,000 feet, officials said earlier. + Pilots and aviation experts said thunderstorms, and requests to gain altitude to avoid them, were not unusual in that area. The Indonesian pilot was experienced and the plane last underwent maintenance in mid-November, the airline said. Online discussion among pilots has centred on unconfirmed secondary radar data from Malaysia that suggested the aircraft was climbing at a speed of 353 knots, about 100 knots too slow, and that it might have stalled. +Investigators are focussing initially on whether the crew took too long to request permission to climb, or could have ascended on their own initiative earlier, said a source close to the probe, adding that poor weather could have played a part as well. + +He cautioned that the investigation was at an early stage and the black box flight recorders had yet to be recovered. + +CLUES WHEN THINGS GO WRONG + The plane, whose engines were made by CFM International, co-owned by General Electric and Safran of France, lacked real-time engine diagnostics or monitoring, a GE spokesman said. Such systems are mainly used on long-haul flights and can provide clues to airlines and investigators when things go wrong. @@ -34,6 +54,8 @@ Three airline disasters involving Malaysian-affiliated carriers in less than a y Malaysian Airlines Flight MH370 went missing on March 8 on a trip from Kuala Lumpur to Beijing with 239 passengers and crew on board and has not been found. On July 17, the same airline's Flight MH17 was shot down over Ukraine, killing all 298 people on board. +Bizarrely, an AirAsia plane from Manila skidded off and overshot the runway on landing at Kalibo in the central Philippines on Tuesday. No one was hurt. + On board Flight QZ8501 were 155 Indonesians, three South Koreans, and one person each from Singapore, Malaysia and Britain. The co-pilot was French. U.S. law enforcement and security officials said passenger and crew lists were being examined but nothing significant had turned up and the incident was regarded as an unexplained accident. @@ -42,6 +64,4 @@ Indonesia AirAsia is 49 percent owned by Malaysia-based budget carrier AirAsia. The AirAsia group, including affiliates in Thailand, the Philippines and India, had not suffered a crash since its Malaysian budget operations began in 2002. -India is waiting to know what went wrong with the missing plane and will investigate if AirAsia India is following all safety procedures, a senior Indian aviation ministry official told Reuters. AirAsia India, a joint venture of the Malaysian carrier, started flying this year and is expanding operations. - -The plane's disappearance comes at a sensitive time for Indonesia's aviation authorities, as they strive to improve the country's safety reputation to match its status as one of the airline industry's fastest growing markets. +(Additional reporting by Fergus Jensen, Wilda Asmarini, Charlotte Greenfield, Fransiska Nangoy, Cindy Silviana, Kanupriya Kapoor, Michael Taylor, Nilufar Rizki and Siva Govindasamy in JAKARTA/SURABAYA, Al-Zaquan Amer Hamzah and Praveen Menon in KUALA LUMPUR, Saeed Azhar, Rujun Shen and Anshuman Daga in SINGAPORE, Jane Wardell in SYDNEY, Tim Hepher in PARIS and Mark Hosenball, David Brunnstrom and Lesley Wroughton in WASHINGTON; Writing by Dean Yates and Robert Birsel; Editing by Nick Macfie and Mike Collett-White) \ No newline at end of file diff --git a/tests/data/text/reuters.com4.txt b/tests/data/text/reuters.com4.txt index 49bc4497..c4811394 100644 --- a/tests/data/text/reuters.com4.txt +++ b/tests/data/text/reuters.com4.txt @@ -1,6 +1,6 @@ 1 of 2. A general view of Gartnavel General Hospital is seen in Glasgow, Scotland December 29, 2014. -(Reuters) - A healthcare worker has been diagnosed with Ebola a day after flying home to Glasgow from Sierra Leone, the Scottish government said on Monday. +LONDON (Reuters) - A healthcare worker has been diagnosed with Ebola a day after flying home to Glasgow from Sierra Leone, the Scottish government said on Monday. The patient is being treated in isolation at Glasgow's Gartnavel Hospital, having flown back to Scotland's largest city late on Sunday on a British Airways flight via Casablanca in Morocco and London's Heathrow. @@ -17,3 +17,5 @@ In August, another British aid worker, William Pooley, contracted the disease af With more than 9,000 cases, Sierra Leone now accounts for nearly half of the known cases of Ebola in this year's West African outbreak, the worst ever. Neighbouring Liberia and Guinea have also been badly hit. The World Health Organization on Monday said the number of people infected by Ebola in Liberia, Sierra Leone and Guinea -- the worst affected by the outbreak -- has passed 20,000, with more than 7,842 deaths in the epidemic so far. + +(Reporting by Andy Bruce in London and Ankur Banerjee in Bengaluru; Editing by Joyjeet Das) \ No newline at end of file diff --git a/tests/data/text/self.com1.txt b/tests/data/text/self.com1.txt index eacfc496..396a4e9e 100644 --- a/tests/data/text/self.com1.txt +++ b/tests/data/text/self.com1.txt @@ -5,3 +5,7 @@ Keep Your Hair’s Movement: Unless you’re fighting with grown-out bangs, refr Go Easy on Styling: The simplest way to keep the accessory from looking stuffy is to skip elaborate updos and ringlet curls. Keep the hair more natural and use the headband to add structure. Make It the Focal Point: With a piece of bling like Taylor’s, keep the rest of the beauty look soft. Although she wore fuchsia lipstick, the singer opted for a matte variety to tone-down the vivid hue. + +RELATED: + +Image Credit: Getty \ No newline at end of file diff --git a/tests/data/text/self.com2.txt b/tests/data/text/self.com2.txt index 13980a9e..0c19cbd5 100644 --- a/tests/data/text/self.com2.txt +++ b/tests/data/text/self.com2.txt @@ -1,7 +1,15 @@ Starting the New Year on a good note means skipping the stress—especially when it comes to your hair! But since many of us will be working in the hours leading up to the ball drop (often with little to no time to adjust between office and party) our suggestion is to prep your locks in the AM. With the right style and a few night-appropriate tricks, your hair won't need much upkeep to make it to midnight. +Loose Waves + Start with structured waves in the morning, wrapping small sections of hair (1-1 1/2-inch pieces) around a 1-inch curling iron for 20 to 30 seconds. Brush the ringlets out and throughout the day, they’ll smooth to soft waves. After work, apply a texturizing spray into the hair, spraying and scrunching each section to increase volume. +Glam Topknot + The secret to a topknot is in the volume. In the morning apply a generous amount of dry shampoo from the scalp to the tip, scrunching the hair to add fullness. Then pull the hair up into a twisted topknot towards the top of the head. Hold with hairspray. At night, either pull a few front pieces out to soften the look or add a glitzy headband. +Messy Fishtail + Pull off a lived-in braid the natural way. Start with a clean fishtail braid in the morning and let it naturally transition to messy-chic—pieces will fall out of the plait over time. At the end of the day, use a dry shampoo at the roots to make the entire style look lived in. Add a bow or leather cuff at the tip of the braid for subtle structure. + +RELATED: \ No newline at end of file diff --git a/tests/data/text/sitepoint.com1.txt b/tests/data/text/sitepoint.com1.txt index e69de29b..b326a6d9 100644 --- a/tests/data/text/sitepoint.com1.txt +++ b/tests/data/text/sitepoint.com1.txt @@ -0,0 +1,85 @@ +The HTTP Archive Report collates information from almost half a million of the web’s most popular websites. The latest figures indicate that average page weight has increased by 15% in one year to reach 1,953Kb — a little under 2Mb — and comprises 95 individual HTTP requests. While this is smaller than the 32% increase in 2013, it remains cause for concern. + +The report analyzes publicly-accessible content and shopping web sites rather than complex web applications and provides a breakdown of the technologies used: + +technology end 2013 end 2014 increase HTML 57Kb 59Kb +4% CSS 46Kb 57Kb +24% JavaScript 276Kb 295Kb +7% Images 1,030Kb 1,243Kb +21% Flash 87Kb 76Kb -13% Other 205Kb 223Kb +9% Total 1,701Kb 1,953Kb +15% + +These are average figures; a large proportion of pages will have greater file sizes. + +A 2Kb rise for HTML seems reasonable although it’s a significant quantity of content given the trend for simpler, more concise text. + +What surprises me most is CSS’s 11Kb rise. Responsive Web Design and CSS3 animations could account for some of this increase but there’s not been a drop in JavaScript. Despite the availability of CSS management and minification tools, the average site also makes six requests for CSS files. + +JavaScript has risen by 19Kb. This is confusing; the need for shims is reducing, effects can be handed to CSS3 and monolithic libraries have fallen from favor. Sites make an average of 18 JavaScript file requests, which is unchanged from last year — although a quarter of sites make more than 30 requests. Perhaps some of the gain can be explained by increasingly sophisticated/bloated social networking scripts? + +27% of sites continue to use Flash — a fall of 5% over the year. The majority is used for advertising, video, and games. Flash hasn’t dropped as fast as expected but its future is clear. + +There’s been a 9% increase for “other” files. That figure doubled in 2013 but, back then, custom fonts and icon fonts were relatively new. + +Finally, images are responsible for 85% of the weight gain. Using high-resolution (Retina) images could account for some of this hike, except: + +Pages contain more than fifty images, which seems excessive. + +Retina accounts for a relatively small proportion of devices. + +SVG, icon fonts, and CSS3 effects can replace many images. + +There are numerous tools to help reduce file sizes. + +Additional Factors + +The survey also reveals: + +95 HTTP requests are made per page — a drop of a single request from last year. + +Pages contain 862 DOM elements. + +Resources are loaded from sixteen domains with a maximum of 52 requests per domain. + +The average PageSpeed score is 78 out of 100 — which is surprisingly good, given the bloat. + +46% of pages use Google libraries. + +47% of pages use custom fonts. + +79% of responses are compressed (gzip’d). + +14% of pages are loaded over HTTPS. + +20% of pages use localStorage. + +65% of pages use iframes (mostly videos and advertising). + +74% of pages use at least one redirect — which seems high. + +The Primary Suspects + +A 15% increase is less extravagant than the 32% rise in 2013 and the 30% rise in 2012, but it’s still too much. Has your bandwidth increased more than 15% in the past twelve months? A third of web users now use mobile devices — will they appreciate the additional weight? + +Let’s put this into context for website owners. Bloated pages adversely affect your profitability: + +Users have a slower experience. It doesn’t matter how great your site looks — people will not wait. There’s little point creating a site that works on mobile devices when your pages are 2Mb. Responsive Web Design != a responsive website. Are you losing up to a third of potential customers? Google will downgrade your site and harm your search engine optimization efforts (though we’re never sure exactly how much this matters to Google’s algorithm). Your hosting costs will increase. The more code you use, the more likely it will break. Updates and maintenance are more difficult, take longer and cost more. + +It’s ironic that web developers praise the benefits of cross-device HTML5 apps when a single page is often larger to download and slower than an equivalent native app. + +Overweight pages are unnecessary. My primary suspects remain bloated CMS templates and frameworks. They offer a cheaper and quicker development route at the expense of quality, efficiency and performance. Many are packed with features you’ll never use, but removing them can be laborious, tedious, and time-consuming. + +We can summarize the problem in one simple word: laziness. Developers are at fault — that’s you and me. We have plenty of excuses: + +there’s never enough time + +the client insisted it should be done this way + +the budget/schedule is too tight + +I inherited a shoddy system + +I don’t have the tools + +Whether it’s technical boundaries or a failure to explain issues, it’s still laziness. We work at the coal face; the final decisions are ours alone. Why create a badly-optimized site when many bloat-blasting solutions are simple and take minutes to implement? + +Clients rarely appreciate the efficiency gains we make but they don’t understand anything we do. We are the experts, and minimizing page weight is an essential part of the job. Do it. It’s easier to beg for forgiveness than to ask for permission. + + + +Are you concerned by the web obesity problem? Are you pleased the scale of increases has dropped? Do you or fellow developers struggle to implement optimization techniques or to explain them to clients? Do you think there are other causes? Is Craig being too simplistic and shouty?! \ No newline at end of file diff --git a/tests/data/text/sitepoint.com2.txt b/tests/data/text/sitepoint.com2.txt index febeaf39..84468749 100644 --- a/tests/data/text/sitepoint.com2.txt +++ b/tests/data/text/sitepoint.com2.txt @@ -4,32 +4,50 @@ This can contain up-to-the-minute information from your application about a user In this article I’m going to take a look at some of the ways you can implement this. +Setting up our Example Application + All the code from this tutorial is available on Github. There’s also an online demo. First, we’ll define our application’s dependencies using Composer: -Now, in , let’s pull in the Composer-generated autoloader, add our statement, initialize our Silex application and setup Twig templating: +"silex/silex": "~2.0@dev", "twig/twig": ">=1.8,<2.0-dev", "smottt/wideimage": "dev-master" + +Now, in index.php , let’s pull in the Composer-generated autoloader, add our use statement, initialize our Silex application and setup Twig templating: + +require_once __DIR__.'/../vendor/autoload.php'; use Symfony\Component\HttpFoundation\Request, Symfony\Component\HttpFoundation\Response, WideImage\WideImage; $app = new Silex\Application(); // Register the Twig service provider $app->register(new Silex\Provider\TwigServiceProvider(), array( 'twig.path' => __DIR__.'/../views', )); Let’s create a data-store of users, with some information about them which will form the basis of our example “badges”. For simplicity, we’ll use a static array; in practice you’d use a database, but it ought to be pretty simple to swap this out for something more dynamic. We’ll inject it into the application container like so: +/** * For simplicity, our datastore is a really simple, static array */ $app['datastore'] = function(){ return [ 'users' => [ 'dave' => [ 'avatar' => 'man1.png', 'trophies' => 1, 'rank' => 'Novice', ], 'jim' => [ 'avatar' => 'man2.png', 'trophies' => 2, 'rank' => 'Intermediate', ], 'helen' => [ 'avatar' => 'woman1.png', 'trophies' => 4, 'rank' => 'Grand Master', ], ] ]; }; + Now that we’ve got a basic application set up along with some data, let’s go through three approaches to how you can provide an embeddable “badge” which displays this data for a given user. +IFrames + IFrames are arguably a dirty word in web circles, perhaps deservedly so. But they are a common and practical approach to embedding content from one site into another. Let’s start with this approach; later we can reuse some of the code when we try out the JavaScript method. -Start with a simple Twig template, which creates some HTML – with inlined styles, to minimize HTTP requests – which contains our “badge” as a : +Start with a simple Twig template, which creates some HTML – with inlined styles, to minimize HTTP requests – which contains our “badge” as a
                                : + + {{ username }}

                                {{ username }}

                                {{ user.rank }}

                                {% for i in 1..user.trophies %} {% endfor %}
                                -This is all pretty straightforward. Notice how we’re incorporating a variable named which we’ll set server-side, which is going to take care of one very important aspect – any images we incorporate must be referenced using absolute URLs. +This is all pretty straightforward. Notice how we’re incorporating a variable named imagepath which we’ll set server-side, which is going to take care of one very important aspect – any images we incorporate must be referenced using absolute URLs. Now the corresponding route: -All pretty straightforward. The bit which populates is a bit quick-and-dirty, but it’ll do the job for now. +/** * Dynamically-generated HTML for embedding in an iFrame */ $app->get('/iframe/{username}', function(Request $request, $username) use ($app) { // Check that the user in question exists if (!isset($app['datastore']['users'][$username])) { // No user with that username, throw a 404 $app->abort(404, "User $username does not exist."); } // Get the user record $user = $app['datastore']['users'][$username]; return $app['twig']->render('badge.twig', [ 'username' => $username, 'imagepath' => ( ($request->server->get('HTTP_PORT') == 443) ? 'https' : 'http' ) . '://' . $request->server->get('HTTP_HOST') . '/images', 'user' => $user, ] ); }); + +All pretty straightforward. The bit which populates $imagepath is a bit quick-and-dirty, but it’ll do the job for now. Embedding this into a third-party site is really simple: + + Later we’ll look at some of the things you need to think about with the iframe approach; for now, let’s move onto method number two. +Dynamically Created Images + One of the simplest ways to implement this is to provide a URL to an image, which gets created server-side. Here’s a screenshot of the sort of image we’re going to create: @@ -40,42 +58,70 @@ You’ll find the necessary resources – the background and trophy images, avat Here’s some example code to dynamically generate an embeddable image: -It’s pretty much self-documented, and should be pretty simple to adapt to your needs or with better images. Note that we’re taking the background image from a non-web accessible directory ( ), but the avatars and trophy icon are in the directory. +/** * Dynamically-generated image */ $app->get('/image/{username}', function($username) use ($app) { // Check that the user in question exists if (!isset($app['datastore']['users'][$username])) { // No user with that username, throw a 404 $app->abort(404, "User $username does not exist."); } // Get the user record $user = $app['datastore']['users'][$username]; // Load the background $background = WideImage::load(__DIR__.'/../resources/images/background.png'); // Load the avatar $avatar = WideImage::load(__DIR__.'/images/' . $user['avatar']); // Load the trophy image $trophy = WideImage::load(__DIR__.'/images/trophy.png'); // Paste the avatar onto the background $im = $background->merge($avatar, 10, 20); // Get the canvas $canvas = $im->getCanvas(); // Set the font for the username $canvas->useFont(__DIR__.'/../resources/fonts/VeraBd.ttf', 12, $im->allocateColor(0, 0, 0)); // Write the username onto the canvas $canvas->writeText(70, 15, $username); // Choose a slightly smaller, non-bold font $canvas->useFont(__DIR__.'/../resources/fonts/Vera.ttf', 9, $im->allocateColor(0, 0, 0)); // Write the rank $canvas->writeText(70, 35, $user['rank']); // Now add the appropriate number of trophies $x = 70; for ($i = 0; $i < $user['trophies']; $i++) { $im = $im->merge($trophy, $x, 55); $x += 20; } // Finally, output the image to the screen return $im->output('png'); }); + +It’s pretty much self-documented, and should be pretty simple to adapt to your needs or with better images. Note that we’re taking the background image from a non-web accessible directory ( resources ), but the avatars and trophy icon are in the public directory. Embedding this into a third-party website couldn’t be simpler: -There are a couple of improvements we could make. For one thing, we’re generating a new image on each request. However you can use WideImage’s method to cache the results, like so: + + +You’ll note that there’s no file extension; this doesn’t matter, though, since WideImage’s output() method will set the appropriate headers for you. + +There are a couple of improvements we could make. For one thing, we’re generating a new image on each request. However you can use WideImage’s saveToFile() method to cache the results, like so: + +$im->saveToFile('/path/to/badge.png'); It might also be better to provide a default image when the requested user cannot be found, rather than issuing a 404 error. Now onto the third and final approach. +Javascript + Using JavaScript to dynamically create embedded content is amongst the most common, and perhaps the most flexible approach. -Again we’re going to generate some HTML, but this time we’ll return a simple snippet of JavaScript that will write it to the host page. All that requires is that the host website insert a simple tag where they want our content to appear. +Again we’re going to generate some HTML, but this time we’ll return a simple snippet of JavaScript that will write it to the host page. All that requires is that the host website insert a simple
                                + Strictly speaking, we don’t even need that container DIV, but it can be used to apply styling on the host site. Now that we’ve examined three approaches, let’s look at some of the things you need to think about when deciding which of these approaches to take. +Considerations + When choosing an approach to this problem, there are a few things you need to weigh up. +CMS’s + If you’re intending people to be able to embed content into the body of CMS-driven content or within blog posts, it’s worth bearing in mind that any CMS or blog software worth its salt will block certain types of content. Inline scripts are almost certainly out. IFrames are probably going to be stripped out. That probably just leaves image tags. +Styling + There are a few ways to approach styling; perhaps you want to control everything, keeping your badges consistent across sites. Alternatively, you could provide default styles but allow site owners the flexibility to override them to better fit the design of their site. Obviously images cannot be customized; aside, perhaps, their sizing. If you use iframes, it’s worth noting that any styling applied to the parent page will not be inherited by your content. On the other hand if you use the JavaScript approach, it may well be possible to override the styling, depending on specificity and how you incorporate your styles. The demo page that comes with the example application shows this in action. +Customization + Perhaps, like Stackoverflow and their “User Flair” badges, you want to provide a number of alternative styles – light and dark, for example. This is entirely possible with any of the approaches I’ve outlined, though it’s arguably slightly more difficult with the image tag approach. +Advanced + So far our embeddable content has been dynamically generated, but in no way interactive. A Facebook “like” button, for example, doesn’t just provide a count – it also allows people to perform the “like” action from within the page. That sort of interactivity will be covered in a later article. +Summary + Embeddable content is great way to promote your site. It can be used not only to link back to your site but to provide “live” content, right there on a third-party “host” website. We’ve looked at three common approaches to this – images, iframes and JavaScript. We’ve looked at some of the things you need to think about when deciding which one to use, along with some pitfalls to be wary of. \ No newline at end of file diff --git a/tests/data/text/slate.com1.txt b/tests/data/text/slate.com1.txt index e69de29b..22997ad2 100644 --- a/tests/data/text/slate.com1.txt +++ b/tests/data/text/slate.com1.txt @@ -0,0 +1 @@ +LAST PARAGRAPH IS MISSED, SHAME BECAUSE THE REST OF THE *HUGE* ARTICLE IS EXTRACTED PROPERLY diff --git a/tests/data/text/space.com1.txt b/tests/data/text/space.com1.txt index 2303491f..8a4fa111 100644 --- a/tests/data/text/space.com1.txt +++ b/tests/data/text/space.com1.txt @@ -1,11 +1,27 @@ +Reference: + +NASA's Hubble Space Telescope snapped this shot of Mars on Aug. 26, 2003, when the Red Planet was 34.7 million miles from Earth. The picture was taken just 11 hours before Mars made its closest approach to us in 60,000 years. + +Credit: NASA/ESA View full size image + Mars is the fourth planet from the sun. Befitting the red planet's bloody color, the Romans named it after their god of war. The Romans copied the ancient Greeks, who also named the planet after their god of war, Ares. Other civilizations also typically gave the planet names based on its color — for example, the Egyptians named it "Her Desher," meaning "the red one," while ancient Chinese astronomers dubbed it "the fire star." +Physical characteristics + +Regolith + The bright rust color Mars is known for is due to iron-rich minerals in its regolith — the loose dust and rock covering its surface. The soil of Earth is a kind of regolith, albeit one loaded with organic content. According to NASA, the iron minerals oxidize, or rust, causing the soil to look red. +Geology + The cold, thin atmosphere means liquid water currently cannot exist on the Martian surface for any length of time. This means that although this desert planet is just half the diameter of Earth, it has the same amount of dry land. The red planet is home to both the highest mountain and the deepest, longest valley in the solar system. Olympus Mons is roughly 17 miles (27 kilometers) high, about three times as tall as Mount Everest, while the Valles Marineris system of valleys — named after the Mariner 9 probe that discovered it in 1971 — can go as deep as 6 miles (10 km) and runs east-west for roughly 2,500 miles (4,000 km), about one-fifth of the distance around Mars and close to the width of Australia or the distance from Philadelphia to San Diego. +Space.com Exclusive T-shirt. Available to Populate Mars. + +Credit: Space.com Store View full size image Space.com Exclusive T-shirt. Available to Populate Mars. Buy Now + Mars has the largest volcanoes in the solar system, including Olympus Mons, which is about 370 miles (600 km) in diameter, wide enough to cover the entire state of New Mexico. It is a shield volcano, with slopes that rise gradually like those of Hawaiian volcanoes, and was created by eruptions of lavas that flowed for long distances before solidifying. Mars also has many other kinds of volcanic landforms, from small, steep-sided cones to enormous plains coated in hardened lava. Some minor eruptions might still occur on the planet. Scientists think the Valles Marineris formed mostly by rifting of the crust as it got stretched. Individual canyons within the system are as much as 60 miles (100 km) wide. They merge in the central part of the Valles Marineris in a region as much as 370 miles (600 km) wide. Large channels emerging from the ends of some canyons and layered sediments within suggest the canyons might once have been filled with liquid water. @@ -16,8 +32,12 @@ Many regions of Mars are flat, low-lying plains. The lowest of the northern plai The number of craters on Mars varies dramatically from place to place, depending on how old the surface is. Much of the surface of the southern hemisphere is extremely old, and so has many craters — including the planet's largest, 1,400-mile-wide (2,300 km) Hellas Planitia — while that of northern hemisphere is younger and so has fewer craters. Some volcanoes have few craters, which suggests they erupted recently, with the resulting lava covering up any old craters. Some craters have unusual-looking deposits of debris around them resembling solidified mudflows, potentially indicating that impactor hit underground water or ice. +Polar caps + Vast deposits of what appear to be finely layered stacks of water ice and dust extend from the poles to latitudes of about 80 degrees in both hemispheres. These were probably deposited by the atmosphere over long spans of time. On top of much of these layered deposits in both hemispheres are caps of water ice that remain frozen all year round. Additional seasonal caps of frost appear in the wintertime. These are made of solid carbon dioxide, also known as "dry ice," which has condensed from carbon dioxide gas in the atmosphere, and in the deepest part of the winter, this frost can extend from the poles to latitudes as low as 45 degrees, or halfway to the equator. The dry ice layer appears to have a fluffy texture, like freshly fallen snow, according to the report in the Journal of Geophysical Research-Planets. +Climate + Mars is much colder than Earth, in large part due to its greater distance from the sun. The average temperature is about minus 80 degrees Fahrenheit (minus 60 degrees Celsius), although they can vary from minus 195 F (minus 125 C) near the poles during the winter to as much as 70 F (20 C) at midday near the equator. The carbon-dioxide-rich atmosphere of Mars is also roughly 100 times less dense than Earth's on average, but it is nevertheless thick enough to support weather, clouds and winds. The density of the atmosphere varies seasonally, as winter forces carbon dioxide to freeze out of the Martian air. @@ -26,22 +46,34 @@ NASA’s Mars Reconnaissance Orbiter found the first definitive detections of ca The dust storms of the Mars are the largest in the solar system, capable of blanketing the entire red planet and lasting for months. One theory as to why dust storms can grow so big on Mars starts with airborne dust particles absorbing sunlight, warming the Martian atmosphere in their vicinity. Warm pockets of air flow toward colder regions, generating winds. Strong winds lift more dust off the ground, which in turn heats the atmosphere, raising more wind and kicking up more dust. +Orbital characteristics + The axis of Mars, like Earth's, is tilted with relation to the sun. This means that like Earth, the amount of sunlight falling on certain parts of the planet can vary widely during the year, giving Mars seasons. However, the seasons that Mars experiences are more extreme than Earth's because the red planet's elliptical, oval-shaped orbit around the sun is more elongated than that of any of the other major planets. When Mars is closest to the sun, its southern hemisphere is tilted toward the sun, giving it a short, very hot summer, while the northern hemisphere experiences a short, cold winter. When Mars is farthest from the sun, the northern hemisphere is tilted toward the sun, giving it a long, mild summer, while the southern hemisphere experiences a long, cold winter. +Composition & structure + +Atmospheric composition (by volume): + +95.32 percent carbon dioxide, 2.7 percent nitrogen, 1.6 percent argon, 0.13 percent oxygen, 0.08 percent carbon monoxide, minor amounts of water, nitrogen oxide, neon, hydrogen-deuterium-oxygen, krypton, xenon + Magnetic field: Mars currently has no global magnetic field, but there are regions of its crust that can be at least 10 times more strongly magnetized than anything measured on Earth, remnants of an ancient global magnetic field. Chemical composition: Mars likely has a solid core composed of iron, nickel, and sulfur. The mantle of Mars is probably similar to Earth's in that it is composed mostly of peridotite, which is made up primarily of silicon, oxygen, iron and magnesium. The crust is probably largely made of the volcanic rock basalt, which is also common in the crusts of the Earth and the moon, although some crustal rocks, especially in the northern hemisphere, may be a form of andesite, a volcanic rock that contains more silica than basalt does. Internal structure: Scientists think that on average, the Martian core is about 1,800 and 2,400 miles in diameter (3,000 and 4,000 km), its mantle is about 900 to 1,200 miles (5,400 to 7,200 km) wide and its crust is about 30 miles (50 km) thick. +Orbit & rotation + Average distance from the sun: 141,633,260 miles (227,936,640 km). By comparison: 1.524 times that of Earth Perihelion (closest): 128,400,000 miles (206,600,000 km). By comparison: 1.404 times that of Earth Aphelion (farthest): 154,900,000 miles (249,200,000 km). By comparison: 1.638 times that of Earth +The moons of Mars + The two moons of Mars, Phobos and Deimos, were discovered by American astronomer Asaph Hall over the course of a week in 1877. Hall had almost given up his search for a moon of Mars, but his wife, Angelina, urged him on — he discovered Deimos the next night, and Phobos six days after that. He named the moons after the sons of the Greek war god Ares — Phobos means "fear," while Deimos means "rout." Both Phobos and Deimos are apparently made of carbon-rich rock mixed with ice and are covered in dust and loose rocks. They are tiny next to Earth's moon, and are irregularly shaped, since they lack enough gravity to pull themselves into a more circular form. The widest Phobos gets is about 17 miles (27 km), and the widest Deimos gets is roughly nine miles (15 km). @@ -54,6 +86,12 @@ Phobos is gradually spiraling toward Mars, drawing about 6 feet (1.8 meters) clo Both moons are potential targets for exploration. One NASA plan envisions bombarding Phobos with small, spiky spherical rovers called hedgehogs. +The planet Mars is the fourth planet from the sun and named after the Roman God of War and is also called the Red Planet. + +Credit: Karl Tate, SPACE.com View full size image The planet Mars is the fourth planet from the sun and named after the Roman God of War and is also called the Red Planet. See what makes Mars tick with this Space.com infographic looking inside the Red Planet + +Research & exploration + The first person to watch Mars with a telescope was Galileo Galilei, and in the century after him, astronomers discovered its polar ice caps. In the 19th and 20th centuries, researchers believed they saw a network of long, straight canals on Mars, hinting at civilization, although later these often proved to be mistaken interpretations of dark regions they saw. Robot spacecraft began observing Mars in the 1960s, with the United States launching Mariner 4 there in 1964 and Mariners 6 and 7 in 1969. They revealed Mars to be a barren world, without any signs of the life or civilizations people had imagined there. In 1971, Mariner 9 orbited Mars, mapping about 80 percent of the planet and discovering its volcanoes and canyons. @@ -72,6 +110,8 @@ In September 2014, India’s Mars Orbiter Mission reached the red planet, making Robots aren’t the only ones looking to buy a ticket to Mars. A workshop group of government, academic, and industry scientists have found that a NASA-led manned mission to Mars should be possible by the 2030s. But NASA isn’t the only one with Martian astronaut hopefuls.. The Mars One colony project is looking to send private citizens on a one-way trip to the red planet. +Possibility of life + Mars could have once harbored life. Some conjecture that life might still exist there even today. A number of researchers have even speculated that life on Earth may have seeded Mars, or that life on Mars seeded Earth. The most public scientific claim for life on Mars came in 1996. Geologist David McKay at NASA's Johnson Space Center in Houston and his colleagues focused on rocks blasted off the surface of Mars by cosmic impacts that landed on Earth. Within they found complex organic molecules, grains of a mineral called magnetite that can form within some kinds of bacteria, and tiny structures that resembled fossilized microbes. However, these claims have proven controversial, and there is no consensus as to whether they are signs of life. @@ -80,4 +120,16 @@ Mars may have possessed oceans on its surface in the past, providing an environm Enthusiasm and excitement on Earth over the possibility of life on the red planet are revealed by the flurry of excitement that greets interesting objects spotted by orbiters and landers. While the Face on Mars garnered attention for the past four decades after it was first spotted by Viking 1, rovers today show close-ups of objects — such as a weathered Martian rock claimed to be a ‘thigh bone’ and a likely shiny rock that raised furor on the internet as a UFO light. +Additional resources + +Read about and see pictures of more than 40 missions to Mars. + +Follow the rover Curiosity as it makes its way across the Martian terrain. + +Learn more about Mars at NASA's Solar System Exploration website. + +Related: + +Also see: + Learn more about each of the primary planets: \ No newline at end of file diff --git a/tests/data/text/space.com2.txt b/tests/data/text/space.com2.txt index ed48d9ad..d7c1105a 100644 --- a/tests/data/text/space.com2.txt +++ b/tests/data/text/space.com2.txt @@ -1,9 +1,17 @@ +The logo of NASA, the National Aeronautics and Space Administration. + +Credit: NASA View full size image + WASHINGTON — NASA will receive $18 billion for 2015, more than a half a billion dollars above the Obama administration’s original request, under the terms of an omnibus spending bill released late Dec. 9. The appropriations bill, which funds NASA and most of the rest of the federal government for the remainder of the fiscal year that began Oct. 1, gives the agency $17.99 billion, including increases for several major exploration and science programs. That total is $530 million above the administration's request of $17.46 billion for the agency, and about $100 million above separate House and Senate appropriations bills considered earlier this year. It's also nearly $350 million above NASA’s enacted 2014 budget of $17.65 billion. Two major elements of NASA's exploration strategy won funding boosts in the final bill. The Space Launch System heavy-lift rocket will receive $1.7 billion, an increase of $320 million over the administration’s request. The Orion spacecraft will get $1.194 billion, an increase of $141.2 million over the request. +Artist’s concept of Orion separating from SLS rocket. + +Credit: NASA View full size image Artist’s concept of Orion separating from SLS rocket. + The bill also requires NASA to submit in its next budget proposal five-year funding projections for the SLS and Orion programs that match the budget and schedule estimates for those programs developed in recent or ongoing reviews. Those estimates, developed by NASA for a program milestone called Key Decision Point C (KDP-C), were completed for SLS in August and will be completed for Orion in the spring of 2015. The bill specifically requires those projections for the SLS to show completion by December 2017. The KDP-C review for SLS estimated that the vehicle would be ready for its first launch no later than November 2018. @@ -12,6 +20,10 @@ NASA's planetary science program won an increase of $157 million, to $1.438 bill NASA's astrophysics program won a $77.5 million increase, to $684.8 million, in the omnibus bill. That includes $70 million for the Stratospheric Observatory for Infrared Astronomy, an airborne telescope for which NASA requested only $12 million. NASA had planned to mothball the flying observatory in 2015 if it could not find partners to fund the telescope's annual operating cost of about $85 million. +The SOFIA flying observatory. + +Credit: NASA View full size image The SOFIA flying observatory. + NASA's commercial crew program received $805 million in the bill, less than the requested $848 million but more than the program received in previous years. The omnibus bill leaves out a controversial provision from the Senate's appropriations bill that would have required certified cost and pricing data from commercial crew companies. That language was opposed by many supporters of the commercial crew program, as well as by the White House. NASA's aeronautics program received $651 million, an increase of $100 million over the administration’s request. In a report accompanying the bill, appropriators directed NASA to apply the additional funds proportionally across the programs in that mission directorate. diff --git a/tests/data/text/spanish.txt b/tests/data/text/spanish.txt index f106063a..34538a25 100644 --- a/tests/data/text/spanish.txt +++ b/tests/data/text/spanish.txt @@ -4,6 +4,8 @@ El fiscal Pedro Horrach está perfilando con el jefe de la Fiscalía Anticorrupc Horrach duda si insistir en su postura de dejar al margen a la hija menor del Rey de los indicios incriminatorios citados por el instructor del ‘caso Nóos’, como ya lo hizo por escrito el pasado mes de noviembre o, por el contrario, se limita exponer de manera sucinta su punto de vista para apoyar el recurso que ya ha anunciado el abogado de la Infanta, Miquel Roca. +Consenso + El fiscal anticorrupción se entrevistó ayer con su superior, Antonio Salinas, con el objetivo de analizar el extenso auto – 227 folios– de Castro y consensuar una postura común. En principio, Horrach no está dispuesto a entrar al trapo de las argumentaciones del titular del Juzgado de Instrucción 3 de Palma, que le reprocha en su resolución que defienda a ultranza a la Infanta y no haga lo mismo, por ejemplo, con otros imputados. «Las meras conjeturas o sospechas no constituyen elementos válidos de imputación», sostenía Horrach, que ya había recurrido el pasado abril contra la primera imputación de la infanta, que anuló la Audiencia de Palma. Asimismo, el juez Castro realizó ayer por la mañana unas breves declaraciones sobre su auto ante un grupo de periodistas a su llegada a los juzgados de Vía Alemania. «Uno lo hace para que se valide, pero luego si se valida o no, no lo sé», dijo el instructor del ‘caso Noos’ en relación a la decisión que debe adoptar la Audiencia de Palma sobre los previsibles recursos anunciados contra la imputación de la infanta Cristina. diff --git a/tests/data/text/syracuse.com2.txt b/tests/data/text/syracuse.com2.txt index fd41b6f8..460c211a 100644 --- a/tests/data/text/syracuse.com2.txt +++ b/tests/data/text/syracuse.com2.txt @@ -2,8 +2,9 @@ The Onondaga Historical Association leads groups through the Hotel Syracuse toda Here's an update from the OHA : - • Reservation can only be made online at www.hotelsyracuseghostwalk.eventbrite.com, not by calling OHA +• Reservation can only be made online at www.hotelsyracuseghostwalk.eventbrite.com, not by calling OHA - • There are no walk-ins allowed +• There are no walk-ins allowed + +• There are a limited amount of tickets being added today, December 29th, at 2 P.M. for the January 3rd tours. - • There are a limited amount of tickets being added today, December 29th, at 2 P.M. for the January 3rd tours. \ No newline at end of file diff --git a/tests/data/text/talkingpointsmemo.com1.txt b/tests/data/text/talkingpointsmemo.com1.txt index 4b49e30d..846d843f 100644 --- a/tests/data/text/talkingpointsmemo.com1.txt +++ b/tests/data/text/talkingpointsmemo.com1.txt @@ -1 +1,37 @@ -You're seeing this error because you have enabled the setting. \ No newline at end of file +It’s a familiar scene: Young people in a makeshift press center sitting around laptops talking about the best way to post a video. Facebook first, or their own news page? Skype isn’t working. When a TV station calls her cell, one woman shushes the room with a waved hand. Behind the camera is Zanyar Omrani, a friendly journalist smuggled in and given permission to record. They could be activists anywhere in the world, except the walls are lined with AK-47s and outside everything is rubble. + +In the beginning of October, Islamic State forces surrounded and besieged the Kurdish town of Kobane in northern Syria at the border with Turkey. IS had swept through Iraq quickly, picking up American tanks and mortars abandoned by the fleeing Iraqi army along the way. Observers feared a humanitarian disaster akin to the IS massacre of ethnic Yazidis on Mount Sinjar in August. The Pentagon insisted that, with no ground troops to fight, Kobane could not be saved. Turkish tanks sat at the border, waiting for an IS victory and a possible ground invasion. + +Months after it was supposed to fall, Kobane still stands, and IS has been forced back. The story of the town’s resistance is an epic tale of determination and self-defense, and it’s one U.S. officials would rather not discuss. When the Pentagon claimed there were no ground troops in Kobane, they weren’t exactly telling the truth. In 2012, Syrian forces withdrew from three northern areas, leaving their administration and defense to the Kurdish Freedom Movement. Now the three territories hold more than four million people, half of them refugees from the Syrian civil war or the Islamic State’s advance, and many of them armed and organized. Kobane is one of the three territories that together constitute the autonomous region of Rojava, and it endures today because of the fiercely egalitarian society the people have built, the kind idealized by many leftists around the world. + +Most Americans, even the ones who pay close attention to national involvement in the Middle East, have little to no idea about what has been happening in Kobane. The Pentagon has been less than forthcoming with support for these enemies of our enemies; their fighters are still on the State Department list of foreign terrorist organizations. The American media has more or less ignored the political project in Rojava; they’ve been more comfortable painting Kurds as desperate fighters fending off the Islamic State. Besides, there are a lot of acronyms and complicated political affiliations to sort through. But while America isn’t watching, something incredible is happening. At the geographical nexus of so many of the world’s conflicts, a new post-national project is emerging. + +First, a little history and some acronyms. Because of our invasion of Iraq, when Americans think of Kurds, we tend to think of the Kurdistan Regional Government (KRG) that controls the oil-rich northern part of the country. It’s a moderate, stable region, and if there were victors of the American invasion, the KRG probably tops the list. Their president, Massoud Barzani, recently finished high on Time’s Person Of The Year list. But the KRG and their peshmerga fighters don’t control Rojava. The governing party there is the Democratic Union Party (PYD), a Syrian affiliate of the Kurdish Worker’s Party (PKK), which is a Marxist political organization founded in Ankara in 1978 that took up arms against Turkey for Kurdish independence. + + + +The PKK’s operations underwent a shift after founder and leader Abdullah Öcalan was captured by Turkey in 1999. The group had established an operations base in the Qandil Mountains at the Iran-Iraq border, but with the Soviet Union dissolved and old sources of support drying up, the Marxist-Leninist revolutionaries were stuck at a historical crossroads. They unilaterally put down their guns, then picked them up again five years later. In 2005, his death sentence reduced to life in prison, Öcalan made a surprising announcement: He renounced nationalism. The PKK and its affiliates followed suit, dropping the fight for a seat at the UN and adopting a unique political ideology called democratic confederalism. When the Syrian government withdrew from what would be Rojava in 2012, they did not declare independence despite controlling the territory. Just when the world’s largest stateless group finally seems poised to join the modern nation state system, when the dream of a Kurdistan (if not a united one) in northern Iraq is nearly at hand, the PKK has abandoned its foundational agenda. What happened? The simplest answer is they thought of a better idea. + +Despite being Turkey’s most notorious prisoner, Öcalan has kept in communication with his comrades (that’s what they call each other) through his writing, some of which has been translated for a western audience that has so far yet to appear. In a pamphlet on democratic confederalism, Öcalan laid out the premise for his unexpected turn: + +So far, with a view to issues of ethnicity and nationhood like the Kurdish question, which have their roots deep in history and at the foundations of society, there seemed to be only one viable solution: the creation of a nation-state, which was the paradigm of the capitalist modernity at that time. We did not believe, however, that any ready-made political blueprints would be able to sustainably improve the situation of the people in the Middle East. Had it not been nationalism and nation-states which had created so many problems in the Middle East? + +The Kurdish freedom struggle, he explains, had always been about “liberating the society and democratizing it” and resisting “the global domination of the modern capitalist system.” Inspired by decolonization struggles throughout the global south, the PKK’s founders followed their model, attempting a Maoist guerrilla war strategy to establish their own nation. They were always a little behind the times; by the PKK’s founding in 1978, the Cold War period of national liberation had drawn to a close. Even if, against all odds, they were to win, the most likely outcome would be a Cuba-style trade embargo enforced by Turkey. The organization needed a new vision, and democratic confederalism was Öcalan’s answer. + +At the geographical nexus of so many of the world’s conflicts, a new post-national project is emerging. + +So what is democratic confederalism? Öcalan calls it “a non-state political administration” or a “democracy without a state.” Instead of centralizing political power in a government, confederalism views society holistically, as an association of associations of intrinsically social people. It aspires to flatten the divisions the state places between public, productive and domestic life, linking together sewing cooperatives, neighborhood communes, refugee assistance groups, schools and self-defense forces in a common framework. The higher levels of organization exist to facilitate the grassroots, and decisions flow from the bottom to the top. It’s designed to work without, beneath, between and against nation-states, depending on the situation. Like the Islamic State, democratic confederalism does not recognize borders except as obstacles to overcome. “It is flexible, multi-cultural, anti-monopolistic, and consensus-oriented,” Öcalan writes. “Ecology and feminism are central pillars.” + +This sounds like the hippie rhetoric of an Occupy Wall Street press release, not the ideological program of an armed group holding territory in wartorn northern Syria. Many of us think of the Middle East as one of the most repressive and authoritarian regions, and what’s happening here is the polar opposite. Stereotypes about the Middle East being nationalist, even xenophobic, have left Americans unprepared to hear about an inclusive, egalitarian platform from someone named “Abdullah.” + +The one aspect of Rojava’s struggle that has captivated the western media— from Breitbart.com to Elle—are the Women’s Protection Units (YPJ), the female-only wing of Rojava’s army. The bands of women study political theory, sing, dance, eat and fight the Islamic State to the death for territory. Always intrigued by young women toting big guns, English-language publications have removed these fighters from their political context in order to make IS look barbaric by comparison. But even as these articles throw around terms like “hero,” they have still managed to undersell the YPJ’s accomplishments. + +Democratic confederalism takes a hard line on the role of women in society. While Americans may like the picture of sexy young women smiling while they kill IS soldiers, their official program verges on what Reddit might call “misandry.” + +“The fundamental question is why man is so jealous, dominant and villainous where woman is concerned,” Öcalan writes, “why he continues to play the rapist.” He calls for a “total divorce” from “the five-thousand-years-old culture of male domination,” including but not limited to capitalism, the state, the nuclear family, prisons and the constructed gender binary. Part of the practical application of this theory includes the formation of women-only organizations like the YPJ, which operate alongside rather than beneath gender-integrated associations like the People’s Protection Units (YPG). Decision-making in Rojava at all levels requires a quota of 40 percent women, and every municipality is led by a male-female team. Hardly the stagnant, patriarchal image Americans assign to Middle Eastern communities. + +It’s an idealistic agenda, and an audacious one, especially given the Kurds’ precarious geo-political position. But democratic confederalism is a resourceful strategy, a way to marshall and combine all of a society’s resources without unaffordable inefficiencies like profit and patriarchal domination. Does it work? In late 2011, a group of German activists traveled into the PKK-controlled area of eastern Turkey to see how, six years after Öcalan’s call, democratic confederalism was progressing in the shadow of the state. They interviewed members of many organizations, from pickle-producers and teachers to political and military leaders, and what they found was the embryo of a new society. Localities have “peace committees” to resolve disputes without the threat of jail. Women’s councils enforce social isolation for spousal abuse. In the city of Gewer, the children’s council designed the playground. To break cycles of inter-family revenge, legal committees seek a peaceful and consensual solution “even in cases of murder.” + +Although IS hasn’t been completely cleared from Kobane, it now seems clear the YPG and YPJ will win, if only by the smallest margin. Without the egalitarian social framework, there’s little doubt the town would have been lost, as American officials expected. Instead, the defense of Kobane was a lesson in the geopolitical importance of democracy beyond the state. In the wake of America’s Occupy moment, many on the left have been looking for a vision of horizontalism that lasts longer than a pop-up sneaker store. Rojava shows us it’s possible. + +Malcolm Harris is an editor at The New Inquiry and a writer based in Brooklyn. \ No newline at end of file diff --git a/tests/data/text/talkingpointsmemo.com2.txt b/tests/data/text/talkingpointsmemo.com2.txt deleted file mode 100644 index 82218109..00000000 --- a/tests/data/text/talkingpointsmemo.com2.txt +++ /dev/null @@ -1,37 +0,0 @@ -Shurtleff, a Republican, served as the state's top legal official from 2001 to 2013. He was succeeded by Swallow, also a Republican, who had been one of his top deputies. Swallow resigned less than a year after taking office, as federal and state investigations into his and Shurtleff's alleged improprieties intensified. Both men had also pursued failed bids for Congress (Shurtleff for a U.S. Senate seat in 2009; Swallow for a House seat in 2002 and 2004.) - -Former Utah attorneys general Mark Shurtleff and John Swallow were accused Tuesday of numerous bribery and obstruction of justice charges, most of them felonies. The charging documents from the Salt Lake City district attorney allege a decadent lifestyle of private jets, all-expense-paid vacations and veiled threats of violence for those who caused trouble. - -Now, the men who served as the top legal officials in Utah stand accused of soliciting bribes, accepting gifts, money laundering and interfering with the resulting criminal investigation. TPM parsed through the charging documents from the district attorney to find a few of the juiciest and most unbelievable details from the massive alleged corruption scheme. - -While one of the duo's alleged associates, Mark Jenson, was on probation for securities fraud, he allegedly paid for Shurtleff and Swallow to vacation at Pelican Hills Resort in Newport Coast, Calif., twice in 2009. The resort was rated by Conde Nast Traveler's readers as one of the best hotels in the world from 2011 to 2014. The trip included massages, golf rounds, dining and men's apparel -- all allegedly paid for by Jenson. Swallow and his wife allegedly stayed there in July 2009 for their wedding anniversary as well. - -They also traveled in style on the dime of their business friends, prosecutors allege. At the same time Swallow was allegedly working behind the scenes to help another associate, Jeremy Johnson, as he was being investigated by the Federal Trade Commission, Swallow allegedly spent two nights on Johnson's 80-foot luxury houseboat in the fall 2010. Shurtleff and Swallow also allegedly took Johnson's private aircraft on flights to and from California and St. George, Utah. - -2. Shurtleff Allegedly Had A Fixer Who Threatened To 'Bust People Up' - -Shurtleff allegedly had an associate who thought of himself as something of a bodyguard or fixer for the attorney general, according to the documents. - -In one instance outlined by prosecutors, the man, Timothy Lawson, allegedly used Shurtleff's name when trying to get someone to back off from trying to collect on a $100,000 business loan that had gone bad in a third-party dispute. Lawson compared himself to "Porter Rockwell" to explain his relationship with Shurtleff, the documents state. The reference was apparently to Orrin Porter Rockwell, the legendary bodyguard for Joseph Smith, the founder of the Mormon Church. - -In his quest to stymie the dispute, Lawson also allegedly told one of the parties that he possessed guns and "Polynesian friends" who would "bust people up." - -One of the primary focuses of the investigation was Shurtleff and Swallow's alleged efforts to help Johnson in securing approval for online poker in Utah and navigating the FTC probe. In September 2010, Swallow allegedly told Johnson that he might be able to help him with the FTC investigation by offering him access to Senate Majority Leader Harry Reid (D-NV) through another associate. The access, he allegedly warned, "won't be cheap." - -Shortly afterward, Johnson allegedly wired a total of $250,000 to Swallow's business associate, Richard Rawle. Rawle then allegedly paid Swallow $8,500. - -Reid's office has previously dismissed the connections to the Shurtleff-Swallow case and the resulting questions from critics, saying that Reid “has never been contacted in regards to this investigation." - -As all of these alleged indiscretions piled up, the defendants -- Swallow in particular -- seemed to realize that they were potentially at risk. Swallow allegedly met with Johnson at a Krispy Kreme in April 2012, to discuss his possible vulnerabilities. Johnson allegedly advised that the $250,000 paid for possible access to Reid be repaid and that all relevant emails deleted. - -Swallow also allegedly told a campaign staffer to purchase a "burner," or pre-paid, cell phone -- with cash -- a few weeks after the Krispy Kreme meeting. He allegedly asked that his government laptop be wiped of its data, telling his office's tech support staff that he had it backed up on an external hard drive. He later claimed he had lost the hard drive during a flight from Phoenix to Salt Lake City, according to the documents. - -Swallow also allegedly obtained death-bed testimony from Rawle, who died of cancer in December 2012, by providing notes to Rawle's attorney for “anyone who would be interested at some point, including the court.” - -Prior to joining the attorney general's office, according to prosecutors, Swallow had worked with Rawle as general counsel for his payday lending business. When he left the company to join Shurtleff's office in 2009, Swallow allegedly received a dozen gold one-ounce coins as payment. - -Then between June 2011 and February 2012, Swallow allegedly sold the coins back to Rawle over 10 transactions for a total sum of $17,000. Swallow allegedly asked that the money be put onto a pre-paid debit card. Swallow then allegedly gave false or inconsistent statements about the coin sales while being deposed by the Utah lieutenant governor's office in October 2013. - -Though not present in the court documents reviewed by TPM, the Deseret News, which broke the news of the arrests on Tuesday morning, reported that Shurtleff is also alleged to have flown to New York in Johnson's jet to meet with actor Vincent D'Onofrio of "Law & Order" and "Men in Black" fame. - -Photos: Former Utah Attorneys General Mark Shurtleff (left) and John Swallow are shown after being released from Salt Lake County jail on Tuesday. \ No newline at end of file diff --git a/tests/data/text/technologyreview.com1.txt b/tests/data/text/technologyreview.com1.txt index 42895540..cc801a7b 100644 --- a/tests/data/text/technologyreview.com1.txt +++ b/tests/data/text/technologyreview.com1.txt @@ -1,3 +1,5 @@ +These Streptococcus pyogenes bacteria use a DNA-cutting defense to battle viruses. The system, called CRISPR, is being harnessed to treat human genetic disease. + Last month in Silicon Valley, biologists Jennifer Doudna and Emmanuelle Charpentier showed up in black gowns to receive the $3 million Breakthrough Prize, a glitzy award put on by Internet billionaires including Mark Zuckerberg. They’d won for developing CRISPR-Cas9, a “powerful and general technology” for editing genomes that’s been hailed as a biotechnology breakthrough. Not dressing up that night was Feng Zhang (see 35 Innovators Under 35, 2013), a researcher in Cambridge at the MIT-Harvard Broad Institute. But earlier this year Zhang claimed his own reward. In April, he won a broad U.S. patent on CRISPR-Cas9 that could give him and his research center control over just about every important commercial use of the technology. @@ -14,6 +16,8 @@ The control of the patents is crucial to several startups that together quickly Zhang cofounded Editas Medicine, and this week the startup announced that it had licensed his patent from the Broad Institute. But Editas doesn’t have CRISPR sewn up. That’s because Doudna, a structural biologist at the University of California, Berkeley, was a cofounder of Editas, too. And since Zhang’s patent came out, she’s broken off with the company, and her intellectual property—in the form of her own pending patent—has been licensed to Intellia, a competing startup unveiled only last month. Making matters still more complicated, Charpentier sold her own rights in the same patent application to CRISPR Therapeutics. +No CRISPR drug yet exists. But if CRISPR turns out to be as important as scientists hope, commercial control over the underlying technology could be worth billions. + In an e-mail, Doudna said she no longer has any involvement with Editas. “I am not part of the company’s team at this point,” she said. Doudna declined to answer further questions, citing the patent dispute. Few researchers are now willing to discuss the patent fight. Lawsuits are certain and they worry anything they say will be used against them. “The technology has brought a lot of excitement, and there is a lot of pressure, too. What are we going to do? What kind of company do we want?” Charpentier says. “It all sounds very confusing for an outsider, and it’s also quite confusing as an insider.” @@ -22,6 +26,8 @@ Academic labs aren’t waiting for the patent claims to get sorted out. Instead, Because of all the new research, Zhang says, the importance of any patent, including his own, isn’t entirely clear. “It’s one important piece, but I don’t really pay attention to patents,” he says. “What the final form of this technology is that changes people’s lives may be very different.” +Twitter CEO Dick Costolo (far left) and actress Cameron Diaz present the Breakthrough Prize to biologists Jennifer Doudna and Emmanuelle Charpentier last month in Mountain View, California. Each won $3 million. + The new gene-editing system was unearthed in bacteria—organisms that use it as a way to identify, and then carve up, the DNA of invading viruses. That work stretched across a decade. Then, in June 2012, a small team led by Doudna and Charpentier published a key paper showing how to turn that natural machinery into a “programmable” editing tool, to cut any DNA strand, at least in a test tube. The next step was clear—scientists needed to see if the editing magic could work on the genomes of human cells, too. In January 2013, the laboratories of Harvard’s Church and Broad’s Zhang were first to publish papers showing that the answer was yes. Doudna published her own results a few weeks later. diff --git a/tests/data/text/technologyreview.com2.txt b/tests/data/text/technologyreview.com2.txt index b243da7e..5351f757 100644 --- a/tests/data/text/technologyreview.com2.txt +++ b/tests/data/text/technologyreview.com2.txt @@ -1,3 +1,5 @@ +A street vendor in India uses a light powered by refurbished battery cells. + Many of the estimated 50 million lithium-ion laptop batteries discarded every year could provide electricity storage sufficient to light homes in poor countries, researchers at IBM say. In work being aired this week at a conference in San Jose, researchers at IBM Research India in Bangalore found that at least 70 percent of all discarded batteries have enough life left to power an LED light at least four hours a day for a year. diff --git a/tests/data/text/telegraph.co.uk1.txt b/tests/data/text/telegraph.co.uk1.txt index 1b070d1a..fd6869ff 100644 --- a/tests/data/text/telegraph.co.uk1.txt +++ b/tests/data/text/telegraph.co.uk1.txt @@ -1,62 +1,231 @@ -09.06 Further confirmation from Reuters: -Debris sighted in Indonesia's Java Sea is from the AirAsia jet presumed to have crashed two days ago, Tatang Zaenudin, an official at the country's search and rescue agency, told Reuters on Tuesday. -09.03 REUTERS - INDONESIA RESCUE AND SEARCH AGENCY OFFICIAL SAYS JAVA SEA DEBRIS IS FROM MISSING AIRASIA JET -08.46 The transcript of the final communication between air-traffic control and the pilot of AirAsia Flight 8501 has been released. -According to The New York Post, It reveals a calm request to redirect the plane and then to climb to avoid a storm. -Air-traffic control couldn’t say yes immediately because six other planes were crowding the higher airspace, forcing Flight 8501 to remain at a lower altitude, the transcript reveals. -By the time the pilot was given the OK, a mere two minutes later, it was apparently too late - there was no response from the cockpit. -Indonesia’s state navigation operator said late Monday that the Airbus 320-200 pilot, Capt. Iriyanto, who goes by one name, had requested permission to turn left to avoid a storm. -The request was granted and the plane turned left for seven miles. -The captain then sent his fateful message at 6:12 a.m., saying, “Request to higher level,” according to AirNav Standards and Safety Director Wisnu Darjono, as quoted by the Jakarta Post. -“Intended to what level?” the controller responded. When Iriyanto said 38,000 feet, he was told to hold off because there were six other planes in that area. -“But when we informed the pilot of the approval at 6:14 a.m., we received no reply,” Wisnu said. -08.41 The Jakarta Post quotes a co-pilot of the Air Force Hercules C130, who flew on Tuesday over waters near Pangkalan Bun in Central Kalimantan in the search for missing AirAsia flight QZ8501. -Lt. Tri Wibowo said that he saw dozens of floating bodies as well as bags and aircraft debris. -QuoteWe thought that the passengers were still alive and waved at us for help. But when we approached closer [we saw] they were already dead. -Tri and the team scoured the area at around 11 a.m. -The Transportation Ministry earlier confirmed that white and red debris found near Pangkalan Bun belonged to the ill-fated flight, which vanished on Sunday en route to Singapore from Indonesia's second-biggest city of Surabaya in East Java. -08.38 Tony Fernandes, the boss of AirAsia said his heart was "filled with sadness" after several bodies were found floating in the sea near where a missing AirAsia flight was last seen. -Earlier, an Indonesia National Search and Rescue spokesman said several pieces of debris seen floating off Borneo might be linked to the missing Flight 8501. -Mr Fernandes, the AirAsia chief executive, said -QuoteMy heart is filled with sadness. Words cannot express how sorry I am. On behalf of AirAsia, my condolences to all. Words cannot express how sorry I am. -08.29 In other plane news, it's just breaking now that a Thai Airways plane TG916 flying from Bangkok to London is dumping fuel and returning to Bangkok, with problems. -08.15 Plane spots 'shadow' on seabed believed to be AirAsia jet: Indonesia search chief -08.10 The Telegraph will not be publishing the images of the victims. The Indonesian search agency chief says the six bodies are swollen but intact and have been transferred to a navy ship. -08.08 As if it couldn't get worse for the families: -08.00 What we know at the moment: -- Items resembling emergency slide, life jackets, plane door and luggage spotted six miles from last point of contact -- Bodies have been spotted by search planes -- 11 divers heading to the debris area -- the waters are warm and relatively shallow at 80ft so teams are holding out hope for surviors -- Officials say they are 95 per cent sure items are from the missing Airbus -- Distraught relatives have been broken the news -07.52 Jonathan Pearlman, our correspondent at the press conference in Jakarta, says Indonesia's search authority has confirmed a body has been found in the water along with debris confirmed to be from the missing Air Asia plane. Indonesia's TV One has shown a rather gruesome photo of one body floating in water. -Henry Bambang Sulistyo, head of Indonesia’s National Search and Rescue Agency, said a search team found the plane's emergency exit at 12.40pm, local time, and confirmed it came from the missing plane. At 1.25pm, further objects were found in the waters, which are apparently about 80 to 100 feet deep. -Mr Sulistyo said the debris is being recovered and taken back to shore at the ton of Pangkalan Bun to be examined. Twenty-one naval and rescue divers are being sent to the area. He said heavy weather is hampering the search. -07.45 Preparing for the grim reality: -07.40 Debris sighted is the emergency exit door, says Indonesia's National Search and Rescue Agency. The chief says he's 95 per cent sure the flotsam is from the Airbus. The waters where the objects have been found are not deep, the team estimate about 80-100ft. -07.38 Indonesia's director general of civil aviation Djoko Murjatmodjo says: "Based on the observation by search and rescue personnel, significant things have been found, such as a passenger door and cargo door. It's in the sea, 100 miles southwest of Pangkalan Bun," he said, referring to the town in Central Kalimantan, on the island of Borneo. -07.30 Distraught family members are being called in to meet officials -07.18 A navy chief spokesman says "crew had visual of people at sea surface, not far from debris". He says he is also checking whether anyone is still alive. The waters in the Java Sea are fairly warm, but it has been more than 40 hours since the crash. -07.12 It seems to have been confirmed. Debris is "from AirAsia plane", Indonesian civil aviation chief says. "For the time being it can be confirmed that it's the AirAsia plane and the transport minister will depart soon to Pangkalan Bun," Djoko Murjatmodjo said. -07.00 Distressing news, but Jakarta-based Kompas TV has said that the co-pilot of an Indonesian air force C130 Hercules is reporting floating objects resembling humans, suitcases, buoys and aircraft debris. A reminder - this is all siz miles from where AirAsia flight QZ8501 was last logged on radar. Airborne search teams are naturally concerned that the debris could be moved by the current in the time it takes one of the ships to reach it. -06.50 It is nearly 2pm in Indonesia, and the sun sets around 6pm, so they have a little time to try to get search ships to the area and gather the debris to analyse. -06.46 For reference, this is what an Airbus 320 emergency slide looks like: -06.45 "The debris is red and white," Djoko Murjatmodjo, acting director general of air transportation at the transportation ministry, told reporters. "We are checking if it's debris from the aircraft. It's probably from the body of the aircraft." -06.38 BREAKING: An Indonesian transport ministry official says debris found in waters near Borneo is likely from Flight QZ8501. The Acting Director General of Transport Murjatmodjo also says he "strongly suspects fragments are part of AirAsia plane." -06.15 The AFP photographer on board the search flight has the taken the clearest photos yet of the debris: -06.05 “These look like items that are not usually seen on sea surface,” Indonesian air force official Agus Dwi Putranto has said. -06.00 Items resembling an emergency slide, plane door and other objects were spotted during an aerial search Tuesday for missing AirAsia flight 8501, Indonesian officials said. -"We spotted about 10 big objects and many more small white-coloured objects which we could not photograph," Indonesian air force official Agus Dwi Putranto told a press conference. "The position is 10 kilometres (six miles) from the location the plane was last captured by radar," he said. -He displayed 10 photos of objects resembling a plane door, emergency slide, and a square box-like object. -An AFP photographer on the same flight that spotted the debris said he had seen objects in the sea resembling a life raft, life jackets and long orange tubes. -05.35 This just in: Items resembling an emergency slide and a plane door have been spotted in AirAsia search, according to Indonesia and as reported by AFP. The debris is said to be to be seen 105 nautical miles from Pangkalan Bun. Jonathan Pearlman, our correspondent in Jakarta, has this: -Quote The search authority has confirmed that an object believed to be from the plane has been spotted in the south-west area of the search zone. It was spotted at 10.15am, local time, apparently by a plane from the National Search and Rescue Agency. An agency official told me that a fisherman spotted the object and reported it to the agency, which sent a plane to investigate. Apparently, they saw numerous objects including what appeared to be a flashing light. They are attempting to retrieve the objects now.05.12 Our correspondent in Surabaya, Tom Phillips, has just been to a press conference with the families. He says they will be flown over the search area tomorrow on a specially chartered Airbus 320 - the same plane as the QZ8051. "They believe their presence, their praying, will help the search and rescue to quickly find the aircraft', the AirAsia Indonesia chief says. -04.20 Officials have said the sea in the general search area was only 150 to 300 feet deep, which would be a help in finding the plane. -"The Java Sea area where they are now searching isn't even an ocean, it's more of an inland sea," Erik van Sebille, a physical oceanographer at the University of New South Wales in Sydney told Reuters. -"It's so shallow that they may just be able to spot the plane," said van Sebille, noting that sunlight travels through water up to about 100 metres. -Oceanographer Pattiaratchi said debris would normally be expected to float on the surface for around 18 days before sinking. -USS Sampson, the American warship on its way to help with the search: -04.05 In answer to our earlier question: why are planes still not fitted with the necessary technology to help find them if they lose contact with ground control? Wall Street Journal is reporting that AirAsia had adopted dedicated tracking technology but the QZ8501 had not been upgraded yet: -04.00 Unfortunately the search area only seems to be getting bigger, not smaller, as the hours pass: -02.50 The cause of the smoke could be any number of factors so should be treated with caution. One aviation expert has an alternate theory as to what is causing it: +17.00 We are going to leave our live coverage here for today, but will be back at first light in Indonesia (around midnight GMT) with further updates as the search continues. + +Meanwhile, Jonathan Pearlman in Pangkalan Bun, and Tom Phillips in Surabaya have this wrap of the day's events: + +16.06 AirAsia QZ8501 made an "unbelievably" steep climb before it crashed, possibly pushing it beyond the Airbus A320's limits, according to radar data, a source close to the investigation has told Reuters. + +"So far, the numbers taken by the radar are unbelievably high. This rate of climb is very high, too high. It appears to be beyond the performance envelope of the aircraft," he said. + +The source, who declined to be named, added that more information was needed to come to a firm conclusion. + +15.25 As night falls in southeast Asia, the search has been called off until tomorrow. + +Meanwhile, Indonesians in Surabaya, East Java are holding a candle-lit vigil for the victims of the crashed AirAsia airplane + +14.53 These latest images from the search mission have just been released by the Malaysian Navy: + +Royal Malaysian Navy ship KD Lekir is show here retrieving the body of a victim + +The same ship is here shown retrieving AirAsia flight QZ8501's emergency evacuation tube + +14.29 Indonesian authorities are hoping to bring the five remaining bodies that have been recovered from the AirAsia plane back to land at around midnight, local time. The bodies have been stranded on an Indonesian naval vessel and could not be flown back from sea due to the poor weather. Only two bodies were able to be transferred by helicopter. + +The new plan is for the naval ship to approach land near the coastal town of Kumai and transfer the bodies to a smaller boat that can enter the town's bay. The bodies will be left in hospital overnight and then flown to Surabaya tomorrow. The plan was due to have happened already but was delayed due to the poor weather. + +14.07 An Indonesian search and rescue official has cast further doubt on claims that one of the bodies recovered from the crashed AirAsia jet had been wearing a life jacket. + +"There is no victim that has been found wearing a life jacket," said Tatang Zaenudin, deputy head of operations at the national search and rescue agency. + +"We found a body at 8.20 a.m. and a life jacket at 10.32 a.m. so there was a time difference. This is the latest information we have," he added. + +The same official told Reuters earlier that one of the recovered bodies had been wearing a life jacket + +13.45 There remain several possible explanations for why QZ8501 crashed. David Millward, the Telegraph's US correspondent and former Transport Editor, explains the most likely: + +The smart money seems to be moving towards the pitots freezing – they are the little detectors near the nose of the aircraft measuring wind velocity they are vital for the pilot when judging the wind speed. This is what did for the Air France airbus, albeit rather a larger one. + +There was also a suggestion on NBC that the fuel flow could have been interfered with because of extreme cold. That is what caused the BA 747 to pancake at Heathrow a few years ago. Simplest way of explaining is to think of it as an aviation equivalent to a cardiac arrest because of blocked fuel intake pipes. It had a different engine, however to the BA jumbo. + +13.08 Tony Fernandes, the Air Asia chief, has been speaking at Surabaya's airport about the rescue operation, badly hampered today by poor weather. + +"The weather unfortunately is not looking good for the next two or three days," he told reporters. "That is slowing us down." + +Mr Fernandes refused to discuss speculation about whether or not a passenger was found wearing a life-jacket but confirmed that the main fuselage of Flight QZ8501 had not been found. + +12.15 More photos are coming in from the search. + +11.48 There are still conflicting reports about whether a body was found wearing a life jacket. + +We are trying to get the answers for you. + +11.23 Our picture desk have compiled a gallery of some of the most powerful images from the search. + +AirAsia plane found in Java Sea, in pictures + +11.18 More from Tom Phillips in Indonesia. + +I've just spoken to one of the relatives who has come to a Disaster Victim Identification centre set up beside the hospital. + +"I hope they can identify my brother soon," said Sigit Heriyanto, who had three family members on the plane. + +Family members are tonight giving blood samples for DNA analysis. But no more bodies are expected to arrive today. + +10.58 The latest updates from Tom Phillips in Indonesia. + +Police forensic chiefs have just been briefing us here at the hospital where 15 forensic doctors will identify victims brought here from crash site. + +They don't expect any more bodies to arrive in Surabaya today because bad weather has hampered the search. + +An AirAsia pilot, who is helping with the identification process, told me just after the bodies arrived: + +It’s already 6 o’clock now. I guess it is very hard to find anything if it dark. + +09.52 In Indonesia, Tom Phillips has been speaking to the family of the pilot, Captain Irianto. + +He writes: + +When the devastating images of Flight QZ8501's wreckage flashed up on television screens on Tuesday, few were left with any illusions about what had happened to the 162 people on board. + +But on Wednesday, as the first of the victims' bodies were flown back to the Indonesian city of Surabaya, one person remained completely in the dark: an 8-year-old boy called Arya Galih Gegana. + +Galih, as he is known, is the son of Irianto, the veteran pilot who was flying the doomed AirAsia flight when it took off in the early hours of Sunday. Relatives have yet to tell him that the father he so adored is gone. + +"I told Galih that his father will not be coming straight home because he has another job to do," said Budi Sutiono, the pilot's 55-year-old brother and the boy's uncle. "He thinks daddy is still at work. + +AirAsia pilot's son 'thinks daddy is still at work' + +08.55 Here's the latest from our video team on the search for wreckage and bodies. + +08.21 Following the offer by the British government to assist in the search for missing Air Asia flight QZ8501, an investigator from the UK's Air Accident Investigation Branch (AAIB) is now helping with the search for the plane's flight recorders. + +The AAIB investigator has joined Singaporean experts to assist the Indonesian operation. They hope to reach the search area on an Indonesian naval vessel as soon as possible. The AAIB investigator is carrying specialist technical equipment that can help locate the flight recorders. + +Moazzam Malik, the British Ambassador to Indonesia, said: + +On behalf of British people and Her Majesty's Government I would like to express our deepest condolences to the family of the passengers and crew. Our thoughts and prayers are with them at this difficult time. + +The UK Air Accident Investigation Branch has long experience, expertise and technical equipment that can help locate flight recorders underwater. I hope their assistance can help the Indonesian authorities locate the black box for Air Asia flight 8501 quickly. We stand ready to help with the subsequent investigation if required." + +07.49 AirAsia boss Tony Fernandes has just tweeted his thanks to the authorities in Indonesia. + +Twitter: Tony Fernandes - Besanas and the Indonesia army, navy and police have been very transparent and very detailed. I thnk them all for their incredible efforts. + +You can read more about Mr Fernandes - Asia's answer to Richard Branson - here. + +AirAsia plane missing: profile of airline boss Tony Fernandes + +07.38 Our correspondent Jonathan Pearlman, in Pangkalan Bun - where the bodies are being brought from the crash site - brings us the following update: + +The two bodies from the crashed AirAsia flight that were returned to land earlier today have now been identified, prepared in coffins and flown to Surabaya. The Indonesian search authority said the bodies belonged to a boy, 140cm tall, wearing a blue shirt and brown trousers, and a woman, 160cm tall, wearing blue jeans and a dark blue shirt. + +Some of the remaining five bodies found so far are due to land here soon and will be identified and flown off in similar fashion. + +Separately, AirAsia owner Tony Fernandes is at the airport in Pangkalan Bun to meet with the Indonesian search authority. + +07.13 + +Twitter: Tom Phillips - The Indonesian gov this morning asked its citizens to cancel outdoor New Year's Eve celebrations as sign of respect to #QZ8501 victims + +06.59 More on that body recovered from the crashed plane. + +Tatang Zaenudin, an official with the search and rescue agency, said: + +This morning, we recovered a total of four bodies and one of them was wearing a life jacket. + +06.48 Reuters - INDONESIA SEARCH AND RESCUE AGENCY SAYS BODY RECOVERED FROM AIRASIA JET WAS WEARING LIFE JACKET + +Relatives of the missing passengers on board the AirAsia plane pray at Juanda Airport, in Surabaya, Indonesia (EPA) + +06.25 Jonathan Pearlman in Palangkan Bun has spoken to the head of the search and rescue agency: + +The Indonesian search authority says it expects bodies of survivors from the AirAsia flight to start washing up ashore along the coastline in the coming days. Air Vice Marshal Sunarbowo Sandi, the mission coordinator in Palangkan Bun, says strong winds and currents have already moved debris up to 30 miles since they were first seen yesterday. He says one more body was found today but poor weather and visibility has made searching all but impossible and is preventing authorities from flying all seven bodies back to shore. The bodies are on naval vessels; two have been flown to land by helicopter. + +He says there has been no sign of the black box but a vessel with detection equipment is being dispatched. He is not aware of any bodies being found in life jackets. More than 90 divers are ready on ships to start searching but have been unable due to the poor weather conditions. He also said there had been no confirmation that the body of the plane has been spotted, though some search personnel have reported seeing something that "seemed" to be the plane. + +06.20 This from Tom Phillips, our correspondent in Surabaya: + +There are 75 ambulances lined up at Surabaya airport this morning in anticipation of the arrival of the victims' bodies. + +It's not clear when the victims will start arriving here, but when they do they will be taken to the city's Bhayankara Hospital, around 6 miles away. Already six bodies have been recovered we now understand, including one wearing an AirAsia uniform. + +Relatives have been holding prayer sessions this morning at the airport as they wait for news about the progress of the rescue operation. + +So many ambulances lined up outside military airport in Surabaya this morning to receive bodies of #QZ8501 victims pic.twitter.com/16xocpihlY — Tom Phillips (@tomphillipsin) December 31, 2014 + +I counted nearly 30 ambulances on standby to take #QZ8501 victims' bodies to hospital in Surabaya for identification pic.twitter.com/PhMGY3uuzf — Tom Phillips (@tomphillipsin) December 31, 2014 + +06.10 Update: total of 7 bodies - 4 men, 3 women - recovered so far, says search chief. One was air hostess Khairunisa Haidar Fauzi,, who was in uniform, one was in a life jacket according to reports, and three were found still holding hands. + +Indonesian police collect AirAsia passenger records from relatives to help identification of the victims at Juanda Airport, in Surabaya, Indonesia (EPA) + +05.20 Seventh body has reportedly just been found. Our correspondent Jonathan Pearlman has just arrived in Pangkalan Bun airport, where Indonesian military just arrived with two bodies. + +#QZ8501: Search teams have just brought a body back to Pangkalan Bun from sea via helicopter http://t.co/fU1QeMQPFC pic.twitter.com/xrDVp5Qaqn — Channel NewsAsia (@ChannelNewsAsia) December 31, 2014 + +04.25 Three people killed on AirAsia flight QZ8501 were holding hands when their bodies were spotted floating in the Java sea off Indonesia, a pilot involved in the search for the missing plane has claimed. + +Lieutenant Airman Tri Wobowo, who co-piloted the C130 Hercules aircraft that first saw debris of the plane on Tuesday, told Indonesian newspaper Kompas: "There are seven to eight people. Three [of them] again hold hands." + +Members of an Indonesian search and rescue team carry a body on a stretcher during the recovery operation (AFP/Getty) + +04.00 Searchers says they've seen a sonar image appears to show an aircraft lying upside down on the ocean floor. The newspaper quoted the agency as saying the image appeared to show an airplane upside down in 24-30 meters of water. + +03.12 We have a little more on the bodies pulled up today: apparently three are male and three female including, according to reports, 20-year-old trainee flight assistant Khairunisa Haidar Fauzi. Ms Khairunisa’s father, Haidar Fauzi, has said: “This is what we expected, and the sooner the body is returned, the sooner we can have her back.” + +#PrayForAirAsia ✈ Air Asia flight attendant Khairunisa Haidar Fauzi was travelling on the missing flight. pic.twitter.com/RBBPdyUNaS — J (@jealousay) December 30, 2014 + +02.50 As we reported earlier, searchers found three more bodies this morning, including a flight attendant still wearing her AirAsia uniform, bringing the total to six so far, Fully clothed bodies could also indicate the plane was intact when it hit the water and support a theory that the Airbus A320-200 suffered an aerodynamic stall and plunged into the sea. + +"The fact that the debris appears fairly contained suggests the aircraft broke up when it hit the water, rather than in the air," said Neil Hansford, a former pilot and chairman of consultancy firm Strategic Aviation Solutions. + +#QZ8501 search: Backpack, food, kid's shoes found by Indonesia ship Bung Tomo: @detikcom http://t.co/fU1QeMQPFC pic.twitter.com/QiXysgpqA4 — Channel NewsAsia (@ChannelNewsAsia) December 31, 2014 + +02.10 AIrAsia boss is on his way to the island receiving bodies to comfort relatives. + +I hope to fly to evacuation site to thank everyone for putting in so much effort in finding the site. And keep everyone informed. — Tony Fernandes (@tonyfernandes) December 31, 2014 + +02.10 Stormy weather has forced Indonesian rescuers to suspend their search for the bodies. + +"We are experiencing bad weather now. Rains and winds prevented us from resuming the search operation this morning," air force rescue coordinator S.B. Supriyadi told AFP. + +Indonesian Navy divers inspect their gear upon arrival for the search operation for the victims of AirAsia flight QZ 8501 at the airport in Pangkalan Bun, Indonesia (AP) + +National Search and Rescue Agency chief Bambang Soelistyo told a press conference Wednesday that six bodies had now been recovered, including a woman in crew uniform. + +"As soon as the weather is clear, the bodies will be brought to Pangkalan Bun," the town with the nearest airstrip to the crash site, said Soelistyo. + +Supriyadi said that hundreds of people from the military, police and national rescue agency were on standby waiting for clear weather in Pangkalan Bun. + +#QZ8501 search: DNA tests on recovered bodies being conducted by police, says police chief http://t.co/fU1QeMQPFC pic.twitter.com/3dieemkTkT — Channel NewsAsia (@ChannelNewsAsia) December 31, 2014 + +02.00 The UK air investigation bureau join will join today's search with special technology to track the exact location of the airplane beacon under water. + +00.35 The Telegraph team in China has been speaking to families of the passengers of Malaysia Airlines Flight MH370 that disappeared in March and, unlike QZ8501, has still yet to be found. + +“Many people say they share the pain and sorrow of the victims’ families, but only we, the families, know exactly how it feels. We have been there before,” said Steve Wang, whose mother was one of 239 people on the plane. + +“I feel so sad that history is repeating itself. I don’t know why it is happening again,” he said. + +Dai Shuqin, a 61-year-old who had five relations on MH370, said she had been constantly thinking about those caught up in the latest Asian air disaster. I pray for them to have at least one people survive,” she said. “If just one person might survive on QZ8501, it would give me hope for MH370.” + +Mrs Dai said she was asking for a “miracle”. “I cannot believe that 10 months later another group of people has been on a lost plane again. All those lives!” + +Jiang Hui, whose mother was on MH370, said: “I don't know how to describe the feeling I had when I heard [about QZ8501]” + +“It was like waking up from a nightmare – I was covered in goose-bumps. How could there be so many disasters this year?” + +Sudarsan Pattnaik works on a sand sculpture portraying the missing AirAsia and Malayasia Airlines planes, on Golden Sea Beach in Puri, in India's eastern Odisha state + +00.30 Aviation experts believe the fuselage may be easily found as the aircraft probably only broke up when it hit the water. + +"The fact that the debris appears fairly contained suggests the aircraft broke up when it hit the water, rather than in the air," Neil Hansford, a former pilot and chairman of consultancy firm Strategic Aviation Solutions, said on Wednesday. + +00.20 Relatives had been holding out hope that against all odds, their family members would be saved. One father learned the horrible news that his daughter Maria Widodo had likely not made it. + +Maria had recently graduated from the National Institute of Education, the Straits Times reported. In college she was part of her university’s guitar ensemble, along with her boyfriend, Andy Paul Chen, who is from Singapore. + +Following news of the plane’s disappearance, Florentina Maria Widodo’s father changed his profile picture on Facebook to one of him and his daughter, and Chen posted the news that his girlfriend was on the flight. + +Our deepest condolences #airasiamissing pic.twitter.com/Nli9H4c2md — Bigi (@DrBigi) December 30, 2014 + +00.04 The grim task of identification of bodies from #QZ8501 will begin today. Family members have already been asked for photos and provided DNA. Due to bad weather, only three bodies have so far been recovered. + +23.46 This is the search area teams will be concentrating on this morning - it's about the size of Wales. + +Malaysia’s @ChiefofNavy: today’s ongoing #QZ8501 search is split into four sea sectors totalling 5378 square km. pic.twitter.com/ha8AezcsjR — John Walton (@thatjohn) December 30, 2014 + +23.45 It is now approaching 7am in Jakarta.The search for more bodies - and clues as to the cause behind AirAsia Flight QZ8501's crash into the Java Sea - continues. It was a long night for families, who learnt the fate of their loved ones in the most devastating of ways. + + + +Relatives of passengers on AirAsia flight QZ 8501 console each other (Getty) \ No newline at end of file diff --git a/tests/data/text/telegraph.co.uk2.txt b/tests/data/text/telegraph.co.uk2.txt index 4b41a9f5..fadebd22 100644 --- a/tests/data/text/telegraph.co.uk2.txt +++ b/tests/data/text/telegraph.co.uk2.txt @@ -4,7 +4,7 @@ In comments which are likely to be seen by Conservatives as a further vindicatio Previously unseen papers show Mrs Thatcher warned six months before GCSEs replaced O-levels in 1986 that she did not “like the sound of the new exam”, and asked for its introduction to be delayed. -However she eventually concluded that to intervene would amount to a public “contradiction” of Keith Joseph, the education secretary and a close friend, and appear as if she was taking the side of teaching unions, which wanted more time to prepare for the new system. She therefore had “no option but to go ahead”, she told aides. +However she eventually concluded that to intervene would amount to a public “contradiction” of Sir Keith Joseph, the education secretary and a close friend, and appear as if she was taking the side of teaching unions, which wanted more time to prepare for the new system. She therefore had “no option but to go ahead”, she told aides. Her previously unknown concerns are revealed in official papers from 1986 released by the National Archive in Kew, west London. @@ -12,9 +12,9 @@ Her fears appear to chime with the views of Conservatives about the GCSE system Michael Gove, who was education secretary until the summer, is said to believe that the introduction of the exam was a “historic mistake” that has led to a dramatic fall in standards. Before stepping aside to become Chief Whip he set in train an overhaul of the curriculum which he said would address “the pernicious damage caused by grade inflation and dumbing down”. -GCSEs were eventually introduced in September 1986 with Kenneth Baker, now Lord Baker of Dorking, as education secretary, after Mr Joseph stepped down in May - weeks after his clash with Mrs Thatcher over the exams. +GCSEs were eventually introduced in September 1986 with Kenneth Baker, now Lord Baker of Dorking, as education secretary, after Sir Keith stepped down in May - weeks after his clash with Mrs Thatcher over the exams. -Mr Joseph had insisted that the new scheme was intended to better stretch teenagers, creating a tougher, “clearer and fairer” system. Under the old system, as part of which more academic teenagers took O-levels and others took CSEs, individual grades were awarded largely based on the relative performance of competing candidates. +Sir Keith had insisted that the new scheme was intended to better stretch teenagers, creating a tougher, “clearer and fairer” system. Under the old system, as part of which more academic teenagers took O-levels and others took CSEs, individual grades were awarded largely based on the relative performance of competing candidates. GCSEs were intended to ensure a focus on “how much or how little pupils understand, know and can do”. @@ -24,7 +24,7 @@ He insisted that the new system would “inject more rigour” and “use-able l Mrs Thatcher marked his three-page briefing note with a series of hand-written annotations, complaining about his use of “an awful lot of high language” and questioning a number of his claims. -Mr Joseph said that under the O-level and CSE system pupils were “simply ranked in merit order, with little regard to how much or how little pupils understand, know and can do”. +Sir Keith said that under the O-level and CSE system pupils were “simply ranked in merit order, with little regard to how much or how little pupils understand, know and can do”. But in a hand-written annotation to his letter Mrs Thatcher said: “This is not a correct judgement of the present examinations system. We were taught to think and apply 50 years ago.” diff --git a/tests/data/text/thai.txt b/tests/data/text/thai.txt new file mode 100644 index 00000000..b2ce780d --- /dev/null +++ b/tests/data/text/thai.txt @@ -0,0 +1,15 @@ +21 ม.ค.2561 เวลาประมาณ 12.00 น. ผู้สื่อข่าวได้รับแจ้งจากนาย ต.(ไม่ประสงค์เปิดเผยชื่อ) บุตรชายของ 'ภูชนะ' คนสนิทของนายสุรชัย แซ่ด่าน ที่หายไปว่า ผลการตรวจ DNA โดยใช้เนื้อเยื่อของศพที่ถูกสังหารด้วยการมัดแขน รัดคอ ทุบจนใบหน้าเละ และท้องถูกผ่ายัดเสาปูนที่ลอยมาติดที่ อ.ธาตุพนม จ.นครพนม นั้น เมื่อตรวจสอบแล้วมีความเกี่ยวพันทางสายเลือดกับนาย ต.จริง + +ภูชนะ (นามแฝง) เป็นคนใกล้ชิดของนายสุรชัย ด่านวัฒนานุสรณ์ หรือ สุรชัย แซ่ด่าน นักเคลื่อนไหวทางการเมืองที่ลี้ภัยออกจากประเทศไทยและได้หายตัวไปจากที่พักในประเทศเพื่อนบ้านในช่วงคืนวันที่ 12-13 ธ.ค.2561 พร้อมกันกับสุรชัยและคนสนิทอีกคนหนึ่ง + +ปรานี ด่านวัฒนานุสรณ์ ภรรยาของสุรชัยกล่าวว่า ได้ทราบข่าวจากบุตรชายของภูชนะแล้ว แต่ยังไม่ขอพูดอะไร โดยเบื้องต้นได้ทำใจแต่แรกแล้วว่าเหตุการณ์ลักษณะนี้จะต้องเกิดขึ้นสักวันหนึ่ง + +ขณะที่ นาย ว. พี่เขยของ 'กาสะลอง' อีกหนึ่งผู้ลี้ภัยที่ได้หายไปพร้อมกับนายสุรชัยกล่าวกับผู้สื่อข่าวว่า หลังจากได้ทราบผลการตรวจสอบ DNA จากลูกชายของภูชนะ ทางญาติของกาสะลองก็ได้ติดต่อไปที่พนักงานสอบสวนเจ้าของคดีและได้คำตอบว่า จะได้ทราบผลการตรวจสอบ DNA ภายในเวลา 2-3 วันนี้ + +สุรชัย (78 ปี) ภูชนะ (54 ปี) กาสะลอง (47 ปี) เป็นนักเคลื่อนไหวทางการเมือง และเป็นผู้ลี้ภัยจากเหตุการณ์รัฐประหาร 2557 ไปยังประเทศเพื่อนบ้านได้หายออกจากที่พักในประเทศเพื่อนบ้านโดยที่ไม่มีใครสามารถติดต่อได้จนปัจจุบันนับเป็นเวลานาน 1 เดือน กับอีก 10 วัน + +เรื่องที่เกี่ยวข้อง + +ลืออุ้ม อ.สุรชัย ผู้ลี้ภัยหายจากที่พัก 12 วันแล้ว + +ตร.เร่งหาเบาะแสคดีฆ่ายัดเสาถ่วงน้ำโขง - เมียไม่เชื่อเป็น 'สุรชัย แซ่ด่าน' \ No newline at end of file diff --git a/tests/data/text/theatlantic.com1.txt b/tests/data/text/theatlantic.com1.txt index b0279ec4..72a89385 100644 --- a/tests/data/text/theatlantic.com1.txt +++ b/tests/data/text/theatlantic.com1.txt @@ -1,21 +1 @@ -After signing up to write a script for Croatian television, I learned that virtually all TV comedies, from Seinfeld to South Park, follow a simple formula. - -As happens to so many of us, I was asked to write a sitcom for Croatian television. I’m an American ex-pat living in Slovenia, and I know next to nothing about Croatia, besides the fact that it’s Slovenia’s southern neighbor, a fellow ex-Yugoslav republic, and that the language resembles Slovene except with a lot more “js” in it. I am a writer of books and articles, and I used to write a lot of plays, but I’ve never written for television. So I immediately said, “Sure, of course I can do that,” before rushing off to Google “How to write a sitcom.” - -In addition to much Googling, I spent a good deal of time watching sitcoms. I was after tips on how they are constructed, and watched actively, looking to crack open their laugh-tracked shiny exterior to get at the goopy mechanism within, to see how they functioned. What I found out surprised me, and changed the way that I watch television. - -From The Simpsons to Seinfeld, from Everybody Loves Raymond to Everybody Hates Chris, from Taxi to Arrested Development to Parks & Recreation, there is a highly-specific, minute-by-minute recipe used to write the vast majority of sitcoms out there. And once you know the formula, it makes it much easier to write them, and much harder to watch them without seeing that formula—the “sitcom code”—everywhere you look. - -My giddy-panicked Googling actually produced fruitful results. With little idea as to where I should begin, I turned to the confidence-inspiring blog, Wise Sloth (whose author, like me, has no TV writing experience), which provided a 15-page breakdown of sitcom formats that I used as a point of departure for my own study. And by study, I mean hopping into my pajamas, cuddling up to my Peruvian Hairless, and watching TV with a notebook in hand. Talib Visram recently wrote in The Atlantic about his experience counting jokes per minute in popular TV shows. My approach was more deconstructionist, and directly applicable to my new gig. I had to figure out how such shows were built, and fast. - -Fortunately, the answer presented itself very quickly. - -First of all, word-processing programs often come with screenwriting templates. FinalDraft, the most popular software for those penning scripts, even has a Sitcom Template, which of course makes life much easier. But as for how to construct an episode, various bloggers, from the Wise Sloth to helpful folks at the BBC, noted a basic structure that I immediately recognized in every sitcom episode I tested. This structure is so formulaic that you’d think it would suck the fun out of writing and watching such shows, but it does nothing of the sort. While knowing the code it changes the way I watch TV, it only increases my admiration for the good writers who do so much within relatively strict confines. - -To demonstrate how this formula works, I’ve chosen an episode of a favorite show, somewhat at random, because it ideally exemplifies the template: episode 4 of season 1 of Parks & Recreation. - -The Sitcom Code breaks down what needs to happen in each episode, by the minute. As Dan Richter of Demand Media notes, “Sitcoms, minus commercials, are typically 22 minutes long [with] a script of 25-40 pages. Every sitcom episode has a main plot (story A), as well as one or two subplots (stories B and C).” There are three main acts, divided by two commercial breaks (in most American TV), with 3-5 scenes per act. One of the distinguishing characteristics of sitcoms, as opposed to other forms of television, is that the main protagonist(s) barely change from one episode to the next, let alone from season to season (Maggie Simpson has been sucking on a pacifier for nearly thirty years). Therefore whatever happens in the episode, the situation must end largely where it began. The Wise Sloth points out that 22 minutes is “not even really time enough to tell a full story. The whole story has to be on fast-forward,” so simplification is key. - -Poet Philip Larkin described all plots as “a beginning, a muddle, and an end,” which is as good a description as any. Each episode begins with the protagonist stating a goal or problem that must be solved, and which we understand will be solved by the end of the episode. If the problem is solved too quickly, then the episode won’t stretch out to 22 minutes, so the first attempt at reaching the goal or solving the problem must fail (“the muddle”), requiring a new approach, before the episode ends and the protagonist either does, or does not, achieve what they set out to do. The goal might be Homer trying to make a fortune by selling recycled grease in The Simpsons, or Job Bluth setting out to sabotage the family’s banana stand in Arrested Development, or the Seinfeld crew looking for where they parked in a vast lot. Another hallmark of sitcoms is that the protagonists frequently fail, and we often want them to, because we do not want our favorite characters to change too much. If Leslie Knope ever left Pawnee for a career as a DC politician, we would be distraught. If Kramer got married and moved to the suburbs—whoa, now! - -When writers sit around and prepare a new episode, many literally map out what will happen, minute-by-minute, in the main storyline and sub-storylines, filling in jokes later. Let’s see how this played out in the Parks & Recreation episode, “Boys' Club.” \ No newline at end of file +MISSES SECOND HALF OF ARTICLE diff --git a/tests/data/text/theatlantic.com2.txt b/tests/data/text/theatlantic.com2.txt index 788692b8..bc5f02c7 100644 --- a/tests/data/text/theatlantic.com2.txt +++ b/tests/data/text/theatlantic.com2.txt @@ -1,9 +1,13 @@ Despite the controversy, the James Franco-Seth Rogen flick is surprisingly relevant, very funny, and deserving of praise. +Richard Vogel/AP + Let's assume you knew absolutely nothing about North Korea and walked into The Interview, which after a very public and protracted to-do, is now showing at a few hundred American cinemas and streaming through a number of outlets online. Given what's been said about the movie, would you expect it to tell you about North Korea's concentration camps? Or the complexity of the dictatorship's propaganda system in which its leaders are touted as deities? Would you expect to hear (twice) that 16 million of the country's 24 million people are malnourished? +Caricature notwithstanding, The Interview has a surprisingly nuanced streak in its celluloid sea of swears and weiner jokes. + Okay, sure, The Interview might feature North Korean dictator Kim Jong Un wondering aloud about the relative "gayness" of drinking margaritas and affirming his love of Katy Perry. But caricature notwithstanding, the comedy has a surprisingly nuanced streak in its celluloid sea of swears and weiner jokes. The Interview centers around Dave Skylark (James Franco), a fratty, self-absorbed, and disconnected celebrity TV journalist, and his producer Aaron Rapoport (Seth Rogen), who are pushed by the CIA to turn a massive scoop (an interview with reclusive North Korean dictator Kim Jong Un) into an assassination mission. Rapoport, believe it or not, laments his station as the producer of tabloid garbage and sees the Kim interview as an opportunity to perform an admirable act of journalism. @@ -16,6 +20,8 @@ In a master stroke (spoilers ahead), the duo decides to flout the CIA's assassin At New York, David Edelstein says that critics of the film who reduce it to a silly and sophomoric bromance, "don't know what the hell they're talking about." He adds: +It means not just to expose Kim Jong-un as a fraud but to emasculate him, which is about the most punk thing you can do to a repressive, totalitarian, murderous, self-proclaimed god of a closed but increasingly porous state. + To that Jay McInerney adds, "Forget what you heard. #TheInterview gets my vote for best picture. Sadly I'm not a member of the Academy." The film doesn't skate away without some problematic parts. In the end, Kim Jong Un's head does explode when a tank shell fired by Rogen and company hits the dictator's helicopter. As Uri Friedman pointed out, the decision to show Kim's death, in slow motion and while Katy Perry's "Firework" twinkles in the background, was worrisome given that "that leader's government, which presides over nuclear weapons...has described the movie as an 'act of war.'" It probably doesn't matter that Kim's death happens during a getaway battle rather than a result of an assassination. diff --git a/tests/data/text/theatlanticcities.com1.txt b/tests/data/text/theatlanticcities.com1.txt index 70930c02..9a9dae6c 100644 --- a/tests/data/text/theatlanticcities.com1.txt +++ b/tests/data/text/theatlanticcities.com1.txt @@ -2,6 +2,8 @@ Antarctica remained largely untouched until roughly 200 years ago, and now, more Isolation has left local wildlife populations particularly vulnerable to diseases commonplace elsewhere in the world. “The effects of both a growing tourism industry and research presence will not be without consequences,” Wray Grimaldi of the University of Otago in Dunedin, New Zealand, said to New Scientist. “Penguins are highly susceptible to infectious diseases.” +One group of researchers found 20 different fecal pathogens on just 15 pairs of tourist boots. + Her team of Antarctic researchers found multiple infectious agents—bacteria such as salmonella and E. coli, viruses such as West Nile and the Avian pox virus—in captive penguins dating back to 1947. Outbreaks from those diseases have killed thousands of penguins over the years, the team reported in a paper published this month in the journal Polar Biology. Another theory is that migrating animals may have brought diseases to Antarctica, as the warming climate is attracting more species than ever before. But previous studies have identified tourist boots as vectors for disease transmission. One group of researchers tested 72 tourists' boots and found 20 different fecal pathogens on just 15 pairs of shoes. diff --git a/tests/data/text/thedailybeast.com1.txt b/tests/data/text/thedailybeast.com1.txt index e8cbf4e1..124d5d55 100644 --- a/tests/data/text/thedailybeast.com1.txt +++ b/tests/data/text/thedailybeast.com1.txt @@ -16,6 +16,8 @@ The clampdown backfired, spawning a social media frenzy with more than a million “There is a danger of backlash because of the esteem [in which] so many Catholics hold nuns,” Martin told The Daily Beast at the height of the scandal. “For many Catholics, sisters are the glue that holds the church together.” +The focus of her investigation was on how nuns live, work and pray. + There was much hope when Pope Francis was elected that he would show mercy to the American sisters. But in April, Gerhard Müller, the prefect for the Congregation for the Doctrine of the Faith, quashed that hope, insisting instead that Francis backed up the 2009 report. Muller criticized the nuns’ choice of speakers for their annual event accusing the sisters of being overtly provocative. “This is a decision that will be seen as a rather open provocation against the Holy See and the doctrinal assessment,” Mueller said in scathing remarks. “Not only that, but it further alienates the LCWR from the bishops, as well.” The Vatican is still working with the LCWR to find a middle ground and, despite what amounted to a lovefest at Tuesday’s press conference, there is still obvious concern. Sister Sharon Holland, the head of the LCWR, who incidentally was the only sister on the stage without a religious habit, told reporters that there were still hard feelings. “I’m concerned about those who may still be angry,” she said. “It’s a concern to me because it’s not healthy to remain angry.” diff --git a/tests/data/text/thedebrief.co.uk1.txt b/tests/data/text/thedebrief.co.uk1.txt index e1e85188..a310292a 100644 --- a/tests/data/text/thedebrief.co.uk1.txt +++ b/tests/data/text/thedebrief.co.uk1.txt @@ -1,8 +1,12 @@ -So in our eyes Ed Sheeran can do no wrong. He’s the most loveable, adorkable (that’s adorable crossed with dorky FYI) pop star EVER. And we love him, we really do. However when it comes to his matchmaking skills for our fave gal, and his BFF, Taylor Swift – well we think he’s a little bit off. +Lena deCasparis | Contributing Writer | 5 days ago +Ed Sheeran Wants To Set Up Taylor Swift, But We’re Not So Sure About His Matchmaking Skills The Debrief: Ed put the cupid bow and arrow down, and stick to playing the guitar +So in our eyes Ed Sheeran can do no wrong. He’s the most loveable, adorkable (that’s adorable crossed with dorky FYI) pop star EVER. And we love him, we really do. However when it comes to his matchmaking skills for our fave gal, and his BFF, Taylor Swift – well we think he’s a little bit off. -This week Ed told Now magazine that he hoped he could set up Taylor with none other that Orlando Bloom. He said ‘He’s lovely, and they live in the same building. [I'm hoping that] the magic might present itself eventually.’ + + +This week Ed told Now magazine that he hoped he could set up Taylor with none other that Orlando Bloom. He said ‘He’s lovely, and they live in the same building. [I'm hoping that] the magic might present itself eventually.’ @@ -20,4 +24,8 @@ Lena Dunham Has Declared Today Taylor Swift Day. Here’s Why We All Need To Get South Park Does A Lorde Skit. But Who’s It Taking The Piss Out Of? -Lorde Gave Taylor Swift The First Listen To Her New Song \ No newline at end of file +Lorde Gave Taylor Swift The First Listen To Her New Song + +Follow Lena on Twitter: @Lenadecasparis + +Pictures: Getty \ No newline at end of file diff --git a/tests/data/text/thedebrief.co.uk2.txt b/tests/data/text/thedebrief.co.uk2.txt index 329602b4..7600dafb 100644 --- a/tests/data/text/thedebrief.co.uk2.txt +++ b/tests/data/text/thedebrief.co.uk2.txt @@ -1,3 +1,7 @@ +Stevie Martin | Staff Writer | Tuesday, 25 November 2014 + +In Which We Discuss Whether Taylor's Or Beyonce's Lyrics Are Better Break-Up Material The Debrief: While Taylor is good for break-ups, there’s a Beyonce lyric for literally every occasion + Some absolute genius of a woman has written a break-up letter that comprises entirely of Taylor Swift lyrics because, let’s face it, pretty much every single Taylor Swift song is about a break-up. Apart from... nope. I’ve got nothing. Anyway, while T-Swift can provide excellent firepower for a particular kind of breakup letter – what if you want to go a bit more ballsy? Like, Beyonce ballsy? Wonder no more because we’ve channelled Queen Bey into the perfect method for getting rid of a cheatin’ man. Or woman. Or someone who hasn’t cheated at all, but you’re just not feeling it anymore. @@ -18,6 +22,8 @@ Sure, it makes you sound a bit all over the place (while Tay mainly does nostalg And Beyonce’s lyrics don’t just go well with relationships, oh no – while Taylor tends to focus on matters of the (broken) heart, Beyonce’s back catalogue works pretty well in most scenarios. Look, there’s a Beyonce for all occasions: +A Beyonce lyric for... + Quitting your job email: Fuck you, pay me. Drunk birthday speech: This goes out to all my girls that’s in the club rocking the latest! Who will buy it for themselves and get more money later. I think I need a barber. @@ -37,3 +43,5 @@ Sold Out Moisturiser Is The Hero Of Beyonce's 7/11 Video Beyonce Behaves A Bit Weirdly, The Internet Gets Even Weirder A Few Signs We Might Be Drifting Apart From Beyonce + +Follow Stevie on Twitter: @5tevieM \ No newline at end of file diff --git a/tests/data/text/theglobeandmail.com1.txt b/tests/data/text/theglobeandmail.com1.txt index 9023d109..730d5fdf 100644 --- a/tests/data/text/theglobeandmail.com1.txt +++ b/tests/data/text/theglobeandmail.com1.txt @@ -2,14 +2,22 @@ A modern airliner’s abrupt disappearance from radar as it flew through a line On Monday, as hopes of finding survivors of Indonesia AirAsia Flight 8501 faded, searchers in ships and aircraft scoured the Java Sea looking for debris to pinpoint where the Singapore-bound Airbus A320 crashed. +Airline incidents and fatalities since: 1975 + With 33 million commercial airline departures and 3.2 billion passengers carried in 2014, the rate of crashes has never been lower, though the death rate of passengers is up slightly from its all time low of 0.09 last year. +Commercial flights and passengers carried: 1975 - 2014 + +Commercial airline accidents and fatalities: 1975 - 2014 (rate per million) + SOURCE: Aviation Safety Network; ICAO; WORLD BANK. * Accidents refers to the number of fatal airliner (14+ passengers) hull-loss accidents. Does not include corporate jet and military transport accidents/hijackings. ** Includes 162 unconfirmed passengers from AirAsia Flight QZ8501 +(AirAsia Flight 8501: What we know so far about the plane’s disappearance) + Nearly two days after the flight vanished, chances of finding anyone alive, even if some of the 162 passengers and crew survived a crash into the sea, were exceedingly remote, but officials were still calling the multinational effort a search-and-rescue operation. The fate of Flight 8501 remains perplexing and until the cockpit voice and flight-data recorders are recovered, investigators won’t be able to determine what exactly went wrong during the plane’s final few minutes. diff --git a/tests/data/text/theglobeandmail.com2.txt b/tests/data/text/theglobeandmail.com2.txt index 7b64d5d8..bd1994b7 100644 --- a/tests/data/text/theglobeandmail.com2.txt +++ b/tests/data/text/theglobeandmail.com2.txt @@ -1,5 +1,9 @@ Oil. Russia. Interest rates. The United States. Europe. Commodities. China. Trade. Japan. The global economy enters 2015 with a lot on its mind, and a long list of competing storylines set to develop as the year progresses. Here, four notable economists share their views on the key issue that will grab their attention in 2015 – and how it might unfold, for better or worse. +Eric Lascelles + +Chief economist, RBC Global Asset Management – Toronto + Collapsing oil prices are poised to reverberate across 2015. If current pricing sticks, global producers will lose over a trillion dollars in annual profits, crimping their investment intentions and impairing their ability to pay dividends and debts. Oil-exporting nations – Canada among them – naturally suffer, with the greatest agony reserved for emerging-market members who must perversely raise interest rates to halt capital flight and combat ballooning U.S.-denominated debt loads. @@ -10,6 +14,10 @@ However, let us not lose sight of the bigger picture. Oil’s extreme dislocatio For that matter, the global economy actually likes low oil prices. For all of the bellyaching, every dollar lost by oil producers is more than recovered across a diffuse set of oil consumers. Indeed, the combination of low oil prices, low bond yields and weakening exchange rates should help revive the moribund European and Japanese economies. This could prove to be the most lasting consequence of oil’s not-so-excellent adventure. +Julian Jessop + +Chief global economist, Capital Economics – London + The year 2014 closed with a focus on the oil price, but developments in other commodity markets will also be worth watching in 2015. The knock-on effects could work in either direction. We expect the price of oil to remain low. As energy is an important cost in the production and transportation of other commodities, this is one reason to expect price pressures more generally to be weak. @@ -18,6 +26,10 @@ Nonetheless, most countries – and the world as a whole – should be better of What’s more, China and India are the two largest markets for gold. While inflation will be lower as a result of the fall in oil prices, global monetary policy is also likely to be looser for longer. We, therefore, expect base and precious metals prices to do well in 2015, improving the prospects for miners and the more diversified commodity economies. +Christopher Ragan + +Professor of economics, McGill University – Montreal + What should we expect from the world of central banking in 2015? I see two major forces. First, given that real gross domestic product in the euro zone is still 2 per cent below the 2008 level, there’s a pretty clear economic case for more fiscal and monetary stimulus. On the other hand, the political forces in opposition to such actions continue to be massive. My best guess for 2015 is that the euro zone will continue to muddle through, without much new stimulus of any kind. @@ -26,6 +38,10 @@ Meanwhile, the United States is showing ever-more promising signs of an economic These different economic scenarios suggest that the U.S. dollar will be strengthening against the euro. How the Canadian dollar fares in such a situation is unclear, although the single best predictor for the Canadian dollar is still the path of world commodity prices – which is pretty much anybody’s guess. +Carl Weinberg + +Chief economist, High Frequency Economics – Valhalla, N.Y. + My biggest worry for 2015 is of a default by Russia on its foreign debt. Vladimir Putin could intentionally renege on his foreign debt obligations – an Argentine approach – or Russia could just run out of liquid foreign-exchange assets. Russia owes $670-billion (U.S.) to foreigners. No one is sure who owns it all. Most of it likely is not in the hands of banks. We should remember that LTCM [Long-Term Capital Management] in the U.S. was not a bank, but its failure did a lot of damage to the financial system. I am also waiting to see if an anti-austerity government comes to power in Greece and defaults on its debts to EFSF, the IMF and Euroland governments. EFSF has sold €180-billion ($254-billion Canadian) in bonds, backed largely by loans to Greece. In the worst case, banks and hedge funds owning these AAA bonds will find them downgraded. Some of them may fail. Euroland’s governments may have to ante up real money to recapitalize EFSF, triggering a fiscal crisis. diff --git a/tests/data/text/thekitchn.com1.txt b/tests/data/text/thekitchn.com1.txt index c88c0fcb..c13f0cd5 100644 --- a/tests/data/text/thekitchn.com1.txt +++ b/tests/data/text/thekitchn.com1.txt @@ -1 +1 @@ -Kristin is a contributing editor for The Kitchn. A former editor at Real Simple, she is compulsively organized and loves solving people's problems. She has a weakness for desserts, especially ice cream. \ No newline at end of file +ONLY EXTRACTED AUTHOR BLURB \ No newline at end of file diff --git a/tests/data/text/thekitchn.com2.txt b/tests/data/text/thekitchn.com2.txt index c88c0fcb..c13f0cd5 100644 --- a/tests/data/text/thekitchn.com2.txt +++ b/tests/data/text/thekitchn.com2.txt @@ -1 +1 @@ -Kristin is a contributing editor for The Kitchn. A former editor at Real Simple, she is compulsively organized and loves solving people's problems. She has a weakness for desserts, especially ice cream. \ No newline at end of file +ONLY EXTRACTED AUTHOR BLURB \ No newline at end of file diff --git a/tests/data/text/thenextweb.com1.txt b/tests/data/text/thenextweb.com1.txt index 236c2954..20ef507a 100644 --- a/tests/data/text/thenextweb.com1.txt +++ b/tests/data/text/thenextweb.com1.txt @@ -1,3 +1 @@ -Crunchyroll is a leading destination and platform for Japanese anime, Korean drama, live-action titles, and tons of other Asian media. - -All videos are professionally translated into multiple languages, and are available for viewing within minutes of their original TV broadcasts through tons of different applications. \ No newline at end of file +FAILED, MISSING ARTICLE AFTR FIRST PARAGRAPH diff --git a/tests/data/text/thenextweb.com2.txt b/tests/data/text/thenextweb.com2.txt deleted file mode 100644 index e06ab695..00000000 --- a/tests/data/text/thenextweb.com2.txt +++ /dev/null @@ -1,3 +0,0 @@ -TNW Weekly The best of the week, handpicked every Friday - -Daily Top stories The most shared stories of the day \ No newline at end of file diff --git a/tests/data/text/theonion.com1.txt b/tests/data/text/theonion.com1.txt index e69de29b..ed05d3cf 100644 --- a/tests/data/text/theonion.com1.txt +++ b/tests/data/text/theonion.com1.txt @@ -0,0 +1 @@ +CHICAGO—Admitting it took them some time to come around to the idea, the parents of local woman Laura Stevens said Wednesday that they had finally accepted their daughter’s mixed-attractiveness relationship with Kyle Baker, a man who is considerably worse-looking than she is. “To be honest, we were quite surprised when Laura brought Kyle to the house for the first time, but eventually we came around to it,” said Stevens’ mother, Janet, who noted that the pair were still met with uncomfortable stares and disapproval from other family members—especially Laura’s grandmother—at last year’s Thanksgiving dinner. “Her father was particularly upset at first, but now I think he’s learned to accept it, and he’s even grown to like Kyle. Besides, Kyle seems to make Laura happy, and that’s all that really matters.” Janet Stevens went on to say that, if the two ever got married, she would love their children no matter how average-looking they are. \ No newline at end of file diff --git a/tests/data/text/theonion.com2.txt b/tests/data/text/theonion.com2.txt deleted file mode 100644 index f9607fc1..00000000 --- a/tests/data/text/theonion.com2.txt +++ /dev/null @@ -1 +0,0 @@ -Explaining that her statements indicated a failure to understand and implement the district’s goal of providing a comprehensive education to all children, Southwest High School officials reportedly fired ninth-grade history teacher Jennifer Steenman today after she was heard saying she learns more from her students than they do from her. Full article. \ No newline at end of file diff --git a/tests/data/text/theroot.com1.txt b/tests/data/text/theroot.com1.txt index 22215427..5c7bc413 100644 --- a/tests/data/text/theroot.com1.txt +++ b/tests/data/text/theroot.com1.txt @@ -1,19 +1 @@ -Within social-justice movements, tensions often surface between older and youngster activists—that’s hardly new. But the Rev. Al Sharpton says that recent criticism that he isn’t doing enough to groom younger, up-and-coming activists within his civil rights tradition is simply not true. - -“This is more about ideology than it is about generational differences,” Sharpton told The Root. He pointed to dozens of his protégés—all in their 20s and 30s—who have committed themselves to the principles of an interracial and nonviolent movement. - -“I have spent an inordinate amount of time trying to make sure that we can continue this movement and National Action Network for the next 30 to 40 years when I am gone,” he said. “Leadership cannot be willed. I can’t pass the torch. I can only keep the flame lit.” - -For his part, Sharpton is not sure who will replace him after he exits the national stage, but he said that it’s not up to him to handpick his successor. That person “has to put in the work and earn it,” he said. - -“I’m sure I was not Jesse’s [Jackson] choice. There were other guys who were probably more palpable to him,” said Sharpton, who as a teenager served as the youth director of Brooklyn’s Operation Breadbasket—the economic arm of the Southern Christian Leadership Conference that Jackson directed at the request of Martin Luther King Jr. - -But by the mid-2000s, Sharpton had gone on to surpass his mentor in national prominence, much to the chagrin of those who had hoped that Sharpton would fade away from the public spotlight. “I was too young to know for sure, but it’s possible that Jesse wasn’t Dr. King’s choice, either,” he said. - -Sharpton said that contrary to some reports, a diverse group of young people were invited to speak at the Dec. 13 Justice for All rally in Washington, D.C. The event drew more than 10,000 people to the nation’s capital to train a spotlight on the killing of unarmed black men in the wake of several high-profile shootings, as well as to urge Congress to take up legislation that would require closer monitoring of police departments across the country. - -“This was not a revolutionary march, and I don’t apologize for that,” said Sharpton, who added that he had not yet arrived at the rally when a group of protesters, whom he did not know, stormed the stage and demanded to speak. He said that when he arrived on the scene later, he granted the activists speaking time after they assured him that they were not going to call for violence or promote inflammatory rhetoric against nonblacks. - -Sharpton recalled that at one point during the protests after the Trayvon Martin shooting, a demonstrator publicly called for a $10,000 bounty to be placed on the life of George Zimmerman, the neighborhood watchman who killed Trayvon. The slain teen’s parents, however, as well as the parents of the other shooting victims who have worked with Sharpton, have called for peaceful demonstrations. - -Sharpton said he was unsure why activists who disagreed with his political tactics and strategy would want to attend the gathering in the first place. “I don’t go to marches spearheaded by people who I disagree with,” said Sharpton. “I just do what I’m doing.” \ No newline at end of file +MISSED LAST PARAGRAPH diff --git a/tests/data/text/theroot.com2.txt b/tests/data/text/theroot.com2.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/data/text/tnr.com1.txt b/tests/data/text/tnr.com1.txt index f8a2b7a6..46237fd5 100644 --- a/tests/data/text/tnr.com1.txt +++ b/tests/data/text/tnr.com1.txt @@ -1,3 +1,5 @@ +Friend, + I was touched that you asked for my advice about going into politics. Anyone whose career in politics was nasty, brutish, and short—as mine was—is grateful that anyone thinks their opinion is worth hearing. All I’d claim is that my thoughts come with what Scott Fitzgerald called “the authority of failure.” First of all, you need to know why you want it. You’d be amazed at how many people who go into politics can’t give you an honest answer to why they want it so badly. @@ -24,10 +26,14 @@ So my advice is: Never let your opponents own your story. If you can’t do this It doesn’t pay, either, to pretend to be better than the business you’re in. You can’t succeed in politics if you give too much appearance of despising the low arts by which we govern ourselves. Fastidious distaste for the roughness and meanness of political life may work in a seminar room, but it’s fatal on the campaign trail. +It is really something in life to be utterly disabused about human motive, and yet still come to work every day. + This distaste is common among people who’ve enjoyed success outside of politics, in academia or journalism or business, and who go into politics with the reasonable assumption that the prestige they achieved in their former profession should automatically transfer into politics. It doesn’t. People who think they’re entitled to standing—because they are brainy, rich, or famous—almost always lose. They forget you earn your standing, you are not entitled to it. That’s the best thing about democracy, the single reason why we’re not yet entirely governed by wealthy oligarchs. I may have come into politics with an unacknowledged condescension toward the game and the people who played it, but I left with more respect for politicians than when I went in. The worst of them—the careerists and predators—you find in all professions. The best of them were a credit to democracy. They knew the difference between an adversary and an enemy, knew when to take half a loaf and when to insist on the whole bakery, knew when to trust their own judgment and when to listen to the people. As I learned while watching wiser colleagues than I in a democratic legislature, it is really something in life to be utterly disabused about human motive, venality, capacity for double-crossing, and yet still come to work every day, trying to get something done. -Liberalism will become an enclave conviction of a shrinking minority unless those who call themselves liberal reconnect their faith in tolerance, equality, opportunity for all with the more difficult faith in the dirty, loud-mouthed, false, lying business of politics itself. This disdain is cynicism, masking as high principle. The ultimate allegiance of a democratic politician is not to party, not even to principle, but to the venal process called politics. So my final advice is this: Politics is not a vulgar means to a goal, it’s a noble life unto itself, and unless you love it, you can’t do it well. I didn’t get there, but I hope you will. \ No newline at end of file +Liberalism will become an enclave conviction of a shrinking minority unless those who call themselves liberal reconnect their faith in tolerance, equality, opportunity for all with the more difficult faith in the dirty, loud-mouthed, false, lying business of politics itself. This disdain is cynicism, masking as high principle. The ultimate allegiance of a democratic politician is not to party, not even to principle, but to the venal process called politics. So my final advice is this: Politics is not a vulgar means to a goal, it’s a noble life unto itself, and unless you love it, you can’t do it well. I didn’t get there, but I hope you will. + +—Warmly, Michael \ No newline at end of file diff --git a/tests/data/text/uproxx.com1.txt b/tests/data/text/uproxx.com1.txt index 900f44f7..fa4fb664 100644 --- a/tests/data/text/uproxx.com1.txt +++ b/tests/data/text/uproxx.com1.txt @@ -1,3 +1,7 @@ +Share This Video + + + It may seem old fashioned, but I sometimes view the internet and the real world as two separate places. Because of that, it seems weird to me when I see something as typically impact-free as a Reddit post have a very real effect on a small town, but that’s exactly what happened to the town of Mammoth, Arizona after a fictional post got taken literally. On the social media site Reddit, a user who goes by the name Lindsey posted a story about an Ebola-like illness taking over the small town. “Eleven days ago, on the 3rd, an older woman who runs a home daycare in my town was found dead in her bathroom by a parent who was picking up his daughter,” Lindsey wrote. “The kids were all really agitated and told police that Mrs. Booker (the deceased) had been yelling at them and bleeding from her eye before she went into the bathroom and collapsed in her bathtub.” Lindsey goes on to say children began dying, then it spread to others around Mammoth. She said everyone who passed away experienced similar symptoms. Adding to the panic, Lindsey wrote: “To the person calling businesses here: that is not us answering. Our downtown has been shut down all day. I called a couple places where I know the business owners and employees and the people who answered are not locals. I don’t know what they told you but they’re not from here.” She said it was a cover-up to keep the sickness contained. But, Lindsey was about to fall victim: “I noticed a small bruise on my arm that upon further f***ing inspection spreads all the way down one side of my back. I’m so f***ing scared. I had a breakdown where I just screamed at the wall and cried and there was blood in my tears and that was like an hour ago.” She said she needed help, and with the exposure of 1.4 million Reddit users, she got the attention she needed. @@ -10,4 +14,11 @@ Curious about the inadvertent internet firestarter behind this? Here’s an exce So, what have we learned? F*ck if I know. These are either terrifying times or a time when the things that terrify us are constantly shown in a loop on a screen that is constantly in front of our faces. That means that people’s freak-out button is a bit more accessible than usual. It’s not good to hit that button, but it’s ultimately not the writer’s fault that people didn’t check to see what was at the heart of this minor panic. -Really, the only defense against something like this is a more diligent inspection of things before we react to them (which would seem to defy our reflexive nature), or a heartier application of skepticism. The former is more time consuming than the latter but we also shouldn’t get to a point where we roll our eyes whenever someone says that they saw a wolf. Cause wolves, man. They out there. \ No newline at end of file +Really, the only defense against something like this is a more diligent inspection of things before we react to them (which would seem to defy our reflexive nature), or a heartier application of skepticism. The former is more time consuming than the latter but we also shouldn’t get to a point where we roll our eyes whenever someone says that they saw a wolf. Cause wolves, man. They out there. + +Share This + + + +Source: KGUN 9 + diff --git a/tests/data/text/uproxx.com2.txt b/tests/data/text/uproxx.com2.txt index e69de29b..e2727ad3 100644 --- a/tests/data/text/uproxx.com2.txt +++ b/tests/data/text/uproxx.com2.txt @@ -0,0 +1,9 @@ +The people at the UK’s Almost Legal Productions want to turn Shaun of the Dead into the next Rocky Horror Picture Show, and thanks to Simon Pegg their dream may be on the fast track to reality. Last year, director Chas Burns* organized a stage show of the 2004 zombie comedy and the culmination was four sold-out shows in October and November, and the show certainly received a little marketing help from Pegg, who tweeted about the event. Burns said that after that glorious 140-character endorsement, people suddenly wanted his new show being performed all over the globe. + +The great news is that those four shows have turned into 15 touring dates for 2015, as Burns told Chortle a simple phone call to Pegg’s assistant led to not only the actor giving his blessing for a Shaun of the Dead stage show, but Working Title and Universal Pictures have also signed off. If we all cross our fingers and toes, that could lead to Shaun and Ed eventually coming to performing arts centers and hipster theaters near you. But don’t worry, this isn’t your typical boring musical. + +‘It’s not a musical. Between you and I, I can’t abide musicals. It’s a production with tongue-in-cheek songs, with some extra stuff thrown in you won’t see in the film and deleted scenes that were left on the cutting room floor. I have always stated that if I ever see anyone yawning at one of our productions I will call it a day.’ ‘It’s incredible the interest we’ve had, people have been waiting for it to happen and we’ve been lucky enough to find ourselves with the rights. ‘Yes we are a small company nobody has ever heard of, and some believe us to be amateur. We are often asked about our professional/amateur position and we merely answer with the same phrase every time: if you measure our professional character by the colour of money then it’s a sad day for theatre.’ (Via Chortle) + +Now the bad news: Those 15 dates are only in Southwest England. So you can either start planning your trips across the pond now, or you can go sit on your front porch steps and pout. That is, unless you live in Gloucester or Worcester, in which case you can go and then brag about it. But then you’d be a jerk, so don’t do that (the bragging part, that is). + +*No relation, but if I had a British cousin, his name would totally be Chas. \ No newline at end of file diff --git a/tests/data/text/upworthy.com1.txt b/tests/data/text/upworthy.com1.txt index 202d1722..05551ff5 100644 --- a/tests/data/text/upworthy.com1.txt +++ b/tests/data/text/upworthy.com1.txt @@ -14,22 +14,34 @@ Benjamin: Worth every minute. It's an honor to be by your side. Joe: One of the biggest things that changed after I got Benjamin was an improvement in my ability to walk. I can't really describe how having a dog on a leash can help someone feel safe and stable to walk but, for me, that was a big deal because, if anyone's ever walked on railroad tracks and if you can even get your hand on a leaf up above you or something, you just feel like okay, I got something and I guess that's what it was. He can brace and let me hold onto him. If I go to fall, I could grab him, and I just felt safer and more comfortable with him there, just knowing that I had help if I needed it. +Benjamin: Always happy to lend a paw, Joe. + +Joe: Benjamin, bring. + +Benjamin: I got this. + Joe: Good boy. I think, for a lot of guys, dogs like Benjamin could literally be the thing that makes them want to wake up in the morning and have a schedule and that cold nose on their elbow could be the reason why they get up and do the things that they need to do. Benjamin, where's my shoe? Benjamin: Right where you left it, Joe. Joe: A typical day with me and Benji usually starts with me trying to find my shoe and Benji can help me with that. Good job, buddy. Good job. Thank you. He always looks like he's thinking. His eyebrows are always going up and down and he always just seems like he's waiting on me to do something dumb so he can give me a look like what are you doing? When I go to school, he goes with me. I plan on going back in the fall and he's going to look forward to that because he likes going to school because all the girls flirt with him. +Benjamin: Hello, ladies. + Joe: Benjamin is a part of my family. I can't imagine my family dynamic without him in it. So, he means an awful lot to me. You wanna play tug a war? If I could ask Benjamin three questions, I would probably ask him who his favorite kid is, which would probably be Izzy, because she just dotes so much attention on him all the time. Benjamin: My favorite? Let me see. Yep, yep, it's Izzy. Joe: I'd probably ask him what his favorite thing to do is, because he's eight years old and I want him to have fun and enjoy things. +Benjamin: Naps, pets, more naps. I'm a simple dog, with simple tastes. + Joe: I would probably ask him if I've made him happy, if I've done right by him, because he's my friend and he's improved my life and I hope that he feels safe and comfortable around me and that I've improved his life as well. Benjamin: Well, now, you're making this old dog get all misty eyed. Joe: I wish that I could express to him how much he means to me, because we always read those things that say, they're only there for a part of yours, but you're there for all of theirs and I'm his whole life and I hope that he is proud of me and that I've made him feel like he's done something incredible in my life, because he has. -Benjamin: Am I proud of Joe? Joe told me once that a hero is someone who does something selfless and who functions even when they are afraid. He may not call himself a hero, but in dog years, I'm older and therefore wiser, and I say he is and I'm proud to be his dog. \ No newline at end of file +Benjamin: Am I proud of Joe? Joe told me once that a hero is someone who does something selfless and who functions even when they are afraid. He may not call himself a hero, but in dog years, I'm older and therefore wiser, and I say he is and I'm proud to be his dog. + +There may be small errors in this transcript. \ No newline at end of file diff --git a/tests/data/text/upworthy.com2.txt b/tests/data/text/upworthy.com2.txt index fa7a3d36..05eb88dd 100644 --- a/tests/data/text/upworthy.com2.txt +++ b/tests/data/text/upworthy.com2.txt @@ -10,8 +10,44 @@ Each year millions of pounds of different fumigants are pumped into California s Without fumigants, farmer say, "Our crops can fail. There's no clear alternative to these chemicals. We need them," they say," to keep consumers happy." It's how you can buy cheap strawberries anytime of the year. And so these toxic pesticides called fumigants continue to be the foundation of our strawberry industry. +DIRECTOR & PRODUCER Ariane Wu + +NARRATOR Roman Mars + +BASED ON REPORTING BY Kendall Taggart, Bernice Yeung & Andrew Donohue + +ILLUSTRATOR & AFTER EFFECTS ARTIST Arthur Jones + +STOP-MOTION ANIMATOR & VIDEO EDITOR Ariane Wu + +MUSIC COMPOSER Jason Kick + +SOUND MIXER Christopher Galipo + +CONSULTING PRODUCER Michael Schiller + +ASSOCIATE PRODUCER Rachel de Leon + +EXECUTIVE PRODUCER Amanda Pike + +EDITORIAL DIRECTOR Robert Salladay + +EXECUTIVE DIRECTOR Robert J. Rosenthal + ARCHIVAL IMAGES AND FOOTAGE COURTESTY OF U.S. Army The U.S. National Archives and Records Administration Sam Hodgson Library of Congress The New York Times - © Woman's Day magazine 1975 +© Woman's Day magazine 1975 + +© Family Circle magazine 1978 California Strawberry Commission + +ADDITIONAL MUSIC + +"Turntable," written by Jack Trombey Courtesy of De Wolfe Music + +SPECIAL THANKS TO Virtual Active Serene Fang + +the center for investigative reporting + +cironline.org - © Family Circle magazine 1978 California Strawberry Commission \ No newline at end of file +There may be small errors in this transcript. \ No newline at end of file diff --git a/tests/data/text/usnews.com1.txt b/tests/data/text/usnews.com1.txt index d91ec7af..faa0f4f2 100644 --- a/tests/data/text/usnews.com1.txt +++ b/tests/data/text/usnews.com1.txt @@ -18,4 +18,5 @@ Finally, in South Carolina, it's GOP Sen. Rand Paul, R-Ky., who has dropped in m There are more charts and maps on Github. All charts include embed codes so that anyone can easily use our charts on their own websites. Data for these charts are current through Monday, Nov. 3. -The Presidential Tracker was created for U.S. News by Lindsey Cook, Andrew Soergel and Dave Catanese using information sourced from www.p2016.org. \ No newline at end of file +The Presidential Tracker was created for U.S. News by Lindsey Cook, Andrew Soergel and Dave Catanese using information sourced from www.p2016.org. + diff --git a/tests/data/text/usnews.com2.txt b/tests/data/text/usnews.com2.txt index 0f42c4ed..26ce3da6 100644 --- a/tests/data/text/usnews.com2.txt +++ b/tests/data/text/usnews.com2.txt @@ -1 +1,7 @@ -It's a formerly 'forbidden fruit' sure to be enjoyed by cigar aficionados on both sides of the Florida Straits. On the list of President Barack Obama's new Cuba policy are legalized Cuban cigars, \ No newline at end of file +Terms of Use Privacy Policy + +Cigar rollers in Havana thrilled with warming U.S.-Cuba policy + +It's a formerly 'forbidden fruit' sure to be enjoyed by cigar aficionados on both sides of the Florida Straits. On the list of President Barack Obama's new Cuba policy are legalized Cuban cigars, + +Source: Reuters \ No newline at end of file diff --git a/tests/data/text/vanityfair.com1.txt b/tests/data/text/vanityfair.com1.txt index e69de29b..9b5c9e32 100644 --- a/tests/data/text/vanityfair.com1.txt +++ b/tests/data/text/vanityfair.com1.txt @@ -0,0 +1,23 @@ +By Kevin Mazur/WireImage. + +Say what you will about Gwyneth Paltrow, the Oscar winner proved to be a pretty great hang when she stopped by Howard Stern’s Sirius XM radio show on Wednesday. Unlike other actors, Paltrow was game to discuss anything that the host asked about, whether it was her high-profile “uncoupling” from Chris Martin, the sexual advice she gave Chelsea Handler, or her past relationships with A-list actors. + +On the last subject, Paltrow opened up about Brad Pitt, whom she started dating after co-starring with him in David Fincher’s 1995 thriller Seven. “I definitely fell in love with him,” she said of her former fiancé. “He was so gorgeous and sweet. I mean, he was Brad Pitt.” + +Paltrow was not the only person to fall in love with him at the time—she revealed that her parents were equally smitten with the star. “My father was so devastated when we broke up,” she said. “My father loved him like a son.” Although she refused to confirm that she dumped Pitt, she did talk about why the relationship didn’t end up working out. + +“I was such a kid,” she shared. “I was 22 when we met. It's taken me until 40 to get my head out of my ass. You can't make that decision when you're 22 years old . . . I wasn't ready, and he was too good for me . . . I didn't know what I was doing.” Although she doesn’t still “hang out with” Pitt, Paltrow said that they are “friendly” if they run into each other. + +The Goop founder offered slightly less glowing remarks about Ben Affleck, whom she dated and co-starred with in the 2000 drama Bounce. + +“He was not in a good place in his life to have a girlfriend,” the actress said of their time together, before suggesting that both her friends and her parents were supportive of her decision to break up with him. When Stern asked about a comment Paltrow previously made about Affleck—specifically, “He’d be better off dating a Scores stripper”—Paltrow laughed, “Well that was true at the time.” + +Paltrow said that she approves of Affleck’s wife, Jennifer Garner, though, and has spent some time with her. “Not only is she beautiful [but] she is so warm and she cooks and is so into her kids. And she’s hilarious . . . I had a girl’s dinner before the summer and she came over. She’s my neighbor. I really like her a lot.” + +Paltrow also opened up about her friends Jay Z and Beyoncé, saying, “They’re the greatest people. They are funny and sweet and they’re self-aware and they’re really intelligent.” When Stern asked whether Paltrow called them after the Solange elevator incident to check in, Paltrow revealed, “If any of my friends ever have anything like that, I call right away. If someone’s clearly going through something, I always go straight to it.” + +And on the subject of Johnny Depp, with whom she co-stars in the upcoming Mordecai, she said, “I just felt like I am not cool enough [for him]. He’s got all of his tattoos and his cool clothes . . You go into the room to rehearse and he is rolling his own cigarettes . . . He listens to music while he acts. He has a little [earbud] in his ear and he listens to music. It’s so cool. He’s creating the whole world in his head.” + +To listen to the complete interview, click here. + +Related: The Paltrow Affair \ No newline at end of file diff --git a/tests/data/text/vanityfair.com2.txt b/tests/data/text/vanityfair.com2.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/data/text/vogue.com1.txt b/tests/data/text/vogue.com1.txt deleted file mode 100644 index 7ac6f9a0..00000000 --- a/tests/data/text/vogue.com1.txt +++ /dev/null @@ -1,7 +0,0 @@ -© 2014 Condé Nast. All rights reserved. - - Use of this site constitutes acceptance of our User Agreement (effective 1/2/2014) and Privacy Policy (Effective 1/2/2014). - - Your California Privacy Rights - - The material on this site may not be reproduced, distributed, transmitted, cached, or otherwise used, except with the prior written permission of Condé Nast. \ No newline at end of file diff --git a/tests/data/text/vogue.com2.txt b/tests/data/text/vogue.com2.txt deleted file mode 100644 index 7ac6f9a0..00000000 --- a/tests/data/text/vogue.com2.txt +++ /dev/null @@ -1,7 +0,0 @@ -© 2014 Condé Nast. All rights reserved. - - Use of this site constitutes acceptance of our User Agreement (effective 1/2/2014) and Privacy Policy (Effective 1/2/2014). - - Your California Privacy Rights - - The material on this site may not be reproduced, distributed, transmitted, cached, or otherwise used, except with the prior written permission of Condé Nast. \ No newline at end of file diff --git a/tests/data/text/vogue.de1.txt b/tests/data/text/vogue.de1.txt index 9ead80d2..252c69e5 100644 --- a/tests/data/text/vogue.de1.txt +++ b/tests/data/text/vogue.de1.txt @@ -1,3 +1,7 @@ +Lieblingsstück: die iPhone-Hülle von Iphoria + +© PR + Im Design von Chanel-Nagellackfläschchen sind die iPhone-Cases von Iphoria der Hingucker bei jedem iPhone. Sie werden Ton in Ton mit der passenden Nagellackfarbe oder im Kontrastlook (oben) getragen. Die Hüllen gibt es in vier Farben: "Rouge Pur" (oben), "Sky Blue", "Sea Mint" und "Candy Pink". Unten sind die iPhone-Cases in Rot, Himmelblau und Mintgrün neben den Chanel-Nagellacken in den Farbtönen "Le Vernis Taboo", "Le Vernis Pirate" und "Le Vernis Lilis" abgebildet. Die Chanel-Nagellacke kosten je 24 Euro. Die Iphoria-iPhone-Cases gibt es über www.iphoria.com, ab 45 Euro. \ No newline at end of file diff --git a/tests/data/text/vogue.de2.txt b/tests/data/text/vogue.de2.txt index 3e3b2eb7..802fbb52 100644 --- a/tests/data/text/vogue.de2.txt +++ b/tests/data/text/vogue.de2.txt @@ -1 +1,3 @@ +Weihnachten + Sie können ironisch-unschön, lustig und überraschend, aber auch klassisch-traditionell sein, doch das Wichtigste ist, dass sie gemütlich sind, Spaß machen und uns an die schönsten Tage des Jahres erinnern: Christmas Sweater! Die Tradition des Weihnachtspullovers ist eine recht junge, erst in den 80er-Jahren vor allem im angelsächsischen Sprachraum entstandene – und vielleicht ist sie gerade deswegen besonders individuell. Weil auch wir bei VOGUE finden, dass man gewisse Traditionen pflegen sollte, zeigen wir unsere liebsten Weihnachtspullover, in denen wir es uns an den Feiertagen gemütlich machen \ No newline at end of file diff --git a/tests/data/text/wetpaint.com1.txt b/tests/data/text/wetpaint.com1.txt index d95c0ae9..5b92bab5 100644 --- a/tests/data/text/wetpaint.com1.txt +++ b/tests/data/text/wetpaint.com1.txt @@ -1,21 +1 @@ -So how did Chris and Whitney sitting in a tree K-I-S-S-I-N-G turn into us imagining their babies? We first got whiff of their whirlwind romance when she got a 1-on-1 date in Des Moines, the final destination before Hometowns. Their romantic evening included a stroll along the Des Moines River Walk followed by dinner at a downtown hotspot. After picking the lobster from their teeth, the duo kissed awkwardly in front of a mural some production assistants who are rethinking their career choices painted of the couple. - - - -Apparently the alley makeout sesh wasn’t as awkward as it seemed, because Chris gifted Whit a rose, for which she gifted him a trip to her favorite Chicago Italian restaurant, Quartino, during their Hometown Date. All must have gone swimmingly, because she then (presumably) has all her fantasies come true in the Fantasy Suite in Bali. - - - -Prince Farming narrows it down to two, taking Whit and one other lucky lady back to… Iowa! Apparently he leads the two final gals to a barn where things go the way of Children of the Corn for the other gal but get all Field of Dreams happy for Whitney. - - - -Chris proposes, Whitney says yee haw, and the two ride off on a tractor into the sunset together. Way to get it, girl! - - - -Are you surprised Whitney is the last woman standing? Tell us below. - - - -Source: Reality Steve \ No newline at end of file +MISSES FIRST TWO PARAGRAPHS BEFORE AN ADVERTISEMENT IN THE MIDDLE OF THE PAGE diff --git a/tests/data/text/wetpaint.com2.txt b/tests/data/text/wetpaint.com2.txt index abf6e4b0..44dfb2c9 100644 --- a/tests/data/text/wetpaint.com2.txt +++ b/tests/data/text/wetpaint.com2.txt @@ -1 +1 @@ -The raven-haired gal bares a striking resemblance to a former Bachelorette who may have broken Chris’s heart last season, with her shiny locks and sparkling eyes. So, what else makes her perfect for Prince Farming? Read on, gentle reader. The 24-year-old from Hamilton, New Jersey was likely the quickest packer of the 30 gals cast on The Bachelor 2015, as she works as a flight attendant. Unfortunately for her, producers sent the lucky ladies on a bizarre tour of ‘Merica rather than letting them dip their toes into foreign oceans. When Alissa isn’t flying the friendly skies, she keeps her head on her shoulders with “family, friends, laughter, hope, and faith,” according to her official ABC bio. Should you want to hop aboard her social media flight, you’re a bit outta luck. Miss Alissa’s Facebook is currently deactivated, and you’ll need a boarding pass to her Instagram, as it’s set to private. Her LinkedIn is also seemingly out of date, as it says she works for Gallo Winery. Needless to say, that job likely gave her the skills needed to hang tough for the hours of drinking before rose ceremonies. Girlfriend keeps it limber between flights with her love of yoga, and she even earned a certificate in being bendy. Let’s hope things work out with Herr Soules, because they might get a touch awkward if not — Alissa says her greatest fear is running into recent exes. She even admits that she once stalked a crush hardcore — and then texted him her findings on accident. Yikes. But just in case you’re wondering, Alissa made the Dean’s List at St. Joseph’s, where she studied business. To sum her up, let’s turn to the gal’s description of her spirit animal: “A wild mustang. Free to run and explore, they're unpredictable and beautiful, and are loyal to their herd.” If that’s not a winning Bachelor contestant, we don’t know who is. Do you think Alissa is Chris’ perfect match? Sound off in the comments below. Sources: Reality Steve, ABC, LinkedIn \ No newline at end of file +MISSES FIRST TWO PARAGRAPHS BEFORE ADVERTISEMENT IN MIDDLE OF PAGE \ No newline at end of file diff --git a/tests/data/text/wired.com1.txt b/tests/data/text/wired.com1.txt index 80b0002f..2f42a695 100644 --- a/tests/data/text/wired.com1.txt +++ b/tests/data/text/wired.com1.txt @@ -2,6 +2,8 @@ A shiny Morpho butterfly is a master of nanoscale light bending. That blue isn If it’s all made of the same stuff, why are some parts so shiny, and other parts not? This video by KQED explains: +How to Build A Butterfly Wing + The shine of butterflies (and beetles) is created by incredibly detailed nanostructures of longitudinal ridges and crossribs. They reflect specific wavelengths of light, creating the insect’s sparkle. The photo at right shows some detail; the white scale bar is is 65 µm long, or 0.0650 mm. Butterflies extrude their scales during metamorphosis from cells on their epidermis, just like we do with hair. And, just like our hair, the final version of a butterfly’s scales are not living cells. diff --git a/tests/data/text/wnet.org1.txt b/tests/data/text/wnet.org1.txt index 26885857..286c5038 100644 --- a/tests/data/text/wnet.org1.txt +++ b/tests/data/text/wnet.org1.txt @@ -2,8 +2,8 @@ AUDIT COMMITTEE (at Skadden, Arps, Slate, Meagher & Flom LLP 4 Times Square, 42n Tuesday, September 24, 2013 at 2:30 PM **Telephonic - Monday, November 25, 2013 at 3:00 PM +Monday, November 25, 2013 at 3:00 PM - Tuesday, April 22, 2014 at 1:00 PM +Tuesday, April 22, 2014 at 1:00 PM ** To request the toll-free conference call-in number, please call (212) 560 – 6928 \ No newline at end of file diff --git a/tests/data/text/wnet.org2.txt b/tests/data/text/wnet.org2.txt index e1bf0334..4035856b 100644 --- a/tests/data/text/wnet.org2.txt +++ b/tests/data/text/wnet.org2.txt @@ -1,5 +1,7 @@ +INTERACTIVE & TECHNOLOGY COMMITTEE (Location as listed below) + Tuesday, October 15, 2013 at 9:00 AM (at Loews Corporation, 667 Madison Avenue at 61st Street, 7th Fl) - Monday, January 27, 2014 at 9:00 AM (at Loews Corporation, 667 Madison Avenue at 61st Street, 7th Fl) +Monday, January 27, 2014 at 9:00 AM (at Loews Corporation, 667 Madison Avenue at 61st Street, 7th Fl) - Tuesday, April 8, 2014 at 9:00 AM (at WNET, 825 Eighth Avenue, 14th Fl. Kellen Board Room) \ No newline at end of file +Tuesday, April 8, 2014 at 9:00 AM (at WNET, 825 Eighth Avenue, 14th Fl. Kellen Board Room) \ No newline at end of file diff --git a/tests/data/text/youbeauty.com1.txt b/tests/data/text/youbeauty.com1.txt index 488f590e..bc5b34af 100644 --- a/tests/data/text/youbeauty.com1.txt +++ b/tests/data/text/youbeauty.com1.txt @@ -1,27 +1,57 @@ +5 Pop Culture Nail Decals & The Totally Appropriate Places to Wear Them + +NailSpin + The joint elevation of fan culture and nail art has left us with something magical: Drake nail decals. It doesn’t stop there – you can now wear your favorite '90s TV characters and even movie quotes on your nails with pride. Although its a personal style game changer, it all costs less than $10. Here are our top five picks and recommendations of totally appropriate places to wear them. But remember the Internet is a wonderland, so don’t stop until you find your own favorite pop culture moment: +Jane's Tiny Things/Forever 21 + +For your high school reunion... + "Saved by the Bell" nail decals at Forever 21, $9 Zack Morris, Kelly Kapowski and the gang reunite right here on your hands. The decals are printed in color and black and white, but if you’re a "Saved by the Bell" fan, you’re obviously choosing the neon. +fingerprintsdecals + + + + + +For a movie marathon... + "Harry Potter" decals by fingerprintsdecals on Etsy, $5.50 Remember when Emma Watson wasn’t an international feminist and was instead a small wizard? The Etsy shop fingerprints has zapped her and her fellow Gryffindors until they’re small enough to fit on a pinky. +PaipurNails + For that one friend… "Mean Girls" quotes by PaipurNails at Etsy, $4 Don’t we all know one person who has memorized every line from "Mean Girls"? For Etsy seller PaipurNails, she is that one person, and she has found her calling with decals that say “Burn Book” and “She doesn’t even go here.” +NailSpin + +For a date... + "The Godfather" by NailSpin at Etsy, $5 What better way to show how cultured you are than adding Vito Corleone to your beauty routine? Your high school friends probably had a "Godfather" poster in their dorm rooms – one up them by putting him on your nails. +NailSpin + +For the afterparty... + +Drake by NailSpin at Etsy, $6 + You can stick a Post-It to your desk that says “‘Live In the Moment” or you can wear YOLO nail decals. We all sometimes need a little daily inspiration – personally, I’d love to get it from Drake’s face on my ring finger. +Related Articles: + Flower Nail Art That Works for Every Season 3 Nail Art Designs That Last a Whole Month \ No newline at end of file diff --git a/tests/data/text/youbeauty.com2.txt b/tests/data/text/youbeauty.com2.txt index 0297e317..56bc3899 100644 --- a/tests/data/text/youbeauty.com2.txt +++ b/tests/data/text/youbeauty.com2.txt @@ -1,9 +1,21 @@ I have confused skin. One moment it is dry and needs hydration and the next it is oily and greasy. So I need a powder that will cut the oil, but still let my skin breathe. Clinique Acne Solutions Powder Makeup gently covers blemishes, evens skin and even absorbs oil to leave skin looking fresh. It's made specifically for "dry combination to oily skin types," so it may be just what I have been looking for. +Product: Clinique Acne Solutions Powder Makeup + +Price: $31.00 + +Tags: + How to Use it: The powder comes with a sponge applicator, but feel free to apply it with a powder brush as well. Apply to oily spots only or use the powder on your entire face for full-coverage. (It's also available in liquid form.) Results: The shade that I received as a trial, Golden, was a bit too dark for my complexion, but I'm expecting to get a tan soon on a trip to the tropics. That aside, my skin has been in its oily phase for the past few days, so I was super excited to see if Clinique's Acne Solutions Powder Makeup really delivered — and it definitely did! The powder immediately cut the shine and gave my skin an even, natural look. It is very light and even thought it builds on your skin, it doesn't look or feel "caked on" like some powders do. I will be purchasing it for myself in the future, although maybe in a lighter shade. +Disclaimer: I received Clinique Acne Solutions Powder as a sample. + +Related Articles: + Could Resveratrol Be the Secret to Clearer Skin? +Neroli Oil: The Ideal Face Oil for Oily Skin + 5 Stubborn Skin Issues and How to Fix Them \ No newline at end of file diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 249f1094..69c05adf 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -7,46 +7,48 @@ import unittest import time import traceback -from collections import defaultdict +import re +from collections import defaultdict, OrderedDict +import concurrent.futures TEST_DIR = os.path.abspath(os.path.dirname(__file__)) PARENT_DIR = os.path.join(TEST_DIR, '..') -# newspaper's unit tests are in their own seperate module, so +# newspaper's unit tests are in their own separate module, so # insert the parent directory manually to gain scope of the # core module sys.path.insert(0, PARENT_DIR) -TEXT_FN = os.path.join(TEST_DIR, 'data/text') -HTML_FN = os.path.join(TEST_DIR, 'data/html') -URLS_FILE = os.path.join(TEST_DIR, 'data/fulltext_url_list.txt') +TEXT_FN = os.path.join(TEST_DIR, 'data', 'text') +HTML_FN = os.path.join(TEST_DIR, 'data', 'html') +URLS_FILE = os.path.join(TEST_DIR, 'data', 'fulltext_url_list.txt') import newspaper -from newspaper import ( - Article, Source, ArticleException, news_pool) +from newspaper import Article, fulltext, Source, ArticleException, news_pool +from newspaper.article import ArticleDownloadState from newspaper.configuration import Configuration from newspaper.urls import get_domain -# from newspaper import Config -# from newspaper.network import multithread_request -# from newspaper.text import (StopWords, StopWordsArabic, -# StopWordsKorean, StopWordsChinese) def print_test(method): - """Utility method for print verbalizing test suite, prints out + """ + Utility method for print verbalizing test suite, prints out time taken for test and functions name, and status """ + def run(*args, **kw): ts = time.time() print('\ttesting function %r' % method.__name__) method(*args, **kw) te = time.time() - print('\t[OK] in %r %2.2f sec' % (method.__name__, te-ts)) + print('\t[OK] in %r %2.2f sec' % (method.__name__, te - ts)) + return run def mock_resource_with(filename, resource_type): - """Mocks an HTTP request by pulling text from a pre-downloaded file + """ + Mocks an HTTP request by pulling text from a pre-downloaded file """ VALID_RESOURCES = ['html', 'txt'] if resource_type not in VALID_RESOURCES: @@ -55,12 +57,13 @@ def mock_resource_with(filename, resource_type): subfolder = 'text' if resource_type == 'txt' else 'html' resource_path = os.path.join(TEST_DIR, "data/%s/%s.%s" % (subfolder, filename, resource_type)) - with open(resource_path, 'r') as f: + with open(resource_path, 'r', encoding='utf-8') as f: return f.read() def get_base_domain(url): - """For example, the base url of uk.reuters.com => reuters.com + """ + For example, the base url of uk.reuters.com => reuters.com """ domain = get_domain(url) tld = '.'.join(domain.split('.')[-2:]) @@ -72,53 +75,89 @@ def get_base_domain(url): return base_domain -class ExhaustiveFullTextCase(unittest.TestCase): +def check_url(*args, **kwargs): + return ExhaustiveFullTextCase.check_url(*args, **kwargs) - def runTest(self): - # The "correct" fulltext needs to be manually checked - # we have 50 so far - FULLTEXT_PREPARED = 50 - domain_counters = {} +@unittest.skipIf('fulltext' not in sys.argv, 'Skipping fulltext tests') +class ExhaustiveFullTextCase(unittest.TestCase): + @staticmethod + def check_url(args): + """ + :param (basestr, basestr) url, res_filename: + :return: (pubdate_failed, fulltext_failed) + """ + url, res_filename = args + pubdate_failed, fulltext_failed = False, False + html = mock_resource_with(res_filename, 'html') + try: + a = Article(url) + a.download(html) + a.parse() + if a.publish_date is None: + pubdate_failed = True + except Exception: + print('<< URL: %s parse ERROR >>' % url) + traceback.print_exc() + pubdate_failed, fulltext_failed = True, True + else: + correct_text = mock_resource_with(res_filename, 'txt') + if not (a.text == correct_text): + # print('Diff: ', simplediff.diff(correct_text, a.text)) + # `correct_text` holds the reason of failure if failure + print('%s -- %s -- %s' % + ('Fulltext failed', + res_filename, correct_text.strip())) + fulltext_failed = True + # TODO: assert statements are commented out for full-text + # extraction tests because we are constantly tweaking the + # algorithm and improving + # assert a.text == correct_text + return pubdate_failed, fulltext_failed + + @print_test + def test_exhaustive(self): with open(URLS_FILE, 'r') as f: urls = [d.strip() for d in f.readlines() if d.strip()] - for url in urls[:FULLTEXT_PREPARED]: + domain_counters = {} + + def get_filename(url): domain = get_base_domain(url) - if domain in domain_counters: - domain_counters[domain] += 1 - else: - domain_counters[domain] = 1 + domain_counters[domain] = domain_counters.get(domain, 0) + 1 + return '{}{}'.format(domain, domain_counters[domain]) - res_filename = domain + str(domain_counters[domain]) - html = mock_resource_with(res_filename, 'html') - try: - a = Article(url) - a.download(html) - a.parse() - except Exception: - print('<< URL: %s parse ERROR >>' % url) - traceback.print_exc() - continue + filenames = map(get_filename, urls) - correct_text = mock_resource_with(res_filename, 'txt') - condensed_url = url[:30] + ' ...' - print('%s -- fulltext status: %s' % - (condensed_url, a.text == correct_text)) - # assert a.text == correct_text + with concurrent.futures.ProcessPoolExecutor() as executor: + test_results = list(executor.map(check_url, zip(urls, filenames))) + + total_pubdates_failed, total_fulltext_failed = \ + list(map(sum, zip(*test_results))) + + print('%s fulltext extractions failed out of %s' % + (total_fulltext_failed, len(urls))) + print('%s pubdate extractions failed out of %s' % + (total_pubdates_failed, len(urls))) + self.assertGreaterEqual(47, total_pubdates_failed) + self.assertGreaterEqual(20, total_fulltext_failed) class ArticleTestCase(unittest.TestCase): - def runTest(self): - self.test_url() - self.test_download_html() - self.test_pre_download_parse() - self.test_parse_html() - self.test_meta_type_extraction() - self.test_meta_extraction() - self.test_pre_download_nlp() - self.test_pre_parse_nlp() - self.test_nlp_body() + def setup_stage(self, stage_name): + stages = OrderedDict([ + ('initial', lambda: None), + ('download', lambda: self.article.download( + mock_resource_with('cnn_article', 'html'))), + ('parse', lambda: self.article.parse()), + ('meta', lambda: None), # Alias for nlp + ('nlp', lambda: self.article.nlp()) + ]) + assert stage_name in stages + for name, action in stages.items(): + if name == stage_name: + break + action() def setUp(self): """Called before the first test case of this unit begins @@ -127,23 +166,45 @@ def setUp(self): url='http://www.cnn.com/2013/11/27/travel/weather-' 'thanksgiving/index.html?iref=allsearch') - def tearDown(self): - """Called after all cases have been completed, intended to - free resources and etc - """ - pass - @print_test def test_url(self): - assert self.article.url == ( + self.assertEqual( 'http://www.cnn.com/2013/11/27/travel/weather-' - 'thanksgiving/index.html?iref=allsearch') + 'thanksgiving/index.html?iref=allsearch', + self.article.url) @print_test def test_download_html(self): + self.setup_stage('download') html = mock_resource_with('cnn_article', 'html') self.article.download(html) - assert len(self.article.html) == 75175 + self.assertEqual(self.article.download_state, ArticleDownloadState.SUCCESS) + self.assertEqual(self.article.download_exception_msg, None) + self.assertEqual(75406, len(self.article.html)) + + @print_test + def test_meta_refresh_redirect(self): + # TODO: We actually hit example.com in this unit test ... which is bad + # Figure out how to mock an actual redirect + config = Configuration() + config.follow_meta_refresh = True + article = Article( + '', config=config) + html = mock_resource_with('google_meta_refresh', 'html') + article.download(input_html=html) + article.parse() + self.assertEqual(article.title, 'Example Domain') + + @print_test + def test_meta_refresh_no_url_redirect(self): + config = Configuration() + config.follow_meta_refresh = True + article = Article( + '', config=config) + html = mock_resource_with('ap_meta_refresh', 'html') + article.download(input_html=html) + article.parse() + self.assertEqual(article.title, 'News from The Associated Press') @print_test def test_pre_download_parse(self): @@ -154,76 +215,96 @@ def test_pre_download_parse(self): @print_test def test_parse_html(self): - AUTHORS = ['Dana Ford', 'Tom Watkins'] + self.setup_stage('parse') + + AUTHORS = ['Chien-Ming Wang', 'Dana A. Ford', 'James S.A. Corey', + 'Tom Watkins'] TITLE = 'After storm, forecasters see smooth sailing for Thanksgiving' LEN_IMGS = 46 META_LANG = 'en' + META_SITE_NAME = 'CNN' self.article.parse() self.article.nlp() text = mock_resource_with('cnn', 'txt') - assert self.article.text == text + self.assertEqual(text, self.article.text) + self.assertEqual(text, fulltext(self.article.html)) # NOTE: top_img extraction requires an internet connection # unlike the rest of this test file TOP_IMG = ('http://i2.cdn.turner.com/cnn/dam/assets/131129200805-' '01-weather-1128-story-top.jpg') - assert self.article.top_img == TOP_IMG + self.assertEqual(TOP_IMG, self.article.top_img) - assert sorted(self.article.authors) == AUTHORS - assert self.article.title == TITLE - assert len(self.article.imgs) == LEN_IMGS - assert self.article.meta_lang == META_LANG + self.assertCountEqual(AUTHORS, self.article.authors) + self.assertEqual(TITLE, self.article.title) + self.assertEqual(LEN_IMGS, len(self.article.imgs)) + self.assertEqual(META_LANG, self.article.meta_lang) + self.assertEqual(META_SITE_NAME, self.article.meta_site_name) + self.assertEqual('2013-11-27 00:00:00', str(self.article.publish_date)) @print_test def test_meta_type_extraction(self): + self.setup_stage('meta') meta_type = self.article.extractor.get_meta_type( self.article.clean_doc) - assert 'article' == meta_type + self.assertEqual('article', meta_type) @print_test def test_meta_extraction(self): + self.setup_stage('meta') meta = self.article.extractor.get_meta_data(self.article.clean_doc) META_DATA = defaultdict(dict, { 'medium': 'news', 'googlebot': 'noarchive', 'pubdate': '2013-11-27T08:36:32Z', 'title': 'After storm, forecasters see smooth sailing for Thanksgiving - CNN.com', - 'og': {'site_name': 'CNN','description': 'A strong storm struck much of the eastern United States on Wednesday, complicating holiday plans for many of the 43 million Americans expected to travel.', 'title': 'After storm, forecasters see smooth sailing for Thanksgiving', 'url': 'http://www.cnn.com/2013/11/27/travel/weather-thanksgiving/index.html', 'image': 'http://i2.cdn.turner.com/cnn/dam/assets/131129200805-01-weather-1128-story-top.jpg', 'type': 'article'}, + 'og': {'site_name': 'CNN', + 'description': 'A strong storm struck much of the eastern United States on Wednesday, complicating holiday plans for many of the 43 million Americans expected to travel.', + 'title': 'After storm, forecasters see smooth sailing for Thanksgiving', + 'url': 'http://www.cnn.com/2013/11/27/travel/weather-thanksgiving/index.html', + 'image': 'http://i2.cdn.turner.com/cnn/dam/assets/131129200805-01-weather-1128-story-top.jpg', + 'type': 'article'}, 'section': 'travel', - 'author': 'Dana Ford and Tom Watkins, CNN', + 'author': 'Dana A. Ford, James S.A. Corey, Chien-Ming Wang, and Tom Watkins, CNN', 'robots': 'index,follow', - 'vr': {'canonical': 'http://edition.cnn.com/2013/11/27/travel/weather-thanksgiving/index.html'}, + 'vr': { + 'canonical': 'http://edition.cnn.com/2013/11/27/travel/weather-thanksgiving/index.html'}, 'source': 'CNN', 'fb': {'page_id': 18793419640, 'app_id': 80401312489}, 'keywords': 'winter storm,holiday travel,Thanksgiving storm,Thanksgiving winter storm', - 'article': {'publisher': 'https://www.facebook.com/cnninternational'}, + 'article': { + 'publisher': 'https://www.facebook.com/cnninternational'}, 'lastmod': '2013-11-28T02:03:23Z', - 'twitter': {'site': {'identifier': '@CNNI', 'id': 2097571}, 'card': 'summary', 'creator': {'identifier': '@cnntravel', 'id': 174377718}}, - 'viewport':'width=1024', + 'twitter': {'site': {'identifier': '@CNNI', 'id': 2097571}, + 'card': 'summary', + 'creator': {'identifier': '@cnntravel', + 'id': 174377718}}, + 'viewport': 'width=1024', 'news_keywords': 'winter storm,holiday travel,Thanksgiving storm,Thanksgiving winter storm' }) - assert meta == META_DATA + self.assertDictEqual(META_DATA, meta) # if the value for a meta key is another dict, that dict ought to be # filled with keys and values dict_values = [v for v in list(meta.values()) if isinstance(v, dict)] - assert all([len(d) > 0 for d in dict_values]) + self.assertTrue(all([len(d) > 0 for d in dict_values])) # there are exactly 5 top-level "og:type" type keys is_dict = lambda v: isinstance(v, dict) - assert len(list(filter(is_dict, list(meta.values())))) == 5 + self.assertEqual(5, len([i for i in meta.values() if is_dict(i)])) # there are exactly 12 top-level "pubdate" type keys is_string = lambda v: isinstance(v, str) - assert len(list(filter(is_string, list(meta.values())))) == 12 + self.assertEqual(12, len([i for i in meta.values() if is_string(i)])) @print_test def test_pre_download_nlp(self): """Test running NLP algos before even downloading the article """ + self.setup_stage('initial') new_article = Article(self.article.url) self.assertRaises(ArticleException, new_article.nlp) @@ -231,33 +312,146 @@ def test_pre_download_nlp(self): def test_pre_parse_nlp(self): """Test running NLP algos before parsing the article """ - new_article = Article(self.article.url) - html = mock_resource_with('cnn_article', 'html') - new_article.download(html) - self.assertRaises(ArticleException, new_article.nlp) + self.setup_stage('parse') + self.assertRaises(ArticleException, self.article.nlp) @print_test def test_nlp_body(self): - KEYWORDS = ['forecasters', 'storm', 'winds', 'sailing', - 'great', 'weather', 'balloons', 'snow', 'good', - 'flight', 'york', 'roads', 'smooth', 'thanksgiving'] + self.setup_stage('nlp') + self.article.nlp() + KEYWORDS = ['balloons', 'delays', 'flight', 'forecasters', + 'good', 'sailing', 'smooth', 'storm', 'thanksgiving', + 'travel', 'weather', 'winds', 'york'] SUMMARY = mock_resource_with('cnn_summary', 'txt') - assert self.article.summary == SUMMARY - assert sorted(self.article.keywords) == sorted(KEYWORDS) + self.assertEqual(SUMMARY, self.article.summary) + self.assertCountEqual(KEYWORDS, self.article.keywords) -class SourceTestCase(unittest.TestCase): - def runTest(self): - self.source_url_input_none() - self.test_cache_categories() - self.test_source_build() +class TestDownloadScheme(unittest.TestCase): + @print_test + def test_download_file_success(self): + url = "file://" + os.path.join(HTML_FN, "cnn_article.html") + article = Article(url=url) + article.download() + self.assertEqual(article.download_state, ArticleDownloadState.SUCCESS) + self.assertEqual(article.download_exception_msg, None) + self.assertEqual(75406, len(article.html)) @print_test - def source_url_input_none(self): - def failfunc(): + def test_download_file_failure(self): + url = "file://" + os.path.join(HTML_FN, "does_not_exist.html") + article = Article(url=url) + article.download() + self.assertEqual(0, len(article.html)) + self.assertEqual(article.download_state, ArticleDownloadState.FAILED_RESPONSE) + self.assertEqual(article.download_exception_msg, "No such file or directory") + + +class ContentExtractorTestCase(unittest.TestCase): + """Test specific element extraction cases""" + + def setUp(self): + self.extractor = newspaper.extractors.ContentExtractor(Configuration()) + self.parser = newspaper.parsers.Parser + + def _get_title(self, html): + doc = self.parser.fromstring(html) + return self.extractor.get_title(doc) + + def test_get_title_basic(self): + html = 'Test title' + self.assertEqual(self._get_title(html), 'Test title') + + def test_get_title_split(self): + html = 'Test page » Test title' + self.assertEqual(self._get_title(html), 'Test title') + + def test_get_title_split_escaped(self): + html = 'Test page » Test title' + self.assertEqual(self._get_title(html), 'Test title') + + def test_get_title_quotes(self): + title = 'Test page and «something in quotes»' + html = '{}'.format(title) + self.assertEqual(self._get_title(html), title) + + def _get_canonical_link(self, article_url, html): + doc = self.parser.fromstring(html) + return self.extractor.get_canonical_link(article_url, doc) + + def test_get_canonical_link_rel_canonical(self): + url = 'http://www.example.com/article.html' + html = ''.format(url) + self.assertEqual(self._get_canonical_link('', html), url) + + def test_get_canonical_link_rel_canonical_absolute_url(self): + url = 'http://www.example.com/article.html' + html = '' + article_url = 'http://www.example.com/article?foo=bar' + self.assertEqual(self._get_canonical_link(article_url, html), url) + + def test_get_canonical_link_og_url_absolute_url(self): + url = 'http://www.example.com/article.html' + html = '' + article_url = 'http://www.example.com/article?foo=bar' + self.assertEqual(self._get_canonical_link(article_url, html), url) + + def test_get_canonical_link_hostname_og_url_absolute_url(self): + url = 'http://www.example.com/article.html' + html = '' + article_url = 'http://www.example.com/article?foo=bar' + self.assertEqual(self._get_canonical_link(article_url, html), url) + + def test_get_top_image_from_meta(self): + html = '' \ + '' + html_empty_og_content = '' \ + '' + html_empty_all = '' \ + '' + html_rel_img_src = html_empty_all + '' + html_rel_img_src2 = html_empty_all + '' + html_rel_icon = html_empty_all + '' + + doc = self.parser.fromstring(html) + self.assertEqual( + self.extractor.get_meta_img_url('http://www.example.com/article?foo=bar', doc), + 'https://example.com/meta_img_filename.jpg' + ) + doc = self.parser.fromstring(html_empty_og_content) + self.assertEqual( + self.extractor.get_meta_img_url('http://www.example.com/article?foo=bar', doc), + 'https://example.com/meta_another_img_filename.jpg' + ) + doc = self.parser.fromstring(html_empty_all) + self.assertEqual( + self.extractor.get_meta_img_url('http://www.example.com/article?foo=bar', doc), + '' + ) + doc = self.parser.fromstring(html_rel_img_src) + self.assertEqual( + self.extractor.get_meta_img_url('http://www.example.com/article?foo=bar', doc), + 'https://example.com/meta_link_image.jpg' + ) + doc = self.parser.fromstring(html_rel_img_src2) + self.assertEqual( + self.extractor.get_meta_img_url('http://www.example.com/article?foo=bar', doc), + 'https://example.com/meta_link_image2.jpg' + ) + doc = self.parser.fromstring(html_rel_icon) + self.assertEqual( + self.extractor.get_meta_img_url('http://www.example.com/article?foo=bar', doc), + 'https://example.com/meta_link_rel_icon.ico' + ) + + +class SourceTestCase(unittest.TestCase): + @print_test + def test_source_url_input_none(self): + with self.assertRaises(Exception): Source(url=None) - self.assertRaises(Exception, failfunc) + @unittest.skip("Need to mock download") @print_test def test_source_build(self): """ @@ -310,6 +504,7 @@ def test_source_build(self): # effect by just mocking CNN's main page HTML. Warning: tedious fix. # assert s.feed_urls() == FEEDS + @unittest.skip("Need to mock download") @print_test def test_cache_categories(self): """Builds two same source objects in a row examines speeds of both @@ -324,13 +519,10 @@ def test_cache_categories(self): saved_urls = s.category_urls() s.categories = [] s.set_categories() - assert sorted(s.category_urls()) == sorted(saved_urls) + self.assertCountEqual(saved_urls, s.category_urls()) class UrlTestCase(unittest.TestCase): - def runTest(self): - self.test_valid_urls() - @print_test def test_valid_urls(self): """Prints out a list of urls with our heuristic guess if it is a @@ -344,30 +536,61 @@ def test_valid_urls(self): # tuples are ('1', 'url_goes_here') form, '1' means valid, # '0' otherwise - for tup in test_tuples: - lst = int(tup[0]) - url = tup[1] - assert len(tup) == 2 - truth_val = True if lst == 1 else False + for lst, url in test_tuples: + truth_val = bool(int(lst)) try: - assert truth_val == valid_url(url, test=True) + self.assertEqual(truth_val, valid_url(url, test=True)) except AssertionError: print('\t\turl: %s is supposed to be %s' % (url, truth_val)) raise + + @print_test + def test_pubdate(self): + """Checks that irrelevant data in url isn't considered as publishing date""" + from newspaper.urls import STRICT_DATE_REGEX + + with open(os.path.join(TEST_DIR, 'data/test_urls_pubdate.txt'), 'r') as f: + lines = f.readlines() + test_tuples = [tuple(l.strip().split(' ')) for l in lines] + # tuples are ('1', 'url_goes_here') form, '1' means publishing date + # is present in the url, '0' otherwise + + for pubdate, url in test_tuples: + is_present = bool(int(pubdate)) + date_match = re.search(STRICT_DATE_REGEX, url) + try: + self.assertEqual(is_present, bool(date_match)) + except AssertionError: + if is_present: + print('\t\tpublishing date in %s should be present' % (url)) + else: + print('\t\tpublishing date in %s should not be present' % (url)) + raise + + + @unittest.skip("Need to write an actual test") @print_test def test_prepare_url(self): """Normalizes a url, removes arguments, hashtags. If a relative url, it merges it with the source domain to make an abs url, etc """ - pass + from newspaper.urls import prepare_url + with open(os.path.join(TEST_DIR, 'data/test_prepare_urls.txt'), 'r') as f: + lines = f.readlines() + test_tuples = [tuple(l.strip().split(' ')) for l in lines] + # tuples are ('real_url', 'url_path', 'source_url') form + + for real, url, source in test_tuples: + try: + self.assertEqual(real, prepare_url(url, source)) + except AssertionError: + print('\t\turl: %s + %s is supposed to be %s' % (url, source, real)) + raise -class APITestCase(unittest.TestCase): - def runTest(self): - self.test_hot_trending() - self.test_popular_urls() +class APITestCase(unittest.TestCase): @print_test def test_hot_trending(self): """Grab google trending, just make sure this runs @@ -381,10 +604,8 @@ def test_popular_urls(self): newspaper.popular_urls() +@unittest.skip("Need to mock download") class MThreadingTestCase(unittest.TestCase): - def runTest(self): - self.test_download_works() - @print_test def test_download_works(self): config = Configuration() @@ -410,48 +631,47 @@ def test_download_works(self): class ConfigBuildTestCase(unittest.TestCase): - def runTest(self): - self.test_config_build() - + """Test if our **kwargs to config building setup actually works. + NOTE: No need to mock responses as we are just initializing the + objects, not actually calling download(..) + """ @print_test - def test_config_build(self): - """Test if our **kwargs to config building setup actually works. - NOTE: No need to mock responses as we are just initializing the - objects, not actually calling download(..) - """ + def test_article_default_params(self): + a = Article(url='http://www.cnn.com/2013/11/27/' - 'travel/weather-thanksgiving/index.html') - assert a.config.language == 'en' - assert a.config.memoize_articles is True - assert a.config.use_meta_language is True + 'travel/weather-thanksgiving/index.html') + self.assertEqual('en', a.config.language) + self.assertTrue(a.config.memoize_articles) + self.assertTrue(a.config.use_meta_language) + @print_test + def test_article_custom_params(self): a = Article(url='http://www.cnn.com/2013/11/27/travel/' - 'weather-thanksgiving/index.html', + 'weather-thanksgiving/index.html', language='zh', memoize_articles=False) - assert a.config.language == 'zh' - assert a.config.memoize_articles is False - assert a.config.use_meta_language is False + self.assertEqual('zh', a.config.language) + self.assertFalse(a.config.memoize_articles) + self.assertFalse(a.config.use_meta_language) + @print_test + def test_source_default_params(self): s = Source(url='http://cnn.com') - assert s.config.language == 'en' - assert s.config.MAX_FILE_MEMO == 20000 - assert s.config.memoize_articles is True - assert s.config.use_meta_language is True + self.assertEqual('en', s.config.language) + self.assertEqual(20000, s.config.MAX_FILE_MEMO) + self.assertTrue(s.config.memoize_articles) + self.assertTrue(s.config.use_meta_language) + @print_test + def test_source_custom_params(self): s = Source(url="http://cnn.com", memoize_articles=False, MAX_FILE_MEMO=10000, language='en') - assert s.config.memoize_articles is False - assert s.config.MAX_FILE_MEMO == 10000 - assert s.config.language == 'en' - assert s.config.use_meta_language is False + self.assertFalse(s.config.memoize_articles) + self.assertEqual(10000, s.config.MAX_FILE_MEMO) + self.assertEqual('en', s.config.language) + self.assertFalse(s.config.use_meta_language) class MultiLanguageTestCase(unittest.TestCase): - def runTest(self): - self.test_chinese_fulltext_extract() - self.test_arabic_fulltext_extract() - self.test_spanish_fulltext_extract() - @print_test def test_chinese_fulltext_extract(self): url = 'http://news.sohu.com/20050601/n225789219.shtml' @@ -460,44 +680,96 @@ def test_chinese_fulltext_extract(self): article.download(html) article.parse() text = mock_resource_with('chinese', 'txt') - assert article.text == text + self.assertEqual(text, article.text) + self.assertEqual(text, fulltext(article.html, 'zh')) @print_test def test_arabic_fulltext_extract(self): - url = 'http://arabic.cnn.com/2013/middle_east/8/3/syria.clashes/'\ + url = 'http://arabic.cnn.com/2013/middle_east/8/3/syria.clashes/' \ 'index.html' article = Article(url=url) html = mock_resource_with('arabic_article', 'html') article.download(html) article.parse() - assert article.meta_lang == 'ar' + self.assertEqual('ar', article.meta_lang) text = mock_resource_with('arabic', 'txt') - assert article.text == text + self.assertEqual(text, article.text) + self.assertEqual(text, fulltext(article.html, 'ar')) @print_test def test_spanish_fulltext_extract(self): - url = 'http://ultimahora.es/mallorca/noticia/noticias/local/fiscal'\ + url = 'http://ultimahora.es/mallorca/noticia/noticias/local/fiscal' \ 'ia-anticorrupcion-estudia-recurre-imputacion-infanta.html' article = Article(url=url, language='es') html = mock_resource_with('spanish_article', 'html') article.download(html) article.parse() text = mock_resource_with('spanish', 'txt') - assert article.text == text + self.assertEqual(text, article.text) + self.assertEqual(text, fulltext(article.html, 'es')) + @print_test + def test_japanese_fulltext_extract(self): + url = 'https://www.nikkei.com/article/DGXMZO31897660Y8A610C1000000/?n_cid=DSTPCS001' + article = Article(url=url, language='ja') + html = mock_resource_with('japanese_article', 'html') + article.download(html) + article.parse() + text = mock_resource_with('japanese', 'txt') + self.assertEqual(text, article.text) + self.assertEqual(text, fulltext(article.html, 'ja')) + + @print_test + def test_japanese_fulltext_extract2(self): + url = 'http://www.afpbb.com/articles/-/3178894' + article = Article(url=url, language='ja') + html = mock_resource_with('japanese_article2', 'html') + article.download(html) + article.parse() + text = mock_resource_with('japanese2', 'txt') + self.assertEqual(text, article.text) + self.assertEqual(text, fulltext(article.html, 'ja')) + + @print_test + def test_thai_fulltext_extract(self): + url = 'https://prachatai.com/journal/2019/01/80642' + article = Article(url=url, language='th') + html = mock_resource_with('thai_article', 'html') + article.download(html) + article.parse() + text = mock_resource_with('thai', 'txt') + self.assertEqual(text, article.text) + self.assertEqual(text, fulltext(article.html, 'th')) + + +class TestNewspaperLanguagesApi(unittest.TestCase): + @print_test + def test_languages_api_call(self): + newspaper.languages() -if __name__ == '__main__': - # unittest.main() # run all units and their cases - suite = unittest.TestSuite() +class TestDownloadPdf(unittest.TestCase): - # suite.addTest(ExhaustiveFullTextCase()) - suite.addTest(ConfigBuildTestCase()) - suite.addTest(MultiLanguageTestCase()) - suite.addTest(UrlTestCase()) - suite.addTest(ArticleTestCase()) - suite.addTest(APITestCase()) - unittest.TextTestRunner().run(suite) + @print_test + def test_article_pdf_ignoring(self): + empty_pdf = "%PDF-" # empty PDF constant + a = Article(url='https://www.adobe.com/pdf/pdfs/ISO32000-1PublicPatentLicense.pdf', + ignored_content_types_defaults={"application/pdf": empty_pdf, + "application/x-pdf": empty_pdf, + "application/x-bzpdf": empty_pdf, + "application/x-gzpdf": empty_pdf}) + a.download() + self.assertEqual(empty_pdf, a.html) + + @print_test + def test_article_pdf_fetching(self): + a = Article(url='https://www.adobe.com/pdf/pdfs/ISO32000-1PublicPatentLicense.pdf') + a.download() + self.assertNotEqual('%PDF-', a.html) + +if __name__ == '__main__': + argv = list(sys.argv) + if 'fulltext' in argv: + argv.remove('fulltext') # remove it here, so it doesn't pass to unittest - # TODO: suite.addTest(SourceTestCase()) - # suite.addTest(MThreadingTestCase()) + unittest.main(verbosity=0, argv=argv)