From bc8e1cd9bfd06d3c909917819966f7cd43a9836e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=B2=D0=B0=D0=BD=20=D0=91=D0=B5=D0=B3=D1=82=D0=B8?= =?UTF-8?q?=D0=BD?= Date: Sat, 3 Feb 2018 10:46:36 +0300 Subject: [PATCH] Added support for dates in page using qddate lib --- newspaper/extractors.py | 13 ++++++++++++- requirements.txt | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/newspaper/extractors.py b/newspaper/extractors.py index b7fe1def..6d3b869f 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -17,7 +17,8 @@ import re from collections import defaultdict -from dateutil.parser import parse as date_parser +from dateutil import parser as date_parser +from qddate import DateParser from tldextract import tldextract from urllib.parse import urljoin, urlparse, urlunparse @@ -230,6 +231,16 @@ def parse_date_str(date_str): if datetime_obj: return datetime_obj + # Uses qddate DateParser to find dates in any strings less than 150 chars long + dp = DateParser(generate=True) + nodes = doc.xpath('//*[string-length(text())<150]') + for node in nodes: + if node.text is None: + continue + datetime_obj = dp.parse(node.text) + if datetime_obj is not None: + return datetime_obj + return None def get_title(self, doc): diff --git a/requirements.txt b/requirements.txt index ccfccd37..58c3c0d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ tldextract>=2.0.1 feedfinder2>=0.0.4 jieba3k>=0.35.1 python-dateutil>=2.5.3 +qddate>=0.1.0 \ No newline at end of file