diff --git a/.gitignore b/.gitignore index a31ce13e..b9c05822 100644 --- a/.gitignore +++ b/.gitignore @@ -4,10 +4,6 @@ .idea .pypirc -# VIM -*.swp -*.swo - # C extensions *.so @@ -43,4 +39,4 @@ nosetests.xml .mr.developer.cfg .project .pydevproject -venv +venv \ No newline at end of file diff --git a/newspaper/article.py b/newspaper/article.py index 2b3da9cf..575533a4 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -11,7 +11,6 @@ import copy import os import glob -import datetime from . import nlp from . import images @@ -169,9 +168,6 @@ def parse(self): title = self.extractor.get_title(self) self.set_title(title) - published_date = self.extractor.get_published_date(self) - self.set_published_date(published_date) - authors = self.extractor.get_authors(self) self.set_authors(authors) @@ -459,15 +455,6 @@ def set_keywords(self, keywords): if keywords: self.keywords = [encodeValue(k) for k in keywords[:self.config.MAX_KEYWORDS]] - def set_published_date(self, date): - """ - Datetime object of the timestamp on the article - """ - if date and not isinstance(date, datetime.datetime): - raise Exception("published date must be a datetime object") - if date: - self.published_date = date - def set_authors(self, authors): """ Authors are in ["firstName lastName", "firstName lastName"] format. diff --git a/newspaper/extractors.py b/newspaper/extractors.py index 81ae1878..2ce7a462 100644 --- a/newspaper/extractors.py +++ b/newspaper/extractors.py @@ -14,7 +14,6 @@ import re import copy import urlparse -import datetime from collections import defaultdict from .packages.tldextract import tldextract @@ -208,44 +207,6 @@ def get_title(self, article): title = MOTLEY_REPLACEMENT.replaceAll(title_text) return title - def get_published_date(self, article): - """ - Find the article published date form either in the page or the url - """ - timestamps = article.doc.xpath( - ("//meta[@property='article:published_time']/@content|" - "//meta[@name='date']/@content|" - "//meta[@name='sailthru.date']/@content") - ) - timestamp = None - if timestamps: - timestamp = timestamps.pop(0).replace( - "T", " ").split(" ").pop(0) - if len(timestamp) != 10: - timestamp = None - if not timestamp: - parts = urlparse.urlparse(article.url) - path_parts = parts.path.strip("/").split("/") - numbered_parts = [part for part in path_parts - if part.isdigit() and len(part) < 5] - if len(numbered_parts) == 3: - timestamp = "-".join(numbered_parts) - - if not timestamp: - return None - - formats = ( - "%Y-%m-%d", # ISO - "%m-%d-%Y", # US - "%d-%m-%Y" # UK - ) - for format in formats: - try: - return datetime.datetime.strptime( - timestamp, format) - except ValueError: - pass - def split_title(self, title, splitter): """ Split the title to best part possible.