From 4d3afeb37bc44841204a422267398fc05e6a2c17 Mon Sep 17 00:00:00 2001
From: Parham Saidi <parham@parha.me>
Date: Wed, 16 Jul 2014 15:59:16 +0000
Subject: [PATCH 1/2] added vim swap files

---
 .gitignore | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index b9c05822..a31ce13e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,10 @@
 .idea
 .pypirc
 
+# VIM
+*.swp
+*.swo
+
 # C extensions
 *.so
 
@@ -39,4 +43,4 @@ nosetests.xml
 .mr.developer.cfg
 .project
 .pydevproject
-venv
\ No newline at end of file
+venv

From 827b8a014e189dfce1f245f9570eb0dd12238079 Mon Sep 17 00:00:00 2001
From: Parham Saidi <parham@parha.me>
Date: Wed, 16 Jul 2014 16:42:53 +0000
Subject: [PATCH 2/2] added published date to the extractor+article

---
 newspaper/article.py    | 13 +++++++++++++
 newspaper/extractors.py | 39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 52 insertions(+)

diff --git a/newspaper/article.py b/newspaper/article.py
index 575533a4..2b3da9cf 100644
--- a/newspaper/article.py
+++ b/newspaper/article.py
@@ -11,6 +11,7 @@
 import copy
 import os
 import glob
+import datetime
 
 from . import nlp
 from . import images
@@ -168,6 +169,9 @@ def parse(self):
         title = self.extractor.get_title(self)
         self.set_title(title)
 
+        published_date = self.extractor.get_published_date(self)
+        self.set_published_date(published_date)
+
         authors = self.extractor.get_authors(self)
         self.set_authors(authors)
 
@@ -455,6 +459,15 @@ def set_keywords(self, keywords):
         if keywords:
             self.keywords = [encodeValue(k) for k in keywords[:self.config.MAX_KEYWORDS]]
 
+    def set_published_date(self, date):
+      """
+      Datetime object of the timestamp on the article
+      """
+      if date and not isinstance(date, datetime.datetime):
+          raise Exception("published date must be a datetime object")
+      if date:
+          self.published_date = date
+
     def set_authors(self, authors):
         """
         Authors are in ["firstName lastName", "firstName lastName"] format.
diff --git a/newspaper/extractors.py b/newspaper/extractors.py
index 2ce7a462..81ae1878 100644
--- a/newspaper/extractors.py
+++ b/newspaper/extractors.py
@@ -14,6 +14,7 @@
 import re
 import copy
 import urlparse
+import datetime
 from collections import defaultdict
 
 from .packages.tldextract import tldextract
@@ -207,6 +208,44 @@ def get_title(self, article):
         title = MOTLEY_REPLACEMENT.replaceAll(title_text)
         return title
 
+    def get_published_date(self, article):
+        """
+        Find the article published date form either in the page or the url
+        """
+        timestamps = article.doc.xpath(
+            ("//meta[@property='article:published_time']/@content|"
+            "//meta[@name='date']/@content|"
+            "//meta[@name='sailthru.date']/@content")
+        )
+        timestamp = None
+        if timestamps:
+            timestamp = timestamps.pop(0).replace(
+                "T", " ").split(" ").pop(0)
+            if len(timestamp) != 10:
+                timestamp = None
+        if not timestamp:
+            parts = urlparse.urlparse(article.url)
+            path_parts = parts.path.strip("/").split("/")
+            numbered_parts = [part for part in path_parts 
+                if part.isdigit() and len(part) < 5]
+            if len(numbered_parts) == 3:
+                timestamp = "-".join(numbered_parts) 
+
+        if not timestamp:
+            return None
+
+        formats = (
+            "%Y-%m-%d",  # ISO 
+            "%m-%d-%Y",  # US
+            "%d-%m-%Y"   # UK
+        )
+        for format in formats:
+            try:
+                return datetime.datetime.strptime(
+                    timestamp, format)
+            except ValueError:
+                pass
+
     def split_title(self, title, splitter):
         """
         Split the title to best part possible.