Clean only current thread tmp files, add timestamp for multithreading

Author: grangier

goose/Crawler.py

@@ -21,6 +21,7 @@
 limitations under the License.
 """
 import os
+import glob
 from copy import deepcopy
 from goose.Article import Article
 from goose.utils import URLHelper
@@ -125,11 +126,10 @@ def getExtractor(self):
         return StandardContentExtractor(self.config)
 
     def releaseResources(self, article):
-        directory = self.config.localStoragePath
-        for fname in os.listdir(directory):
-            f = '%s/%s' % (self.config.localStoragePath, fname)
+        path = '%s/%s_*' % (self.config.localStoragePath, article.linkhash)
+        for fname in glob.glob(path):
             try:
-                os.remove(f)
+                os.remove(fname)
             except OSError:
                 # TODO better log handeling
                 pass

goose/utils/__init__.py

@@ -20,6 +20,7 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+import time
 import hashlib
 import re
 import os
@@ -92,7 +93,7 @@ def getCleanedUrl(self, urlToCrawl):
         finalUrl = urlToCrawl.replace('#!', '?_escaped_fragment_=') \
                     if '#!' in urlToCrawl else urlToCrawl
 
-        linkhash = hashlib.md5(finalUrl).hexdigest()
+        linkhash = '%s.%s' % (hashlib.md5(finalUrl).hexdigest(), time.time())
 
         return ParsingCandidate(finalUrl, linkhash, finalUrl)