pep8

Author: grangier

goose/Configuration.py

@@ -20,48 +20,49 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+
+
 class Configuration(object):
-    
+
     def __init__(self):
         # this is the local storage path used to place
         # images to inspect them, should be writable
         self.localStoragePath = "/tmp/goosetmp"
-        
+
         # What's the minimum bytes for an image we'd accept is,
         # alot of times we want to filter out the author's little images
         # in the beginning of the article
         self.minBytesForImages = 4500
-        
+
         # set this guy to false if you don't care about getting images,
         # otherwise you can either use the default
         # image extractor to implement the ImageExtractor
         # interface to build your own
         self.enableImageFetching = True
-        
+
         # path to your imagemagick convert executable,
         # on the mac using mac ports this is the default listed
         self.imagemagickConvertPath = "/opt/local/bin/convert"
-        
+
         # path to your imagemagick identify executable
         self.imagemagickIdentifyPath = "/opt/local/bin/identify"
-        
+
         # used as the user agent that
         # is sent with your web requests to extract an article
         # self.browserUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2)"\
         #                         " AppleWebKit/534.52.7 (KHTML, like Gecko) "\
         #                         "Version/5.1.2 Safari/534.52.7"
         self.browserUserAgent = 'Goose/1.0'
-        
+
         # TODO
         self.publishDateExtractor = None
-        
+
         # TODO
         self.additionalDataExtractor = None
-    
-    
+
     def getPublishDateExtractor(self):
         return self.publishDateExtractor
-    
+
     def setPublishDateExtractor(self, extractor):
         """\
         Pass in to extract article publish dates.
@@ -70,10 +71,10 @@ def setPublishDateExtractor(self, extractor):
         if not extractor:
             raise ValueError("extractor must not be null!")
         self.publishDateExtractor = extractor
-    
+
     def getAdditionalDataExtractor(self):
         return self.additionalDataExtractor
-    
+
     def setAdditionalDataExtractor(self, extractor):
         """\
         Pass in to extract any additional data not defined within
@@ -82,5 +83,3 @@ def setAdditionalDataExtractor(self, extractor):
         if not extractor:
             raise ValueError("extractor must not be null!")
         self.additionalDataExtractor = extractor
-    
-    

goose/Crawler.py

@@ -31,37 +31,36 @@
 from goose.images.UpgradedImageExtractor import UpgradedImageIExtractor
 from goose.network import HtmlFetcher
 
+
 class CrawlCandidate(object):
-    
+
     def __init__(self, config, url, rawHTML):
         self.config = config
         self.url = url
         self.rawHTML = rawHTML
 
 
-
 class Crawler(object):
-    
+
     def __init__(self, config):
         self.config = config
         self.logPrefix = "crawler:"
-    
+
     def crawl(self, crawlCandidate):
         article = Article()
-        
+
         parseCandidate = URLHelper.getCleanedUrl(crawlCandidate.url)
         rawHtml = self.getHTML(crawlCandidate, parseCandidate)
-        
+
         if rawHtml is None:
             return article
-        
+
         doc = self.getDocument(parseCandidate.url, rawHtml)
-        
-        
+
         extractor = self.getExtractor()
         docCleaner = self.getDocCleaner()
         outputFormatter = self.getOutputFormatter()
-        
+
         # article
         article.finalUrl = parseCandidate.url
         article.linkhash = parseCandidate.linkhash
@@ -81,7 +80,7 @@ def crawl(self, crawlCandidate):
         article.tags = extractor.extractTags(article)
         # # before we do any calcs on the body itself let's clean up the document
         article.doc = docCleaner.clean(article)
-        
+
         # big stuff
         article.topNode = extractor.calculateBestNodeBasedOnClustering(article)
         if article.topNode is not None:
@@ -91,43 +90,37 @@ def crawl(self, crawlCandidate):
             if self.config.enableImageFetching:
                 imageExtractor = self.getImageExtractor(article)
                 article.topImage = imageExtractor.getBestImage(article.rawDoc, article.topNode)
-            
+
             article.topNode = extractor.postExtractionCleanup(article.topNode)
             article.cleanedArticleText = outputFormatter.getFormattedText(article.topNode)
-        
+
         return article
-    
+
     def getHTML(self, crawlCandidate, parsingCandidate):
         if crawlCandidate.rawHTML:
             return crawlCandidate.rawHTML
         else:
             # fetch HTML
             html = HtmlFetcher().getHtml(self.config, parsingCandidate.url)
             return html
-    
-    
+
     def getImageExtractor(self, article):
         httpClient = None
         return UpgradedImageIExtractor(httpClient, article, self.config)
-    
-    
+
     def getOutputFormatter(self):
         return StandardOutputFormatter()
-    
-    
+
     def getDocCleaner(self):
         return StandardDocumentCleaner()
-    
-    
+
     def getDocument(self, url, rawHtml):
         doc = Parser.fromstring(rawHtml)
         return doc
-    
-    
+
     def getExtractor(self):
         return StandardContentExtractor()
-    
-    
+
     def releaseResources(self, article):
         directory = self.config.localStoragePath
         for fname in os.listdir(directory):
@@ -137,7 +130,3 @@ def releaseResources(self, article):
             except OSError:
                 # TODO better log handeling
                 pass
-                
-        
-            
-            

goose/Goose.py

@@ -25,46 +25,43 @@
 from goose.Crawler import CrawlCandidate
 from goose.Crawler import Crawler
 
+
 class Goose(object):
     """\
-    
+
     """
     def __init__(self, config=None):
         self.config = config or Configuration()
         self.initializeEnvironment()
-    
-    
+
     def extractContent(self, url=None, rawHTML=None):
         """\
         Main method to extract an article object from a URL,
         pass in a url and get back a Article
         """
         cc = CrawlCandidate(self.config, url, rawHTML)
         return self.sendToActor(cc)
-    
-    
+
     def shutdownNetwork(self):
         pass
-    
-    
+
     def sendToActor(self, crawlCandiate):
         crawler = Crawler(self.config)
         article = crawler.crawl(crawlCandiate)
         return article
-    
-    
+
     def initializeEnvironment(self):
         # test if config.localStoragePath
         # is a directory
         if not os.path.isdir(self.config.localStoragePath):
             os.makedirs(self.config.localStoragePath)
-        
+
         if not os.path.isdir(self.config.localStoragePath):
             raise Exception(self.config.localStoragePath +
                 " directory does not seem to exist, "
                 "you need to set this for image processing downloads"
             )
-        
+
         # test to write a dummy file to the directory
         # to check is directory is writtable
         path = '%s/test.txt' % self.config.localStoragePath
@@ -77,7 +74,3 @@ def initializeEnvironment(self):
                 " directory is not writeble, "
                 "you need to set this for image processing downloads"
             )
-        
-        
-    
-    

goose/Video.py

@@ -21,7 +21,8 @@
 limitations under the License.
 """
 
+
 class Video(object):
-    
+
     def __init_(self):
-        pass
+        pass