move lxml stuff to parser class

Author: grangier

goose/cleaners.py

@@ -79,13 +79,13 @@ def clean_em_tags(self, doc):
         for node in ems:
             images = Parser.getElementsByTag(node, tag='img')
             if len(images) == 0:
-                node.drop_tag()
+                Parser.drop_tag(node)
         return doc
 
     def remove_drop_caps(self, doc):
         items = Parser.css_select(doc, "span[class~=dropcap], span[class~=drop_cap]")
         for item in items:
-            item.drop_tag()
+            Parser.drop_tag(item)
 
         return doc
 
@@ -108,22 +108,18 @@ def remove_scripts_styles(self, doc):
         return doc
 
     def clean_bad_tags(self, doc):
-
         # ids
-        naughty_list = doc.xpath(self.nauthy_ids_re,
-                                        namespaces={'re': self.regexp_namespace})
+        naughty_list = Parser.xpath_re(doc, self.nauthy_ids_re)
         for node in naughty_list:
             Parser.remove(node)
 
         # class
-        naughty_classes = doc.xpath(self.nauthy_classes_re,
-                                        namespaces={'re': self.regexp_namespace})
+        naughty_classes = Parser.xpath_re(doc, self.nauthy_classes_re)
         for node in naughty_classes:
             Parser.remove(node)
 
         # name
-        naughty_names = doc.xpath(self.nauthy_names_re,
-                                        namespaces={'re': self.regexp_namespace})
+        naughty_names = Parser.xpath_re(doc, self.nauthy_names_re)
         for node in naughty_names:
             Parser.remove(node)
 
@@ -132,15 +128,15 @@ def clean_bad_tags(self, doc):
     def remove_nodes_regex(self, doc, pattern):
         for selector in ['id', 'class']:
             reg = "//*[re:test(@%s, '%s', 'i')]" % (selector, pattern)
-            naughty_list = doc.xpath(reg, namespaces={'re': self.regexp_namespace})
+            naughty_list = Parser.xpath_re(doc, reg)
             for node in naughty_list:
                 Parser.remove(node)
         return doc
 
     def clean_para_spans(self, doc):
         spans = Parser.css_select(doc, 'p > span')
         for item in spans:
-            item.drop_tag()
+            Parser.drop_tag(item)
         return doc
 
     def get_flushed_buffer(self, replacement_text, doc):

goose/extractors.py

@@ -131,7 +131,7 @@ def get_favicon(self, article):
         kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'icon'}
         meta = Parser.getElementsByTag(article.doc, **kwargs)
         if meta:
-            favicon = meta[0].attrib.get('href')
+            favicon = Parser.getAttribute(meta[0], 'href')
             return favicon
         return ''
 
@@ -169,7 +169,7 @@ def get_meta_content(self, doc, metaName):
         content = None
 
         if meta is not None and len(meta) > 0:
-            content = meta[0].attrib.get('content')
+            content = Parser.getAttribute(meta[0], 'content')
 
         if content:
             return content.strip()
@@ -196,7 +196,7 @@ def get_canonical_link(self, article):
             kwargs = {'tag': 'link', 'attr': 'rel', 'value': 'canonical'}
             meta = Parser.getElementsByTag(article.doc, **kwargs)
             if meta is not None and len(meta) > 0:
-                href = meta[0].attrib.get('href')
+                href = Parser.getAttribute(meta[0], 'href')
                 if href:
                     href = href.strip()
                     o = urlparse(href)
@@ -277,10 +277,10 @@ def calculate_best_node(self, article):
             # parent node
             parent_node = Parser.getParent(node)
             self.update_score(parent_node, upscore)
-            self.update_node_count(node.getparent(), 1)
+            self.update_node_count(parent_node, 1)
 
-            if node.getparent() not in parent_nodes:
-                parent_nodes.append(node.getparent())
+            if parent_node not in parent_nodes:
+                parent_nodes.append(parent_node)
 
             # parentparent node
             parent_parent_node = Parser.getParent(parent_node)
@@ -414,24 +414,24 @@ def update_score(self, node, addToScore):
         we're passing in to the current
         """
         current_score = 0
-        score_string = node.attrib.get('gravityScore')
+        score_string = Parser.getAttribute(node, 'gravityScore')
         if score_string:
             current_score = int(score_string)
 
         new_score = current_score + addToScore
-        node.set("gravityScore", str(new_score))
+        Parser.setAttribute(node, "gravityScore", str(new_score))
 
     def update_node_count(self, node, add_to_count):
         """\
         stores how many decent nodes are under a parent node
         """
         current_score = 0
-        count_string = node.attrib.get('gravityNodes')
+        count_string = Parser.getAttribute(node, 'gravityNodes')
         if count_string:
             current_score = int(count_string)
 
         new_score = current_score + add_to_count
-        node.set("gravityNodes", str(new_score))
+        Parser.setAttribute(node, "gravityNodes", str(new_score))
 
     def is_highlink_density(self, e):
         """\
@@ -468,7 +468,7 @@ def get_score(self, node):
         return self.get_node_gravity_score(node) or 0
 
     def get_node_gravity_score(self, node):
-        grvScoreString = node.attrib.get('gravityScore')
+        grvScoreString = Parser.getAttribute(node, 'gravityScore')
         if not grvScoreString:
             return None
         return int(grvScoreString)

goose/outputformatters.py

@@ -79,7 +79,8 @@ def remove_negativescores_nodes(self):
         """
         gravity_items = Parser.css_select(self.top_node, "*[gravityScore]")
         for item in gravity_items:
-            score = int(item.attrib.get('gravityScore'), 0)
+            score = Parser.getAttribute(item, 'gravityScore')
+            score = int(score, 0)
             if score < 1:
                 item.getparent().remove(item)
 

goose/parsers.py

@@ -29,10 +29,19 @@
 
 class Parser(object):
 
+    @classmethod
+    def xpath_re(self, node, expression):
+        regexp_namespace = "http://exslt.org/regular-expressions"
+        items = node.xpath(expression, namespaces={'re': regexp_namespace})
+        return items
+
     @classmethod
     def drop_tag(self, nodes):
-        for node in nodes:
-            node.drop_tag()
+        if isinstance(nodes, list):
+            for node in nodes:
+                node.drop_tag()
+        else:
+            nodes.drop_tag()
 
     @classmethod
     def css_select(self, node, selector):