diff --git a/newspaper/cleaners.py b/newspaper/cleaners.py index 47b6f1a8..6cd1fdc3 100644 --- a/newspaper/cleaners.py +++ b/newspaper/cleaners.py @@ -8,7 +8,6 @@ class DocumentCleaner(object): - def __init__(self, config): """Set appropriate tag names and regexes of tags to remove from the HTML @@ -30,12 +29,11 @@ def __init__(self, config): "|legende|ajoutVideo|timestamp|js_replies" ) self.regexp_namespace = "http://exslt.org/regular-expressions" - self.nauthy_ids_re = ("//*[re:test(@id, '%s', 'i')]" % - self.remove_nodes_re) - self.nauthy_classes_re = ("//*[re:test(@class, '%s', 'i')]" % - self.remove_nodes_re) - self.nauthy_names_re = ("//*[re:test(@name, '%s', 'i')]" % - self.remove_nodes_re) + self.nauthy_ids_re = "//*[re:test(@id, '%s', 'i')]" % self.remove_nodes_re + self.nauthy_classes_re = ( + "//*[re:test(@class, '%s', 'i')]" % self.remove_nodes_re + ) + self.nauthy_names_re = "//*[re:test(@name, '%s', 'i')]" % self.remove_nodes_re self.div_to_p_re = r"<(a|blockquote|dl|div|img|ol|p|pre|table|ul)" self.caption_re = "^caption$" self.google_re = " google " @@ -43,11 +41,31 @@ def __init__(self, config): self.facebook_re = "[^-]facebook" self.facebook_broadcasting_re = "facebook-broadcasting" self.twitter_re = "[^-]twitter" - self.tablines_replacements = ReplaceSequence()\ - .create("\n", "\n\n")\ - .append("\t")\ - .append("^\\s+$") - self.contains_article = './/article|.//*[@id="article"]|.//*[@itemprop="articleBody"]' + self.tablines_replacements = ( + ReplaceSequence().create("\n", "\n\n").append("\t").append("^\\s+$") + ) + self.contains_article_xp = ( + './/article|.//*[@id="article"]|.//*[@itemprop="articleBody"]' + ) + self.contains_article_tag = "article" + self.contains_article_attrs = {"id": ["article"], "itemprop": ["articleBody"]} + + def node_contains_article(self, node): + """ + orignal xpath(self.contains_article) doesn't work when applied directly the node + it look only on the childs whil the tag can be on the node ! + """ + if not len(node) or node is None: + return False + # + for key, value in node.items(): + if value in self.contains_article_attrs.get(key, []): + return True + # + if node.xpath(self.contains_article_xp): + return True + # + return False def clean(self, doc_to_clean): """Remove chunks of the DOM as specified @@ -62,13 +80,14 @@ def clean(self, doc_to_clean): doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.google_re) doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.entries_re) doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.facebook_re) - doc_to_clean = self.remove_nodes_regex(doc_to_clean, - self.facebook_broadcasting_re) + doc_to_clean = self.remove_nodes_regex( + doc_to_clean, self.facebook_broadcasting_re + ) doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.twitter_re) doc_to_clean = self.clean_para_spans(doc_to_clean) - doc_to_clean = self.div_to_para(doc_to_clean, 'div') - doc_to_clean = self.div_to_para(doc_to_clean, 'span') - doc_to_clean = self.div_to_para(doc_to_clean, 'section') + doc_to_clean = self.div_to_para(doc_to_clean, "div") + doc_to_clean = self.div_to_para(doc_to_clean, "span") + doc_to_clean = self.div_to_para(doc_to_clean, "section") return doc_to_clean def clean_body_classes(self, doc): @@ -81,34 +100,35 @@ def clean_body_classes(self, doc): return doc def clean_article_tags(self, doc): - articles = self.parser.getElementsByTag(doc, tag='article') + articles = self.parser.getElementsByTag(doc, tag="article") for article in articles: - for attr in ['id', 'name', 'class']: + for attr in ["id", "name", "class"]: self.parser.delAttribute(article, attr=attr) return doc def clean_em_tags(self, doc): - ems = self.parser.getElementsByTag(doc, tag='em') + ems = self.parser.getElementsByTag(doc, tag="em") for node in ems: - images = self.parser.getElementsByTag(node, tag='img') + images = self.parser.getElementsByTag(node, tag="img") if len(images) == 0: self.parser.drop_tag(node) return doc def remove_drop_caps(self, doc): - items = self.parser.css_select(doc, 'span[class~=dropcap], ' - 'span[class~=drop_cap]') + items = self.parser.css_select( + doc, "span[class~=dropcap], " "span[class~=drop_cap]" + ) for item in items: self.parser.drop_tag(item) return doc def remove_scripts_styles(self, doc): # remove scripts - scripts = self.parser.getElementsByTag(doc, tag='script') + scripts = self.parser.getElementsByTag(doc, tag="script") for item in scripts: self.parser.remove(item) # remove styles - styles = self.parser.getElementsByTag(doc, tag='style') + styles = self.parser.getElementsByTag(doc, tag="style") for item in styles: self.parser.remove(item) # remove comments @@ -122,22 +142,23 @@ def clean_bad_tags(self, doc): # ids naughty_list = self.parser.xpath_re(doc, self.nauthy_ids_re) for node in naughty_list: - if not node.xpath(self.contains_article): + # if not node.xpath(self.contains_article): + if not self.node_contains_article(node): self.parser.remove(node) # class naughty_classes = self.parser.xpath_re(doc, self.nauthy_classes_re) for node in naughty_classes: - if not node.xpath(self.contains_article): + if not self.node_contains_article(node): self.parser.remove(node) # name naughty_names = self.parser.xpath_re(doc, self.nauthy_names_re) for node in naughty_names: - if not node.xpath(self.contains_article): + if not self.node_contains_article(node): self.parser.remove(node) return doc def remove_nodes_regex(self, doc, pattern): - for selector in ['id', 'class']: + for selector in ["id", "class"]: reg = "//*[re:test(@%s, '%s', 'i')]" % (selector, pattern) naughty_list = self.parser.xpath_re(doc, reg) for node in naughty_list: @@ -145,7 +166,7 @@ def remove_nodes_regex(self, doc, pattern): return doc def clean_para_spans(self, doc): - spans = self.parser.css_select(doc, 'p span') + spans = self.parser.css_select(doc, "p span") for item in spans: self.parser.drop_tag(item) return doc @@ -153,34 +174,33 @@ def clean_para_spans(self, doc): def get_flushed_buffer(self, replacement_text, doc): return self.parser.textToPara(replacement_text) - def replace_walk_left_right(self, kid, kid_text, - replacement_text, nodes_to_remove): + def replace_walk_left_right(self, kid, kid_text, replacement_text, nodes_to_remove): kid_text_node = kid replace_text = self.tablines_replacements.replaceAll(kid_text) if len(replace_text) > 1: prev_node = self.parser.previousSibling(kid_text_node) - while prev_node is not None \ - and self.parser.getTag(prev_node) == "a" \ - and self.parser.getAttribute( - prev_node, 'grv-usedalready') != 'yes': + while ( + prev_node is not None + and self.parser.getTag(prev_node) == "a" + and self.parser.getAttribute(prev_node, "grv-usedalready") != "yes" + ): outer = " " + self.parser.outerHtml(prev_node) + " " replacement_text.append(outer) nodes_to_remove.append(prev_node) - self.parser.setAttribute(prev_node, attr='grv-usedalready', - value='yes') + self.parser.setAttribute(prev_node, attr="grv-usedalready", value="yes") prev_node = self.parser.previousSibling(prev_node) replacement_text.append(replace_text) next_node = self.parser.nextSibling(kid_text_node) - while next_node is not None \ - and self.parser.getTag(next_node) == "a" \ - and self.parser.getAttribute( - next_node, 'grv-usedalready') != 'yes': + while ( + next_node is not None + and self.parser.getTag(next_node) == "a" + and self.parser.getAttribute(next_node, "grv-usedalready") != "yes" + ): outer = " " + self.parser.outerHtml(next_node) + " " replacement_text.append(outer) nodes_to_remove.append(next_node) - self.parser.setAttribute(next_node, attr='grv-usedalready', - value='yes') + self.parser.setAttribute(next_node, attr="grv-usedalready", value="yes") next_node = self.parser.nextSibling(next_node) def get_replacement_nodes(self, doc, div): @@ -190,23 +210,23 @@ def get_replacement_nodes(self, doc, div): kids = self.parser.childNodesWithText(div) for kid in kids: # The node is a

and already has some replacement text - if self.parser.getTag(kid) == 'p' and len(replacement_text) > 0: - new_node = self.get_flushed_buffer( - ''.join(replacement_text), doc) + if self.parser.getTag(kid) == "p" and len(replacement_text) > 0: + new_node = self.get_flushed_buffer("".join(replacement_text), doc) nodes_to_return.append(new_node) replacement_text = [] nodes_to_return.append(kid) # The node is a text node elif self.parser.isTextNode(kid): kid_text = self.parser.getText(kid) - self.replace_walk_left_right(kid, kid_text, replacement_text, - nodes_to_remove) + self.replace_walk_left_right( + kid, kid_text, replacement_text, nodes_to_remove + ) else: nodes_to_return.append(kid) # flush out anything still remaining - if(len(replacement_text) > 0): - new_node = self.get_flushed_buffer(''.join(replacement_text), doc) + if len(replacement_text) > 0: + new_node = self.get_flushed_buffer("".join(replacement_text), doc) nodes_to_return.append(new_node) replacement_text = [] @@ -216,14 +236,13 @@ def get_replacement_nodes(self, doc, div): return nodes_to_return def replace_with_para(self, doc, div): - self.parser.replaceTag(div, 'p') + self.parser.replaceTag(div, "p") def div_to_para(self, doc, dom_type): bad_divs = 0 else_divs = 0 divs = self.parser.getElementsByTag(doc, tag=dom_type) - tags = ['a', 'blockquote', 'dl', 'div', 'img', 'ol', 'p', - 'pre', 'table', 'ul'] + tags = ["a", "blockquote", "dl", "div", "img", "ol", "p", "pre", "table", "ul"] for div in divs: items = self.parser.getElementsByTags(div, tags) if div is not None and len(items) == 0: