diff --git a/newspaper/__init__.py b/newspaper/__init__.py index eaef8855..b49c2486 100644 --- a/newspaper/__init__.py +++ b/newspaper/__init__.py @@ -12,6 +12,9 @@ popular_urls, NewsPool, Configuration as Config) from .source import Source from .version import __version__ +from .extractors import ContentExtractor +from .parsers import Parser +from .network import get_html news_pool = NewsPool() diff --git a/newspaper/article.py b/newspaper/article.py index 5fe75d88..a92cd8bd 100644 --- a/newspaper/article.py +++ b/newspaper/article.py @@ -33,14 +33,17 @@ class ArticleException(Exception): class Article(object): """Article objects abstract an online news article page """ - def __init__(self, url, title='', source_url='', config=None, **kwargs): + def __init__(self, url, title='', source_url='', config=None, content_extractor=None, **kwargs): """The **kwargs argument may be filled with config values, which is added into the config object """ self.config = config or Configuration() self.config = extend_config(self.config, kwargs) - self.extractor = ContentExtractor(self.config) + if content_extractor is None: + self.extractor = ContentExtractor(self.config) + else: + self.extractor = content_extractor if source_url == '': source_url = urls.get_scheme(url) + '://' + urls.get_domain(url) @@ -212,6 +215,7 @@ def parse(self): text = '' self.top_node = self.extractor.calculate_best_node(self.doc) + #self.top_node = document_cleaner.clean(self.top_node) if self.top_node is not None: video_extractor = VideoExtractor(self.config, self.top_node) self.set_movies(video_extractor.get_videos()) diff --git a/newspaper/cleaners.py b/newspaper/cleaners.py index 5de87b06..70f26dfc 100644 --- a/newspaper/cleaners.py +++ b/newspaper/cleaners.py @@ -64,7 +64,7 @@ def clean(self, doc_to_clean): self.facebook_braodcasting_re) doc_to_clean = self.remove_nodes_regex(doc_to_clean, self.twitter_re) doc_to_clean = self.clean_para_spans(doc_to_clean) - doc_to_clean = self.div_to_para(doc_to_clean, 'div') + #doc_to_clean = self.div_to_para(doc_to_clean, 'div') doc_to_clean = self.div_to_para(doc_to_clean, 'span') return doc_to_clean diff --git a/newspaper/outputformatters.py b/newspaper/outputformatters.py index c5726717..2e8ade4f 100644 --- a/newspaper/outputformatters.py +++ b/newspaper/outputformatters.py @@ -54,6 +54,7 @@ def get_formatted(self, top_node): self.remove_trailing_media_div() text = self.convert_to_text() # print(self.parser.nodeToString(self.get_top_node())) + #print(text) return (text, html) def convert_to_text(self):