useless_junk.py

# -*- coding: utf-8 -*-

"""
import os
import glob
from copy import deepcopy
from goose.article import Article
from goose.utils import URLHelper, RawHelper
from goose.extractors import StandardContentExtractor
from goose.cleaners import StandardDocumentCleaner
from goose.outputformatters import StandardOutputFormatter
from goose.images.extractors import UpgradedImageIExtractor
from goose.videos.extractors import VideoExtractor
from goose.network import HtmlFetcher


class CrawlCandidate(object):

    def __init__(self, config, url, raw_html):
        self.config = config
        # parser
        self.parser = self.config.get_parser()
        self.url = url
        self.raw_html = raw_html

class Crawler(object):

    def __init__(self, config):
        self.config = config
        # parser
        self.parser = self.config.get_parser()
        self.logPrefix = "crawler:"

    def crawl(self, crawl_candidate):
        article = Article()

        parse_candidate = self.get_parse_candidate(crawl_candidate)
        raw_html = self.get_html(crawl_candidate, parse_candidate)

        if raw_html is None:
            return article

        doc = self.get_document(raw_html)

        extractor = self.get_extractor()
        document_cleaner = self.get_document_cleaner()
        output_formatter = self.get_output_formatter()

        # article
        article.final_url = parse_candidate.url
        article.link_hash = parse_candidate.link_hash
        article.raw_html = raw_html
        article.doc = doc
        article.raw_doc = deepcopy(doc)
        article.title = extractor.get_title(article)
        # TODO
        # article.publish_date = config.publishDateExtractor.extract(doc)
        # article.additional_data = config.get_additionaldata_extractor.extract(doc)
        article.meta_lang = extractor.get_meta_lang(article)
        article.meta_favicon = extractor.get_favicon(article)
        article.meta_description = extractor.get_meta_description(article)
        article.meta_keywords = extractor.get_meta_keywords(article)
        article.canonical_link = extractor.get_canonical_link(article)
        article.domain = extractor.get_domain(article.final_url)
        article.tags = extractor.extract_tags(article)
        # # before we do any calcs on the body itself let's clean up the document
        article.doc = document_cleaner.clean(article)

        # big stuff
        article.top_node = extractor.calculate_best_node(article)
        if article.top_node is not None:
            # video handeling
            video_extractor = self.get_video_extractor(article)
            video_extractor.get_videos()
            # image handeling
            if self.config.enable_image_fetching:
                image_extractor = self.get_image_extractor(article)
                article.top_image = image_extractor.get_best_image(article.raw_doc, article.top_node)
            # post cleanup
            article.top_node = extractor.post_cleanup(article.top_node)
            # clean_text
            article.cleaned_text = output_formatter.get_formatted_text(article)

        # cleanup tmp file
        self.relase_resources(article)

        return article

    def get_parse_candidate(self, crawl_candidate):
        if crawl_candidate.raw_html:
            return RawHelper.get_parsing_candidate(crawl_candidate.url, crawl_candidate.raw_html)
        return URLHelper.get_parsing_candidate(crawl_candidate.url)

    def get_html(self, crawl_candidate, parsing_candidate):
        if crawl_candidate.raw_html:
            return crawl_candidate.raw_html
        # fetch HTML
        html = HtmlFetcher().get_html(self.config, parsing_candidate.url)
        return html

    def get_image_extractor(self, article):
        http_client = None
        return UpgradedImageIExtractor(http_client, article, self.config)

    def get_video_extractor(self, article):
        return VideoExtractor(article, self.config)

    def get_output_formatter(self):
        return StandardOutputFormatter(self.config)

    def get_document_cleaner(self):
        return StandardDocumentCleaner(self.config)

    def get_document(self, raw_html):
        doc = self.parser.fromstring(raw_html)
        return doc

    def get_extractor(self):
        return StandardContentExtractor(self.config)

    def relase_resources(self, article):
        path = os.path.join(self.config.local_storage_path, '%s_*' % article.link_hash)
        for fname in glob.glob(path):
            try:
                os.remove(fname)
            except OSError:
                # TODO better log handeling
                pass

if method == 'soup':
    safe_img = (doc.find('meta', attrs={'property':'og:image'})
            or doc.find('meta', attrs={'name':'og:image'}))
    if safe_img:
        safe_img = safe_img.get('content')

    if not safe_img:
        safe_img = (doc.find('link', attrs={'rel':'img_src'})
                or doc.find('link', attrs={'rel':'icon'}))

    if safe_img:
        safe_img = safe_img.get('content')

    if not safe_img:
        safe_img = ''

            return safe_img

def fix_unicode(inputstr):

    if inputstr is None:
        return u''

    if not isinstance(inputstr, unicode):
        try:
            inputstr = inputstr.decode('utf8', errors='ignore')
        except ValueError, e:
            log.debug(e)
            inputstr = u''

    inputstr = inputstr.strip()
    return inputstr

if method == 'soup':
    img_tags = doc.findAll('img')
    img_links = [i.get('src') for i in img_tags if i.get('src')]

all_tuples = []
for feed in self.feeds:
    dom = feed.dom
    if dom.get('entries'):
        ll = dom['entries']
        tuples = [(l['link'], l['title']) for l in ll
                        if l.get('link') and l.get('title')]
        all_tuples.extend(tuples)

"""

"""
class GrequestsTestCase(unittest.TestCase):
    def runTest(self):
        print 'testing grequests unit'
        #self.test_ordering()
        self.test_capacity()

    @print_test
    def test_ordering(self):
        TEST_SIZE = 25
        dd = {}
        urls = read_urls(amount=TEST_SIZE)

        # don't count feeds, they always redirect to some other url
        urls = [u for u in urls if 'feeds' not in urlparse.urlparse(u).netloc.split('.')]

        for index, url in enumerate(urls):
            _ul = urlparse.urlparse(url)
            normalized = _ul.netloc + _ul.path
            dd[index] = normalized

        responses = async_request(urls, timeout=3)
        for index, resp in enumerate(responses):
            _ul = urlparse.urlparse(resp.url)
            normalized = _ul.netloc + _ul.path
            # print dd[index], '==', normalized
            assert dd[index] == normalized

    @print_test
    def test_capacity(self):
        TEST_SIZE = 450
        urls = read_urls(amount=TEST_SIZE)
        responses = async_request(urls, timeout=3)
        failed = 0
        for index, r in enumerate(responses):
            if r is not None:
                pass
            else:
                #print '[FAIL]', urls[index]
                failed += 1
        print '\t\ttotal:', len(urls), 'failed', failed

"""