benchmarks.py

# -*- coding: utf-8 -*-
"""
Async IO vs multi-threading

Multi-thread:           5.9 secs (10 threads) for 100 requests
Async-IO with Gevent:   10.5 secs  for 100 requests
Single thread:          86.0 secs for 100 requests
"""
import sys
import logging
import queue
import os

from threading import activeCount
from threading import Thread
from http.cookiejar import CookieJar as cj
from .unit_tests import read_urls

try:  # Python 2.7+
    from logging import NullHandler
except ImportError:
    class NullHandler(logging.Handler):
        def emit(self, record):
            pass

logging.getLogger(__name__).addHandler(NullHandler())
log = logging.getLogger(__name__)

PARENT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.join(PARENT_DIR, '..'))

from newspaper.network import multithread_request, sync_request
from newspaper.utils import print_duration


@print_duration
def naive_run(urls):
    """no multithreading or async io
    """
    resps = []
    for url in urls:
        resps.append(sync_request(url))
    print(resps)


@print_duration
def mthread_run(urls):
    """download a bunch of urls via multithreading
    """
    reqs = multithread_request(urls)
    resps = [req.resp for req in reqs]


@print_duration
def asyncio_run(urls):
    """download a bunch of urls via async io
    """
    pass
    # rs = (grequests.request('GET', u, **req_kwargs) for u in urls)
    # responses = async_request(urls)
    # print(responses)


def benchmark():
    """multi-threading vs async-io vs regular
    """
    urls = read_urls(amount=1000)
    # naive_run(urls)
    mthread_run(urls)
    # asyncio_run(urls)


if __name__ == '__main__':
    benchmark()