diff --git a/search_engine_parser/core/utils.py b/search_engine_parser/core/utils.py index f7211e7..ac77596 100644 --- a/search_engine_parser/core/utils.py +++ b/search_engine_parser/core/utils.py @@ -2,11 +2,23 @@ import random import pickle import hashlib +import logging import aiohttp from fake_useragent import UserAgent FILEPATH = os.path.dirname(os.path.abspath(__file__)) +# fake-useragent already recovers internally when its cache server is +# unreachable or returns unexpected data (it falls back to querying the +# cache server for raw JSON, see its utils.py:load()), but it logs that +# recovery as a warning with a full traceback attached (exc_info=exc). +# get_rand_user_agent() below is a second layer of defense on top of +# that, so the traceback is just noise that led users to file bug +# reports for something that was never actually failing (see +# bisohns/search-engine-parser#159). Scoped to fake_useragent's own +# logger only, so we don't hide warnings from anything else. +logging.getLogger('fake_useragent').setLevel(logging.ERROR) + # prevent caching USER_AGENT_LIST = [ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:11.0) Gecko/20100101 Firefox/11.0", diff --git a/search_engine_parser/tests/test_utils.py b/search_engine_parser/tests/test_utils.py new file mode 100644 index 0000000..3f588d6 --- /dev/null +++ b/search_engine_parser/tests/test_utils.py @@ -0,0 +1,63 @@ +import logging +import unittest +from unittest.mock import patch, MagicMock, PropertyMock + +from search_engine_parser.core.utils import get_rand_user_agent, USER_AGENT_LIST + +FAKE_USERAGENT_LOGGER = logging.getLogger('fake_useragent') + + +class ListHandler(logging.Handler): + """ Collects emitted log records for later assertions """ + + def __init__(self): + super().__init__() + self.records = [] + + def emit(self, record): + self.records.append(record) + + +def raise_index_error(*args, **kwargs): + # Mirrors fake-useragent's own internal behaviour when its cache + # server is unreachable or returns unexpected data: it logs a + # WARNING with a full traceback attached (exc_info) before falling + # back, see fake_useragent/utils.py:load() in the fake-useragent + # package itself. + FAKE_USERAGENT_LOGGER.warning( + 'Error occurred during loading data. ' + 'Trying to use cache server %s', + 'https://fake-useragent.herokuapp.com', + exc_info=IndexError('list index out of range'), + ) + raise IndexError('list index out of range') + + +class UtilsTest(unittest.TestCase): + + @patch('search_engine_parser.core.utils.UserAgent') + def test_get_rand_user_agent_falls_back_and_suppresses_warning(self, mock_user_agent_class): + """ + get_rand_user_agent() must fall back to USER_AGENT_LIST when + fake_useragent fails internally (bisohns/search-engine-parser#159), + and the fake_useragent logger must not surface a WARNING-or-above + record while it does so. + """ + mock_instance = MagicMock() + type(mock_instance).random = PropertyMock(side_effect=raise_index_error) + mock_user_agent_class.return_value = mock_instance + + handler = ListHandler() + FAKE_USERAGENT_LOGGER.addHandler(handler) + try: + result = get_rand_user_agent() + finally: + FAKE_USERAGENT_LOGGER.removeHandler(handler) + + warning_or_above = [r for r in handler.records if r.levelno >= logging.WARNING] + self.assertEqual( + warning_or_above, [], + "fake_useragent logger emitted a WARNING-or-above record; " + "utils.py's suppression should prevent this (see #159)" + ) + self.assertIn(result, USER_AGENT_LIST)