""" This module is used for reading HTML pages using either bs4.BeautifulSoup objects or url strings """ from bs4 import BeautifulSoup from .utils import get_url_status from .color import color def read(link, *, response=False, show_msg=False, headers=None, schemes=None): """ Attempts to retrieve HTML from link Args: headers (dict) schemes (list) Returns: resp.text (str): html from page """ headers = {'User-Agent': 'XXXX-XXXXX-XXXX'} if not headers else headers # Attempts to connect directly to site if no scheme is passed if not schemes: if show_msg: print(f"Attempting to connect to {link}") resp = get_url_status(link, headers) if resp != 0: if response: return resp.text, resp return resp.text schemes = ['https://', 'http://'] if not schemes else schemes for scheme in schemes: temp_url = scheme + link if show_msg: print(f"Attempting to connect to {link}") resp = get_url_status(temp_url, headers) if resp != 0: if response: return resp.text, resp return resp.text raise ConnectionError def display(link): """ Prints the status of a link """ resp = get_url_status(link) if resp != 0: try: title = BeautifulSoup(resp.text, 'html.parser').title.string link_status = color(link, 'green') except AttributeError: title = "Not Found" link_status = color(link, 'red') else: title = "Not Found" link_status = color(link, 'red') print("%-80s %-30s" % (link_status, title)) def display_ip(): """Returns users tor ip address https://check.torproject.org/ tells you if you are using tor and it displays your IP address which we scape and return """ page = read('https://check.torproject.org/') page = BeautifulSoup(page, 'html.parser') ip_cont = page.find('strong') ip_addr = ip_cont.renderContents() ip_string = color(ip_addr.decode("utf-8"), 'yellow') print(f'Tor IP Address: {ip_string}')