"""
This module is used for reading HTML pages using either bs4.BeautifulSoup
objects or url strings
"""
import requests.exceptions
from bs4 import BeautifulSoup
from .link import LinkNode
from .utils import multi_thread
from .color import color
class LinkIO:
"""
Class to interact with and interrogate links.
"""
@staticmethod
def display_children(root):
"""
Static method to display status of child nodes.
Args:
root (LinkNode): root of children to be displayed.
"""
sucess_msg = color(f'Links Found - {len(root.links)}', 'green')
print(sucess_msg + '\n' + '---------------------------------')
multi_thread(root.links, LinkIO.display)
@staticmethod
def read(link, *, response=False, show_msg=False, headers=None, schemes=None):
"""
Attempts to retrieve HTML from link.
Args:
link (str): Link to read.
response (bool): Determines if response is returned.
show_msg (bool): Determines if message is displayed for connection.
headers (dict): Header for request, defaults to None.
schemes (list): Different schemes to attempt to use.
Returns:
str: HTML from page.
requests.Response (optional): Response returned from requests.
"""
headers = {'User-Agent': 'XXXX-XXXXX-XXXX'} if not headers else headers
# Attempts to connect directly to site if no scheme is passed
if not schemes:
if show_msg:
print(f'Attempting to connect to {link}')
if LinkNode.valid_link(link):
node = LinkNode(link)
if response:
return node.response.text, node.response
return node.response.text
schemes = ['https://', 'http://'] if not schemes else schemes
# Attempt to use different schemes until one is successful
for scheme in schemes:
temp_url = scheme + link
if show_msg:
print(f'Attempting to connect to {link}')
if LinkNode.valid_link(temp_url):
node = LinkNode(temp_url)
if response:
return node.response.text, node.response
return node.response.text
raise ConnectionError
@staticmethod
def display(link):
"""
Prints the status of a link based on it's connection status.
Args:
link (str): Link to return status of.
"""
if LinkNode.valid_link(link):
try:
node = LinkNode(link)
title = node.name
link_status = node.status
except (requests.exceptions.HTTPError,
requests.exceptions.ConnectionError,
ConnectionError):
title = 'Not Found'
link_status = color(link, 'red')
status_msg = "%-80s %-30s" % (link_status, title)
print(status_msg)
@staticmethod
def display_ip():
"""
Uses https://check.torproject.org/ to determine if you
are using Tor which is then scraped and displayed.
"""
page = LinkIO.read('https://check.torproject.org/', show_msg=True)
page = BeautifulSoup(page, 'html.parser')
ip_cont = page.find('strong')
ip_addr = ip_cont.renderContents()
ip_string = color(ip_addr.decode("utf-8"), 'yellow')
print(f'Tor IP Address: {ip_string}')