Fixed -e flag

To use dash -e flag, pass url name like www.url.com
and it will try to establish an https connection than http if
the secure conneciton fails. If both fail then an error message is
printed. If the connection is successful than we search for the
onion domain name and the others that were passed with the flag.

Author: KingAkeem

modules/getweblinks.py

@@ -7,28 +7,40 @@
 
 
 def valid_url(url):
-    """Checks if url is valid using regular expression matching
+    """Checks for any valid url using regular expression matching
 
-        Matches all possible url patterns with the url that is passed and returns
-        True if it is a url and returns False if it is not.
+        Matches all possible url patterns with the url that is passed and
+        returns True if it is a url and returns False if it is not.
 
         Args:
             url: string representing url to be checked
 
         Returns:
-            bool: True if valid url format, False if not
+            bool: True if valid url format and False if not
     """
-    regex = re.compile(
-            r'^(?:http|ftp)s?://'  # http:// or https://
-            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # domain...
-            r'localhost|'  # localhost...
-            r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'  # ...or ip
-            r'(?::\d+)?'  # optional port
-            r'(?:/?|[/?]\S+)$', re.IGNORECASE)
 
+    pattern = r"^https?:\/\/(www\.)?([a-z,A-Z,0-9]*)\.([a-z, A-Z]+)(.*)"
+    regex = re.compile(pattern)
     if regex.match(url):
         return True
+    return False
+
+
+def valid_onion_url(url):
+    """Checks for valid onion url using regular expression matching
+
+        Only matches onion urls
 
+        Args:
+            url: string representing url to be checked
+
+        Returns:
+            bool: True if valid onion url format, False if not
+    """
+    pattern = r"^https?:\/\/(www\.)?([a-z,A-Z,0-9]*)\.([a-z,A-Z]+)(.*)"
+    regex = re.compile(pattern)
+    if regex.match(url):
+        return True
     return False
 
 
@@ -68,24 +80,25 @@ def getLinks(soup, ext=False, live=False):
     b_colors = Bcolors()
     if isinstance(soup, BeautifulSoup):
         websites = []
+
         links = soup.find_all('a')
         for ref in links:
             url = ref.get('href')
-            if url and valid_url(url):
-                if ext:
-                    for extension in ext:
-                        if not url.endswith(extension):
-                            continue
-                websites.append(url)
+            if ext:
+                if url and valid_url(url):
+                    websites.append(url)
+            else:
+                if url and valid_onion_url(url):
+                    websites.append(url)
 
         """Pretty print output as below"""
         print(''.join((b_colors.OKGREEN,
               'Websites Found - ', b_colors.ENDC, str(len(websites)))))
         print('------------------------------------')
-        if live:
-            for link in websites:
-                print(next(get_link_status(link, b_colors)))
+
+        for link in websites:
+            print(next(get_link_status(link, b_colors)))
+        return websites
+
     else:
         raise('Method parameter is not of instance BeautifulSoup')
-
-    return websites

modules/pagereader.py

@@ -2,23 +2,52 @@
 
 from bs4 import BeautifulSoup
 from modules.bcolors import Bcolors
+from requests.exceptions import ConnectionError, HTTPError
+from sys import exit
 
 
-def readPage(site):
+def connection_msg(site):
+    yield "Attempting to connect to {site}".format(site=site)
+
+
+def readPage(site, extension=False):
     headers = {'User-Agent':
                'TorBot - Onion crawler | www.github.com/DedSecInside/TorBot'}
     attempts_left = 3
-
-    while (attempts_left):
+    err = " "
+    while attempts_left:
         try:
-            response = requests.get(site, headers=headers)
-            page = BeautifulSoup(response.text, 'html.parser')
-            return page
-        except Exception as e:
+            if not extension:
+                print(next(connection_msg(site)))
+                response = requests.get(site, headers=headers)
+                print("Connection successful.")
+                page = BeautifulSoup(response.text, 'html.parser')
+                return page
+            if extension and attempts_left == 3:
+                print(next(connection_msg('https://'+site)))
+                response = requests.get('https://'+site, headers=headers)
+                print("Connection successful.")
+                page = BeautifulSoup(response.text, 'html.parser')
+                return page
+            if extension and attempts_left == 2:
+                print(next(connection_msg('http://'+site)))
+                response = requests.get('http://'+site, headers=headers)
+                print("Connection successful.")
+                page = BeautifulSoup(response.text, 'html.parser')
+                return page
+            if extension and attempts_left == 1:
+                msg = ''.join(("There has been an {err} while attempting to ",
+                              "connect to {site}.")).format(err=err, site=site)
+                exit(msg)
+
+        except (HTTPError, ConnectionError) as e:
             attempts_left -= 1
-            error = e
+            err = e
 
-    raise error
+    if err == HTTPError:
+        raise("There has been an HTTP error after three attempts.")
+    if err == ConnectionError:
+        raise("There has been a connection error after three attempts.")
 
 
 def get_ip():

torBot.py

@@ -86,8 +86,8 @@ def header():
                 {FAIL} + {BOLD}
                            __  ____  ____  __        ______
                           / /_/ __ \/ __ \/ /_  ____/_  __/
-                         / __/ / / / /_/ / __ \/ __ \/ / 
-                        / /_/ /_/ / _, _/ /_/ / /_/ / /  
+                         / __/ / / / /_/ / __ \/ __ \/ /
+                        / /_/ /_/ / _, _/ /_/ / /_/ / /
                         \__/\____/_/ |_/_____/\____/_/  V{VERSION}
                 {FAIL} + {On_Black}
                 #######################################################
@@ -98,13 +98,14 @@ def header():
                       {FAIL} + "LICENSE: GNU Public License" + {END}""".format(
                 D3DSEC=D3DSEC, INS1DE=INS1DE, FAIL=b_color.FAIL,
                 BOLD=b_color.BOLD, VERSION=__VERSION, END=b_color.ENDC,
-                On_Black=b_color.On_Black,WHITE=b_color.WHITE
+                On_Black=b_color.On_Black, WHITE=b_color.WHITE
                 )
     print(header)
 
 
-def main():
-    connect(LOCALHOST, PORT)
+def main(conn=False):
+    if conn:
+        connect(LOCALHOST, PORT)
     parser = argparse.ArgumentParser()
     parser.add_argument("-v", "--version",
                         action="store_true",
@@ -137,6 +138,13 @@ def main():
                                        "scanned site, (very slow)")))
     args = parser.parse_args()
 
+    link = args.url
+    if '.onion' in link:
+        pass
+    else:
+        print("You must use -e/--extension to use domains other than .onion.")
+        exit()
+
     # If flag is -v, --update, -q/--quiet then user only runs that operation
     # because these are single flags only
     if args.version:
@@ -147,13 +155,11 @@ def main():
         exit()
     if not args.quiet:
         header()
-
     # If url flag is set then check for accompanying flag set. Only one
     # additional flag can be set with -u/--url flag
     if args.url:
         print("Tor IP Address :", pagereader.get_ip())
-        link = args.url
-        html_content = pagereader.readPage(link)
+        html_content = pagereader.readPage(link, args.extension)
         # -m/--mail
         if args.mail:
             emails = getemails.getMails(html_content)
@@ -166,15 +172,9 @@ def main():
             if args.save:
                 print('Nothing to save.\n')
         else:
-            if args.live:
-                live = True
-            else:
-                live = False
-            if args.extension:
-                ext = True
-            else:
-                ext = False
-            links = getweblinks.getLinks(soup=html_content, live=live, ext=ext)
+            links = getweblinks.getLinks(soup=html_content,
+                                         live=args.live,
+                                         ext=args.extension)
             if args.save:
                 savefile.saveJson("Links", links)
     else:
@@ -186,7 +186,7 @@ def main():
 if __name__ == '__main__':
 
     try:
-        main()
+        main(conn=True)
 
     except KeyboardInterrupt:
         print("Interrupt received! Exiting cleanly...")