From 03d59ee60a303fcc9ec8561e3e23f05289369a8f Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Tue, 25 Jul 2017 10:49:41 +0200 Subject: [PATCH 01/19] [auth] Translate auth module to python 3 --- securitybot/auth/auth.py | 38 ++++++++++++++++------------- securitybot/auth/duo.py | 52 +++++++++++++++++++--------------------- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/securitybot/auth/auth.py b/securitybot/auth/auth.py index b70ec3f..8478174 100644 --- a/securitybot/auth/auth.py +++ b/securitybot/auth/auth.py @@ -4,19 +4,18 @@ __author__ = 'Alex Bertsch' __email__ = 'abertsch@dropbox.com' -from securitybot.util import enum from datetime import timedelta from abc import ABCMeta, abstractmethod +from enum import Enum, unique -AUTH_STATES = enum('NONE', - 'PENDING', - 'AUTHORIZED', - 'DENIED', -) -# Allowable time before 2FA is checked again. -# Ideally this should be as low as possible without being annoying. -AUTH_TIME = timedelta(hours=2) +@unique +class AuthState(Enum): + NONE = 1 + PENDING = 2 + AUTHORIZED = 3 + DENIED = 4 + class Auth(object): ''' @@ -26,8 +25,16 @@ class Auth(object): __metaclass__ = ABCMeta @abstractmethod - def can_auth(self): - # type: () -> bool + def __init__(self, config: dict=None, **kwargs): + ''' + Initialise default values for global config + ''' + if dict is None: + return + self.auth_time = config.get('auth_time', 7200) + + @abstractmethod + def can_auth(self) -> bool: ''' Returns: (bool) Whether 2FA is available. @@ -35,8 +42,7 @@ def can_auth(self): pass @abstractmethod - def auth(self, reason=None): - # type: (str) -> None + def auth(self, reason: str=None) -> None: ''' Begins an authorization request, which should be non-blocking. @@ -46,8 +52,7 @@ def auth(self, reason=None): pass @abstractmethod - def auth_status(self): - # type: () -> int + def auth_status(self) -> int: ''' Returns: (enum) The current auth status, one of AUTH_STATES. @@ -55,8 +60,7 @@ def auth_status(self): pass @abstractmethod - def reset(self): - # type: () -> None + def reset(self) -> None: ''' Resets auth status. ''' diff --git a/securitybot/auth/duo.py b/securitybot/auth/duo.py index 3721651..ec7100e 100644 --- a/securitybot/auth/duo.py +++ b/securitybot/auth/duo.py @@ -1,33 +1,33 @@ ''' Authentication using Duo. ''' -__author__ = 'Alex Bertsch' -__email__ = 'abertsch@dropbox.com' +__author__ = 'Alex Bertsch, Antoine Cardon' +__email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' import logging from datetime import datetime -from urllib import urlencode -from securitybot.auth.auth import Auth, AUTH_STATES, AUTH_TIME +from urllib.parse import urlencode +from securitybot.auth.auth import Auth, AuthState +from typing import Callable -from typing import Any class DuoAuth(Auth): - def __init__(self, duo_api, username): - # type: (Any, str) -> None + + def __init__(self, config: dict=None, duo_api: Callable=None) -> None: ''' Args: duo_api (duo_client.Auth): An Auth API client from Duo. username (str): The username of the person authorized through this object. ''' - self.client = duo_api - self.username = username - self.txid = None # type: str + super().__init__() + self.client: Callable = duo_api + self.username: str = config.get('duo', {}).get('username', "") + self.txid: str = None self.auth_time = datetime.min - self.state = AUTH_STATES.NONE + self.state = AuthState.NONE - def can_auth(self): - # type: () -> bool + def can_auth(self) -> bool: # Use Duo preauth to look for a device with Push # TODO: This won't work for anyone who's set to auto-allow, but # I don't believe we have anyone like that... @@ -39,8 +39,7 @@ def can_auth(self): return True return False - def auth(self, reason=None): - # type: (str) -> None + def auth(self, reason: str=None) -> None: logging.debug('Sending Duo Push request for {}'.format(self.username)) pushinfo = 'from=Securitybot' if reason: @@ -56,29 +55,26 @@ def auth(self, reason=None): pushinfo=pushinfo ) self.txid = res['txid'] - self.state = AUTH_STATES.PENDING + self.state = AuthState.PENDING - def _recently_authed(self): - # type: () -> bool + def _recently_authed(self) -> bool: return (datetime.now() - self.auth_time) < AUTH_TIME - def auth_status(self): - # type: () -> int - if self.state == AUTH_STATES.PENDING: + def auth_status(self) -> int: + if self.state == AuthState.PENDING: res = self.client.auth_status(self.txid) if not res['waiting']: if res['success']: - self.state = AUTH_STATES.AUTHORIZED + self.state = AuthState.AUTHORIZED self.auth_time = datetime.now() else: - self.state = AUTH_STATES.DENIED + self.state = AuthState.DENIED self.auth_time = datetime.min - elif self.state == AUTH_STATES.AUTHORIZED: + elif self.state == AuthState.AUTHORIZED: if not self._recently_authed(): - self.state = AUTH_STATES.NONE + self.state = AuthState.NONE return self.state - def reset(self): - # type: () -> None + def reset(self) -> None: self.txid = None - self.state = AUTH_STATES.NONE + self.state = AuthState.NONE From 10a1f6bfa7f653fa75381d3b26bb033e9ec3a888 Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Tue, 25 Jul 2017 10:50:18 +0200 Subject: [PATCH 02/19] [auth] Add a dummy null auth authenticator. Always returns True. --- securitybot/auth/nullauth.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 securitybot/auth/nullauth.py diff --git a/securitybot/auth/nullauth.py b/securitybot/auth/nullauth.py new file mode 100644 index 0000000..5ac5feb --- /dev/null +++ b/securitybot/auth/nullauth.py @@ -0,0 +1,29 @@ +''' +Dummy authenticator which returns always True. +''' +__author__ = 'Antoine Cardon' +__email__ = 'antoine.cardon@algolia.com' + +from securitybot.auth.auth import Auth + + +class NullAuth(Auth): + + def __init__(self) -> None: + pass + + def can_auth(self) -> bool: + return False + + def auth(self, reason: str=None) -> None: + pass + + def _recently_authed(self) -> str: + # type: () -> bool + return 'AUTHORIZED' + + def auth_status(self) -> bool: + return True + + def reset(self) -> None: + pass From 9c8a75ef496410a7b03c8751d891a4599341a360 Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Tue, 25 Jul 2017 10:52:47 +0200 Subject: [PATCH 03/19] [auth] auth_time is now a timedelta again --- securitybot/auth/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/securitybot/auth/auth.py b/securitybot/auth/auth.py index 8478174..d581b5e 100644 --- a/securitybot/auth/auth.py +++ b/securitybot/auth/auth.py @@ -31,7 +31,7 @@ def __init__(self, config: dict=None, **kwargs): ''' if dict is None: return - self.auth_time = config.get('auth_time', 7200) + self.auth_time = timedelta(seconds=config.get('auth_time', 7200)) @abstractmethod def can_auth(self) -> bool: From 0dc17e309a3374babaed5ff5965acf734e2e8118 Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Thu, 27 Jul 2017 17:14:06 +0200 Subject: [PATCH 04/19] Huge and messy commit: Port to python3, Generic database driver in progress, config in yaml file. Working. --- config/bot.yaml | 33 +++++++-- main.py | 34 ++++----- securitybot/auth/auth.py | 19 +++-- securitybot/auth/duo.py | 22 +++--- securitybot/auth/nullauth.py | 4 +- securitybot/blacklist/blacklist.py | 14 ++-- securitybot/blacklist/sql_blacklist.py | 15 ++-- securitybot/bot.py | 99 ++++++++------------------ securitybot/chat/chat.py | 25 +++---- securitybot/chat/slack.py | 48 ++++++------- securitybot/commands.py | 14 +++- securitybot/config.py | 33 +++++++++ securitybot/db/__init__.py | 3 + securitybot/db/engine.py | 61 ++++++++++++++++ securitybot/db/sql.py | 78 ++++++++++++++++++++ securitybot/ignored_alerts.py | 21 +++--- securitybot/sql.py | 93 ------------------------ securitybot/tasker/sql_tasker.py | 45 +++++++----- securitybot/tasker/tasker.py | 27 ++++--- securitybot/user.py | 26 ++++--- securitybot/util.py | 29 ++++---- util/db_up.py | 23 +++--- 22 files changed, 425 insertions(+), 341 deletions(-) create mode 100644 securitybot/config.py create mode 100644 securitybot/db/__init__.py create mode 100644 securitybot/db/engine.py create mode 100644 securitybot/db/sql.py delete mode 100644 securitybot/sql.py diff --git a/config/bot.yaml b/config/bot.yaml index 9a953ac..b08a77b 100644 --- a/config/bot.yaml +++ b/config/bot.yaml @@ -1,4 +1,29 @@ ---- -messages_path: config/messages.yaml -commands_path: config/commands.yaml -... +includes: + messages_path: config/messages.yaml + commands_path: config/commands.yaml +database: + type: sql + engine: SQLEngine + user: root + password: password + host: host.example.com + db: securitybot +user: + fetcher: + name: ldap + options: + username: 'user' + password: changeme + host: ldap.example.com +duo: + ikey: 'IKEY' + skey: 'SKEY' + endpoint: 'XXXXXX.duosecurity.com' +slack: + username: 'username' + token: 'TOKEN' + reporting_channel: 'ChannelID' + icon_url: '' +debug: + level: 'debug' + diff --git a/main.py b/main.py index 9d81a72..7c42bde 100644 --- a/main.py +++ b/main.py @@ -5,40 +5,42 @@ from securitybot.chat.slack import Slack from securitybot.tasker.sql_tasker import SQLTasker from securitybot.auth.duo import DuoAuth -from securitybot.sql import init_sql +from securitybot.db.engine import DbEngine import duo_client +from securitybot.config import config -CONFIG = {} -SLACK_KEY = 'slack_api_token' -DUO_INTEGRATION = 'duo_integration_key' -DUO_SECRET = 'duo_secret_key' -DUO_ENDPOINT = 'duo_endpoint' -REPORTING_CHANNEL = 'some_slack_channel_id' -ICON_URL = 'https://dl.dropboxusercontent.com/s/t01pwfrqzbz3gzu/securitybot.png' def init(): # Setup logging + config.load_config('config/bot.yaml') logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s %(levelname)s] %(message)s') logging.getLogger('requests').setLevel(logging.WARNING) logging.getLogger('usllib3').setLevel(logging.WARNING) + def main(): init() - init_sql() + # init_sql() - # Create components needed for Securitybot + # Create components needed for SecurityBot duo_api = duo_client.Auth( - ikey=DUO_INTEGRATION, - skey=DUO_SECRET, - host=DUO_ENDPOINT + ikey=config['duo']['ikey'], + skey=config['duo']['skey'], + host=config['duo']['endpoint'] ) duo_builder = lambda name: DuoAuth(duo_api, name) - - chat = Slack('securitybot', SLACK_KEY, ICON_URL) + try: + # Initialise DbEngine here + DbEngine(config['database']) + except KeyError: + logging.error('No database configuration') + raise + + chat = Slack(config['slack']) tasker = SQLTasker() - sb = SecurityBot(chat, tasker, duo_builder, REPORTING_CHANNEL, 'config/bot.yaml') + sb = SecurityBot(chat, tasker, duo_builder, config['slack']['reporting_channel']) sb.run() if __name__ == '__main__': diff --git a/securitybot/auth/auth.py b/securitybot/auth/auth.py index d581b5e..9770a1b 100644 --- a/securitybot/auth/auth.py +++ b/securitybot/auth/auth.py @@ -1,8 +1,8 @@ ''' An authentication object for doing 2FA on Slack users. ''' -__author__ = 'Alex Bertsch' -__email__ = 'abertsch@dropbox.com' +__author__ = 'Alex Bertsch, Antoine Cardon' +__email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' from datetime import timedelta from abc import ABCMeta, abstractmethod @@ -10,26 +10,25 @@ @unique -class AuthState(Enum): +class AuthStates(Enum): NONE = 1 PENDING = 2 AUTHORIZED = 3 DENIED = 4 -class Auth(object): +class Auth(object, metaclass=ABCMeta): ''' When designing Auth subclasses, try to make sure that the authorization attempt is as non-blocking as possible. ''' - __metaclass__ = ABCMeta @abstractmethod def __init__(self, config: dict=None, **kwargs): ''' Initialise default values for global config ''' - if dict is None: + if config is None: return self.auth_time = timedelta(seconds=config.get('auth_time', 7200)) @@ -39,7 +38,7 @@ def can_auth(self) -> bool: Returns: (bool) Whether 2FA is available. ''' - pass + raise NotImplementedError() @abstractmethod def auth(self, reason: str=None) -> None: @@ -49,7 +48,7 @@ def auth(self, reason: str=None) -> None: Args: reason (str): Optional reason string that may be provided ''' - pass + raise NotImplementedError() @abstractmethod def auth_status(self) -> int: @@ -57,11 +56,11 @@ def auth_status(self) -> int: Returns: (enum) The current auth status, one of AUTH_STATES. ''' - pass + raise NotImplementedError() @abstractmethod def reset(self) -> None: ''' Resets auth status. ''' - pass + raise NotImplementedError() diff --git a/securitybot/auth/duo.py b/securitybot/auth/duo.py index ec7100e..dbb9a2a 100644 --- a/securitybot/auth/duo.py +++ b/securitybot/auth/duo.py @@ -7,13 +7,13 @@ import logging from datetime import datetime from urllib.parse import urlencode -from securitybot.auth.auth import Auth, AuthState +from securitybot.auth.auth import Auth, AuthStates from typing import Callable class DuoAuth(Auth): - def __init__(self, config: dict=None, duo_api: Callable=None) -> None: + def __init__(self, duo_api: Callable=None, username="") -> None: ''' Args: duo_api (duo_client.Auth): An Auth API client from Duo. @@ -22,10 +22,10 @@ def __init__(self, config: dict=None, duo_api: Callable=None) -> None: ''' super().__init__() self.client: Callable = duo_api - self.username: str = config.get('duo', {}).get('username', "") + self.username: str = username self.txid: str = None self.auth_time = datetime.min - self.state = AuthState.NONE + self.state = AuthStates.NONE def can_auth(self) -> bool: # Use Duo preauth to look for a device with Push @@ -55,26 +55,26 @@ def auth(self, reason: str=None) -> None: pushinfo=pushinfo ) self.txid = res['txid'] - self.state = AuthState.PENDING + self.state = AuthStates.PENDING def _recently_authed(self) -> bool: return (datetime.now() - self.auth_time) < AUTH_TIME def auth_status(self) -> int: - if self.state == AuthState.PENDING: + if self.state == AuthStates.PENDING: res = self.client.auth_status(self.txid) if not res['waiting']: if res['success']: - self.state = AuthState.AUTHORIZED + self.state = AuthStates.AUTHORIZED self.auth_time = datetime.now() else: - self.state = AuthState.DENIED + self.state = AuthStates.DENIED self.auth_time = datetime.min - elif self.state == AuthState.AUTHORIZED: + elif self.state == AuthStates.AUTHORIZED: if not self._recently_authed(): - self.state = AuthState.NONE + self.state = AuthStates.NONE return self.state def reset(self) -> None: self.txid = None - self.state = AuthState.NONE + self.state = AuthStates.NONE diff --git a/securitybot/auth/nullauth.py b/securitybot/auth/nullauth.py index 5ac5feb..0806bdd 100644 --- a/securitybot/auth/nullauth.py +++ b/securitybot/auth/nullauth.py @@ -4,7 +4,7 @@ __author__ = 'Antoine Cardon' __email__ = 'antoine.cardon@algolia.com' -from securitybot.auth.auth import Auth +from securitybot.auth.auth import Auth, AuthState class NullAuth(Auth): @@ -20,7 +20,7 @@ def auth(self, reason: str=None) -> None: def _recently_authed(self) -> str: # type: () -> bool - return 'AUTHORIZED' + return AuthState.AUTHORIZED def auth_status(self) -> bool: return True diff --git a/securitybot/blacklist/blacklist.py b/securitybot/blacklist/blacklist.py index 175f853..59caa49 100644 --- a/securitybot/blacklist/blacklist.py +++ b/securitybot/blacklist/blacklist.py @@ -1,13 +1,13 @@ ''' A generic blacklist class. ''' -__author__ = 'Alex Bertsch' -__email__ = 'abertsch@dropbox.com' +__author__ = 'Alex Bertsch, Antoine Cardon' +__email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' from abc import ABCMeta, abstractmethod -class Blacklist(object): - __metaclass__ = ABCMeta + +class Blacklist(object, metaclass=ABCMeta): @abstractmethod def is_present(self, name): @@ -18,7 +18,7 @@ def is_present(self, name): Args: name (str): The name to check. ''' - pass + raise NotImplementedError() @abstractmethod def add(self, name): @@ -29,7 +29,7 @@ def add(self, name): Args: name (str): The name to add to the blacklist. ''' - pass + raise NotImplementedError() @abstractmethod def remove(self, name): @@ -40,4 +40,4 @@ def remove(self, name): Args: name (str): The name to remove from the blacklist. ''' - pass + raise NotImplementedError() diff --git a/securitybot/blacklist/sql_blacklist.py b/securitybot/blacklist/sql_blacklist.py index 3a8d629..596b4d8 100644 --- a/securitybot/blacklist/sql_blacklist.py +++ b/securitybot/blacklist/sql_blacklist.py @@ -1,20 +1,23 @@ ''' A MySQL-based blacklist class. ''' -__author__ = 'Alex Bertsch' -__email__ = 'abertsch@dropbox.com' +__author__ = 'Alex Bertsch, Antoine Cardon' +__email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' from securitybot.blacklist.blacklist import Blacklist -from securitybot.sql import SQLEngine +from securitybot.db.engine import DbEngine + class SQLBlacklist(Blacklist): + def __init__(self): # type: () -> None ''' Creates a new blacklist tied to a table named "blacklist". ''' # Load from table - names = SQLEngine.execute('SELECT * FROM blacklist') + self._db_engine = DbEngine() + names = self._db_engine.execute('SELECT * FROM blacklist') # Break tuples into names self._blacklist = {name[0] for name in names} @@ -37,7 +40,7 @@ def add(self, name): name (str): The name to add to the blacklist. ''' self._blacklist.add(name) - SQLEngine.execute('INSERT INTO blacklist (ldap) VALUES (%s)', (name,)) + self._db_engine.execute('INSERT INTO blacklist (ldap) VALUES (%s)', (name,)) def remove(self, name): # type: (str) -> None @@ -48,4 +51,4 @@ def remove(self, name): name (str): The name to remove from the blacklist. ''' self._blacklist.remove(name) - SQLEngine.execute('DELETE FROM blacklist WHERE ldap = %s', (name,)) + self._db_engine.execute('DELETE FROM blacklist WHERE ldap = %s', (name,)) diff --git a/securitybot/bot.py b/securitybot/bot.py index 4f3ed90..64e55b7 100644 --- a/securitybot/bot.py +++ b/securitybot/bot.py @@ -2,8 +2,8 @@ The internals of securitybot. Defines a core class SecurityBot that manages most of the bot's behavior. ''' -__author__ = 'Alex Bertsch' -__email__ = 'abertsch@dropbox.com' +__author__ = 'Alex Bertsch, Antoine Cardon' +__email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' import logging from securitybot.user import User @@ -11,16 +11,12 @@ from datetime import datetime, timedelta import pytz import shlex -import yaml -import string - +from re import sub import securitybot.commands as bot_commands from securitybot.blacklist.sql_blacklist import SQLBlacklist -from securitybot.chat.chat import Chat -from securitybot.tasker.tasker import Task, Tasker -from securitybot.auth.auth import Auth +from securitybot.config import config -from typing import Any, Callable, Dict, List, Tuple +# from typing import Any, Callable, Dict, List, Tuple TASK_POLL_TIME = timedelta(minutes=1) REPORTING_TIME = timedelta(hours=1) @@ -34,6 +30,7 @@ 'failure_msg': None, } + def clean_input(text): # type: (unicode) -> str ''' @@ -43,32 +40,32 @@ def clean_input(text): # smart quote, as happens with auto-formatting. text = (text.replace(u'\u2018', '\'') .replace(u'\u2019', '\'') - .replace(u'\u201c','"') + .replace(u'\u201c', '"') .replace(u'\u201d', '"')) # Undo autoformatting of dashes text = (text.replace(u'\u2013', '--') .replace(u'\u2014', '--')) - return text.encode('utf-8') + return text + +PUNCTUATION = r'[.,!?\'"`]' -PUNCTUATION = '.,!?\'"`' def clean_command(command): # type: (str) -> str '''Cleans a command.''' command = command.lower() - # Force to str - command = command.encode('utf-8') # Remove punctuation people are likely to use and won't interfere with command names - command = command.translate(string.maketrans('', ''), PUNCTUATION) + command = sub(PUNCTUATION, '', command) return command + class SecurityBot(object): ''' It's always dangerous naming classes the same name as the project... ''' - def __init__(self, chat, tasker, auth_builder, reporting_channel, config_path): + def __init__(self, chat, tasker, auth_builder, reporting_channel): # type: (Chat, Tasker, Callable[[str], Auth], str, str) -> None ''' Args: @@ -86,7 +83,8 @@ def __init__(self, chat, tasker, auth_builder, reporting_channel, config_path): self._last_task_poll = datetime.min.replace(tzinfo=pytz.utc) self._last_report = datetime.min.replace(tzinfo=pytz.utc) - self._load_config(config_path) + self._load_commands() + self.messages = config['messages'] self.chat = chat chat.connect() @@ -95,72 +93,33 @@ def __init__(self, chat, tasker, auth_builder, reporting_channel, config_path): self.blacklist = SQLBlacklist() # A dictionary to be populated with all members of the team - self.users = {} # type: Dict[str, User] - self.users_by_name = {} # type: Dict[str, User] + self.users = {} # type: Dict[str, User] + self.users_by_name = {} # type: Dict[str, User] self._populate_users() # Dictionary of users who have outstanding tasks - self.active_users = {} # type: Dict[str, User] + self.active_users = {} # type: Dict[str, User] # Recover tasks self.recover_in_progress_tasks() logging.info('Done!') - # Initialization functions - - def _load_config(self, config_path): - # type: (str) -> None - ''' - Loads a configuration file for the bot. - ''' - logging.info('Loading configuration.') - with open(config_path, 'r') as f: - config = yaml.safe_load(f) - - # Required parameters - try: - self._load_messages(config['messages_path']) - self._load_commands(config['commands_path']) - except KeyError as e: - logging.error('Missing parameter: {0}'.format(e)) - raise SecurityBotException('Configuration file missing parameters.') - - # Optional parameters - self.icon_url = config.get('icon_url', 'https://placehold.it/256x256') - - def _load_messages(self, messages_path): - # type: (str) -> None - ''' - Loads messages from a YAML file. - - Args: - messages_path (str): Path to messages file. - ''' - self.messages = yaml.safe_load(open(messages_path)) - - def _load_commands(self, commands_path): - # type: (str) -> None + def _load_commands(self) -> None: ''' Loads commands from a configuration file. - - Args: - commands_path (str): Path to commands file. ''' - with open(commands_path, 'r') as f: - commands = yaml.safe_load(f) - - self.commands = {} # type: Dict[str, Any] - for name, cmd in commands.items(): - new_cmd = DEFAULT_COMMAND.copy() - new_cmd.update(cmd) + self.commands = {} # type: Dict[str, Any] + for name, cmd in config['commands'].items(): + new_cmd = DEFAULT_COMMAND.copy() + new_cmd.update(cmd) - try: - new_cmd['fn'] = getattr(bot_commands, format(cmd['fn'])) - except AttributeError as e: - raise SecurityBotException('Invalid function: {0}'.format(e)) + try: + new_cmd['fn'] = getattr(bot_commands, format(cmd['fn'])) + except AttributeError as e: + raise SecurityBotException('Invalid function: {0}'.format(e)) - self.commands[name] = new_cmd + self.commands[name] = new_cmd logging.info('Loaded commands: {0}'.format(self.commands.keys())) # Bot functions @@ -288,7 +247,6 @@ def recover_in_progress_tasks(self): self._add_task(task) - def handle_verifying_tasks(self): # type: () -> None ''' @@ -415,5 +373,6 @@ def parse_command(self, command): return (clean_command(split[0]), split[1:]) + class SecurityBotException(Exception): pass diff --git a/securitybot/chat/chat.py b/securitybot/chat/chat.py index 6a51a0a..8ed6b0d 100644 --- a/securitybot/chat/chat.py +++ b/securitybot/chat/chat.py @@ -2,29 +2,26 @@ A simple wrapper over an abstract chat/messaging system like Slack. ''' -__author__ = 'Alex Bertsch' -__email__ = 'abertsch@dropbox.com' +__author__ = 'Alex Bertsch, Antoine Cardon' +__email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' from securitybot.user import User +from typing import List, Dict, Any from abc import ABCMeta, abstractmethod -from typing import Any, Dict, List -class Chat(object): +class Chat(object, metaclass=ABCMeta): ''' A wrapper over various chat frameworks, like Slack. ''' - __metaclass__ = ABCMeta @abstractmethod - def connect(self): - # type: () -> None + def connect(self) -> None: '''Connects to the chat system.''' pass @abstractmethod - def get_users(self): - # type: () -> List[Dict[str, Any]] + def get_users(self) -> List[Dict[str, Any]]: ''' Returns a list of all users in the chat system. @@ -43,8 +40,7 @@ def get_users(self): pass @abstractmethod - def get_messages(self): - # type () -> List[Dict[str, Any]] + def get_messages(self) -> List[Dict[str, Any]]: ''' Gets a list of all new messages received by the bot in direct messaging channels. That is, this function ignores all messages @@ -59,8 +55,7 @@ def get_messages(self): pass @abstractmethod - def send_message(self, channel, message): - # type: (Any, str) -> None + def send_message(self, channel: Any, message: str) -> None: ''' Sends some message to a desired channel. As channels are possibly chat-system specific, this function has a horrible @@ -69,12 +64,12 @@ def send_message(self, channel, message): pass @abstractmethod - def message_user(self, user, message): - # type: (User, str) -> None + def message_user(self, user: User, message: str) -> None: ''' Sends some message to a desired user, using a User object and a string message. ''' pass + class ChatException(Exception): pass diff --git a/securitybot/chat/slack.py b/securitybot/chat/slack.py index 6d9510c..5c590a6 100644 --- a/securitybot/chat/slack.py +++ b/securitybot/chat/slack.py @@ -1,44 +1,42 @@ ''' A wrapper over the Slack API. ''' -__author__ = 'Alex Bertsch' -__email__ = 'abertsch@dropbox.com' +__author__ = 'Alex Bertsch, Antoine Cardon' +__email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' import logging from slackclient import SlackClient -import json - from securitybot.user import User from securitybot.chat.chat import Chat, ChatException -from typing import Any, Dict, List +from typing import Callable, Any, Dict, List + class Slack(Chat): ''' A wrapper around the Slack API designed for Securitybot. ''' - def __init__(self, username, token, icon_url): - # type: (str, str, str) -> None + + # username: str, token: str, icon_url: str=None) -> None: + def __init__(self, config: Dict=None) -> None: ''' Constructs the Slack API object using the bot's username, a Slack token, and a URL to what the bot's profile pic should be. ''' - self._username = username - self._icon_url = icon_url + self._username = config['username'] + self._icon_url = config['icon_url'] - self._slack = SlackClient(token) + self._slack = SlackClient(config['token']) self._validate() - def _validate(self): - # type: () -> None + def _validate(self) -> None: '''Validates Slack API connection.''' response = self._api_call('api.test') if not response['ok']: raise ChatException('Unable to connect to Slack API.') logging.info('Connection to Slack API successful!') - def _api_call(self, method, **kwargs): - # type: (str, **Any) -> Dict[str, Any] + def _api_call(self, method: Callable, **kwargs) -> Dict[str, Any]: ''' Performs a _validated_ Slack API call. After performing a normal API call using SlackClient, validate that the call returned 'ok'. If not, @@ -58,7 +56,7 @@ def _api_call(self, method, **kwargs): logging.error('Bad Slack API request on {}'.format(method)) return response - def connect(self): + def connect(self) -> None: # type: () -> None '''Connects to the chat system.''' logging.info('Attempting to start Slack RTM session.') @@ -67,8 +65,7 @@ def connect(self): else: raise ChatException('Unable to start Slack RTM session') - def get_users(self): - # type: () -> List[Dict[str, Any]] + def get_users(self) -> List[Dict[str, Any]]: ''' Returns a list of all users in the chat system. @@ -86,8 +83,7 @@ def get_users(self): ''' return self._api_call('users.list')['members'] - def get_messages(self): - # type () -> List[Dict[str, Any]] + def get_messages(self) -> List[Dict[str, Any]]: ''' Gets a list of all new messages received by the bot in direct messaging channels. That is, this function ignores all messages @@ -103,21 +99,19 @@ def get_messages(self): messages = [e for e in events if e['type'] == 'message'] return [m for m in messages if 'user' in m and m['channel'].startswith('D')] - def send_message(self, channel, message): - # type: (Any, str) -> None + def send_message(self, channel: Any, message: str) -> None: ''' Sends some message to a desired channel. As channels are possibly chat-system specific, this function has a horrible type signature. ''' self._api_call('chat.postMessage', channel=channel, - text=message, - username=self._username, - as_user=False, - icon_url=self._icon_url) + text=message, + username=self._username, + as_user=False, + icon_url=self._icon_url) - def message_user(self, user, message): - # type: (User, str) -> None + def message_user(self, user: User, message: str=None): ''' Sends some message to a desired user, using a User object and a string message. ''' diff --git a/securitybot/commands.py b/securitybot/commands.py index 099f052..c2fa9dd 100644 --- a/securitybot/commands.py +++ b/securitybot/commands.py @@ -6,6 +6,9 @@ They return True upon success and False upon failure, or just None if the command doesn't have success/failure messages. ''' +__author__ = 'Alex Bertsch, Antoine Cardon' +__email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' + import re from datetime import timedelta @@ -13,10 +16,12 @@ import securitybot.ignored_alerts as ignored_alerts from securitybot.util import create_new_alert + def hi(bot, user, args): '''Says hello to a user.''' bot.chat.message_user(user, bot.messages['hi'].format(user.get_name())) + def help(bot, user, args): '''Prints help for each command.''' msg = '{0}\n\n'.format(bot.messages['help_header']) @@ -29,6 +34,7 @@ def help(bot, user, args): msg += bot.messages['help_footer'] bot.chat.message_user(user, msg) + def add_to_blacklist(bot, user, args): '''Adds a user to the blacklist.''' name = user['name'] @@ -37,6 +43,7 @@ def add_to_blacklist(bot, user, args): return True return False + def remove_from_blacklist(bot, user, args): '''Removes a user from the blacklist.''' name = user['name'] @@ -45,10 +52,12 @@ def remove_from_blacklist(bot, user, args): return True return False + def positive_response(bot, user, args): '''Registers a postive response from a user.''' user.positive_response(' '.join(args)) + def negative_response(bot, user, args): '''Registers a negative response from a user.''' user.negative_response(' '.join(args)) @@ -57,6 +66,7 @@ def negative_response(bot, user, args): OUTATIME = timedelta() TIME_LIMIT = timedelta(hours=4) + def ignore(bot, user, args): '''Ignores a specific alert for a user for some period of time.''' if len(args) != 2: @@ -92,8 +102,10 @@ def ignore(bot, user, args): ignored_alerts.ignore_task(user['name'], task.title, 'ignored', ignoretime) return True + def test(bot, user, args): '''Creates a new test alert in Maniphest for a user.''' - create_new_alert('testing_alert', user['name'], 'Testing alert', 'Testing Securitybot') + create_new_alert('testing_alert', user['name'], + 'Testing alert', 'Testing Securitybot') return True diff --git a/securitybot/config.py b/securitybot/config.py new file mode 100644 index 0000000..cd928c3 --- /dev/null +++ b/securitybot/config.py @@ -0,0 +1,33 @@ +__author__ = 'Antoine Cardon' +__email__ = 'antoine.cardon@algolia.com' + +import yaml +from typing import Dict +import logging + + +class Config: + + def __init__(self): + self._config = {} + + def load_config(self, config_path: str="") -> Dict: + logging.info('Loading configuration.') + with open(config_path, 'r') as f: + self._config = yaml.safe_load(f) + try: + self._config['messages'] = yaml.safe_load( + open(self._config['includes']['messages_path'])) + self._config['commands'] = yaml.safe_load( + open(self._config['includes']['commands_path'])) + except KeyError as e: + logging.error('Missing parameter: {0}'.format(e)) + raise Exception('Configuration file missing parameters.') + + def get(self, arg, default=None): + return self._config.get(arg, default) + + def __getitem__(self, arg): + return self._config[arg] + +config = Config() diff --git a/securitybot/db/__init__.py b/securitybot/db/__init__.py new file mode 100644 index 0000000..dfab5b9 --- /dev/null +++ b/securitybot/db/__init__.py @@ -0,0 +1,3 @@ +from securitybot.db.sql import SQLEngine + +__all__ = ["SQLEngine"] diff --git a/securitybot/db/engine.py b/securitybot/db/engine.py new file mode 100644 index 0000000..da302ec --- /dev/null +++ b/securitybot/db/engine.py @@ -0,0 +1,61 @@ +''' +An abstract to the db backend +''' + +__author__ = 'Antoine Cardon' +__email__ = 'antoine.cardon@algolia.com' + +from typing import Any, List, Dict +from abc import ABCMeta, abstractmethod + +engines = {} + + +def register_engine(cls): + engines[cls.__name__] = cls + return cls + + +class EngineInterface(object, metaclass=ABCMeta): + + @abstractmethod + def __init__(self, config: Dict[str, Any], **kwargs): + ''' + Initializes the DB backend with the config dictionnary. + + Args: + config (dict): The configuration forn the database. This may contains implementation specific paramters + ''' + raise NotImplementedError() + + @abstractmethod + def execute(query: str, params: List[Any]=None): + ''' + Instructs the backend to run the following query and return a list of results. + ''' + raise NotImplementedError() + + +class EngineException(Exception): + pass + + +class Singleton(type): + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] + + +class DbEngine(object, metaclass=Singleton): + + def __init__(self, config: Dict[str, Any]=None): + if config.get('engine', None) not in engines.keys(): + raise EngineException("Engine not found") + else: + self._engine = engines[config['engine']](config) + + def execute(self, query: str, params: List[Any]=None): + return self._engine.execute(query, params) diff --git a/securitybot/db/sql.py b/securitybot/db/sql.py new file mode 100644 index 0000000..c9f17ff --- /dev/null +++ b/securitybot/db/sql.py @@ -0,0 +1,78 @@ +''' +A wrapper for the securitybot to access its database. +''' +__author__ = 'Alex Bertsch, Antoine Cardon' +__email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' + +import MySQLdb +import logging + +from securitybot.db.engine import EngineInterface, EngineException, register_engine +from typing import Any, Dict, Sequence + + +@register_engine +class SQLEngine(EngineInterface): + + def __init__(self, config: Dict[str, Any]): + ''' + Initializes the SQL connection to be used for the bot. + + Args: + config (Dict): Configuration for this engine. + ''' + self._host = config.get('host', None) + self._user = config.get('user', None) + self._passwd = config.get('password', None) + self._db = config.get('db', None) + self._create_engine() + + def _create_engine(self): + # type: (str, str, str, str) -> None + ''' + Args: + host (str): The hostname of the SQL server. + user (str): The username to use. + passwd (str): Password for MySQL user. + db (str): The name of the database to connect to. + ''' + self._conn = MySQLdb.connect(host=self._host, + user=self._user, + passwd=self._passwd, + db=self._db) + self._cursor = self._conn.cursor() + + def execute(self, query: str, params: Sequence=None): + # type: (str, Sequence[Any]) -> Sequence[Sequence[Any]] + ''' + Executes a given SQL query with some possible params. + + Args: + query (str): The query to perform. + params (Tuple[str]): Optional parameters to pass to the query. + Returns: + Tuple[Tuple[str]]: The output from the SQL query. + ''' + if params is None: + params = () + try: + logging.debug('Executing: ' + query + str(params)) + self._cursor.execute(query, params) + rows = self._cursor.fetchall() + self._conn.commit() + except (AttributeError, MySQLdb.OperationalError): + # Recover from lost connection + logging.warn('Recovering from lost MySQL connection.') + self._create_engine() + return self.execute(query, params) + except MySQLdb.Error as e: + try: + raise SQLEngineException('MySQL error [{0}]: {1}'.format(e.args[0], e.args[1])) + except IndexError: + raise SQLEngineException('MySQL error: {0}'.format(e)) + logging.debug('Result: ' + str(rows)) + return rows + + +class SQLEngineException(EngineException): + pass diff --git a/securitybot/ignored_alerts.py b/securitybot/ignored_alerts.py index 59a177f..5b34dfc 100644 --- a/securitybot/ignored_alerts.py +++ b/securitybot/ignored_alerts.py @@ -2,19 +2,20 @@ A small file for keeping track of ignored alerts in the database. ''' import pytz -from datetime import datetime, timedelta -from securitybot.sql import SQLEngine +from datetime import timedelta, datetime +from securitybot.db.engine import DbEngine from typing import Dict -def __update_ignored_list(): + +def __update_ignored_list() -> None: # type: () -> None ''' Prunes the ignored table of old ignored alerts. ''' - SQLEngine.execute('''DELETE FROM ignored WHERE until <= NOW()''') + DbEngine().execute('''DELETE FROM ignored WHERE until <= NOW()''') + -def get_ignored(username): - # type: (str) -> Dict[str, str] +def get_ignored(username: str) -> Dict[str, str]: ''' Returns a dictionary of ignored alerts to reasons why the ignored are ignored. @@ -25,11 +26,11 @@ def get_ignored(username): Dict[str, str]: A mapping of ignored alert titles to reasons ''' __update_ignored_list() - rows = SQLEngine.execute('''SELECT title, reason FROM ignored WHERE ldap = %s''', (username,)) + rows = DbEngine().execute('''SELECT title, reason FROM ignored WHERE ldap = %s''', (username,)) return {row[0]: row[1] for row in rows} -def ignore_task(username, title, reason, ttl): - # type: (str, str, str, timedelta) -> None + +def ignore_task(username: str, title: str, reason: str, ttl: timedelta) -> None: ''' Adds a task with the given title to the ignore list for the given amount of time. Additionally adds an optional message to specify the @@ -43,7 +44,7 @@ def ignore_task(username, title, reason, ttl): ''' expiry_time = datetime.now(tz=pytz.utc) + ttl # NB: Non-standard MySQL specific query - SQLEngine.execute('''INSERT INTO ignored (ldap, title, reason, until) + DbEngine().execute('''INSERT INTO ignored (ldap, title, reason, until) VALUES (%s, %s, %s, %s) ON DUPLICATE KEY UPDATE reason=VALUES(reason), until=VALUES(until) ''', (username, title, reason, expiry_time.strftime('%Y-%m-%d %H:%M:%S'))) diff --git a/securitybot/sql.py b/securitybot/sql.py deleted file mode 100644 index 43ebd7d..0000000 --- a/securitybot/sql.py +++ /dev/null @@ -1,93 +0,0 @@ -''' -A wrapper for the securitybot to access its database. -''' -import MySQLdb -import logging - -from typing import Any, Sequence - -class SQLEngine(object): - # Whether the singleton has been instantiated - _host = None # type: str - _user = None # type: str - _passwd = None # type: str - _db = None # type: str - _created = False # type: bool - _conn = None - _cursor = None - - def __init__(self, host, user, passwd, db): - # type: (str, str, str, str) -> None - ''' - Initializes the SQL connection to be used for the bot. - - Args: - host (str): The hostname of the SQL server. - user (str): The username to use. - passwd (str): Password for MySQL user. - db (str): The name of the database to connect to. - ''' - if not SQLEngine._created: - SQLEngine._host = host - SQLEngine._user = user - SQLEngine._passwd = passwd - SQLEngine._db = db - SQLEngine._create_engine(host, user, passwd, db) - SQLEngine._created = True - - @staticmethod - def _create_engine(host, user, passwd, db): - # type: (str, str, str, str) -> None - ''' - Args: - host (str): The hostname of the SQL server. - user (str): The username to use. - passwd (str): Password for MySQL user. - db (str): The name of the database to connect to. - ''' - SQLEngine._conn = MySQLdb.connect(host=host, - user=user, - passwd=passwd, - db=db) - SQLEngine._cursor = SQLEngine._conn.cursor() - - @staticmethod - def execute(query, params=None): - # type: (str, Sequence[Any]) -> Sequence[Sequence[Any]] - ''' - Executes a given SQL query with some possible params. - - Args: - query (str): The query to perform. - params (Tuple[str]): Optional parameters to pass to the query. - Returns: - Tuple[Tuple[str]]: The output from the SQL query. - ''' - if params is None: - params = () - try: - SQLEngine._cursor.execute(query, params) - rows = SQLEngine._cursor.fetchall() - SQLEngine._conn.commit() - except (AttributeError, MySQLdb.OperationalError): - # Recover from lost connection - logging.warn('Recovering from lost MySQL connection.') - SQLEngine._create_engine(SQLEngine._host, - SQLEngine._user, - SQLEngine._passwd, - SQLEngine._db) - return SQLEngine.execute(query, params) - except MySQLdb.Error as e: - try: - raise SQLEngineException('MySQL error [{0}]: {1}'.format(e.args[0], e.args[1])) - except IndexError: - raise SQLEngineException('MySQL error: {0}'.format(e)) - return rows - -class SQLEngineException(Exception): - pass - -def init_sql(): - # type: () -> None - '''Initializes SQL.''' - SQLEngine('localhost', 'root', '', 'securitybot') diff --git a/securitybot/tasker/sql_tasker.py b/securitybot/tasker/sql_tasker.py index e5db55f..b3e83d0 100644 --- a/securitybot/tasker/sql_tasker.py +++ b/securitybot/tasker/sql_tasker.py @@ -1,8 +1,11 @@ ''' A tasker on top of a SQL database. ''' -from securitybot.tasker.tasker import Task, Tasker, STATUS_LEVELS -from securitybot.sql import SQLEngine +__author__ = 'Alex Bertsch, Antoine Cardon' +__email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' + +from securitybot.tasker.tasker import Task, Tasker, StatusLevel +from securitybot.db.engine import DbEngine from typing import List @@ -24,32 +27,36 @@ WHERE status = %s ''' + class SQLTasker(Tasker): - def _get_tasks(self, level): + + def __init__(self): + self._db_engine = DbEngine() + + def _get_tasks(self, level) -> List[Task]: # type: (int) -> List[Task] ''' Gets all tasks of a certain level. Args: - level (int): One of STATUS_LEVELS + level (int): One of StatusLevel Returns: List of SQLTasks. ''' - alerts = SQLEngine.execute(GET_ALERTS, (level,)) + alerts = self._db_engine.execute(GET_ALERTS, (level,)) return [SQLTask(*alert) for alert in alerts] def get_new_tasks(self): # type: () -> List[Task] - return self._get_tasks(STATUS_LEVELS.OPEN) - + return self._get_tasks(StatusLevel.OPEN.value) def get_active_tasks(self): # type: () -> List[Task] - return self._get_tasks(STATUS_LEVELS.INPROGRESS) + return self._get_tasks(StatusLevel.INPROGRESS.value) def get_pending_tasks(self): # type: () -> List[Task] - return self._get_tasks(STATUS_LEVELS.VERIFICATION) + return self._get_tasks(StatusLevel.VERIFICATION.value) SET_STATUS = ''' UPDATE alert_status @@ -65,7 +72,9 @@ def get_pending_tasks(self): WHERE hash=UNHEX(%s) ''' + class SQLTask(Task): + def __init__(self, hsh, title, username, reason, description, url, performed, comment, authenticated, status): # type: (str, str, str, str, str, str, bool, str, bool, int) -> None @@ -74,7 +83,7 @@ def __init__(self, hsh, title, username, reason, description, url, hsh (str): SHA256 primary key hash. ''' super(SQLTask, self).__init__(title, username, reason, description, url, - performed, comment, authenticated, status) + performed, comment, authenticated, status) self.hash = hsh def _set_status(self, status): @@ -85,24 +94,24 @@ def _set_status(self, status): Args: status (int): The new status to use. ''' - SQLEngine.execute(SET_STATUS, (status, self.hash)) + self._db_engine.execute(SET_STATUS, (status, self.hash)) def _set_response(self): # type: () -> None ''' Updates the user response for this task. ''' - SQLEngine.execute(SET_RESPONSE, (self.comment, - self.performed, - self.authenticated, - self.hash)) + self._db_engine.execute(SET_RESPONSE, (self.comment, + self.performed, + self.authenticated, + self.hash)) def set_open(self): - self._set_status(STATUS_LEVELS.OPEN) + self._set_status(StatusLevel.OPEN.value) def set_in_progress(self): - self._set_status(STATUS_LEVELS.INPROGRESS) + self._set_status(StatusLevel.INPROGRESS.value) def set_verifying(self): - self._set_status(STATUS_LEVELS.VERIFICATION) + self._set_status(StatusLevel.VERIFICATION.value) self._set_response() diff --git a/securitybot/tasker/tasker.py b/securitybot/tasker/tasker.py index 4167489..f02851c 100644 --- a/securitybot/tasker/tasker.py +++ b/securitybot/tasker/tasker.py @@ -4,17 +4,18 @@ Tasker and Task, which define a class to manage tasks and a task class respectively. ''' -__author__ = 'Alex Bertsch' -__email__ = 'abertsch@dropbox.com' +__author__ = 'Alex Bertsch, Antoine Cardon' +__email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' from abc import ABCMeta, abstractmethod -from securitybot.util import enum +from enum import Enum, unique +from securitybot.db.engine import DbEngine -class Tasker(object): + +class Tasker(object, metaclass=ABCMeta): ''' A simple interface to retrieve tasks on which the bot should act upon. ''' - __metaclass__ = ABCMeta @abstractmethod def get_new_tasks(self): @@ -44,14 +45,19 @@ def get_pending_tasks(self): ''' pass -# Task status levels -STATUS_LEVELS = enum('OPEN', 'INPROGRESS', 'VERIFICATION') -class Task(object): - __metaclass__ = ABCMeta +@unique +class StatusLevel(Enum): + # Task status levels + OPEN = 0 + INPROGRESS = 1 + VERIFICATION = 2 + + +class Task(object, metaclass=ABCMeta): def __init__(self, title, username, reason, description, url, performed, comment, - authenticated, status): + authenticated, status): # type: (str, str, str, str, str, bool, str, bool, int) -> None ''' Creates a new Task for an alert that should go to `username` and is @@ -79,6 +85,7 @@ def __init__(self, title, username, reason, description, url, performed, comment self.comment = comment self.authenticated = authenticated self.status = status + self._db_engine = DbEngine() @abstractmethod def set_open(self): diff --git a/securitybot/user.py b/securitybot/user.py index f7e47cf..3120770 100644 --- a/securitybot/user.py +++ b/securitybot/user.py @@ -9,16 +9,14 @@ import pytz from datetime import datetime, timedelta import securitybot.ignored_alerts as ignored_alerts -from securitybot.tasker.tasker import Task -from securitybot.auth.auth import AUTH_STATES +from securitybot.auth.auth import AuthStates from securitybot.state_machine import StateMachine from securitybot.util import tuple_builder, get_expiration_time -from typing import Any, Dict, List - -ESCALATION_TIME = timedelta(hours=2) +ESCALATION_TIME = timedelta(minutes=2) BACKOFF_TIME = timedelta(hours=21) + class User(object): ''' A user to be contacted by the security bot. Each user stores all of the @@ -35,9 +33,9 @@ def __init__(self, user, auth, parent): auth (Auth): The authentication object to use. parent (Bot): The bot object that spawned this user. ''' - self._user = user # type: Dict[str, Any] - self.tasks = [] # type: List[Task] - self.pending_task = None # type: Task + self._user = user # type: Dict[str, Any] + self.tasks = [] # type: List[Task] + self.pending_task = None # type: Task # Authetnication object specific to this user self.auth = auth @@ -48,7 +46,7 @@ def __init__(self, user, auth, parent): self._last_message = tuple_builder() # Last authorization status - self._last_auth = AUTH_STATES.NONE + self._last_auth = AuthStates.NONE # Task auto-escalation time self._escalation_time = datetime.max.replace(tzinfo=pytz.utc) @@ -178,7 +176,7 @@ def _already_authed(self): Checks if the user performed the last action and if they are already authorized. ''' - return self._performed_action() and self.auth_status() == AUTH_STATES.AUTHORIZED + return self._performed_action() and self.auth_status() == AuthStates.AUTHORIZED def _cannot_2fa(self): # type: () -> bool @@ -212,7 +210,7 @@ def _denies_authorization(self): def _auth_completed(self): # type: () -> bool '''Checks if authentication has been completed.''' - return self._last_auth is AUTH_STATES.AUTHORIZED or self._last_auth is AUTH_STATES.DENIED + return self._last_auth is AuthStates.AUTHORIZED or self._last_auth is AuthStates.DENIED # State actions @@ -220,7 +218,7 @@ def _auto_escalate(self): # type: () -> None '''Marks the current task as needing verification and moves on.''' logging.info('Silently escalating {0} for {1}' - .format(self.pending_task.description, self['name'])) + .format(self.pending_task.description, self['name'])) # Append in the case that this is called when waiting for auth permission self.pending_task.comment += 'Automatically escalated. No response received.' self.pending_task.set_verifying() @@ -251,7 +249,6 @@ def _act_on_not_performed(self): comment=comment, url=self.pending_task.url)) - # Exit actions def _update_task_response(self): @@ -270,7 +267,7 @@ def _update_task_auth(self): ''' Updates the task with authorization permission. ''' - if self._last_auth is AUTH_STATES.AUTHORIZED: + if self._last_auth is AuthStates.AUTHORIZED: self.send_message('good_auth') self.pending_task.authenticated = True else: @@ -415,5 +412,6 @@ def get_name(self): return self._user['profile']['first_name'] return self._user['name'] + class UserException(Exception): pass diff --git a/securitybot/util.py b/securitybot/util.py index 33ccca3..b7310ce 100644 --- a/securitybot/util.py +++ b/securitybot/util.py @@ -1,18 +1,14 @@ -__author__ = 'Alex Bertsch' -__email__ = 'abertsch@dropbox.com' +__author__ = 'Alex Bertsch, Antoine Cardon' +__email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' import pytz import binascii import os from datetime import datetime, timedelta from collections import namedtuple +from securitybot.db.engine import DbEngine +from securitybot.tasker.tasker import StatusLevel -from securitybot.sql import SQLEngine - -# http://stackoverflow.com/questions/36932/how-can-i-represent-an-enum-in-python -def enum(*sequential, **named): - enums = dict(zip(sequential, range(len(sequential))), **named) - return type('Enum', (), enums) def tuple_builder(answer=None, text=None): tup = namedtuple('Response', ['answer', 'text']) @@ -24,6 +20,7 @@ def tuple_builder(answer=None, text=None): CLOSING_HOUR = 18 LOCAL_TZ = pytz.timezone('America/Los_Angeles') + def during_business_hours(time): ''' Checks if a given time is within business hours. Currently is true @@ -39,6 +36,7 @@ def during_business_hours(time): return (OPENING_HOUR <= here.hour < CLOSING_HOUR and 1 <= time.isoweekday() <= 5) + def get_expiration_time(start, time): ''' Gets an expiration time for an alert. @@ -69,6 +67,7 @@ def get_expiration_time(start, time): end = next_day + delta return end + def create_new_alert(title, ldap, description, reason, url='N/A', key=None): # type: (str, str, str, str, str, str) -> None ''' @@ -77,19 +76,19 @@ def create_new_alert(title, ldap, description, reason, url='N/A', key=None): # Generate random key if none provided if key is None: key = binascii.hexlify(os.urandom(32)) - + db_engine = DbEngine() # Insert that into the database as a new alert - SQLEngine.execute(''' + db_engine.execute(''' INSERT INTO alerts (hash, ldap, title, description, reason, url, event_time) VALUES (UNHEX(%s), %s, %s, %s, %s, %s, NOW()) ''', - (key, ldap, title, description, reason, url)) + (key, ldap, title, description, reason, url)) - SQLEngine.execute(''' + db_engine.execute(''' INSERT INTO user_responses (hash, comment, performed, authenticated) VALUES (UNHEX(%s), '', false, false) ''', - (key,)) - - SQLEngine.execute('INSERT INTO alert_status (hash, status) VALUES (UNHEX(%s), 0)', (key,)) + + db_engine.execute('INSERT INTO alert_status (hash, status) VALUES (UNHEX(%s), %s)', + (key, StatusLevel.OPEN.value)) diff --git a/util/db_up.py b/util/db_up.py index fd1fdd1..ca593a5 100644 --- a/util/db_up.py +++ b/util/db_up.py @@ -1,11 +1,10 @@ #!/usr/bin/env python import MySQLdb -import sys # DB CONFIG GOES HERE host = 'localhost' user = 'root' -passwd= '' +passwd = 'password' db = MySQLdb.connect(host=host, user=user, @@ -15,19 +14,19 @@ cur = db.cursor() # Start fresh -print 'Removing all tables' +print('Removing all tables') cur.execute('SHOW TABLES') tables = cur.fetchall() for table in tables: table = table[0] - print 'Dropping {0}'.format(table) - cur.execute('DROP TABLE {0}'.format(MySQLdb.escape_string(table))) + print('Dropping {0}'.format(table)) + cur.execute('DROP TABLE {0}'.format(table)) # Create tables -print 'Creating tables...' +print('Creating tables...') cur.execute( -''' + ''' CREATE TABLE blacklist ( ldap VARCHAR(255) NOT NULL, PRIMARY KEY ( ldap ) @@ -36,7 +35,7 @@ ) cur.execute( -''' + ''' CREATE TABLE ignored ( ldap VARCHAR(255) NOT NULL, title VARCHAR(255) NOT NULL, @@ -48,7 +47,7 @@ ) cur.execute( -''' + ''' CREATE TABLE alerts ( hash BINARY(32) NOT NULL, ldap VARCHAR(255) NOT NULL, @@ -63,7 +62,7 @@ ) cur.execute( -''' + ''' CREATE TABLE alert_status ( hash BINARY(32) NOT NULL, status TINYINT UNSIGNED NOT NULL, @@ -73,7 +72,7 @@ ) cur.execute( -''' + ''' CREATE TABLE user_responses( hash BINARY(32) NOT NULL, comment TEXT, @@ -84,4 +83,4 @@ ''' ) -print 'Done!' +print('Done!') From 873af47a8e69bba76de9337acb8063a06f4a4c95 Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Fri, 28 Jul 2017 11:16:18 +0200 Subject: [PATCH 05/19] Make AUTH_TIME a configurable variable --- securitybot/auth/duo.py | 3 +- securitybot/{blacklist => }/blacklist.py | 0 securitybot/blacklist/__init__.py | 0 securitybot/blacklist/sql_blacklist.py | 54 ----------- securitybot/{tasker => }/tasker.py | 0 securitybot/tasker/__init__.py | 0 securitybot/tasker/sql_tasker.py | 117 ----------------------- 7 files changed, 2 insertions(+), 172 deletions(-) rename securitybot/{blacklist => }/blacklist.py (100%) delete mode 100644 securitybot/blacklist/__init__.py delete mode 100644 securitybot/blacklist/sql_blacklist.py rename securitybot/{tasker => }/tasker.py (100%) delete mode 100644 securitybot/tasker/__init__.py delete mode 100644 securitybot/tasker/sql_tasker.py diff --git a/securitybot/auth/duo.py b/securitybot/auth/duo.py index dbb9a2a..63b55e5 100644 --- a/securitybot/auth/duo.py +++ b/securitybot/auth/duo.py @@ -8,6 +8,7 @@ from datetime import datetime from urllib.parse import urlencode from securitybot.auth.auth import Auth, AuthStates +from securitybot.config import config from typing import Callable @@ -58,7 +59,7 @@ def auth(self, reason: str=None) -> None: self.state = AuthStates.PENDING def _recently_authed(self) -> bool: - return (datetime.now() - self.auth_time) < AUTH_TIME + return (datetime.now() - self.auth_time) < timedelta(seconds=config['duo']['auth_time']) def auth_status(self) -> int: if self.state == AuthStates.PENDING: diff --git a/securitybot/blacklist/blacklist.py b/securitybot/blacklist.py similarity index 100% rename from securitybot/blacklist/blacklist.py rename to securitybot/blacklist.py diff --git a/securitybot/blacklist/__init__.py b/securitybot/blacklist/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/securitybot/blacklist/sql_blacklist.py b/securitybot/blacklist/sql_blacklist.py deleted file mode 100644 index 596b4d8..0000000 --- a/securitybot/blacklist/sql_blacklist.py +++ /dev/null @@ -1,54 +0,0 @@ -''' -A MySQL-based blacklist class. -''' -__author__ = 'Alex Bertsch, Antoine Cardon' -__email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' - -from securitybot.blacklist.blacklist import Blacklist -from securitybot.db.engine import DbEngine - - -class SQLBlacklist(Blacklist): - - def __init__(self): - # type: () -> None - ''' - Creates a new blacklist tied to a table named "blacklist". - ''' - # Load from table - self._db_engine = DbEngine() - names = self._db_engine.execute('SELECT * FROM blacklist') - # Break tuples into names - self._blacklist = {name[0] for name in names} - - def is_present(self, name): - # type: (str) -> bool - ''' - Checks if a name is on the blacklist. - - Args: - name (str): The name to check. - ''' - return name in self._blacklist - - def add(self, name): - # type: (str) -> None - ''' - Adds a name to the blacklist. - - Args: - name (str): The name to add to the blacklist. - ''' - self._blacklist.add(name) - self._db_engine.execute('INSERT INTO blacklist (ldap) VALUES (%s)', (name,)) - - def remove(self, name): - # type: (str) -> None - ''' - Removes a name to the blacklist. - - Args: - name (str): The name to remove from the blacklist. - ''' - self._blacklist.remove(name) - self._db_engine.execute('DELETE FROM blacklist WHERE ldap = %s', (name,)) diff --git a/securitybot/tasker/tasker.py b/securitybot/tasker.py similarity index 100% rename from securitybot/tasker/tasker.py rename to securitybot/tasker.py diff --git a/securitybot/tasker/__init__.py b/securitybot/tasker/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/securitybot/tasker/sql_tasker.py b/securitybot/tasker/sql_tasker.py deleted file mode 100644 index b3e83d0..0000000 --- a/securitybot/tasker/sql_tasker.py +++ /dev/null @@ -1,117 +0,0 @@ -''' -A tasker on top of a SQL database. -''' -__author__ = 'Alex Bertsch, Antoine Cardon' -__email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' - -from securitybot.tasker.tasker import Task, Tasker, StatusLevel -from securitybot.db.engine import DbEngine - -from typing import List - -# Note: this order is provided to match the SQLTask constructor -GET_ALERTS = ''' -SELECT HEX(alerts.hash), - title, - ldap, - reason, - description, - url, - performed, - comment, - authenticated, - status -FROM alerts -JOIN user_responses ON alerts.hash = user_responses.hash -JOIN alert_status ON alerts.hash = alert_status.hash -WHERE status = %s -''' - - -class SQLTasker(Tasker): - - def __init__(self): - self._db_engine = DbEngine() - - def _get_tasks(self, level) -> List[Task]: - # type: (int) -> List[Task] - ''' - Gets all tasks of a certain level. - - Args: - level (int): One of StatusLevel - Returns: - List of SQLTasks. - ''' - alerts = self._db_engine.execute(GET_ALERTS, (level,)) - return [SQLTask(*alert) for alert in alerts] - - def get_new_tasks(self): - # type: () -> List[Task] - return self._get_tasks(StatusLevel.OPEN.value) - - def get_active_tasks(self): - # type: () -> List[Task] - return self._get_tasks(StatusLevel.INPROGRESS.value) - - def get_pending_tasks(self): - # type: () -> List[Task] - return self._get_tasks(StatusLevel.VERIFICATION.value) - -SET_STATUS = ''' -UPDATE alert_status -SET status=%s -WHERE hash=UNHEX(%s) -''' - -SET_RESPONSE = ''' -UPDATE user_responses -SET comment=%s, - performed=%s, - authenticated=%s -WHERE hash=UNHEX(%s) -''' - - -class SQLTask(Task): - - def __init__(self, hsh, title, username, reason, description, url, - performed, comment, authenticated, status): - # type: (str, str, str, str, str, str, bool, str, bool, int) -> None - ''' - Args: - hsh (str): SHA256 primary key hash. - ''' - super(SQLTask, self).__init__(title, username, reason, description, url, - performed, comment, authenticated, status) - self.hash = hsh - - def _set_status(self, status): - # type: (int) -> None - ''' - Sets the status of a task in the DB. - - Args: - status (int): The new status to use. - ''' - self._db_engine.execute(SET_STATUS, (status, self.hash)) - - def _set_response(self): - # type: () -> None - ''' - Updates the user response for this task. - ''' - self._db_engine.execute(SET_RESPONSE, (self.comment, - self.performed, - self.authenticated, - self.hash)) - - def set_open(self): - self._set_status(StatusLevel.OPEN.value) - - def set_in_progress(self): - self._set_status(StatusLevel.INPROGRESS.value) - - def set_verifying(self): - self._set_status(StatusLevel.VERIFICATION.value) - self._set_response() From deebe6d2bfec7687faa238ce0c7653d47f1f199c Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Fri, 28 Jul 2017 11:16:40 +0200 Subject: [PATCH 06/19] Update requirements.txt --- requirements.txt | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/requirements.txt b/requirements.txt index d8d3663..9343dea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,28 +1,6 @@ -backports-abc==0.4 -certifi==2016.8.8 -configparser==3.5.0 duo-client==3.0 -enum34==1.1.6 -flake8==3.0.4 -funcsigs==1.0.2 -linecache2==1.0.0 -mccabe==0.5.2 -mock==2.0.0 -MySQL-python==1.2.5 -nose==1.3.7 -pbr==1.10.0 -py==1.4.31 -pycodestyle==2.0.0 -pyflakes==1.2.3 -pytest==3.0.1 -pytz==2016.6.1 -PyYAML==3.11 -requests==2.11.1 -singledispatch==3.4.0.3 -six==1.10.0 -slackclient==1.0.1 -tornado==4.4.1 -traceback2==1.4.0 -typing==3.5.2.2 -unittest2==1.1.0 -websocket-client==0.37.0 +mysqlclient==1.3.10 +pytz==2017.2 +PyYAML==3.12 +requests==2.18.1 +slackclient==1.0.6 \ No newline at end of file From dfa6f21ac430957d774cd98536cc3f4a454c28c3 Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Fri, 28 Jul 2017 11:17:48 +0200 Subject: [PATCH 07/19] Remove asbtract from blacklist as no more class needs to inherit from it. blacklist is now generic. Same for tasker and SQLTasker. --- securitybot/blacklist.py | 27 +++++++--- securitybot/tasker.py | 111 ++++++++++++++++++++------------------- 2 files changed, 77 insertions(+), 61 deletions(-) diff --git a/securitybot/blacklist.py b/securitybot/blacklist.py index 59caa49..cb15071 100644 --- a/securitybot/blacklist.py +++ b/securitybot/blacklist.py @@ -4,12 +4,23 @@ __author__ = 'Alex Bertsch, Antoine Cardon' __email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' -from abc import ABCMeta, abstractmethod +from securitybot.db.engine import DbEngine +from securitybot.config import config -class Blacklist(object, metaclass=ABCMeta): +class Blacklist(object): + + def __init__(self): + # type: () -> None + ''' + Creates a new blacklist tied to a table named "blacklist". + ''' + # Load from table + self._db_engine = DbEngine() + names = self._db_engine.execute(config['queries']['blacklist_list']) + # Break tuples into names + self._blacklist = {name[0] for name in names} - @abstractmethod def is_present(self, name): # type: (str) -> bool ''' @@ -18,9 +29,8 @@ def is_present(self, name): Args: name (str): The name to check. ''' - raise NotImplementedError() + return name in self._blacklist - @abstractmethod def add(self, name): # type: (str) -> None ''' @@ -29,9 +39,9 @@ def add(self, name): Args: name (str): The name to add to the blacklist. ''' - raise NotImplementedError() + self._blacklist.add(name) + self._db_engine.execute(config['queries']['blacklist_add'], (name,)) - @abstractmethod def remove(self, name): # type: (str) -> None ''' @@ -40,4 +50,5 @@ def remove(self, name): Args: name (str): The name to remove from the blacklist. ''' - raise NotImplementedError() + self._blacklist.remove(name) + self._db_engine.execute(config['queries']['blacklist_remove'], (name,)) diff --git a/securitybot/tasker.py b/securitybot/tasker.py index f02851c..6ac0019 100644 --- a/securitybot/tasker.py +++ b/securitybot/tasker.py @@ -7,43 +7,10 @@ __author__ = 'Alex Bertsch, Antoine Cardon' __email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' -from abc import ABCMeta, abstractmethod from enum import Enum, unique from securitybot.db.engine import DbEngine - - -class Tasker(object, metaclass=ABCMeta): - ''' - A simple interface to retrieve tasks on which the bot should act upon. - ''' - - @abstractmethod - def get_new_tasks(self): - # type: () -> List[Task] - ''' - Returns a list of new Task objects that need to be acted upon, i.e. - the intial message needs to be sent out to the alertee. - ''' - pass - - @abstractmethod - def get_active_tasks(self): - # type: () -> List[Task] - ''' - Returns a list of Task objects for which the alertees have been - contacted but have not replied. Periodically this list should be polled - and stale tasks should have their alertees pinged. - ''' - pass - - @abstractmethod - def get_pending_tasks(self): - # type: () -> List[Task] - ''' - Retrieves a list of tasks for which the user has responded and it now - waiting for manual closure. - ''' - pass +from securitybot.config import config +from typing import List @unique @@ -54,9 +21,9 @@ class StatusLevel(Enum): VERIFICATION = 2 -class Task(object, metaclass=ABCMeta): +class Task(object): - def __init__(self, title, username, reason, description, url, performed, comment, + def __init__(self, hsh, title, username, reason, description, url, performed, comment, authenticated, status): # type: (str, str, str, str, str, bool, str, bool, int) -> None ''' @@ -86,30 +53,68 @@ def __init__(self, title, username, reason, description, url, performed, comment self.authenticated = authenticated self.status = status self._db_engine = DbEngine() + self.hash = hsh - @abstractmethod - def set_open(self): - # type: () -> None + def _set_status(self, status): + # type: (int) -> None ''' - Sets this task to be open and performs any needed actions to ensure that - the corresponding tasker will be able to properly see it as such. + Sets the status of a task in the DB. + + Args: + status (int): The new status to use. ''' - pass + self._db_engine.execute(config['queries']['set_status'], (status, self.hash)) - @abstractmethod - def set_in_progress(self): + def _set_response(self): # type: () -> None ''' - Sets this task to be in progress and performs any needed actions to - ensure the corresponding tasker will be able to see it as such. + Updates the user response for this task. ''' - pass + self._db_engine.execute(config['queries']['set_response'], (self.comment, + self.performed, + self.authenticated, + self.hash)) + + def set_open(self): + self._set_status(StatusLevel.OPEN.value) + + def set_in_progress(self): + self._set_status(StatusLevel.INPROGRESS.value) - @abstractmethod def set_verifying(self): - # type: () -> None + self._set_status(StatusLevel.VERIFICATION.value) + self._set_response() + + +class Tasker(object): + ''' + A simple class to retrieve tasks on which the bot should act upon. + ''' + + def __init__(self): + self._db_engine = DbEngine() + + def _get_tasks(self, level) -> List[Task]: + # type: (int) -> List[Task] ''' - Sets this task to be waiting for verification and performs and needed - actions to ensure that the corresponding tasker sees it as such. + Gets all tasks of a certain level. + + Args: + level (int): One of StatusLevel + Returns: + List of SQLTasks. ''' - pass + alerts = self._db_engine.execute(config['queries']['get_alerts'], (level,)) + return [Task(*alert) for alert in alerts] + + def get_new_tasks(self): + # type: () -> List[Task] + return self._get_tasks(StatusLevel.OPEN.value) + + def get_active_tasks(self): + # type: () -> List[Task] + return self._get_tasks(StatusLevel.INPROGRESS.value) + + def get_pending_tasks(self): + # type: () -> List[Task] + return self._get_tasks(StatusLevel.VERIFICATION.value) From 72e6f6cd1f02ee68641345040037566a94ac519e Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Fri, 28 Jul 2017 11:18:14 +0200 Subject: [PATCH 08/19] Remove SQL hard coded string from util.py --- securitybot/util.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/securitybot/util.py b/securitybot/util.py index b7310ce..4f14a64 100644 --- a/securitybot/util.py +++ b/securitybot/util.py @@ -7,7 +7,8 @@ from datetime import datetime, timedelta from collections import namedtuple from securitybot.db.engine import DbEngine -from securitybot.tasker.tasker import StatusLevel +from securitybot.config import config +from securitybot.tasker import StatusLevel def tuple_builder(answer=None, text=None): @@ -78,17 +79,9 @@ def create_new_alert(title, ldap, description, reason, url='N/A', key=None): key = binascii.hexlify(os.urandom(32)) db_engine = DbEngine() # Insert that into the database as a new alert - db_engine.execute(''' - INSERT INTO alerts (hash, ldap, title, description, reason, url, event_time) - VALUES (UNHEX(%s), %s, %s, %s, %s, %s, NOW()) - ''', + db_engine.execute(config['queries']['new_alert_alerts'], (key, ldap, title, description, reason, url)) - db_engine.execute(''' - INSERT INTO user_responses (hash, comment, performed, authenticated) - VALUES (UNHEX(%s), '', false, false) - ''', - (key,)) + db_engine.execute(config['queries']['new_alert_user_response'], (key,)) - db_engine.execute('INSERT INTO alert_status (hash, status) VALUES (UNHEX(%s), %s)', - (key, StatusLevel.OPEN.value)) + db_engine.execute(config['queries']['new_alert'], (key, StatusLevel.OPEN.value)) From 7031df1b349893318c1370c418b1bdd023b78864 Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Fri, 28 Jul 2017 11:19:01 +0200 Subject: [PATCH 09/19] Moving Singleton class to a class helper file --- securitybot/db/engine.py | 12 ++---------- securitybot/utils/__init__.py | 0 securitybot/utils/class_helper.py | 7 +++++++ 3 files changed, 9 insertions(+), 10 deletions(-) create mode 100644 securitybot/utils/__init__.py create mode 100644 securitybot/utils/class_helper.py diff --git a/securitybot/db/engine.py b/securitybot/db/engine.py index da302ec..b06e64a 100644 --- a/securitybot/db/engine.py +++ b/securitybot/db/engine.py @@ -6,6 +6,7 @@ __email__ = 'antoine.cardon@algolia.com' from typing import Any, List, Dict +from securitybot.utils.class_helper import Singleton from abc import ABCMeta, abstractmethod engines = {} @@ -40,22 +41,13 @@ class EngineException(Exception): pass -class Singleton(type): - _instances = {} - - def __call__(cls, *args, **kwargs): - if cls not in cls._instances: - cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) - return cls._instances[cls] - - class DbEngine(object, metaclass=Singleton): def __init__(self, config: Dict[str, Any]=None): if config.get('engine', None) not in engines.keys(): raise EngineException("Engine not found") else: - self._engine = engines[config['engine']](config) + self._engine: EngineInterface = engines[config['engine']](config) def execute(self, query: str, params: List[Any]=None): return self._engine.execute(query, params) diff --git a/securitybot/utils/__init__.py b/securitybot/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/securitybot/utils/class_helper.py b/securitybot/utils/class_helper.py new file mode 100644 index 0000000..3776cb9 --- /dev/null +++ b/securitybot/utils/class_helper.py @@ -0,0 +1,7 @@ +class Singleton(type): + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] From 5f2adce3becd74ae78a68fc8b138832d70156086 Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Fri, 28 Jul 2017 11:19:38 +0200 Subject: [PATCH 10/19] Makes config a singleton to prevent overrides --- securitybot/config.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/securitybot/config.py b/securitybot/config.py index cd928c3..1c82ccc 100644 --- a/securitybot/config.py +++ b/securitybot/config.py @@ -2,19 +2,21 @@ __email__ = 'antoine.cardon@algolia.com' import yaml -from typing import Dict import logging +from typing import Dict +from securitybot.utils.class_helper import Singleton -class Config: +class Config(object, metaclass=Singleton): - def __init__(self): + def __init__(self, *args, **kwargs): self._config = {} def load_config(self, config_path: str="") -> Dict: logging.info('Loading configuration.') with open(config_path, 'r') as f: self._config = yaml.safe_load(f) + self._config['queries'] = {} try: self._config['messages'] = yaml.safe_load( open(self._config['includes']['messages_path'])) @@ -24,6 +26,9 @@ def load_config(self, config_path: str="") -> Dict: logging.error('Missing parameter: {0}'.format(e)) raise Exception('Configuration file missing parameters.') + with open(self._config['includes']['queries'][self._config['database']['queries']], 'r') as qfile: + self._config['queries'] = yaml.safe_load(qfile) + def get(self, arg, default=None): return self._config.get(arg, default) From afaa05aeeac80cf8c068a4d767baf0a71fc48784 Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Fri, 28 Jul 2017 11:19:59 +0200 Subject: [PATCH 11/19] Remove SQL specialized code --- securitybot/bot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/securitybot/bot.py b/securitybot/bot.py index 64e55b7..d4bd6d1 100644 --- a/securitybot/bot.py +++ b/securitybot/bot.py @@ -13,7 +13,7 @@ import shlex from re import sub import securitybot.commands as bot_commands -from securitybot.blacklist.sql_blacklist import SQLBlacklist +from securitybot.blacklist import Blacklist from securitybot.config import config # from typing import Any, Callable, Dict, List, Tuple @@ -89,8 +89,8 @@ def __init__(self, chat, tasker, auth_builder, reporting_channel): self.chat = chat chat.connect() - # Load blacklist from SQL - self.blacklist = SQLBlacklist() + # Load blacklist from DB + self.blacklist = Blacklist() # A dictionary to be populated with all members of the team self.users = {} # type: Dict[str, User] From a6c40e6c2fbe5e3d649a9479d37dcc5172df6b97 Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Fri, 28 Jul 2017 11:20:23 +0200 Subject: [PATCH 12/19] Remove SQL specialized code --- securitybot/ignored_alerts.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/securitybot/ignored_alerts.py b/securitybot/ignored_alerts.py index 5b34dfc..619a607 100644 --- a/securitybot/ignored_alerts.py +++ b/securitybot/ignored_alerts.py @@ -4,6 +4,7 @@ import pytz from datetime import timedelta, datetime from securitybot.db.engine import DbEngine +from securitybot.config import config from typing import Dict @@ -12,7 +13,8 @@ def __update_ignored_list() -> None: ''' Prunes the ignored table of old ignored alerts. ''' - DbEngine().execute('''DELETE FROM ignored WHERE until <= NOW()''') + if config['queries'].get('update_ignored_list', False): + DbEngine().execute(config['queries']['update_ignored_list']) def get_ignored(username: str) -> Dict[str, str]: @@ -26,7 +28,7 @@ def get_ignored(username: str) -> Dict[str, str]: Dict[str, str]: A mapping of ignored alert titles to reasons ''' __update_ignored_list() - rows = DbEngine().execute('''SELECT title, reason FROM ignored WHERE ldap = %s''', (username,)) + rows = DbEngine().execute(config['queries']['get_ignored'], (username,)) return {row[0]: row[1] for row in rows} @@ -44,7 +46,5 @@ def ignore_task(username: str, title: str, reason: str, ttl: timedelta) -> None: ''' expiry_time = datetime.now(tz=pytz.utc) + ttl # NB: Non-standard MySQL specific query - DbEngine().execute('''INSERT INTO ignored (ldap, title, reason, until) - VALUES (%s, %s, %s, %s) - ON DUPLICATE KEY UPDATE reason=VALUES(reason), until=VALUES(until) - ''', (username, title, reason, expiry_time.strftime('%Y-%m-%d %H:%M:%S'))) + DbEngine().execute(config['queries']['ignore_task'], + (username, title, reason, expiry_time.strftime('%Y-%m-%d %H:%M:%S'))) From 41483ea807f4703de812a80c4cea98cf46ff5cec Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Fri, 28 Jul 2017 11:23:59 +0200 Subject: [PATCH 13/19] New config file template --- config/bot.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/config/bot.yaml b/config/bot.yaml index b08a77b..89ad044 100644 --- a/config/bot.yaml +++ b/config/bot.yaml @@ -1,9 +1,12 @@ includes: messages_path: config/messages.yaml commands_path: config/commands.yaml + queries: + mysql: config/queries/mysql.yaml database: type: sql engine: SQLEngine + queries: mysql user: root password: password host: host.example.com @@ -19,11 +22,22 @@ duo: ikey: 'IKEY' skey: 'SKEY' endpoint: 'XXXXXX.duosecurity.com' + auth_time: 3600 slack: username: 'username' token: 'TOKEN' reporting_channel: 'ChannelID' icon_url: '' +bot: + timers: + task_poll_time: 60 + reporting_time: 60 + time: + opening_hour: 10 + closing_hour: 18 + local_tz: pytz.timezone('America/Los_Angeles') + sanitizing: + punctuation: "[.,!?\'\"`]" debug: level: 'debug' From bcb9b594e927f3567e01be0724a9043f6b402943 Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Fri, 28 Jul 2017 11:28:19 +0200 Subject: [PATCH 14/19] Remove bad option in config file --- config/bot.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/bot.yaml b/config/bot.yaml index 89ad044..df3f18f 100644 --- a/config/bot.yaml +++ b/config/bot.yaml @@ -36,8 +36,6 @@ bot: opening_hour: 10 closing_hour: 18 local_tz: pytz.timezone('America/Los_Angeles') - sanitizing: - punctuation: "[.,!?\'\"`]" debug: level: 'debug' From e3e737d651092241fb1aaa26442fe795fc8ead72 Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Fri, 28 Jul 2017 11:28:59 +0200 Subject: [PATCH 15/19] Remove SQL artifacts and make the main able to load another config file in argv[1] --- main.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/main.py b/main.py index 7c42bde..6d70c1a 100644 --- a/main.py +++ b/main.py @@ -3,20 +3,26 @@ from securitybot.bot import SecurityBot from securitybot.chat.slack import Slack -from securitybot.tasker.sql_tasker import SQLTasker +from securitybot.tasker import Tasker from securitybot.auth.duo import DuoAuth from securitybot.db.engine import DbEngine import duo_client from securitybot.config import config +from sys import argv def init(): # Setup logging - config.load_config('config/bot.yaml') logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s %(levelname)s] %(message)s') logging.getLogger('requests').setLevel(logging.WARNING) logging.getLogger('usllib3').setLevel(logging.WARNING) + import pdb + pdb.set_trace() + if len(argv) > 1 and argv[1] != "": + config.load_config(argv[1]) + else: + config.load_config('config/bot.yaml') def main(): @@ -38,7 +44,7 @@ def main(): raise chat = Slack(config['slack']) - tasker = SQLTasker() + tasker = Tasker() sb = SecurityBot(chat, tasker, duo_builder, config['slack']['reporting_channel']) sb.run() From ea008e28e5b53070b95ea60fb47f5591d336eb71 Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Fri, 28 Jul 2017 11:29:23 +0200 Subject: [PATCH 16/19] Add sql queries templates --- config/queries/mysql.yaml | 61 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 config/queries/mysql.yaml diff --git a/config/queries/mysql.yaml b/config/queries/mysql.yaml new file mode 100644 index 0000000..a458810 --- /dev/null +++ b/config/queries/mysql.yaml @@ -0,0 +1,61 @@ +--- +update_ignored_list: > + DELETE FROM ignored WHERE until <= NOW() + +get_ignored: > + SELECT title, reason FROM ignored WHERE ldap = %s + +ignore_task: > + INSERT INTO ignored (ldap, title, reason, until) + VALUES (%s, %s, %s, %s) + ON DUPLICATE KEY UPDATE reason=VALUES(reason), until=VALUES(until) + +blacklist_list: > + SELECT * FROM blacklist + +blacklist_add: > + INSERT INTO blacklist (ldap) VALUES (%s) + +blacklist_remove: > + DELETE FROM blacklist WHERE ldap = %s + +new_alert_status: > + INSERT INTO alert_status (hash, status) VALUES (UNHEX(%s), %s) + +new_alert_alerts: > + INSERT INTO alerts (hash, ldap, title, description, reason, url, event_time) + VALUES (UNHEX(%s), %s, %s, %s, %s, %s, NOW()) + +new_alert_user_response: > + INSERT INTO user_responses (hash, comment, performed, authenticated) + VALUES (UNHEX(%s), '', false, false) + +get_alerts: > + SELECT HEX(alerts.hash), + title, + ldap, + reason, + description, + url, + performed, + comment, + authenticated, + status + FROM alerts + JOIN user_responses ON alerts.hash = user_responses.hash + JOIN alert_status ON alerts.hash = alert_status.hash + WHERE status = %s + +set_status: > + UPDATE alert_status + SET status=%s + WHERE hash=UNHEX(%s) + +set_response: > + UPDATE user_responses + SET comment=%s, + performed=%s, + authenticated=%s + WHERE hash=UNHEX(%s) + +... \ No newline at end of file From b6e3293504f00aede93f89ee2aa0e3ba9e09b20e Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Fri, 28 Jul 2017 11:32:24 +0200 Subject: [PATCH 17/19] Fix missing import --- securitybot/auth/duo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/securitybot/auth/duo.py b/securitybot/auth/duo.py index 63b55e5..820dbed 100644 --- a/securitybot/auth/duo.py +++ b/securitybot/auth/duo.py @@ -5,7 +5,7 @@ __email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' import logging -from datetime import datetime +from datetime import datetime, timedelta from urllib.parse import urlencode from securitybot.auth.auth import Auth, AuthStates from securitybot.config import config From 4e2d57254b4d636dfdb08f98592cd8d3feef8b14 Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Fri, 28 Jul 2017 11:33:28 +0200 Subject: [PATCH 18/19] Remove unused import --- securitybot/state_machine.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/securitybot/state_machine.py b/securitybot/state_machine.py index 0129363..4bd9684 100644 --- a/securitybot/state_machine.py +++ b/securitybot/state_machine.py @@ -8,7 +8,6 @@ import logging from collections import defaultdict -from typing import Callable class StateMachine(object): ''' @@ -81,18 +80,19 @@ def __init__(self, states, transitions, initial, during=None, on_enter=None, # Validate transition for correct states if transition['source'] not in self._states: raise StateMachineException('Invalid source state: {0}' - .format(transition['source'])) + .format(transition['source'])) if transition['dest'] not in self._states: raise StateMachineException('Invalid destination state: {0}' - .format(transition['dest'])) + .format(transition['dest'])) source_state = self._states[transition['source']] dest_state = self._states[transition['dest']] self._transitions[transition['source']].append(Transition(source_state, - dest_state, - transition.get('condition', None), - transition.get('action', None) - )) + dest_state, + transition.get( + 'condition', None), + transition.get('action', None) + )) def step(self): # type: () -> None @@ -114,12 +114,14 @@ def step(self): self.state.on_enter() break + class State(object): ''' A simple representation of a state in `StateMachine`. Each state has a function to perform while it's active, when it's entered into, and when it's exited. These functions may be None. ''' + def __init__(self, name, during, on_enter, on_exit): # type: (str, Callable[..., None], Callable[..., None], Callable[..., None]) -> None ''' @@ -163,6 +165,7 @@ def on_exit(self): if self._on_exit is not None: self._on_exit() + class Transition(object): ''' A transition object to move between states. Each transition object holds @@ -207,5 +210,6 @@ def action(self): if self._action is not None: self._action() + class StateMachineException(Exception): pass From 4ec54f9887f601b610a90c17645ca2551613fff9 Mon Sep 17 00:00:00 2001 From: Antoine Cardon Date: Fri, 28 Jul 2017 11:49:03 +0200 Subject: [PATCH 19/19] [OCD] Rearrange imports --- securitybot/auth/duo.py | 3 ++- securitybot/bot.py | 7 ++++--- securitybot/chat/chat.py | 5 +++-- securitybot/chat/slack.py | 4 ++-- securitybot/commands.py | 1 - securitybot/config.py | 1 + securitybot/db/engine.py | 3 ++- securitybot/db/sql.py | 2 +- securitybot/ignored_alerts.py | 3 ++- securitybot/tasker.py | 3 ++- securitybot/user.py | 1 + securitybot/util.py | 1 + 12 files changed, 21 insertions(+), 13 deletions(-) diff --git a/securitybot/auth/duo.py b/securitybot/auth/duo.py index 820dbed..c4cfb6a 100644 --- a/securitybot/auth/duo.py +++ b/securitybot/auth/duo.py @@ -7,9 +7,10 @@ import logging from datetime import datetime, timedelta from urllib.parse import urlencode +from typing import Callable + from securitybot.auth.auth import Auth, AuthStates from securitybot.config import config -from typing import Callable class DuoAuth(Auth): diff --git a/securitybot/bot.py b/securitybot/bot.py index d4bd6d1..c608062 100644 --- a/securitybot/bot.py +++ b/securitybot/bot.py @@ -6,13 +6,14 @@ __email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' import logging -from securitybot.user import User -import time -from datetime import datetime, timedelta import pytz import shlex +import time +from datetime import datetime, timedelta from re import sub + import securitybot.commands as bot_commands +from securitybot.user import User from securitybot.blacklist import Blacklist from securitybot.config import config diff --git a/securitybot/chat/chat.py b/securitybot/chat/chat.py index 8ed6b0d..4761778 100644 --- a/securitybot/chat/chat.py +++ b/securitybot/chat/chat.py @@ -5,9 +5,10 @@ __author__ = 'Alex Bertsch, Antoine Cardon' __email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' -from securitybot.user import User -from typing import List, Dict, Any from abc import ABCMeta, abstractmethod +from typing import List, Dict, Any + +from securitybot.user import User class Chat(object, metaclass=ABCMeta): diff --git a/securitybot/chat/slack.py b/securitybot/chat/slack.py index 5c590a6..2c9ce45 100644 --- a/securitybot/chat/slack.py +++ b/securitybot/chat/slack.py @@ -6,11 +6,11 @@ import logging from slackclient import SlackClient +from typing import Callable, Any, Dict, List + from securitybot.user import User from securitybot.chat.chat import Chat, ChatException -from typing import Callable, Any, Dict, List - class Slack(Chat): ''' diff --git a/securitybot/commands.py b/securitybot/commands.py index c2fa9dd..93b5884 100644 --- a/securitybot/commands.py +++ b/securitybot/commands.py @@ -10,7 +10,6 @@ __email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' import re - from datetime import timedelta import securitybot.ignored_alerts as ignored_alerts diff --git a/securitybot/config.py b/securitybot/config.py index 1c82ccc..4458b82 100644 --- a/securitybot/config.py +++ b/securitybot/config.py @@ -4,6 +4,7 @@ import yaml import logging from typing import Dict + from securitybot.utils.class_helper import Singleton diff --git a/securitybot/db/engine.py b/securitybot/db/engine.py index b06e64a..f22ca83 100644 --- a/securitybot/db/engine.py +++ b/securitybot/db/engine.py @@ -6,9 +6,10 @@ __email__ = 'antoine.cardon@algolia.com' from typing import Any, List, Dict -from securitybot.utils.class_helper import Singleton from abc import ABCMeta, abstractmethod +from securitybot.utils.class_helper import Singleton + engines = {} diff --git a/securitybot/db/sql.py b/securitybot/db/sql.py index c9f17ff..3543879 100644 --- a/securitybot/db/sql.py +++ b/securitybot/db/sql.py @@ -6,9 +6,9 @@ import MySQLdb import logging +from typing import Any, Dict, Sequence from securitybot.db.engine import EngineInterface, EngineException, register_engine -from typing import Any, Dict, Sequence @register_engine diff --git a/securitybot/ignored_alerts.py b/securitybot/ignored_alerts.py index 619a607..ab06e3b 100644 --- a/securitybot/ignored_alerts.py +++ b/securitybot/ignored_alerts.py @@ -3,9 +3,10 @@ ''' import pytz from datetime import timedelta, datetime +from typing import Dict + from securitybot.db.engine import DbEngine from securitybot.config import config -from typing import Dict def __update_ignored_list() -> None: diff --git a/securitybot/tasker.py b/securitybot/tasker.py index 6ac0019..6ffa690 100644 --- a/securitybot/tasker.py +++ b/securitybot/tasker.py @@ -8,9 +8,10 @@ __email__ = 'abertsch@dropbox.com, antoine.cardon@algolia.com' from enum import Enum, unique +from typing import List + from securitybot.db.engine import DbEngine from securitybot.config import config -from typing import List @unique diff --git a/securitybot/user.py b/securitybot/user.py index 3120770..9d7c61c 100644 --- a/securitybot/user.py +++ b/securitybot/user.py @@ -8,6 +8,7 @@ import logging import pytz from datetime import datetime, timedelta + import securitybot.ignored_alerts as ignored_alerts from securitybot.auth.auth import AuthStates from securitybot.state_machine import StateMachine diff --git a/securitybot/util.py b/securitybot/util.py index 4f14a64..2854df1 100644 --- a/securitybot/util.py +++ b/securitybot/util.py @@ -6,6 +6,7 @@ import os from datetime import datetime, timedelta from collections import namedtuple + from securitybot.db.engine import DbEngine from securitybot.config import config from securitybot.tasker import StatusLevel