diff --git a/.gitignore b/.gitignore index c3f77ce1a6..9ca157a05b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# usual mac files +.DS_Store + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/README.md b/README.md index d72b2d933e..84a315e2f3 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,34 @@ +security-content ![security-content](static/logo.png) +===== + +Contains a collection of security stories with their corresponding detection, investigative, contexual and support splunk searches | branch | build status | | --- | --- | | develop| [![develop status](https://circleci.com/gh/splunk/security-content/tree/develop.svg?style=svg&circle-token=67ad1fa7779c57d7e5bcfc42bd617baf607ec269)](https://circleci.com/gh/splunk/security-content/tree/develop)| | master | [![master status](https://circleci.com/gh/splunk/security-content/tree/master.svg?style=svg&circle-token=67ad1fa7779c57d7e5bcfc42bd617baf607ec269)](https://circleci.com/gh/splunk/security-content/tree/master)| -# security-content -Contains a collection of security stories with their corresponding detection, investigative, contexual and support splunk searches - -# Consumption +# Usage Can be consumed using: -* API (https://api.splunksecuritycontent.com) -* CLI + +#### [API](https://github.com/splunk/security-content-api) +`curl https://g7jbilqdth.execute-api.us-west-2.amazonaws.com/api/` + +#### [CLI](https://github.com/splunk/security-content-api/blob/master/content-update.py) +`python content-update.py -o $SPLUNK_HOME/etc/apps/DA-ESS-ContentUpdate --splunk_user admin --splunk_password xxxx` # Structure -`src/` - splunk content app source files, includes lookups, binaries, and defaul config files +* [stories/](stories/) - contains all analytics stories/use cases for ESCU +* [detections/](detections/) - splunk, uba and phantom detections that power stories +* [investigations/](investigations/) - splunk, and phantom investigation content that are used in stories +* [responses/](responses/) - automated splunk and phantom responses that are used in stories +* [baselines/](baselines/) - phantom and Splunk baseline needed to support detections in stories +* [src/](src/) - splunk content app source files, includes lookups, binaries, and defaul config files +* [bin/](bin/) - where all binaries to produce, and test content lives +* [spec/](spec/) - location of all spec files that describe ESCU content +* [docs/](docs/) - documentation for all of the spec files # Developing - For getting pre-commit checks, install the hooks see below for steps: 1. Install circleci [CLI Tool](https://circleci.com/docs/2.0/local-cli/#installation) 2. create virtualenv and install requirements: `virtualenv venv && source venv/bin/activate && pip install -r requirements.txt` @@ -25,3 +37,6 @@ For getting pre-commit checks, install the hooks see below for steps: To test a local change to CI or build make sure you are running docker and then `circleci local execute -e GITHUB_TOKEN=$GITHUB_TOKEN --branch ` +To generate docs from schema automatically +1. install https://github.com/adobe/jsonschema2md +2. `jsonschema2md -d spec/v2/detections.json.spec -o docs` diff --git a/escu/searches/support_add_prohibited_processes_to_es.json b/baselines/add_prohibited_processes_to_es.json similarity index 100% rename from escu/searches/support_add_prohibited_processes_to_es.json rename to baselines/add_prohibited_processes_to_es.json diff --git a/escu/searches/support_approved_aws_service_accounts.json b/baselines/approved_aws_service_accounts.json similarity index 100% rename from escu/searches/support_approved_aws_service_accounts.json rename to baselines/approved_aws_service_accounts.json diff --git a/escu/searches/support_assets_sorted_by_category.json b/baselines/assets_sorted_by_category.json similarity index 100% rename from escu/searches/support_assets_sorted_by_category.json rename to baselines/assets_sorted_by_category.json diff --git a/escu/searches/support_baseline_api_calls_per_user_arn.json b/baselines/baseline_api_calls_per_user_arn.json similarity index 100% rename from escu/searches/support_baseline_api_calls_per_user_arn.json rename to baselines/baseline_api_calls_per_user_arn.json diff --git a/escu/searches/support_baseline_aws_regions.json b/baselines/baseline_aws_regions.json similarity index 100% rename from escu/searches/support_baseline_aws_regions.json rename to baselines/baseline_aws_regions.json diff --git a/escu/searches/support_baseline_blocked_vpc_traffic.json b/baselines/baseline_blocked_vpc_traffic.json similarity index 100% rename from escu/searches/support_baseline_blocked_vpc_traffic.json rename to baselines/baseline_blocked_vpc_traffic.json diff --git a/escu/searches/support_baseline_network_acl_modifications.json b/baselines/baseline_network_acl_modifications.json similarity index 100% rename from escu/searches/support_baseline_network_acl_modifications.json rename to baselines/baseline_network_acl_modifications.json diff --git a/escu/searches/support_baseline_s3_deletion_by_arn.json b/baselines/baseline_s3_deletion_by_arn.json similarity index 100% rename from escu/searches/support_baseline_s3_deletion_by_arn.json rename to baselines/baseline_s3_deletion_by_arn.json diff --git a/escu/searches/support_baseline_security_group_activity_by_arn.json b/baselines/baseline_security_group_activity_by_arn.json similarity index 100% rename from escu/searches/support_baseline_security_group_activity_by_arn.json rename to baselines/baseline_security_group_activity_by_arn.json diff --git a/escu/searches/support_brand_abuse_dnstwist_domains.json b/baselines/brand_abuse_dnstwist_domains.json similarity index 100% rename from escu/searches/support_brand_abuse_dnstwist_domains.json rename to baselines/brand_abuse_dnstwist_domains.json diff --git a/escu/searches/support_discover_dns_records.json b/baselines/discover_dns_records.json similarity index 100% rename from escu/searches/support_discover_dns_records.json rename to baselines/discover_dns_records.json diff --git a/escu/searches/support_identify_ports_on_network.json b/baselines/identify_ports_on_network.json similarity index 100% rename from escu/searches/support_identify_ports_on_network.json rename to baselines/identify_ports_on_network.json diff --git a/escu/searches/support_identify_systems_creating_rdp_traffic.json b/baselines/identify_systems_creating_rdp_traffic.json similarity index 100% rename from escu/searches/support_identify_systems_creating_rdp_traffic.json rename to baselines/identify_systems_creating_rdp_traffic.json diff --git a/escu/searches/support_identify_systems_receiving_rdp_traffic.json b/baselines/identify_systems_receiving_rdp_traffic.json similarity index 100% rename from escu/searches/support_identify_systems_receiving_rdp_traffic.json rename to baselines/identify_systems_receiving_rdp_traffic.json diff --git a/escu/searches/support_identify_systems_using_remote_desktop.json b/baselines/identify_systems_using_remote_desktop.json similarity index 100% rename from escu/searches/support_identify_systems_using_remote_desktop.json rename to baselines/identify_systems_using_remote_desktop.json diff --git a/escu/searches/support_monitor_successful_backups.json b/baselines/monitor_successful_backups.json similarity index 100% rename from escu/searches/support_monitor_successful_backups.json rename to baselines/monitor_successful_backups.json diff --git a/escu/searches/support_monitor_successful_windows_updates.json b/baselines/monitor_successful_windows_updates.json similarity index 100% rename from escu/searches/support_monitor_successful_windows_updates.json rename to baselines/monitor_successful_windows_updates.json diff --git a/escu/searches/support_monitor_unsuccessful_backups.json b/baselines/monitor_unsuccessful_backups.json similarity index 100% rename from escu/searches/support_monitor_unsuccessful_backups.json rename to baselines/monitor_unsuccessful_backups.json diff --git a/escu/searches/support_monitor_unsuccessful_windows_updates.json b/baselines/monitor_unsuccessful_windows_updates.json similarity index 100% rename from escu/searches/support_monitor_unsuccessful_windows_updates.json rename to baselines/monitor_unsuccessful_windows_updates.json diff --git a/escu/searches/support_previously_seen_aws_cross_account_activity.json b/baselines/previously_seen_aws_cross_account_activity.json similarity index 100% rename from escu/searches/support_previously_seen_aws_cross_account_activity.json rename to baselines/previously_seen_aws_cross_account_activity.json diff --git a/escu/searches/support_previously_seen_cmd_arguments.json b/baselines/previously_seen_cmd_arguments.json similarity index 100% rename from escu/searches/support_previously_seen_cmd_arguments.json rename to baselines/previously_seen_cmd_arguments.json diff --git a/escu/searches/support_previously_seen_ec2_amis.json b/baselines/previously_seen_ec2_amis.json similarity index 100% rename from escu/searches/support_previously_seen_ec2_amis.json rename to baselines/previously_seen_ec2_amis.json diff --git a/escu/searches/support_previously_seen_ec2_instance_types.json b/baselines/previously_seen_ec2_instance_types.json similarity index 100% rename from escu/searches/support_previously_seen_ec2_instance_types.json rename to baselines/previously_seen_ec2_instance_types.json diff --git a/escu/searches/support_previously_seen_ec2_launches_by_user.json b/baselines/previously_seen_ec2_launches_by_user.json similarity index 100% rename from escu/searches/support_previously_seen_ec2_launches_by_user.json rename to baselines/previously_seen_ec2_launches_by_user.json diff --git a/escu/searches/support_previously_seen_ec2_modifications.json b/baselines/previously_seen_ec2_modifications.json similarity index 100% rename from escu/searches/support_previously_seen_ec2_modifications.json rename to baselines/previously_seen_ec2_modifications.json diff --git a/escu/searches/support_previously_seen_provisioning_activity_src.json b/baselines/previously_seen_provisioning_activity_src.json similarity index 100% rename from escu/searches/support_previously_seen_provisioning_activity_src.json rename to baselines/previously_seen_provisioning_activity_src.json diff --git a/escu/searches/support_previously_seen_s3_remote_ip.json b/baselines/previously_seen_s3_remote_ip.json similarity index 100% rename from escu/searches/support_previously_seen_s3_remote_ip.json rename to baselines/previously_seen_s3_remote_ip.json diff --git a/escu/searches/support_previously_seen_user_roles.json b/baselines/previously_seen_user_roles.json similarity index 100% rename from escu/searches/support_previously_seen_user_roles.json rename to baselines/previously_seen_user_roles.json diff --git a/escu/searches/support_previously_seen_users.json b/baselines/previously_seen_users.json similarity index 100% rename from escu/searches/support_previously_seen_users.json rename to baselines/previously_seen_users.json diff --git a/escu/searches/support_previously_seen_windows_service_starts.json b/baselines/previously_seen_windows_service_starts.json similarity index 100% rename from escu/searches/support_previously_seen_windows_service_starts.json rename to baselines/previously_seen_windows_service_starts.json diff --git a/escu/searches/support_systems_ready_for_spectre_meltdown_patch.json b/baselines/systems_ready_for_spectre_meltdown_patch.json similarity index 100% rename from escu/searches/support_systems_ready_for_spectre_meltdown_patch.json rename to baselines/systems_ready_for_spectre_meltdown_patch.json diff --git a/bin/config.ini b/bin/config.ini deleted file mode 100644 index 894a621d2f..0000000000 --- a/bin/config.ini +++ /dev/null @@ -1,2 +0,0 @@ -[confluence] -url=https://confluence.splunk.com diff --git a/bin/generate.py b/bin/generate.py new file mode 100644 index 0000000000..9807e8a981 --- /dev/null +++ b/bin/generate.py @@ -0,0 +1,204 @@ +#!/usr/bin/python + +''' +Generates splunk configurations from manifest files under the security-content repo. +''' + +import glob +import json +import argparse +from os import path + +ALL_UUIDS = [] + + +def markdown(x): + markdown = str(x) + markdown = markdown.replace("", "`") + markdown = markdown.replace("", "`") + markdown = markdown.replace("", "**") + markdown = markdown.replace("", "**") + # list tag replacements + markdown = markdown.replace("
  1. ", "\\\n\\\n1. ") + markdown = markdown.replace("
  2. ", "\\\n\\\n1. ") + markdown = markdown.replace("
", "") + markdown = markdown.replace("", "") + markdown = markdown.replace("