diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d1790ddfbe6..d51104e0810 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,3 @@ -* @patel-bhavin @ljstella @nasbench +* @patel-bhavin @ljstella @nasbench @pyth0n1c @RavenTait @tccontre @rosplk @p4t12ick @onurmerdogan /response_templates/ @kbouchardherjavecgroup @ccl0utier @henryy-splunk -/.github/workflows/response_templates/mcopenapi_public.yml @kbouchardherjavecgroup @ccl0utier @henryy-splunk \ No newline at end of file +/.github/workflows/response_templates/mcopenapi_public.yml @kbouchardherjavecgroup @ccl0utier @henryy-splunk diff --git a/.github/workflows/appinspect.yml b/.github/workflows/appinspect.yml index 6eab313976c..36be91d3b69 100644 --- a/.github/workflows/appinspect.yml +++ b/.github/workflows/appinspect.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repository code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 diff --git a/.github/workflows/archived-ta-tracker.yml b/.github/workflows/archived-ta-tracker.yml new file mode 100644 index 00000000000..0f45898107d --- /dev/null +++ b/.github/workflows/archived-ta-tracker.yml @@ -0,0 +1,32 @@ +name: Archived TA tracker + +on: + workflow_dispatch: + schedule: + - cron: "17 7 * * *" + +permissions: + contents: read + issues: write + +jobs: + check-archived-tas: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install YAML parser + run: python -m pip install PyYAML==6.0.3 + + - name: Create or update archived TA issues + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_REF_NAME: ${{ github.ref_name }} + run: python scripts/check_archived_tas.py diff --git a/.github/workflows/build-response-templates.yml b/.github/workflows/build-response-templates.yml index 5af3719003d..075f46864ef 100644 --- a/.github/workflows/build-response-templates.yml +++ b/.github/workflows/build-response-templates.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repository code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - uses: actions/setup-python@v6 with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 323fe272956..7b65e001459 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repository code - uses: actions/checkout@v6 + uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v6 diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index 0b07ac92a32..752b6063e6f 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -6,53 +6,58 @@ on: - cron: '55 06 * * *' # Runs daily at midnight jobs: - data-source-validation-and-update: + update-data-source-tas-and-install-yml-apps: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: ref: 'develop' token: ${{ secrets.DATA_SOURCES_DEPENDABOT }} - + - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.11' + python-version: '3.14' architecture: 'x64' # or the version your script requires - name: Install Python Dependencies and ContentCTL and Atomic Red Team run: | echo "- Contentctl version - $(cat requirements.txt)" pip install -r requirements.txt - - - name: Run ContentCTL Data source TA validation - id: validate - run: | - pwd - contentctl validate --data-source-TA-validation 2>&1 | tee data_source_validation.log - continue-on-error: true - - - name: Print Validation Log - run: | - cat data_source_validation.log - rm -f =4.0.0 - - name: Update Data Sources if Validation Fails + - name: Update data source TAs and install.yml apps run: | - python .github/workflows/update_data_sources_ta.py - git status + echo "Updating data source supported_TA and repo install.yml app versions" + contentctl-ng update-data-source-tas --content-dirs . + + # TODO: This comment is intentionally left in this workflow to demonstrate how individuals + # TAs can be intentionally excluded from updates, for example when we have intentionally pinned + # to an older version of a TA. In the future, we will try to make this a formally documented + # feature of contentctl-ng, but needing to do this is fairly rare and with out other priorities, + # does not fit at this time. + # The demonstration below shows how to exlcude the windows TA from updates, if necessary. + # git checkout -- $(grep -rl "https://splunkbase.splunk.com/app/742" data_sources/) + echo "Changed data source and install.yml files:" + git status --short -- data_sources install.yml - - name: Create Pull Request + - name: Create data source TA and install.yml app update pull request uses: peter-evans/create-pull-request@v8 with: token: ${{ secrets.DATA_SOURCES_DEPENDABOT }} - commit-message: Updated TAs + commit-message: Update data source TA and install.yml app versions branch: auto-ta-update-${{ github.run_number }} base: develop - title: Automated Splunk TA Update ${{ github.run_number }} - body: "This PR contains updates to Splunk TAs made by GitHub Actions workflow." + title: Automated Data Source TA and install.yml App Update ${{ github.run_number }} + body: | + This PR contains data source supported_TA and install.yml app version updates generated by contentctl-ng. + + The workflow runs: + + ```bash + contentctl-ng update-data-source-tas --content-dirs . + ``` add-paths: | data_sources/** - contentctl.yml + install.yml diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index f9383b78c0f..32893e4b612 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -9,10 +9,10 @@ jobs: pull-requests: write runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: repository: "splunk/security_content" - - uses: actions/labeler@v6 + - uses: actions/labeler@v7 with: sync-labels: true configuration-path: '.github/labeler.yml' \ No newline at end of file diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 486722289f4..ad8abf2ddaf 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -2,6 +2,12 @@ name: unit-testing on: pull_request: types: [opened, reopened, synchronize] + paths: + - data_sources/** + - detections/** + - lookups/** + - macros/** + - install.yml jobs: unit-testing: runs-on: large-ubuntu-22.04-32core @@ -9,11 +15,11 @@ jobs: steps: #For fork PRs, always check out security_content and the PR target in security content! - name: Check out the repository code - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: 'splunk/security_content' #this should be the TARGET repo of the PR. we hardcode it for now ref: ${{ github.base_ref }} - + - name: Print out information abour PR target run: | echo "The PR target branch is: ${{ github.base_ref }}" @@ -21,8 +27,8 @@ jobs: echo "My current branch is" git branch --show-current git rev-parse HEAD - - + + - name: Set up Python uses: actions/setup-python@v6 with: @@ -34,38 +40,38 @@ jobs: run: | echo "- Build Tool Version - $(cat requirements.txt)" pip install -r requirements.txt - + # Check out the PR, even if it lives in a fork. - # Instructions for pulling a PR were taken from: + # Instructions for pulling a PR were taken from: # https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally - name: Checkout the PR branch and the target to calculate changed files for testing run: | - + echo "Current Branch (Head Ref): ${{ github.head_ref }}" echo "Target Branch (Base Ref): ${{ github.base_ref }}" git pull > /dev/null 2>&1 #We checkout into a new branch - new_branch_for_testing to avoid name collisions with develop incase the forked PR is from develop git fetch origin pull/${{ github.event.pull_request.number }}/head:new_branch_for_testing - #We must specifically get the PR's target branch from security_content, not the one that resides in the fork PR's forked repo + #We must specifically get the PR's target branch from security_content, not the one that resides in the fork PR's forked repo git switch new_branch_for_testing - + - name: Run a contentctl-ng build command to create a package that will be tested. run: | contentctl-ng build - + - name: Start the test environment run: | docker run -d --platform linux/amd64 -p 8088:8088 -p 8089:8089 -p 8000:8000 -e 'SPLUNK_START_ARGS=--accept-license' -e 'SPLUNK_GENERAL_TERMS=--accept-sgt-current-at-splunk-com' -e 'SPLUNK_PASSWORD=Chang3d!' --name splunk splunk/splunk:latest # Wait some time for this environment to be ready sleep 180 - + - name: Run a contentctl-ng install to configure the testing environment env: APPINSPECTUSERNAME: "${{ secrets.APPINSPECTUSERNAME }}" - APPINSPECTPASSWORD: "${{ secrets.APPINSPECTPASSWORD }}" + APPINSPECTPASSWORD: "${{ secrets.APPINSPECTPASSWORD }}" run : | - contentctl-ng install --splunkbase-username "$APPINSPECTUSERNAME" --splunkbase-password "$APPINSPECTPASSWORD" - + contentctl-ng install --splunkbase-username "$APPINSPECTUSERNAME" --splunkbase-password "$APPINSPECTPASSWORD" + - name: Test content which has changed between this branch and the target branch run: | contentctl-ng test --verbose --post-test-behavior NEVER_PAUSE --mode CHANGED --git-ref ${{ github.base_ref }} @@ -81,7 +87,7 @@ jobs: test_results/summary.yml dist/*.tar.gz - # Print entire result summary so that the users can view it in the Github Actions logs + # Print entire result summary so that the users can view it in the Github Actions logs - name: Print entire test_results/summary.yml if: always() run: cat test_results/summary.yml @@ -89,7 +95,7 @@ jobs: # Run a simple custom script created to pretty print results in a markdown friendly format in Github Actions Summary - name: Check the test_results/summary.yml for pass/fail. if: always() - run: | - echo "This job will fail if there are failures in unit-testing" + run: | + echo "This job will fail if there are failures in unit-testing" python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY - echo "The Unit testing is completed. See details in the unit-testing job summary UI " \ No newline at end of file + echo "The Unit testing is completed. See details in the unit-testing job summary UI " diff --git a/.github/workflows/update_data_sources_ta.py b/.github/workflows/update_data_sources_ta.py deleted file mode 100644 index 77ae138a3ee..00000000000 --- a/.github/workflows/update_data_sources_ta.py +++ /dev/null @@ -1,98 +0,0 @@ -import os -import yaml -from collections import OrderedDict - -# Custom YAML loader to preserve the order of keys -class OrderedLoader(yaml.SafeLoader): - pass - -def construct_mapping(loader, node): - loader.flatten_mapping(node) - return OrderedDict(loader.construct_pairs(node)) - -OrderedLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - construct_mapping -) - -# Custom YAML dumper to preserve the order of keys -class OrderedDumper(yaml.SafeDumper): - pass - -def dict_representer(dumper, data): - return dumper.represent_dict(data.items()) - -OrderedDumper.add_representer(OrderedDict, dict_representer) - -# Define the paths -log_file_path = 'data_source_validation.log' -data_sources_dir = 'data_sources' -contentctl_file_path = 'contentctl.yml' - -def update_data_sources(ta_name, latest_version): - # Update the YAML files in the data sources directory - for filename in os.listdir(data_sources_dir): - if filename.endswith('.yml'): - file_path = os.path.join(data_sources_dir, filename) - with open(file_path, 'r') as yml_file: - data = yaml.load(yml_file, Loader=OrderedLoader) - - # Check if the TA name matches and update the version - updated = False - for ta in data.get('supported_TA', []): - if ta['name'] == ta_name: - if ta['version'] != latest_version: - ta['version'] = latest_version - updated = True - - # Write the updated data back to the YAML file - if updated: - with open(file_path, 'w') as yml_file: - yaml.dump(data, yml_file, Dumper=OrderedDumper) - -def update_contentctl_yml(title, new_version): - # Load the existing YAML file - with open(contentctl_file_path, 'r') as file: - content = yaml.load(file, Loader=OrderedLoader) - - # Iterate over the apps to find the title and update the version and hardcoded_path - updated = False - for app in content.get('apps', []): - if app.get('title') == title: - if app.get('version') != new_version: - app['version'] = new_version - updated = True - print(f"Updated {title} in contentctl.yml to version {new_version}") - - # Update the hardcoded_path if it exists - if 'hardcoded_path' in app: - base_url, current_version = app['hardcoded_path'].rsplit('_', 1) - new_hardcoded_path = f"{base_url}_{new_version.replace('.', '')}.tgz" - app['hardcoded_path'] = new_hardcoded_path - print(f"Updated hardcoded_path for {title} to {new_hardcoded_path}") - - # Write the updated content back to the YAML file if changes were made - if updated: - with open(contentctl_file_path, 'w') as file: - yaml.dump(content, file, Dumper=OrderedDumper, default_flow_style=False) - -def main(): - # Read the log file to find version mismatches - with open(log_file_path, 'r') as log_file: - log_lines = log_file.readlines() - - # Parse the log file to find the TA name and the latest version - for i, line in enumerate(log_lines): - if 'Version mismatch' in line: - ta_name = log_lines[i].split("'")[3].strip() - latest_version = log_lines[i + 1].split(':')[1].strip() - print(f"Found version mismatch for TA: {ta_name}, updating to version: {latest_version}") - - # Update data sources and contentctl.yml - update_data_sources(ta_name, latest_version) - update_contentctl_yml(ta_name, latest_version) - - print("Version updates completed.") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/.github/workflows/update_nist_ai_rmf_detection_map.yml b/.github/workflows/update_nist_ai_rmf_detection_map.yml new file mode 100644 index 00000000000..1d362f48781 --- /dev/null +++ b/.github/workflows/update_nist_ai_rmf_detection_map.yml @@ -0,0 +1,59 @@ +name: NIST AI RMF Detection Map Update + +on: + workflow_dispatch: + schedule: + - cron: '25 06 * * *' + +permissions: + contents: write + pull-requests: write + +jobs: + update-nist-ai-rmf-detection-map: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + ref: develop + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.14' + architecture: x64 + + - name: Install Python dependencies + run: | + pip install PyYAML==6.0.3 + + - name: Generate NIST AI RMF detection map + run: | + python scripts/nist_dashboard/generate_nist_ai_rmf_detection_map.py --modified-date "$(date -u +%F)" + echo "Changed NIST AI RMF lookup files:" + git status --short -- lookups/csv/detection_subcategory_map.csv lookups/csv/detection_subcategory_map.yml + + - name: Create NIST AI RMF detection map update pull request + uses: peter-evans/create-pull-request@v8 + with: + token: ${{ github.token }} + commit-message: Update NIST AI RMF detection map + branch: auto-nist-ai-rmf-detection-map-${{ github.run_number }} + base: develop + title: Automated NIST AI RMF lookup table update + body: | + This PR updates the generated NIST AI RMF detection-to-subcategory lookup. + + The workflow runs: + + ```bash + python scripts/nist_dashboard/generate_nist_ai_rmf_detection_map.py --modified-date "$(date -u +%F)" + ``` + + When `lookups/csv/detection_subcategory_map.csv` changes, the generator also bumps `lookups/csv/detection_subcategory_map.yml` by incrementing `version` and setting `modification_date`. + add-paths: | + lookups/csv/detection_subcategory_map.csv + lookups/csv/detection_subcategory_map.yml diff --git a/.github/workflows/update_splunk_tas.yml b/.github/workflows/update_splunk_tas.yml index 2fcce67b1fe..0caa429b36e 100644 --- a/.github/workflows/update_splunk_tas.yml +++ b/.github/workflows/update_splunk_tas.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: ref: 'develop' diff --git a/.github/workflows/yaml-validation.yml b/.github/workflows/yaml-validation.yml index e9bf0ad42b9..fa5640e36c6 100644 --- a/.github/workflows/yaml-validation.yml +++ b/.github/workflows/yaml-validation.yml @@ -5,12 +5,26 @@ on: paths: - 'detections/**/*.yml' - 'detections/**/*.yaml' + - '.yamlfieldorder' + - '.yamlfmt' + - '.yamllint' + - '.pre-commit-config.yaml' + - 'scripts/enforce_yaml_field_order.py' + - 'scripts/validate_yaml.py' + - '.github/workflows/yaml-validation.yml' push: branches: - develop paths: - 'detections/**/*.yml' - 'detections/**/*.yaml' + - '.yamlfieldorder' + - '.yamlfmt' + - '.yamllint' + - '.pre-commit-config.yaml' + - 'scripts/enforce_yaml_field_order.py' + - 'scripts/validate_yaml.py' + - '.github/workflows/yaml-validation.yml' jobs: validate: @@ -19,7 +33,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 @@ -33,7 +47,7 @@ jobs: run: pip install yamllint - name: Set up Go - uses: actions/setup-go@v6 + uses: actions/setup-go@v7 - name: Install yamlfmt run: | diff --git a/.gitignore b/.gitignore index 2a6e590e07b..5133d1798ee 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ dist/ContentPack-*.appinspect_api_results.html dist/ContentPack-*.appinspect_api_results.json atomic-red-team/ external_repos/ +schemas/ # IDE .vscode/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3b5877a65c5..c71a44920b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,3 +22,11 @@ repos: pass_filenames: true # Optional: Specify custom yamlfmt binary path if not in PATH # args: [--yamlfmt-path, /path/to/yamlfmt] + + - id: yaml-field-order + name: YAML field order (detections only) + description: Enforce configured root field order in detection YAML files + entry: python scripts/enforce_yaml_field_order.py --fix + language: python + files: ^detections/.*\.(yml|yaml)$ + pass_filenames: true diff --git a/.vscode/escu_6.code-snippets b/.vscode/escu_6.code-snippets new file mode 100644 index 00000000000..623d309083e --- /dev/null +++ b/.vscode/escu_6.code-snippets @@ -0,0 +1,229 @@ +{ + "New Detection": { + "scope": "yaml", + "prefix": "detection_escu6", + "description": "Scaffold a Splunk Security Content detection YAML", + "body": [ + "name: ${1:${TM_FILENAME_BASE/(^|[_-])(\\w)/${1:+ }${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${2:Your Name, Splunk}", + "status: production", + "type: ${3|TTP,Anomaly,Hunting,Correlation|}", + "description: ${4}", + "data_source:", + " - ", + "search: >-", + " your search here", + " | `${TM_FILENAME_BASE}_filter`", + "how_to_implement: ", + "known_false_positives: ", + "references:", + " - ", + "drilldown_searches:", + " - name: View the detection results for - \"$risk_object$\"", + " search: '%original_detection_search% | search risk_object = \"$risk_object$\"'", + " earliest_offset: $info_min_time$", + " latest_offset: $info_max_time$", + " - name: View risk events for the last 7 days for - \"$risk_object$\"", + " search: '| from datamodel Risk.All_Risk | search normalized_risk_object IN (\"$risk_object$\") | stats count min(_time) as firstTime max(_time) as lastTime values(search_name) as \"Search Name\" values(risk_message) as \"Risk Message\" values(analyticstories) as \"Analytic Stories\" values(annotations._all) as \"Annotations\" values(annotations.mitre_attack.mitre_tactic) as \"ATT&CK Tactics\" by normalized_risk_object | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`'", + " earliest_offset: 7d", + " latest_offset: \"0\"", + "finding:", + " title: ", + " entity:", + " field: ", + " type: ", + " score: ", + "intermediate_findings:", + " entities:", + " - field: ", + " type: ", + " score: ", + " message:", + "threat_objects:", + " - field: ", + " type: ", + "analytic_story:", + " - ", + "asset_type: ", + "mitre_attack_id:", + " - ", + "product:", + " - Splunk Enterprise", + " - Splunk Enterprise Security", + " - Splunk Cloud", + "category: ", + "security_domain: ", + "tests:", + " - name: True Positive Test", + " attack_data:", + " - data: ", + " source: ", + " sourcetype: ", + " test_type: unit", + ] + }, + "New Macro": { + "scope": "yaml", + "prefix": "macro_escu6", + "body": [ + "name: ${1:${TM_FILENAME_BASE/([_-])(\\w)/ ${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${1:Splunk Threat Research Team}", + "description: ${2}", + "definition: ${3}", + "$0" + ], + "description": "Scaffold a new Splunk security content macro YAML file" + }, + + "New Data Source": { + "scope": "yaml", + "prefix": "data_source_escu6", + "description": "Scaffold a new Splunk security content data source YAML file", + "body": [ + "name: ${1:${TM_FILENAME_BASE/(^|[_-])(\\w)/${1:+ }${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${1:Splunk Threat Research Team}", + "description: ${3}", + "source: ${4:sourcename}", + "sourcetype: ${5:sourcetype_value}", + "separator: ${6:EventCode}", + "separator_value: '${7}'", + "supported_TA:", + " - name: ${8}", + " url: ${9:hxxps://splunkbase.splunk.com/app/....}", + " version: ${10}", + "fields:", + " - ", + "output_fields:", + " - ", + "mitre_components:", + " - ", + "example_log: '${11}'", + "$0" + ] + }, + + "New Story": { + "scope": "yaml", + "prefix": "story_escu6", + "description": "Scaffold a new Splunk security content analytic story YAML file", + "body": [ + "name: ${1:${TM_FILENAME_BASE/(^|[_-])(\\w)/${1:+ }${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${2:Splunk Threat Research Team}", + "status: production", + "description: ${3}", + "narrative: ${4}", + "references:", + " - ", + "category:", + " - ", + "product:", + " - Splunk Enterprise", + " - Splunk Enterprise Security", + " - Splunk Cloud", + "usecase: ", + "$0" + ] + }, + + "New Lookup (CSV)": { + "scope": "yaml", + "prefix": "lookup_csv_escu6", + "description": "Scaffold a new Splunk security content CSV lookup YAML file", + "body": [ + "name: ${1:${TM_FILENAME_BASE/([_-])(\\w)/ ${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${2:Splunk Threat Research Team}", + "lookup_type: csv", + "description: ${3}", + "match_type:", + " - ${4:WILDCARD()}", + "min_matches: REMOVE ME IF NO MIN MATCHES REQUIRED", + "max_matches: REMOVE ME IF NO MAX MATCHES REQUIRED", + "case_sensitive_match: REMOVE ME IF NO CASE SENSITIVE MATCH REQUIRED", + "$0" + ] + }, + + "New Lookup (KVStore)": { + "scope": "yaml", + "prefix": "lookup_kvstore_escu6", + "description": "Scaffold a new Splunk security content KVStore lookup YAML file", + "body": [ + "name: ${1:${TM_FILENAME_BASE/([_-])(\\w)/ ${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${2:Splunk Threat Research Team}", + "lookup_type: kvstore", + "description: ${3}", + "fields:", + " - _key", + " - ${4:field_name}", + "$0" + ] + }, + + "New Dashboard": { + "scope": "yaml", + "prefix": "dashboard_escu6", + "description": "Scaffold a new Splunk security content dashboard YAML file", + "body": [ + "name: ${1:${TM_FILENAME_BASE/(^|[_-])(\\w)/${1:+ }${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${2:Splunk Threat Research Team}", + "description: ${3:Utilize this dashboard to...}", + "$0" + ] + }, + + "New Baseline": { + "scope": "yaml", + "prefix": "baseline_escu6", + "description": "Scaffold a new Splunk security content baseline YAML file", + "body": [ + "name: ${1:${TM_FILENAME_BASE/(^|[_-])(\\w)/${1:+ }${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${2:Splunk Threat Research Team}", + "status: production", + "description: ${3:This search...}", + "search: ${4}", + "how_to_implement: ${5}", + "known_false_positives: ${6}", + "references: []", + "product:", + " - Splunk Enterprise", + " - Splunk Enterprise Security", + " - Splunk Cloud", + "security_domain: ${7}", + "schedule: Default Baseline", + "$0" + ] + } + +} diff --git a/.yamlfieldorder b/.yamlfieldorder new file mode 100644 index 00000000000..36f5054b522 --- /dev/null +++ b/.yamlfieldorder @@ -0,0 +1,46 @@ +# Root field order for detection YAML files. +# +# The formatter moves whole top-level YAML blocks and does not re-emit YAML. +# This preserves indentation, spaces, comments, scalar style, and nested content +# inside each field block. +target_paths: + - detections/ + +extensions: + - .yml + - .yaml + +# Unknown top-level fields are kept intact and appended after configured fields. +unknown_fields: append + +root_field_order: + - name + - id + - version + - creation_date + - modification_date + - author + - status + - type + - description + - data_source + - search + - how_to_implement + - known_false_positives + - references + - drilldown_searches + - finding + - intermediate_findings + - threat_objects + - analytic_story + - threat_group + - asset_type + - cve + - atomic_guid + - mitre_attack_id + - product + - category + - security_domain + - baselines + - tests + - deprecation_info diff --git a/README.md b/README.md index c196f78ae63..340b89cf323 100644 --- a/README.md +++ b/README.md @@ -25,13 +25,13 @@ Welcome to the Splunk Security Content This project gives you access to our repository of Analytic Stories, security guides that provide background on tactics, techniques and procedures (TTPs), mapped to the MITRE ATT&CK Framework, the Lockheed Martin Cyber Kill Chain, and CIS Controls. They include Splunk searches, machine learning algorithms and Splunk Phantom playbooks (where available)—all designed to work together to detect, investigate, and respond to threats. -**Note:** We have sister projects that enable us to build the industry's best security content. These projects are the Splunk Attack Range, an attack simulation lab built around Splunk, and Contentctl, the tool that enables us to build, test, and package our content for distribution. +**Note:** We have sister projects that enable us to build the industry's best security content. These projects are the Splunk Attack Range, an attack simulation lab built around Splunk, and contentctl-ng, the tool that enables us to build, test, and package our content for distribution. # Tools 🧰 - [Splunk Attack Range](https://github.com/splunk/attack_range): The Splunk Attack Range is an open-source project maintained by the Splunk Threat Research Team. It builds instrumented cloud (AWS, Azure) and local environments (Virtualbox), simulates attacks, and forwards the data into a Splunk instance. This environment can then be used to develop and test the effectiveness of detections. -- [Contentctl](https://github.com/splunk/contentctl): The tool that enables us to build, test, and package our content for distribution. -- [Attack data](https://github.com/splunk/attack_data): The is a collection of attack data that is used to test our content. +- [contentctl-ng](https://pypi.org/project/contentctl-ng/): The tool that enables us to build, test, and package our content for distribution. +- [Attack data](https://github.com/splunk/attack_data): This is a collection of attack data that is used to test our content. - [Atomic Red Team](https://github.com/redcanaryco/atomic-red-team): Atomic Red Team™ is a library of tests mapped to the MITRE ATT&CK® framework. Security teams can use Atomic Red Team to quickly, portably, and reproducibly test their environments. # Get Content🛡 @@ -55,74 +55,74 @@ Follow these steps to get started with Splunk Security Content. 1. Clone this repository using `git clone https://github.com/splunk/security_content.git` 2. Navigate to the repository directory using `cd security_content` -3. Install contentctl using `pip install contentctl` to install the latest version of contentctl, this is a pre-requisite to validate, build and test the content like the Splunk Threat Research team -4. Install pre-commit using `pip install pre-commit` then proceed to installing the hooks via `pre-commit install`. this is a pre-requisite to validate and apply the proper formatting. +3. Install contentctl-ng using `pip install contentctl-ng` to install the latest version of contentctl-ng, this is a prerequisite to validate, build, and test the content like the Splunk Threat Research team +4. Install pre-commit using `pip install pre-commit` then proceed to installing the hooks via `pre-commit install`. This is a prerequisite to validate and apply the proper formatting. # Quick Start 🚀 -1. Setup the environment +1. Set up the environment ``` git clone https://github.com/splunk/security_content.git cd security_content python3.11 -m venv .venv source .venv/bin/activate -pip install contentctl +pip install contentctl-ng ``` -2. Create a new **detection.yml** and answer the questions - ``` - contentctl new - ``` +2. Create a new content YML using VSCode Snippets detailed below - -**NOTE** - Make sure you update the detection.yml with the required fields and values. If you need help, please refer to the [yaml-spec](docs/yaml-spec) to check out the list of allowed values - -4. Validate your content +3. Validate your content and build an app ``` -contentctl validate +contentctl-ng build ``` -**NOTE** - The contentctl validate command ensures that all YAML files adhere to the defined specifications and are up-to-date. It checks for required fields, correct data types, and overall consistency, helping maintain the integrity and quality of the content. +**NOTE** - The contentctl build command ensures that all YML files adhere to the defined specifications and are up-to-date. It checks for required fields, correct data types, and overall consistency, helping maintain the integrity and quality of the content. -5. Build an ESCU app with enrichments from Atomic Red Team and Mitre CTI repositories -``` -contentctl build --enrichments -``` +# VS Code Snippets ✂️ -6. Test the content - Our testing framework is based on [contentctl](https://github.com/splunk/contentctl) and is extensive and flexible. Refer to the [contentctl test documentation](https://github.com/splunk/contentctl?tab=readme-ov-file#contentctl-test) to learn more about the testing framework. +This repo ships snippets in [.vscode/escu_6.code-snippets](.vscode/escu_6.code-snippets) to scaffold new content quickly in VS Code. +1. Create a new `.yml` file in the appropriate content directory (e.g. `detections/`, `macros/`, `stories/`) and give it a descriptive name. This name will be used to populate the `name: ` field of your YML as well. +2. Type one of the snippet prefixes below and press `Tab` (or select it from the IntelliSense popup and press `Enter`) to expand a pre-filled template. +3. Fill in the relevant fields for your content. You may remove any optional fields which are not relevant to this piece of content. -# Recommendations 💡 +Available prefixes exist for `detection_escu6` , `macro_escu6`, `data_source_escu6`, `story_escu6`, `lookup_csv_escu6`, `lookup_kvstore_escu6` , `dashboard_escu6`, and `baseline_escu6` . -- 🚨 NOTE: If you are just getting started with managing your Splunk detection as code, we recommend that you keep the YAML structure of the detections as close as possible to the original structure of the detections. This will make it easier to manage your detections and will also make it easier to contribute back to the community by creating a pull request to the Splunk Security Content project. +Please see the demo video below for more information -- In order to build an content app that specific for your organization, we strongly recommend that you start with keeping only the detections that are related to your organization and remove other yamls that are not related to your organization. This includes selecting detections, stories, macros, lookups that are used by the detection ymls. +https://github.com/user-attachments/assets/1be02afa-36b5-4d49-91cb-8ebbcf4123d0 -- If your detections are using macros and lookups, please make sure that you have the same macros and lookups in those directories.. This will ensure that the content app is self-contained and does not rely on external files. +# Recommendations 💡 -- We recommend that you follow the errors produced by the `contentctl` tool while developing this content. The errors are descriptive enough to guide you in getting the right values. If you need help, please open a GitHub issue in the `contentctl` repository or refer to the [yaml-spec](docs/yaml-spec) to check out the list of allowed values +- 🚨 NOTE: If you are just getting started with managing your Splunk detection as code, we recommend that you keep the YML structure of the detections as close as possible to the original structure of the detections. This will make it easier to manage your detections and will also make it easier to contribute back to the community by creating a pull request to the Splunk Security Content project. +- In order to build a content app that specific for your organization, we strongly recommend that you start with keeping only the detections that are related to your organization and remove other YML files that are not related to your organization. This includes selecting detections, stories, macros, lookups that are used by the detection YML files. -# YAML Specs 📄 +- If your detections are using macros and lookups, please make sure that you have the same macros and lookups in those directories. This will ensure that the content app is self-contained and does not rely on external files. -YAML specs define the structure and required fields for various YAML configuration files used in the project. These specifications ensure consistency and validation across different types of YAML files, such as macros, lookups, and analytic stories. Each spec outlines the expected data types, descriptions, and whether the fields are mandatory, providing a clear schema for developers to follow. +- We recommend that you follow the errors produced by the `contentctl-ng` tool while developing this content. The errors are descriptive enough to guide you in getting the right values. If you need help, please refer to the [JSON Schemas](#json-schemas) to check out the list of allowed values -- [detection](https://github.com/splunk/security_content/blob/develop/docs/yaml-spec/detection_spec.yml) -- [stories](https://github.com/splunk/security_content/blob/develop/docs/yaml-spec/stories_spec.yml) -- [macros](https://github.com/splunk/security_content/blob/develop/docs/yaml-spec/macros_spec.yml) -- [lookups](https://github.com/splunk/security_content/blob/develop/docs/yaml-spec/lookups_spec.yml) +# Json Schemas 📄 + +The JSON schemas defined in the [/schemas](/schemas/) folder, define the structure and required fields for various YML files used in the project. These specifications ensure consistency and validation across different types of YML files, such as macros, lookups, and analytic stories. Each spec outlines the expected data types, descriptions, and whether the fields are mandatory, providing a clear schema for developers to follow. + +- [detection](schemas/EventBasedDetection.schema.json) +- [stories](schemas/Story.schema.json) +- [macros](schemas/FileBackedMacro.schema.json) +- [lookups](schemas/CsvLookup.schema.json) +- [baseline](schemas/Baseline.schema.json) +- [playbook](schemas/Playbook.schema.json) +- [data_source](schemas/DataSource.schema.json) # Content Parts 🧩 -* [detections/](detections/): Contains all detection searches to-date and growing. -* [stories/](stories/): All Analytic Stories that are group detections or also known as Use Cases -* [deployments/](deployments/): Configuration for the schedule and alert action for all content -* [playbooks/](playbooks/): Incident Response Playbooks/Workflow for responding to a specific Use Case or Threat. * [baselines/](baselines/): Searches that must be executed before a detection runs. It is specifically useful for collecting data on a system before running your detection on the collected data. -* [investigations/](investigations/): Investigations to further analyze the output from detections. For more information, you can refer to the [Splunk Enterprise Security documentation on timelines](https://help.splunk.com/en/splunk-enterprise-security-7/user-guide/7.3/investigations/investigations-in-splunk-enterprise-security). -* [macros/](macros/): Implements Splunk’s search macros, shortcuts to commonly used search patterns like sysmon source type. More on how macros are used to customize content below. -* [lookups/](lookups/): Implements Splunk’s lookup, usually to provide a list of static values like commonly used ransomware extensions. * [data_sources/](data_sources/): Defines the data sources, the necessary TA or App to collect them and the fields provided that can be used by the detections. - +* [deployments/](deployments/): Configuration for the schedule and alert action for all content +* [detections/](detections/): Contains all detection searches to-date and growing. +* [lookups/](lookups/): Implements Splunk’s lookup, usually to provide a list of static values like commonly used ransomware extensions. +* [macros/](macros/): Implements Splunk’s search macros, shortcuts to commonly used search patterns like sysmon source type. More on how macros are used to customize content below. +* [playbooks/](playbooks/): Incident Response Playbooks/Workflow for responding to a specific Use Case or Threat. +* [stories/](stories/): All Analytic Stories that are group detections or also known as Use Cases # MITRE ATT&CK ⚔️ ### Detection Coverage @@ -142,7 +142,7 @@ If you have questions or need support, you can: * Join the [#security-research](https://splunkcommunity.slack.com/archives/CDNHXVBGS) channel in the [Splunk Community Slack.](https://splk.it/slack) ## License -Copyright 2022 Splunk Inc. +Copyright 2026 Splunk Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/app_template/default/data/ui/nav/default.xml b/app_template/default/data/ui/nav/default.xml index 87cd34acd6f..1738ee90689 100644 --- a/app_template/default/data/ui/nav/default.xml +++ b/app_template/default/data/ui/nav/default.xml @@ -1,9 +1,12 @@