diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d1790ddfbe6..d51104e0810 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,3 +1,3 @@ -* @patel-bhavin @ljstella @nasbench +* @patel-bhavin @ljstella @nasbench @pyth0n1c @RavenTait @tccontre @rosplk @p4t12ick @onurmerdogan /response_templates/ @kbouchardherjavecgroup @ccl0utier @henryy-splunk -/.github/workflows/response_templates/mcopenapi_public.yml @kbouchardherjavecgroup @ccl0utier @henryy-splunk \ No newline at end of file +/.github/workflows/response_templates/mcopenapi_public.yml @kbouchardherjavecgroup @ccl0utier @henryy-splunk diff --git a/.github/workflows/archived-ta-tracker.yml b/.github/workflows/archived-ta-tracker.yml new file mode 100644 index 00000000000..0f45898107d --- /dev/null +++ b/.github/workflows/archived-ta-tracker.yml @@ -0,0 +1,32 @@ +name: Archived TA tracker + +on: + workflow_dispatch: + schedule: + - cron: "17 7 * * *" + +permissions: + contents: read + issues: write + +jobs: + check-archived-tas: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install YAML parser + run: python -m pip install PyYAML==6.0.3 + + - name: Create or update archived TA issues + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_REF_NAME: ${{ github.ref_name }} + run: python scripts/check_archived_tas.py diff --git a/.github/workflows/datasource-dependabot.yml b/.github/workflows/datasource-dependabot.yml index 654ac85d846..752b6063e6f 100644 --- a/.github/workflows/datasource-dependabot.yml +++ b/.github/workflows/datasource-dependabot.yml @@ -31,6 +31,14 @@ jobs: run: | echo "Updating data source supported_TA and repo install.yml app versions" contentctl-ng update-data-source-tas --content-dirs . + + # TODO: This comment is intentionally left in this workflow to demonstrate how individuals + # TAs can be intentionally excluded from updates, for example when we have intentionally pinned + # to an older version of a TA. In the future, we will try to make this a formally documented + # feature of contentctl-ng, but needing to do this is fairly rare and with out other priorities, + # does not fit at this time. + # The demonstration below shows how to exlcude the windows TA from updates, if necessary. + # git checkout -- $(grep -rl "https://splunkbase.splunk.com/app/742" data_sources/) echo "Changed data source and install.yml files:" git status --short -- data_sources install.yml diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index c15bbc1aa35..ad8abf2ddaf 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -3,10 +3,11 @@ on: pull_request: types: [opened, reopened, synchronize] paths: - - detections/** - data_sources/** + - detections/** - lookups/** - macros/** + - install.yml jobs: unit-testing: runs-on: large-ubuntu-22.04-32core diff --git a/.github/workflows/update_nist_ai_rmf_detection_map.yml b/.github/workflows/update_nist_ai_rmf_detection_map.yml new file mode 100644 index 00000000000..1d362f48781 --- /dev/null +++ b/.github/workflows/update_nist_ai_rmf_detection_map.yml @@ -0,0 +1,59 @@ +name: NIST AI RMF Detection Map Update + +on: + workflow_dispatch: + schedule: + - cron: '25 06 * * *' + +permissions: + contents: write + pull-requests: write + +jobs: + update-nist-ai-rmf-detection-map: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + ref: develop + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.14' + architecture: x64 + + - name: Install Python dependencies + run: | + pip install PyYAML==6.0.3 + + - name: Generate NIST AI RMF detection map + run: | + python scripts/nist_dashboard/generate_nist_ai_rmf_detection_map.py --modified-date "$(date -u +%F)" + echo "Changed NIST AI RMF lookup files:" + git status --short -- lookups/csv/detection_subcategory_map.csv lookups/csv/detection_subcategory_map.yml + + - name: Create NIST AI RMF detection map update pull request + uses: peter-evans/create-pull-request@v8 + with: + token: ${{ github.token }} + commit-message: Update NIST AI RMF detection map + branch: auto-nist-ai-rmf-detection-map-${{ github.run_number }} + base: develop + title: Automated NIST AI RMF lookup table update + body: | + This PR updates the generated NIST AI RMF detection-to-subcategory lookup. + + The workflow runs: + + ```bash + python scripts/nist_dashboard/generate_nist_ai_rmf_detection_map.py --modified-date "$(date -u +%F)" + ``` + + When `lookups/csv/detection_subcategory_map.csv` changes, the generator also bumps `lookups/csv/detection_subcategory_map.yml` by incrementing `version` and setting `modification_date`. + add-paths: | + lookups/csv/detection_subcategory_map.csv + lookups/csv/detection_subcategory_map.yml diff --git a/.github/workflows/yaml-validation.yml b/.github/workflows/yaml-validation.yml index fe4aa381c5b..fa5640e36c6 100644 --- a/.github/workflows/yaml-validation.yml +++ b/.github/workflows/yaml-validation.yml @@ -5,12 +5,26 @@ on: paths: - 'detections/**/*.yml' - 'detections/**/*.yaml' + - '.yamlfieldorder' + - '.yamlfmt' + - '.yamllint' + - '.pre-commit-config.yaml' + - 'scripts/enforce_yaml_field_order.py' + - 'scripts/validate_yaml.py' + - '.github/workflows/yaml-validation.yml' push: branches: - develop paths: - 'detections/**/*.yml' - 'detections/**/*.yaml' + - '.yamlfieldorder' + - '.yamlfmt' + - '.yamllint' + - '.pre-commit-config.yaml' + - 'scripts/enforce_yaml_field_order.py' + - 'scripts/validate_yaml.py' + - '.github/workflows/yaml-validation.yml' jobs: validate: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3b5877a65c5..c71a44920b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,3 +22,11 @@ repos: pass_filenames: true # Optional: Specify custom yamlfmt binary path if not in PATH # args: [--yamlfmt-path, /path/to/yamlfmt] + + - id: yaml-field-order + name: YAML field order (detections only) + description: Enforce configured root field order in detection YAML files + entry: python scripts/enforce_yaml_field_order.py --fix + language: python + files: ^detections/.*\.(yml|yaml)$ + pass_filenames: true diff --git a/.yamlfieldorder b/.yamlfieldorder new file mode 100644 index 00000000000..36f5054b522 --- /dev/null +++ b/.yamlfieldorder @@ -0,0 +1,46 @@ +# Root field order for detection YAML files. +# +# The formatter moves whole top-level YAML blocks and does not re-emit YAML. +# This preserves indentation, spaces, comments, scalar style, and nested content +# inside each field block. +target_paths: + - detections/ + +extensions: + - .yml + - .yaml + +# Unknown top-level fields are kept intact and appended after configured fields. +unknown_fields: append + +root_field_order: + - name + - id + - version + - creation_date + - modification_date + - author + - status + - type + - description + - data_source + - search + - how_to_implement + - known_false_positives + - references + - drilldown_searches + - finding + - intermediate_findings + - threat_objects + - analytic_story + - threat_group + - asset_type + - cve + - atomic_guid + - mitre_attack_id + - product + - category + - security_domain + - baselines + - tests + - deprecation_info diff --git a/README.md b/README.md index e01cf2677f0..340b89cf323 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ This project gives you access to our repository of Analytic Stories, security gu - [Splunk Attack Range](https://github.com/splunk/attack_range): The Splunk Attack Range is an open-source project maintained by the Splunk Threat Research Team. It builds instrumented cloud (AWS, Azure) and local environments (Virtualbox), simulates attacks, and forwards the data into a Splunk instance. This environment can then be used to develop and test the effectiveness of detections. - [contentctl-ng](https://pypi.org/project/contentctl-ng/): The tool that enables us to build, test, and package our content for distribution. -- [Attack data](https://github.com/splunk/attack_data): The is a collection of attack data that is used to test our content. +- [Attack data](https://github.com/splunk/attack_data): This is a collection of attack data that is used to test our content. - [Atomic Red Team](https://github.com/redcanaryco/atomic-red-team): Atomic Red Team™ is a library of tests mapped to the MITRE ATT&CK® framework. Security teams can use Atomic Red Team to quickly, portably, and reproducibly test their environments. # Get Content🛡 @@ -55,12 +55,12 @@ Follow these steps to get started with Splunk Security Content. 1. Clone this repository using `git clone https://github.com/splunk/security_content.git` 2. Navigate to the repository directory using `cd security_content` -3. Install contentctl-ng using `pip install contentctl-ng` to install the latest version of contentctl-ng, this is a pre-requisite to validate, build and test the content like the Splunk Threat Research team -4. Install pre-commit using `pip install pre-commit` then proceed to installing the hooks via `pre-commit install`. this is a pre-requisite to validate and apply the proper formatting. +3. Install contentctl-ng using `pip install contentctl-ng` to install the latest version of contentctl-ng, this is a prerequisite to validate, build, and test the content like the Splunk Threat Research team +4. Install pre-commit using `pip install pre-commit` then proceed to installing the hooks via `pre-commit install`. This is a prerequisite to validate and apply the proper formatting. # Quick Start 🚀 -1. Setup the environment +1. Set up the environment ``` git clone https://github.com/splunk/security_content.git cd security_content @@ -95,9 +95,9 @@ https://github.com/user-attachments/assets/1be02afa-36b5-4d49-91cb-8ebbcf4123d0 - 🚨 NOTE: If you are just getting started with managing your Splunk detection as code, we recommend that you keep the YML structure of the detections as close as possible to the original structure of the detections. This will make it easier to manage your detections and will also make it easier to contribute back to the community by creating a pull request to the Splunk Security Content project. -- In order to build an content app that specific for your organization, we strongly recommend that you start with keeping only the detections that are related to your organization and remove other YML files that are not related to your organization. This includes selecting detections, stories, macros, lookups that are used by the detection YML files. +- In order to build a content app that specific for your organization, we strongly recommend that you start with keeping only the detections that are related to your organization and remove other YML files that are not related to your organization. This includes selecting detections, stories, macros, lookups that are used by the detection YML files. -- If your detections are using macros and lookups, please make sure that you have the same macros and lookups in those directories.. This will ensure that the content app is self-contained and does not rely on external files. +- If your detections are using macros and lookups, please make sure that you have the same macros and lookups in those directories. This will ensure that the content app is self-contained and does not rely on external files. - We recommend that you follow the errors produced by the `contentctl-ng` tool while developing this content. The errors are descriptive enough to guide you in getting the right values. If you need help, please refer to the [JSON Schemas](#json-schemas) to check out the list of allowed values diff --git a/app_template/default/data/ui/nav/default.xml b/app_template/default/data/ui/nav/default.xml index 87cd34acd6f..c4aeaca59d4 100644 --- a/app_template/default/data/ui/nav/default.xml +++ b/app_template/default/data/ui/nav/default.xml @@ -1,9 +1,13 @@