diff --git a/.github/actions/contentctl-build/action.yml b/.github/actions/contentctl-build/action.yml new file mode 100644 index 0000000000..826ad4317d --- /dev/null +++ b/.github/actions/contentctl-build/action.yml @@ -0,0 +1,19 @@ +name: contentctl-build +description: Install contentctl-ng and run contentctl-ng build. +inputs: + build-tarball-path: + description: Path to the built content tarball produced by contentctl-ng build. + required: false + default: dist/DA-ESS-ContentUpdate-latest.tar.gz +runs: + using: composite + steps: + - name: Install contentctl-ng + uses: ./.github/actions/contentctl-install + + - name: Run build + shell: bash + run: | + contentctl-ng build + mkdir -p artifacts + mv "${{ inputs.build-tarball-path }}" artifacts/ diff --git a/.github/actions/contentctl-install/action.yml b/.github/actions/contentctl-install/action.yml new file mode 100644 index 0000000000..99c74d2030 --- /dev/null +++ b/.github/actions/contentctl-install/action.yml @@ -0,0 +1,22 @@ +name: contentctl-install +description: Set up Python and install contentctl-ng from requirements.txt. +inputs: + python-version: + description: Python version used for setup. + required: false + default: '3.14' + +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ inputs.python-version }} + architecture: x64 + + - name: Install contentctl-ng + shell: bash + run: | + echo "- Build Tool Version - $(cat requirements.txt)" + pip install -r requirements.txt diff --git a/.github/workflows/appinspect.yml b/.github/workflows/appinspect.yml index 383715c130..9e36ec1e27 100644 --- a/.github/workflows/appinspect.yml +++ b/.github/workflows/appinspect.yml @@ -1,45 +1,45 @@ +# This workflow performs a simple build action. +# It is intentionally separate from the build +# workflow to provide granular feedback and insight +# into when a build passes but an appinspect fails. name: appinspect + on: pull_request: types: [opened, reopened, synchronize] + jobs: appinspect: runs-on: ubuntu-latest steps: - - name: Check out repository (PR) - if: ${{ github.event_name == 'pull_request' }} + - name: Check out the repository code uses: actions/checkout@v6 - with: - ref: refs/pull/${{ github.event.pull_request.number }}/merge - - uses: actions/setup-python@v6 - with: - python-version: '3.11' - architecture: 'x64' - - - name: Install Python Dependencies and ContentCTL and Atomic Red Team - run: | - echo "- Contentctl version - $(cat requirements.txt)" - pip install -r requirements.txt - git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git external_repos/atomic-red-team - git clone --depth=1 --single-branch --branch="master" https://github.com/mitre-attack/attack-stix-data external_repos/cti + - name: Install dependencies and run build + uses: ./.github/actions/contentctl-build + - - name: Running appinspect with enrichments + - name: Run appinspect with enrichments env: APPINSPECTUSERNAME: "${{ secrets.APPINSPECTUSERNAME }}" APPINSPECTPASSWORD: "${{ secrets.APPINSPECTPASSWORD }}" run: | - echo $APPINSPECTUSERNAME - contentctl inspect --splunk-api-username "$APPINSPECTUSERNAME" --splunk-api-password "$APPINSPECTPASSWORD" --enrichments --enable-metadata-validation --suppress-missing-content-exceptions --enforce-deprecation-mapping-requirement - echo "done appinspect" + # Download the most recent Release in order to enforce metadata validation + # --location flag required to follow redirects from the "latest" URL + curl --location -o DA-ESS-ContentUpdate-latest.tar.gz https://github.com/splunk/security_content/releases/latest/download/DA-ESS-ContentUpdate-latest.tar.gz + + # Inspect, using the release we downloaded above + # Also, we intentionally ignore missing content exceptions - private content is + # not expected to be present when running this check in GitHub + contentctl-ng inspect --splunkbase-username "$APPINSPECTUSERNAME" --splunkbase-password "$APPINSPECTPASSWORD" --old-app-path DA-ESS-ContentUpdate-latest.tar.gz --ignore-missing-content-exceptions mkdir -p artifacts/app_inspect_report cp -r dist/*.html artifacts/app_inspect_report cp -r dist/*.tar.gz artifacts/ - - name: store_artifacts + - name: Store artifacts uses: actions/upload-artifact@v7 with: - name: content-latest + name: content-latest-with-appinspect-report path: | artifacts/DA-ESS-ContentUpdate-latest.tar.gz artifacts/app_inspect_report diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a218f914bb..8f05e25118 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,10 +1,17 @@ +# This workflow performs a simple build action. +# It is intentionally separate from the appinspect +# workflow to provide granular feedback and insight +# into when a build passes but an appinspect fails. name: build + on: pull_request: types: [opened, reopened, synchronize] push: branches: - develop + + jobs: build: runs-on: ubuntu-latest @@ -12,28 +19,5 @@ jobs: - name: Check out the repository code uses: actions/checkout@v6 - - uses: actions/setup-python@v6 - with: - python-version: '3.11' - architecture: 'x64' - - - name: Install Python Dependencies and ContentCTL and Atomic Red Team - run: | - echo "- Contentctl version - $(cat requirements.txt)" - pip install -r requirements.txt - git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git external_repos/atomic-red-team - git clone --depth=1 --single-branch --branch="master" https://github.com/mitre-attack/attack-stix-data external_repos/cti - - - name: Running build with enrichments - run: | - contentctl build --enrichments --enforce_deprecation_mapping_requirement - mkdir artifacts - mv dist/DA-ESS-ContentUpdate-latest.tar.gz artifacts/ - - - name: store_artifacts - uses: actions/upload-artifact@v7 - with: - name: content-latest - path: | - artifacts/DA-ESS-ContentUpdate-latest.tar.gz - dist/api + - name: Install dependencies and run build + uses: ./.github/actions/contentctl-build \ No newline at end of file diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 8fa96affb8..266b05652f 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -4,7 +4,7 @@ on: types: [opened, reopened, synchronize] jobs: unit-testing: - runs-on: ubuntu-latest + runs-on: large-ubuntu-22.04-32core if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job steps: #For fork PRs, always check out security_content and the PR target in security content! @@ -15,23 +15,14 @@ jobs: ref: ${{ github.base_ref }} - - uses: actions/setup-python@v6 - with: - python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy - architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified - - name: Install Python Dependencies and ContentCTL - run: | - python -m pip install --upgrade pip - echo "- Contentctl version - $(cat requirements.txt)" - pip install -r requirements.txt - - - # Running contentctl test with a few arguments, before running the command make sure you checkout into the current branch of the pull request. This step only performs unit testing on all the changes against the target-branch. In most cases this target branch will be develop - # Make sure we check out the PR, even if it actually lives in a fork + uses: ./.github/actions/contentctl-install + + + # Check out the PR, even if it lives in a fork. # Instructions for pulling a PR were taken from: # https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally - - name: Run ContentCTL test for changes against target branch + - name: Checkout the PR branch and the target to calculate changed files for testing run: | echo "Current Branch (Head Ref): ${{ github.head_ref }}" @@ -41,7 +32,27 @@ jobs: git fetch origin pull/${{ github.event.pull_request.number }}/head:new_branch_for_testing #We must specifically get the PR's target branch from security_content, not the one that resides in the fork PR's forked repo git switch new_branch_for_testing - contentctl test --verbose --disable-tqdm --no-enable-integration-testing --container-settings.num-containers 1 --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }} + + + - name: Run a contentctl-ng build to ensure that the branch in syntactically correct + uses: ./.github/actions/contentctl-build + + - name: Start the test environment + run: | + docker run -d --platform linux/amd64 -p 8088:8088 -p 8089:8089 -p 8000:8000 -e 'SPLUNK_START_ARGS=--accept-license' -e 'SPLUNK_GENERAL_TERMS=--accept-sgt-current-at-splunk-com' -e 'SPLUNK_PASSWORD=Chang3d!' --name splunk splunk/splunk:latest + # Wait some time for this environment to be ready + sleep 180 + + - name: Run a contentctl-ng install to configure the testing environment + env: + APPINSPECTUSERNAME: "${{ secrets.APPINSPECTUSERNAME }}" + APPINSPECTPASSWORD: "${{ secrets.APPINSPECTPASSWORD }}" + run : | + contentctl-ng install --splunkbase-username "$APPINSPECTUSERNAME" --splunkbase-password "$APPINSPECTPASSWORD" + + - name: Test content which has changed between this branch and the target branch + run: | + contentctl-ng test --verbose --post-test-behavior NEVER_PAUSE --mode CHANGED --git-ref ${{ github.base_ref }} echo "contentctl test - COMPLETED" # Store test_results/summary.yml and dist/DA-ESS-ContentUpdate-latest.tar.gz to job artifact-test_summary_results.zip @@ -65,4 +76,4 @@ jobs: run: | echo "This job will fail if there are failures in unit-testing" python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY - echo "The Unit testing is completed. See details in the unit-testing job summary UI " \ No newline at end of file + echo "The Unit testing is completed. See details in the unit-testing job summary UI " diff --git a/.vscode/escu_6.code-snippets b/.vscode/escu_6.code-snippets new file mode 100644 index 0000000000..d5990d8b33 --- /dev/null +++ b/.vscode/escu_6.code-snippets @@ -0,0 +1,229 @@ +{ + "New Detection": { + "scope": "yaml", + "prefix": "detection_escu6", + "description": "Scaffold a Splunk Security Content detection YAML", + "body": [ + "name: ${1:${TM_FILENAME_BASE/(^|[_-])(\\w)/${1:+ }${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${2:Your Name, Splunk}", + "status: production", + "type: ${3|TTP,Anomaly,Hunting,Correlation|}", + "description: ${4}", + "data_source:", + " - ", + "search: >-", + " your search here", + " | `${TM_FILENAME_BASE}_filter`", + "how_to_implement: ", + "known_false_positives: ", + "references:", + " - ", + "drilldown_searches:", + " - name: View the detection results for - \"$risk_object$\"", + " search: '%original_detection_search% | search risk_object = \"$risk_object$\"'", + " earliest_offset: $info_min_time$", + " latest_offset: $info_max_time$", + " - name: View risk events for the last 7 days for - \"$risk_object$\"", + " search: '| from datamodel Risk.All_Risk | search normalized_risk_object IN (\"$risk_object$\") | stats count min(_time) as firstTime max(_time) as lastTime values(search_name) as \"Search Name\" values(risk_message) as \"Risk Message\" values(analyticstories) as \"Analytic Stories\" values(annotations._all) as \"Annotations\" values(annotations.mitre_attack.mitre_tactic) as \"ATT&CK Tactics\" by normalized_risk_object | `security_content_ctime(firstTime)` | `security_content_ctime(lastTime)`'", + " earliest_offset: 7d", + " latest_offset: \"0\"", + "finding:", + " title: ", + " entity:", + " field: ", + " type: ", + " score: ", + "intermediate_findings:", + " entities:", + " - field: ", + " type: ", + " score: ", + " message:", + "threat_objects:", + " - field: ", + " type: ", + "analytic_story:", + " - ", + "asset_type: ", + "mitre_attack_id:", + " - ", + "product:", + " - Splunk Enterprise", + " - Splunk Enterprise Security", + " - Splunk Cloud", + "category: ", + "security_domain: ", + "tests:", + " - name: True Positive Test", + " attack_data:", + " - data: ", + " source: ", + " sourcetype: ", + " test_type: unit", + ] + }, + "New Macro": { + "scope": "yaml", + "prefix": "macro_escu6", + "body": [ + "name: ${1:${TM_FILENAME_BASE/([_-])(\\w)/ ${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${1:Splunk Threat Research Team}", + "description: ${2}", + "definition: ${3}", + "$0" + ], + "description": "Scaffold a new Splunk security content macro YAML file" + }, + + "New Data Source": { + "scope": "yaml", + "prefix": "data_source_escu6", + "description": "Scaffold a new Splunk security content data source YAML file", + "body": [ + "name: ${1:${TM_FILENAME_BASE/(^|[_-])(\\w)/${1:+ }${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${1:Splunk Threat Research Team}", + "description: ${3}", + "source: ${4:sourcename}", + "sourcetype: ${5:sourcetype_value}", + "separator: ${6:EventCode}", + "separator_value: '${7}'", + "supported_TA:", + " - name: ${8}", + " url: ${9:hxxps://splunkbase.splunk.com/app/....}", + " version: ${10}", + "fields:", + " - ", + "output_fields:", + " - ", + "mitre_components:", + " - ", + "example_log: '${11}'", + "$0" + ] + }, + + "New Story": { + "scope": "yaml", + "prefix": "story_escu6", + "description": "Scaffold a new Splunk security content analytic story YAML file", + "body": [ + "name: ${1:${TM_FILENAME_BASE/(^|[_-])(\\w)/${1:+ }${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${2:Splunk Threat Research Team}", + "status: production", + "description: ${3}", + "narrative: ${4}", + "references:", + " - ", + "category:", + " - ", + "product:", + " - Splunk Enterprise", + " - Splunk Enterprise Security", + " - Splunk Cloud", + "usecase: ", + "$0" + ] + }, + + "New Lookup (CSV)": { + "scope": "yaml", + "prefix": "lookup_csv_escu6", + "description": "Scaffold a new Splunk security content CSV lookup YAML file", + "body": [ + "name: ${1:${TM_FILENAME_BASE/([_-])(\\w)/ ${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${2:Splunk Threat Research Team}", + "lookup_type: csv", + "description: ${3}", + "match_type:", + " - ${4:WILDCARD()}", + "min_matches: REMOVE ME IF NO MIN MATCHES REQUIRED", + "max_matches: REMOVE ME IF NO MAX MATCHES REQUIRED", + "case_sensitive_match: REMOVE ME IF NO CASE SENSITIVE MATCH REQUIRED", + "$0" + ] + }, + + "New Lookup (KVStore)": { + "scope": "yaml", + "prefix": "lookup_kvstore_escu6", + "description": "Scaffold a new Splunk security content KVStore lookup YAML file", + "body": [ + "name: ${1:${TM_FILENAME_BASE/([_-])(\\w)/ ${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${2:Splunk Threat Research Team}", + "lookup_type: kvstore", + "description: ${3}", + "fields:", + " - _key", + " - ${4:field_name}", + "$0" + ] + }, + + "New Dashboard": { + "scope": "yaml", + "prefix": "dashboard_escu6", + "description": "Scaffold a new Splunk security content dashboard YAML file", + "body": [ + "name: ${1:${TM_FILENAME_BASE/(^|[_-])(\\w)/${1:+ }${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${2:Splunk Threat Research Team}", + "description: ${3:Utilize this dashboard to...}", + "$0" + ] + }, + + "New Baseline": { + "scope": "yaml", + "prefix": "baseline_escu6", + "description": "Scaffold a new Splunk security content baseline YAML file", + "body": [ + "name: ${1:${TM_FILENAME_BASE/(^|[_-])(\\w)/${1:+ }${2:/upcase}/g}}", + "id: $UUID", + "version: 1", + "creation_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "modification_date: '${CURRENT_YEAR}-${CURRENT_MONTH}-${CURRENT_DATE}'", + "author: ${2:Splunk Threat Research Team}", + "status: production", + "description: ${3:This search...}", + "search: ${4}", + "how_to_implement: ${5}", + "known_false_positives: ${6}", + "references: []", + "product:", + " - Splunk Enterprise", + " - Splunk Enterprise Security", + " - Splunk Cloud", + "security_domain: ${7}", + "schedule: Default Baseline", + "$0" + ] + } + +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4fc98151bf..1be27eb086 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -contentctl==5.6.0 +contentctl-ng==0.4.0