Workflow updates

.github/actions/contentctl-build/action.yml

@@ -0,0 +1,19 @@
+name: contentctl-build
+description: Install contentctl-ng and run contentctl-ng build.
+inputs:
+  build-tarball-path:
+    description: Path to the built content tarball produced by contentctl-ng build.
+    required: false
+    default: dist/DA-ESS-ContentUpdate-latest.tar.gz
+runs:
+  using: composite
+  steps:
+    - name: Install contentctl-ng
+      uses: ./.github/actions/contentctl-install
+
+    - name: Run build
+      shell: bash
+      run: |
+        contentctl-ng build
+        mkdir -p artifacts
+        mv "${{ inputs.build-tarball-path }}" artifacts/

.github/actions/contentctl-install/action.yml

@@ -0,0 +1,22 @@
+name: contentctl-install
+description: Set up Python and install contentctl-ng from requirements.txt.
+inputs:
+  python-version:
+    description: Python version used for setup.
+    required: false
+    default: '3.14'
+
+runs:
+  using: composite
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ inputs.python-version }}
+        architecture: x64
+
+    - name: Install contentctl-ng
+      shell: bash
+      run: |
+        echo "- Build Tool Version - $(cat requirements.txt)"
+        pip install -r requirements.txt

.github/workflows/appinspect.yml

@@ -1,45 +1,45 @@
+# This workflow performs a simple build action.
+# It is intentionally separate from the build
+# workflow to provide granular feedback and insight
+# into when a build passes but an appinspect fails.
 name: appinspect
+
 on:
   pull_request:
     types: [opened, reopened, synchronize]
+
 jobs:
   appinspect:
     runs-on: ubuntu-latest
     steps:
-      - name: Check out repository (PR)
-        if: ${{ github.event_name == 'pull_request' }}
+      - name: Check out the repository code
         uses: actions/checkout@v6
-        with:
-          ref: refs/pull/${{ github.event.pull_request.number }}/merge
 
-      - uses: actions/setup-python@v6
-        with:
-          python-version: '3.11'
-          architecture: 'x64'
-
-      - name: Install Python Dependencies and ContentCTL and Atomic Red Team
-        run: |
-          echo "- Contentctl version - $(cat requirements.txt)"
-          pip install -r requirements.txt
-          git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git external_repos/atomic-red-team
-          git clone --depth=1 --single-branch --branch="master" https://github.com/mitre-attack/attack-stix-data external_repos/cti
+      - name: Install dependencies and run build
+        uses: ./.github/actions/contentctl-build
+
 
-      - name: Running appinspect with enrichments
+      - name: Run appinspect with enrichments
         env:
           APPINSPECTUSERNAME: "${{ secrets.APPINSPECTUSERNAME }}"
           APPINSPECTPASSWORD: "${{ secrets.APPINSPECTPASSWORD }}"
         run: |
-          echo $APPINSPECTUSERNAME
-          contentctl inspect --splunk-api-username "$APPINSPECTUSERNAME" --splunk-api-password "$APPINSPECTPASSWORD" --enrichments --enable-metadata-validation --suppress-missing-content-exceptions --enforce-deprecation-mapping-requirement
-          echo "done appinspect"
+          # Download the most recent Release in order to enforce metadata validation
+          # --location flag required to follow redirects from the "latest" URL
+          curl --location -o  DA-ESS-ContentUpdate-latest.tar.gz https://github.com/splunk/security_content/releases/latest/download/DA-ESS-ContentUpdate-latest.tar.gz
+
+          # Inspect, using the release we downloaded above
+          # Also, we intentionally ignore missing content exceptions - private content is 
+          # not expected to be present when running this check in GitHub
+          contentctl-ng inspect --splunkbase-username "$APPINSPECTUSERNAME" --splunkbase-password "$APPINSPECTPASSWORD" --old-app-path DA-ESS-ContentUpdate-latest.tar.gz --ignore-missing-content-exceptions
           mkdir -p artifacts/app_inspect_report
           cp -r dist/*.html artifacts/app_inspect_report
           cp -r dist/*.tar.gz artifacts/
 
-      - name: store_artifacts
+      - name: Store artifacts
         uses: actions/upload-artifact@v7
         with:
-          name: content-latest
+          name: content-latest-with-appinspect-report
           path: |
             artifacts/DA-ESS-ContentUpdate-latest.tar.gz
             artifacts/app_inspect_report

.github/workflows/build.yml

@@ -1,39 +1,23 @@
+# This workflow performs a simple build action.
+# It is intentionally separate from the appinspect
+# workflow to provide granular feedback and insight
+# into when a build passes but an appinspect fails.
 name: build
+
 on:
   pull_request:
     types: [opened, reopened, synchronize]
   push:
     branches:
       - develop
+
+
 jobs:
   build:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repository code
         uses: actions/checkout@v6
 
-      - uses: actions/setup-python@v6
-        with:
-          python-version: '3.11'
-          architecture: 'x64'
-
-      - name: Install Python Dependencies and ContentCTL and Atomic Red Team
-        run: |
-          echo "- Contentctl version - $(cat requirements.txt)"
-          pip install -r requirements.txt
-          git clone --depth=1 --single-branch --branch=master https://github.com/redcanaryco/atomic-red-team.git external_repos/atomic-red-team
-          git clone --depth=1 --single-branch --branch="master" https://github.com/mitre-attack/attack-stix-data external_repos/cti
-
-      - name: Running build with enrichments
-        run: |
-          contentctl build --enrichments --enforce_deprecation_mapping_requirement
-          mkdir artifacts
-          mv dist/DA-ESS-ContentUpdate-latest.tar.gz artifacts/
-
-      - name: store_artifacts
-        uses: actions/upload-artifact@v7
-        with:
-          name: content-latest
-          path: |
-            artifacts/DA-ESS-ContentUpdate-latest.tar.gz
-            dist/api
+      - name: Install dependencies and run build
+        uses: ./.github/actions/contentctl-build

.github/workflows/unit-testing.yml

@@ -4,7 +4,7 @@ on:
     types: [opened, reopened, synchronize]
 jobs:
   unit-testing:
-    runs-on: ubuntu-latest
+    runs-on: large-ubuntu-22.04-32core
     if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job
     steps:
         #For fork PRs, always check out security_content and the PR target in security content!
@@ -15,23 +15,14 @@ jobs:
             ref: ${{ github.base_ref }}
 
 
-        - uses: actions/setup-python@v6
-          with:
-            python-version: '3.11' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
-            architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
-
         - name: Install Python Dependencies and ContentCTL
-          run: |
-            python -m pip install --upgrade pip
-            echo "- Contentctl version - $(cat requirements.txt)"
-            pip install -r requirements.txt
-
-
-        # Running contentctl test with a few arguments, before running the command make sure you checkout into the current branch of the pull request. This step only performs unit testing on all the changes against the target-branch. In most cases this target branch will be develop
-        # Make sure we check out the PR, even if it actually lives in a fork
+          uses: ./.github/actions/contentctl-install
+
+
+        # Check out the PR, even if it lives in a fork.
         # Instructions for pulling a PR were taken from: 
         # https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally
-        - name: Run ContentCTL test for changes against target branch
+        - name: Checkout the PR branch and the target to calculate changed files for testing
           run: |
 
             echo "Current Branch (Head Ref): ${{ github.head_ref }}"
@@ -41,7 +32,27 @@ jobs:
             git fetch origin pull/${{ github.event.pull_request.number }}/head:new_branch_for_testing
             #We must specifically get the PR's target branch from security_content, not the one that resides in the fork PR's forked repo           
             git switch new_branch_for_testing
-            contentctl test --verbose --disable-tqdm --no-enable-integration-testing --container-settings.num-containers 1 --post-test-behavior never_pause mode:changes --mode.target-branch ${{ github.base_ref }}
+
+
+        - name: Run a contentctl-ng build to ensure that the branch in syntactically correct
+          uses: ./.github/actions/contentctl-build
+
+        - name: Start the test environment
+          run: |
+            docker run -d --platform linux/amd64 -p 8088:8088 -p 8089:8089 -p 8000:8000 -e 'SPLUNK_START_ARGS=--accept-license' -e 'SPLUNK_GENERAL_TERMS=--accept-sgt-current-at-splunk-com' -e 'SPLUNK_PASSWORD=Chang3d!' --name splunk splunk/splunk:latest
+            # Wait some time for this environment to be ready
+            sleep 180
+
+        - name: Run a contentctl-ng install to configure the testing environment
+          env:
+            APPINSPECTUSERNAME: "${{ secrets.APPINSPECTUSERNAME }}"
+            APPINSPECTPASSWORD: "${{ secrets.APPINSPECTPASSWORD }}"  
+          run : |
+            contentctl-ng install --splunkbase-username "$APPINSPECTUSERNAME" --splunkbase-password "$APPINSPECTPASSWORD" 
+
+        - name: Test content which has changed between this branch and the target branch
+          run: |
+            contentctl-ng test --verbose --post-test-behavior NEVER_PAUSE --mode CHANGED --git-ref ${{ github.base_ref }}
             echo "contentctl test - COMPLETED"
 
         # Store test_results/summary.yml and dist/DA-ESS-ContentUpdate-latest.tar.gz to job artifact-test_summary_results.zip
@@ -65,4 +76,4 @@ jobs:
           run: |       
             echo "This job will fail if there are failures in unit-testing"  
             python .github/workflows/format_test_results.py >> $GITHUB_STEP_SUMMARY
-            echo "The Unit testing is completed. See details in the unit-testing job summary UI "
+            echo "The Unit testing is completed. See details in the unit-testing job summary UI "