test:conflict resolved

Author: Detection Testing Service

.github/workflows/build-and-validate.yml

@@ -147,23 +147,19 @@ jobs:
           source venv/bin/activate
           python3 contentctl.py --path . --verbose generate --product ESCU --output dist/escu
           python3 contentctl.py --path . --verbose generate --product SAAWS --output dist/saaws
-          python3 contentctl.py --path . --verbose generate --product DevSecOps --output dist/devsecops
           python3 contentctl.py --path . --verbose generate --product SSA --output dist/ssa
 
       - name: Copy lookups .csv files
         run: |
           # clean up current lookups
           rm -rf dist/escu/lookups
           rm -rf dist/saaws/lookups
-          rm -rf dist/devsecops/lookups
           mkdir dist/escu/lookups
           mkdir dist/saaws/lookups
-          mkdir dist/devsecops/lookups
           #copy over lookups
           cd lookups
           cp -rv *.csv ../dist/escu/lookups
           cp -rv *.csv ../dist/saaws/lookups
-          cp -rv *.csv ../dist/devsecops/lookups
 
       #Tag is '' for non-tagged push and the tag name for a tagged release
       - name: Set tag
@@ -203,9 +199,6 @@ jobs:
             sed -i "s/version = .*$/version = $CONTENT_VERSION/g" dist/saaws/default/content-version.conf
             tar -czf content-pack-build-saaws.tar.gz dist/saaws/*
 
-            # update build number and version for devsecops
-            tar -czf content-pack-build-devsecops.tar.gz dist/devsecops/*
-
             # update build number and version for ssa
             tar -czf content-pack-build-ssa.tar.gz dist/ssa/*
 
@@ -216,7 +209,6 @@ jobs:
           path: |
             content-pack-build-escu.tar.gz
             content-pack-build-saaws.tar.gz
-            content-pack-build-devsecops.tar.gz
             content-pack-build-ssa.tar.gz
 
 
@@ -261,17 +253,14 @@ jobs:
           cd build
           tar -zxf content-pack-build-escu.tar.gz
           tar -zxf content-pack-build-saaws.tar.gz
-          tar -zxf content-pack-build-devsecops.tar.gz
           tar -zxf content-pack-build-ssa.tar.gz
 
           mv dist/escu      DA-ESS-ContentUpdate
           mv dist/saaws     DA-ESS_AmazonWebServices_Content
-          mv dist/devsecops dev_sec_ops_analytics
           mv dist/ssa       SSA_Content
 
           slim package -o upload DA-ESS-ContentUpdate
           slim package -o upload DA-ESS_AmazonWebServices_Content
-          slim package -o upload dev_sec_ops_analytics
           #slim package -o upload SSA_Content
 
           cp upload/DA-ESS-ContentUpdate-*.tar.gz DA-ESS-ContentUpdate-latest.tar.gz
@@ -280,9 +269,6 @@ jobs:
           cp upload/DA-ESS_AmazonWebServices_Content-*tar.gz DA-ESS_AmazonWebServices_Content-latest.tar.gz
           sha256sum DA-ESS_AmazonWebServices_Content-latest.tar.gz >> checksum.txt
 
-          cp upload/dev_sec_ops_analytics-*tar.gz dev_sec_ops_analytics-latest.tar.gz
-          sha256sum dev_sec_ops_analytics-latest.tar.gz >> checksum.txt
-
           #Do this copy so that we conform as much as possible, and have to make 
           #as few changes as possible, once we start generating this as a real,
           #properly packaged app
@@ -305,7 +291,6 @@ jobs:
           path: |
             build/DA-ESS-ContentUpdate-latest.tar.gz
             build/DA-ESS_AmazonWebServices_Content-latest.tar.gz
-            build/dev_sec_ops_analytics-latest.tar.gz
             build/SSA_Content-latest.tar.gz
             build/checksum.txt
 
@@ -350,7 +335,6 @@ jobs:
           #Better not to pass these arguments on the command line, if possible
           ./appinspect.sh ../ DA-ESS-ContentUpdate-latest.tar.gz "$APPINSPECT_USERNAME" "$APPINSPECT_PASSWORD"
           ./appinspect.sh ../ DA-ESS_AmazonWebServices_Content-latest.tar.gz "$APPINSPECT_USERNAME" "$APPINSPECT_PASSWORD"
-          ./appinspect.sh ../ dev_sec_ops_analytics-latest.tar.gz "$APPINSPECT_USERNAME" "$APPINSPECT_PASSWORD"
 
       - name: Create report artifact
         if: always()

.gitlab-ci.yml

@@ -11,6 +11,7 @@ stages:
   - publish_smoketest_runner
   - publish_deployer
   - smoketest_staging
+  - publish_build_to_pre_qa
 
 
 publish_deployer:
@@ -59,3 +60,24 @@ smoketest_staging:
   only:
     - /^ssa.*$/
     - develop
+
+publish_build_to_pre_qa:
+  stage: publish_build_to_pre_qa
+  artifacts:
+    when: always
+    paths:
+      - artifacts/*
+  image: python:3.8-alpine
+  before_script:
+    - apk add --update --no-cache make curl bash git
+    - curl -L https://github.com/screwdriver-cd/gitversion/releases/download/v1.1.1/gitversion_linux_amd64 -o /usr/local/bin/gitversion && chmod +x /usr/local/bin/gitversion
+    - eval $(ssh-agent -s)
+  script:
+    - mkdir -p artifacts
+    - pip install requests
+    - python security_content_automation/publish_build_to_pre_qa/publish_build_to_pre_qa.py --version $CI_COMMIT_REF_NAME --builds DA-ESS_AmazonWebServices_Content DA-ESS-ContentUpdate
+  after_script:
+    - cp publish_build_to_pre_qa.log artifacts/publish_build_to_pre_qa.log
+  rules:
+    - if: '$CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+\.[0-9]$/'
+      when: always

bin/generate.py

@@ -104,15 +104,26 @@ def generate_ssa_yaml(detections, TEMPLATE_PATH, OUTPUT_PATH):
     # disable yaml pointers https://stackoverflow.com/questions/51272814/python-yaml-dumping-pointer-references
     yaml.Dumper.ignore_aliases = lambda *args : True
 
+    # wiping old detections for SSA
+    shutil.rmtree(OUTPUT_PATH + '/detections/*', ignore_errors=True)
+
     for d in detections:
-        manifest_file = OUTPUT_PATH + '/detections/' + d['name'].lower().replace(" ", "_") + '.yml'
+        manifest_file = OUTPUT_PATH + '/detections/ssa___' + d['name'].lower().replace(" ", "_") + '.yml'
 
         # remove unused fields
         del d['risk']
         del d['deployment']
         del d['mappings']
         del d['savedsearch_annotations']
 
+        # add detection test
+        test_file = 'ssa___' + d['name'].lower().replace(" ", "_") + '.test.yml'
+        for file in glob.glob('tests/*/*'):
+            if test_file == file.split("/")[-1]:
+                with open(file, 'r') as file:
+                    test_yaml = yaml.safe_load(file)
+                    d['test'] = test_yaml
+
         with open(manifest_file, 'w') as file:
             documents = yaml.dump(d, file, sort_keys=True)
 
@@ -414,7 +425,6 @@ def add_playbook(detection, playbooks):
             if detection['name'] in playbook['tags']['detections']:
                 detection['how_to_implement'] = detection['how_to_implement'] + preface + playbook['name']
         except KeyError:
-            print("playbook " + playbook['name'] + " has no detections, passing....")
             pass
     return detection
 
@@ -664,6 +674,43 @@ def main(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE):
     global global_product
     global_product = PRODUCT
 
+    TEMPLATE_PATH = path.join(REPO_PATH, 'bin/jinja2_templates')
+
+    objects = get_objects(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE)
+
+    try:
+        if VERBOSE:
+            print("generating Mitre lookups")
+    #    generate_mitre_lookup(OUTPUT_PATH)
+    except Exception as e:
+        print('Error: ' + str(e))
+        print("WARNING: Generation of Mitre lookup failed.")
+
+    # calculate deprecation totals
+    deprecated = []
+    for d in objects['detections']:
+        if 'deprecated' in d:
+            deprecated.append(d)
+
+    detection_path = ''
+    lookups_path = ''
+    lookups_files= ''
+    use_case_lib_path = ''
+    macros_path = ''
+    workbench_panels_objects = ''
+
+    if global_product == 'SSA':
+        detection_path = generate_ssa_yaml(objects["detections"], TEMPLATE_PATH, OUTPUT_PATH)
+        objects["macros"] = []
+    else:
+        detection_path = generate_savedsearches_conf(objects["detections"], objects["deployments"], TEMPLATE_PATH, OUTPUT_PATH)
+        lookups_path = generate_transforms_conf(objects["lookups"], TEMPLATE_PATH, OUTPUT_PATH)
+        lookups_path = generate_collections_conf(objects["lookups"], TEMPLATE_PATH, OUTPUT_PATH)
+        lookups_files = generate_lookup_files(objects["lookups"], TEMPLATE_PATH, OUTPUT_PATH,REPO_PATH)
+        use_case_lib_path = generate_use_case_library_conf(objects["stories"], objects["detections"], TEMPLATE_PATH, OUTPUT_PATH)
+        macros_path = generate_macros_conf(objects["macros"], objects["detections"], TEMPLATE_PATH, OUTPUT_PATH)
+        workbench_panels_objects = generate_workbench_panels(objects["detections"], objects["stories"], TEMPLATE_PATH, OUTPUT_PATH)
+
     if global_product == 'API':
         json_types = []
         # List of all YAML types to search in repo
@@ -673,63 +720,25 @@ def main(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE):
         if VERBOSE: print("JSON output directory: " + output_dir)
         # remove any pre-existing output directories
         shutil.rmtree(output_dir, ignore_errors=True)
-        if VERBOSE: print("Remove pre-existing JSON directory")
+        if VERBOSE: print("remove pre-existing JSON directory")
         # create output directory
         os.mkdir(output_dir)
-        if VERBOSE: print("Created output directory")
+        if VERBOSE: print("created output directory")
         # Generate all YAML types
         for yt in yml_types:
             processor = Yaml2Json(yt, REPO_PATH)
             with open(os.path.join(output_dir, yt + '.json'), 'w') as json_out:
                 # write out YAML type
                 json.dump(processor.list_objects(yt), json_out)
-                if VERBOSE: print("Writing %s JSON" % yt)
-
-    else:
-        TEMPLATE_PATH = path.join(REPO_PATH, 'bin/jinja2_templates')
-
-        objects = get_objects(REPO_PATH, OUTPUT_PATH, PRODUCT, VERBOSE)
-
-        try:
-            if VERBOSE:
-                print("generating Mitre lookups")
-        #    generate_mitre_lookup(OUTPUT_PATH)
-        except Exception as e:
-            print('Error: ' + str(e))
-            print("WARNING: Generation of Mitre lookup failed.")
-
-        # calculate deprecation totals
-        deprecated = []
-        for d in objects['detections']:
-            if 'deprecated' in d:
-                deprecated.append(d)
-
-        detection_path = ''
-        lookups_path = ''
-        lookups_files= ''
-        use_case_lib_path = ''
-        macros_path = ''
-        workbench_panels_objects = ''
-
-        if global_product == 'SSA':
-            detection_path = generate_ssa_yaml(objects["detections"], TEMPLATE_PATH, OUTPUT_PATH)
-            objects["macros"] = []
-        else:
-            detection_path = generate_savedsearches_conf(objects["detections"], objects["deployments"], TEMPLATE_PATH, OUTPUT_PATH)
-            lookups_path = generate_transforms_conf(objects["lookups"], TEMPLATE_PATH, OUTPUT_PATH)
-            lookups_path = generate_collections_conf(objects["lookups"], TEMPLATE_PATH, OUTPUT_PATH)
-            lookups_files = generate_lookup_files(objects["lookups"], TEMPLATE_PATH, OUTPUT_PATH,REPO_PATH)
-            use_case_lib_path = generate_use_case_library_conf(objects["stories"], objects["detections"], TEMPLATE_PATH, OUTPUT_PATH)
-            macros_path = generate_macros_conf(objects["macros"], objects["detections"], TEMPLATE_PATH, OUTPUT_PATH)
-            workbench_panels_objects = generate_workbench_panels(objects["detections"], objects["stories"], TEMPLATE_PATH, OUTPUT_PATH)
-
-        if VERBOSE:
-            print("{0} stories have been successfully written to {1}".format(len(objects["stories"]), use_case_lib_path))
-            print("{0} detections have been successfully written to {1}".format(len(objects["detections"]), detection_path))
-            print("{0} detections have been marked deprecated on {1}".format(len(deprecated), detection_path))
-            print("{0} macros have been successfully written to {1}".format(len(objects["macros"]), macros_path))
-            print("{0} workbench panels have been successfully written to {1}, {2} and {3}".format(len(workbench_panels_objects), OUTPUT_PATH + "/default/es_investigations.conf", OUTPUT_PATH + "/default/workflow_actions.conf", OUTPUT_PATH + "/default/data/ui/panels/*"))
-            print("security content generation completed..")
+                if VERBOSE: print("writing {0} JSON".format(yt))
+
+    if VERBOSE:
+        print("{0} stories have been successfully written to {1}".format(len(objects["stories"]), use_case_lib_path))
+        print("{0} detections have been successfully written to {1}".format(len(objects["detections"]), detection_path))
+        print("{0} detections have been marked deprecated on {1}".format(len(deprecated), detection_path))
+        print("{0} macros have been successfully written to {1}".format(len(objects["macros"]), macros_path))
+        print("{0} workbench panels have been successfully written to {1}, {2} and {3}".format(len(workbench_panels_objects), OUTPUT_PATH + "/default/es_investigations.conf", OUTPUT_PATH + "/default/workflow_actions.conf", OUTPUT_PATH + "/default/data/ui/panels/*"))
+        print("security content generation completed..")
 
 
 if __name__ == "__main__":

bin/jinja2_templates/detection.j2

@@ -49,3 +49,7 @@ tags:
   {% for observable in observables -%}
   - {{observable}}
   {% endfor -%}
+  nist:
+  -
+  cis20:
+  -

bin/jinja2_templates/doc_playbooks.j2

@@ -39,7 +39,7 @@ tags:
 {{ playbook.how_to_implement}}
 
 #### Playbooks
-![](https://raw.githubusercontent.com/splunk/security_content/develop/playbooks/{{ playbook.name | lower | replace(" ", "_")}}.png)
+![](https://raw.githubusercontent.com/splunk/security_content/develop/playbooks/{{ playbook.playbook | lower | replace(" ", "_")}}.png)
 
 #### Required field
 {% for field in playbook.tags.playbook_fields -%}
@@ -54,4 +54,4 @@ tags:
 {% endif %}
 
 
-[*source*](https://github.com/splunk/security_content/tree/develop/playbooks/{{ playbook.name | lower | replace (" ", "_") }}.yml) \| *version*: **{{playbook.version}}**
+[*source*](https://github.com/splunk/security_content/tree/develop/playbooks/{{ playbook.playbook | lower | replace (" ", "_") }}.yml) \| *version*: **{{playbook.version}}**