From 52c80236788d5ce2844269590115d2f5e258685a Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 29 Apr 2022 13:31:56 -0700
Subject: [PATCH 01/38] Added support for enumerating all of the different
 actions that content_changer can make.  This list of actions will be
 dynamically updated base on the names of the functions added in
 content_changer.py!

---
 .../application/use_cases/content_changer.py               | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/bin/contentctl_project/contentctl_core/application/use_cases/content_changer.py b/bin/contentctl_project/contentctl_core/application/use_cases/content_changer.py
index 0573ab422c..2f557c5ecb 100644
--- a/bin/contentctl_project/contentctl_core/application/use_cases/content_changer.py
+++ b/bin/contentctl_project/contentctl_core/application/use_cases/content_changer.py
@@ -26,9 +26,14 @@ def execute(self, input_dto: ContentChangerInputDto) -> None:
     
         input_dto.adapter.writeObjectsInPlace(objects)
 
+    @staticmethod
+    def enumerate_content_changer_functions(exclude_functions: list[str] = ["execute", "enumerate_content_changer_functions"]) -> list[str]:
+        import inspect
+        members = inspect.getmembers(ContentChanger, predicate=inspect.isfunction)
+        function_names = [function_object[0] for function_object in members if function_object[0] not in exclude_functions]
+        return function_names
 
     # Define Converter Functions here
-
     def example_converter_func(self, objects : list) -> None:
         for obj in objects:
             obj['author'] = obj['author'].upper()

From 3c5a342479b7e667acdf6444c38bdc3fcedc7ee5 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 29 Apr 2022 15:43:01 -0700
Subject: [PATCH 02/38] Better support in contentctl and content_changer for
 updating yml files. The help statement for content_changer in content_ctl now
 dynamically lists the transforms that can be performed on detections.  In
 addition, a redundant transfrom was removed.

---
 .../application/use_cases/content_changer.py  | 39 +++++++++----------
 contentctl.py                                 |  8 ++--
 2 files changed, 23 insertions(+), 24 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/application/use_cases/content_changer.py b/bin/contentctl_project/contentctl_core/application/use_cases/content_changer.py
index 2f557c5ecb..0606ed8ad9 100644
--- a/bin/contentctl_project/contentctl_core/application/use_cases/content_changer.py
+++ b/bin/contentctl_project/contentctl_core/application/use_cases/content_changer.py
@@ -1,6 +1,8 @@
+import argparse
+from ast import arg
 import re
 import uuid
-
+import inspect
 from dataclasses import dataclass
 
 from bin.contentctl_project.contentctl_core.application.factory.object_factory import ObjectFactory, ObjectFactoryInputDto
@@ -13,7 +15,6 @@ class ContentChangerInputDto:
     factory_input_dto : ObjectFactoryInputDto
     converter_func_name : str
 
-
 class ContentChanger:
 
     def execute(self, input_dto: ContentChangerInputDto) -> None:
@@ -27,8 +28,7 @@ def execute(self, input_dto: ContentChangerInputDto) -> None:
         input_dto.adapter.writeObjectsInPlace(objects)
 
     @staticmethod
-    def enumerate_content_changer_functions(exclude_functions: list[str] = ["execute", "enumerate_content_changer_functions"]) -> list[str]:
-        import inspect
+    def enumerate_content_changer_functions(exclude_functions: list[str] = ["enumerate_content_changer_functions", "execute", "example_converter_func"]) -> list[str]:
         members = inspect.getmembers(ContentChanger, predicate=inspect.isfunction)
         function_names = [function_object[0] for function_object in members if function_object[0] not in exclude_functions]
         return function_names
@@ -36,16 +36,7 @@ def enumerate_content_changer_functions(exclude_functions: list[str] = ["execute
     # Define Converter Functions here
     def example_converter_func(self, objects : list) -> None:
         for obj in objects:
-            obj['author'] = obj['author'].upper()
-
-    def add_default_risk_values(self, objects : list) -> None:
-        for obj in objects:
-            if not 'confidence' in obj['tags']:
-                obj['tags']['confidence'] = 50
-            if not 'impact' in obj['tags']:
-                obj['tags']['impact'] = 50
-            if not 'risk_score' in obj['tags']:
-                obj['tags']['risk_score'] = 25
+            obj['author'] = obj['author'].upper()                
 
     def add_unknown_context(self, objects : list) -> None:
         for obj in objects:
@@ -109,8 +100,9 @@ def add_default_kill_chain_phases(self, objects : list) -> None:
 
     def fix_wrong_calculated_risk_score(self, objects : list) -> None:
         for obj in objects:
-            calculated_risk_score = (int(obj['tags']['impact']))*(int(obj['tags']['confidence']))/100
-            if calculated_risk_score != int(obj['tags']['risk_score']):
+            #Risk score must be an integer, so we round it to the nearest integer
+            calculated_risk_score = round((obj['tags']['impact'] * obj['tags']['confidence'])/100)
+            if calculated_risk_score != round(obj['tags']['risk_score']):
                 obj['tags']['risk_score'] = calculated_risk_score
 
     def add_asset_type_to_endpoint_detections(self, objects : list) -> None:
@@ -179,13 +171,18 @@ def fix_kill_chain(self, objects : list) -> None:
 
     def add_default_confidence_impact_risk_score(self, objects : list) -> None:
         for obj in objects:
-            if 'confidence' not in obj['tags']:
+            updated = True
+            if not 'confidence' in obj['tags']:
+                updated = True
                 obj['tags']['confidence'] = 50
-            if 'impact' not in obj['tags']:
+            if not 'impact' in obj['tags']:
+                updated = True
                 obj['tags']['impact'] = 50
-            if 'risk_score' not in obj['tags']:
-                calculated_risk_score = (int(obj['tags']['impact']))*(int(obj['tags']['confidence']))/100
-                obj['tags']['risk_score'] = calculated_risk_score
+            if not 'risk_score' in obj['tags'] or updated == True:
+                #Recalculate the risk score if we have added/updated
+                #the confidence or impact fields OR the risk_score
+                #was missing in the first place
+                self.fix_wrong_calculated_risk_score([obj])
 
     def fix_cc(self, objects : list) -> None:
         for obj in objects:
diff --git a/contentctl.py b/contentctl.py
index f0e4026e8f..14c29ed49d 100644
--- a/contentctl.py
+++ b/contentctl.py
@@ -257,7 +257,7 @@ def main(args):
     parser = argparse.ArgumentParser(
         description="Use `contentctl.py action -h` to get help with any Splunk Security Content action")
     parser.add_argument("-p", "--path", required=True, 
-                                        help="path to the Splunk Security Content folder")
+                                        help="path to the Splunk Security Content folder",)
     parser.set_defaults(func=lambda _: parser.print_help())
 
     actions_parser = parser.add_subparsers(title="Splunk Security Content actions", dest="action")
@@ -287,8 +287,10 @@ def main(args):
         help="Type of package to create, choose between `ESCU`, `SSA` or `API`.")
     generate_parser.set_defaults(func=generate)
     
-    content_changer_parser.add_argument("-cf", "--change_function", required=True, type=str,
-        help="Define a change funtion defined in bin/contentctl_core/contentctl/application/use_cases/content_changer.py")
+    content_changer_choices = ContentChanger.enumerate_content_changer_functions()
+    content_changer_parser.add_argument("-cf", "--change_function", required=True, metavar='{ ' + ', '.join(content_changer_choices) +' }' , type=str, choices=content_changer_choices, 
+                                        help= "Choose from the functions above defined in \nbin/contentctl_core/contentctl/application/use_cases/content_changer.py")
+    
     content_changer_parser.set_defaults(func=content_changer)
 
     docgen_parser.add_argument("-o", "--output", required=True, type=str,

From 710b582b15be8c81ebe87438948785143729bafe Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 29 Apr 2022 15:58:03 -0700
Subject: [PATCH 03/38] Added an option for all, meaning that all of the
 transforms \(besides the transform on test files\) will be run.

---
 .../application/use_cases/content_changer.py              | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/bin/contentctl_project/contentctl_core/application/use_cases/content_changer.py b/bin/contentctl_project/contentctl_core/application/use_cases/content_changer.py
index 0606ed8ad9..dadcb7a07c 100644
--- a/bin/contentctl_project/contentctl_core/application/use_cases/content_changer.py
+++ b/bin/contentctl_project/contentctl_core/application/use_cases/content_changer.py
@@ -33,6 +33,14 @@ def enumerate_content_changer_functions(exclude_functions: list[str] = ["enumera
         function_names = [function_object[0] for function_object in members if function_object[0] not in exclude_functions]
         return function_names
 
+    def all(self, objects : list) -> None:
+        for func_name in ContentChanger.enumerate_content_changer_functions():
+            if func_name not in ["all", "change_test_file_format"]:
+                print(f"calling {func_name}")
+                func_object = getattr(self, func_name)
+                func_object(objects)
+
+
     # Define Converter Functions here
     def example_converter_func(self, objects : list) -> None:
         for obj in objects:

From f81a103b014736641e60c48cf6cf9530f8839840 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 29 Apr 2022 16:53:20 -0700
Subject: [PATCH 04/38] Added the majority of support for offline and cached
 mode for contentctl.  This means that developers can much more rapidly test
 new builds without waiting on slow API end points \(taking build and validate
 job from upwards of 10 minutes to a few seconds\). It also means that the
 tool can be used in completely offline, secure environments that do not
 mirror the required HTTP endpoints.

---
 .../application/factory/factory.py            |  3 +-
 .../builder/attack_enrichment.py              |  4 +-
 .../builder/cve_enrichment.py                 | 47 +++++++++++++++++--
 .../security_content_detection_builder.py     |  5 +-
 .../builder/splunk_app_enrichment.py          | 45 ++++++++++++++++--
 contentctl.py                                 |  2 +
 6 files changed, 94 insertions(+), 12 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/application/factory/factory.py b/bin/contentctl_project/contentctl_core/application/factory/factory.py
index 1b41524232..8ffb5db699 100644
--- a/bin/contentctl_project/contentctl_core/application/factory/factory.py
+++ b/bin/contentctl_project/contentctl_core/application/factory/factory.py
@@ -27,6 +27,7 @@ class FactoryInputDto:
     playbook_builder: PlaybookBuilder
     director: Director
     attack_enrichment: dict
+    force_cached_or_offline: bool = False
     
 
 @dataclass()
@@ -116,7 +117,7 @@ def createSecurityContent(self, type: SecurityContentType) -> list:
                               self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, 
                                    self.output_dto.deployments, self.output_dto.playbooks, self.output_dto.baselines,
                                    self.output_dto.tests, self.input_dto.attack_enrichment, self.output_dto.macros,
-                                   self.output_dto.lookups)
+                                   self.output_dto.lookups, self.input_dto.force_cached_or_offline)
                               detection = self.input_dto.detection_builder.getObject()
                               self.output_dto.detections.append(detection)
                     
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py
index b74f9cbde1..cdfa844755 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py
@@ -13,11 +13,13 @@
 class AttackEnrichment():
 
     @classmethod
-    def get_attack_lookup(self, store_csv = None) -> dict:
+    def get_attack_lookup(self, store_csv = None, force_cached_or_offline: bool = False) -> dict:
         attack_lookup = dict()
         file_path = os.path.join(os.path.dirname(__file__), '../../../../lookups/mitre_enrichment.csv')
 
         try:
+            if force_cached_or_offline is True:
+                raise(Exception("WARNING - Using cached MITRE Attack Enrichment.  Attack Enrichment may be out of date. Only use this setting for offline environments and development purposes."))
             lift = attack_client()
             all_enterprise = lift.get_enterprise(stix_format=False)
             enterprise_relationships = lift.get_enterprise_relationships()
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py
index a6e2ae5cf7..2ac463296f 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py
@@ -1,17 +1,55 @@
 
 from pycvesearch import CVESearch
+import functools
+import pickle
 
 CVESSEARCH_API_URL = 'https://cve.circl.lu'
 
+CVE_CACHE_FILENAME = "lookups/CVE_CACHE.PICKLE"
+
+@functools.cache
+def cvesearch_helper(url:str, cve_id:str):
+    
+
+    try:
+        with open(CVE_CACHE_FILENAME, 'rb') as p:
+            dat = pickle.load(p)
+            if cve_id in dat:
+                return dat[cve_id]
+    except Exception as e:
+        print(f"error loading pickle file {CVE_CACHE_FILENAME}- it probably didn't exist. we will create it")
+        dat = {}
+    
+    try:
+        print(f"Even though 'force_cached_or_offline' was enabled, we were unable to find {cve_id}, so we will attempt to resolve it at {CVESSEARCH_API_URL}")
+        cve = cvesearch_id_helper(url)
+        result = cve.id(cve_id)
+    except Exception as e:
+        raise(Exception(f"The option 'force_cached_or_offline' was used, but {cve_id} not found in {CVE_CACHE_FILENAME} and unable to connect to {CVESSEARCH_API_URL}"))
+    with open(CVE_CACHE_FILENAME, 'wb') as p:
+        dat[cve_id] = result
+        pickle.dump(dat, p)
+    
+    return result
+
+@functools.cache
+def cvesearch_id_helper(url:str):
+    cve = CVESearch(CVESSEARCH_API_URL)
+    return cve
+
+
 
 class CveEnrichment():
 
     @classmethod
-    def enrich_cve(self, cve_id: str) -> dict:
+    def enrich_cve(self, cve_id: str, force_cached_or_offline: bool = False) -> dict:
         cve_enriched = dict()
         try:
-            cve = CVESearch(CVESSEARCH_API_URL)
-            result = cve.id(cve_id)
+            if force_cached_or_offline is True:
+                result = cvesearch_helper(CVESSEARCH_API_URL, cve_id)
+            else:
+                cve = CVESearch(CVESSEARCH_API_URL)
+                result = cve.id(cve_id)
             cve_enriched['id'] = cve_id
             cve_enriched['cvss'] = result['cvss']
             cve_enriched['summary'] = result['summary']
@@ -19,5 +57,8 @@ def enrich_cve(self, cve_id: str) -> dict:
             # there was a error calling the circl api lets just empty the object
             print("WARNING, issue enriching {0}, with error: {1}".format(cve_id, str(TypeErr)))
             cve_enriched = dict()
+        except Exception as e:
+            print("WARNING - {0}".format(str(e)))
+            cve_enriched = dict()
     
         return cve_enriched
\ No newline at end of file
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py
index 072facd9c0..0453f7db21 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py
@@ -16,6 +16,7 @@
 
 class SecurityContentDetectionBuilder(DetectionBuilder):
     security_content_obj : SecurityContentObject
+    force_cached_or_offline: bool = False
 
 
     def setObject(self, path: str) -> None:
@@ -219,14 +220,14 @@ def addCve(self) -> None:
             self.security_content_obj.cve_enrichment = []
             if self.security_content_obj.tags.cve:
                 for cve in self.security_content_obj.tags.cve:
-                    self.security_content_obj.cve_enrichment.append(CveEnrichment.enrich_cve(cve))
+                    self.security_content_obj.cve_enrichment.append(CveEnrichment.enrich_cve(cve, force_cached_or_offline = self.force_cached_or_offline))
 
     def addSplunkApp(self) -> None:
         if self.security_content_obj:
             self.security_content_obj.splunk_app_enrichment = []
             if self.security_content_obj.tags.supported_tas:
                 for splunk_app in self.security_content_obj.tags.supported_tas:
-                    self.security_content_obj.splunk_app_enrichment.append(SplunkAppEnrichment.enrich_splunk_app(splunk_app))
+                    self.security_content_obj.splunk_app_enrichment.append(SplunkAppEnrichment.enrich_splunk_app(splunk_app, force_cached_or_offline=self.force_cached_or_offline))
 
     def reset(self) -> None:
         self.security_content_obj = None
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py
index 0a3c520bdf..65cc4dc608 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py
@@ -1,19 +1,50 @@
 import requests
 import xmltodict
 import json
+import functools
+import pickle
 
 SPLUNKBASE_API_URL = "https://apps.splunk.com/api/apps/entriesbyid/"
 
+APP_ENRICHMENT_CACHE_FILENAME = "lookups/APP_ENRICHMENT_CACHE.PICKLE"
+
+@functools.cache
+def requests_get_helper(url:str)->bytes:
+    try:
+        with open(APP_ENRICHMENT_CACHE_FILENAME, 'rb') as p:
+            dat = pickle.load(p)
+            if url in dat:
+                return dat[url]
+    except Exception as e:
+        print(f"error loading pickle file {APP_ENRICHMENT_CACHE_FILENAME}- it probably didn't exist. we will create it")
+        dat = {}
+
+    try:
+        print(f"Even though 'force_cached_or_offline' is enabled, the url {url} was not found. We will try to resolve it.")
+        req = requests.get(url)
+        with open(APP_ENRICHMENT_CACHE_FILENAME, 'wb') as p:
+            dat[url] = req.content
+            pickle.dump(dat, p)
+    except Exception as e:
+        raise(Exception(f"ERROR - Failed to get Splunk App Enrichment at {SPLUNKBASE_API_URL}"))
+
+    return req.content
+
+
 
 class SplunkAppEnrichment():
 
     @classmethod
-    def enrich_splunk_app(self, splunk_ta: str) -> dict:
+    def enrich_splunk_app(self, splunk_ta: str, force_cached_or_offline: bool = False) -> dict:
         appurl = SPLUNKBASE_API_URL + splunk_ta
         splunk_app_enriched = dict()
         try:
-            response = requests.get(appurl)
-            response_dict = xmltodict.parse(response.content)
+            if force_cached_or_offline is True:
+                content = requests_get_helper(appurl)
+                response_dict = xmltodict.parse(content)
+            else:
+                response = requests.get(appurl)
+                response_dict = xmltodict.parse(response.content)
             # check if list since data changes depending on answer
             url, results = self._parse_splunkbase_response(response_dict)
             # grab the app name
@@ -22,8 +53,12 @@ def enrich_splunk_app(self, splunk_ta: str) -> dict:
                     splunk_app_enriched['name'] = i['#text']
             # grab out the splunkbase url  
             if 'entriesbyid' in url:
-                response = requests.get(url)
-                response_dict = xmltodict.parse(response.content)
+                if force_cached_or_offline is True:
+                    content = requests_get_helper(url)
+                    response_dict = xmltodict.parse(content)
+                else:
+                    response = requests.get(url)
+                    response_dict = xmltodict.parse(response.content)
                 #print(json.dumps(response_dict, indent=2))
                 url, results = self._parse_splunkbase_response(response_dict)
                 # chop the url so we grab the splunkbase portion but not direct download
diff --git a/contentctl.py b/contentctl.py
index 14c29ed49d..46406c5f4a 100644
--- a/contentctl.py
+++ b/contentctl.py
@@ -258,6 +258,8 @@ def main(args):
         description="Use `contentctl.py action -h` to get help with any Splunk Security Content action")
     parser.add_argument("-p", "--path", required=True, 
                                         help="path to the Splunk Security Content folder",)
+    parser.add_argument("-c", "--cached_and_offline", required=False, type=bool, default=False,
+        help="Force cached/offline resources.  While this makes execution much faster, it may result in enrichment which is out of date. This is suitable for use only in development or disconnected environments.")
     parser.set_defaults(func=lambda _: parser.print_help())
 
     actions_parser = parser.add_subparsers(title="Splunk Security Content actions", dest="action")

From 31149c3e61e6f519987ae5c45029c71d5e957d68 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 29 Apr 2022 17:04:57 -0700
Subject: [PATCH 05/38] More changes to defaults and argument parsing to
 support offline and cached mode.

---
 .../contentctl_core/application/factory/factory.py   |  2 +-
 .../builder/cve_enrichment.py                        |  2 --
 contentctl.py                                        | 12 ++++++------
 3 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/application/factory/factory.py b/bin/contentctl_project/contentctl_core/application/factory/factory.py
index 8ffb5db699..b79c3f996f 100644
--- a/bin/contentctl_project/contentctl_core/application/factory/factory.py
+++ b/bin/contentctl_project/contentctl_core/application/factory/factory.py
@@ -27,7 +27,7 @@ class FactoryInputDto:
     playbook_builder: PlaybookBuilder
     director: Director
     attack_enrichment: dict
-    force_cached_or_offline: bool = False
+    force_cached_or_offline: bool = True
     
 
 @dataclass()
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py
index 2ac463296f..f10457a1b8 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py
@@ -9,8 +9,6 @@
 
 @functools.cache
 def cvesearch_helper(url:str, cve_id:str):
-    
-
     try:
         with open(CVE_CACHE_FILENAME, 'rb') as p:
             dat = pickle.load(p)
diff --git a/contentctl.py b/contentctl.py
index 46406c5f4a..2c3f2762d1 100644
--- a/contentctl.py
+++ b/contentctl.py
@@ -101,7 +101,7 @@ def generate(args) -> None:
         SecurityContentInvestigationBuilder(),
         SecurityContentPlaybookBuilder(),
         SecurityContentDirector(),
-        AttackEnrichment.get_attack_lookup(store_csv=True)
+        AttackEnrichment.get_attack_lookup(store_csv=True, force_cached_or_offline=args.cached_and_offline)
     )
 
     ba_factory_input_dto = BAFactoryInputDto(
@@ -158,7 +158,7 @@ def validate(args) -> None:
         SecurityContentInvestigationBuilder(),
         SecurityContentPlaybookBuilder(),
         SecurityContentDirector(),
-        AttackEnrichment.get_attack_lookup()
+        AttackEnrichment.get_attack_lookup(force_cached_or_offline=args.cached_and_offline)
     )
 
     ba_factory_input_dto = BAFactoryInputDto(
@@ -198,7 +198,7 @@ def doc_gen(args) -> None:
         SecurityContentInvestigationBuilder(),
         SecurityContentPlaybookBuilder(),
         SecurityContentDirector(),
-        AttackEnrichment.get_attack_lookup()
+        AttackEnrichment.get_attack_lookup(force_cached_or_offline=args.cached_and_offline)
     )
 
     doc_gen_input_dto = DocGenInputDto(
@@ -236,7 +236,7 @@ def reporting(args) -> None:
         SecurityContentInvestigationBuilder(),
         SecurityContentPlaybookBuilder(),
         SecurityContentDirector(),
-        AttackEnrichment.get_attack_lookup()
+        AttackEnrichment.get_attack_lookup(force_cached_or_offline=args.cached_and_offline)
     )
 
     reporting_input_dto = ReportingInputDto(
@@ -258,9 +258,9 @@ def main(args):
         description="Use `contentctl.py action -h` to get help with any Splunk Security Content action")
     parser.add_argument("-p", "--path", required=True, 
                                         help="path to the Splunk Security Content folder",)
-    parser.add_argument("-c", "--cached_and_offline", required=False, type=bool, default=False,
+    parser.add_argument("--cached_and_offline", action=argparse.BooleanOptionalAction,
         help="Force cached/offline resources.  While this makes execution much faster, it may result in enrichment which is out of date. This is suitable for use only in development or disconnected environments.")
-    parser.set_defaults(func=lambda _: parser.print_help())
+    parser.set_defaults(cached_and_offline=False, func=lambda _: parser.print_help())
 
     actions_parser = parser.add_subparsers(title="Splunk Security Content actions", dest="action")
     #new_parser = actions_parser.add_parser("new", help="Create new content (detection, story, baseline)")

From 03035d1c78173dcb3524b966d0cc46ba9d6516b4 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 2 May 2022 12:59:28 -0700
Subject: [PATCH 06/38] Enhancement to use threaded detection construction,
 which really speeds up verification of URLS.  Also, got the cache working and
 the force cache argument passed all the way through to the detection builder.
  Please note that this will take much longer to run the first time, because
 the cache has not been populated. Subsequent runs will be slower.  If data
 exists in this cache, it is NOT updated and will become stale if the web
 resource is updated.  This is only suitable for development purposes and
 offline environments.

---
 .../application/factory/factory.py            | 177 +++++++++++++++---
 .../domain/entities/detection.py              |   6 +-
 .../builder/security_content_director.py      |   2 +-
 contentctl.py                                 |   5 +-
 4 files changed, 165 insertions(+), 25 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/application/factory/factory.py b/bin/contentctl_project/contentctl_core/application/factory/factory.py
index b79c3f996f..a8d0ede095 100644
--- a/bin/contentctl_project/contentctl_core/application/factory/factory.py
+++ b/bin/contentctl_project/contentctl_core/application/factory/factory.py
@@ -68,6 +68,7 @@ def execute(self, input_dto: FactoryInputDto) -> None:
 
 
      def createSecurityContent(self, type: SecurityContentType) -> list:
+          
           objects = []
           if type == SecurityContentType.deployments:
                files = Utils.get_all_yml_files_from_directory(os.path.join(self.input_dto.input_path, str(type.name), 'ESCU'))
@@ -78,58 +79,190 @@ def createSecurityContent(self, type: SecurityContentType) -> list:
           
           validation_error_found = False
 
-          for file in files:
-               if not 'ssa__' in file:
+          import copy
+          import threading
+          NUM_THREADS = 50
+          
+          def make_detections_thread(index:int):
+               my_dto = copy.deepcopy(self.input_dto)
+               for i in range(index, len(files), NUM_THREADS):
+                    file = files[i]
+                    if 'ssa__' in file:
+                         continue
                     try:
                          if type == SecurityContentType.lookups:
-                              self.input_dto.director.constructLookup(self.input_dto.basic_builder, file)
-                              self.output_dto.lookups.append(self.input_dto.basic_builder.getObject())
+                              my_dto.director.constructLookup(my_dto.basic_builder, file)
+                              self.output_dto.lookups.append(my_dto.basic_builder.getObject())
                          
                          elif type == SecurityContentType.macros:
-                              self.input_dto.director.constructMacro(self.input_dto.basic_builder, file)
-                              self.output_dto.macros.append(self.input_dto.basic_builder.getObject())
+                              my_dto.director.constructMacro(my_dto.basic_builder, file)
+                              self.output_dto.macros.append(my_dto.basic_builder.getObject())
                          
                          elif type == SecurityContentType.deployments:
-                              self.input_dto.director.constructDeployment(self.input_dto.basic_builder, file)
-                              self.output_dto.deployments.append(self.input_dto.basic_builder.getObject())
+                              my_dto.director.constructDeployment(my_dto.basic_builder, file)
+                              self.output_dto.deployments.append(my_dto.basic_builder.getObject())
                          
                          elif type == SecurityContentType.playbooks:
-                              self.input_dto.director.constructPlaybook(self.input_dto.playbook_builder, file)
-                              self.output_dto.playbooks.append(self.input_dto.playbook_builder.getObject())                    
+                              my_dto.director.constructPlaybook(my_dto.playbook_builder, file)
+                              self.output_dto.playbooks.append(my_dto.playbook_builder.getObject())                    
                          
                          elif type == SecurityContentType.baselines:
-                              self.input_dto.director.constructBaseline(self.input_dto.baseline_builder, file, self.output_dto.deployments)
-                              baseline = self.input_dto.baseline_builder.getObject()
+                              my_dto.director.constructBaseline(my_dto.baseline_builder, file, self.output_dto.deployments)
+                              baseline = my_dto.baseline_builder.getObject()
                               self.output_dto.baselines.append(baseline)
                          
                          elif type == SecurityContentType.investigations:
-                              self.input_dto.director.constructInvestigation(self.input_dto.investigation_builder, file)
-                              investigation = self.input_dto.investigation_builder.getObject()
+                              my_dto.director.constructInvestigation(my_dto.investigation_builder, file)
+                              investigation = my_dto.investigation_builder.getObject()
                               self.output_dto.investigations.append(investigation)
 
                          elif type == SecurityContentType.stories:
-                              self.input_dto.director.constructStory(self.input_dto.story_builder, file, 
+                              my_dto.director.constructStory(my_dto.story_builder, file, 
                                    self.output_dto.detections, self.output_dto.baselines, self.output_dto.investigations)
-                              story = self.input_dto.story_builder.getObject()
+                              story = my_dto.story_builder.getObject()
                               self.output_dto.stories.append(story)
                     
                          elif type == SecurityContentType.detections:
-                              self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, 
+                              my_dto.director.constructDetection(my_dto.detection_builder, file, 
                                    self.output_dto.deployments, self.output_dto.playbooks, self.output_dto.baselines,
-                                   self.output_dto.tests, self.input_dto.attack_enrichment, self.output_dto.macros,
-                                   self.output_dto.lookups, self.input_dto.force_cached_or_offline)
-                              detection = self.input_dto.detection_builder.getObject()
+                                   self.output_dto.tests, my_dto.attack_enrichment, self.output_dto.macros,
+                                   self.output_dto.lookups, my_dto.force_cached_or_offline)
+                              detection = my_dto.detection_builder.getObject()
                               self.output_dto.detections.append(detection)
                     
                          elif type == SecurityContentType.unit_tests:
-                              self.input_dto.director.constructTest(self.input_dto.basic_builder, file)
-                              test = self.input_dto.basic_builder.getObject()
+                              my_dto.director.constructTest(my_dto.basic_builder, file)
+                              test = my_dto.basic_builder.getObject()
                               self.output_dto.tests.append(test)
                     
                     except ValidationError as e:
                          print('\nValidation Error for file ' + file)
                          print(e)
                          validation_error_found = True
+                    
+               #print(f"Builder thread {index} Done!")
+
+          builder_threads = []
+          #print("starting builder threads")
+          for t in range(NUM_THREADS):
+               thread = threading.Thread(target=make_detections_thread,args=(t,))
+               thread.start()
+               builder_threads.append(thread)
+          #print("All threads started")
+          for thread in builder_threads:
+               thread.join()
+               #print("thread joined")
+          #print("all threads joined")
+          
+
+
+               
+          #      if not 'ssa__' in file:
+          #           try:
+          #                if type == SecurityContentType.lookups:
+          #                     self.input_dto.director.constructLookup(self.input_dto.basic_builder, file)
+          #                     self.output_dto.lookups.append(self.input_dto.basic_builder.getObject())
+                         
+          #                elif type == SecurityContentType.macros:
+          #                     self.input_dto.director.constructMacro(self.input_dto.basic_builder, file)
+          #                     self.output_dto.macros.append(self.input_dto.basic_builder.getObject())
+                         
+          #                elif type == SecurityContentType.deployments:
+          #                     self.input_dto.director.constructDeployment(self.input_dto.basic_builder, file)
+          #                     self.output_dto.deployments.append(self.input_dto.basic_builder.getObject())
+                         
+          #                elif type == SecurityContentType.playbooks:
+          #                     self.input_dto.director.constructPlaybook(self.input_dto.playbook_builder, file)
+          #                     self.output_dto.playbooks.append(self.input_dto.playbook_builder.getObject())                    
+                         
+          #                elif type == SecurityContentType.baselines:
+          #                     self.input_dto.director.constructBaseline(self.input_dto.baseline_builder, file, self.output_dto.deployments)
+          #                     baseline = self.input_dto.baseline_builder.getObject()
+          #                     self.output_dto.baselines.append(baseline)
+                         
+          #                elif type == SecurityContentType.investigations:
+          #                     self.input_dto.director.constructInvestigation(self.input_dto.investigation_builder, file)
+          #                     investigation = self.input_dto.investigation_builder.getObject()
+          #                     self.output_dto.investigations.append(investigation)
+
+          #                elif type == SecurityContentType.stories:
+          #                     self.input_dto.director.constructStory(self.input_dto.story_builder, file, 
+          #                          self.output_dto.detections, self.output_dto.baselines, self.output_dto.investigations)
+          #                     story = self.input_dto.story_builder.getObject()
+          #                     self.output_dto.stories.append(story)
+                    
+          #                elif type == SecurityContentType.detections:
+          #                     self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, 
+          #                          self.output_dto.deployments, self.output_dto.playbooks, self.output_dto.baselines,
+          #                          self.output_dto.tests, self.input_dto.attack_enrichment, self.output_dto.macros,
+          #                          self.output_dto.lookups, self.input_dto.force_cached_or_offline)
+          #                     detection = self.input_dto.detection_builder.getObject()
+          #                     self.output_dto.detections.append(detection)
+                    
+          #                elif type == SecurityContentType.unit_tests:
+          #                     self.input_dto.director.constructTest(self.input_dto.basic_builder, file)
+          #                     test = self.input_dto.basic_builder.getObject()
+          #                     self.output_dto.tests.append(test)
+                    
+          #           except ValidationError as e:
+          #                print('\nValidation Error for file ' + file)
+          #                print(e)
+          #                validation_error_found = True
+
+          
+
+          # for file in files:
+          #      if not 'ssa__' in file:
+          #           try:
+          #                if type == SecurityContentType.lookups:
+          #                     self.input_dto.director.constructLookup(self.input_dto.basic_builder, file)
+          #                     self.output_dto.lookups.append(self.input_dto.basic_builder.getObject())
+                         
+          #                elif type == SecurityContentType.macros:
+          #                     self.input_dto.director.constructMacro(self.input_dto.basic_builder, file)
+          #                     self.output_dto.macros.append(self.input_dto.basic_builder.getObject())
+                         
+          #                elif type == SecurityContentType.deployments:
+          #                     self.input_dto.director.constructDeployment(self.input_dto.basic_builder, file)
+          #                     self.output_dto.deployments.append(self.input_dto.basic_builder.getObject())
+                         
+          #                elif type == SecurityContentType.playbooks:
+          #                     self.input_dto.director.constructPlaybook(self.input_dto.playbook_builder, file)
+          #                     self.output_dto.playbooks.append(self.input_dto.playbook_builder.getObject())                    
+                         
+          #                elif type == SecurityContentType.baselines:
+          #                     self.input_dto.director.constructBaseline(self.input_dto.baseline_builder, file, self.output_dto.deployments)
+          #                     baseline = self.input_dto.baseline_builder.getObject()
+          #                     self.output_dto.baselines.append(baseline)
+                         
+          #                elif type == SecurityContentType.investigations:
+          #                     self.input_dto.director.constructInvestigation(self.input_dto.investigation_builder, file)
+          #                     investigation = self.input_dto.investigation_builder.getObject()
+          #                     self.output_dto.investigations.append(investigation)
+
+          #                elif type == SecurityContentType.stories:
+          #                     self.input_dto.director.constructStory(self.input_dto.story_builder, file, 
+          #                          self.output_dto.detections, self.output_dto.baselines, self.output_dto.investigations)
+          #                     story = self.input_dto.story_builder.getObject()
+          #                     self.output_dto.stories.append(story)
+                    
+          #                elif type == SecurityContentType.detections:
+          #                     self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, 
+          #                          self.output_dto.deployments, self.output_dto.playbooks, self.output_dto.baselines,
+          #                          self.output_dto.tests, self.input_dto.attack_enrichment, self.output_dto.macros,
+          #                          self.output_dto.lookups, self.input_dto.force_cached_or_offline)
+          #                     detection = self.input_dto.detection_builder.getObject()
+          #                     self.output_dto.detections.append(detection)
+                    
+          #                elif type == SecurityContentType.unit_tests:
+          #                     self.input_dto.director.constructTest(self.input_dto.basic_builder, file)
+          #                     test = self.input_dto.basic_builder.getObject()
+          #                     self.output_dto.tests.append(test)
+                    
+          #           except ValidationError as e:
+          #                print('\nValidation Error for file ' + file)
+          #                print(e)
+          #                validation_error_found = True
 
           if validation_error_found:
                sys.exit(1)
\ No newline at end of file
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/detection.py b/bin/contentctl_project/contentctl_core/domain/entities/detection.py
index d2bb95035f..dbe001277e 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/detection.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/detection.py
@@ -125,9 +125,13 @@ def name_max_length(cls, values):
 
     # @validator('references')
     # def references_check(cls, v, values):
+    #     #import traceback
+    #     #import sys
+    #     #traceback.print_stack()
+    #     #sys.exit(0)
     #     for reference in v:
     #         try:
-    #             get = requests.get(reference)
+    #             get = requests.head(reference)
     #             if not get.status_code == 200:
     #                 raise ValueError('Reference ' + reference + ' is not reachable: ' + values["name"])
     #         except requests.exceptions.RequestException as e:
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_director.py b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_director.py
index 5475ded773..414822839e 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_director.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_director.py
@@ -12,7 +12,7 @@
 
 class SecurityContentDirector(Director):
 
-    def constructDetection(self, builder: DetectionBuilder, path: str, deployments: list, playbooks: list, baselines: list, tests: list, attack_enrichment: dict, macros: list, lookups: list) -> None:
+    def constructDetection(self, builder: DetectionBuilder, path: str, deployments: list, playbooks: list, baselines: list, tests: list, attack_enrichment: dict, macros: list, lookups: list, force_cached_or_offline: bool = False) -> None:
         builder.reset()
         builder.setObject(os.path.join(os.path.dirname(__file__), path))
         builder.addDeployment(deployments)
diff --git a/contentctl.py b/contentctl.py
index 2c3f2762d1..a760a7fb30 100644
--- a/contentctl.py
+++ b/contentctl.py
@@ -92,6 +92,7 @@ def generate(args) -> None:
         print("ERROR: invalid product. valid products are ESCU, SSA or API.")
         sys.exit(1)
 
+    
     factory_input_dto = FactoryInputDto(
         os.path.abspath(args.path),
         SecurityContentBasicBuilder(),
@@ -149,10 +150,12 @@ def validate(args) -> None:
         print("ERROR: invalid product. valid products are all, ESCU or SSA.")
         sys.exit(1)
 
+    detection_builder = SecurityContentDetectionBuilder()
+    detection_builder.force_cached_or_offline = args.cached_and_offline
     factory_input_dto = FactoryInputDto(
         os.path.abspath(args.path),
         SecurityContentBasicBuilder(),
-        SecurityContentDetectionBuilder(),
+        detection_builder,
         SecurityContentStoryBuilder(),
         SecurityContentBaselineBuilder(),
         SecurityContentInvestigationBuilder(),

From 578e6bb088d21712408039d9bb7d62f2d55533e5 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 2 May 2022 17:12:50 -0700
Subject: [PATCH 07/38] Updated a very large number of detections whose
 references were returning HTTP Status 301 - resource moved. For example, this
 includes a large number of fireeye reports, which are now under mandiant,
 cobaltstrike info, and microsoft links.

---
 .../aws_create_policy_version_to_allow_all_resources.yml    | 2 +-
 detections/cloud/aws_createaccesskey.yml                    | 2 +-
 detections/cloud/aws_createloginprofile.yml                 | 2 +-
 detections/cloud/aws_iam_assume_role_policy_brute_force.yml | 2 +-
 .../aws_saml_access_by_provider_user_and_principal.yml      | 2 +-
 detections/cloud/aws_saml_update_identity_provider.yml      | 2 +-
 detections/cloud/aws_setdefaultpolicyversion.yml            | 2 +-
 detections/cloud/aws_updateloginprofile.yml                 | 2 +-
 .../gsuite_email_suspicious_subject_with_attachment.yml     | 2 +-
 detections/cloud/gsuite_suspicious_shared_file_name.yml     | 2 +-
 .../cloud/o365_add_app_role_assignment_grant_user.yml       | 2 +-
 detections/cloud/o365_added_service_principal.yml           | 4 ++--
 detections/cloud/o365_new_federated_domain_added.yml        | 4 ++--
 detections/deprecated/suspicious_rundll32_rename.yml        | 2 +-
 detections/endpoint/active_setup_registry_autostart.yml     | 2 +-
 .../allow_file_and_printing_sharing_in_firewall.yml         | 2 +-
 detections/endpoint/allow_network_discovery_in_firewall.yml | 2 +-
 detections/endpoint/clop_common_exec_parameter.yml          | 2 +-
 detections/endpoint/clop_ransomware_known_service_name.yml  | 2 +-
 .../endpoint/cmdline_tool_not_executed_in_cmd_shell.yml     | 2 +-
 detections/endpoint/cobalt_strike_named_pipes.yml           | 6 +++---
 .../create_or_delete_windows_shares_using_net_exe.yml       | 2 +-
 detections/endpoint/delete_shadowcopy_with_powershell.yml   | 4 ++--
 detections/endpoint/detect_rclone_command_line_usage.yml    | 2 +-
 detections/endpoint/detect_renamed_rclone.yml               | 2 +-
 ...detect_rundll32_application_control_bypass___advpack.yml | 2 +-
 ...etect_rundll32_application_control_bypass___setupapi.yml | 2 +-
 ...etect_rundll32_application_control_bypass___syssetup.yml | 2 +-
 .../dllhost_with_no_command_line_arguments_with_network.yml | 2 +-
 detections/endpoint/drop_icedid_license_dat.yml             | 2 +-
 detections/endpoint/eventvwr_uac_bypass.yml                 | 2 +-
 detections/endpoint/extraction_of_registry_hives.yml        | 2 +-
 detections/endpoint/firewall_allowed_program_enable.yml     | 2 +-
 detections/endpoint/fodhelper_uac_bypass.yml                | 2 +-
 .../getcurrent_user_with_powershell_script_block.yml        | 2 +-
 ...gpupdate_with_no_command_line_arguments_with_network.yml | 2 +-
 detections/endpoint/high_process_termination_frequency.yml  | 2 +-
 .../endpoint/icedid_exfiltrated_archived_file_creation.yml  | 2 +-
 .../impacket_lateral_movement_commandline_parameters.yml    | 2 +-
 detections/endpoint/jscript_execution_using_cscript_app.yml | 2 +-
 detections/endpoint/kerberos_user_enumeration.yml           | 2 +-
 .../endpoint/logon_script_event_trigger_execution.yml       | 2 +-
 .../endpoint/ms_scripting_process_loading_ldap_module.yml   | 2 +-
 .../endpoint/ms_scripting_process_loading_wmi_module.yml    | 2 +-
 .../msbuild_suspicious_spawned_by_script_process.yml        | 2 +-
 .../endpoint/network_discovery_using_route_windows_app.yml  | 2 +-
 detections/endpoint/nishang_powershelltcponeline.yml        | 2 +-
 detections/endpoint/office_application_drop_executable.yml  | 2 +-
 .../office_document_spawned_child_process_to_download.yml   | 2 +-
 detections/endpoint/office_spawning_control.yml             | 2 +-
 .../endpoint/possible_lateral_movement_powershell_spawn.yml | 2 +-
 detections/endpoint/powershell_4104_hunting.yml             | 2 +-
 .../endpoint/print_spooler_adding_a_printer_driver.yml      | 4 ++--
 .../endpoint/print_spooler_failed_to_load_a_plug_in.yml     | 4 ++--
 .../endpoint/process_deleting_its_process_file_path.yml     | 2 +-
 detections/endpoint/ransomware_notes_bulk_creation.yml      | 2 +-
 .../regsvr32_silent_and_install_param_dll_loading.yml       | 2 +-
 .../endpoint/remcos_rat_file_creation_in_remcos_folder.yml  | 2 +-
 detections/endpoint/resize_shadowstorage_volume.yml         | 2 +-
 detections/endpoint/runas_execution_in_commandline.yml      | 2 +-
 ...rundll32_with_no_command_line_arguments_with_network.yml | 2 +-
 detections/endpoint/sdclt_uac_bypass.yml                    | 2 +-
 ...searchprotocolhost_with_no_command_line_with_network.yml | 2 +-
 detections/endpoint/services_escalate_exe.yml               | 2 +-
 detections/endpoint/slui_runas_elevated.yml                 | 4 ++--
 detections/endpoint/slui_spawning_a_process.yml             | 2 +-
 detections/endpoint/spoolsv_spawning_rundll32.yml           | 4 ++--
 detections/endpoint/spoolsv_suspicious_process_access.yml   | 4 ++--
 detections/endpoint/spoolsv_writing_a_dll.yml               | 4 ++--
 detections/endpoint/spoolsv_writing_a_dll___sysmon.yml      | 4 ++--
 detections/endpoint/sqlite_module_in_temp_folder.yml        | 2 +-
 .../endpoint/ssa___detect_rclone_command_line_usage.yml     | 2 +-
 detections/endpoint/ssa___resize_shadowstorage_volume.yml   | 2 +-
 detections/endpoint/ssa___windows_eventvwr_uac_bypass.yml   | 2 +-
 .../endpoint/ssa___windows_rasautou_dll_execution.yml       | 2 +-
 .../suspicious_dllhost_no_command_line_arguments.yml        | 2 +-
 .../suspicious_gpupdate_no_command_line_arguments.yml       | 2 +-
 .../suspicious_image_creation_in_appdata_folder.yml         | 2 +-
 detections/endpoint/suspicious_reg_exe_process.yml          | 2 +-
 .../endpoint/suspicious_rundll32_dllregisterserver.yml      | 4 ++--
 detections/endpoint/suspicious_rundll32_startw.yml          | 4 ++--
 .../suspicious_rundll32_with_no_command_line_arguments.yml  | 2 +-
 ...picious_searchprotocolhost_no_command_line_arguments.yml | 2 +-
 .../endpoint/suspicious_wav_file_in_appdata_folder.yml      | 2 +-
 .../unified_messaging_service_spawning_a_process.yml        | 2 +-
 detections/endpoint/windows_adfind_exe.yml                  | 2 +-
 .../endpoint/windows_high_file_deletion_frequency.yml       | 2 +-
 .../windows_powerview_constrained_delegation_discovery.yml  | 2 +-
 detections/endpoint/windows_rasautou_dll_execution.yml      | 2 +-
 ...windows_service_created_with_suspicious_service_path.yml | 2 +-
 detections/endpoint/xsl_script_execution_with_wmic.yml      | 2 +-
 .../cloud/gsuite_suspicious_calendar_invite.yml             | 2 +-
 .../endpoint/ssa___high_file_deletion_frequency.yml         | 2 +-
 .../endpoint/sunburst_correlation_dll_and_network_event.yml | 2 +-
 .../endpoint/suspicious_curl_network_connection.yml         | 2 +-
 .../experimental/endpoint/suspicious_plistbuddy_usage.yml   | 2 +-
 .../endpoint/suspicious_plistbuddy_usage_via_osquery.yml    | 2 +-
 .../endpoint/suspicious_sqlite3_lsquarantine_behavior.yml   | 2 +-
 .../experimental/network/detect_snicat_sni_exfiltration.yml | 2 +-
 .../experimental/network/detect_zerologon_via_zeek.yml      | 2 +-
 detections/experimental/web/supernova_webshell.yml          | 2 +-
 .../network/multiple_archive_files_http_post_traffic.yml    | 2 +-
 102 files changed, 116 insertions(+), 116 deletions(-)

diff --git a/detections/cloud/aws_create_policy_version_to_allow_all_resources.yml b/detections/cloud/aws_create_policy_version_to_allow_all_resources.yml
index 2132c89ba9..fb067003a4 100644
--- a/detections/cloud/aws_create_policy_version_to_allow_all_resources.yml
+++ b/detections/cloud/aws_create_policy_version_to_allow_all_resources.yml
@@ -22,7 +22,7 @@ known_false_positives: While this search has no known false positives, it is pos
   resources. That said, AWS strongly advises against granting full control to all
   AWS resources
 references:
-- https://labs.bishopfox.com/tech-blog/privilege-escalation-in-aws
+- https://bishopfox.com/blog/privilege-escalation-in-aws
 - https://rhinosecuritylabs.com/aws/aws-privilege-escalation-methods-mitigation-part-2/
 tags:
   analytic_story:
diff --git a/detections/cloud/aws_createaccesskey.yml b/detections/cloud/aws_createaccesskey.yml
index ff06ff2563..02642b1e49 100644
--- a/detections/cloud/aws_createaccesskey.yml
+++ b/detections/cloud/aws_createaccesskey.yml
@@ -24,7 +24,7 @@ how_to_implement: You must install splunk AWS add on and Splunk App for AWS. Thi
 known_false_positives: While this search has no known false positives, it is possible
   that an AWS admin has legitimately created keys for another user.
 references:
-- https://labs.bishopfox.com/tech-blog/privilege-escalation-in-aws
+- https://bishopfox.com/blog/privilege-escalation-in-aws
 - https://rhinosecuritylabs.com/aws/aws-privilege-escalation-methods-mitigation-part-2/
 tags:
   analytic_story:
diff --git a/detections/cloud/aws_createloginprofile.yml b/detections/cloud/aws_createloginprofile.yml
index 050e1d6a5c..bebf5ac260 100644
--- a/detections/cloud/aws_createloginprofile.yml
+++ b/detections/cloud/aws_createloginprofile.yml
@@ -22,7 +22,7 @@ how_to_implement: You must install splunk AWS add on and Splunk App for AWS. Thi
 known_false_positives: While this search has no known false positives, it is possible
   that an AWS admin has legitimately created a login profile for another user.
 references:
-- https://labs.bishopfox.com/tech-blog/privilege-escalation-in-aws
+- https://bishopfox.com/blog/privilege-escalation-in-aws
 - https://rhinosecuritylabs.com/aws/aws-privilege-escalation-methods-mitigation-part-2/
 tags:
   analytic_story:
diff --git a/detections/cloud/aws_iam_assume_role_policy_brute_force.yml b/detections/cloud/aws_iam_assume_role_policy_brute_force.yml
index 3848d3b9bb..2630f63548 100644
--- a/detections/cloud/aws_iam_assume_role_policy_brute_force.yml
+++ b/detections/cloud/aws_iam_assume_role_policy_brute_force.yml
@@ -24,7 +24,7 @@ known_false_positives: This detection will require tuning to provide high fideli
   detection capabilties. Tune based on src addresses (corporate offices, VPN terminations)
   or by groups of users.
 references:
-- https://www.praetorian.com/blog/aws-iam-assume-role-vulnerabilities
+- https://www.praetorian.com/blog/aws-iam-assume-role-vulnerabilities/
 - https://rhinosecuritylabs.com/aws/assume-worst-aws-assume-role-enumeration/
 - https://www.elastic.co/guide/en/security/current/aws-iam-brute-force-of-assume-role-policy.html
 tags:
diff --git a/detections/cloud/aws_saml_access_by_provider_user_and_principal.yml b/detections/cloud/aws_saml_access_by_provider_user_and_principal.yml
index 003d880703..022dafe665 100644
--- a/detections/cloud/aws_saml_access_by_provider_user_and_principal.yml
+++ b/detections/cloud/aws_saml_access_by_provider_user_and_principal.yml
@@ -23,7 +23,7 @@ known_false_positives: Attacks using a Golden SAML or SAML assertion hijacks or
   access and abuse detection searches can provide the necessary context to detect
   these attacks.
 references:
-- https://us-cert.cisa.gov/ncas/alerts/aa21-008a
+- https://www.cisa.gov/uscert/ncas/alerts/aa21-008a
 - https://www.splunk.com/en_us/blog/security/a-golden-saml-journey-solarwinds-continued.html
 - https://www.fireeye.com/content/dam/fireeye-www/blog/pdfs/wp-m-unc2452-2021-000343-01.pdf
 - https://www.cyberark.com/resources/threat-research-blog/golden-saml-newly-discovered-attack-technique-forges-authentication-to-cloud-apps
diff --git a/detections/cloud/aws_saml_update_identity_provider.yml b/detections/cloud/aws_saml_update_identity_provider.yml
index 34aef7434e..dac7c0d864 100644
--- a/detections/cloud/aws_saml_update_identity_provider.yml
+++ b/detections/cloud/aws_saml_update_identity_provider.yml
@@ -19,7 +19,7 @@ how_to_implement: You must install splunk AWS add on and Splunk App for AWS. Thi
 known_false_positives: Updating a SAML provider or creating a new one may not necessarily
   be malicious however it needs to be closely monitored.
 references:
-- https://us-cert.cisa.gov/ncas/alerts/aa21-008a
+- https://www.cisa.gov/uscert/ncas/alerts/aa21-008a
 - https://www.splunk.com/en_us/blog/security/a-golden-saml-journey-solarwinds-continued.html
 - https://www.fireeye.com/content/dam/fireeye-www/blog/pdfs/wp-m-unc2452-2021-000343-01.pdf
 - https://www.cyberark.com/resources/threat-research-blog/golden-saml-newly-discovered-attack-technique-forges-authentication-to-cloud-apps
diff --git a/detections/cloud/aws_setdefaultpolicyversion.yml b/detections/cloud/aws_setdefaultpolicyversion.yml
index 6e9d444e48..83f36e3050 100644
--- a/detections/cloud/aws_setdefaultpolicyversion.yml
+++ b/detections/cloud/aws_setdefaultpolicyversion.yml
@@ -21,7 +21,7 @@ known_false_positives: While this search has no known false positives, it is pos
   all resources. That said, AWS strongly advises against granting full control to
   all AWS resources
 references:
-- https://labs.bishopfox.com/tech-blog/privilege-escalation-in-aws
+- https://bishopfox.com/blog/privilege-escalation-in-aws
 - https://rhinosecuritylabs.com/aws/aws-privilege-escalation-methods-mitigation-part-2/
 tags:
   analytic_story:
diff --git a/detections/cloud/aws_updateloginprofile.yml b/detections/cloud/aws_updateloginprofile.yml
index 20a56ccfc2..aabeb5a952 100644
--- a/detections/cloud/aws_updateloginprofile.yml
+++ b/detections/cloud/aws_updateloginprofile.yml
@@ -24,7 +24,7 @@ how_to_implement: You must install splunk AWS add on and Splunk App for AWS. Thi
 known_false_positives: While this search has no known false positives, it is possible
   that an AWS admin has legitimately created keys for another user.
 references:
-- https://labs.bishopfox.com/tech-blog/privilege-escalation-in-aws
+- https://bishopfox.com/blog/privilege-escalation-in-aws
 - https://rhinosecuritylabs.com/aws/aws-privilege-escalation-methods-mitigation-part-2/
 tags:
   analytic_story:
diff --git a/detections/cloud/gsuite_email_suspicious_subject_with_attachment.yml b/detections/cloud/gsuite_email_suspicious_subject_with_attachment.yml
index 25722961dd..b544c0a87b 100644
--- a/detections/cloud/gsuite_email_suspicious_subject_with_attachment.yml
+++ b/detections/cloud/gsuite_email_suspicious_subject_with_attachment.yml
@@ -32,7 +32,7 @@ known_false_positives: normal user or normal transaction may contain the subject
   file type attachment that this detection try to search.
 references:
 - https://www.redhat.com/en/topics/devops/what-is-devsecops
-- https://www.fireeye.com/content/dam/fireeye-www/global/en/current-threats/pdfs/rpt-top-spear-phishing-words.pdf
+- https://www.mandiant.com/resources/top-words-used-in-spear-phishing-attacks
 tags:
   analytic_story:
   - Dev Sec Ops
diff --git a/detections/cloud/gsuite_suspicious_shared_file_name.yml b/detections/cloud/gsuite_suspicious_shared_file_name.yml
index 90cc35bc48..33fcc963af 100644
--- a/detections/cloud/gsuite_suspicious_shared_file_name.yml
+++ b/detections/cloud/gsuite_suspicious_shared_file_name.yml
@@ -31,7 +31,7 @@ known_false_positives: normal user or normal transaction may contain the subject
   file type attachment that this detection try to search
 references:
 - https://www.redhat.com/en/topics/devops/what-is-devsecops
-- https://www.fireeye.com/content/dam/fireeye-www/global/en/current-threats/pdfs/rpt-top-spear-phishing-words.pdf
+- https://www.mandiant.com/resources/top-words-used-in-spear-phishing-attacks
 tags:
   analytic_story:
   - Dev Sec Ops
diff --git a/detections/cloud/o365_add_app_role_assignment_grant_user.yml b/detections/cloud/o365_add_app_role_assignment_grant_user.yml
index 764103e793..4ee794bd2d 100644
--- a/detections/cloud/o365_add_app_role_assignment_grant_user.yml
+++ b/detections/cloud/o365_add_app_role_assignment_grant_user.yml
@@ -19,7 +19,7 @@ known_false_positives: The creation of a new Federation is not necessarily malic
   abuse or backdoor via federated identities at a different cloud provider.
 references:
 - https://www.fireeye.com/content/dam/fireeye-www/blog/pdfs/wp-m-unc2452-2021-000343-01.pdf
-- https://us-cert.cisa.gov/ncas/alerts/aa21-008a
+- https://www.cisa.gov/uscert/ncas/alerts/aa21-008a
 tags:
   analytic_story:
   - Office 365 Detections
diff --git a/detections/cloud/o365_added_service_principal.yml b/detections/cloud/o365_added_service_principal.yml
index 4eb8e927bf..2b47480363 100644
--- a/detections/cloud/o365_added_service_principal.yml
+++ b/detections/cloud/o365_added_service_principal.yml
@@ -20,9 +20,9 @@ known_false_positives: The creation of a new Federation is not necessarily malic
   abuse or backdoor via federated identities at a different cloud provider.
 references:
 - https://www.fireeye.com/content/dam/fireeye-www/blog/pdfs/wp-m-unc2452-2021-000343-01.pdf
-- https://us-cert.cisa.gov/ncas/alerts/aa21-008a
+- https://www.cisa.gov/uscert/ncas/alerts/aa21-008a
 - https://www.splunk.com/en_us/blog/security/a-golden-saml-journey-solarwinds-continued.html
-- https://www.sygnia.co/golden-saml-advisory
+- https://blog.sygnia.co/detection-and-hunting-of-golden-saml-attack?hsLang=en
 tags:
   analytic_story:
   - Office 365 Detections
diff --git a/detections/cloud/o365_new_federated_domain_added.yml b/detections/cloud/o365_new_federated_domain_added.yml
index beeb34270c..f9662fce58 100644
--- a/detections/cloud/o365_new_federated_domain_added.yml
+++ b/detections/cloud/o365_new_federated_domain_added.yml
@@ -18,9 +18,9 @@ known_false_positives: The creation of a new Federated domain is not necessarily
   abuse or backdoor via federated identities at a similar or different cloud provider.
 references:
 - https://www.fireeye.com/content/dam/fireeye-www/blog/pdfs/wp-m-unc2452-2021-000343-01.pdf
-- https://us-cert.cisa.gov/ncas/alerts/aa21-008a
+- https://www.cisa.gov/uscert/ncas/alerts/aa21-008a
 - https://www.splunk.com/en_us/blog/security/a-golden-saml-journey-solarwinds-continued.html
-- https://www.sygnia.co/golden-saml-advisory
+- https://blog.sygnia.co/detection-and-hunting-of-golden-saml-attack?hsLang=en
 - https://o365blog.com/post/aadbackdoor/
 tags:
   analytic_story:
diff --git a/detections/deprecated/suspicious_rundll32_rename.yml b/detections/deprecated/suspicious_rundll32_rename.yml
index f772449975..ec83425324 100644
--- a/detections/deprecated/suspicious_rundll32_rename.yml
+++ b/detections/deprecated/suspicious_rundll32_rename.yml
@@ -28,7 +28,7 @@ known_false_positives: Although unlikely, some legitimate applications may use a
 references:
 - https://attack.mitre.org/techniques/T1218/011/
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1218.011/T1218.011.md
-- https://lolbas-project.github.io/lolbas/Binaries/Rundll32
+- https://lolbas-project.github.io/lolbas/Binaries/Rundll32/
 tags:
   analytic_story:
   - Suspicious Rundll32 Activity
diff --git a/detections/endpoint/active_setup_registry_autostart.yml b/detections/endpoint/active_setup_registry_autostart.yml
index aba210b24e..824682c8f9 100644
--- a/detections/endpoint/active_setup_registry_autostart.yml
+++ b/detections/endpoint/active_setup_registry_autostart.yml
@@ -34,7 +34,7 @@ how_to_implement: To successfully implement this search, you must be ingesting d
   to the registry.
 known_false_positives: Active setup installer may add or modify this registry.
 references:
-- https://www.microsoft.com/en-us/wdsi/threats/malware-encyclopedia-description?Name=Backdoor%3aWin32%2fPoisonivy.E
+- https://www.microsoft.com/en-us/wdsi/threats/malware-encyclopedia-description?Name=Backdoor%3AWin32%2FPoisonivy.E
 - https://attack.mitre.org/techniques/T1547/014/
 tags:
   analytic_story:
diff --git a/detections/endpoint/allow_file_and_printing_sharing_in_firewall.yml b/detections/endpoint/allow_file_and_printing_sharing_in_firewall.yml
index 1ccf2ebe82..85441224e3 100644
--- a/detections/endpoint/allow_file_and_printing_sharing_in_firewall.yml
+++ b/detections/endpoint/allow_file_and_printing_sharing_in_firewall.yml
@@ -24,7 +24,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: network admin may modify this firewall feature that may cause
   this rule to be triggered.
 references:
-- https://kb.fortinet.com/kb/documentLink.do?externalID=FD52469
+- https://community.fortinet.com:443/t5/FortiEDR/How-FortiEDR-detects-and-blocks-Revil-Ransomware-aka-sodinokibi/ta-p/189638?externalID=FD52469
 - https://app.any.run/tasks/c0f98850-af65-4352-9746-fbebadee4f05/
 tags:
   analytic_story:
diff --git a/detections/endpoint/allow_network_discovery_in_firewall.yml b/detections/endpoint/allow_network_discovery_in_firewall.yml
index fd7b605d0d..22d738824e 100644
--- a/detections/endpoint/allow_network_discovery_in_firewall.yml
+++ b/detections/endpoint/allow_network_discovery_in_firewall.yml
@@ -25,7 +25,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: network admin may modify this firewall feature that may cause
   this rule to be triggered.
 references:
-- https://kb.fortinet.com/kb/documentLink.do?externalID=FD52469
+- https://community.fortinet.com:443/t5/FortiEDR/How-FortiEDR-detects-and-blocks-Revil-Ransomware-aka-sodinokibi/ta-p/189638?externalID=FD52469
 - https://app.any.run/tasks/c0f98850-af65-4352-9746-fbebadee4f05/
 tags:
   analytic_story:
diff --git a/detections/endpoint/clop_common_exec_parameter.yml b/detections/endpoint/clop_common_exec_parameter.yml
index f3deda0e64..ba8b92a48e 100644
--- a/detections/endpoint/clop_common_exec_parameter.yml
+++ b/detections/endpoint/clop_common_exec_parameter.yml
@@ -25,7 +25,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
   your endpoints into the `Endpoint` datamodel in the `Processes` node.
 known_false_positives: Operators can execute third party tools using these parameters.
 references:
-- https://www.fireeye.com/blog/threat-research/2020/10/fin11-email-campaigns-precursor-for-ransomware-data-theft.html
+- https://www.mandiant.com/resources/fin11-email-campaigns-precursor-for-ransomware-data-theft
 - https://blog.virustotal.com/2020/11/keep-your-friends-close-keep-ransomware.html
 tags:
   analytic_story:
diff --git a/detections/endpoint/clop_ransomware_known_service_name.yml b/detections/endpoint/clop_ransomware_known_service_name.yml
index 9df5fb9fd3..6fd13099f6 100644
--- a/detections/endpoint/clop_ransomware_known_service_name.yml
+++ b/detections/endpoint/clop_ransomware_known_service_name.yml
@@ -19,7 +19,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   from your endpoints.
 known_false_positives: unknown
 references:
-- https://www.fireeye.com/blog/threat-research/2020/10/fin11-email-campaigns-precursor-for-ransomware-data-theft.html
+- https://www.mandiant.com/resources/fin11-email-campaigns-precursor-for-ransomware-data-theft
 - https://blog.virustotal.com/2020/11/keep-your-friends-close-keep-ransomware.html
 tags:
   analytic_story:
diff --git a/detections/endpoint/cmdline_tool_not_executed_in_cmd_shell.yml b/detections/endpoint/cmdline_tool_not_executed_in_cmd_shell.yml
index f69af6ee9f..c8f69706d3 100644
--- a/detections/endpoint/cmdline_tool_not_executed_in_cmd_shell.yml
+++ b/detections/endpoint/cmdline_tool_not_executed_in_cmd_shell.yml
@@ -31,7 +31,7 @@ known_false_positives: A network operator or systems administrator may utilize a
   automated host discovery application that may generate false positives. Filter as
   needed.
 references:
-- https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html
+- https://www.mandiant.com/resources/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation
 - https://attack.mitre.org/groups/G0046/
 tags:
   analytic_story:
diff --git a/detections/endpoint/cobalt_strike_named_pipes.yml b/detections/endpoint/cobalt_strike_named_pipes.yml
index f1dfc7f922..a0db163a0d 100644
--- a/detections/endpoint/cobalt_strike_named_pipes.yml
+++ b/detections/endpoint/cobalt_strike_named_pipes.yml
@@ -33,10 +33,10 @@ known_false_positives: The idea of using named pipes with Cobalt Strike is to bl
 references:
 - https://attack.mitre.org/techniques/T1218/009/
 - https://docs.microsoft.com/en-us/windows/win32/ipc/named-pipes
-- https://www.cobaltstrike.com/help-smb-beacon
-- https://blog.cobaltstrike.com/2021/02/09/learn-pipe-fitting-for-all-of-your-offense-projects/
+- https://hstechdocs.helpsystems.com/manuals/cobaltstrike/current/userguide/index.htm#cshid=1040
+- https://www.cobaltstrike.com/blog/learn-pipe-fitting-for-all-of-your-offense-projects/
 - https://gist.github.com/MHaggis/6c600e524045a6d49c35291a21e10752
-- https://www.fireeye.com/blog/threat-research/2021/05/shining-a-light-on-darkside-ransomware-operations.html
+- https://www.mandiant.com/resources/shining-a-light-on-darkside-ransomware-operations
 tags:
   analytic_story:
   - Cobalt Strike
diff --git a/detections/endpoint/create_or_delete_windows_shares_using_net_exe.yml b/detections/endpoint/create_or_delete_windows_shares_using_net_exe.yml
index 049158682e..3826c9058d 100644
--- a/detections/endpoint/create_or_delete_windows_shares_using_net_exe.yml
+++ b/detections/endpoint/create_or_delete_windows_shares_using_net_exe.yml
@@ -22,7 +22,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: Administrators often leverage net.exe to create or delete network
   shares. You should verify that the activity was intentional and is legitimate.
 references:
-- https://attack.mitre.org/techniques/T1070/005
+- https://attack.mitre.org/techniques/T1070/005/
 tags:
   analytic_story:
   - Hidden Cobra Malware
diff --git a/detections/endpoint/delete_shadowcopy_with_powershell.yml b/detections/endpoint/delete_shadowcopy_with_powershell.yml
index 7688461d92..2d6a8474dd 100644
--- a/detections/endpoint/delete_shadowcopy_with_powershell.yml
+++ b/detections/endpoint/delete_shadowcopy_with_powershell.yml
@@ -20,8 +20,8 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   registry to monitor this event.
 known_false_positives: unknown
 references:
-- https://www.fireeye.com/blog/threat-research/2021/05/shining-a-light-on-darkside-ransomware-operations.html
-- https://searchwindowsserver.techtarget.com/tutorial/Set-up-PowerShell-script-block-logging-for-added-security
+- https://www.mandiant.com/resources/shining-a-light-on-darkside-ransomware-operations
+- https://www.techtarget.com/searchwindowsserver/tutorial/Set-up-PowerShell-script-block-logging-for-added-security
 tags:
   analytic_story:
   - DarkSide Ransomware
diff --git a/detections/endpoint/detect_rclone_command_line_usage.yml b/detections/endpoint/detect_rclone_command_line_usage.yml
index 5d1051b1d3..37048de5ad 100644
--- a/detections/endpoint/detect_rclone_command_line_usage.yml
+++ b/detections/endpoint/detect_rclone_command_line_usage.yml
@@ -29,7 +29,7 @@ known_false_positives: False positives should be limited as this is restricted t
   the Rclone process name. Filter or tune the analytic as needed.
 references:
 - https://redcanary.com/blog/rclone-mega-extortion/
-- https://www.fireeye.com/blog/threat-research/2021/05/shining-a-light-on-darkside-ransomware-operations.html
+- https://www.mandiant.com/resources/shining-a-light-on-darkside-ransomware-operations
 - https://thedfirreport.com/2021/03/29/sodinokibi-aka-revil-ransomware/
 - https://thedfirreport.com/2021/11/29/continuing-the-bazar-ransomware-story/
 tags:
diff --git a/detections/endpoint/detect_renamed_rclone.yml b/detections/endpoint/detect_renamed_rclone.yml
index 31f9611618..9c39130b20 100644
--- a/detections/endpoint/detect_renamed_rclone.yml
+++ b/detections/endpoint/detect_renamed_rclone.yml
@@ -27,7 +27,7 @@ known_false_positives: False positives should be limited as this analytic identi
   use case.
 references:
 - https://redcanary.com/blog/rclone-mega-extortion/
-- https://www.fireeye.com/blog/threat-research/2021/05/shining-a-light-on-darkside-ransomware-operations.html
+- https://www.mandiant.com/resources/shining-a-light-on-darkside-ransomware-operations
 - https://thedfirreport.com/2021/03/29/sodinokibi-aka-revil-ransomware/
 tags:
   analytic_story:
diff --git a/detections/endpoint/detect_rundll32_application_control_bypass___advpack.yml b/detections/endpoint/detect_rundll32_application_control_bypass___advpack.yml
index a3e262f36e..916d615608 100644
--- a/detections/endpoint/detect_rundll32_application_control_bypass___advpack.yml
+++ b/detections/endpoint/detect_rundll32_application_control_bypass___advpack.yml
@@ -31,7 +31,7 @@ known_false_positives: Although unlikely, some legitimate applications may use a
 references:
 - https://attack.mitre.org/techniques/T1218/011/
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1218.011/T1218.011.md
-- https://lolbas-project.github.io/lolbas/Binaries/Rundll32
+- https://lolbas-project.github.io/lolbas/Binaries/Rundll32/
 - https://lolbas-project.github.io/lolbas/Libraries/Advpack/
 - https://bohops.com/2018/02/26/leveraging-inf-sct-fetch-execute-techniques-for-bypass-evasion-persistence/
 tags:
diff --git a/detections/endpoint/detect_rundll32_application_control_bypass___setupapi.yml b/detections/endpoint/detect_rundll32_application_control_bypass___setupapi.yml
index 1a4e2bb1cd..5baa5324b6 100644
--- a/detections/endpoint/detect_rundll32_application_control_bypass___setupapi.yml
+++ b/detections/endpoint/detect_rundll32_application_control_bypass___setupapi.yml
@@ -29,7 +29,7 @@ known_false_positives: Although unlikely, some legitimate applications may use s
 references:
 - https://attack.mitre.org/techniques/T1218/011/
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1218.011/T1218.011.md
-- https://lolbas-project.github.io/lolbas/Binaries/Rundll32
+- https://lolbas-project.github.io/lolbas/Binaries/Rundll32/
 - https://lolbas-project.github.io/lolbas/Libraries/Setupapi/
 - https://bohops.com/2018/02/26/leveraging-inf-sct-fetch-execute-techniques-for-bypass-evasion-persistence/
 tags:
diff --git a/detections/endpoint/detect_rundll32_application_control_bypass___syssetup.yml b/detections/endpoint/detect_rundll32_application_control_bypass___syssetup.yml
index 78f7368a0e..4e207fb56c 100644
--- a/detections/endpoint/detect_rundll32_application_control_bypass___syssetup.yml
+++ b/detections/endpoint/detect_rundll32_application_control_bypass___syssetup.yml
@@ -31,7 +31,7 @@ known_false_positives: Although unlikely, some legitimate applications may use s
 references:
 - https://attack.mitre.org/techniques/T1218/011/
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1218.011/T1218.011.md
-- https://lolbas-project.github.io/lolbas/Binaries/Rundll32
+- https://lolbas-project.github.io/lolbas/Binaries/Rundll32/
 - https://lolbas-project.github.io/lolbas/Libraries/Syssetup/
 - https://bohops.com/2018/02/26/leveraging-inf-sct-fetch-execute-techniques-for-bypass-evasion-persistence/
 tags:
diff --git a/detections/endpoint/dllhost_with_no_command_line_arguments_with_network.yml b/detections/endpoint/dllhost_with_no_command_line_arguments_with_network.yml
index 614061636c..f6e063ff1c 100644
--- a/detections/endpoint/dllhost_with_no_command_line_arguments_with_network.yml
+++ b/detections/endpoint/dllhost_with_no_command_line_arguments_with_network.yml
@@ -28,7 +28,7 @@ known_false_positives: Although unlikely, some legitimate third party applicatio
   may use a moved copy of dllhost, triggering a false positive.
 references:
 - https://raw.githubusercontent.com/threatexpress/malleable-c2/c3385e481159a759f79b8acfe11acf240893b830/jquery-c2.4.2.profile
-- https://blog.cobaltstrike.com/2021/02/09/learn-pipe-fitting-for-all-of-your-offense-projects/
+- https://www.cobaltstrike.com/blog/learn-pipe-fitting-for-all-of-your-offense-projects/
 tags:
   analytic_story:
   - Cobalt Strike
diff --git a/detections/endpoint/drop_icedid_license_dat.yml b/detections/endpoint/drop_icedid_license_dat.yml
index f37b89a86d..5f354529b2 100644
--- a/detections/endpoint/drop_icedid_license_dat.yml
+++ b/detections/endpoint/drop_icedid_license_dat.yml
@@ -19,7 +19,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   Sysmon TA.
 known_false_positives: unknown
 references:
-- https://www.cisecurity.org/white-papers/security-primer-icedid/
+- https://www.cisecurity.org/insights/white-papers/security-primer-icedid
 tags:
   analytic_story:
   - IcedID
diff --git a/detections/endpoint/eventvwr_uac_bypass.yml b/detections/endpoint/eventvwr_uac_bypass.yml
index e4c6563593..691b33b0c9 100644
--- a/detections/endpoint/eventvwr_uac_bypass.yml
+++ b/detections/endpoint/eventvwr_uac_bypass.yml
@@ -34,7 +34,7 @@ known_false_positives: Some false positives may be present and will need to be f
 references:
 - https://blog.malwarebytes.com/malwarebytes-news/2021/02/lazyscripter-from-empire-to-double-rat/
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1548.002/T1548.002.md
-- https://attack.mitre.org/techniques/T1548/002
+- https://attack.mitre.org/techniques/T1548/002/
 - https://enigma0x3.net/2016/08/15/fileless-uac-bypass-using-eventvwr-exe-and-registry-hijacking/
 tags:
   analytic_story:
diff --git a/detections/endpoint/extraction_of_registry_hives.yml b/detections/endpoint/extraction_of_registry_hives.yml
index 0da0a0fd51..9c32749838 100644
--- a/detections/endpoint/extraction_of_registry_hives.yml
+++ b/detections/endpoint/extraction_of_registry_hives.yml
@@ -23,7 +23,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: It is possible some agent based products will generate false
   positives. Filter as needed.
 references:
-- https://www.fireeye.com/blog/threat-research/2021/05/shining-a-light-on-darkside-ransomware-operations.html
+- https://www.mandiant.com/resources/shining-a-light-on-darkside-ransomware-operations
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1003.002/T1003.002.md
 tags:
   analytic_story:
diff --git a/detections/endpoint/firewall_allowed_program_enable.yml b/detections/endpoint/firewall_allowed_program_enable.yml
index 1e90cd0a8d..0d664b3f46 100644
--- a/detections/endpoint/firewall_allowed_program_enable.yml
+++ b/detections/endpoint/firewall_allowed_program_enable.yml
@@ -27,7 +27,7 @@ known_false_positives: A network operator or systems administrator may utilize a
   automated or manual execution of this firewall rule that may generate false positives.
   Filter as needed.
 references:
-- https://app.any.run/tasks/ad4c3cda-41f2-4401-8dba-56cc2d245488/#
+- https://app.any.run/tasks/ad4c3cda-41f2-4401-8dba-56cc2d245488/
 tags:
   analytic_story:
   - Windows Defense Evasion Tactics
diff --git a/detections/endpoint/fodhelper_uac_bypass.yml b/detections/endpoint/fodhelper_uac_bypass.yml
index a90168a952..10fad03051 100644
--- a/detections/endpoint/fodhelper_uac_bypass.yml
+++ b/detections/endpoint/fodhelper_uac_bypass.yml
@@ -33,7 +33,7 @@ references:
 - https://blog.malwarebytes.com/malwarebytes-news/2021/02/lazyscripter-from-empire-to-double-rat/
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1548.002/T1548.002.md
 - https://github.com/gushmazuko/WinBypass/blob/master/FodhelperBypass.ps1
-- https://attack.mitre.org/techniques/T1548/002
+- https://attack.mitre.org/techniques/T1548/002/
 tags:
   analytic_story:
   - Windows Defense Evasion Tactics
diff --git a/detections/endpoint/getcurrent_user_with_powershell_script_block.yml b/detections/endpoint/getcurrent_user_with_powershell_script_block.yml
index 626ea184b4..7fa024201b 100644
--- a/detections/endpoint/getcurrent_user_with_powershell_script_block.yml
+++ b/detections/endpoint/getcurrent_user_with_powershell_script_block.yml
@@ -21,7 +21,7 @@ known_false_positives: Administrators or power users may use this PowerShell com
   for troubleshooting.
 references:
 - https://attack.mitre.org/techniques/T1033/
-- https://docs.microsoft.com/en-us/dotnet/api/system.security.principal.windowsidentity.getcurrent?view=net-5.0
+- https://docs.microsoft.com/en-us/dotnet/api/system.security.principal.windowsidentity.getcurrent?view=net-6.0&viewFallbackFrom=net-5.0
 tags:
   analytic_story:
   - Active Directory Discovery
diff --git a/detections/endpoint/gpupdate_with_no_command_line_arguments_with_network.yml b/detections/endpoint/gpupdate_with_no_command_line_arguments_with_network.yml
index b19ba37540..6e144d84d4 100644
--- a/detections/endpoint/gpupdate_with_no_command_line_arguments_with_network.yml
+++ b/detections/endpoint/gpupdate_with_no_command_line_arguments_with_network.yml
@@ -29,7 +29,7 @@ known_false_positives: Limited false positives may be present in small environme
   Tuning may be required based on parent process.
 references:
 - https://raw.githubusercontent.com/xx0hcd/Malleable-C2-Profiles/0ef8cf4556e26f6d4190c56ba697c2159faa5822/crimeware/trick_ryuk.profile
-- https://blog.cobaltstrike.com/2021/02/09/learn-pipe-fitting-for-all-of-your-offense-projects/
+- https://www.cobaltstrike.com/blog/learn-pipe-fitting-for-all-of-your-offense-projects/
 tags:
   analytic_story:
   - Cobalt Strike
diff --git a/detections/endpoint/high_process_termination_frequency.yml b/detections/endpoint/high_process_termination_frequency.yml
index 2ab9945819..b204013bde 100644
--- a/detections/endpoint/high_process_termination_frequency.yml
+++ b/detections/endpoint/high_process_termination_frequency.yml
@@ -19,7 +19,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   If you are using Sysmon, you must have at least version 6.0.4 of the Sysmon TA.
 known_false_positives: admin or user tool that can terminate multiple process.
 references:
-- https://www.fireeye.com/blog/threat-research/2020/10/fin11-email-campaigns-precursor-for-ransomware-data-theft.html
+- https://www.mandiant.com/resources/fin11-email-campaigns-precursor-for-ransomware-data-theft
 - https://blog.virustotal.com/2020/11/keep-your-friends-close-keep-ransomware.html
 tags:
   analytic_story:
diff --git a/detections/endpoint/icedid_exfiltrated_archived_file_creation.yml b/detections/endpoint/icedid_exfiltrated_archived_file_creation.yml
index fe1ef7e880..a183c225cd 100644
--- a/detections/endpoint/icedid_exfiltrated_archived_file_creation.yml
+++ b/detections/endpoint/icedid_exfiltrated_archived_file_creation.yml
@@ -19,7 +19,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   Sysmon TA.
 known_false_positives: unknown
 references:
-- https://www.cisecurity.org/white-papers/security-primer-icedid/
+- https://www.cisecurity.org/insights/white-papers/security-primer-icedid
 tags:
   analytic_story:
   - IcedID
diff --git a/detections/endpoint/impacket_lateral_movement_commandline_parameters.yml b/detections/endpoint/impacket_lateral_movement_commandline_parameters.yml
index 051e857767..2862b783a3 100644
--- a/detections/endpoint/impacket_lateral_movement_commandline_parameters.yml
+++ b/detections/endpoint/impacket_lateral_movement_commandline_parameters.yml
@@ -31,7 +31,7 @@ references:
 - https://attack.mitre.org/techniques/T1021/003/
 - https://attack.mitre.org/techniques/T1047/
 - https://attack.mitre.org/techniques/T1053/
-- https://attack.mitre.org/techniques/T1053/005
+- /
 - https://github.com/SecureAuthCorp/impacket
 - https://vk9-sec.com/impacket-remote-code-execution-rce-on-windows-from-linux/
 - https://www.microsoft.com/security/blog/2022/01/15/destructive-malware-targeting-ukrainian-organizations/
diff --git a/detections/endpoint/jscript_execution_using_cscript_app.yml b/detections/endpoint/jscript_execution_using_cscript_app.yml
index 964c8a046b..7a0a6585a2 100644
--- a/detections/endpoint/jscript_execution_using_cscript_app.yml
+++ b/detections/endpoint/jscript_execution_using_cscript_app.yml
@@ -24,7 +24,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   Sysmon TA.
 known_false_positives: unknown
 references:
-- https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html
+- https://www.mandiant.com/resources/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation
 - https://attack.mitre.org/groups/G0046/
 tags:
   analytic_story:
diff --git a/detections/endpoint/kerberos_user_enumeration.yml b/detections/endpoint/kerberos_user_enumeration.yml
index 7b0b57fe32..fcf31e87b0 100644
--- a/detections/endpoint/kerberos_user_enumeration.yml
+++ b/detections/endpoint/kerberos_user_enumeration.yml
@@ -28,7 +28,7 @@ known_false_positives: Possible false positive scenarios include but are not lim
 references:
 - https://github.com/ropnop/kerbrute
 - https://attack.mitre.org/techniques/T1589/002/ 
-- https://www.redsiege.com/blog/2020/04/user-enumeration-part-3-windows/
+- https://redsiege.com/tools-techniques/2020/04/user-enumeration-part-3-windows/
 tags:
   analytic_story:
   - Active Directory Kerberos Attacks
diff --git a/detections/endpoint/logon_script_event_trigger_execution.yml b/detections/endpoint/logon_script_event_trigger_execution.yml
index 46e8bea28d..c2789af5d0 100644
--- a/detections/endpoint/logon_script_event_trigger_execution.yml
+++ b/detections/endpoint/logon_script_event_trigger_execution.yml
@@ -23,7 +23,7 @@ how_to_implement: To successfully implement this search, you must be ingesting d
   to the registry.
 known_false_positives: unknown
 references:
-- https://attack.mitre.org/techniques/T1037/001
+- https://attack.mitre.org/techniques/T1037/001/
 tags:
   analytic_story:
   - Windows Persistence Techniques
diff --git a/detections/endpoint/ms_scripting_process_loading_ldap_module.yml b/detections/endpoint/ms_scripting_process_loading_ldap_module.yml
index aef0f2abc1..f2dcbe9f8e 100644
--- a/detections/endpoint/ms_scripting_process_loading_ldap_module.yml
+++ b/detections/endpoint/ms_scripting_process_loading_ldap_module.yml
@@ -25,7 +25,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
 known_false_positives: automation scripting language may used by network operator
   to do ldap query.
 references:
-- https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html
+- https://www.mandiant.com/resources/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation
 - https://attack.mitre.org/groups/G0046/
 tags:
   analytic_story:
diff --git a/detections/endpoint/ms_scripting_process_loading_wmi_module.yml b/detections/endpoint/ms_scripting_process_loading_wmi_module.yml
index 9d2c2c28f0..9454179e9b 100644
--- a/detections/endpoint/ms_scripting_process_loading_wmi_module.yml
+++ b/detections/endpoint/ms_scripting_process_loading_wmi_module.yml
@@ -26,7 +26,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
 known_false_positives: automation scripting language may used by network operator
   to do ldap query.
 references:
-- https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html
+- https://www.mandiant.com/resources/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation
 - https://attack.mitre.org/groups/G0046/
 tags:
   analytic_story:
diff --git a/detections/endpoint/msbuild_suspicious_spawned_by_script_process.yml b/detections/endpoint/msbuild_suspicious_spawned_by_script_process.yml
index 3b9815d26f..5472eb5965 100644
--- a/detections/endpoint/msbuild_suspicious_spawned_by_script_process.yml
+++ b/detections/endpoint/msbuild_suspicious_spawned_by_script_process.yml
@@ -27,7 +27,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: False positives should be limited as developers do not spawn
   MSBuild via a WSH.
 references:
-- https://app.any.run/tasks/dc93ee63-050c-4ff8-b07e-8277af9ab939/#
+- https://app.any.run/tasks/dc93ee63-050c-4ff8-b07e-8277af9ab939/
 tags:
   analytic_story:
   - Trusted Developer Utilities Proxy Execution MSBuild
diff --git a/detections/endpoint/network_discovery_using_route_windows_app.yml b/detections/endpoint/network_discovery_using_route_windows_app.yml
index 6815de87b5..ebf5e23634 100644
--- a/detections/endpoint/network_discovery_using_route_windows_app.yml
+++ b/detections/endpoint/network_discovery_using_route_windows_app.yml
@@ -24,7 +24,7 @@ known_false_positives: A network operator or systems administrator may utilize a
   automated host discovery application that may generate false positives or an amazon
   ec2 script that uses this application. Filter as needed.
 references:
-- https://app.any.run/tasks/ad4c3cda-41f2-4401-8dba-56cc2d245488/#
+- https://app.any.run/tasks/ad4c3cda-41f2-4401-8dba-56cc2d245488/
 tags:
   analytic_story:
   - Active Directory Discovery
diff --git a/detections/endpoint/nishang_powershelltcponeline.yml b/detections/endpoint/nishang_powershelltcponeline.yml
index 81d98127c6..ec9059bce4 100644
--- a/detections/endpoint/nishang_powershelltcponeline.yml
+++ b/detections/endpoint/nishang_powershelltcponeline.yml
@@ -29,7 +29,7 @@ references:
 - https://github.com/samratashok/nishang/blob/master/Shells/Invoke-PowerShellTcpOneLine.ps1
 - https://www.volexity.com/blog/2021/03/02/active-exploitation-of-microsoft-exchange-zero-day-vulnerabilities/
 - https://www.microsoft.com/security/blog/2021/03/02/hafnium-targeting-exchange-servers/
-- https://blog.rapid7.com/2021/03/03/rapid7s-insightidr-enables-detection-and-response-to-microsoft-exchange-0-day/
+- https://www.rapid7.com/blog/post/2021/03/03/rapid7s-insightidr-enables-detection-and-response-to-microsoft-exchange-0-day/
 tags:
   analytic_story:
   - HAFNIUM Group
diff --git a/detections/endpoint/office_application_drop_executable.yml b/detections/endpoint/office_application_drop_executable.yml
index ea59025eb4..8fa06d009e 100644
--- a/detections/endpoint/office_application_drop_executable.yml
+++ b/detections/endpoint/office_application_drop_executable.yml
@@ -31,7 +31,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   Sysmon TA. Tune and filter known instances where renamed rundll32.exe may be used.
 known_false_positives: office macro for automation may do this behavior
 references:
-- https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html
+- https://www.mandiant.com/resources/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation
 - https://attack.mitre.org/groups/G0046/
 tags:
   analytic_story:
diff --git a/detections/endpoint/office_document_spawned_child_process_to_download.yml b/detections/endpoint/office_document_spawned_child_process_to_download.yml
index 3dde6f4156..d26b64a7d2 100644
--- a/detections/endpoint/office_document_spawned_child_process_to_download.yml
+++ b/detections/endpoint/office_document_spawned_child_process_to_download.yml
@@ -25,7 +25,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   used.
 known_false_positives: Default browser not in the filter list.
 references:
-- https://app.any.run/tasks/92d7ef61-bfd7-4c92-bc15-322172b4ebec/#
+- https://app.any.run/tasks/92d7ef61-bfd7-4c92-bc15-322172b4ebec/
 tags:
   analytic_story:
   - Spearphishing Attachments
diff --git a/detections/endpoint/office_spawning_control.yml b/detections/endpoint/office_spawning_control.yml
index d566a598d1..8d514f5f02 100644
--- a/detections/endpoint/office_spawning_control.yml
+++ b/detections/endpoint/office_spawning_control.yml
@@ -30,7 +30,7 @@ references:
 - https://strontic.github.io/xcyclopedia/library/control.exe-1F13E714A0FEA8887707DFF49287996F.html
 - https://app.any.run/tasks/36c14029-9df8-439c-bba0-45f2643b0c70/
 - https://attack.mitre.org/techniques/T1218/011/
-- https://www.echotrail.io/insights/search/control.exe
+- https://www.echotrail.io/insights/search/control.exe/
 - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-40444
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1218.002/T1218.002.yaml
 tags:
diff --git a/detections/endpoint/possible_lateral_movement_powershell_spawn.yml b/detections/endpoint/possible_lateral_movement_powershell_spawn.yml
index 0f02d539bc..8eae0df7bf 100644
--- a/detections/endpoint/possible_lateral_movement_powershell_spawn.yml
+++ b/detections/endpoint/possible_lateral_movement_powershell_spawn.yml
@@ -30,7 +30,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
 known_false_positives: Legitimate applications may spawn PowerShell as a child process
   of the the identified processes. Filter as needed.
 references:
-- https://attack.mitre.org/techniques/T1021/003
+- https://attack.mitre.org/techniques/T1021/003/
 - https://attack.mitre.org/techniques/T1021/006/
 - https://attack.mitre.org/techniques/T1047/
 - https://attack.mitre.org/techniques/T1053.005/
diff --git a/detections/endpoint/powershell_4104_hunting.yml b/detections/endpoint/powershell_4104_hunting.yml
index 65732d0319..07aab110ed 100644
--- a/detections/endpoint/powershell_4104_hunting.yml
+++ b/detections/endpoint/powershell_4104_hunting.yml
@@ -45,7 +45,7 @@ references:
 - https://github.com/marcurdy/dfir-toolset/blob/master/Powershell%20Blueteam.txt
 - https://devblogs.microsoft.com/powershell/powershell-the-blue-team/
 - https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_logging?view=powershell-5.1
-- https://www.fireeye.com/blog/threat-research/2016/02/greater_visibilityt.html
+- https://www.mandiant.com/resources/greater-visibilityt
 - https://hurricanelabs.com/splunk-tutorials/how-to-use-powershell-transcription-logs-in-splunk/
 tags:
   analytic_story:
diff --git a/detections/endpoint/print_spooler_adding_a_printer_driver.yml b/detections/endpoint/print_spooler_adding_a_printer_driver.yml
index fba2cdcbc3..824349bf6f 100644
--- a/detections/endpoint/print_spooler_adding_a_printer_driver.yml
+++ b/detections/endpoint/print_spooler_adding_a_printer_driver.yml
@@ -28,8 +28,8 @@ how_to_implement: You will need to ensure PrintService Admin and Operational log
 known_false_positives: Unknown. This may require filtering.
 references:
 - https://twitter.com/MalwareJake/status/1410421445608476679?s=20
-- https://blog.truesec.com/2021/06/30/fix-for-printnightmare-cve-2021-1675-exploit-to-keep-your-print-servers-running-while-a-patch-is-not-available/
-- https://blog.truesec.com/2021/06/30/exploitable-critical-rce-vulnerability-allows-regular-users-to-fully-compromise-active-directory-printnightmare-cve-2021-1675/
+- https://www.truesec.com/hub/blog/fix-for-printnightmare-cve-2021-1675-exploit-to-keep-your-print-servers-running-while-a-patch-is-not-available
+- https://www.truesec.com/hub/blog/exploitable-critical-rce-vulnerability-allows-regular-users-to-fully-compromise-active-directory-printnightmare-cve-2021-1675
 - https://www.reddit.com/r/msp/comments/ob6y02/critical_vulnerability_printnightmare_exposes
 tags:
   analytic_story:
diff --git a/detections/endpoint/print_spooler_failed_to_load_a_plug_in.yml b/detections/endpoint/print_spooler_failed_to_load_a_plug_in.yml
index 6d4944b43d..03692c30ac 100644
--- a/detections/endpoint/print_spooler_failed_to_load_a_plug_in.yml
+++ b/detections/endpoint/print_spooler_failed_to_load_a_plug_in.yml
@@ -27,8 +27,8 @@ how_to_implement: You will need to ensure PrintService Admin and Operational log
   are being logged to Splunk from critical or all systems.
 known_false_positives: False positives are unknown and filtering may be required.
 references:
-- https://blog.truesec.com/2021/06/30/fix-for-printnightmare-cve-2021-1675-exploit-to-keep-your-print-servers-running-while-a-patch-is-not-available/
-- https://blog.truesec.com/2021/06/30/exploitable-critical-rce-vulnerability-allows-regular-users-to-fully-compromise-active-directory-printnightmare-cve-2021-1675/
+- https://www.truesec.com/hub/blog/fix-for-printnightmare-cve-2021-1675-exploit-to-keep-your-print-servers-running-while-a-patch-is-not-available
+- https://www.truesec.com/hub/blog/exploitable-critical-rce-vulnerability-allows-regular-users-to-fully-compromise-active-directory-printnightmare-cve-2021-1675
 - https://www.reddit.com/r/msp/comments/ob6y02/critical_vulnerability_printnightmare_exposes
 tags:
   analytic_story:
diff --git a/detections/endpoint/process_deleting_its_process_file_path.yml b/detections/endpoint/process_deleting_its_process_file_path.yml
index 91963eddde..ee649e7d5b 100644
--- a/detections/endpoint/process_deleting_its_process_file_path.yml
+++ b/detections/endpoint/process_deleting_its_process_file_path.yml
@@ -24,7 +24,7 @@ how_to_implement: You must be ingesting data that records process activity from
   model.
 known_false_positives: unknown
 references:
-- https://www.fireeye.com/blog/threat-research/2020/10/fin11-email-campaigns-precursor-for-ransomware-data-theft.html
+- https://www.mandiant.com/resources/fin11-email-campaigns-precursor-for-ransomware-data-theft
 - https://blog.virustotal.com/2020/11/keep-your-friends-close-keep-ransomware.html
 - https://www.microsoft.com/security/blog/2022/01/15/destructive-malware-targeting-ukrainian-organizations/
 tags:
diff --git a/detections/endpoint/ransomware_notes_bulk_creation.yml b/detections/endpoint/ransomware_notes_bulk_creation.yml
index 4beb9bbadd..7f18f31534 100644
--- a/detections/endpoint/ransomware_notes_bulk_creation.yml
+++ b/detections/endpoint/ransomware_notes_bulk_creation.yml
@@ -22,7 +22,7 @@ how_to_implement: You must be ingesting data that records the filesystem activit
   you want to collect data.
 known_false_positives: unknown
 references:
-- https://www.fireeye.com/blog/threat-research/2020/10/fin11-email-campaigns-precursor-for-ransomware-data-theft.html
+- https://www.mandiant.com/resources/fin11-email-campaigns-precursor-for-ransomware-data-theft
 - https://blog.virustotal.com/2020/11/keep-your-friends-close-keep-ransomware.html
 tags:
   analytic_story:
diff --git a/detections/endpoint/regsvr32_silent_and_install_param_dll_loading.yml b/detections/endpoint/regsvr32_silent_and_install_param_dll_loading.yml
index 25a35dcb5b..1aab04eb9a 100644
--- a/detections/endpoint/regsvr32_silent_and_install_param_dll_loading.yml
+++ b/detections/endpoint/regsvr32_silent_and_install_param_dll_loading.yml
@@ -26,7 +26,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: Other third part application may used this parameter but not
   so common in base windows environment.
 references:
-- https://app.any.run/tasks/dc93ee63-050c-4ff8-b07e-8277af9ab939/#
+- https://app.any.run/tasks/dc93ee63-050c-4ff8-b07e-8277af9ab939/
 - https://attack.mitre.org/techniques/T1218/010/
 tags:
   analytic_story:
diff --git a/detections/endpoint/remcos_rat_file_creation_in_remcos_folder.yml b/detections/endpoint/remcos_rat_file_creation_in_remcos_folder.yml
index 5a7db5ddf9..99f25fd29a 100644
--- a/detections/endpoint/remcos_rat_file_creation_in_remcos_folder.yml
+++ b/detections/endpoint/remcos_rat_file_creation_in_remcos_folder.yml
@@ -21,7 +21,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   Sysmon TA.
 known_false_positives: unknown
 references:
-- https://success.trendmicro.com/solution/1123281-remcos-malware-information
+- https://success.trendmicro.com/dcx/s/solution/1123281-remcos-malware-information?language=en_US
 - https://blog.malwarebytes.com/threat-intelligence/2021/07/remcos-rat-delivered-via-visual-basic/
 tags:
   analytic_story:
diff --git a/detections/endpoint/resize_shadowstorage_volume.yml b/detections/endpoint/resize_shadowstorage_volume.yml
index e26140cd64..5b8102197c 100644
--- a/detections/endpoint/resize_shadowstorage_volume.yml
+++ b/detections/endpoint/resize_shadowstorage_volume.yml
@@ -28,7 +28,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   Sysmon TA.
 known_false_positives: network admin can resize the shadowstorage for valid purposes.
 references:
-- https://www.fireeye.com/blog/threat-research/2020/10/fin11-email-campaigns-precursor-for-ransomware-data-theft.html
+- https://www.mandiant.com/resources/fin11-email-campaigns-precursor-for-ransomware-data-theft
 - https://blog.virustotal.com/2020/11/keep-your-friends-close-keep-ransomware.html
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1490/T1490.md
 - https://redcanary.com/blog/blackbyte-ransomware/
diff --git a/detections/endpoint/runas_execution_in_commandline.yml b/detections/endpoint/runas_execution_in_commandline.yml
index 3fd74c50d6..74202d6ecc 100644
--- a/detections/endpoint/runas_execution_in_commandline.yml
+++ b/detections/endpoint/runas_execution_in_commandline.yml
@@ -27,7 +27,7 @@ known_false_positives: A network operator or systems administrator may utilize a
   automated or manual execute this command that may generate false positives. filter
   is needed.
 references:
-- https://app.any.run/tasks/ad4c3cda-41f2-4401-8dba-56cc2d245488/#
+- https://app.any.run/tasks/ad4c3cda-41f2-4401-8dba-56cc2d245488/
 tags:
   analytic_story:
   - Windows Privilege Escalation
diff --git a/detections/endpoint/rundll32_with_no_command_line_arguments_with_network.yml b/detections/endpoint/rundll32_with_no_command_line_arguments_with_network.yml
index bc4cd4c43b..4ece57a8bc 100644
--- a/detections/endpoint/rundll32_with_no_command_line_arguments_with_network.yml
+++ b/detections/endpoint/rundll32_with_no_command_line_arguments_with_network.yml
@@ -35,7 +35,7 @@ known_false_positives: Although unlikely, some legitimate applications may use a
 references:
 - https://attack.mitre.org/techniques/T1218/011/
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1218.011/T1218.011.md
-- https://lolbas-project.github.io/lolbas/Binaries/Rundll32
+- https://lolbas-project.github.io/lolbas/Binaries/Rundll32/
 - https://bohops.com/2018/02/26/leveraging-inf-sct-fetch-execute-techniques-for-bypass-evasion-persistence/
 tags:
   analytic_story:
diff --git a/detections/endpoint/sdclt_uac_bypass.yml b/detections/endpoint/sdclt_uac_bypass.yml
index a217ae1d8d..6e0bfbad04 100644
--- a/detections/endpoint/sdclt_uac_bypass.yml
+++ b/detections/endpoint/sdclt_uac_bypass.yml
@@ -33,7 +33,7 @@ known_false_positives: Limited to no false positives are expected.
 references:
 - https://enigma0x3.net/2017/03/17/fileless-uac-bypass-using-sdclt-exe/
 - https://github.com/hfiref0x/UACME
-- https://www.cyborgsecurity.com/cyborg_labs/threat-hunt-deep-dives-user-account-control-bypass-via-registry-modification/
+- https://www.cyborgsecurity.com/cyborg-labs/threat-hunt-deep-dives-user-account-control-bypass-via-registry-modification/
 tags:
   analytic_story:
   - Windows Defense Evasion Tactics
diff --git a/detections/endpoint/searchprotocolhost_with_no_command_line_with_network.yml b/detections/endpoint/searchprotocolhost_with_no_command_line_with_network.yml
index 72bb777f36..344ea9a9db 100644
--- a/detections/endpoint/searchprotocolhost_with_no_command_line_with_network.yml
+++ b/detections/endpoint/searchprotocolhost_with_no_command_line_with_network.yml
@@ -28,7 +28,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: Limited false positives may be present in small environments.
   Tuning may be required based on parent process.
 references:
-- https://github.com/fireeye/red_team_tool_countermeasures/blob/master/rules/PGF/supplemental/hxioc/SUSPICIOUS%20EXECUTION%20OF%20SEARCHPROTOCOLHOST%20(METHODOLOGY).ioc
+- https://github.com/mandiant/red_team_tool_countermeasures/blob/master/rules/PGF/supplemental/hxioc/SUSPICIOUS%20EXECUTION%20OF%20SEARCHPROTOCOLHOST%20(METHODOLOGY).ioc
 tags:
   analytic_story:
   - Cobalt Strike
diff --git a/detections/endpoint/services_escalate_exe.yml b/detections/endpoint/services_escalate_exe.yml
index c92a1c97ba..1acb3164ea 100644
--- a/detections/endpoint/services_escalate_exe.yml
+++ b/detections/endpoint/services_escalate_exe.yml
@@ -31,7 +31,7 @@ known_false_positives: False positives should be limited as `services.exe` shoul
 references:
 - https://thedfirreport.com/2021/03/29/sodinokibi-aka-revil-ransomware/
 - https://attack.mitre.org/techniques/T1548/
-- https://www.cobaltstrike.com/help-beacon
+- https://hstechdocs.helpsystems.com/manuals/cobaltstrike/current/userguide/index.htm#cshid=1085
 tags:
   analytic_story:
   - Cobalt Strike
diff --git a/detections/endpoint/slui_runas_elevated.yml b/detections/endpoint/slui_runas_elevated.yml
index 4dbc34bea1..2a13cda060 100644
--- a/detections/endpoint/slui_runas_elevated.yml
+++ b/detections/endpoint/slui_runas_elevated.yml
@@ -25,10 +25,10 @@ known_false_positives: Limited false positives should be present as this is not
   used by legitimate applications.
 references:
 - https://www.exploit-db.com/exploits/46998
-- https://medium.com/@mattharr0ey/privilege-escalation-uac-bypass-in-changepk-c40b92818d1b
+- https://mattharr0ey.medium.com/privilege-escalation-uac-bypass-in-changepk-c40b92818d1b
 - https://gist.github.com/r00t-3xp10it/0c92cd554d3156fd74f6c25660ccc466
 - https://www.rapid7.com/db/modules/exploit/windows/local/bypassuac_sluihijack/
-- https://www.fireeye.com/blog/threat-research/2021/05/shining-a-light-on-darkside-ransomware-operations.html
+- https://www.mandiant.com/resources/shining-a-light-on-darkside-ransomware-operations
 tags:
   analytic_story:
   - DarkSide Ransomware
diff --git a/detections/endpoint/slui_spawning_a_process.yml b/detections/endpoint/slui_spawning_a_process.yml
index 3215584ac4..cd58e631f7 100644
--- a/detections/endpoint/slui_spawning_a_process.yml
+++ b/detections/endpoint/slui_spawning_a_process.yml
@@ -26,7 +26,7 @@ known_false_positives: Certain applications may spawn from `slui.exe` that are l
 references:
 - https://www.exploit-db.com/exploits/46998
 - https://www.rapid7.com/db/modules/exploit/windows/local/bypassuac_sluihijack/
-- https://www.fireeye.com/blog/threat-research/2021/05/shining-a-light-on-darkside-ransomware-operations.html
+- https://www.mandiant.com/resources/shining-a-light-on-darkside-ransomware-operations
 tags:
   analytic_story:
   - DarkSide Ransomware
diff --git a/detections/endpoint/spoolsv_spawning_rundll32.yml b/detections/endpoint/spoolsv_spawning_rundll32.yml
index 5b326fbfa3..f899e7cb85 100644
--- a/detections/endpoint/spoolsv_spawning_rundll32.yml
+++ b/detections/endpoint/spoolsv_spawning_rundll32.yml
@@ -26,8 +26,8 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: Limited false positives have been identified. There are limited
   instances where `rundll32.exe` may be spawned by a legitimate print driver.
 references:
-- https://blog.truesec.com/2021/06/30/fix-for-printnightmare-cve-2021-1675-exploit-to-keep-your-print-servers-running-while-a-patch-is-not-available/
-- https://blog.truesec.com/2021/06/30/exploitable-critical-rce-vulnerability-allows-regular-users-to-fully-compromise-active-directory-printnightmare-cve-2021-1675/
+- https://www.truesec.com/hub/blog/fix-for-printnightmare-cve-2021-1675-exploit-to-keep-your-print-servers-running-while-a-patch-is-not-available
+- https://www.truesec.com/hub/blog/exploitable-critical-rce-vulnerability-allows-regular-users-to-fully-compromise-active-directory-printnightmare-cve-2021-1675
 - https://www.reddit.com/r/msp/comments/ob6y02/critical_vulnerability_printnightmare_exposes
 tags:
   analytic_story:
diff --git a/detections/endpoint/spoolsv_suspicious_process_access.yml b/detections/endpoint/spoolsv_suspicious_process_access.yml
index 121e22215b..5e8615a259 100644
--- a/detections/endpoint/spoolsv_suspicious_process_access.yml
+++ b/detections/endpoint/spoolsv_suspicious_process_access.yml
@@ -23,8 +23,8 @@ how_to_implement: To successfully implement this search, you need to be ingestin
 known_false_positives: Unknown. Filter as needed.
 references:
 - https://github.com/cube0x0/impacket/commit/73b9466c17761384ece11e1028ec6689abad6818
-- https://blog.truesec.com/2021/06/30/fix-for-printnightmare-cve-2021-1675-exploit-to-keep-your-print-servers-running-while-a-patch-is-not-available/
-- https://blog.truesec.com/2021/06/30/exploitable-critical-rce-vulnerability-allows-regular-users-to-fully-compromise-active-directory-printnightmare-cve-2021-1675/
+- https://www.truesec.com/hub/blog/fix-for-printnightmare-cve-2021-1675-exploit-to-keep-your-print-servers-running-while-a-patch-is-not-available
+- https://www.truesec.com/hub/blog/exploitable-critical-rce-vulnerability-allows-regular-users-to-fully-compromise-active-directory-printnightmare-cve-2021-1675
 - https://www.reddit.com/r/msp/comments/ob6y02/critical_vulnerability_printnightmare_exposes
 tags:
   analytic_story:
diff --git a/detections/endpoint/spoolsv_writing_a_dll.yml b/detections/endpoint/spoolsv_writing_a_dll.yml
index 8c5aebff2d..537821a9cf 100644
--- a/detections/endpoint/spoolsv_writing_a_dll.yml
+++ b/detections/endpoint/spoolsv_writing_a_dll.yml
@@ -28,8 +28,8 @@ how_to_implement: To successfully implement this search you need to be ingesting
   node.
 known_false_positives: Unknown.
 references:
-- https://blog.truesec.com/2021/06/30/fix-for-printnightmare-cve-2021-1675-exploit-to-keep-your-print-servers-running-while-a-patch-is-not-available/
-- https://blog.truesec.com/2021/06/30/exploitable-critical-rce-vulnerability-allows-regular-users-to-fully-compromise-active-directory-printnightmare-cve-2021-1675/
+- https://www.truesec.com/hub/blog/fix-for-printnightmare-cve-2021-1675-exploit-to-keep-your-print-servers-running-while-a-patch-is-not-available
+- https://www.truesec.com/hub/blog/exploitable-critical-rce-vulnerability-allows-regular-users-to-fully-compromise-active-directory-printnightmare-cve-2021-1675
 - https://www.reddit.com/r/msp/comments/ob6y02/critical_vulnerability_printnightmare_exposes
 tags:
   analytic_story:
diff --git a/detections/endpoint/spoolsv_writing_a_dll___sysmon.yml b/detections/endpoint/spoolsv_writing_a_dll___sysmon.yml
index c725e2030e..a1a2b581fe 100644
--- a/detections/endpoint/spoolsv_writing_a_dll___sysmon.yml
+++ b/detections/endpoint/spoolsv_writing_a_dll___sysmon.yml
@@ -23,8 +23,8 @@ how_to_implement: To successfully implement this search, you need to be ingestin
 known_false_positives: Limited false positives. Filter as needed.
 references:
 - https://github.com/cube0x0/impacket/commit/73b9466c17761384ece11e1028ec6689abad6818
-- https://blog.truesec.com/2021/06/30/fix-for-printnightmare-cve-2021-1675-exploit-to-keep-your-print-servers-running-while-a-patch-is-not-available/
-- https://blog.truesec.com/2021/06/30/exploitable-critical-rce-vulnerability-allows-regular-users-to-fully-compromise-active-directory-printnightmare-cve-2021-1675/
+- https://www.truesec.com/hub/blog/fix-for-printnightmare-cve-2021-1675-exploit-to-keep-your-print-servers-running-while-a-patch-is-not-available
+- https://www.truesec.com/hub/blog/exploitable-critical-rce-vulnerability-allows-regular-users-to-fully-compromise-active-directory-printnightmare-cve-2021-1675
 - https://www.reddit.com/r/msp/comments/ob6y02/critical_vulnerability_printnightmare_exposes
 tags:
   analytic_story:
diff --git a/detections/endpoint/sqlite_module_in_temp_folder.yml b/detections/endpoint/sqlite_module_in_temp_folder.yml
index 7611e81ac0..35f4d81ff1 100644
--- a/detections/endpoint/sqlite_module_in_temp_folder.yml
+++ b/detections/endpoint/sqlite_module_in_temp_folder.yml
@@ -21,7 +21,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   Sysmon TA.
 known_false_positives: unknown
 references:
-- https://www.cisecurity.org/white-papers/security-primer-icedid/
+- https://www.cisecurity.org/insights/white-papers/security-primer-icedid
 tags:
   analytic_story:
   - IcedID
diff --git a/detections/endpoint/ssa___detect_rclone_command_line_usage.yml b/detections/endpoint/ssa___detect_rclone_command_line_usage.yml
index fb2d973337..a1ba6e1c39 100644
--- a/detections/endpoint/ssa___detect_rclone_command_line_usage.yml
+++ b/detections/endpoint/ssa___detect_rclone_command_line_usage.yml
@@ -37,7 +37,7 @@ known_false_positives: False positives should be limited as this is restricted t
   the Rclone process name. Filter or tune the analytic as needed.
 references:
 - https://redcanary.com/blog/rclone-mega-extortion/
-- https://www.fireeye.com/blog/threat-research/2021/05/shining-a-light-on-darkside-ransomware-operations.html
+- https://www.mandiant.com/resources/shining-a-light-on-darkside-ransomware-operations
 - https://thedfirreport.com/2021/03/29/sodinokibi-aka-revil-ransomware/
 - https://thedfirreport.com/2021/11/29/continuing-the-bazar-ransomware-story/
 tags:
diff --git a/detections/endpoint/ssa___resize_shadowstorage_volume.yml b/detections/endpoint/ssa___resize_shadowstorage_volume.yml
index 8051f68180..2e976b397c 100644
--- a/detections/endpoint/ssa___resize_shadowstorage_volume.yml
+++ b/detections/endpoint/ssa___resize_shadowstorage_volume.yml
@@ -30,7 +30,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
 known_false_positives: System administrators may resize the shadowstorage for valid
   purposes. Filter as needed.
 references:
-- https://www.fireeye.com/blog/threat-research/2020/10/fin11-email-campaigns-precursor-for-ransomware-data-theft.html
+- https://www.mandiant.com/resources/fin11-email-campaigns-precursor-for-ransomware-data-theft
 - https://blog.virustotal.com/2020/11/keep-your-friends-close-keep-ransomware.html
 tags:
   analytic_story:
diff --git a/detections/endpoint/ssa___windows_eventvwr_uac_bypass.yml b/detections/endpoint/ssa___windows_eventvwr_uac_bypass.yml
index 0fbedc0c94..236ae03d09 100644
--- a/detections/endpoint/ssa___windows_eventvwr_uac_bypass.yml
+++ b/detections/endpoint/ssa___windows_eventvwr_uac_bypass.yml
@@ -35,7 +35,7 @@ known_false_positives: None known at this time.
 references:
 - https://blog.malwarebytes.com/malwarebytes-news/2021/02/lazyscripter-from-empire-to-double-rat/
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1548.002/T1548.002.md
-- https://attack.mitre.org/techniques/T1548/002
+- https://attack.mitre.org/techniques/T1548/002/
 - https://enigma0x3.net/2016/08/15/fileless-uac-bypass-using-eventvwr-exe-and-registry-hijacking/
 tags:
   analytic_story:
diff --git a/detections/endpoint/ssa___windows_rasautou_dll_execution.yml b/detections/endpoint/ssa___windows_rasautou_dll_execution.yml
index ff0ec076d5..1e567f0bcc 100644
--- a/detections/endpoint/ssa___windows_rasautou_dll_execution.yml
+++ b/detections/endpoint/ssa___windows_rasautou_dll_execution.yml
@@ -34,7 +34,7 @@ references:
 - https://github.com/mandiant/DueDLLigence
 - https://github.com/MHaggis/notes/blob/master/utilities/Invoke-SPLDLLigence.ps1
 - https://gist.github.com/NickTyrer/c6043e4b302d5424f701f15baf136513
-- https://www.fireeye.com/blog/threat-research/2019/10/staying-hidden-on-the-endpoint-evading-detection-with-shellcode.html
+- https://www.mandiant.com/resources/staying-hidden-on-the-endpoint-evading-detection-with-shellcode
 tags:
   analytic_story:
   - Windows Defense Evasion Tactics
diff --git a/detections/endpoint/suspicious_dllhost_no_command_line_arguments.yml b/detections/endpoint/suspicious_dllhost_no_command_line_arguments.yml
index 1312e658f1..eb9e665c8f 100644
--- a/detections/endpoint/suspicious_dllhost_no_command_line_arguments.yml
+++ b/detections/endpoint/suspicious_dllhost_no_command_line_arguments.yml
@@ -26,7 +26,7 @@ known_false_positives: Limited false positives may be present in small environme
   Tuning may be required based on parent process.
 references:
 - https://raw.githubusercontent.com/threatexpress/malleable-c2/c3385e481159a759f79b8acfe11acf240893b830/jquery-c2.4.2.profile
-- https://blog.cobaltstrike.com/2021/02/09/learn-pipe-fitting-for-all-of-your-offense-projects/
+- https://www.cobaltstrike.com/blog/learn-pipe-fitting-for-all-of-your-offense-projects/
 tags:
   analytic_story:
   - Cobalt Strike
diff --git a/detections/endpoint/suspicious_gpupdate_no_command_line_arguments.yml b/detections/endpoint/suspicious_gpupdate_no_command_line_arguments.yml
index 63799d764f..9b4e2687db 100644
--- a/detections/endpoint/suspicious_gpupdate_no_command_line_arguments.yml
+++ b/detections/endpoint/suspicious_gpupdate_no_command_line_arguments.yml
@@ -25,7 +25,7 @@ known_false_positives: Limited false positives may be present in small environme
   Tuning may be required based on parent process.
 references:
 - https://raw.githubusercontent.com/xx0hcd/Malleable-C2-Profiles/0ef8cf4556e26f6d4190c56ba697c2159faa5822/crimeware/trick_ryuk.profile
-- https://blog.cobaltstrike.com/2021/02/09/learn-pipe-fitting-for-all-of-your-offense-projects/
+- https://www.cobaltstrike.com/blog/learn-pipe-fitting-for-all-of-your-offense-projects/
 tags:
   analytic_story:
   - Cobalt Strike
diff --git a/detections/endpoint/suspicious_image_creation_in_appdata_folder.yml b/detections/endpoint/suspicious_image_creation_in_appdata_folder.yml
index 3c7ddb8699..76b8e82de4 100644
--- a/detections/endpoint/suspicious_image_creation_in_appdata_folder.yml
+++ b/detections/endpoint/suspicious_image_creation_in_appdata_folder.yml
@@ -28,7 +28,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   Sysmon TA.
 known_false_positives: unknown
 references:
-- https://success.trendmicro.com/solution/1123281-remcos-malware-information
+- https://success.trendmicro.com/dcx/s/solution/1123281-remcos-malware-information?language=en_US
 - https://blog.malwarebytes.com/threat-intelligence/2021/07/remcos-rat-delivered-via-visual-basic/
 tags:
   analytic_story:
diff --git a/detections/endpoint/suspicious_reg_exe_process.yml b/detections/endpoint/suspicious_reg_exe_process.yml
index a0c4ac1abd..8eb801fb52 100644
--- a/detections/endpoint/suspicious_reg_exe_process.yml
+++ b/detections/endpoint/suspicious_reg_exe_process.yml
@@ -28,7 +28,7 @@ known_false_positives: It's possible for system administrators to write scripts
   exhibit this behavior. If this is the case, the search will need to be modified
   to filter them out.
 references:
-- https://car.mitre.org/wiki/CAR-2013-03-001
+- https://car.mitre.org/wiki/CAR-2013-03-001/
 tags:
   analytic_story:
   - Windows Defense Evasion Tactics
diff --git a/detections/endpoint/suspicious_rundll32_dllregisterserver.yml b/detections/endpoint/suspicious_rundll32_dllregisterserver.yml
index 01c948bca9..c8be5358d0 100644
--- a/detections/endpoint/suspicious_rundll32_dllregisterserver.yml
+++ b/detections/endpoint/suspicious_rundll32_dllregisterserver.yml
@@ -30,11 +30,11 @@ known_false_positives: This is likely to produce false positives and will requir
 references:
 - https://attack.mitre.org/techniques/T1218/011/
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1218.011/T1218.011.md
-- https://lolbas-project.github.io/lolbas/Binaries/Rundll32
+- https://lolbas-project.github.io/lolbas/Binaries/Rundll32/
 - https://symantec-enterprise-blogs.security.com/blogs/threat-intelligence/seedworm-apt-iran-middle-east
 - https://github.com/pan-unit42/tweets/blob/master/2020-12-10-IOCs-from-Ursnif-infection-with-Delf-variant.txt
 - https://www.crowdstrike.com/blog/duck-hunting-with-falcon-complete-qakbot-zip-based-campaign/
-- https://msdn.microsoft.com/en-us/library/windows/desktop/ms682162(v=vs.85).aspx
+- https://docs.microsoft.com/en-us/windows/win32/api/olectl/nf-olectl-dllregisterserver?redirectedfrom=MSDN
 tags:
   analytic_story:
   - Suspicious Rundll32 Activity
diff --git a/detections/endpoint/suspicious_rundll32_startw.yml b/detections/endpoint/suspicious_rundll32_startw.yml
index f490971e25..9917f0f74b 100644
--- a/detections/endpoint/suspicious_rundll32_startw.yml
+++ b/detections/endpoint/suspicious_rundll32_startw.yml
@@ -31,8 +31,8 @@ known_false_positives: Although unlikely, some legitimate applications may use S
 references:
 - https://attack.mitre.org/techniques/T1218/011/
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1218.011/T1218.011.md
-- https://www.cobaltstrike.com/help-windows-executable
-- https://lolbas-project.github.io/lolbas/Binaries/Rundll32
+- https://hstechdocs.helpsystems.com/manuals/cobaltstrike/current/userguide/index.htm#cshid=1036
+- https://lolbas-project.github.io/lolbas/Binaries/Rundll32/
 - https://bohops.com/2018/02/26/leveraging-inf-sct-fetch-execute-techniques-for-bypass-evasion-persistence/
 tags:
   analytic_story:
diff --git a/detections/endpoint/suspicious_rundll32_with_no_command_line_arguments.yml b/detections/endpoint/suspicious_rundll32_with_no_command_line_arguments.yml
index 9923bf96eb..69bf3619f8 100644
--- a/detections/endpoint/suspicious_rundll32_with_no_command_line_arguments.yml
+++ b/detections/endpoint/suspicious_rundll32_with_no_command_line_arguments.yml
@@ -27,7 +27,7 @@ known_false_positives: Although unlikely, some legitimate applications may use a
 references:
 - https://attack.mitre.org/techniques/T1218/011/
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1218.011/T1218.011.md
-- https://lolbas-project.github.io/lolbas/Binaries/Rundll32
+- https://lolbas-project.github.io/lolbas/Binaries/Rundll32/
 - https://bohops.com/2018/02/26/leveraging-inf-sct-fetch-execute-techniques-for-bypass-evasion-persistence/
 tags:
   analytic_story:
diff --git a/detections/endpoint/suspicious_searchprotocolhost_no_command_line_arguments.yml b/detections/endpoint/suspicious_searchprotocolhost_no_command_line_arguments.yml
index 0a667c2244..91d7f49a76 100644
--- a/detections/endpoint/suspicious_searchprotocolhost_no_command_line_arguments.yml
+++ b/detections/endpoint/suspicious_searchprotocolhost_no_command_line_arguments.yml
@@ -26,7 +26,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: Limited false positives may be present in small environments.
   Tuning may be required based on parent process.
 references:
-- https://github.com/fireeye/red_team_tool_countermeasures/blob/master/rules/PGF/supplemental/hxioc/SUSPICIOUS%20EXECUTION%20OF%20SEARCHPROTOCOLHOST%20(METHODOLOGY).ioc
+- https://github.com/mandiant/red_team_tool_countermeasures/blob/master/rules/PGF/supplemental/hxioc/SUSPICIOUS%20EXECUTION%20OF%20SEARCHPROTOCOLHOST%20(METHODOLOGY).ioc
 tags:
   analytic_story:
   - Cobalt Strike
diff --git a/detections/endpoint/suspicious_wav_file_in_appdata_folder.yml b/detections/endpoint/suspicious_wav_file_in_appdata_folder.yml
index 111d718101..ff532958aa 100644
--- a/detections/endpoint/suspicious_wav_file_in_appdata_folder.yml
+++ b/detections/endpoint/suspicious_wav_file_in_appdata_folder.yml
@@ -27,7 +27,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   version 6.0.4 of the Sysmon TA.
 known_false_positives: unknown
 references:
-- https://success.trendmicro.com/solution/1123281-remcos-malware-information
+- https://success.trendmicro.com/dcx/s/solution/1123281-remcos-malware-information?language=en_US
 - https://blog.malwarebytes.com/threat-intelligence/2021/07/remcos-rat-delivered-via-visual-basic/
 tags:
   analytic_story:
diff --git a/detections/endpoint/unified_messaging_service_spawning_a_process.yml b/detections/endpoint/unified_messaging_service_spawning_a_process.yml
index 9089986a8c..6848554959 100644
--- a/detections/endpoint/unified_messaging_service_spawning_a_process.yml
+++ b/detections/endpoint/unified_messaging_service_spawning_a_process.yml
@@ -31,7 +31,7 @@ known_false_positives: Unknown. Tune out child processes as needed to limit volu
 references:
 - https://www.volexity.com/blog/2021/03/02/active-exploitation-of-microsoft-exchange-zero-day-vulnerabilities/
 - https://www.microsoft.com/security/blog/2021/03/02/hafnium-targeting-exchange-servers/
-- https://blog.rapid7.com/2021/03/03/rapid7s-insightidr-enables-detection-and-response-to-microsoft-exchange-0-day/
+- https://www.rapid7.com/blog/post/2021/03/03/rapid7s-insightidr-enables-detection-and-response-to-microsoft-exchange-0-day/
 tags:
   analytic_story:
   - HAFNIUM Group
diff --git a/detections/endpoint/windows_adfind_exe.yml b/detections/endpoint/windows_adfind_exe.yml
index 1a48f71173..f0faeb2522 100644
--- a/detections/endpoint/windows_adfind_exe.yml
+++ b/detections/endpoint/windows_adfind_exe.yml
@@ -26,7 +26,7 @@ known_false_positives: administrators rarely use adfind, usually not used for le
   reasons
 references:
 - https://www.volexity.com/blog/2020/12/14/dark-halo-leverages-solarwinds-compromise-to-breach-organizations/
-- https://www.fireeye.com/blog/threat-research/2019/01/a-nasty-trick-from-credential-theft-malware-to-business-disruption.html
+- https://www.mandiant.com/resources/a-nasty-trick-from-credential-theft-malware-to-business-disruption
 tags:
   analytic_story:
   - NOBELIUM Group
diff --git a/detections/endpoint/windows_high_file_deletion_frequency.yml b/detections/endpoint/windows_high_file_deletion_frequency.yml
index e36112b07b..9aaf0c761e 100644
--- a/detections/endpoint/windows_high_file_deletion_frequency.yml
+++ b/detections/endpoint/windows_high_file_deletion_frequency.yml
@@ -22,7 +22,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   If you are using Sysmon, you must have at least version 6.0.4 of the Sysmon TA.
 known_false_positives: user may delete bunch of pictures or files in a folder.
 references:
-- https://www.fireeye.com/blog/threat-research/2020/10/fin11-email-campaigns-precursor-for-ransomware-data-theft.html
+- https://www.mandiant.com/resources/fin11-email-campaigns-precursor-for-ransomware-data-theft
 - https://blog.virustotal.com/2020/11/keep-your-friends-close-keep-ransomware.html
 - https://www.microsoft.com/security/blog/2022/01/15/destructive-malware-targeting-ukrainian-organizations/
 tags:
diff --git a/detections/endpoint/windows_powerview_constrained_delegation_discovery.yml b/detections/endpoint/windows_powerview_constrained_delegation_discovery.yml
index d32ac79dcd..6e52ede05b 100644
--- a/detections/endpoint/windows_powerview_constrained_delegation_discovery.yml
+++ b/detections/endpoint/windows_powerview_constrained_delegation_discovery.yml
@@ -21,7 +21,7 @@ references:
 - https://adsecurity.org/?p=1667
 - https://docs.microsoft.com/en-us/defender-for-identity/cas-isp-unconstrained-kerberos
 - https://www.guidepointsecurity.com/blog/delegating-like-a-boss-abusing-kerberos-delegation-in-active-directory/
-- https://book.hacktricks.xyz/windows/active-directory-methodology/constrained-delegation
+- https://book.hacktricks.xyz/windows-hardening/active-directory-methodology/constrained-delegation
 - https://www.cyberark.com/resources/threat-research-blog/weakness-within-kerberos-delegation
 tags:
   analytic_story:
diff --git a/detections/endpoint/windows_rasautou_dll_execution.yml b/detections/endpoint/windows_rasautou_dll_execution.yml
index a0dde062a7..55f9249c45 100644
--- a/detections/endpoint/windows_rasautou_dll_execution.yml
+++ b/detections/endpoint/windows_rasautou_dll_execution.yml
@@ -27,7 +27,7 @@ references:
 - https://github.com/mandiant/DueDLLigence
 - https://github.com/MHaggis/notes/blob/master/utilities/Invoke-SPLDLLigence.ps1
 - https://gist.github.com/NickTyrer/c6043e4b302d5424f701f15baf136513
-- https://www.fireeye.com/blog/threat-research/2019/10/staying-hidden-on-the-endpoint-evading-detection-with-shellcode.html
+- https://www.mandiant.com/resources/staying-hidden-on-the-endpoint-evading-detection-with-shellcode
 tags:
   analytic_story:
   - Windows Defense Evasion Tactics
diff --git a/detections/endpoint/windows_service_created_with_suspicious_service_path.yml b/detections/endpoint/windows_service_created_with_suspicious_service_path.yml
index 6303ab57ba..5be65564e7 100644
--- a/detections/endpoint/windows_service_created_with_suspicious_service_path.yml
+++ b/detections/endpoint/windows_service_created_with_suspicious_service_path.yml
@@ -22,7 +22,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
 known_false_positives: Legitimate applications may install services with uncommon
   services paths.
 references:
-- https://www.fireeye.com/blog/threat-research/2020/10/fin11-email-campaigns-precursor-for-ransomware-data-theft.html
+- https://www.mandiant.com/resources/fin11-email-campaigns-precursor-for-ransomware-data-theft
 - https://blog.virustotal.com/2020/11/keep-your-friends-close-keep-ransomware.html
 tags:
   analytic_story:
diff --git a/detections/endpoint/xsl_script_execution_with_wmic.yml b/detections/endpoint/xsl_script_execution_with_wmic.yml
index 2d6353c745..b1e1d43adf 100644
--- a/detections/endpoint/xsl_script_execution_with_wmic.yml
+++ b/detections/endpoint/xsl_script_execution_with_wmic.yml
@@ -23,7 +23,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   Sysmon TA.
 known_false_positives: unknown
 references:
-- https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html
+- https://www.mandiant.com/resources/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation
 - https://attack.mitre.org/groups/G0046/
 - https://web.archive.org/web/20190814201250/https://subt0x11.blogspot.com/2018/04/wmicexe-whitelisting-bypass-hacking.html
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1220/T1220.md#atomic-test-3---wmic-bypass-using-local-xsl-file
diff --git a/detections/experimental/cloud/gsuite_suspicious_calendar_invite.yml b/detections/experimental/cloud/gsuite_suspicious_calendar_invite.yml
index e7d37480b9..5ed8af2f8c 100644
--- a/detections/experimental/cloud/gsuite_suspicious_calendar_invite.yml
+++ b/detections/experimental/cloud/gsuite_suspicious_calendar_invite.yml
@@ -24,7 +24,7 @@ known_false_positives: This search will also produce normal activity statistics.
   use email parameter.
 references:
 - https://www.techrepublic.com/article/how-to-avoid-the-dreaded-google-calendar-malicious-invite-issue/
-- https://gcn.com/articles/2012/09/26/20-most-common-words-phishing-attacks.aspx
+- https://gcn.com/cybersecurity/2012/09/the-20-most-common-words-in-phishing-attacks/280956/
 tags:
   analytic_story:
   - Spearphishing Attachments
diff --git a/detections/experimental/endpoint/ssa___high_file_deletion_frequency.yml b/detections/experimental/endpoint/ssa___high_file_deletion_frequency.yml
index 4c80844d43..58b8f09f88 100644
--- a/detections/experimental/endpoint/ssa___high_file_deletion_frequency.yml
+++ b/detections/experimental/endpoint/ssa___high_file_deletion_frequency.yml
@@ -29,7 +29,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
   your endpoints into the `Endpoint` datamodel in the `Filesytem` node.
 known_false_positives: user may delete bunch of pictures or files in a folder.
 references:
-- https://www.fireeye.com/blog/threat-research/2020/10/fin11-email-campaigns-precursor-for-ransomware-data-theft.html
+- https://www.mandiant.com/resources/fin11-email-campaigns-precursor-for-ransomware-data-theft
 - https://blog.virustotal.com/2020/11/keep-your-friends-close-keep-ransomware.html
 tags:
   analytic_story:
diff --git a/detections/experimental/endpoint/sunburst_correlation_dll_and_network_event.yml b/detections/experimental/endpoint/sunburst_correlation_dll_and_network_event.yml
index f040044a13..2fd0388a69 100644
--- a/detections/experimental/endpoint/sunburst_correlation_dll_and_network_event.yml
+++ b/detections/experimental/endpoint/sunburst_correlation_dll_and_network_event.yml
@@ -20,7 +20,7 @@ how_to_implement: This detection relies on sysmon logs with the Event ID 7, Driv
   We suggest to run this detection at least once a day over the last 14 days.
 known_false_positives: unknown
 references:
-- https://www.fireeye.com/blog/threat-research/2020/12/evasive-attacker-leverages-solarwinds-supply-chain-compromises-with-sunburst-backdoor.html
+- https://www.mandiant.com/resources/evasive-attacker-leverages-solarwinds-supply-chain-compromises-with-sunburst-backdoor
 tags:
   analytic_story:
   - NOBELIUM Group
diff --git a/detections/experimental/endpoint/suspicious_curl_network_connection.yml b/detections/experimental/endpoint/suspicious_curl_network_connection.yml
index e6c08c0328..7c203d23b2 100644
--- a/detections/experimental/endpoint/suspicious_curl_network_connection.yml
+++ b/detections/experimental/endpoint/suspicious_curl_network_connection.yml
@@ -22,7 +22,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: Unknown. Filter as needed.
 references:
 - https://redcanary.com/blog/clipping-silver-sparrows-wings/
-- https://marcosantadev.com/manage-plist-files-plistbuddy/
+- https://www.marcosantadev.com/manage-plist-files-plistbuddy/
 tags:
   analytic_story:
   - Silver Sparrow
diff --git a/detections/experimental/endpoint/suspicious_plistbuddy_usage.yml b/detections/experimental/endpoint/suspicious_plistbuddy_usage.yml
index 561a1ffe12..24b026e856 100644
--- a/detections/experimental/endpoint/suspicious_plistbuddy_usage.yml
+++ b/detections/experimental/endpoint/suspicious_plistbuddy_usage.yml
@@ -43,7 +43,7 @@ known_false_positives: Some legitimate applications may use PlistBuddy to create
   modify property lists and possibly generate false positives. Review the property
   list being modified or created to confirm.
 references:
-- https://marcosantadev.com/manage-plist-files-plistbuddy/
+- https://www.marcosantadev.com/manage-plist-files-plistbuddy/
 tags:
   analytic_story:
   - Silver Sparrow
diff --git a/detections/experimental/endpoint/suspicious_plistbuddy_usage_via_osquery.yml b/detections/experimental/endpoint/suspicious_plistbuddy_usage_via_osquery.yml
index 4a0f6a3dee..df66383b81 100644
--- a/detections/experimental/endpoint/suspicious_plistbuddy_usage_via_osquery.yml
+++ b/detections/experimental/endpoint/suspicious_plistbuddy_usage_via_osquery.yml
@@ -38,7 +38,7 @@ known_false_positives: Some legitimate applications may use PlistBuddy to create
   modify property lists and possibly generate false positives. Review the property
   list being modified or created to confirm.
 references:
-- https://marcosantadev.com/manage-plist-files-plistbuddy/
+- https://www.marcosantadev.com/manage-plist-files-plistbuddy/
 tags:
   analytic_story:
   - Silver Sparrow
diff --git a/detections/experimental/endpoint/suspicious_sqlite3_lsquarantine_behavior.yml b/detections/experimental/endpoint/suspicious_sqlite3_lsquarantine_behavior.yml
index 36e24b86b2..95230c6f41 100644
--- a/detections/experimental/endpoint/suspicious_sqlite3_lsquarantine_behavior.yml
+++ b/detections/experimental/endpoint/suspicious_sqlite3_lsquarantine_behavior.yml
@@ -22,7 +22,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: Unknown.
 references:
 - https://redcanary.com/blog/clipping-silver-sparrows-wings/
-- https://marcosantadev.com/manage-plist-files-plistbuddy/
+- https://www.marcosantadev.com/manage-plist-files-plistbuddy/
 tags:
   analytic_story:
   - Silver Sparrow
diff --git a/detections/experimental/network/detect_snicat_sni_exfiltration.yml b/detections/experimental/network/detect_snicat_sni_exfiltration.yml
index dad4a82700..6843d4d652 100644
--- a/detections/experimental/network/detect_snicat_sni_exfiltration.yml
+++ b/detections/experimental/network/detect_snicat_sni_exfiltration.yml
@@ -18,7 +18,7 @@ how_to_implement: You must be ingesting Zeek SSL data into Splunk. Zeek data sho
   or disprove if any data exfiltration has taken place.
 known_false_positives: Unknown
 references:
-- https://www.mnemonic.no/blog/introducing-snicat/
+- https://www.mnemonic.io/resources/blog/introducing-snicat/
 - https://github.com/mnemonic-no/SNIcat
 - https://attack.mitre.org/techniques/T1041/
 tags:
diff --git a/detections/experimental/network/detect_zerologon_via_zeek.yml b/detections/experimental/network/detect_zerologon_via_zeek.yml
index 8523d2505f..add1fbb7aa 100644
--- a/detections/experimental/network/detect_zerologon_via_zeek.yml
+++ b/detections/experimental/network/detect_zerologon_via_zeek.yml
@@ -20,7 +20,7 @@ known_false_positives: unknown
 references:
 - https://www.secura.com/blog/zero-logon
 - https://github.com/SecuraBV/CVE-2020-1472
-- https://portal.msrc.microsoft.com/en-US/security-guidance/advisory/CVE-2020-1472
+- https://msrc.microsoft.com/en-US/security-guidance/advisory/CVE-2020-1472
 tags:
   analytic_story:
   - Detect Zerologon Attack
diff --git a/detections/experimental/web/supernova_webshell.yml b/detections/experimental/web/supernova_webshell.yml
index 0c59ce3170..272047d798 100644
--- a/detections/experimental/web/supernova_webshell.yml
+++ b/detections/experimental/web/supernova_webshell.yml
@@ -20,7 +20,7 @@ known_false_positives: There might be false positives associted with this detect
   since items like args as a web argument is pretty generic.
 references:
 - https://www.splunk.com/en_us/blog/security/detecting-supernova-malware-solarwinds-continued.html
-- https://www.guidepointsecurity.com/supernova-solarwinds-net-webshell-analysis/
+- https://www.guidepointsecurity.com/blog/supernova-solarwinds-net-webshell-analysis/
 tags:
   analytic_story:
   - NOBELIUM Group
diff --git a/detections/network/multiple_archive_files_http_post_traffic.yml b/detections/network/multiple_archive_files_http_post_traffic.yml
index 3c52e5ca0b..2406e5f37e 100644
--- a/detections/network/multiple_archive_files_http_post_traffic.yml
+++ b/detections/network/multiple_archive_files_http_post_traffic.yml
@@ -29,7 +29,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
 known_false_positives: Normal archive transfer via HTTP protocol may trip this detection.
 references:
 - https://attack.mitre.org/techniques/T1560/001/
-- https://www.fireeye.com/blog/threat-research/2019/01/apt39-iranian-cyber-espionage-group-focused-on-personal-information.html
+- https://www.mandiant.com/resources/apt39-iranian-cyber-espionage-group-focused-on-personal-information
 - https://www.microsoft.com/security/blog/2021/01/20/deep-dive-into-the-solorigate-second-stage-activation-from-sunburst-to-teardrop-and-raindrop/
 tags:
   analytic_story:

From 646afbf214625cbe189c7365f645e8606935a865 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Mon, 2 May 2022 17:15:09 -0700
Subject: [PATCH 08/38] A number of small updates to support resolving and
 verifying all of the reference URLs in detections at scale. Also, fixed a few
 more HTTP Status Code 301 references in the tests folder.

---
 .../application/factory/factory.py            |  2 +-
 .../domain/entities/detection.py              | 52 +++++++++++++------
 .../endpoint/cobalt_strike_named_pipes.yml    |  2 +-
 .../stories/ransomware_darkside.yml           |  2 +-
 .../detections/cobalt_strike_named_pipes.yml  |  2 +-
 .../test_data/story/ransomware_darkside.yml   |  2 +-
 6 files changed, 42 insertions(+), 20 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/application/factory/factory.py b/bin/contentctl_project/contentctl_core/application/factory/factory.py
index a8d0ede095..0cd3e155d7 100644
--- a/bin/contentctl_project/contentctl_core/application/factory/factory.py
+++ b/bin/contentctl_project/contentctl_core/application/factory/factory.py
@@ -81,7 +81,7 @@ def createSecurityContent(self, type: SecurityContentType) -> list:
 
           import copy
           import threading
-          NUM_THREADS = 50
+          NUM_THREADS = 20
           
           def make_detections_thread(index:int):
                my_dto = copy.deepcopy(self.input_dto)
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/detection.py b/bin/contentctl_project/contentctl_core/domain/entities/detection.py
index dbe001277e..a9f9a1a551 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/detection.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/detection.py
@@ -19,6 +19,16 @@
 from bin.contentctl_project.contentctl_core.domain.entities.baseline import Baseline
 from bin.contentctl_project.contentctl_core.domain.entities.playbook import Playbook
 
+
+
+import functools
+DEFAULT_CHECK_REFERENCE_TIMEOUT_SECONDS = 5
+DEFAULT_USER_AGENT_STRING = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36"
+@functools.cache
+def check_reference(reference: str, timeout_seconds = DEFAULT_CHECK_REFERENCE_TIMEOUT_SECONDS, user_agent: str = DEFAULT_USER_AGENT_STRING)-> requests.Response:
+    get = requests.head(reference, timeout=timeout_seconds, headers = {"User-Agent": user_agent}, allow_redirects=True)
+    return get
+
 class Detection(BaseModel, SecurityContentObject):
     # detection spec
     name: str
@@ -122,22 +132,34 @@ def name_max_length(cls, values):
                 raise ValueError('name is longer then 67 chars: ' + values["name"])
         return values
 
+   
+        
+
+
+    @validator('references')
+    def references_check(cls, v, values):
+        #import traceback
+        #import sys
+        #traceback.print_stack()
+        #sys.exit(0)
+        for reference in v:
+            try:
+                get = check_reference(reference)
+                if get.status_code == 200:
+                    if reference != get.url:
+                        print(f"Redirect from {reference} to {get.url}")#follows all redirects    
+                    continue
+                elif get.status_code == 301:
+                    
+                    print(f"Redirect from {reference} to {get.url}")#follows all redirects
+                #If we get here, it means it was not a 200 response code. Raise an error
+                raise ValueError('Reference ' + str(get.status_code) + ' ' + reference + ' is not reachable: ' + values["name"])
+
+            except requests.exceptions.RequestException as e:
+                raise ValueError('Reference ' + reference + ' is not reachable: ' + values["name"])
+                #raise ValueError('Reference ' + reference + ' is not reachable: ' + values["name"])
 
-    # @validator('references')
-    # def references_check(cls, v, values):
-    #     #import traceback
-    #     #import sys
-    #     #traceback.print_stack()
-    #     #sys.exit(0)
-    #     for reference in v:
-    #         try:
-    #             get = requests.head(reference)
-    #             if not get.status_code == 200:
-    #                 raise ValueError('Reference ' + reference + ' is not reachable: ' + values["name"])
-    #         except requests.exceptions.RequestException as e:
-    #             raise ValueError('Reference ' + reference + ' is not reachable: ' + values["name"])
-
-    #     return v
+        return v
 
     @validator('search')
     def search_validate(cls, v, values):
diff --git a/bin/contentctl_project/contentctl_core/tests/application/use_cases/test_data_input/detections/endpoint/cobalt_strike_named_pipes.yml b/bin/contentctl_project/contentctl_core/tests/application/use_cases/test_data_input/detections/endpoint/cobalt_strike_named_pipes.yml
index 2515803d69..01e540af90 100644
--- a/bin/contentctl_project/contentctl_core/tests/application/use_cases/test_data_input/detections/endpoint/cobalt_strike_named_pipes.yml
+++ b/bin/contentctl_project/contentctl_core/tests/application/use_cases/test_data_input/detections/endpoint/cobalt_strike_named_pipes.yml
@@ -34,7 +34,7 @@ references:
 - https://attack.mitre.org/techniques/T1218/009/
 - https://docs.microsoft.com/en-us/windows/win32/ipc/named-pipes
 - https://gist.github.com/MHaggis/6c600e524045a6d49c35291a21e10752
-- https://www.fireeye.com/blog/threat-research/2021/05/shining-a-light-on-darkside-ransomware-operations.html
+- https://www.mandiant.com/resources/shining-a-light-on-darkside-ransomware-operations
 tags:
   analytic_story:
   - Cobalt Strike
diff --git a/bin/contentctl_project/contentctl_core/tests/application/use_cases/test_data_input/stories/ransomware_darkside.yml b/bin/contentctl_project/contentctl_core/tests/application/use_cases/test_data_input/stories/ransomware_darkside.yml
index f2e6d8bfd9..b9c2d45c48 100644
--- a/bin/contentctl_project/contentctl_core/tests/application/use_cases/test_data_input/stories/ransomware_darkside.yml
+++ b/bin/contentctl_project/contentctl_core/tests/application/use_cases/test_data_input/stories/ransomware_darkside.yml
@@ -8,7 +8,7 @@ description: Leverage searches that allow you to detect and investigate unusual
 narrative: 'This story addresses Darkside ransomware. This ransomware payload has many similarities to common ransomware however there are certain items particular to it. The creation of a .TXT log that shows every item being encrypted as well as the creation of ransomware notes and files adding a machine ID created based on CRC32 checksum algorithm. This ransomware payload leaves machines in minimal operation level,enough to browse the attackers websites. A customized URI with leaked information is presented to each victim.This is the ransomware payload that shut down the Colonial pipeline. The story is composed of several detection searches covering similar items to other ransomware payloads and those particular to Darkside payload.'
 references:
 - https://www.splunk.com/en_us/blog/security/the-darkside-of-the-ransomware-pipeline.htmlbig-game-hunting-with-ryuk-another-lucrative-targeted-ransomware/
-- https://www.fireeye.com/blog/threat-research/2021/05/shining-a-light-on-darkside-ransomware-operations.html
+- https://www.mandiant.com/resources/shining-a-light-on-darkside-ransomware-operations
 tags:
   analytic_story: DarkSide Ransomware
   category:
diff --git a/bin/contentctl_project/contentctl_core/tests/application/use_cases/test_data_output/detections/cobalt_strike_named_pipes.yml b/bin/contentctl_project/contentctl_core/tests/application/use_cases/test_data_output/detections/cobalt_strike_named_pipes.yml
index 4638e6c952..8cea32f024 100644
--- a/bin/contentctl_project/contentctl_core/tests/application/use_cases/test_data_output/detections/cobalt_strike_named_pipes.yml
+++ b/bin/contentctl_project/contentctl_core/tests/application/use_cases/test_data_output/detections/cobalt_strike_named_pipes.yml
@@ -34,7 +34,7 @@ references:
 - https://attack.mitre.org/techniques/T1218/009/
 - https://docs.microsoft.com/en-us/windows/win32/ipc/named-pipes
 - https://gist.github.com/MHaggis/6c600e524045a6d49c35291a21e10752
-- https://www.fireeye.com/blog/threat-research/2021/05/shining-a-light-on-darkside-ransomware-operations.html
+- https://www.mandiant.com/resources/shining-a-light-on-darkside-ransomware-operations
 tags:
   analytic_story:
   - Cobalt Strike
diff --git a/bin/contentctl_project/contentctl_infrastructure/tests/builder/test_data/story/ransomware_darkside.yml b/bin/contentctl_project/contentctl_infrastructure/tests/builder/test_data/story/ransomware_darkside.yml
index f2e6d8bfd9..b9c2d45c48 100644
--- a/bin/contentctl_project/contentctl_infrastructure/tests/builder/test_data/story/ransomware_darkside.yml
+++ b/bin/contentctl_project/contentctl_infrastructure/tests/builder/test_data/story/ransomware_darkside.yml
@@ -8,7 +8,7 @@ description: Leverage searches that allow you to detect and investigate unusual
 narrative: 'This story addresses Darkside ransomware. This ransomware payload has many similarities to common ransomware however there are certain items particular to it. The creation of a .TXT log that shows every item being encrypted as well as the creation of ransomware notes and files adding a machine ID created based on CRC32 checksum algorithm. This ransomware payload leaves machines in minimal operation level,enough to browse the attackers websites. A customized URI with leaked information is presented to each victim.This is the ransomware payload that shut down the Colonial pipeline. The story is composed of several detection searches covering similar items to other ransomware payloads and those particular to Darkside payload.'
 references:
 - https://www.splunk.com/en_us/blog/security/the-darkside-of-the-ransomware-pipeline.htmlbig-game-hunting-with-ryuk-another-lucrative-targeted-ransomware/
-- https://www.fireeye.com/blog/threat-research/2021/05/shining-a-light-on-darkside-ransomware-operations.html
+- https://www.mandiant.com/resources/shining-a-light-on-darkside-ransomware-operations
 tags:
   analytic_story: DarkSide Ransomware
   category:

From 900bd9e7e23c264b301db4ffbe5d7d82031b66e5 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 3 May 2022 10:34:10 -0700
Subject: [PATCH 09/38] Updated a large number of the links in other
 detections.  Also, adjusted the number of threads that run and tweaked the
 timeout time.

---
 .../contentctl_core/application/factory/factory.py   |  2 +-
 .../contentctl_core/domain/entities/detection.py     | 12 +++++++-----
 detections/cloud/kubernetes_nginx_ingress_rfi.yml    |  2 +-
 .../endpoint/account_discovery_with_net_app.yml      |  2 +-
 ...il_download_with_urlcache_and_split_arguments.yml |  2 +-
 .../detect_html_help_spawn_child_process.yml         |  2 +-
 .../detect_html_help_url_in_command_line.yml         |  2 +-
 ...ect_html_help_using_infotech_storage_handlers.yml |  2 +-
 .../detect_wmi_event_subscription_persistence.yml    |  4 ++--
 detections/endpoint/disable_show_hidden_files.yml    |  2 +-
 .../endpoint/disabling_firewall_with_netsh.yml       |  2 +-
 .../endpoint/dns_exfiltration_using_nslookup_app.yml |  4 ++--
 .../endpoint/excessive_usage_of_nslookup_app.yml     |  4 ++--
 ...able_file_written_in_administrative_smb_share.yml |  2 +-
 ...kerberoasting_spn_request_with_rc4_encryption.yml |  2 +-
 .../endpoint/nltest_domain_trust_discovery.yml       |  2 +-
 .../possible_lateral_movement_powershell_spawn.yml   |  2 +-
 .../powershell_using_memory_as_backing_store.yml     |  2 +-
 .../schedule_task_with_rundll32_command_trigger.yml  |  2 +-
 ...rviceprincipalnames_discovery_with_powershell.yml |  2 +-
 .../serviceprincipalnames_discovery_with_setspn.yml  |  2 +-
 .../ssa___dns_exfiltration_using_nslookup_app.yml    |  4 ++--
 .../ssa___windows_certutil_urlcache_download.yml     |  2 +-
 .../system_information_discovery_detection.yml       |  2 +-
 detections/endpoint/trickbot_named_pipe.yml          |  2 +-
 ...r_process_connecting_to_ip_check_web_services.yml |  2 +-
 .../wermgr_process_create_executable_file.yml        |  2 +-
 ...mgr_process_spawned_cmd_or_powershell_process.yml |  2 +-
 .../wmi_permanent_event_subscription___sysmon.yml    |  4 ++--
 .../wmi_recon_running_process_or_services.yml        |  4 ++--
 .../endpoint/winrm_spawning_a_process.yml            |  2 +-
 31 files changed, 43 insertions(+), 41 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/application/factory/factory.py b/bin/contentctl_project/contentctl_core/application/factory/factory.py
index 0cd3e155d7..1beafa9362 100644
--- a/bin/contentctl_project/contentctl_core/application/factory/factory.py
+++ b/bin/contentctl_project/contentctl_core/application/factory/factory.py
@@ -81,7 +81,7 @@ def createSecurityContent(self, type: SecurityContentType) -> list:
 
           import copy
           import threading
-          NUM_THREADS = 20
+          NUM_THREADS = 10
           
           def make_detections_thread(index:int):
                my_dto = copy.deepcopy(self.input_dto)
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/detection.py b/bin/contentctl_project/contentctl_core/domain/entities/detection.py
index a9f9a1a551..9604a43ffd 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/detection.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/detection.py
@@ -22,11 +22,11 @@
 
 
 import functools
-DEFAULT_CHECK_REFERENCE_TIMEOUT_SECONDS = 5
+DEFAULT_CHECK_REFERENCE_TIMEOUT_SECONDS = 60
 DEFAULT_USER_AGENT_STRING = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36"
 @functools.cache
 def check_reference(reference: str, timeout_seconds = DEFAULT_CHECK_REFERENCE_TIMEOUT_SECONDS, user_agent: str = DEFAULT_USER_AGENT_STRING)-> requests.Response:
-    get = requests.head(reference, timeout=timeout_seconds, headers = {"User-Agent": user_agent}, allow_redirects=True)
+    get = requests.get(reference, timeout=timeout_seconds, headers = {"User-Agent": user_agent, "referrer": "https://google.com"}, allow_redirects=True)
     return get
 
 class Detection(BaseModel, SecurityContentObject):
@@ -147,11 +147,13 @@ def references_check(cls, v, values):
                 get = check_reference(reference)
                 if get.status_code == 200:
                     if reference != get.url:
-                        print(f"Redirect from {reference} to {get.url}")#follows all redirects    
+                        print(f"Redirect from {reference} to {get.url}") #follows all redirects    
                     continue
                 elif get.status_code == 301:
-                    
-                    print(f"Redirect from {reference} to {get.url}")#follows all redirects
+                    #print(f"Redirect from {reference} to {get.url}")
+                    #Redirects here have typically added a guid or something similar to the URL which is not persistent
+                    #and will change across calls.  Don't throw an error on this.
+                    continue
                 #If we get here, it means it was not a 200 response code. Raise an error
                 raise ValueError('Reference ' + str(get.status_code) + ' ' + reference + ' is not reachable: ' + values["name"])
 
diff --git a/detections/cloud/kubernetes_nginx_ingress_rfi.yml b/detections/cloud/kubernetes_nginx_ingress_rfi.yml
index a647e3df2d..7ff7ad1c68 100644
--- a/detections/cloud/kubernetes_nginx_ingress_rfi.yml
+++ b/detections/cloud/kubernetes_nginx_ingress_rfi.yml
@@ -18,7 +18,7 @@ how_to_implement: You must ingest Kubernetes logs through Splunk Connect for Kub
 known_false_positives: unknown
 references:
 - https://github.com/splunk/splunk-connect-for-kubernetes
-- https://www.netsparker.com/blog/web-security/remote-file-inclusion-vulnerability/
+- https://www.invicti.com/blog/web-security/remote-file-inclusion-vulnerability/
 tags:
   analytic_story:
   - Dev Sec Ops
diff --git a/detections/endpoint/account_discovery_with_net_app.yml b/detections/endpoint/account_discovery_with_net_app.yml
index 50656c9f33..d22b22d34f 100644
--- a/detections/endpoint/account_discovery_with_net_app.yml
+++ b/detections/endpoint/account_discovery_with_net_app.yml
@@ -28,7 +28,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: admin or power user may used this series of command.
 references:
 - https://labs.vipre.com/trickbot-and-its-modules/
-- https://blog.whitehat.eu/2019/05/incident-trickbot-ryuk-2.html
+- https://whitehat.eu/incident-response-case-study-featuring-ryuk-and-trickbot-part-2/
 - https://app.any.run/tasks/48414a33-3d66-4a46-afe5-c2003bb55ccf/
 tags:
   analytic_story:
diff --git a/detections/endpoint/certutil_download_with_urlcache_and_split_arguments.yml b/detections/endpoint/certutil_download_with_urlcache_and_split_arguments.yml
index 8c970d2088..6bbe7450e2 100644
--- a/detections/endpoint/certutil_download_with_urlcache_and_split_arguments.yml
+++ b/detections/endpoint/certutil_download_with_urlcache_and_split_arguments.yml
@@ -29,7 +29,7 @@ known_false_positives: Limited false positives in most environments, however tun
 references:
 - https://attack.mitre.org/techniques/T1105/
 - https://www.avira.com/en/blog/certutil-abused-by-attackers-to-spread-threats
-- https://www.fireeye.com/blog/threat-research/2019/10/certutil-qualms-they-came-to-drop-fombs.html
+- https://web.archive.org/web/20210921110637/https://www.fireeye.com/blog/threat-research/2019/10/certutil-qualms-they-came-to-drop-fombs.html
 tags:
   analytic_story:
   - Ingress Tool Transfer
diff --git a/detections/endpoint/detect_html_help_spawn_child_process.yml b/detections/endpoint/detect_html_help_spawn_child_process.yml
index 7803c03508..ce9659c565 100644
--- a/detections/endpoint/detect_html_help_spawn_child_process.yml
+++ b/detections/endpoint/detect_html_help_spawn_child_process.yml
@@ -32,7 +32,7 @@ references:
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1218.001/T1218.001.md
 - https://lolbas-project.github.io/lolbas/Binaries/Hh/
 - https://gist.github.com/mgeeky/cce31c8602a144d8f2172a73d510e0e7
-- https://cyberforensicator.com/2019/01/20/silence-dissecting-malicious-chm-files-and-performing-forensic-analysis/
+- https://web.archive.org/web/20220119133748/https://cyberforensicator.com/2019/01/20/silence-dissecting-malicious-chm-files-and-performing-forensic-analysis/
 tags:
   analytic_story:
   - Suspicious Compiled HTML Activity
diff --git a/detections/endpoint/detect_html_help_url_in_command_line.yml b/detections/endpoint/detect_html_help_url_in_command_line.yml
index 71053ab030..1772887629 100644
--- a/detections/endpoint/detect_html_help_url_in_command_line.yml
+++ b/detections/endpoint/detect_html_help_url_in_command_line.yml
@@ -36,7 +36,7 @@ references:
 - https://lolbas-project.github.io/lolbas/Binaries/Hh/
 - https://blog.sevagas.com/?Hacking-around-HTA-files
 - https://gist.github.com/mgeeky/cce31c8602a144d8f2172a73d510e0e7
-- https://cyberforensicator.com/2019/01/20/silence-dissecting-malicious-chm-files-and-performing-forensic-analysis/
+- https://web.archive.org/web/20220119133748/https://cyberforensicator.com/2019/01/20/silence-dissecting-malicious-chm-files-and-performing-forensic-analysis/
 tags:
   analytic_story:
   - Suspicious Compiled HTML Activity
diff --git a/detections/endpoint/detect_html_help_using_infotech_storage_handlers.yml b/detections/endpoint/detect_html_help_using_infotech_storage_handlers.yml
index 1c1298b8fe..7c1a5b5be0 100644
--- a/detections/endpoint/detect_html_help_using_infotech_storage_handlers.yml
+++ b/detections/endpoint/detect_html_help_using_infotech_storage_handlers.yml
@@ -38,7 +38,7 @@ references:
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1218.001/T1218.001.md
 - https://lolbas-project.github.io/lolbas/Binaries/Hh/
 - https://gist.github.com/mgeeky/cce31c8602a144d8f2172a73d510e0e7
-- https://cyberforensicator.com/2019/01/20/silence-dissecting-malicious-chm-files-and-performing-forensic-analysis/
+- https://web.archive.org/web/20220119133748/https://cyberforensicator.com/2019/01/20/silence-dissecting-malicious-chm-files-and-performing-forensic-analysis/
 tags:
   analytic_story:
   - Suspicious Compiled HTML Activity
diff --git a/detections/endpoint/detect_wmi_event_subscription_persistence.yml b/detections/endpoint/detect_wmi_event_subscription_persistence.yml
index aaf397cc5b..1cb56a143a 100644
--- a/detections/endpoint/detect_wmi_event_subscription_persistence.yml
+++ b/detections/endpoint/detect_wmi_event_subscription_persistence.yml
@@ -38,8 +38,8 @@ known_false_positives: It is possible some applications will create a consumer a
 references:
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1546.003/T1546.003.md
 - https://www.eideon.com/2018-03-02-THL03-WMIBackdoors/
-- https://github.com/trustedsec/SysmonCommunityGuide/blob/master/WMI-events.md
-- https://in.security/an-intro-into-abusing-and-identifying-wmi-event-subscriptions-for-persistence/
+- https://github.com/trustedsec/SysmonCommunityGuide/blob/master/chapters/WMI-events.md
+- https://in.security/2019/04/03/an-intro-into-abusing-and-identifying-wmi-event-subscriptions-for-persistence/
 tags:
   analytic_story:
   - Suspicious WMI Use
diff --git a/detections/endpoint/disable_show_hidden_files.yml b/detections/endpoint/disable_show_hidden_files.yml
index 4b38910f2a..a11229d9b5 100644
--- a/detections/endpoint/disable_show_hidden_files.yml
+++ b/detections/endpoint/disable_show_hidden_files.yml
@@ -31,7 +31,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
   that this registry was included in your config files ex. sysmon config to be monitored.
 known_false_positives: unknown
 references:
-- https://www.sophos.com/en-us/threat-center/threat-analyses/viruses-and-spyware/W32~Tiotua-P/detailed-analysis.aspx
+- https://www.sophos.com/en-us/threat-center/threat-analyses/viruses-and-spyware/W32~Tiotua-P/detailed-analysis
 tags:
   analytic_story:
   - Windows Defense Evasion Tactics
diff --git a/detections/endpoint/disabling_firewall_with_netsh.yml b/detections/endpoint/disabling_firewall_with_netsh.yml
index 0354fb1152..64d2057372 100644
--- a/detections/endpoint/disabling_firewall_with_netsh.yml
+++ b/detections/endpoint/disabling_firewall_with_netsh.yml
@@ -23,7 +23,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: admin may disable firewall during testing or fixing network
   problem.
 references:
-- https://tccontre.blogspot.com/2020/01/remcos-rat-evading-windows-defender-av.htm
+- https://tccontre.blogspot.com/2020/01/remcos-rat-evading-windows-defender-av.html
 tags:
   analytic_story:
   - Windows Defense Evasion Tactics
diff --git a/detections/endpoint/dns_exfiltration_using_nslookup_app.yml b/detections/endpoint/dns_exfiltration_using_nslookup_app.yml
index ea24ac1917..48d6aa3ef3 100644
--- a/detections/endpoint/dns_exfiltration_using_nslookup_app.yml
+++ b/detections/endpoint/dns_exfiltration_using_nslookup_app.yml
@@ -26,8 +26,8 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   Sysmon TA. Tune and filter known instances of nslookup.exe may be used.
 known_false_positives: admin nslookup usage
 references:
-- https://www.fireeye.com/blog/threat-research/2017/03/fin7_spear_phishing.html
-- https://www.varonis.com/blog/dns-tunneling/
+- https://www.mandiant.com/resources/fin7-spear-phishing-campaign-targets-personnel-involved-sec-filings
+- https://www.varonis.com/blog/dns-tunneling
 - https://www.microsoft.com/security/blog/2021/01/20/deep-dive-into-the-solorigate-second-stage-activation-from-sunburst-to-teardrop-and-raindrop/
 tags:
   analytic_story:
diff --git a/detections/endpoint/excessive_usage_of_nslookup_app.yml b/detections/endpoint/excessive_usage_of_nslookup_app.yml
index 2a9fc6e0b7..03fd45e04d 100644
--- a/detections/endpoint/excessive_usage_of_nslookup_app.yml
+++ b/detections/endpoint/excessive_usage_of_nslookup_app.yml
@@ -24,8 +24,8 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   Sysmon TA. Tune and filter known instances of nslookup.exe may be used.
 known_false_positives: unknown
 references:
-- https://www.fireeye.com/blog/threat-research/2017/03/fin7_spear_phishing.html
-- https://www.varonis.com/blog/dns-tunneling/
+- https://www.mandiant.com/resources/fin7-spear-phishing-campaign-targets-personnel-involved-sec-filings
+- https://www.varonis.com/blog/dns-tunneling
 - https://www.microsoft.com/security/blog/2021/01/20/deep-dive-into-the-solorigate-second-stage-activation-from-sunburst-to-teardrop-and-raindrop/
 tags:
   analytic_story:
diff --git a/detections/endpoint/executable_file_written_in_administrative_smb_share.yml b/detections/endpoint/executable_file_written_in_administrative_smb_share.yml
index e8c96f3fb8..288d0e71c6 100644
--- a/detections/endpoint/executable_file_written_in_administrative_smb_share.yml
+++ b/detections/endpoint/executable_file_written_in_administrative_smb_share.yml
@@ -27,7 +27,7 @@ references:
 - https://attack.mitre.org/techniques/T1021/002/
 - https://www.rapid7.com/blog/post/2013/03/09/psexec-demystified/
 - https://labs.vipre.com/trickbot-and-its-modules/
-- https://blog.whitehat.eu/2019/05/incident-trickbot-ryuk-2.html
+- https://whitehat.eu/incident-response-case-study-featuring-ryuk-and-trickbot-part-2/
 tags:
   analytic_story:
   - Data Destruction
diff --git a/detections/endpoint/kerberoasting_spn_request_with_rc4_encryption.yml b/detections/endpoint/kerberoasting_spn_request_with_rc4_encryption.yml
index e066626974..4ef578f990 100644
--- a/detections/endpoint/kerberoasting_spn_request_with_rc4_encryption.yml
+++ b/detections/endpoint/kerberoasting_spn_request_with_rc4_encryption.yml
@@ -24,7 +24,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
 known_false_positives: Older systems that support kerberos RC4 by default like NetApp
   may generate false positives. Filter as needed
 references:
-- https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1208/T1208.md
+- https://github.com/redcanaryco/atomic-red-team/blob/4e3e9c8096dde00639a6b98845ec349135554ed5/atomics/T1208/T1208.md
 - https://www.trimarcsecurity.com/post/trimarcresearch-detecting-kerberoasting-activity
 tags:
   analytic_story:
diff --git a/detections/endpoint/nltest_domain_trust_discovery.yml b/detections/endpoint/nltest_domain_trust_discovery.yml
index 26070b6bfe..45919d38b8 100644
--- a/detections/endpoint/nltest_domain_trust_discovery.yml
+++ b/detections/endpoint/nltest_domain_trust_discovery.yml
@@ -25,7 +25,7 @@ references:
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1482/T1482.md
 - https://malware.news/t/lets-learn-trickbot-implements-network-collector-module-leveraging-cmd-wmi-ldap/19104
 - https://attack.mitre.org/techniques/T1482/
-- https://www.owasp.org/images/4/4b/Red_Team_Operating_in_a_Modern_Environment.pdf
+- https://owasp.org/www-pdf-archive/Red_Team_Operating_in_a_Modern_Environment.pdf
 - https://ss64.com/nt/nltest.html
 - https://redcanary.com/threat-detection-report/techniques/domain-trust-discovery/
 - https://thedfirreport.com/2020/10/08/ryuks-return/
diff --git a/detections/endpoint/possible_lateral_movement_powershell_spawn.yml b/detections/endpoint/possible_lateral_movement_powershell_spawn.yml
index 8eae0df7bf..94fd63e776 100644
--- a/detections/endpoint/possible_lateral_movement_powershell_spawn.yml
+++ b/detections/endpoint/possible_lateral_movement_powershell_spawn.yml
@@ -33,7 +33,7 @@ references:
 - https://attack.mitre.org/techniques/T1021/003/
 - https://attack.mitre.org/techniques/T1021/006/
 - https://attack.mitre.org/techniques/T1047/
-- https://attack.mitre.org/techniques/T1053.005/
+- https://attack.mitre.org/techniques/T1053/005/
 - https://attack.mitre.org/techniques/T1543/003/
 tags:
   analytic_story:
diff --git a/detections/endpoint/powershell_using_memory_as_backing_store.yml b/detections/endpoint/powershell_using_memory_as_backing_store.yml
index 16e253edc3..544055d93a 100644
--- a/detections/endpoint/powershell_using_memory_as_backing_store.yml
+++ b/detections/endpoint/powershell_using_memory_as_backing_store.yml
@@ -20,7 +20,7 @@ how_to_implement: To successfully implement this analytic, you will need to enab
 known_false_positives: powershell may used this function to store out object into
   memory.
 references:
-- https://www.carbonblack.com/blog/decoding-malicious-powershell-streams/
+- https://web.archive.org/web/20201112031711/https://www.carbonblack.com/blog/decoding-malicious-powershell-streams/
 - https://docs.splunk.com/Documentation/UBA/5.0.4.1/GetDataIn/AddPowerShell#Configure_module_logging_for_PowerShell.
 - https://blog.palantir.com/tampering-with-windows-event-tracing-background-offense-and-defense-4be7ac62ac63
 - https://static1.squarespace.com/static/552092d5e4b0661088167e5c/t/59c1814829f18782e24f1fe2/1505853768977/Windows+PowerShell+Logging+Cheat+Sheet+ver+Sept+2017+v2.1.pdf
diff --git a/detections/endpoint/schedule_task_with_rundll32_command_trigger.yml b/detections/endpoint/schedule_task_with_rundll32_command_trigger.yml
index 85cfe1293e..342bdbb973 100644
--- a/detections/endpoint/schedule_task_with_rundll32_command_trigger.yml
+++ b/detections/endpoint/schedule_task_with_rundll32_command_trigger.yml
@@ -30,7 +30,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
 known_false_positives: unknown
 references:
 - https://labs.vipre.com/trickbot-and-its-modules/
-- https://blog.whitehat.eu/2019/05/incident-trickbot-ryuk-2.html
+- https://whitehat.eu/incident-response-case-study-featuring-ryuk-and-trickbot-part-2/
 tags:
   analytic_story:
   - Windows Persistence Techniques
diff --git a/detections/endpoint/serviceprincipalnames_discovery_with_powershell.yml b/detections/endpoint/serviceprincipalnames_discovery_with_powershell.yml
index e928600d1e..78ed3308a1 100644
--- a/detections/endpoint/serviceprincipalnames_discovery_with_powershell.yml
+++ b/detections/endpoint/serviceprincipalnames_discovery_with_powershell.yml
@@ -37,7 +37,7 @@ references:
 - https://strontic.github.io/xcyclopedia/library/setspn.exe-5C184D581524245DAD7A0A02B51FD2C2.html
 - https://attack.mitre.org/techniques/T1558/003/
 - https://social.technet.microsoft.com/wiki/contents/articles/717.service-principal-names-spn-setspn-syntax.aspx
-- https://www.harmj0y.net/blog/powershell/kerberoasting-without-mimikatz/
+- https://web.archive.org/web/20220212163642/https://www.harmj0y.net/blog/powershell/kerberoasting-without-mimikatz/
 - https://blog.zsec.uk/paving-2-da-wholeset/
 - https://msitpros.com/?p=3113
 - https://adsecurity.org/?p=3466
diff --git a/detections/endpoint/serviceprincipalnames_discovery_with_setspn.yml b/detections/endpoint/serviceprincipalnames_discovery_with_setspn.yml
index 895d40e049..05e43bedb0 100644
--- a/detections/endpoint/serviceprincipalnames_discovery_with_setspn.yml
+++ b/detections/endpoint/serviceprincipalnames_discovery_with_setspn.yml
@@ -49,7 +49,7 @@ references:
 - https://strontic.github.io/xcyclopedia/library/setspn.exe-5C184D581524245DAD7A0A02B51FD2C2.html
 - https://attack.mitre.org/techniques/T1558/003/
 - https://social.technet.microsoft.com/wiki/contents/articles/717.service-principal-names-spn-setspn-syntax.aspx
-- https://www.harmj0y.net/blog/powershell/kerberoasting-without-mimikatz/
+- https://web.archive.org/web/20220212163642/https://www.harmj0y.net/blog/powershell/kerberoasting-without-mimikatz/
 - https://blog.zsec.uk/paving-2-da-wholeset/
 - https://msitpros.com/?p=3113
 - https://adsecurity.org/?p=3466
diff --git a/detections/endpoint/ssa___dns_exfiltration_using_nslookup_app.yml b/detections/endpoint/ssa___dns_exfiltration_using_nslookup_app.yml
index 6463d7e1a0..6948bda87f 100644
--- a/detections/endpoint/ssa___dns_exfiltration_using_nslookup_app.yml
+++ b/detections/endpoint/ssa___dns_exfiltration_using_nslookup_app.yml
@@ -32,8 +32,8 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: It is possible for some legitimate administrative utilities
   to use similar cmd_line parameters. Filter as needed.
 references:
-- https://www.fireeye.com/blog/threat-research/2017/03/fin7_spear_phishing.html
-- https://www.varonis.com/blog/dns-tunneling/
+- https://www.mandiant.com/resources/fin7-spear-phishing-campaign-targets-personnel-involved-sec-filings
+- https://www.varonis.com/blog/dns-tunneling
 - https://www.microsoft.com/security/blog/2021/01/20/deep-dive-into-the-solorigate-second-stage-activation-from-sunburst-to-teardrop-and-raindrop/
 tags:
   analytic_story:
diff --git a/detections/endpoint/ssa___windows_certutil_urlcache_download.yml b/detections/endpoint/ssa___windows_certutil_urlcache_download.yml
index 1cc747b227..9559de6e58 100644
--- a/detections/endpoint/ssa___windows_certutil_urlcache_download.yml
+++ b/detections/endpoint/ssa___windows_certutil_urlcache_download.yml
@@ -34,7 +34,7 @@ known_false_positives: Limited false positives in most environments, however tun
 references:
 - https://attack.mitre.org/techniques/T1105/
 - https://www.avira.com/en/blog/certutil-abused-by-attackers-to-spread-threats
-- https://www.fireeye.com/blog/threat-research/2019/10/certutil-qualms-they-came-to-drop-fombs.html
+- https://web.archive.org/web/20210921110637/https://www.fireeye.com/blog/threat-research/2019/10/certutil-qualms-they-came-to-drop-fombs.html
 tags:
   analytic_story:
   - Ingress Tool Transfer
diff --git a/detections/endpoint/system_information_discovery_detection.yml b/detections/endpoint/system_information_discovery_detection.yml
index fb110f5366..fe8728e82d 100644
--- a/detections/endpoint/system_information_discovery_detection.yml
+++ b/detections/endpoint/system_information_discovery_detection.yml
@@ -21,7 +21,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
   your endpoints into the `Endpoint` datamodel in the `Processes` node.
 known_false_positives: Administrators debugging servers
 references:
-- https://oscp.infosecsanyam.in/priv-escalation/windows-priv-escalation
+- https://web.archive.org/web/20210119205146/https://oscp.infosecsanyam.in/priv-escalation/windows-priv-escalation
 tags:
   analytic_story:
   - Discovery Techniques
diff --git a/detections/endpoint/trickbot_named_pipe.yml b/detections/endpoint/trickbot_named_pipe.yml
index 547a6d02b8..292aa13b47 100644
--- a/detections/endpoint/trickbot_named_pipe.yml
+++ b/detections/endpoint/trickbot_named_pipe.yml
@@ -19,7 +19,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
 known_false_positives: unknown
 references:
 - https://labs.vipre.com/trickbot-and-its-modules/
-- https://blog.whitehat.eu/2019/05/incident-trickbot-ryuk-2.html
+- https://whitehat.eu/incident-response-case-study-featuring-ryuk-and-trickbot-part-2/
 tags:
   analytic_story:
   - Trickbot
diff --git a/detections/endpoint/wermgr_process_connecting_to_ip_check_web_services.yml b/detections/endpoint/wermgr_process_connecting_to_ip_check_web_services.yml
index 7df34f9c2c..ab1e9a6d0e 100644
--- a/detections/endpoint/wermgr_process_connecting_to_ip_check_web_services.yml
+++ b/detections/endpoint/wermgr_process_connecting_to_ip_check_web_services.yml
@@ -27,7 +27,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
 known_false_positives: unknown
 references:
 - https://labs.vipre.com/trickbot-and-its-modules/
-- https://blog.whitehat.eu/2019/05/incident-trickbot-ryuk-2.html
+- https://whitehat.eu/incident-response-case-study-featuring-ryuk-and-trickbot-part-2/
 tags:
   analytic_story:
   - Trickbot
diff --git a/detections/endpoint/wermgr_process_create_executable_file.yml b/detections/endpoint/wermgr_process_create_executable_file.yml
index 0a4ae33869..1f6e626961 100644
--- a/detections/endpoint/wermgr_process_create_executable_file.yml
+++ b/detections/endpoint/wermgr_process_create_executable_file.yml
@@ -23,7 +23,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
 known_false_positives: unknown
 references:
 - https://labs.vipre.com/trickbot-and-its-modules/
-- https://blog.whitehat.eu/2019/05/incident-trickbot-ryuk-2.html
+- https://whitehat.eu/incident-response-case-study-featuring-ryuk-and-trickbot-part-2/
 tags:
   analytic_story:
   - Trickbot
diff --git a/detections/endpoint/wermgr_process_spawned_cmd_or_powershell_process.yml b/detections/endpoint/wermgr_process_spawned_cmd_or_powershell_process.yml
index 27e26fb8c8..19baae8d94 100644
--- a/detections/endpoint/wermgr_process_spawned_cmd_or_powershell_process.yml
+++ b/detections/endpoint/wermgr_process_spawned_cmd_or_powershell_process.yml
@@ -25,7 +25,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: unknown
 references:
 - https://labs.vipre.com/trickbot-and-its-modules/
-- https://blog.whitehat.eu/2019/05/incident-trickbot-ryuk-2.html
+- https://whitehat.eu/incident-response-case-study-featuring-ryuk-and-trickbot-part-2/
 tags:
   analytic_story:
   - Trickbot
diff --git a/detections/endpoint/wmi_permanent_event_subscription___sysmon.yml b/detections/endpoint/wmi_permanent_event_subscription___sysmon.yml
index ce9c28550f..a218df1b1b 100644
--- a/detections/endpoint/wmi_permanent_event_subscription___sysmon.yml
+++ b/detections/endpoint/wmi_permanent_event_subscription___sysmon.yml
@@ -37,8 +37,8 @@ known_false_positives: Although unlikely, administrators may use event subscript
 references:
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1546.003/T1546.003.md
 - https://www.eideon.com/2018-03-02-THL03-WMIBackdoors/
-- https://github.com/trustedsec/SysmonCommunityGuide/blob/master/WMI-events.md
-- https://in.security/an-intro-into-abusing-and-identifying-wmi-event-subscriptions-for-persistence/
+- https://github.com/trustedsec/SysmonCommunityGuide/blob/master/chapters/WMI-events.md
+- https://in.security/2019/04/03/an-intro-into-abusing-and-identifying-wmi-event-subscriptions-for-persistence/
 tags:
   analytic_story:
   - Suspicious WMI Use
diff --git a/detections/endpoint/wmi_recon_running_process_or_services.yml b/detections/endpoint/wmi_recon_running_process_or_services.yml
index bd73f3bf21..269b959417 100644
--- a/detections/endpoint/wmi_recon_running_process_or_services.yml
+++ b/detections/endpoint/wmi_recon_running_process_or_services.yml
@@ -22,8 +22,8 @@ known_false_positives: network administrator may used this command for checking
 references:
 - https://news.sophos.com/en-us/2020/05/12/maze-ransomware-1-year-counting/
 - https://www.eideon.com/2018-03-02-THL03-WMIBackdoors/
-- https://github.com/trustedsec/SysmonCommunityGuide/blob/master/WMI-events.md
-- https://in.security/an-intro-into-abusing-and-identifying-wmi-event-subscriptions-for-persistence/
+- https://github.com/trustedsec/SysmonCommunityGuide/blob/master/chapters/WMI-events.md
+- https://in.security/2019/04/03/an-intro-into-abusing-and-identifying-wmi-event-subscriptions-for-persistence/
 tags:
   analytic_story:
   - Hermetic Wiper 
diff --git a/detections/experimental/endpoint/winrm_spawning_a_process.yml b/detections/experimental/endpoint/winrm_spawning_a_process.yml
index 69f91b9d8a..520c21f52d 100644
--- a/detections/experimental/endpoint/winrm_spawning_a_process.yml
+++ b/detections/experimental/endpoint/winrm_spawning_a_process.yml
@@ -24,7 +24,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: Unknown. Add new processes or filter as needed. It is possible
   system management software may spawn processes from `wsmprovhost.exe`.
 references:
-- https://github.com/SigmaHQ/sigma/blob/master/rules/windows/process_access/win_susp_shell_spawn_from_winrm.yml
+- https://github.com/SigmaHQ/sigma/blob/9b7fb0c0f3af2e53ed483e29e0d0f88ccf1c08ca/rules/windows/process_access/win_susp_shell_spawn_from_winrm.yml
 - https://www.zerodayinitiative.com/blog/2021/5/17/cve-2021-31166-a-wormable-code-execution-bug-in-httpsys
 - https://github.com/0vercl0k/CVE-2021-31166/blob/main/cve-2021-31166.py
 tags:

From bc7a22ae4a1b855d549a0c95809673d19c71c365 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 3 May 2022 14:21:30 -0700
Subject: [PATCH 10/38] Fixed reference links in two more detection YMLs

---
 .../endpoint/kerberoasting_spn_request_with_rc4_encryption.yml  | 2 +-
 ...s_disable_windows_group_policy_features_through_registry.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/detections/endpoint/kerberoasting_spn_request_with_rc4_encryption.yml b/detections/endpoint/kerberoasting_spn_request_with_rc4_encryption.yml
index 4ef578f990..a8f708c1d4 100644
--- a/detections/endpoint/kerberoasting_spn_request_with_rc4_encryption.yml
+++ b/detections/endpoint/kerberoasting_spn_request_with_rc4_encryption.yml
@@ -25,7 +25,7 @@ known_false_positives: Older systems that support kerberos RC4 by default like N
   may generate false positives. Filter as needed
 references:
 - https://github.com/redcanaryco/atomic-red-team/blob/4e3e9c8096dde00639a6b98845ec349135554ed5/atomics/T1208/T1208.md
-- https://www.trimarcsecurity.com/post/trimarcresearch-detecting-kerberoasting-activity
+- https://www.hub.trimarcsecurity.com/post/trimarc-research-detecting-kerberoasting-activity
 tags:
   analytic_story:
   - Windows Privilege Escalation
diff --git a/detections/endpoint/windows_disable_windows_group_policy_features_through_registry.yml b/detections/endpoint/windows_disable_windows_group_policy_features_through_registry.yml
index 1dcf8918c4..ac3844dbc3 100644
--- a/detections/endpoint/windows_disable_windows_group_policy_features_through_registry.yml
+++ b/detections/endpoint/windows_disable_windows_group_policy_features_through_registry.yml
@@ -32,7 +32,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: unknown
 references:
 - https://hybrid-analysis.com/sample/ef1c427394c205580576d18ba68d5911089c7da0386f19d1ca126929d3e671ab?environmentId=120&lang=en
-- https://www.sophos.com/de-de/threat-center/threat-analyses/viruses-and-spyware/Troj~Krotten-N/detailed-analysis
+- https://www.sophos.com/en-us/threat-center/threat-analyses/viruses-and-spyware/Troj~Krotten-N/detailed-analysis
 - https://www.virustotal.com/gui/file/2d7855bf6470aa323edf2949b54ce2a04d9e38770f1322c3d0420c2303178d91/details
 tags:
   analytic_story:

From ee8c0b9bb65e312d4a8cf001768bcc46536e9b48 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 3 May 2022 16:17:41 -0700
Subject: [PATCH 11/38] Fixed the remaining broken or outdated references.  The
 remaining references that fail validation cannot be easily validated by
 requests, due to website requirements, or sometimes return errors or take a
 long time to load.  Becuase they usually work, we will leave them as-is.

---
 detections/endpoint/dsquery_domain_discovery.yml                | 2 +-
 detections/endpoint/get_domaintrust_with_powershell.yml         | 2 +-
 .../endpoint/get_domaintrust_with_powershell_script_block.yml   | 2 +-
 .../impacket_lateral_movement_commandline_parameters.yml        | 2 +-
 detections/endpoint/rubeus_command_line_parameters.yml          | 2 +-
 .../rubeus_kerberos_ticket_exports_through_winlogon_access.yml  | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/detections/endpoint/dsquery_domain_discovery.yml b/detections/endpoint/dsquery_domain_discovery.yml
index c88919c256..6d3d833e1a 100644
--- a/detections/endpoint/dsquery_domain_discovery.yml
+++ b/detections/endpoint/dsquery_domain_discovery.yml
@@ -40,7 +40,7 @@ known_false_positives: Limited false positives. If there is a true false positiv
   filter based on command-line or parent process.
 references:
 - https://github.com/redcanaryco/atomic-red-team/blob/master/atomics/T1482/T1482.md
-- http://www.harmj0y.net/blog/redteaming/a-guide-to-attacking-domain-trusts/
+- https://blog.harmj0y.net/redteaming/a-guide-to-attacking-domain-trusts/
 - https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2012-R2-and-2012/cc732952(v=ws.11)
 - https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2012-R2-and-2012/cc754232(v=ws.11)
 tags:
diff --git a/detections/endpoint/get_domaintrust_with_powershell.yml b/detections/endpoint/get_domaintrust_with_powershell.yml
index 6a8ad0c082..f129d1717d 100644
--- a/detections/endpoint/get_domaintrust_with_powershell.yml
+++ b/detections/endpoint/get_domaintrust_with_powershell.yml
@@ -25,7 +25,7 @@ how_to_implement: To successfully implement this search you need to be ingesting
 known_false_positives: Limited false positives as this requires an active Administrator
   or adversary to bring in, import, and execute.
 references:
-- http://www.harmj0y.net/blog/redteaming/a-guide-to-attacking-domain-trusts/
+- https://blog.harmj0y.net/redteaming/a-guide-to-attacking-domain-trusts/
 tags:
   analytic_story:
   - Active Directory Discovery
diff --git a/detections/endpoint/get_domaintrust_with_powershell_script_block.yml b/detections/endpoint/get_domaintrust_with_powershell_script_block.yml
index 910ee5d880..f90d423fed 100644
--- a/detections/endpoint/get_domaintrust_with_powershell_script_block.yml
+++ b/detections/endpoint/get_domaintrust_with_powershell_script_block.yml
@@ -26,7 +26,7 @@ how_to_implement: To successfully implement this analytic, you will need to enab
 known_false_positives: It is possible certain system management frameworks utilize
   this command to gather trust information.
 references:
-- http://www.harmj0y.net/blog/redteaming/a-guide-to-attacking-domain-trusts/
+- https://blog.harmj0y.net/redteaming/a-guide-to-attacking-domain-trusts/
 - https://docs.splunk.com/Documentation/UBA/5.0.4.1/GetDataIn/AddPowerShell#Configure_module_logging_for_PowerShell.
 - https://blog.palantir.com/tampering-with-windows-event-tracing-background-offense-and-defense-4be7ac62ac63
 - https://static1.squarespace.com/static/552092d5e4b0661088167e5c/t/59c1814829f18782e24f1fe2/1505853768977/Windows+PowerShell+Logging+Cheat+Sheet+ver+Sept+2017+v2.1.pdf
diff --git a/detections/endpoint/impacket_lateral_movement_commandline_parameters.yml b/detections/endpoint/impacket_lateral_movement_commandline_parameters.yml
index 2862b783a3..0ef2da8938 100644
--- a/detections/endpoint/impacket_lateral_movement_commandline_parameters.yml
+++ b/detections/endpoint/impacket_lateral_movement_commandline_parameters.yml
@@ -31,7 +31,7 @@ references:
 - https://attack.mitre.org/techniques/T1021/003/
 - https://attack.mitre.org/techniques/T1047/
 - https://attack.mitre.org/techniques/T1053/
-- /
+- https://attack.mitre.org/techniques/T1053/005/
 - https://github.com/SecureAuthCorp/impacket
 - https://vk9-sec.com/impacket-remote-code-execution-rce-on-windows-from-linux/
 - https://www.microsoft.com/security/blog/2022/01/15/destructive-malware-targeting-ukrainian-organizations/
diff --git a/detections/endpoint/rubeus_command_line_parameters.yml b/detections/endpoint/rubeus_command_line_parameters.yml
index fc1f7a21f5..6e99edde4c 100644
--- a/detections/endpoint/rubeus_command_line_parameters.yml
+++ b/detections/endpoint/rubeus_command_line_parameters.yml
@@ -34,7 +34,7 @@ known_false_positives: Although unlikely, legitimate applications may use the sa
   command line parameters as Rubeus. Filter as needed.
 references:
 - https://github.com/GhostPack/Rubeus
-- http://www.harmj0y.net/blog/redteaming/from-kekeo-to-rubeus/
+- https://web.archive.org/web/20210725005734/http://www.harmj0y.net/blog/redteaming/from-kekeo-to-rubeus/
 - https://attack.mitre.org/techniques/T1550/003/
 - https://en.hackndo.com/kerberos-silver-golden-tickets/
 tags:
diff --git a/detections/endpoint/rubeus_kerberos_ticket_exports_through_winlogon_access.yml b/detections/endpoint/rubeus_kerberos_ticket_exports_through_winlogon_access.yml
index ce3cf7ae65..5e8df260a9 100644
--- a/detections/endpoint/rubeus_kerberos_ticket_exports_through_winlogon_access.yml
+++ b/detections/endpoint/rubeus_kerberos_ticket_exports_through_winlogon_access.yml
@@ -31,7 +31,7 @@ known_false_positives: Legitimate applications may obtain a handle for winlogon.
   Filter as needed
 references:
 - https://github.com/GhostPack/Rubeus
-- http://www.harmj0y.net/blog/redteaming/from-kekeo-to-rubeus/
+- https://web.archive.org/web/20210725005734/http://www.harmj0y.net/blog/redteaming/from-kekeo-to-rubeus/
 - https://attack.mitre.org/techniques/T1550/003/
 tags:
   analytic_story:

From 0957d0de030be069d6ef32b426a1de484ba23029 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 3 May 2022 16:24:29 -0700
Subject: [PATCH 12/38] Updates to the detection ref validator. Still needs a
 few tweaks.

---
 .../domain/entities/detection.py              | 32 +++++++++++++++----
 1 file changed, 25 insertions(+), 7 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/domain/entities/detection.py b/bin/contentctl_project/contentctl_core/domain/entities/detection.py
index 9604a43ffd..96a55804c8 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/detection.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/detection.py
@@ -3,10 +3,13 @@
 import string
 import re
 import requests
-
+import time
 from pydantic import BaseModel, validator, root_validator
 from dataclasses import dataclass
 from datetime import datetime
+import urllib3, urllib3.exceptions
+
+
 
 from bin.contentctl_project.contentctl_core.domain.entities.security_content_object import SecurityContentObject
 from bin.contentctl_project.contentctl_core.domain.entities.enums.enums import AnalyticsType
@@ -22,11 +25,18 @@
 
 
 import functools
-DEFAULT_CHECK_REFERENCE_TIMEOUT_SECONDS = 60
+DEFAULT_CHECK_REFERENCE_TIMEOUT_SECONDS = 20
 DEFAULT_USER_AGENT_STRING = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36"
+VERIFY_SSL = False
+if VERIFY_SSL is False:
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+count_not_cached = 0
+total_count = 0
 @functools.cache
 def check_reference(reference: str, timeout_seconds = DEFAULT_CHECK_REFERENCE_TIMEOUT_SECONDS, user_agent: str = DEFAULT_USER_AGENT_STRING)-> requests.Response:
-    get = requests.get(reference, timeout=timeout_seconds, headers = {"User-Agent": user_agent, "referrer": "https://google.com"}, allow_redirects=True)
+    global count_not_cached
+    count_not_cached+=1
+    get = requests.get(reference, timeout=timeout_seconds, headers = {"User-Agent": user_agent}, allow_redirects=True, verify=VERIFY_SSL)
     return get
 
 class Detection(BaseModel, SecurityContentObject):
@@ -138,16 +148,22 @@ def name_max_length(cls, values):
 
     @validator('references')
     def references_check(cls, v, values):
+        global total_count
+        #return v
         #import traceback
         #import sys
         #traceback.print_stack()
         #sys.exit(0)
         for reference in v:
+            #Get the start time of the request so that, if it fails, we can print out how long we waited
+            start_time = time.time()
+            total_count += 1
             try:
                 get = check_reference(reference)
                 if get.status_code == 200:
                     if reference != get.url:
-                        print(f"Redirect from {reference} to {get.url}") #follows all redirects    
+                        #print(f"Redirect from {reference} to {get.url}") #follows all redirects    
+                        pass
                     continue
                 elif get.status_code == 301:
                     #print(f"Redirect from {reference} to {get.url}")
@@ -155,11 +171,13 @@ def references_check(cls, v, values):
                     #and will change across calls.  Don't throw an error on this.
                     continue
                 #If we get here, it means it was not a 200 response code. Raise an error
-                raise ValueError('Reference ' + str(get.status_code) + ' ' + reference + ' is not reachable: ' + values["name"])
+                raise ValueError(f"Reference {reference} returned status code {get.status_code} after {round(time.time()-start_time)} seconds: {values['name']}")
 
             except requests.exceptions.RequestException as e:
-                raise ValueError('Reference ' + reference + ' is not reachable: ' + values["name"])
-                #raise ValueError('Reference ' + reference + ' is not reachable: ' + values["name"])
+                raise ValueError(f"Reference {reference} is not reachable after {round(time.time()-start_time)} seconds: {values['name']}")
+                
+            finally:
+                print(f"Request [{count_not_cached} / {total_count} - cache ratio {(1 - count_not_cached/total_count):.3f}]")
 
         return v
 

From c0dc8541b44be326090a4b13eb33e86a385587e1 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 4 May 2022 16:22:40 -0700
Subject: [PATCH 13/38] Added a class for validating links.  This class
 performs its own type of caching, similar to functools caching, but keeps
 additional metrics suitable for dumping when all validation has completed.

---
 .../domain/entities/link_validator.py         | 109 ++++++++++++++++++
 1 file changed, 109 insertions(+)
 create mode 100644 bin/contentctl_project/contentctl_core/domain/entities/link_validator.py

diff --git a/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
new file mode 100644
index 0000000000..bbe49628f7
--- /dev/null
+++ b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
@@ -0,0 +1,109 @@
+from lib2to3.pytree import Base
+import re
+from tracemalloc import start
+from unittest.mock import DEFAULT
+from pydantic import BaseModel, validator, root_validator,Field
+from typing import Union, Callable
+import requests
+import urllib3, urllib3.exceptions
+import time
+import abc
+
+DEFAULT_USER_AGENT_STRING = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36"
+class LinkStats(BaseModel):
+    #Static Values
+    method: Callable = requests.get
+    allowed_http_codes: list[int] = [200] 
+    access_count: int = 0
+    timeout_seconds: int = 30
+    allow_redirects: bool = True
+    headers: dict = {"User-Agent": DEFAULT_USER_AGENT_STRING}
+    verify_ssl: bool = False
+    if verify_ssl is False:
+        urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+    
+    #Values generated at runtime.
+    #We need to assign these some default values to get the 
+    #validation working since ComputedField has not yet been 
+    #introduced to Pydantic
+    reference: str
+    redirect: Union[str,None] = None
+    status_code: int = 0
+    valid: bool = False
+    resolution_time: float = 0 
+    
+    
+    def is_link_valid(self)->bool:
+        self.access_count += 1
+        return self.valid
+    
+    @root_validator
+    def check_reference(cls, values):
+        start_time = time.time()
+        #Get out all the fields names to make them easier to reference
+        method = values['method']
+        reference = values['reference']
+        timeout_seconds = values['timeout_seconds']
+        headers = values['headers']
+        allow_redirects = values['allow_redirects']
+        verify_ssl = values['verify_ssl']
+        allowed_http_codes = values['allowed_http_codes']
+        if not (reference.startswith("http://") or reference.startswith("https://")):
+            raise(ValueError(f"Reference {reference} does not begin with http(s). Only http(s) references are supported"))
+        
+        try:
+            print(f"timeout seconds is {timeout_seconds}")
+            get = method(reference, timeout=timeout_seconds, 
+                            headers = headers, 
+                            allow_redirects=allow_redirects, verify=verify_ssl)
+            resolution_time = time.time() - start_time
+
+            
+            values['status_code'] = get.status_code
+            values['resolution_time'] = resolution_time
+            if reference != get.url:
+                values['redirect'] = get.url
+            else:
+                values['redirect'] = None #None is also already the default
+
+            #Returns the updated values and sets them for the object
+            if get.status_code in allowed_http_codes:
+                values['valid'] = True
+            else:
+                print(f"Unacceptable HTTP Status Code {get.status_code} received for {reference}")
+                values['valid'] = False
+            return values    
+
+        except Exception as e:
+            resolution_time = time.time() - start_time
+            print(f"Reference {reference} was not reachable after {resolution_time:.2f} seconds")
+            values['status_code'] = 0
+            values['valid'] = False
+            values['redirect'] = None
+            values['resolution_time'] = resolution_time
+            return values
+
+
+class LinkValidator(abc.ABC):
+    cache: dict[str,LinkStats] = {}
+    #cache: dict[str,LinkStats] = {}
+
+    
+    @staticmethod
+    def validate_reference(reference: str, raise_exception_if_failure: bool = False) -> bool:
+        
+        if reference not in LinkValidator.cache:
+            LinkValidator.cache[reference] = LinkStats(reference=reference)
+        result = LinkValidator.cache[reference].is_link_valid()
+        if result is True:
+            return True
+        elif raise_exception_if_failure is True:
+            raise(Exception(f"Reference Link Failed: {reference}"))
+        else:
+            return False
+        
+
+
+
+
+

From 54798186a9d8867ea6f6aadff65e0bdcc282ec90 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 4 May 2022 16:24:02 -0700
Subject: [PATCH 14/38] Saving changes made to support URL verification which
 is working - but we have now broken these out into a more robust class
 link_validator.py

---
 .../domain/entities/detection.py              | 28 +++++++++++++++----
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/domain/entities/detection.py b/bin/contentctl_project/contentctl_core/domain/entities/detection.py
index 96a55804c8..ff83628732 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/detection.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/detection.py
@@ -21,7 +21,7 @@
 from bin.contentctl_project.contentctl_core.domain.entities.lookup import Lookup
 from bin.contentctl_project.contentctl_core.domain.entities.baseline import Baseline
 from bin.contentctl_project.contentctl_core.domain.entities.playbook import Playbook
-
+import sys
 
 
 import functools
@@ -32,11 +32,21 @@
     urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 count_not_cached = 0
 total_count = 0
+CACHER = dict()
 @functools.cache
 def check_reference(reference: str, timeout_seconds = DEFAULT_CHECK_REFERENCE_TIMEOUT_SECONDS, user_agent: str = DEFAULT_USER_AGENT_STRING)-> requests.Response:
     global count_not_cached
     count_not_cached+=1
     get = requests.get(reference, timeout=timeout_seconds, headers = {"User-Agent": user_agent}, allow_redirects=True, verify=VERIFY_SSL)
+    CACHER[reference] = get.text
+
+    totalSize = sys.getsizeof(CACHER)
+    for key in CACHER:
+        totalSize += sys.getsizeof(CACHER[key])
+                
+    print(f"Request [{count_not_cached} / {total_count} - cache ratio {(1 - count_not_cached/total_count):.3f}], CACHE: {(totalSize / (1024*1024)):.2f} MB")
+
+
     return get
 
 class Detection(BaseModel, SecurityContentObject):
@@ -52,8 +62,10 @@ class Detection(BaseModel, SecurityContentObject):
     search: str
     how_to_implement: str
     known_false_positives: str
+    check_references: bool = False #Validation is done in order, this field must be defined first
     references: list
     tags: DetectionTags
+    
 
     # enrichments
     deprecated: bool = None
@@ -143,9 +155,6 @@ def name_max_length(cls, values):
         return values
 
    
-        
-
-
     @validator('references')
     def references_check(cls, v, values):
         global total_count
@@ -154,6 +163,14 @@ def references_check(cls, v, values):
         #import sys
         #traceback.print_stack()
         #sys.exit(0)
+        
+        if 'check_references' in values and values['check_references'] is False:
+            #Reference checking is NOT enabled
+            return v
+        elif 'check_references' not in values:
+            raise(Exception("Member 'check_references' missing from Detection!"))
+        
+
         for reference in v:
             #Get the start time of the request so that, if it fails, we can print out how long we waited
             start_time = time.time()
@@ -177,8 +194,7 @@ def references_check(cls, v, values):
                 raise ValueError(f"Reference {reference} is not reachable after {round(time.time()-start_time)} seconds: {values['name']}")
                 
             finally:
-                print(f"Request [{count_not_cached} / {total_count} - cache ratio {(1 - count_not_cached/total_count):.3f}]")
-
+                pass
         return v
 
     @validator('search')

From b60db04b718902f902f643aabb4b001a80547f21 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 4 May 2022 16:50:18 -0700
Subject: [PATCH 15/38] Added link validator code to baselines, detections,
 investigations, and stories. Still some testing to be done.

---
 .../application/factory/factory.py            |  2 +-
 .../domain/entities/baseline.py               | 20 ++++---
 .../domain/entities/detection.py              | 59 ++-----------------
 .../domain/entities/investigation.py          | 20 ++++---
 .../domain/entities/link_validator.py         |  8 ++-
 .../contentctl_core/domain/entities/story.py  | 24 ++++----
 .../security_content_baseline_builder.py      |  7 +++
 .../security_content_detection_builder.py     |  8 ++-
 .../security_content_investigation_builder.py |  5 ++
 .../builder/security_content_story_builder.py |  6 ++
 contentctl.py                                 | 18 ++++--
 11 files changed, 87 insertions(+), 90 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/application/factory/factory.py b/bin/contentctl_project/contentctl_core/application/factory/factory.py
index 1beafa9362..28febb7900 100644
--- a/bin/contentctl_project/contentctl_core/application/factory/factory.py
+++ b/bin/contentctl_project/contentctl_core/application/factory/factory.py
@@ -81,7 +81,7 @@ def createSecurityContent(self, type: SecurityContentType) -> list:
 
           import copy
           import threading
-          NUM_THREADS = 10
+          NUM_THREADS = 1
           
           def make_detections_thread(index:int):
                my_dto = copy.deepcopy(self.input_dto)
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/baseline.py b/bin/contentctl_project/contentctl_core/domain/entities/baseline.py
index cd0b5ee32d..ec40824447 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/baseline.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/baseline.py
@@ -10,6 +10,7 @@
 from bin.contentctl_project.contentctl_core.domain.entities.enums.enums import DataModel
 from bin.contentctl_project.contentctl_core.domain.entities.baseline_tags import BaselineTags
 from bin.contentctl_project.contentctl_core.domain.entities.deployment import Deployment
+from bin.contentctl_project.contentctl_core.domain.entities.link_validator import LinkValidator
 
 
 class Baseline(BaseModel, SecurityContentObject):
@@ -25,6 +26,7 @@ class Baseline(BaseModel, SecurityContentObject):
     search: str
     how_to_implement: str
     known_false_positives: str
+    check_references: bool = False #Validation is done in order, this field must be defined first
     references: list
     tags: BaselineTags
 
@@ -84,15 +86,17 @@ def encode_error(cls, v, values, field):
 
     @validator('references')
     def references_check(cls, v, values):
-        for reference in v:
-            try:
-                get = requests.get(reference)
-                if not get.status_code == 200:
-                    raise ValueError('Reference ' + reference + ' is not reachable: ' + values["name"])
-            except requests.exceptions.RequestException as e:
-                raise ValueError('Reference ' + reference + ' is not reachable: ' + values["name"])
+        
+        
+        if 'check_references' in values and values['check_references'] is False:
+            #Reference checking is NOT enabled
+            return v
+        elif 'check_references' not in values:
+            raise(Exception("Member 'check_references' missing from Detection!"))
+        
 
-        return v
+        for reference in v:
+            LinkValidator.validate_reference(reference)
 
     @validator('search')
     def search_validate(cls, v, values):
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/detection.py b/bin/contentctl_project/contentctl_core/domain/entities/detection.py
index ff83628732..af07cc43be 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/detection.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/detection.py
@@ -1,13 +1,11 @@
-import enum
 import uuid
 import string
-import re
 import requests
 import time
 from pydantic import BaseModel, validator, root_validator
 from dataclasses import dataclass
 from datetime import datetime
-import urllib3, urllib3.exceptions
+
 
 
 
@@ -21,34 +19,12 @@
 from bin.contentctl_project.contentctl_core.domain.entities.lookup import Lookup
 from bin.contentctl_project.contentctl_core.domain.entities.baseline import Baseline
 from bin.contentctl_project.contentctl_core.domain.entities.playbook import Playbook
+from bin.contentctl_project.contentctl_core.domain.entities.link_validator import LinkValidator
 import sys
 
 
-import functools
-DEFAULT_CHECK_REFERENCE_TIMEOUT_SECONDS = 20
-DEFAULT_USER_AGENT_STRING = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36"
-VERIFY_SSL = False
-if VERIFY_SSL is False:
-    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-count_not_cached = 0
-total_count = 0
-CACHER = dict()
-@functools.cache
-def check_reference(reference: str, timeout_seconds = DEFAULT_CHECK_REFERENCE_TIMEOUT_SECONDS, user_agent: str = DEFAULT_USER_AGENT_STRING)-> requests.Response:
-    global count_not_cached
-    count_not_cached+=1
-    get = requests.get(reference, timeout=timeout_seconds, headers = {"User-Agent": user_agent}, allow_redirects=True, verify=VERIFY_SSL)
-    CACHER[reference] = get.text
-
-    totalSize = sys.getsizeof(CACHER)
-    for key in CACHER:
-        totalSize += sys.getsizeof(CACHER[key])
-                
-    print(f"Request [{count_not_cached} / {total_count} - cache ratio {(1 - count_not_cached/total_count):.3f}], CACHE: {(totalSize / (1024*1024)):.2f} MB")
 
 
-    return get
-
 class Detection(BaseModel, SecurityContentObject):
     # detection spec
     name: str
@@ -157,12 +133,7 @@ def name_max_length(cls, values):
    
     @validator('references')
     def references_check(cls, v, values):
-        global total_count
-        #return v
-        #import traceback
-        #import sys
-        #traceback.print_stack()
-        #sys.exit(0)
+        
         
         if 'check_references' in values and values['check_references'] is False:
             #Reference checking is NOT enabled
@@ -172,29 +143,7 @@ def references_check(cls, v, values):
         
 
         for reference in v:
-            #Get the start time of the request so that, if it fails, we can print out how long we waited
-            start_time = time.time()
-            total_count += 1
-            try:
-                get = check_reference(reference)
-                if get.status_code == 200:
-                    if reference != get.url:
-                        #print(f"Redirect from {reference} to {get.url}") #follows all redirects    
-                        pass
-                    continue
-                elif get.status_code == 301:
-                    #print(f"Redirect from {reference} to {get.url}")
-                    #Redirects here have typically added a guid or something similar to the URL which is not persistent
-                    #and will change across calls.  Don't throw an error on this.
-                    continue
-                #If we get here, it means it was not a 200 response code. Raise an error
-                raise ValueError(f"Reference {reference} returned status code {get.status_code} after {round(time.time()-start_time)} seconds: {values['name']}")
-
-            except requests.exceptions.RequestException as e:
-                raise ValueError(f"Reference {reference} is not reachable after {round(time.time()-start_time)} seconds: {values['name']}")
-                
-            finally:
-                pass
+            LinkValidator.validate_reference(reference)
         return v
 
     @validator('search')
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/investigation.py b/bin/contentctl_project/contentctl_core/domain/entities/investigation.py
index b177465927..380b568d29 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/investigation.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/investigation.py
@@ -12,6 +12,7 @@
 from bin.contentctl_project.contentctl_core.domain.entities.enums.enums import AnalyticsType
 from bin.contentctl_project.contentctl_core.domain.entities.enums.enums import DataModel
 from bin.contentctl_project.contentctl_core.domain.entities.investigation_tags import InvestigationTags
+from bin.contentctl_project.contentctl_core.domain.entities.link_validator import LinkValidator
 
 
 class Investigation(BaseModel, SecurityContentObject):
@@ -27,6 +28,7 @@ class Investigation(BaseModel, SecurityContentObject):
     search: str
     how_to_implement: str
     known_false_positives: str
+    check_references: bool = False #Validation is done in order, this field must be defined first
     references: list
     inputs: list = None
     tags: InvestigationTags
@@ -81,15 +83,17 @@ def encode_error(cls, v, values, field):
 
     @validator('references')
     def references_check(cls, v, values):
-        for reference in v:
-            try:
-                get = requests.get(reference)
-                if not get.status_code == 200:
-                    raise ValueError('Reference ' + reference + ' is not reachable: ' + values["name"])
-            except requests.exceptions.RequestException as e:
-                raise ValueError('Reference ' + reference + ' is not reachable: ' + values["name"])
+        
+        
+        if 'check_references' in values and values['check_references'] is False:
+            #Reference checking is NOT enabled
+            return v
+        elif 'check_references' not in values:
+            raise(Exception("Member 'check_references' missing from Detection!"))
+        
 
-        return v
+        for reference in v:
+            LinkValidator.validate_reference(reference)
 
     @validator('search')
     def search_validate(cls, v, values):
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
index bbe49628f7..48f6c6dd54 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
@@ -52,7 +52,6 @@ def check_reference(cls, values):
             raise(ValueError(f"Reference {reference} does not begin with http(s). Only http(s) references are supported"))
         
         try:
-            print(f"timeout seconds is {timeout_seconds}")
             get = method(reference, timeout=timeout_seconds, 
                             headers = headers, 
                             allow_redirects=allow_redirects, verify=verify_ssl)
@@ -86,13 +85,18 @@ def check_reference(cls, values):
 
 class LinkValidator(abc.ABC):
     cache: dict[str,LinkStats] = {}
+    uncached_checks: int = 0
+    total_checks: int = 0
+    
     #cache: dict[str,LinkStats] = {}
 
     
     @staticmethod
     def validate_reference(reference: str, raise_exception_if_failure: bool = False) -> bool:
-        
+        LinkValidator.total_checks += 1
+        print(f"Total Checks: {LinkValidator.total_checks}")
         if reference not in LinkValidator.cache:
+            LinkValidator.uncached_checks += 1
             LinkValidator.cache[reference] = LinkStats(reference=reference)
         result = LinkValidator.cache[reference].is_link_valid()
         if result is True:
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/story.py b/bin/contentctl_project/contentctl_core/domain/entities/story.py
index cae85af56f..e72b509ad2 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/story.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/story.py
@@ -7,6 +7,7 @@
 
 from bin.contentctl_project.contentctl_core.domain.entities.security_content_object import SecurityContentObject
 from bin.contentctl_project.contentctl_core.domain.entities.story_tags import StoryTags
+from bin.contentctl_project.contentctl_core.domain.entities.link_validator import LinkValidator
 
 class Story(BaseModel, SecurityContentObject):
     # story spec
@@ -17,6 +18,7 @@ class Story(BaseModel, SecurityContentObject):
     author: str
     description: str
     narrative: str
+    check_references: bool = False #Validation is done in order, this field must be defined first
     references: list
     tags: StoryTags
 
@@ -61,14 +63,16 @@ def encode_error(cls, v, values, field):
             raise ValueError('encoding error in ' + field.name + ': ' + values["name"])
         return v
 
-    # @validator('references')
-    # def references_check(cls, v, values):
-    #     for reference in v:
-    #         try:
-    #             get = requests.get(reference)
-    #             if not get.status_code == 200:
-    #                 raise ValueError('Reference ' + reference + ' is not reachable: ' + values["name"])
-    #         except requests.exceptions.RequestException as e:
-    #             raise ValueError('Reference ' + reference + ' is not reachable: ' + values["name"])
+      @validator('references')
+    def references_check(cls, v, values):
+        
+        
+        if 'check_references' in values and values['check_references'] is False:
+            #Reference checking is NOT enabled
+            return v
+        elif 'check_references' not in values:
+            raise(Exception("Member 'check_references' missing from Detection!"))
+        
 
-    #     return v
\ No newline at end of file
+        for reference in v:
+            LinkValidator.validate_reference(reference)
\ No newline at end of file
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_baseline_builder.py b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_baseline_builder.py
index 8dc85d55fb..62a9e243c9 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_baseline_builder.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_baseline_builder.py
@@ -11,12 +11,19 @@
 
 class SecurityContentBaselineBuilder(BaselineBuilder):
     baseline : Baseline
+    check_references: bool
+
+    def __init__(self, check_references: bool = False):
+        self.check_references = check_references
 
     def setObject(self, path: str) -> None:
         yml_dict = YmlReader.load_file(path)
         yml_dict["tags"]["name"] = yml_dict["name"]
+
+        yml_dict["check_references"] = self.check_references
         try:
             self.baseline = Baseline.parse_obj(yml_dict)
+            del(yml_dict["check_references"])
         except ValidationError as e:
             print('Validation Error for file ' + path)
             print(e)
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py
index 0453f7db21..d894e2cb6a 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py
@@ -16,13 +16,19 @@
 
 class SecurityContentDetectionBuilder(DetectionBuilder):
     security_content_obj : SecurityContentObject
-    force_cached_or_offline: bool = False
+    force_cached_or_offline: bool
+    check_references: bool
 
+    def __init__(self, force_cached_or_offline: bool = False, check_references: bool = False):
+        self.force_cached_or_offline = force_cached_or_offline
+        self.check_references = check_references
 
     def setObject(self, path: str) -> None:
         yml_dict = YmlReader.load_file(path)
         yml_dict["tags"]["name"] = yml_dict["name"]
+        yml_dict["check_references"] = self.check_references
         self.security_content_obj = Detection.parse_obj(yml_dict)
+        del(yml_dict["check_references"])
         self.security_content_obj.source = os.path.split(os.path.dirname(self.security_content_obj.file_path))[-1]      
 
 
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_investigation_builder.py b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_investigation_builder.py
index a9d5869d6b..618bc454e1 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_investigation_builder.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_investigation_builder.py
@@ -11,12 +11,17 @@
 
 class SecurityContentInvestigationBuilder(InvestigationBuilder):
     investigation: Investigation
+    check_references: bool
 
+    def __init__(self, check_references: bool = False):
+        self.check_references = check_references
 
     def setObject(self, path: str) -> None:
         yml_dict = YmlReader.load_file(path)
         try:
+            yml_dict["check_references"] = self.check_references
             self.investigation = Investigation.parse_obj(yml_dict)
+            del(yml_dict["check_references"])
         except ValidationError as e:
             print('Validation Error for file ' + path)
             print(e)
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_story_builder.py b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_story_builder.py
index 1518b69f31..cfad698d70 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_story_builder.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_story_builder.py
@@ -11,12 +11,18 @@
 
 class SecurityContentStoryBuilder(StoryBuilder):
     story: Story
+    check_references: bool
+
+    def __init__(self, check_references: bool = False):
+        self.check_references = check_references
 
     def setObject(self, path: str) -> None:
         yml_dict = YmlReader.load_file(path)
         yml_dict["tags"]["name"] = yml_dict["name"]
+        yml_dict["check_references"] = self.check_references
         try:
             self.story = Story.parse_obj(yml_dict)
+            del(yml_dict["check_references"])
         except ValidationError as e:
             print('Validation Error for file ' + path)
             print(e)
diff --git a/contentctl.py b/contentctl.py
index a760a7fb30..d95f6f4dc9 100644
--- a/contentctl.py
+++ b/contentctl.py
@@ -150,12 +150,10 @@ def validate(args) -> None:
         print("ERROR: invalid product. valid products are all, ESCU or SSA.")
         sys.exit(1)
 
-    detection_builder = SecurityContentDetectionBuilder()
-    detection_builder.force_cached_or_offline = args.cached_and_offline
     factory_input_dto = FactoryInputDto(
         os.path.abspath(args.path),
         SecurityContentBasicBuilder(),
-        detection_builder,
+        SecurityContentDetectionBuilder(force_cached_or_offline=args.cached_and_offline, check_references=args.check_references),
         SecurityContentStoryBuilder(),
         SecurityContentBaselineBuilder(),
         SecurityContentInvestigationBuilder(),
@@ -167,7 +165,7 @@ def validate(args) -> None:
     ba_factory_input_dto = BAFactoryInputDto(
         os.path.abspath(args.path),
         SecurityContentBasicBuilder(),
-        SecurityContentDetectionBuilder(),
+        SecurityContentDetectionBuilder(force_cached_or_offline = False, check_references=args.check_references),
         SecurityContentDirector()
     )
 
@@ -274,6 +272,8 @@ def main(args):
     new_content_parser = actions_parser.add_parser("new_content", help="Create new security content object")
     reporting_parser = actions_parser.add_parser("reporting", help="Create security content reporting")
 
+    
+
     # # new arguments
     # new_parser.add_argument("-t", "--type", required=False, type=str, default="detection",
     #                              help="Type of new content to create, please choose between `detection`, `baseline` or `story`. Defaults to `detection`")
@@ -283,7 +283,12 @@ def main(args):
 
     validate_parser.add_argument("-pr", "--product", required=True, type=str, default='all', 
         help="Type of package to create, choose between all, `ESCU` or `SSA`.")
-    validate_parser.set_defaults(func=validate, epilog="""
+    validate_parser.add_argument('--check_references', action=argparse.BooleanOptionalAction, help="The number of threads to use to resolve references.  "
+                                   "Larger numbers will result in faster resolution, but will be more likely to hit rate limits or use a large amount of "
+                                   "bandwidth.  A larger number of threads is particularly useful on high-bandwidth connections, but does not improve "
+                                   "performance on slow connections.")
+    
+    validate_parser.set_defaults(func=validate, check_references=False, epilog="""
                 Validates security manifest for correctness, adhering to spec and other common items.""")
 
     generate_parser.add_argument("-o", "--output", required=True, type=str,
@@ -308,6 +313,9 @@ def main(args):
 
     reporting_parser.set_defaults(func=reporting)
 
+    
+    
+
     # # parse them
     args = parser.parse_args()
     return args.func(args)

From d814d4d74a482a30183842966ff88c99bcebdb64 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 4 May 2022 17:08:41 -0700
Subject: [PATCH 16/38] Added link validation to playbooks and performed
 successful tests against all Stories, Playbooks, Baselines, Detections, and
 Investigations. Also tested when the check_references or
 --no-check_references flags are passed successfully!

---
 .../application/factory/factory.py              |  2 +-
 .../contentctl_core/domain/entities/baseline.py |  2 +-
 .../domain/entities/investigation.py            |  2 +-
 .../domain/entities/link_validator.py           |  7 +------
 .../contentctl_core/domain/entities/playbook.py | 17 +++++++++++++++++
 .../contentctl_core/domain/entities/story.py    |  4 ++--
 .../security_content_playbook_builder.py        |  7 ++++++-
 7 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/application/factory/factory.py b/bin/contentctl_project/contentctl_core/application/factory/factory.py
index 28febb7900..1beafa9362 100644
--- a/bin/contentctl_project/contentctl_core/application/factory/factory.py
+++ b/bin/contentctl_project/contentctl_core/application/factory/factory.py
@@ -81,7 +81,7 @@ def createSecurityContent(self, type: SecurityContentType) -> list:
 
           import copy
           import threading
-          NUM_THREADS = 1
+          NUM_THREADS = 10
           
           def make_detections_thread(index:int):
                my_dto = copy.deepcopy(self.input_dto)
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/baseline.py b/bin/contentctl_project/contentctl_core/domain/entities/baseline.py
index ec40824447..38816b4299 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/baseline.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/baseline.py
@@ -92,7 +92,7 @@ def references_check(cls, v, values):
             #Reference checking is NOT enabled
             return v
         elif 'check_references' not in values:
-            raise(Exception("Member 'check_references' missing from Detection!"))
+            raise(Exception("Member 'check_references' missing from Baseline!"))
         
 
         for reference in v:
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/investigation.py b/bin/contentctl_project/contentctl_core/domain/entities/investigation.py
index 380b568d29..639e3c46a3 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/investigation.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/investigation.py
@@ -89,7 +89,7 @@ def references_check(cls, v, values):
             #Reference checking is NOT enabled
             return v
         elif 'check_references' not in values:
-            raise(Exception("Member 'check_references' missing from Detection!"))
+            raise(Exception("Member 'check_references' missing from Investigation!"))
         
 
         for reference in v:
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
index 48f6c6dd54..d85c628c11 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
@@ -105,9 +105,4 @@ def validate_reference(reference: str, raise_exception_if_failure: bool = False)
             raise(Exception(f"Reference Link Failed: {reference}"))
         else:
             return False
-        
-
-
-
-
-
+            
\ No newline at end of file
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/playbook.py b/bin/contentctl_project/contentctl_core/domain/entities/playbook.py
index c1696ec72c..2854778910 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/playbook.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/playbook.py
@@ -6,6 +6,7 @@
 
 from bin.contentctl_project.contentctl_core.domain.entities.security_content_object import SecurityContentObject
 from bin.contentctl_project.contentctl_core.domain.entities.playbook_tags import PlaybookTag
+from bin.contentctl_project.contentctl_core.domain.entities.link_validator import LinkValidator
 
 
 
@@ -19,7 +20,23 @@ class Playbook(BaseModel, SecurityContentObject):
     description: str
     how_to_implement: str
     playbook: str
+    check_references: bool = False #Validation is done in order, this field must be defined first
     references: list
     app_list: list
     tags: PlaybookTag
 
+    
+    @validator('references')
+    def references_check(cls, v, values):
+        
+        
+        if 'check_references' in values and values['check_references'] is False:
+            #Reference checking is NOT enabled
+            return v
+        elif 'check_references' not in values:
+            raise(Exception("Member 'check_references' missing from Playbook!"))
+        
+
+        for reference in v:
+            LinkValidator.validate_reference(reference)
+
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/story.py b/bin/contentctl_project/contentctl_core/domain/entities/story.py
index e72b509ad2..bcf3c8320c 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/story.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/story.py
@@ -63,7 +63,7 @@ def encode_error(cls, v, values, field):
             raise ValueError('encoding error in ' + field.name + ': ' + values["name"])
         return v
 
-      @validator('references')
+    @validator('references')
     def references_check(cls, v, values):
         
         
@@ -71,7 +71,7 @@ def references_check(cls, v, values):
             #Reference checking is NOT enabled
             return v
         elif 'check_references' not in values:
-            raise(Exception("Member 'check_references' missing from Detection!"))
+            raise(Exception("Member 'check_references' missing from Story!"))
         
 
         for reference in v:
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_playbook_builder.py b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_playbook_builder.py
index 59edc3dd21..f76823322a 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_playbook_builder.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_playbook_builder.py
@@ -12,12 +12,17 @@
 
 class SecurityContentPlaybookBuilder(PlaybookBuilder):
     playbook: Playbook
-
+    check_references: bool
+    
+    def __init__(self, check_references: bool = False):
+        self.check_references = check_references
 
     def setObject(self, path: str) -> None:
         yml_dict = YmlReader.load_file(path)
+        yml_dict["check_references"] = self.check_references
         try:
             self.playbook = Playbook.parse_obj(yml_dict)
+            del(yml_dict["check_references"])
         except ValidationError as e:
             print('Validation Error for file ' + path)
             print(e)

From c1ab5a491be85864d4c2e2e7840c41c32b5dd3a6 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 4 May 2022 17:09:36 -0700
Subject: [PATCH 17/38] Added check_references passthrough from contentctl to
 the builders.

---
 contentctl.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/contentctl.py b/contentctl.py
index d95f6f4dc9..23c52a503e 100644
--- a/contentctl.py
+++ b/contentctl.py
@@ -154,10 +154,10 @@ def validate(args) -> None:
         os.path.abspath(args.path),
         SecurityContentBasicBuilder(),
         SecurityContentDetectionBuilder(force_cached_or_offline=args.cached_and_offline, check_references=args.check_references),
-        SecurityContentStoryBuilder(),
-        SecurityContentBaselineBuilder(),
-        SecurityContentInvestigationBuilder(),
-        SecurityContentPlaybookBuilder(),
+        SecurityContentStoryBuilder(check_references=args.check_references),
+        SecurityContentBaselineBuilder(check_references=args.check_references),
+        SecurityContentInvestigationBuilder(check_references=args.check_references),
+        SecurityContentPlaybookBuilder(check_references=args.check_references),
         SecurityContentDirector(),
         AttackEnrichment.get_attack_lookup(force_cached_or_offline=args.cached_and_offline)
     )

From 67a0ed2821f559f9af3a440882e6b3624faa2b40 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 10 May 2022 12:00:31 -0700
Subject: [PATCH 18/38] Removed some un needed prints from Link Validator. 
 Added progress indicators to the steps of the factory so that users have
 feedback on what is going on when contentctl is running. This is helpful
 since it can run for an extended period of time, especially on a slower or
 high latency connection, when links are being validated, or when caching is
 not enabled.

---
 .../application/factory/factory.py            | 170 +++++++-----------
 .../domain/entities/link_validator.py         |   8 +-
 2 files changed, 67 insertions(+), 111 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/application/factory/factory.py b/bin/contentctl_project/contentctl_core/application/factory/factory.py
index 1beafa9362..eee0a51f61 100644
--- a/bin/contentctl_project/contentctl_core/application/factory/factory.py
+++ b/bin/contentctl_project/contentctl_core/application/factory/factory.py
@@ -78,10 +78,13 @@ def createSecurityContent(self, type: SecurityContentType) -> list:
                files = Utils.get_all_yml_files_from_directory(os.path.join(self.input_dto.input_path, str(type.name)))
           
           validation_error_found = False
-
+          
+          #Code below us commented out, but allows threaded construction of detections
+          #No issues have been observed running construction in parallel
+          '''
           import copy
           import threading
-          NUM_THREADS = 10
+          NUM_THREADS = 1
           
           def make_detections_thread(index:int):
                my_dto = copy.deepcopy(self.input_dto)
@@ -140,129 +143,84 @@ def make_detections_thread(index:int):
                          print(e)
                          validation_error_found = True
                     
-               #print(f"Builder thread {index} Done!")
+               
 
           builder_threads = []
-          #print("starting builder threads")
+          
           for t in range(NUM_THREADS):
                thread = threading.Thread(target=make_detections_thread,args=(t,))
                thread.start()
                builder_threads.append(thread)
-          #print("All threads started")
+          
           for thread in builder_threads:
                thread.join()
-               #print("thread joined")
-          #print("all threads joined")
-          
-
-
-               
-          #      if not 'ssa__' in file:
-          #           try:
-          #                if type == SecurityContentType.lookups:
-          #                     self.input_dto.director.constructLookup(self.input_dto.basic_builder, file)
-          #                     self.output_dto.lookups.append(self.input_dto.basic_builder.getObject())
-                         
-          #                elif type == SecurityContentType.macros:
-          #                     self.input_dto.director.constructMacro(self.input_dto.basic_builder, file)
-          #                     self.output_dto.macros.append(self.input_dto.basic_builder.getObject())
-                         
-          #                elif type == SecurityContentType.deployments:
-          #                     self.input_dto.director.constructDeployment(self.input_dto.basic_builder, file)
-          #                     self.output_dto.deployments.append(self.input_dto.basic_builder.getObject())
-                         
-          #                elif type == SecurityContentType.playbooks:
-          #                     self.input_dto.director.constructPlaybook(self.input_dto.playbook_builder, file)
-          #                     self.output_dto.playbooks.append(self.input_dto.playbook_builder.getObject())                    
-                         
-          #                elif type == SecurityContentType.baselines:
-          #                     self.input_dto.director.constructBaseline(self.input_dto.baseline_builder, file, self.output_dto.deployments)
-          #                     baseline = self.input_dto.baseline_builder.getObject()
-          #                     self.output_dto.baselines.append(baseline)
-                         
-          #                elif type == SecurityContentType.investigations:
-          #                     self.input_dto.director.constructInvestigation(self.input_dto.investigation_builder, file)
-          #                     investigation = self.input_dto.investigation_builder.getObject()
-          #                     self.output_dto.investigations.append(investigation)
-
-          #                elif type == SecurityContentType.stories:
-          #                     self.input_dto.director.constructStory(self.input_dto.story_builder, file, 
-          #                          self.output_dto.detections, self.output_dto.baselines, self.output_dto.investigations)
-          #                     story = self.input_dto.story_builder.getObject()
-          #                     self.output_dto.stories.append(story)
-                    
-          #                elif type == SecurityContentType.detections:
-          #                     self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, 
-          #                          self.output_dto.deployments, self.output_dto.playbooks, self.output_dto.baselines,
-          #                          self.output_dto.tests, self.input_dto.attack_enrichment, self.output_dto.macros,
-          #                          self.output_dto.lookups, self.input_dto.force_cached_or_offline)
-          #                     detection = self.input_dto.detection_builder.getObject()
-          #                     self.output_dto.detections.append(detection)
-                    
-          #                elif type == SecurityContentType.unit_tests:
-          #                     self.input_dto.director.constructTest(self.input_dto.basic_builder, file)
-          #                     test = self.input_dto.basic_builder.getObject()
-          #                     self.output_dto.tests.append(test)
-                    
-          #           except ValidationError as e:
-          #                print('\nValidation Error for file ' + file)
-          #                print(e)
-          #                validation_error_found = True
-
+          '''
           
+          #Non threaded, production version of the construction code
 
-          # for file in files:
-          #      if not 'ssa__' in file:
-          #           try:
-          #                if type == SecurityContentType.lookups:
-          #                     self.input_dto.director.constructLookup(self.input_dto.basic_builder, file)
-          #                     self.output_dto.lookups.append(self.input_dto.basic_builder.getObject())
+          for index,file in enumerate(files):
+               if not 'ssa__' in file:
+                    progress_percent = (index/len(files)) * 100
+                    try:
+                         if type == SecurityContentType.lookups:
+                              print(f"\r{'Lookups Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                              self.input_dto.director.constructLookup(self.input_dto.basic_builder, file)
+                              self.output_dto.lookups.append(self.input_dto.basic_builder.getObject())
                          
-          #                elif type == SecurityContentType.macros:
-          #                     self.input_dto.director.constructMacro(self.input_dto.basic_builder, file)
-          #                     self.output_dto.macros.append(self.input_dto.basic_builder.getObject())
+                         elif type == SecurityContentType.macros:
+                              print(f"\r{'Playbooks Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                              self.input_dto.director.constructMacro(self.input_dto.basic_builder, file)
+                              self.output_dto.macros.append(self.input_dto.basic_builder.getObject())
                          
-          #                elif type == SecurityContentType.deployments:
-          #                     self.input_dto.director.constructDeployment(self.input_dto.basic_builder, file)
-          #                     self.output_dto.deployments.append(self.input_dto.basic_builder.getObject())
+                         elif type == SecurityContentType.deployments:
+                              print(f"\r{'Playbooks Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                              self.input_dto.director.constructDeployment(self.input_dto.basic_builder, file)
+                              self.output_dto.deployments.append(self.input_dto.basic_builder.getObject())
                          
-          #                elif type == SecurityContentType.playbooks:
-          #                     self.input_dto.director.constructPlaybook(self.input_dto.playbook_builder, file)
-          #                     self.output_dto.playbooks.append(self.input_dto.playbook_builder.getObject())                    
+                         elif type == SecurityContentType.playbooks:
+                              print(f"\r{'Playbooks Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                              self.input_dto.director.constructPlaybook(self.input_dto.playbook_builder, file)
+                              self.output_dto.playbooks.append(self.input_dto.playbook_builder.getObject())                    
                          
-          #                elif type == SecurityContentType.baselines:
-          #                     self.input_dto.director.constructBaseline(self.input_dto.baseline_builder, file, self.output_dto.deployments)
-          #                     baseline = self.input_dto.baseline_builder.getObject()
-          #                     self.output_dto.baselines.append(baseline)
+                         elif type == SecurityContentType.baselines:
+                              print(f"\r{'Baselines Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                              self.input_dto.director.constructBaseline(self.input_dto.baseline_builder, file, self.output_dto.deployments)
+                              baseline = self.input_dto.baseline_builder.getObject()
+                              self.output_dto.baselines.append(baseline)
                          
-          #                elif type == SecurityContentType.investigations:
-          #                     self.input_dto.director.constructInvestigation(self.input_dto.investigation_builder, file)
-          #                     investigation = self.input_dto.investigation_builder.getObject()
-          #                     self.output_dto.investigations.append(investigation)
+                         elif type == SecurityContentType.investigations:
+                              print(f"\r{'Investigations Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                              self.input_dto.director.constructInvestigation(self.input_dto.investigation_builder, file)
+                              investigation = self.input_dto.investigation_builder.getObject()
+                              self.output_dto.investigations.append(investigation)
 
-          #                elif type == SecurityContentType.stories:
-          #                     self.input_dto.director.constructStory(self.input_dto.story_builder, file, 
-          #                          self.output_dto.detections, self.output_dto.baselines, self.output_dto.investigations)
-          #                     story = self.input_dto.story_builder.getObject()
-          #                     self.output_dto.stories.append(story)
+                         elif type == SecurityContentType.stories:
+                              print(f"\r{'Stories Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                              self.input_dto.director.constructStory(self.input_dto.story_builder, file, 
+                                   self.output_dto.detections, self.output_dto.baselines, self.output_dto.investigations)
+                              story = self.input_dto.story_builder.getObject()
+                              self.output_dto.stories.append(story)
                     
-          #                elif type == SecurityContentType.detections:
-          #                     self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, 
-          #                          self.output_dto.deployments, self.output_dto.playbooks, self.output_dto.baselines,
-          #                          self.output_dto.tests, self.input_dto.attack_enrichment, self.output_dto.macros,
-          #                          self.output_dto.lookups, self.input_dto.force_cached_or_offline)
-          #                     detection = self.input_dto.detection_builder.getObject()
-          #                     self.output_dto.detections.append(detection)
+                         elif type == SecurityContentType.detections:
+                              print(f"\r{'Detections Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                              self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, 
+                                   self.output_dto.deployments, self.output_dto.playbooks, self.output_dto.baselines,
+                                   self.output_dto.tests, self.input_dto.attack_enrichment, self.output_dto.macros,
+                                   self.output_dto.lookups, self.input_dto.force_cached_or_offline)
+                              detection = self.input_dto.detection_builder.getObject()
+                              self.output_dto.detections.append(detection)
                     
-          #                elif type == SecurityContentType.unit_tests:
-          #                     self.input_dto.director.constructTest(self.input_dto.basic_builder, file)
-          #                     test = self.input_dto.basic_builder.getObject()
-          #                     self.output_dto.tests.append(test)
+                         elif type == SecurityContentType.unit_tests:
+                              print(f"\r{'Detections Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                              self.input_dto.director.constructTest(self.input_dto.basic_builder, file)
+                              test = self.input_dto.basic_builder.getObject()
+                              self.output_dto.tests.append(test)
                     
-          #           except ValidationError as e:
-          #                print('\nValidation Error for file ' + file)
-          #                print(e)
-          #                validation_error_found = True
+                    except ValidationError as e:
+                         print('\nValidation Error for file ' + file)
+                         print(e)
+                         validation_error_found = True
+          print("...Done!")
 
           if validation_error_found:
                sys.exit(1)
\ No newline at end of file
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
index d85c628c11..b4ca525294 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
@@ -56,8 +56,6 @@ def check_reference(cls, values):
                             headers = headers, 
                             allow_redirects=allow_redirects, verify=verify_ssl)
             resolution_time = time.time() - start_time
-
-            
             values['status_code'] = get.status_code
             values['resolution_time'] = resolution_time
             if reference != get.url:
@@ -87,22 +85,22 @@ class LinkValidator(abc.ABC):
     cache: dict[str,LinkStats] = {}
     uncached_checks: int = 0
     total_checks: int = 0
-    
     #cache: dict[str,LinkStats] = {}
 
     
     @staticmethod
     def validate_reference(reference: str, raise_exception_if_failure: bool = False) -> bool:
         LinkValidator.total_checks += 1
-        print(f"Total Checks: {LinkValidator.total_checks}")
         if reference not in LinkValidator.cache:
             LinkValidator.uncached_checks += 1
             LinkValidator.cache[reference] = LinkStats(reference=reference)
         result = LinkValidator.cache[reference].is_link_valid()
+
+        #print(f"Total Checks: {LinkValidator.total_checks}, Percent Cached: {100*(1 - LinkValidator.uncached_checks / LinkValidator.total_checks):.2f}")
+
         if result is True:
             return True
         elif raise_exception_if_failure is True:
             raise(Exception(f"Reference Link Failed: {reference}"))
         else:
             return False
-            
\ No newline at end of file

From 6e6c7f5f4a00d023b96abc68bd8921f705564b8c Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 10 May 2022 13:15:52 -0700
Subject: [PATCH 19/38] More finishing touches on link validation.  Now,
 failures are printed out at the end after all links have been checked instead
 of when the tool is running, making it easier to read.  Files that referernce
 the link are also included.

---
 .../application/factory/factory.py            | 129 +++++++++---------
 .../domain/entities/baseline.py               |   2 +-
 .../domain/entities/detection.py              |   2 +-
 .../domain/entities/investigation.py          |   2 +-
 .../domain/entities/link_validator.py         |  25 ++--
 .../domain/entities/playbook.py               |   2 +-
 .../contentctl_core/domain/entities/story.py  |   2 +-
 7 files changed, 88 insertions(+), 76 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/application/factory/factory.py b/bin/contentctl_project/contentctl_core/application/factory/factory.py
index eee0a51f61..3c5e5dd5df 100644
--- a/bin/contentctl_project/contentctl_core/application/factory/factory.py
+++ b/bin/contentctl_project/contentctl_core/application/factory/factory.py
@@ -14,7 +14,7 @@
 from bin.contentctl_project.contentctl_core.application.builder.playbook_builder import PlaybookBuilder
 from bin.contentctl_project.contentctl_core.application.builder.director import Director
 from bin.contentctl_project.contentctl_core.application.factory.utils.utils import Utils
-
+from bin.contentctl_project.contentctl_core.domain.entities.link_validator import LinkValidator
 
 @dataclass(frozen=True)
 class FactoryInputDto:
@@ -54,7 +54,7 @@ def __init__(self, output_dto: FactoryOutputDto) -> None:
 
      def execute(self, input_dto: FactoryInputDto) -> None:
           self.input_dto = input_dto
-
+          print("Creating Security Content. This may take some time...")
           # order matters to load and enrich security content types
           self.createSecurityContent(SecurityContentType.unit_tests)
           self.createSecurityContent(SecurityContentType.lookups)
@@ -65,7 +65,8 @@ def execute(self, input_dto: FactoryInputDto) -> None:
           self.createSecurityContent(SecurityContentType.playbooks)
           self.createSecurityContent(SecurityContentType.detections)
           self.createSecurityContent(SecurityContentType.stories)
-
+          LinkValidator.print_link_validation_errors()
+          
 
      def createSecurityContent(self, type: SecurityContentType) -> list:
           
@@ -157,69 +158,71 @@ def make_detections_thread(index:int):
           '''
           
           #Non threaded, production version of the construction code
-
-          for index,file in enumerate(files):
-               if not 'ssa__' in file:
-                    progress_percent = (index/len(files)) * 100
-                    try:
-                         if type == SecurityContentType.lookups:
-                              print(f"\r{'Lookups Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
-                              self.input_dto.director.constructLookup(self.input_dto.basic_builder, file)
-                              self.output_dto.lookups.append(self.input_dto.basic_builder.getObject())
-                         
-                         elif type == SecurityContentType.macros:
-                              print(f"\r{'Playbooks Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
-                              self.input_dto.director.constructMacro(self.input_dto.basic_builder, file)
-                              self.output_dto.macros.append(self.input_dto.basic_builder.getObject())
-                         
-                         elif type == SecurityContentType.deployments:
-                              print(f"\r{'Playbooks Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
-                              self.input_dto.director.constructDeployment(self.input_dto.basic_builder, file)
-                              self.output_dto.deployments.append(self.input_dto.basic_builder.getObject())
-                         
-                         elif type == SecurityContentType.playbooks:
-                              print(f"\r{'Playbooks Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
-                              self.input_dto.director.constructPlaybook(self.input_dto.playbook_builder, file)
-                              self.output_dto.playbooks.append(self.input_dto.playbook_builder.getObject())                    
-                         
-                         elif type == SecurityContentType.baselines:
-                              print(f"\r{'Baselines Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
-                              self.input_dto.director.constructBaseline(self.input_dto.baseline_builder, file, self.output_dto.deployments)
-                              baseline = self.input_dto.baseline_builder.getObject()
-                              self.output_dto.baselines.append(baseline)
-                         
-                         elif type == SecurityContentType.investigations:
-                              print(f"\r{'Investigations Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
-                              self.input_dto.director.constructInvestigation(self.input_dto.investigation_builder, file)
-                              investigation = self.input_dto.investigation_builder.getObject()
-                              self.output_dto.investigations.append(investigation)
-
-                         elif type == SecurityContentType.stories:
-                              print(f"\r{'Stories Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
-                              self.input_dto.director.constructStory(self.input_dto.story_builder, file, 
-                                   self.output_dto.detections, self.output_dto.baselines, self.output_dto.investigations)
-                              story = self.input_dto.story_builder.getObject()
-                              self.output_dto.stories.append(story)
+          files_without_ssa = [f for f in files if 'ssa___' not in f]
+          for index,file in enumerate(files_without_ssa):
+          
+               #Index + 1 because we are zero indexed, not 1 indexed.  This ensures
+               # that printouts end at 100%, not some other number 
+               progress_percent = ((index+1)/len(files_without_ssa)) * 100
+               try:
+                    if type == SecurityContentType.lookups:
+                         print(f"\r{'Lookups Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         self.input_dto.director.constructLookup(self.input_dto.basic_builder, file)
+                         self.output_dto.lookups.append(self.input_dto.basic_builder.getObject())
                     
-                         elif type == SecurityContentType.detections:
-                              print(f"\r{'Detections Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
-                              self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, 
-                                   self.output_dto.deployments, self.output_dto.playbooks, self.output_dto.baselines,
-                                   self.output_dto.tests, self.input_dto.attack_enrichment, self.output_dto.macros,
-                                   self.output_dto.lookups, self.input_dto.force_cached_or_offline)
-                              detection = self.input_dto.detection_builder.getObject()
-                              self.output_dto.detections.append(detection)
+                    elif type == SecurityContentType.macros:
+                         print(f"\r{'Playbooks Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         self.input_dto.director.constructMacro(self.input_dto.basic_builder, file)
+                         self.output_dto.macros.append(self.input_dto.basic_builder.getObject())
                     
-                         elif type == SecurityContentType.unit_tests:
-                              print(f"\r{'Detections Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
-                              self.input_dto.director.constructTest(self.input_dto.basic_builder, file)
-                              test = self.input_dto.basic_builder.getObject()
-                              self.output_dto.tests.append(test)
+                    elif type == SecurityContentType.deployments:
+                         print(f"\r{'Deployments Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         self.input_dto.director.constructDeployment(self.input_dto.basic_builder, file)
+                         self.output_dto.deployments.append(self.input_dto.basic_builder.getObject())
                     
-                    except ValidationError as e:
-                         print('\nValidation Error for file ' + file)
-                         print(e)
-                         validation_error_found = True
+                    elif type == SecurityContentType.playbooks:
+                         print(f"\r{'Playbooks Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         self.input_dto.director.constructPlaybook(self.input_dto.playbook_builder, file)
+                         self.output_dto.playbooks.append(self.input_dto.playbook_builder.getObject())                    
+                    
+                    elif type == SecurityContentType.baselines:
+                         print(f"\r{'Baselines Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         self.input_dto.director.constructBaseline(self.input_dto.baseline_builder, file, self.output_dto.deployments)
+                         baseline = self.input_dto.baseline_builder.getObject()
+                         self.output_dto.baselines.append(baseline)
+                    
+                    elif type == SecurityContentType.investigations:
+                         print(f"\r{'Investigations Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         self.input_dto.director.constructInvestigation(self.input_dto.investigation_builder, file)
+                         investigation = self.input_dto.investigation_builder.getObject()
+                         self.output_dto.investigations.append(investigation)
+
+                    elif type == SecurityContentType.stories:
+                         print(f"\r{'Stories Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         self.input_dto.director.constructStory(self.input_dto.story_builder, file, 
+                              self.output_dto.detections, self.output_dto.baselines, self.output_dto.investigations)
+                         story = self.input_dto.story_builder.getObject()
+                         self.output_dto.stories.append(story)
+               
+                    elif type == SecurityContentType.detections:
+                         print(f"\r{'Detections Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, 
+                              self.output_dto.deployments, self.output_dto.playbooks, self.output_dto.baselines,
+                              self.output_dto.tests, self.input_dto.attack_enrichment, self.output_dto.macros,
+                              self.output_dto.lookups, self.input_dto.force_cached_or_offline)
+                         detection = self.input_dto.detection_builder.getObject()
+                         self.output_dto.detections.append(detection)
+               
+                    elif type == SecurityContentType.unit_tests:
+                         print(f"\r{'Unit Tests Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         self.input_dto.director.constructTest(self.input_dto.basic_builder, file)
+                         test = self.input_dto.basic_builder.getObject()
+                         self.output_dto.tests.append(test)
+               
+               except ValidationError as e:
+                    print('\nValidation Error for file ' + file)
+                    print(e)
+                    validation_error_found = True
           print("...Done!")
 
           if validation_error_found:
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/baseline.py b/bin/contentctl_project/contentctl_core/domain/entities/baseline.py
index 38816b4299..6f6026ed2e 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/baseline.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/baseline.py
@@ -96,7 +96,7 @@ def references_check(cls, v, values):
         
 
         for reference in v:
-            LinkValidator.validate_reference(reference)
+            LinkValidator.validate_reference(reference, values['name'])
 
     @validator('search')
     def search_validate(cls, v, values):
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/detection.py b/bin/contentctl_project/contentctl_core/domain/entities/detection.py
index af07cc43be..2db26c076c 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/detection.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/detection.py
@@ -143,7 +143,7 @@ def references_check(cls, v, values):
         
 
         for reference in v:
-            LinkValidator.validate_reference(reference)
+            LinkValidator.validate_reference(reference, values['name'])
         return v
 
     @validator('search')
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/investigation.py b/bin/contentctl_project/contentctl_core/domain/entities/investigation.py
index 639e3c46a3..340a544c2b 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/investigation.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/investigation.py
@@ -93,7 +93,7 @@ def references_check(cls, v, values):
         
 
         for reference in v:
-            LinkValidator.validate_reference(reference)
+            LinkValidator.validate_reference(reference, values['name'])
 
     @validator('search')
     def search_validate(cls, v, values):
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
index b4ca525294..1f35c9e724 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
@@ -14,8 +14,8 @@ class LinkStats(BaseModel):
     #Static Values
     method: Callable = requests.get
     allowed_http_codes: list[int] = [200] 
-    access_count: int = 0
-    timeout_seconds: int = 30
+    access_count: int = 1 #when constructor is called, it has been accessed once!
+    timeout_seconds: int = 15
     allow_redirects: bool = True
     headers: dict = {"User-Agent": DEFAULT_USER_AGENT_STRING}
     verify_ssl: bool = False
@@ -27,14 +27,16 @@ class LinkStats(BaseModel):
     #validation working since ComputedField has not yet been 
     #introduced to Pydantic
     reference: str
+    referencing_files: set[str]
     redirect: Union[str,None] = None
     status_code: int = 0
     valid: bool = False
     resolution_time: float = 0 
     
     
-    def is_link_valid(self)->bool:
+    def is_link_valid(self, referencing_file:str)->bool:
         self.access_count += 1
+        self.referencing_files.add(referencing_file)
         return self.valid
     
     @root_validator
@@ -67,13 +69,13 @@ def check_reference(cls, values):
             if get.status_code in allowed_http_codes:
                 values['valid'] = True
             else:
-                print(f"Unacceptable HTTP Status Code {get.status_code} received for {reference}")
+                #print(f"Unacceptable HTTP Status Code {get.status_code} received for {reference}")
                 values['valid'] = False
             return values    
 
         except Exception as e:
             resolution_time = time.time() - start_time
-            print(f"Reference {reference} was not reachable after {resolution_time:.2f} seconds")
+            #print(f"Reference {reference} was not reachable after {resolution_time:.2f} seconds")
             values['status_code'] = 0
             values['valid'] = False
             values['redirect'] = None
@@ -89,12 +91,12 @@ class LinkValidator(abc.ABC):
 
     
     @staticmethod
-    def validate_reference(reference: str, raise_exception_if_failure: bool = False) -> bool:
+    def validate_reference(reference: str, referencing_file:str, raise_exception_if_failure: bool = False) -> bool:
         LinkValidator.total_checks += 1
         if reference not in LinkValidator.cache:
             LinkValidator.uncached_checks += 1
-            LinkValidator.cache[reference] = LinkStats(reference=reference)
-        result = LinkValidator.cache[reference].is_link_valid()
+            LinkValidator.cache[reference] = LinkStats(reference=reference, referencing_files = set([referencing_file]))
+        result = LinkValidator.cache[reference].is_link_valid(referencing_file)
 
         #print(f"Total Checks: {LinkValidator.total_checks}, Percent Cached: {100*(1 - LinkValidator.uncached_checks / LinkValidator.total_checks):.2f}")
 
@@ -104,3 +106,10 @@ def validate_reference(reference: str, raise_exception_if_failure: bool = False)
             raise(Exception(f"Reference Link Failed: {reference}"))
         else:
             return False
+    @staticmethod
+    def print_link_validation_errors():
+        for k in LinkValidator.cache.keys():
+            if LinkValidator.cache[k].valid is False:
+                print(f"Link {k} invalid with HTTP Status Code [{LinkValidator.cache[k].status_code}] and referenced by the following files:")
+                for ref in LinkValidator.cache[k].referencing_files:
+                    print(f"\t* {ref}")
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/playbook.py b/bin/contentctl_project/contentctl_core/domain/entities/playbook.py
index 2854778910..9fcafb78c1 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/playbook.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/playbook.py
@@ -38,5 +38,5 @@ def references_check(cls, v, values):
         
 
         for reference in v:
-            LinkValidator.validate_reference(reference)
+            LinkValidator.validate_reference(reference, values['name'])
 
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/story.py b/bin/contentctl_project/contentctl_core/domain/entities/story.py
index bcf3c8320c..107879cc0b 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/story.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/story.py
@@ -75,4 +75,4 @@ def references_check(cls, v, values):
         
 
         for reference in v:
-            LinkValidator.validate_reference(reference)
\ No newline at end of file
+            LinkValidator.validate_reference(reference, values['name'])
\ No newline at end of file

From cdb66ae1434386ab587ab0d7932510b69699cfab Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 10 May 2022 15:49:45 -0700
Subject: [PATCH 20/38] More optimizations in content_ctl. No longer generate
 ba_factory or factory if they are not needed - only generate the one or ones
 that are required. Also, provide status printouts for the AttackEnrichment so
 that the user can monitor progress and be sure something is happening.

---
 .../application/factory/ba_factory.py         | 15 +++-
 .../application/factory/factory.py            | 22 ++---
 .../application/use_cases/generate.py         |  5 +-
 .../application/use_cases/validate.py         |  5 +-
 .../builder/attack_enrichment.py              | 19 +++-
 .../security_content_detection_builder.py     |  4 +-
 contentctl.py                                 | 88 ++++++++++---------
 7 files changed, 98 insertions(+), 60 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py b/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py
index e611a15304..d5050d854c 100644
--- a/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py
+++ b/bin/contentctl_project/contentctl_core/application/factory/ba_factory.py
@@ -33,9 +33,10 @@ def __init__(self, output_dto: BAFactoryOutputDto) -> None:
 
     def execute(self, input_dto: BAFactoryInputDto) -> None:
         self.input_dto = input_dto
-
+        print("Creating Security Content - SSA. This may take some time...")
         self.createSecurityContent(SecurityContentType.unit_tests)
         self.createSecurityContent(SecurityContentType.detections)
+        
 
         
 
@@ -48,15 +49,24 @@ def createSecurityContent(self, type: SecurityContentType) -> list:
 
         validation_error_found = False
 
-        for file in files:
+        files_with_ssa = [f for f in files if 'ssa___' in f]
+        for index,file in enumerate(files_with_ssa):
+          
+            #Index + 1 because we are zero indexed, not 1 indexed.  This ensures
+            # that printouts end at 100%, not some other number 
+            progress_percent = ((index+1)/len(files_with_ssa)) * 100
+        
             if 'ssa__' in file:
+                progress_percent = ((index+1)/len(files_with_ssa)) * 100
                 try:
                     if type == SecurityContentType.detections:
+                        print(f"\r{'Detections Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
                         self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, [], [], [], self.output_dto.tests, {}, [], [])
                         detection = self.input_dto.detection_builder.getObject()
                         if not detection.deprecated and not detection.experimental:
                             self.output_dto.detections.append(detection)
                     elif type == SecurityContentType.unit_tests:
+                        print(f"\r{'Unit Tests Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
                         self.input_dto.director.constructTest(self.input_dto.basic_builder, file)
                         test = self.input_dto.basic_builder.getObject()
                         self.output_dto.tests.append(test)
@@ -65,6 +75,7 @@ def createSecurityContent(self, type: SecurityContentType) -> list:
                     print('\nValidation Error for file ' + file)
                     print(e)
                     validation_error_found = True
+        print("Done!")
 
         if validation_error_found:
             sys.exit(1)
\ No newline at end of file
diff --git a/bin/contentctl_project/contentctl_core/application/factory/factory.py b/bin/contentctl_project/contentctl_core/application/factory/factory.py
index 3c5e5dd5df..9fa6e81cbe 100644
--- a/bin/contentctl_project/contentctl_core/application/factory/factory.py
+++ b/bin/contentctl_project/contentctl_core/application/factory/factory.py
@@ -54,7 +54,7 @@ def __init__(self, output_dto: FactoryOutputDto) -> None:
 
      def execute(self, input_dto: FactoryInputDto) -> None:
           self.input_dto = input_dto
-          print("Creating Security Content. This may take some time...")
+          print("Creating Security Content - ESCU. This may take some time...")
           # order matters to load and enrich security content types
           self.createSecurityContent(SecurityContentType.unit_tests)
           self.createSecurityContent(SecurityContentType.lookups)
@@ -166,46 +166,46 @@ def make_detections_thread(index:int):
                progress_percent = ((index+1)/len(files_without_ssa)) * 100
                try:
                     if type == SecurityContentType.lookups:
-                         print(f"\r{'Lookups Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         print(f"\r{'Lookups Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
                          self.input_dto.director.constructLookup(self.input_dto.basic_builder, file)
                          self.output_dto.lookups.append(self.input_dto.basic_builder.getObject())
                     
                     elif type == SecurityContentType.macros:
-                         print(f"\r{'Playbooks Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         print(f"\r{'Playbooks Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
                          self.input_dto.director.constructMacro(self.input_dto.basic_builder, file)
                          self.output_dto.macros.append(self.input_dto.basic_builder.getObject())
                     
                     elif type == SecurityContentType.deployments:
-                         print(f"\r{'Deployments Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         print(f"\r{'Deployments Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
                          self.input_dto.director.constructDeployment(self.input_dto.basic_builder, file)
                          self.output_dto.deployments.append(self.input_dto.basic_builder.getObject())
                     
                     elif type == SecurityContentType.playbooks:
-                         print(f"\r{'Playbooks Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         print(f"\r{'Playbooks Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
                          self.input_dto.director.constructPlaybook(self.input_dto.playbook_builder, file)
                          self.output_dto.playbooks.append(self.input_dto.playbook_builder.getObject())                    
                     
                     elif type == SecurityContentType.baselines:
-                         print(f"\r{'Baselines Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         print(f"\r{'Baselines Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
                          self.input_dto.director.constructBaseline(self.input_dto.baseline_builder, file, self.output_dto.deployments)
                          baseline = self.input_dto.baseline_builder.getObject()
                          self.output_dto.baselines.append(baseline)
                     
                     elif type == SecurityContentType.investigations:
-                         print(f"\r{'Investigations Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         print(f"\r{'Investigations Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
                          self.input_dto.director.constructInvestigation(self.input_dto.investigation_builder, file)
                          investigation = self.input_dto.investigation_builder.getObject()
                          self.output_dto.investigations.append(investigation)
 
                     elif type == SecurityContentType.stories:
-                         print(f"\r{'Stories Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         print(f"\r{'Stories Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
                          self.input_dto.director.constructStory(self.input_dto.story_builder, file, 
                               self.output_dto.detections, self.output_dto.baselines, self.output_dto.investigations)
                          story = self.input_dto.story_builder.getObject()
                          self.output_dto.stories.append(story)
                
                     elif type == SecurityContentType.detections:
-                         print(f"\r{'Detections Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         print(f"\r{'Detections Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
                          self.input_dto.director.constructDetection(self.input_dto.detection_builder, file, 
                               self.output_dto.deployments, self.output_dto.playbooks, self.output_dto.baselines,
                               self.output_dto.tests, self.input_dto.attack_enrichment, self.output_dto.macros,
@@ -214,7 +214,7 @@ def make_detections_thread(index:int):
                          self.output_dto.detections.append(detection)
                
                     elif type == SecurityContentType.unit_tests:
-                         print(f"\r{'Unit Tests Progress'.rjust(23)}: [{progress_percent:3.0f}%]", end="", flush=True)
+                         print(f"\r{'Unit Tests Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
                          self.input_dto.director.constructTest(self.input_dto.basic_builder, file)
                          test = self.input_dto.basic_builder.getObject()
                          self.output_dto.tests.append(test)
@@ -223,7 +223,7 @@ def make_detections_thread(index:int):
                     print('\nValidation Error for file ' + file)
                     print(e)
                     validation_error_found = True
-          print("...Done!")
+          print("Done!")
 
           if validation_error_found:
                sys.exit(1)
\ No newline at end of file
diff --git a/bin/contentctl_project/contentctl_core/application/use_cases/generate.py b/bin/contentctl_project/contentctl_core/application/use_cases/generate.py
index 3f3f4f5ce2..e0732c5840 100644
--- a/bin/contentctl_project/contentctl_core/application/use_cases/generate.py
+++ b/bin/contentctl_project/contentctl_core/application/use_cases/generate.py
@@ -2,6 +2,7 @@
 import shutil
 
 from dataclasses import dataclass
+from typing import Union
 
 from bin.contentctl_project.contentctl_core.domain.entities.enums.enums import SecurityContentProduct, SecurityContentType
 from bin.contentctl_project.contentctl_core.application.adapter.adapter import Adapter
@@ -13,8 +14,8 @@
 @dataclass(frozen=True)
 class GenerateInputDto:
     output_path: str
-    factory_input_dto: FactoryInputDto
-    ba_factory_input_dto: BAFactoryInputDto
+    factory_input_dto: Union[FactoryInputDto,None]
+    ba_factory_input_dto: Union[BAFactoryInputDto,None]
     adapter : Adapter
     product: SecurityContentProduct
 
diff --git a/bin/contentctl_project/contentctl_core/application/use_cases/validate.py b/bin/contentctl_project/contentctl_core/application/use_cases/validate.py
index a83f831c6e..65637518d9 100644
--- a/bin/contentctl_project/contentctl_core/application/use_cases/validate.py
+++ b/bin/contentctl_project/contentctl_core/application/use_cases/validate.py
@@ -3,6 +3,7 @@
 from dataclasses import dataclass
 
 from pydantic import ValidationError
+from typing import Union
 
 from bin.contentctl_project.contentctl_core.domain.entities.enums.enums import SecurityContentProduct
 from bin.contentctl_project.contentctl_core.application.factory.factory import FactoryInputDto, Factory, FactoryOutputDto
@@ -11,8 +12,8 @@
 
 @dataclass(frozen=True)
 class ValidateInputDto:
-    factory_input_dto: FactoryInputDto
-    ba_factory_input_dto: BAFactoryInputDto
+    factory_input_dto: Union[FactoryInputDto,None]
+    ba_factory_input_dto: Union[BAFactoryInputDto,None]
     product: SecurityContentProduct
 
 
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py
index cdfa844755..f23007dd8c 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py
@@ -14,18 +14,32 @@ class AttackEnrichment():
 
     @classmethod
     def get_attack_lookup(self, store_csv = None, force_cached_or_offline: bool = False) -> dict:
+        print("Getting MITRE Attack Enrichment Data. This may take some time...")
         attack_lookup = dict()
         file_path = os.path.join(os.path.dirname(__file__), '../../../../lookups/mitre_enrichment.csv')
 
         try:
             if force_cached_or_offline is True:
                 raise(Exception("WARNING - Using cached MITRE Attack Enrichment.  Attack Enrichment may be out of date. Only use this setting for offline environments and development purposes."))
+            print(f"\r{'Client'.rjust(23)}: [{0:3.0f}%]...", end="", flush=True)
             lift = attack_client()
+            print(f"\r{'Client'.rjust(23)}: [{100:3.0f}%]...Done!", end="\n", flush=True)
+            
+            print(f"\r{'Enterprise'.rjust(23)}: [{0.0:3.0f}%]...", end="", flush=True)
             all_enterprise = lift.get_enterprise(stix_format=False)
+            print(f"\r{'Enterprise'.rjust(23)}: [{100:3.0f}%]...Done!", end="\n", flush=True)
+            
+            print(f"\r{'Relationships'.rjust(23)}: [{0.0:3.0f}%]...", end="", flush=True)
             enterprise_relationships = lift.get_enterprise_relationships()
+            print(f"\r{'Relationships'.rjust(23)}: [{100:3.0f}%]...Done!", end="\n", flush=True)
+            
+            print(f"\r{'Groups'.rjust(23)}: [{0:3.0f}%]...", end="", flush=True)
             enterprise_groups = lift.get_enterprise_groups()
-
-            for technique in all_enterprise['techniques']:
+            print(f"\r{'Groups'.rjust(23)}: [{100:3.0f}%]...Done!", end="\n", flush=True)
+            
+            for index, technique in enumerate(all_enterprise['techniques']):
+                progress_percent = ((index+1)/len(all_enterprise['techniques'])) * 100
+                print(f"\r\t{'MITRE Technique Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
                 apt_groups = []
                 for relationship in enterprise_relationships:
                     if (relationship['target_ref'] == technique['id']) and relationship['source_ref'].startswith('intrusion-set'):
@@ -69,4 +83,5 @@ def get_attack_lookup(self, store_csv = None, force_cached_or_offline: bool = Fa
                 attack_lookup = {rows[0]:{'technique': rows[1], 'tactics': rows[2].split('|'), 'groups': rows[3].split('|')} for rows in reader}
             attack_lookup.pop('mitre_id')
 
+        print("Done!")
         return attack_lookup
\ No newline at end of file
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py
index d894e2cb6a..9e07f641d8 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py
@@ -16,7 +16,9 @@
 
 class SecurityContentDetectionBuilder(DetectionBuilder):
     security_content_obj : SecurityContentObject
-    force_cached_or_offline: bool
+    #Only SecurityContentDetectionBuilder cares about force_cached_or_offline because it is
+    #used for CveEnrichment.enrich_cve and SplunkAppEnrichment.enrich_splunk_app
+    force_cached_or_offline: bool 
     check_references: bool
 
     def __init__(self, force_cached_or_offline: bool = False, check_references: bool = False):
diff --git a/contentctl.py b/contentctl.py
index 23c52a503e..92ac2c1628 100644
--- a/contentctl.py
+++ b/contentctl.py
@@ -92,25 +92,29 @@ def generate(args) -> None:
         print("ERROR: invalid product. valid products are ESCU, SSA or API.")
         sys.exit(1)
 
-    
-    factory_input_dto = FactoryInputDto(
-        os.path.abspath(args.path),
-        SecurityContentBasicBuilder(),
-        SecurityContentDetectionBuilder(),
-        SecurityContentStoryBuilder(),
-        SecurityContentBaselineBuilder(),
-        SecurityContentInvestigationBuilder(),
-        SecurityContentPlaybookBuilder(),
-        SecurityContentDirector(),
-        AttackEnrichment.get_attack_lookup(store_csv=True, force_cached_or_offline=args.cached_and_offline)
-    )
+    #Save runtime by only generating the required factory inputs
+    factory_input_dto = None
+    ba_factory_input_dto = None
+    if args.product in ["ESCU", "API"]:
+        factory_input_dto = FactoryInputDto(
+            os.path.abspath(args.path),
+            SecurityContentBasicBuilder(),
+            SecurityContentDetectionBuilder(force_cached_or_offline=args.cached_and_offline),
+            SecurityContentStoryBuilder(),
+            SecurityContentBaselineBuilder(),
+            SecurityContentInvestigationBuilder(),
+            SecurityContentPlaybookBuilder(),
+            SecurityContentDirector(),
+            AttackEnrichment.get_attack_lookup(force_cached_or_offline=args.cached_and_offline)
+        )
+    if args.product in ["SSA", "API"]:
+        ba_factory_input_dto = BAFactoryInputDto(
+            os.path.abspath(args.path),
+            SecurityContentBasicBuilder(),
+            SecurityContentDetectionBuilder(force_cached_or_offline = args.cached_and_offline),
+            SecurityContentDirector()
+        )
 
-    ba_factory_input_dto = BAFactoryInputDto(
-        os.path.abspath(args.path),
-        SecurityContentBasicBuilder(),
-        SecurityContentDetectionBuilder(),
-        SecurityContentDirector()
-    )
 
     if args.product == "ESCU":
         generate_input_dto = GenerateInputDto(
@@ -129,6 +133,7 @@ def generate(args) -> None:
             SecurityContentProduct.API
         )
     else:
+        print("making dto")
         generate_input_dto = GenerateInputDto(
             os.path.abspath(args.output),
             factory_input_dto,
@@ -136,7 +141,6 @@ def generate(args) -> None:
             ObjToYmlAdapter(),
             SecurityContentProduct.SSA
         ) 
-
     generate = Generate()
     generate.execute(generate_input_dto)
 
@@ -150,25 +154,29 @@ def validate(args) -> None:
         print("ERROR: invalid product. valid products are all, ESCU or SSA.")
         sys.exit(1)
 
-    factory_input_dto = FactoryInputDto(
-        os.path.abspath(args.path),
-        SecurityContentBasicBuilder(),
-        SecurityContentDetectionBuilder(force_cached_or_offline=args.cached_and_offline, check_references=args.check_references),
-        SecurityContentStoryBuilder(check_references=args.check_references),
-        SecurityContentBaselineBuilder(check_references=args.check_references),
-        SecurityContentInvestigationBuilder(check_references=args.check_references),
-        SecurityContentPlaybookBuilder(check_references=args.check_references),
-        SecurityContentDirector(),
-        AttackEnrichment.get_attack_lookup(force_cached_or_offline=args.cached_and_offline)
-    )
-
-    ba_factory_input_dto = BAFactoryInputDto(
-        os.path.abspath(args.path),
-        SecurityContentBasicBuilder(),
-        SecurityContentDetectionBuilder(force_cached_or_offline = False, check_references=args.check_references),
-        SecurityContentDirector()
-    )
-
+    #Save runtime by only generating the required factory inputs
+    factory_input_dto = None
+    ba_factory_input_dto = None
+    if args.product in ["ESCU", "all"]:
+        factory_input_dto = FactoryInputDto(
+            os.path.abspath(args.path),
+            SecurityContentBasicBuilder(),
+            SecurityContentDetectionBuilder(force_cached_or_offline=args.cached_and_offline, check_references=args.check_references),
+            SecurityContentStoryBuilder(check_references=args.check_references),
+            SecurityContentBaselineBuilder(check_references=args.check_references),
+            SecurityContentInvestigationBuilder(check_references=args.check_references),
+            SecurityContentPlaybookBuilder(check_references=args.check_references),
+            SecurityContentDirector(),
+            AttackEnrichment.get_attack_lookup(force_cached_or_offline=args.cached_and_offline)
+        )
+    if args.product in ["SSA", "all"]:
+        ba_factory_input_dto = BAFactoryInputDto(
+            os.path.abspath(args.path),
+            SecurityContentBasicBuilder(),
+            SecurityContentDetectionBuilder(force_cached_or_offline = args.cached_and_offline, check_references=args.check_references),
+            SecurityContentDirector()
+        )
+    
     if args.product == "ESCU" or args.product == "all":
         validate_input_dto = ValidateInputDto(
             factory_input_dto,
@@ -193,7 +201,7 @@ def doc_gen(args) -> None:
     factory_input_dto = FactoryInputDto(
         os.path.abspath(args.path),
         SecurityContentBasicBuilder(),
-        SecurityContentDetectionBuilder(),
+        SecurityContentDetectionBuilder(force_cached_or_offline=args.cached_and_offline),
         SecurityContentStoryBuilder(),
         SecurityContentBaselineBuilder(),
         SecurityContentInvestigationBuilder(),
@@ -231,7 +239,7 @@ def reporting(args) -> None:
     factory_input_dto = FactoryInputDto(
         os.path.abspath(args.path),
         SecurityContentBasicBuilder(),
-        SecurityContentDetectionBuilder(),
+        SecurityContentDetectionBuilder(force_cached_or_offline=args.cached_and_offline),
         SecurityContentStoryBuilder(),
         SecurityContentBaselineBuilder(),
         SecurityContentInvestigationBuilder(),

From e7580faf5f7bb20f90215061089fd73fd4ad0f71 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 10 May 2022 17:08:24 -0700
Subject: [PATCH 21/38] Added caching of the results of link-verification to
 the --cached_and_offline argument.  This means that if a link returns
 successfully and you run the tool again, the link won't be checked again. 
 However, this means that if that cache sticks around for a long time, the
 link may actually stop returning but we won't continue checking it.  Still,
 this makes it significantly easier to quickly find broken links, fix the
 broken links, and then run the tool again in a short period of time, only
 re-checking the broken links that have been fixed.  This makes repairing and
 checking broken links a must faster workflow.

---
 .../domain/entities/link_validator.py         | 61 +++++++++++++++++--
 contentctl.py                                 | 14 +++++
 stories/hidden_cobra_malware.yml              |  2 +-
 stories/netsh_abuse.yml                       |  2 +-
 stories/ransomware.yml                        |  2 +-
 .../router_and_infrastructure_security.yml    |  2 +-
 stories/suspicious_gcp_storage_activities.yml |  2 +-
 stories/suspicious_wmi_use.yml                |  2 +-
 stories/unusual_processes.yml                 |  2 +-
 9 files changed, 76 insertions(+), 13 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
index 1f35c9e724..5bd4c34f23 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
@@ -8,12 +8,15 @@
 import urllib3, urllib3.exceptions
 import time
 import abc
+import pickle 
+import os
 
 DEFAULT_USER_AGENT_STRING = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36"
+ALLOWED_HTTP_CODES = [200]
 class LinkStats(BaseModel):
     #Static Values
     method: Callable = requests.get
-    allowed_http_codes: list[int] = [200] 
+    allowed_http_codes: list[int] = ALLOWED_HTTP_CODES 
     access_count: int = 1 #when constructor is called, it has been accessed once!
     timeout_seconds: int = 15
     allow_redirects: bool = True
@@ -89,7 +92,52 @@ class LinkValidator(abc.ABC):
     total_checks: int = 0
     #cache: dict[str,LinkStats] = {}
 
+    use_file_cache: bool = False
+    reference_cache_file: str ="lookups/REFERENCE_CACHE.PICKLE"
+
+    @staticmethod
+    def initialize_cache(use_file_cache: bool = False):
+        LinkValidator.use_file_cache = use_file_cache
+        if use_file_cache is False:
+            return
+        if not os.path.exists(LinkValidator.reference_cache_file):
+            print(f"Reference cache not found. Creating cache at {LinkValidator.reference_cache_file} after this run is complete")
+            return
+        #Load the cache into memory
+        try:
+            with open(LinkValidator.reference_cache_file, "rb") as cache_file:
+                LinkValidator.cache = pickle.load(cache_file)
+                print("reference cache found!")
+        except Exception as e:
+            #Most common error will be that the file did not exist
+            print(f"Reference cache read error: {str(e)}. Creating cache at {LinkValidator.reference_cache_file} after this run is complete")
     
+    @staticmethod
+    def dump_cache():
+        if LinkValidator.use_file_cache is False:
+            #No need to dump the cache
+            return
+            
+        try:
+            
+            #If an entry failed, we do not want to cache its success! So remove failed 
+            #refs from the dict before caching it
+            failed_refs = []
+            for ref in LinkValidator.cache.keys():
+                if LinkValidator.cache[ref].status_code not in ALLOWED_HTTP_CODES:
+                    failed_refs.append(ref)
+                    #can't remove it here because this will throw an error:
+                    #cannot change size of dictionary while iterating over it
+            for ref in failed_refs:
+                del(LinkValidator.cache[ref])
+            
+            with open(LinkValidator.reference_cache_file, "wb") as cache_file:
+                 pickle.dump(LinkValidator.cache, cache_file)
+        except Exception as e:
+            print(f"Error writing to the reference cache at {LinkValidator.reference_cache_file}: {str(e)}")
+        
+            
+
     @staticmethod
     def validate_reference(reference: str, referencing_file:str, raise_exception_if_failure: bool = False) -> bool:
         LinkValidator.total_checks += 1
@@ -108,8 +156,9 @@ def validate_reference(reference: str, referencing_file:str, raise_exception_if_
             return False
     @staticmethod
     def print_link_validation_errors():
-        for k in LinkValidator.cache.keys():
-            if LinkValidator.cache[k].valid is False:
-                print(f"Link {k} invalid with HTTP Status Code [{LinkValidator.cache[k].status_code}] and referenced by the following files:")
-                for ref in LinkValidator.cache[k].referencing_files:
-                    print(f"\t* {ref}")
+        failures = [LinkValidator.cache[k] for k in LinkValidator.cache if LinkValidator.cache[k].valid is False]
+        failures.sort(key=lambda d: d.status_code)
+        for failure in failures:
+            print(f"Link {failure.reference} invalid with HTTP Status Code [{failure.status_code}] and referenced by the following files:")
+            for ref in failure.referencing_files:
+                print(f"\t* {ref}")
diff --git a/contentctl.py b/contentctl.py
index 92ac2c1628..41a23127be 100644
--- a/contentctl.py
+++ b/contentctl.py
@@ -2,6 +2,8 @@
 import argparse
 import os
 
+from bin.contentctl_project.contentctl_core.domain.entities.link_validator import LinkValidator
+
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'bin/contentctl_project')))
 
 from bin.contentctl_project.contentctl_core.application.use_cases.content_changer import ContentChanger, ContentChangerInputDto
@@ -92,6 +94,10 @@ def generate(args) -> None:
         print("ERROR: invalid product. valid products are ESCU, SSA or API.")
         sys.exit(1)
 
+
+    if args.cached_and_offline:
+        LinkValidator.initialize_cache(args.cached_and_offline)
+
     #Save runtime by only generating the required factory inputs
     factory_input_dto = None
     ba_factory_input_dto = None
@@ -144,6 +150,8 @@ def generate(args) -> None:
     generate = Generate()
     generate.execute(generate_input_dto)
 
+    if args.cached_and_offline:
+        LinkValidator.dump_cache()
 
 def validate(args) -> None:
     if not args.product:
@@ -154,6 +162,9 @@ def validate(args) -> None:
         print("ERROR: invalid product. valid products are all, ESCU or SSA.")
         sys.exit(1)
 
+    if args.cached_and_offline:
+        LinkValidator.initialize_cache(args.cached_and_offline)
+
     #Save runtime by only generating the required factory inputs
     factory_input_dto = None
     ba_factory_input_dto = None
@@ -195,6 +206,8 @@ def validate(args) -> None:
         validate = Validate()
         validate.execute(validate_input_dto)
 
+    if args.cached_and_offline:
+        LinkValidator.dump_cache()
 
 
 def doc_gen(args) -> None:
@@ -323,6 +336,7 @@ def main(args):
 
     
     
+    
 
     # # parse them
     args = parser.parse_args()
diff --git a/stories/hidden_cobra_malware.yml b/stories/hidden_cobra_malware.yml
index 0c62805bc0..ee5087e413 100644
--- a/stories/hidden_cobra_malware.yml
+++ b/stories/hidden_cobra_malware.yml
@@ -35,7 +35,7 @@ narrative: 'North Korea''s government-sponsored "cyber army" has been slowly bui
   files associated with the malware. You can also use a search in this story to investigate
   activity that indicates that malware is sending email back to the attackers.'
 references:
-- https://www.us-cert.gov/HIDDEN-COBRA-North-Korean-Malicious-Cyber-Activity
+- https://web.archive.org/web/20191220004307/https://www.us-cert.gov/HIDDEN-COBRA-North-Korean-Malicious-Cyber-Activity
 - https://www.operationblockbuster.com/wp-content/uploads/2016/02/Operation-Blockbuster-Destructive-Malware-Report.pdf
 tags:
   analytic_story: Hidden Cobra Malware
diff --git a/stories/netsh_abuse.yml b/stories/netsh_abuse.yml
index 83a715d79a..f6bea10719 100644
--- a/stories/netsh_abuse.yml
+++ b/stories/netsh_abuse.yml
@@ -18,7 +18,7 @@ narrative: 'It is a common practice for attackers of all types to leverage nativ
 references:
 - https://docs.microsoft.com/en-us/previous-versions/tn-archive/bb490939(v=technet.10)
 - https://htmlpreview.github.io/?https://github.com/MatthewDemaske/blogbackup/blob/master/netshell.html
-- http://blog.jpcert.or.jp/2016/01/windows-commands-abused-by-attackers.html
+- https://blogs.jpcert.or.jp/en/2016/01/windows-commands-abused-by-attackers.html
 tags:
   analytic_story: Netsh Abuse
   category:
diff --git a/stories/ransomware.yml b/stories/ransomware.yml
index 1835219ce4..f7e4597f27 100644
--- a/stories/ransomware.yml
+++ b/stories/ransomware.yml
@@ -17,7 +17,7 @@ narrative: Ransomware is an ever-present risk to the enterprise, wherein an infe
   apply several techniques--such as those in this Analytic Story--to detect and or
   mitigate the effects of ransomware.
 references:
-- https://www.carbonblack.com/2017/06/28/carbon-black-threat-research-technical-analysis-petya-notpetya-ransomware/
+- https://web.archive.org/web/20190826231258/https://www.carbonblack.com/2017/06/28/carbon-black-threat-research-technical-analysis-petya-notpetya-ransomware/
 - https://www.splunk.com/blog/2017/06/27/closing-the-detection-to-mitigation-gap-or-to-petya-or-notpetya-whocares-.html
 tags:
   analytic_story: Ransomware
diff --git a/stories/router_and_infrastructure_security.yml b/stories/router_and_infrastructure_security.yml
index 1d6f3a161c..9c8a14a05f 100644
--- a/stories/router_and_infrastructure_security.yml
+++ b/stories/router_and_infrastructure_security.yml
@@ -17,7 +17,7 @@ narrative: 'Networking devices, such as routers and switches, are often overlook
   can obtain direct access to the company''s internal infrastructure&#151; effectively
   increasing the attack surface and accessing private services/data.'
 references:
-- https://www.fireeye.com/blog/executive-perspective/2015/09/the_new_route_toper.html
+- https://web.archive.org/web/20210420020040/https://www.fireeye.com/blog/executive-perspective/2015/09/the_new_route_toper.html
 - https://www.cisco.com/c/en/us/about/security-center/event-response/synful-knock.html
 tags:
   analytic_story: Router and Infrastructure Security
diff --git a/stories/suspicious_gcp_storage_activities.yml b/stories/suspicious_gcp_storage_activities.yml
index 3964d460bb..125b4d2f7a 100644
--- a/stories/suspicious_gcp_storage_activities.yml
+++ b/stories/suspicious_gcp_storage_activities.yml
@@ -13,7 +13,7 @@ narrative: Similar to other cloud providers, GCP operates on a shared responsibi
   on detecting things like open storage buckets (both read and write) along with storage
   bucket access from unfamiliar users and IP addresses.
 references:
-- https://cloud.google.com/blog/product/gcp/4-steps-for-hardening-your-cloud-storage-buckets-taking-charge-of-your-security
+- https://cloud.google.com/blog/products/gcp/4-steps-for-hardening-your-cloud-storage-buckets-taking-charge-of-your-security
 - https://rhinosecuritylabs.com/gcp/google-cloud-platform-gcp-bucket-enumeration/
 tags:
   analytic_story: Suspicious GCP Storage Activities
diff --git a/stories/suspicious_wmi_use.yml b/stories/suspicious_wmi_use.yml
index a54992248c..c8c5c7b3a5 100644
--- a/stories/suspicious_wmi_use.yml
+++ b/stories/suspicious_wmi_use.yml
@@ -21,7 +21,7 @@ narrative: WMI is a Microsoft infrastructure for management data and operations
   These details may provide insights related to how WMI was used and to what end.
 references:
 - https://www.blackhat.com/docs/us-15/materials/us-15-Graeber-Abusing-Windows-Management-Instrumentation-WMI-To-Build-A-Persistent%20Asynchronous-And-Fileless-Backdoor-wp.pdf
-- https://www.fireeye.com/blog/threat-research/2017/03/wmimplant_a_wmi_ba.html
+- https://web.archive.org/web/20210921091529/https://www.fireeye.com/blog/threat-research/2017/03/wmimplant_a_wmi_ba.html
 tags:
   analytic_story: Suspicious WMI Use
   category:
diff --git a/stories/unusual_processes.yml b/stories/unusual_processes.yml
index d25056b6d0..063a71c912 100644
--- a/stories/unusual_processes.yml
+++ b/stories/unusual_processes.yml
@@ -21,7 +21,7 @@ narrative: 'Being able to profile a host''s processes within your environment ca
   other hosts are affected. This extra information may provide clues that can help
   the analyst further investigate any suspicious activity.'
 references:
-- https://www.fireeye.com/blog/threat-research/2017/08/monitoring-windows-console-activity-part-two.html
+- https://web.archive.org/web/20210921093439/https://www.fireeye.com/blog/threat-research/2017/08/monitoring-windows-console-activity-part-two.html
 - https://www.splunk.com/pdfs/technical-briefs/advanced-threat-detection-and-response-tech-brief.pdf
 - https://www.sans.org/reading-room/whitepapers/logging/detecting-security-incidents-windows-workstation-event-logs-34262
 tags:

From bf27f47d1f8c833b91238327c814b48707dbf083 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 11 May 2022 12:00:47 -0700
Subject: [PATCH 22/38] Moved from using a pickled dictionary, loaded and
 written manually, to shelve.Shelf for Reference cache.  The interface is
 nearly identical, but it results in much cleaner code.  We will likely do the
 same in the next commit for CVE cache.

---
 .../domain/entities/link_validator.py         | 69 +++++++++----------
 .../builder/cve_enrichment.py                 |  2 +-
 contentctl.py                                 |  4 +-
 3 files changed, 35 insertions(+), 40 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
index 5bd4c34f23..a68d7436ab 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
@@ -8,8 +8,9 @@
 import urllib3, urllib3.exceptions
 import time
 import abc
-import pickle 
+
 import os
+import shelve
 
 DEFAULT_USER_AGENT_STRING = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.41 Safari/537.36"
 ALLOWED_HTTP_CODES = [200]
@@ -87,13 +88,13 @@ def check_reference(cls, values):
 
 
 class LinkValidator(abc.ABC):
-    cache: dict[str,LinkStats] = {}
+    cache: Union[dict[str,LinkStats], shelve.Shelf] = {}
     uncached_checks: int = 0
     total_checks: int = 0
     #cache: dict[str,LinkStats] = {}
 
     use_file_cache: bool = False
-    reference_cache_file: str ="lookups/REFERENCE_CACHE.PICKLE"
+    reference_cache_file: str ="lookups/REFERENCE_CACHE.db"
 
     @staticmethod
     def initialize_cache(use_file_cache: bool = False):
@@ -101,42 +102,36 @@ def initialize_cache(use_file_cache: bool = False):
         if use_file_cache is False:
             return
         if not os.path.exists(LinkValidator.reference_cache_file):
-            print(f"Reference cache not found. Creating cache at {LinkValidator.reference_cache_file} after this run is complete")
-            return
-        #Load the cache into memory
-        try:
-            with open(LinkValidator.reference_cache_file, "rb") as cache_file:
-                LinkValidator.cache = pickle.load(cache_file)
-                print("reference cache found!")
-        except Exception as e:
-            #Most common error will be that the file did not exist
-            print(f"Reference cache read error: {str(e)}. Creating cache at {LinkValidator.reference_cache_file} after this run is complete")
-    
-    @staticmethod
-    def dump_cache():
-        if LinkValidator.use_file_cache is False:
-            #No need to dump the cache
-            return
-            
+            print(f"Reference cache at {LinkValidator.reference_cache_file} not found. Creating it.")
+        
         try:
-            
-            #If an entry failed, we do not want to cache its success! So remove failed 
-            #refs from the dict before caching it
-            failed_refs = []
-            for ref in LinkValidator.cache.keys():
-                if LinkValidator.cache[ref].status_code not in ALLOWED_HTTP_CODES:
-                    failed_refs.append(ref)
-                    #can't remove it here because this will throw an error:
-                    #cannot change size of dictionary while iterating over it
-            for ref in failed_refs:
-                del(LinkValidator.cache[ref])
-            
-            with open(LinkValidator.reference_cache_file, "wb") as cache_file:
-                 pickle.dump(LinkValidator.cache, cache_file)
-        except Exception as e:
-            print(f"Error writing to the reference cache at {LinkValidator.reference_cache_file}: {str(e)}")
+            LinkValidator.cache = shelve.open(LinkValidator.reference_cache_file, flag='c', writeback=True)
+        except:
+            print(f"Failed to create the cache file {LinkValidator.reference_cache_file}.  Reference info will not be cached.")
+            LinkValidator.cache = {}
+
+        #Remove all of the failures to force those resources to be resolved again
+        failed_refs = []
+        for ref in LinkValidator.cache.keys():
+            if LinkValidator.cache[ref].status_code not in ALLOWED_HTTP_CODES:
+                failed_refs.append(ref)
+                #can't remove it here because this will throw an error:
+                #cannot change size of dictionary while iterating over it
+            else:
+                #Set the reference count to 0 and referencing files to empty set
+                LinkValidator.cache[ref].access_count = 0
+                LinkValidator.cache[ref].referencing_files = set()
         
-            
+        for ref in failed_refs:
+            del(LinkValidator.cache[ref])
+
+
+        
+             
+    @staticmethod
+    def close_cache():
+        if LinkValidator.use_file_cache:
+            LinkValidator.cache.close()
 
     @staticmethod
     def validate_reference(reference: str, referencing_file:str, raise_exception_if_failure: bool = False) -> bool:
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py
index f10457a1b8..8454b05185 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py
@@ -19,7 +19,7 @@ def cvesearch_helper(url:str, cve_id:str):
         dat = {}
     
     try:
-        print(f"Even though 'force_cached_or_offline' was enabled, we were unable to find {cve_id}, so we will attempt to resolve it at {CVESSEARCH_API_URL}")
+        #print(f"Even though 'force_cached_or_offline' was enabled, we were unable to find {cve_id}, so we will attempt to resolve it at {CVESSEARCH_API_URL}")
         cve = cvesearch_id_helper(url)
         result = cve.id(cve_id)
     except Exception as e:
diff --git a/contentctl.py b/contentctl.py
index 41a23127be..e806591e5c 100644
--- a/contentctl.py
+++ b/contentctl.py
@@ -151,7 +151,7 @@ def generate(args) -> None:
     generate.execute(generate_input_dto)
 
     if args.cached_and_offline:
-        LinkValidator.dump_cache()
+        LinkValidator.close_cache()
 
 def validate(args) -> None:
     if not args.product:
@@ -207,7 +207,7 @@ def validate(args) -> None:
         validate.execute(validate_input_dto)
 
     if args.cached_and_offline:
-        LinkValidator.dump_cache()
+        LinkValidator.close_cache()
 
 
 def doc_gen(args) -> None:

From fa9179fd9ae10a1b4cfaf9b14ea774fdee189226 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 11 May 2022 13:51:00 -0700
Subject: [PATCH 23/38] Added the shelve/shelf functionality to the CVE
 Enrichment Cache as well. This also replaced plain dict functionality and
 simplified the code.

---
 .../builder/cve_enrichment.py                 | 32 ++++++++++---------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py
index 8454b05185..198c353c90 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py
@@ -1,37 +1,39 @@
 
 from pycvesearch import CVESearch
 import functools
-import pickle
+import os
+import shelve
 
 CVESSEARCH_API_URL = 'https://cve.circl.lu'
 
-CVE_CACHE_FILENAME = "lookups/CVE_CACHE.PICKLE"
+CVE_CACHE_FILENAME = "lookups/CVE_CACHE.db"
 
 @functools.cache
 def cvesearch_helper(url:str, cve_id:str):
-    try:
-        with open(CVE_CACHE_FILENAME, 'rb') as p:
-            dat = pickle.load(p)
-            if cve_id in dat:
-                return dat[cve_id]
-    except Exception as e:
-        print(f"error loading pickle file {CVE_CACHE_FILENAME}- it probably didn't exist. we will create it")
-        dat = {}
+    if not os.path.exists(CVE_CACHE_FILENAME):
+        print(f"Reference cache at {CVE_CACHE_FILENAME} not found. Creating it.")
+    cache = shelve.open(CVE_CACHE_FILENAME, flag='c', writeback=True)
+    if cve_id in cache:
+        return cache[cve_id]
+    
+
     
     try:
-        #print(f"Even though 'force_cached_or_offline' was enabled, we were unable to find {cve_id}, so we will attempt to resolve it at {CVESSEARCH_API_URL}")
         cve = cvesearch_id_helper(url)
         result = cve.id(cve_id)
     except Exception as e:
         raise(Exception(f"The option 'force_cached_or_offline' was used, but {cve_id} not found in {CVE_CACHE_FILENAME} and unable to connect to {CVESSEARCH_API_URL}"))
-    with open(CVE_CACHE_FILENAME, 'wb') as p:
-        dat[cve_id] = result
-        pickle.dump(dat, p)
-    
+    if result is None:
+        raise(Exception(f'CveEnrichment for [ {cve_id} ] failed - CVE does not exist'))
+    cache[cve_id] = result
+    cache.close()
+
     return result
 
 @functools.cache
 def cvesearch_id_helper(url:str):
+    #The initial CVESearch call takes some time.
+    #We cache it to avoid making this call each time we need to do a lookup
     cve = CVESearch(CVESSEARCH_API_URL)
     return cve
 

From 926317f5c42efb89d9c512582d0446324daf8430 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Fri, 13 May 2022 07:51:31 -0400
Subject: [PATCH 24/38] Updated status print information around the creation of
 App Enrichment, CVE, and link validator caches.  Made sure to close CVE and
 App Enrichment caches after they are opened and accessed instead of relying
 on Python to close them for us when their reference count is zero. Also
 tested to ensure that if a previous validate and generate succeeded, it can
 be run again with no internet access! This is perfect not just for
 performance reasons, but offline/disconnected environments.

---
 .../domain/entities/link_validator.py         |  2 +-
 .../builder/cve_enrichment.py                 |  6 ++--
 .../builder/splunk_app_enrichment.py          | 31 +++++++++----------
 3 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
index a68d7436ab..a022f2a963 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/link_validator.py
@@ -102,7 +102,7 @@ def initialize_cache(use_file_cache: bool = False):
         if use_file_cache is False:
             return
         if not os.path.exists(LinkValidator.reference_cache_file):
-            print(f"Reference cache at {LinkValidator.reference_cache_file} not found. Creating it.")
+            print(f"Cache at {LinkValidator.reference_cache_file} not found - Creating it.")
         
         try:
             LinkValidator.cache = shelve.open(LinkValidator.reference_cache_file, flag='c', writeback=True)
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py
index 198c353c90..0e509377a5 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/cve_enrichment.py
@@ -11,10 +11,12 @@
 @functools.cache
 def cvesearch_helper(url:str, cve_id:str):
     if not os.path.exists(CVE_CACHE_FILENAME):
-        print(f"Reference cache at {CVE_CACHE_FILENAME} not found. Creating it.")
+        print(f"Cache at {CVE_CACHE_FILENAME} not found - Creating it.")
     cache = shelve.open(CVE_CACHE_FILENAME, flag='c', writeback=True)
     if cve_id in cache:
-        return cache[cve_id]
+        req = cache[cve_id]
+        cache.close()
+        return req
     
 
     
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py
index 65cc4dc608..8dbcab908f 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/splunk_app_enrichment.py
@@ -3,34 +3,31 @@
 import json
 import functools
 import pickle
+import shelve
+import os
 
 SPLUNKBASE_API_URL = "https://apps.splunk.com/api/apps/entriesbyid/"
 
-APP_ENRICHMENT_CACHE_FILENAME = "lookups/APP_ENRICHMENT_CACHE.PICKLE"
+APP_ENRICHMENT_CACHE_FILENAME = "lookups/APP_ENRICHMENT_CACHE.db"
 
 @functools.cache
 def requests_get_helper(url:str)->bytes:
+    if not os.path.exists(APP_ENRICHMENT_CACHE_FILENAME):
+        print(f"Cache at {APP_ENRICHMENT_CACHE_FILENAME} not found - Creating it.")
+    cache = shelve.open(APP_ENRICHMENT_CACHE_FILENAME, flag='c', writeback=True)
+    if url in cache:
+        req_content = cache[url]
+        cache.close()
+        return req_content
     try:
-        with open(APP_ENRICHMENT_CACHE_FILENAME, 'rb') as p:
-            dat = pickle.load(p)
-            if url in dat:
-                return dat[url]
-    except Exception as e:
-        print(f"error loading pickle file {APP_ENRICHMENT_CACHE_FILENAME}- it probably didn't exist. we will create it")
-        dat = {}
-
-    try:
-        print(f"Even though 'force_cached_or_offline' is enabled, the url {url} was not found. We will try to resolve it.")
         req = requests.get(url)
-        with open(APP_ENRICHMENT_CACHE_FILENAME, 'wb') as p:
-            dat[url] = req.content
-            pickle.dump(dat, p)
+        req_content = req.content
+        cache[url] = req_content
+        cache.close()
+        return req_content
     except Exception as e:
         raise(Exception(f"ERROR - Failed to get Splunk App Enrichment at {SPLUNKBASE_API_URL}"))
 
-    return req.content
-
-
 
 class SplunkAppEnrichment():
 

From a74ebee88f84e5e149ee816f7a1facf748698910 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 22 Jun 2022 13:44:17 -0700
Subject: [PATCH 25/38] Added the --skip_enrichment option to the contentctl
 tool.  This can save a huge amount of time when generating an app.  Also
 added a progress meter when the documentation is being generated. Finally,
 correctly initialized the default values of some class member fields in order
 to get all the contentctl steps working.

---
 .../domain/entities/detection_tags.py         |  2 +-
 .../domain/entities/story_tags.py             |  8 ++---
 .../adapter/obj_to_md_adapter.py              | 24 +++++++++++----
 .../builder/attack_enrichment.py              |  6 +++-
 .../security_content_detection_builder.py     | 14 +++++++--
 contentctl.py                                 | 29 ++++++++++---------
 6 files changed, 57 insertions(+), 26 deletions(-)

diff --git a/bin/contentctl_project/contentctl_core/domain/entities/detection_tags.py b/bin/contentctl_project/contentctl_core/domain/entities/detection_tags.py
index e506ad08d9..51081a8e89 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/detection_tags.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/detection_tags.py
@@ -29,7 +29,7 @@ class DetectionTags(BaseModel):
     supported_tas: list = None
 
     # enrichment
-    mitre_attack_enrichments: list[MitreAttackEnrichment] = None
+    mitre_attack_enrichments: list[MitreAttackEnrichment] = []
     confidence_id: int = None
     impact_id: int = None
     context_ids: list = None
diff --git a/bin/contentctl_project/contentctl_core/domain/entities/story_tags.py b/bin/contentctl_project/contentctl_core/domain/entities/story_tags.py
index 14f905f8b8..639bf033c5 100644
--- a/bin/contentctl_project/contentctl_core/domain/entities/story_tags.py
+++ b/bin/contentctl_project/contentctl_core/domain/entities/story_tags.py
@@ -13,10 +13,10 @@ class StoryTags(BaseModel):
     usecase: str
 
     # enrichment
-    mitre_attack_enrichments: list[MitreAttackEnrichment] = None
-    mitre_attack_tactics: list = None
-    datamodels: list = None
-    kill_chain_phases: list = None
+    mitre_attack_enrichments: list[MitreAttackEnrichment] = []
+    mitre_attack_tactics: list = []
+    datamodels: list = []
+    kill_chain_phases: list = []
 
 
     @validator('product')
diff --git a/bin/contentctl_project/contentctl_infrastructure/adapter/obj_to_md_adapter.py b/bin/contentctl_project/contentctl_infrastructure/adapter/obj_to_md_adapter.py
index bbc3ccf846..e8cdfb5d03 100644
--- a/bin/contentctl_project/contentctl_infrastructure/adapter/obj_to_md_adapter.py
+++ b/bin/contentctl_project/contentctl_infrastructure/adapter/obj_to_md_adapter.py
@@ -1,5 +1,5 @@
 import os
-
+import asyncio
 
 from bin.contentctl_project.contentctl_core.application.adapter.adapter import Adapter
 from bin.contentctl_project.contentctl_core.domain.entities.enums.enums import SecurityContentType
@@ -7,8 +7,14 @@
 
 
 class ObjToMdAdapter(Adapter):
-
+    index = 0
+    files_to_write = 0
+    
     def writeObjects(self, objects: list, output_path: str, type: SecurityContentType = None) -> None:
+        self.files_to_write = sum([len(obj) for obj in objects])
+        self.index = 0
+        progress_percent = ((self.index+1)/self.files_to_write) * 100
+        print(f"\r{'Docgen Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
 
         attack_tactics = set()
         datamodels = set()
@@ -36,7 +42,7 @@ def writeObjects(self, objects: list, output_path: str, type: SecurityContentTyp
         self.writeNavigationPageObjects(sorted(list(datamodels)), output_path)
         self.writeNavigationPageObjects(sorted(list(attack_tactics)), output_path)
         self.writeNavigationPageObjects(sorted(list(categories)), output_path)
-
+        
         JinjaWriter.writeObjectsList('doc_story_page.j2', os.path.join(output_path, '_pages/stories.md'), sorted(objects[0], key=lambda x: x.name))
         self.writeObjectsMd(objects[0], os.path.join(output_path, '_stories'), 'doc_stories.j2')
 
@@ -45,8 +51,8 @@ def writeObjects(self, objects: list, output_path: str, type: SecurityContentTyp
 
         JinjaWriter.writeObjectsList('doc_playbooks_page.j2', os.path.join(output_path, '_pages/paybooks.md'), sorted(objects[2], key=lambda x: x.name))
         self.writeObjectsMd(objects[2], os.path.join(output_path, '_playbooks'), 'doc_playbooks.j2')
-
-
+        
+        print("Done!")
     def writeNavigationPageObjects(self, objects: list, output_path: str) -> None:
         for obj in objects:
             JinjaWriter.writeObject('doc_navigation_pages.j2', os.path.join(output_path, '_pages', obj.lower().replace(' ', '_') + '.md'),
@@ -57,8 +63,16 @@ def writeNavigationPageObjects(self, objects: list, output_path: str) -> None:
 
     def writeObjectsMd(self, objects, output_path: str, template_name: str) -> None:
         for obj in objects:
+            progress_percent = ((self.index+1)/self.files_to_write) * 100
+            self.index+=1
+            print(f"\r{'Docgen Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
+
             JinjaWriter.writeObject(template_name, os.path.join(output_path, obj.name.lower().replace(' ', '_') + '.md'), obj)
 
     def writeDetectionsMd(self, objects, output_path: str, template_name: str) -> None:
         for obj in objects:
+            progress_percent = ((self.index+1)/self.files_to_write) * 100
+            self.index+=1
+            print(f"\r{'Docgen Progress'.rjust(23)}: [{progress_percent:3.0f}%]...", end="", flush=True)
+
             JinjaWriter.writeObject(template_name, os.path.join(output_path, obj.date + '-' + obj.name.lower().replace(' ', '_') + '.md'), obj)
\ No newline at end of file
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py b/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py
index f23007dd8c..5a648fca71 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/attack_enrichment.py
@@ -13,12 +13,16 @@
 class AttackEnrichment():
 
     @classmethod
-    def get_attack_lookup(self, store_csv = None, force_cached_or_offline: bool = False) -> dict:
+    def get_attack_lookup(self, store_csv = None, force_cached_or_offline: bool = False, skip_enrichment:bool = False) -> dict:
         print("Getting MITRE Attack Enrichment Data. This may take some time...")
         attack_lookup = dict()
         file_path = os.path.join(os.path.dirname(__file__), '../../../../lookups/mitre_enrichment.csv')
 
+        if skip_enrichment is True:
+            print("Skipping enrichment")
+            return attack_lookup
         try:
+
             if force_cached_or_offline is True:
                 raise(Exception("WARNING - Using cached MITRE Attack Enrichment.  Attack Enrichment may be out of date. Only use this setting for offline environments and development purposes."))
             print(f"\r{'Client'.rjust(23)}: [{0:3.0f}%]...", end="", flush=True)
diff --git a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py
index 9e07f641d8..5924a6fc19 100644
--- a/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py
+++ b/bin/contentctl_project/contentctl_infrastructure/builder/security_content_detection_builder.py
@@ -20,10 +20,12 @@ class SecurityContentDetectionBuilder(DetectionBuilder):
     #used for CveEnrichment.enrich_cve and SplunkAppEnrichment.enrich_splunk_app
     force_cached_or_offline: bool 
     check_references: bool
+    skip_enrichment: bool
 
-    def __init__(self, force_cached_or_offline: bool = False, check_references: bool = False):
+    def __init__(self, force_cached_or_offline: bool = False, check_references: bool = False, skip_enrichment:bool = False):
         self.force_cached_or_offline = force_cached_or_offline
         self.check_references = check_references
+        self.skip_enrichment = skip_enrichment
 
     def setObject(self, path: str) -> None:
         yml_dict = YmlReader.load_file(path)
@@ -174,6 +176,10 @@ def addMitreAttackEnrichment(self, attack_enrichment: dict) -> None:
             if attack_enrichment:
                 if self.security_content_obj.tags.mitre_attack_id:
                     self.security_content_obj.tags.mitre_attack_enrichments = []
+                    if self.skip_enrichment:
+                        # We just want to make the above, mitre_attack_enrichments, and
+                        # empty list since we didn't actually fetch those enrichments
+                        return
                     for mitre_attack_id in self.security_content_obj.tags.mitre_attack_id:
                         if mitre_attack_id in attack_enrichment:
                             mitre_attack_enrichment = MitreAttackEnrichment(
@@ -185,7 +191,7 @@ def addMitreAttackEnrichment(self, attack_enrichment: dict) -> None:
                             self.security_content_obj.tags.mitre_attack_enrichments.append(mitre_attack_enrichment)
                         else:
                             #print("mitre_attack_id " + mitre_attack_id + " doesn't exist for detecction " + self.security_content_obj.name)
-                            raise ValueError("mitre_attack_id " + mitre_attack_id + " doesn't exist for detecction " + self.security_content_obj.name)
+                            raise ValueError("mitre_attack_id " + mitre_attack_id + " doesn't exist for detection " + self.security_content_obj.name)
 
 
     def addMacros(self, macros: list) -> None:
@@ -224,6 +230,8 @@ def addLookups(self, lookups: list) -> None:
 
 
     def addCve(self) -> None:
+        if self.skip_enrichment:
+            return None
         if self.security_content_obj:
             self.security_content_obj.cve_enrichment = []
             if self.security_content_obj.tags.cve:
@@ -231,6 +239,8 @@ def addCve(self) -> None:
                     self.security_content_obj.cve_enrichment.append(CveEnrichment.enrich_cve(cve, force_cached_or_offline = self.force_cached_or_offline))
 
     def addSplunkApp(self) -> None:
+        if self.skip_enrichment:
+            return None
         if self.security_content_obj:
             self.security_content_obj.splunk_app_enrichment = []
             if self.security_content_obj.tags.supported_tas:
diff --git a/contentctl.py b/contentctl.py
index e806591e5c..5b79db40f8 100644
--- a/contentctl.py
+++ b/contentctl.py
@@ -105,19 +105,19 @@ def generate(args) -> None:
         factory_input_dto = FactoryInputDto(
             os.path.abspath(args.path),
             SecurityContentBasicBuilder(),
-            SecurityContentDetectionBuilder(force_cached_or_offline=args.cached_and_offline),
+            SecurityContentDetectionBuilder(force_cached_or_offline=args.cached_and_offline, skip_enrichment=args.skip_enrichment),
             SecurityContentStoryBuilder(),
             SecurityContentBaselineBuilder(),
             SecurityContentInvestigationBuilder(),
             SecurityContentPlaybookBuilder(),
             SecurityContentDirector(),
-            AttackEnrichment.get_attack_lookup(force_cached_or_offline=args.cached_and_offline)
+            AttackEnrichment.get_attack_lookup(force_cached_or_offline=args.cached_and_offline, skip_enrichment=args.skip_enrichment)
         )
     if args.product in ["SSA", "API"]:
         ba_factory_input_dto = BAFactoryInputDto(
             os.path.abspath(args.path),
             SecurityContentBasicBuilder(),
-            SecurityContentDetectionBuilder(force_cached_or_offline = args.cached_and_offline),
+            SecurityContentDetectionBuilder(force_cached_or_offline = args.cached_and_offline, skip_enrichment=args.skip_enrichment),
             SecurityContentDirector()
         )
 
@@ -172,19 +172,19 @@ def validate(args) -> None:
         factory_input_dto = FactoryInputDto(
             os.path.abspath(args.path),
             SecurityContentBasicBuilder(),
-            SecurityContentDetectionBuilder(force_cached_or_offline=args.cached_and_offline, check_references=args.check_references),
+            SecurityContentDetectionBuilder(force_cached_or_offline=args.cached_and_offline, check_references=args.check_references, skip_enrichment=args.skip_enrichment),
             SecurityContentStoryBuilder(check_references=args.check_references),
             SecurityContentBaselineBuilder(check_references=args.check_references),
             SecurityContentInvestigationBuilder(check_references=args.check_references),
             SecurityContentPlaybookBuilder(check_references=args.check_references),
             SecurityContentDirector(),
-            AttackEnrichment.get_attack_lookup(force_cached_or_offline=args.cached_and_offline)
+            AttackEnrichment.get_attack_lookup(force_cached_or_offline=args.cached_and_offline, skip_enrichment=args.skip_enrichment)
         )
     if args.product in ["SSA", "all"]:
         ba_factory_input_dto = BAFactoryInputDto(
             os.path.abspath(args.path),
             SecurityContentBasicBuilder(),
-            SecurityContentDetectionBuilder(force_cached_or_offline = args.cached_and_offline, check_references=args.check_references),
+            SecurityContentDetectionBuilder(force_cached_or_offline = args.cached_and_offline, check_references=args.check_references, skip_enrichment=args.skip_enrichment),
             SecurityContentDirector()
         )
     
@@ -214,13 +214,13 @@ def doc_gen(args) -> None:
     factory_input_dto = FactoryInputDto(
         os.path.abspath(args.path),
         SecurityContentBasicBuilder(),
-        SecurityContentDetectionBuilder(force_cached_or_offline=args.cached_and_offline),
+        SecurityContentDetectionBuilder(force_cached_or_offline=args.cached_and_offline, skip_enrichment=args.skip_enrichment),
         SecurityContentStoryBuilder(),
         SecurityContentBaselineBuilder(),
         SecurityContentInvestigationBuilder(),
         SecurityContentPlaybookBuilder(),
         SecurityContentDirector(),
-        AttackEnrichment.get_attack_lookup(force_cached_or_offline=args.cached_and_offline)
+        AttackEnrichment.get_attack_lookup(force_cached_or_offline=args.cached_and_offline, skip_enrichment=args.skip_enrichment)
     )
 
     doc_gen_input_dto = DocGenInputDto(
@@ -235,14 +235,14 @@ def doc_gen(args) -> None:
 
 def new_content(args) -> None:
     if args.type == 'detection':
-        type = SecurityContentType.detections
+        contentType = SecurityContentType.detections
     elif args.type == 'story':
-        type = SecurityContentType.stories
+        contentType = SecurityContentType.stories
     else:
         print("ERROR: type " + args.type + " not supported")
         sys.exit(1)
 
-    new_content_factory_input_dto = NewContentFactoryInputDto(type)
+    new_content_factory_input_dto = NewContentFactoryInputDto(contentType)
     new_content_input_dto = NewContentInputDto(new_content_factory_input_dto, ObjToYmlAdapter())
     new_content = NewContent()
     new_content.execute(new_content_input_dto)
@@ -252,13 +252,13 @@ def reporting(args) -> None:
     factory_input_dto = FactoryInputDto(
         os.path.abspath(args.path),
         SecurityContentBasicBuilder(),
-        SecurityContentDetectionBuilder(force_cached_or_offline=args.cached_and_offline),
+        SecurityContentDetectionBuilder(force_cached_or_offline=args.cached_and_offline, skip_enrichment=args.skip_enrichment),
         SecurityContentStoryBuilder(),
         SecurityContentBaselineBuilder(),
         SecurityContentInvestigationBuilder(),
         SecurityContentPlaybookBuilder(),
         SecurityContentDirector(),
-        AttackEnrichment.get_attack_lookup(force_cached_or_offline=args.cached_and_offline)
+        AttackEnrichment.get_attack_lookup(force_cached_or_offline=args.cached_and_offline, skip_enrichment=args.skip_enrichment)
     )
 
     reporting_input_dto = ReportingInputDto(
@@ -282,6 +282,9 @@ def main(args):
                                         help="path to the Splunk Security Content folder",)
     parser.add_argument("--cached_and_offline", action=argparse.BooleanOptionalAction,
         help="Force cached/offline resources.  While this makes execution much faster, it may result in enrichment which is out of date. This is suitable for use only in development or disconnected environments.")
+    parser.add_argument("--skip_enrichment", action=argparse.BooleanOptionalAction,
+        help="Skip enrichment of CVEs.  This can significantly decrease the amount of time needed to run content_ctl.")
+
     parser.set_defaults(cached_and_offline=False, func=lambda _: parser.print_help())
 
     actions_parser = parser.add_subparsers(title="Splunk Security Content actions", dest="action")

From 130f5e3c991aedd9e140cdaec17b88f344c85279 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 22 Jun 2022 15:49:56 -0700
Subject: [PATCH 26/38] Simplified the build-and-validate workflow, collapsing
 several jobs into one.

---
 .github/workflows/build-and-validate.yml | 156 +++++++----------------
 1 file changed, 43 insertions(+), 113 deletions(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index cb6f09b7c0..e83b43cd22 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -40,7 +40,7 @@ jobs:
           exit 0
           fi
 
-  validate-content:
+  validate-and-build:
     #Note that the CircleCI job used a Container.  The way to do this with Github Actions
     #is to first start up a Virtual Machine, then we can by following:
     # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idcontainer
@@ -62,10 +62,6 @@ jobs:
           architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
 
 
-      #TODO: CircleCI restore_cache equivalent
-
-      #don't need to install python3 or python3-dev since it was handled by the action above?
-      #Also, no support for YAML anchors/aliases in Github Actions...
       - name: Install System Packages
         run: |
           sudo apt update -qq
@@ -76,24 +72,17 @@ jobs:
         run: |
           #Get the virtualenv set up
           rm -rf venv
-          python3 -m venv --clear venv
-          source venv/bin/activate
-          python3 -m pip install --upgrade pip
-          python3 -m pip install wheel
-          python3 -m pip install -q -r requirements.txt
+          python -m venv --clear .venv
+          source .venv/bin/activate
+          python -m pip install --upgrade pip
+          python -m pip install wheel
+          python -m pip install -q -r requirements.txt
       
-      - name: run validate
+      - name: content_ctl validate
         run: |
-          source venv/bin/activate
-          python3 contentctl.py -p . validate -pr ESCU
-          python3 contentctl.py -p . validate -pr SSA
-
-
-      # - name: Get CTI Repo for Mitre context
-      #   uses:  actions/checkout@v2
-      #   with:
-      #     repository: mitre/cti #check out https://github.com/mitre/cti.git, defaults to HEAD
-      #     path: "cti/"
+          source .venv/bin/activate
+          python contentctl.py -p . validate -pr ESCU
+          python contentctl.py -p . validate -pr SSA
 
 
       #Now generate the documentation (uses Node)
@@ -101,65 +90,34 @@ jobs:
         with:
           node-version: '14' #can easily be changed to a different version
       
-      - name: Generate documentation
+      - name: contentctl docgen
         run: |
-          ls -lah
-          #Enter the virtualenv and run the docgen
-          source venv/bin/activate
-          python3 contentctl.py -p . docgen -o docs
+          source .venv/bin/activate
+          python contentctl.py -p . docgen -o docs
           #Now generate the spec docs
           npm install -g @adobe/jsonschema2md
           jsonschema2md -d spec -o docs/spec -f yaml -e spec.json -x -
           #Clean up extra properties on docs
           rm -rf docs/spec/*-*.md
-          echo "****** BRANCH INFORMATION ******"
-          git branch
-          git branch --show-current
-  
-  build-sources:
-    runs-on: ubuntu-latest
-    needs: validate-content
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v2
-
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
-          architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
-
-      - name: Install System Packages
-        run: |
-          sudo apt update -qq
-          sudo apt install jq -qq
       
-      - name: Install Python Dependencies
+      - name: contentctl generate
         run: |
-          #Get the virtualenv set up
-          rm -rf venv
-          python3 -m venv --clear venv
-          source venv/bin/activate
-          python3 -m pip install --upgrade pip
-          python3 -m pip install wheel
-          python3 -m pip install -q -r requirements.txt
-      
-      - name: Run Generate
-        run: |
-          source venv/bin/activate
+          source .venv/bin/activate
           rm -rf dist/escu/default/data/ui/panels/*.xml
-          python3 contentctl.py --path . generate --product ESCU --output dist/escu
-          python3 contentctl.py --path . generate --product SSA --output dist/ssa
+          python contentctl.py --path . generate --product ESCU --output dist/escu
+          python contentctl.py --path . generate --product SSA --output dist/ssa
       
       - name: Copy lookups .mlmodel files
         run: |
           cd lookups
           count=`ls -1 *.mlmodel 2>/dev/null | wc -l`
           if [ $count != 0 ]
-          then cp -rv *.mlmodel ../dist/escu/lookups
-          else echo "No mlmodel files to copy"
+          then 
+             cp -rv *.mlmodel ../dist/escu/lookups
+          else 
+             echo "No mlmodel files to copy"
           fi
       
-      #Tag is '' for non-tagged push and the tag name for a tagged release
       - name: Set tag
         id: vars
         run: |
@@ -188,57 +146,24 @@ jobs:
             sed -i "s/^version = .*$/version = $CONTENT_VERSION/g" dist/escu/default/app.conf
             sed -i "s/\"version\": .*$/\"version\": \"$CONTENT_VERSION\"/g" dist/escu/app.manifest
             sed -i "s/version = .*$/version = $CONTENT_VERSION/g" dist/escu/default/content-version.conf
-            tar -czf content-pack-build-escu.tar.gz dist/escu/*
+            mkdir build
+            tar -czf build/content-pack-build-escu.tar.gz dist/escu/*
             # update build number and version for ssa
-            tar -czf content-pack-build-ssa.tar.gz dist/ssa/*
+            tar -czf build/content-pack-build-ssa.tar.gz dist/ssa/*
       
-      - name: Persist to Workspace
-        uses: actions/upload-artifact@v2
-        with:
-          name: content-pack-build
-          path: |
-            content-pack-build-escu.tar.gz
-            content-pack-build-ssa.tar.gz
-
-  build-package:
-    runs-on: ubuntu-latest
-    needs: [validate-content, build-sources]
-
-    steps:
-      - uses: actions/download-artifact@v2
-        with:
-          name: content-pack-build
-          path: build/
-
-      #This explicitly uses a different version of python (2.7)
-      - uses: actions/setup-python@v2
-        with:
-          python-version: '2.7' #Available versions here - https://github.com/actions/python-versions/releases  easy to change/make a matrix/use pypy
-          architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified
-
-      - name: Get virtualenv for Python 2.7
-        run: |
-          sudo apt install virtualenv
-      
-      - name: Grab Splunk Packaging Toolkit
+      - name: Download and Install Splunk Packaging Toolkit
         run : |
-          curl -Ls https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-0.9.0.tar.gz -o splunk-packaging-toolkit-latest.tar.gz
+          source .venv/bin/activate
+          curl -Ls https://download.splunk.com/misc/packaging-toolkit/splunk-packaging-toolkit-1.0.1.tar.gz -o splunk-packaging-toolkit-latest.tar.gz
           mkdir slim-latest
           tar -zxf splunk-packaging-toolkit-latest.tar.gz -C slim-latest --strip-components=1
-      
-      - name: Install Splunk Packaging Toolkit (slim)
-        run: |
           cd slim-latest
-          virtualenv --python=/usr/bin/python2.7 --clear venv
-          source venv/bin/activate
-          python2 -m pip install --upgrade pip
-          python2 -m pip install wheel
-          python2 -m pip install semantic_version
-          python2 -m pip install .
+          python -m pip install .
+          cd ..
       
-      - name: Create a .spl for this Build Using Slim
+      - name: Build ESCU
         run: |
-          source slim-latest/venv/bin/activate
+          source .venv/bin/activate
           cd build
           tar -zxf content-pack-build-escu.tar.gz
           tar -zxf content-pack-build-ssa.tar.gz
@@ -253,7 +178,8 @@ jobs:
           tar -zcf upload/SSA_Content-NO_SLIM.tar.gz SSA_Content  
           cp upload/SSA_Content-*.tar.gz SSA_Content-latest.tar.gz
           sha256sum SSA_Content-latest.tar.gz >> checksum.txt
-          
+
+
       - name: store_artifacts
         uses: actions/upload-artifact@v2
         with:
@@ -269,13 +195,17 @@ jobs:
             build/DA-ESS-ContentUpdate-latest.tar.gz
             build/SSA_Content-latest.tar.gz
             build/checksum.txt
+
+
+      
+
   
 
   #Everything below this line should ONLY run on a tag and nothing else
   #We still want all of the above checks to run and pass before running these
   run-appinspect:
     runs-on: ubuntu-latest
-    needs: [validate-content, build-sources, build-package]
+    needs: [validate-content, validate-and-build]
     #Only run when tagged
     if: startsWith(github.ref, 'refs/tags/')
     steps:
@@ -331,7 +261,7 @@ jobs:
   
   create-report:
     runs-on: ubuntu-latest
-    needs: [validate-content, build-sources, build-package, run-appinspect]
+    needs: [validate-content, validate-and-build, run-appinspect]
     #Only run when tagged
     if: startsWith(github.ref, 'refs/tags/')
     steps:
@@ -379,7 +309,7 @@ jobs:
   
   update-sources-github:
     runs-on: ubuntu-latest
-    needs: [validate-content, build-sources, build-package, run-appinspect, create-report]
+    needs: [validate-content, validate-and-build, run-appinspect, create-report]
     #Only run when tagged
     if: startsWith(github.ref, 'refs/tags/')
     steps:
@@ -448,7 +378,7 @@ jobs:
     #Github-maintained release action is in archived state: https://github.com/actions/create-release
     #They recommend several and we use the following with the most stars: https://github.com/softprops/action-gh-release
     runs-on: ubuntu-latest
-    needs: [validate-content, build-sources, build-package, run-appinspect, create-report, update-sources-github]
+    needs: [validate-content, validate-and-build, run-appinspect, create-report]
     #Only run when tagged
     if: startsWith(github.ref, 'refs/tags/')
     steps:
@@ -488,7 +418,7 @@ jobs:
   
   attack-range-update:
     runs-on: ubuntu-latest
-    needs: [validate-content, build-sources, build-package, run-appinspect, create-report, update-sources-github, publish-github-release]
+    needs: [validate-content, validate-and-build, run-appinspect, create-report]
     #Only run when tagged
     if: startsWith(github.ref, 'refs/tags/')
     steps:
@@ -514,7 +444,7 @@ jobs:
   
   master-api-update:
     runs-on: ubuntu-latest
-    needs: [validate-content, build-sources, build-package, run-appinspect, create-report, update-sources-github, publish-github-release, attack-range-update]
+    needs: [validate-content, validate-and-build, run-appinspect, create-report]
     #Only run when tagged
     if: startsWith(github.ref, 'refs/tags/')
     steps:

From 4f6d30dc5199666d18f5eeb76e33eef21777a0b5 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 22 Jun 2022 15:58:21 -0700
Subject: [PATCH 27/38] Fixes to needs dependencies after updating job names

---
 .github/workflows/build-and-validate.yml | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index e83b43cd22..2528cd6552 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -66,7 +66,7 @@ jobs:
         run: |
           sudo apt update -qq
           sudo apt install jq -qq
-      #TODO: CircleCI save_cache equivalent
+      
 
       - name: Install Python Dependencies
         run: |
@@ -196,16 +196,12 @@ jobs:
             build/SSA_Content-latest.tar.gz
             build/checksum.txt
 
-
-      
-
-  
-
+          
   #Everything below this line should ONLY run on a tag and nothing else
   #We still want all of the above checks to run and pass before running these
   run-appinspect:
     runs-on: ubuntu-latest
-    needs: [validate-content, validate-and-build]
+    needs: [validate-and-build]
     #Only run when tagged
     if: startsWith(github.ref, 'refs/tags/')
     steps:
@@ -261,7 +257,7 @@ jobs:
   
   create-report:
     runs-on: ubuntu-latest
-    needs: [validate-content, validate-and-build, run-appinspect]
+    needs: [validate-and-build, run-appinspect]
     #Only run when tagged
     if: startsWith(github.ref, 'refs/tags/')
     steps:
@@ -309,7 +305,7 @@ jobs:
   
   update-sources-github:
     runs-on: ubuntu-latest
-    needs: [validate-content, validate-and-build, run-appinspect, create-report]
+    needs: [validate-and-build, run-appinspect, create-report]
     #Only run when tagged
     if: startsWith(github.ref, 'refs/tags/')
     steps:
@@ -378,7 +374,7 @@ jobs:
     #Github-maintained release action is in archived state: https://github.com/actions/create-release
     #They recommend several and we use the following with the most stars: https://github.com/softprops/action-gh-release
     runs-on: ubuntu-latest
-    needs: [validate-content, validate-and-build, run-appinspect, create-report]
+    needs: [validate-and-build, run-appinspect, create-report]
     #Only run when tagged
     if: startsWith(github.ref, 'refs/tags/')
     steps:
@@ -418,7 +414,7 @@ jobs:
   
   attack-range-update:
     runs-on: ubuntu-latest
-    needs: [validate-content, validate-and-build, run-appinspect, create-report]
+    needs: [validate-and-build, run-appinspect, create-report]
     #Only run when tagged
     if: startsWith(github.ref, 'refs/tags/')
     steps:
@@ -444,7 +440,7 @@ jobs:
   
   master-api-update:
     runs-on: ubuntu-latest
-    needs: [validate-content, validate-and-build, run-appinspect, create-report]
+    needs: [validate-and-build, run-appinspect, create-report]
     #Only run when tagged
     if: startsWith(github.ref, 'refs/tags/')
     steps:

From d858cd6875d6b107828cb7f7172229e7c140bcd7 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Wed, 22 Jun 2022 16:06:29 -0700
Subject: [PATCH 28/38] included the --skip_enrichment flag

---
 .github/workflows/build-and-validate.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index 2528cd6552..269fb052b0 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -81,8 +81,8 @@ jobs:
       - name: content_ctl validate
         run: |
           source .venv/bin/activate
-          python contentctl.py -p . validate -pr ESCU
-          python contentctl.py -p . validate -pr SSA
+          python contentctl.py -p . --skip_enrichment validate -pr ESCU
+          python contentctl.py -p . --skip_enrichment validate -pr SSA
 
 
       #Now generate the documentation (uses Node)
@@ -93,7 +93,7 @@ jobs:
       - name: contentctl docgen
         run: |
           source .venv/bin/activate
-          python contentctl.py -p . docgen -o docs
+          python contentctl.py -p . --skip_enrichment docgen -o docs
           #Now generate the spec docs
           npm install -g @adobe/jsonschema2md
           jsonschema2md -d spec -o docs/spec -f yaml -e spec.json -x -
@@ -104,8 +104,8 @@ jobs:
         run: |
           source .venv/bin/activate
           rm -rf dist/escu/default/data/ui/panels/*.xml
-          python contentctl.py --path . generate --product ESCU --output dist/escu
-          python contentctl.py --path . generate --product SSA --output dist/ssa
+          python contentctl.py --path . --skip_enrichment generate --product ESCU --output dist/escu
+          python contentctl.py --path . --skip_enrichment generate --product SSA --output dist/ssa
       
       - name: Copy lookups .mlmodel files
         run: |
@@ -289,7 +289,7 @@ jobs:
       - name: run reporting
         run: |
           source venv/bin/activate
-          python3 contentctl.py -p . reporting
+          python3 contentctl.py -p . --skip_enrichment reporting
       
       #Official, Verified Amazon-AWS Github Account Provided Action
       - uses: aws-actions/configure-aws-credentials@v1
@@ -350,7 +350,7 @@ jobs:
       - name: Run doc-gen
         run: |
           source venv/bin/activate
-          python3 contentctl.py -p . docgen -o docs
+          python3 contentctl.py -p . --skip_enrichment docgen -o docs
 
       - name: Update github with new docs and package bits
         run: |
@@ -468,7 +468,7 @@ jobs:
       - name: Create YML to JSON Folder
         run: |
           source venv/bin/activate
-          python3 contentctl.py --path . generate --product API --output dist/api
+          python3 contentctl.py --path . --skip_enrichment generate --product API --output dist/api
 
       - name: Generate content version and timestamp JSON
         run : |

From 82af76d67fbbbd94682305946cffce5426ead7ee Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Thu, 23 Jun 2022 13:50:15 -0700
Subject: [PATCH 29/38] Added ability to add or remove the --skip_enrichment
 argument to content_ctl based on whether or not we are running CI/CD workflow
 on a tagged release

---
 .github/workflows/build-and-validate.yml | 43 +++++++++++++-----------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index 269fb052b0..58012adac9 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -47,6 +47,22 @@ jobs:
     runs-on: ubuntu-latest
     needs: [validate-tag-if-present]
     steps:
+
+      - name: Configure Enrichment and Tag
+        id: vars
+        run: |
+          if [ $(echo ${GITHUB_REF} | grep "^refs/tags/*") ]; then
+            #failed to find the refs/tags/ beginning, grab and set the tag
+            echo "Release is TAGGED!"
+            echo "::set-output name=tag::${GITHUB_REF#refs/tags/}"
+            echo "::set-output name=skip_enrichment_var::"
+          else
+            #Not a tagged relese
+            echo "Release is NOT TAGGED!"
+            echo "::set-output name=tag::"
+            echo "::set-output name=skipenrichment_var::--skip_enrichment"
+          fi
+
       #Previous config chose which branch/tag to operate on.
       #I think Github is smart enough to choose based on whether it's a pull request or push + other info?
       - name: Check out the repository code
@@ -81,8 +97,8 @@ jobs:
       - name: content_ctl validate
         run: |
           source .venv/bin/activate
-          python contentctl.py -p . --skip_enrichment validate -pr ESCU
-          python contentctl.py -p . --skip_enrichment validate -pr SSA
+          python contentctl.py -p . ${{ steps.vars.outputs.skip_enrichment_var }} validate -pr ESCU
+          python contentctl.py -p . ${{ steps.vars.outputs.skip_enrichment_var }} validate -pr SSA
 
 
       #Now generate the documentation (uses Node)
@@ -104,8 +120,8 @@ jobs:
         run: |
           source .venv/bin/activate
           rm -rf dist/escu/default/data/ui/panels/*.xml
-          python contentctl.py --path . --skip_enrichment generate --product ESCU --output dist/escu
-          python contentctl.py --path . --skip_enrichment generate --product SSA --output dist/ssa
+          python contentctl.py --path . ${{ steps.vars.outputs.skip_enrichment_var }} generate --product ESCU --output dist/escu
+          python contentctl.py --path . ${{ steps.vars.outputs.skip_enrichment_var }} --product SSA --output dist/ssa
       
       - name: Copy lookups .mlmodel files
         run: |
@@ -118,19 +134,6 @@ jobs:
              echo "No mlmodel files to copy"
           fi
       
-      - name: Set tag
-        id: vars
-        run: |
-          if [ $(echo ${GITHUB_REF} | grep "^refs/tags/*") ]; then
-            #failed to find the refs/tags/ beginning, grab and set the tag
-            echo "Release is TAGGED!"
-            echo "::set-output name=tag::${GITHUB_REF#refs/tags/}"
-          else
-            #Not a tagged relese
-            echo "Release is NOT TAGGED!"
-            echo "::set-output name=tag::"
-          fi
-      
       - name: Update Version and Build number
         run : |
             # check if tag is set, get build number from the tag if set
@@ -289,7 +292,7 @@ jobs:
       - name: run reporting
         run: |
           source venv/bin/activate
-          python3 contentctl.py -p . --skip_enrichment reporting
+          python3 contentctl.py -p . reporting
       
       #Official, Verified Amazon-AWS Github Account Provided Action
       - uses: aws-actions/configure-aws-credentials@v1
@@ -350,7 +353,7 @@ jobs:
       - name: Run doc-gen
         run: |
           source venv/bin/activate
-          python3 contentctl.py -p . --skip_enrichment docgen -o docs
+          python3 contentctl.py -p . docgen -o docs
 
       - name: Update github with new docs and package bits
         run: |
@@ -468,7 +471,7 @@ jobs:
       - name: Create YML to JSON Folder
         run: |
           source venv/bin/activate
-          python3 contentctl.py --path . --skip_enrichment generate --product API --output dist/api
+          python3 contentctl.py --path . generate --product API --output dist/api
 
       - name: Generate content version and timestamp JSON
         run : |

From b7955f84b9a8fdb5ce2225e9ba303210eaf84627 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Thu, 23 Jun 2022 13:51:56 -0700
Subject: [PATCH 30/38] Typo naming new variable in workflow.

---
 .github/workflows/build-and-validate.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index 58012adac9..18a3230006 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -60,7 +60,7 @@ jobs:
             #Not a tagged relese
             echo "Release is NOT TAGGED!"
             echo "::set-output name=tag::"
-            echo "::set-output name=skipenrichment_var::--skip_enrichment"
+            echo "::set-output name=skip_enrichment_var::--skip_enrichment"
           fi
 
       #Previous config chose which branch/tag to operate on.

From a133549d6cdc4be748c46b8b9364e5babe87e38d Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Thu, 23 Jun 2022 13:58:13 -0700
Subject: [PATCH 31/38] Fixing a call to content_ctl generate.  This passes
 initial testing using nektos act on my own system!

---
 .github/workflows/build-and-validate.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index 18a3230006..2ac03eec16 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -121,7 +121,7 @@ jobs:
           source .venv/bin/activate
           rm -rf dist/escu/default/data/ui/panels/*.xml
           python contentctl.py --path . ${{ steps.vars.outputs.skip_enrichment_var }} generate --product ESCU --output dist/escu
-          python contentctl.py --path . ${{ steps.vars.outputs.skip_enrichment_var }} --product SSA --output dist/ssa
+          python contentctl.py --path . ${{ steps.vars.outputs.skip_enrichment_var }} generate --product SSA --output dist/ssa
       
       - name: Copy lookups .mlmodel files
         run: |

From 4931458ff917ac15528b9df35a638805933a1c05 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Thu, 23 Jun 2022 15:02:26 -0700
Subject: [PATCH 32/38] Updated one broken reference link.

---
 detections/web/vmware_server_side_template_injection_hunt.yml   | 2 +-
 ..._workspace_one_freemarker_server_side_template_injection.yml | 2 +-
 .../vmware_server_side_injection_and_privilege_escalation.yml   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/detections/web/vmware_server_side_template_injection_hunt.yml b/detections/web/vmware_server_side_template_injection_hunt.yml
index 535d254d2c..62edaaec13 100644
--- a/detections/web/vmware_server_side_template_injection_hunt.yml
+++ b/detections/web/vmware_server_side_template_injection_hunt.yml
@@ -20,7 +20,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   by known good.
 known_false_positives: False positives may be present if the activity is blocked or was not successful. Filter known vulnerablity scanners. Filter as needed.
 references:
-  - https://www.cisa.gov/uscert/sites/default/files/publications/CISA_CSA_Threat_Actors_Chaining_VMware_Unpatched_Vulnerabilities_for_Full_System_Control.pdf
+  - https://www.cisa.gov/uscert/ncas/alerts/aa22-138b
   - https://github.com/wvu/metasploit-framework/blob/master/modules/exploits/linux/http/vmware_workspace_one_access_cve_2022_22954.rb
   - https://github.com/sherlocksecurity/VMware-CVE-2022-22954
   - https://www.vmware.com/security/advisories/VMSA-2022-0011.html
diff --git a/detections/web/vmware_workspace_one_freemarker_server_side_template_injection.yml b/detections/web/vmware_workspace_one_freemarker_server_side_template_injection.yml
index 3162fafb0d..b13bdb98c7 100644
--- a/detections/web/vmware_workspace_one_freemarker_server_side_template_injection.yml
+++ b/detections/web/vmware_workspace_one_freemarker_server_side_template_injection.yml
@@ -21,7 +21,7 @@ how_to_implement: To successfully implement this search, you need to be ingestin
   by known good.
 known_false_positives: False positives may be present if the activity is blocked or was not successful. Filter known vulnerablity scanners. Filter as needed.
 references:
-  - https://www.cisa.gov/uscert/sites/default/files/publications/CISA_CSA_Threat_Actors_Chaining_VMware_Unpatched_Vulnerabilities_for_Full_System_Control.pdf
+  - https://www.cisa.gov/uscert/ncas/alerts/aa22-138b
   - https://github.com/wvu/metasploit-framework/blob/master/modules/exploits/linux/http/vmware_workspace_one_access_cve_2022_22954.rb
   - https://github.com/sherlocksecurity/VMware-CVE-2022-22954
   - https://www.vmware.com/security/advisories/VMSA-2022-0011.html
diff --git a/stories/vmware_server_side_injection_and_privilege_escalation.yml b/stories/vmware_server_side_injection_and_privilege_escalation.yml
index b3b23f8355..da5ea81156 100644
--- a/stories/vmware_server_side_injection_and_privilege_escalation.yml
+++ b/stories/vmware_server_side_injection_and_privilege_escalation.yml
@@ -9,7 +9,7 @@ narrative: 'On April 6, 2022, VMware published VMSA-2022-0011, which discloses m
   To further exacerbate this issue, VMware also disclosed a local privilege escalation issue, CVE-2022-22960, which permits the attacker to gain root after exploiting CVE-2022-22954. Products affected include - VMware Workspace ONE Access (Access) 20.10.0.0 - 20.10.0.1, 21.08.0.0 - 21.08.0.1 and VMware Identity Manager (vIDM) 3.3.3 - 3.3.6.'
 references:
   - https://attackerkb.com/topics/BDXyTqY1ld/cve-2022-22954/rapid7-analysis
-  - https://www.cisa.gov/uscert/sites/default/files/publications/CISA_CSA_Threat_Actors_Chaining_VMware_Unpatched_Vulnerabilities_for_Full_System_Control.pdf 
+  - https://www.cisa.gov/uscert/ncas/alerts/aa22-138b 
 tags:
   analytic_story: VMware Server Side Injection and Privilege Escalation
   category:

From 0181c9fd63ef62723ff98ac8204781077de70cf8 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Thu, 23 Jun 2022 15:18:56 -0700
Subject: [PATCH 33/38] Update the docker detection testing code to use the
 latest contentctl from develop as well as skip enrichment.

---
 bin/docker_detection_tester/detection_testing_execution.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/docker_detection_tester/detection_testing_execution.py b/bin/docker_detection_tester/detection_testing_execution.py
index 2b58110445..01a457ae11 100644
--- a/bin/docker_detection_tester/detection_testing_execution.py
+++ b/bin/docker_detection_tester/detection_testing_execution.py
@@ -177,10 +177,10 @@ def generate_escu_app(persist_security_content: bool = False) -> str:
                     ". ./.venv/bin/activate",
                     "python -m pip install wheel",
                     "python -m pip install -r requirements.txt",
-                    "python3 contentctl.py --path . generate --product ESCU --output dist/escu"]
+                    "python3 ../../../contentctl.py --path . --skip_enrichment generate --product ESCU --output dist/escu"]
     else:
         commands = [". ./.venv/bin/activate",
-                    "python3 contentctl.py --path . generate --product ESCU --output dist/escu"]
+                    "python3 ../../../contentctl.py --path . --skip_enrichment generate --product ESCU --output dist/escu"]
     ret = subprocess.run("; ".join(commands),
                          shell=True, capture_output=True)
     if ret.returncode != 0:

From 050491a1cc2f81be212017341ca8d443d5ebb9cc Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Thu, 23 Jun 2022 15:28:24 -0700
Subject: [PATCH 34/38] Forgot to include a print statement, causing lines to
 be jumbled together.

---
 bin/docker_detection_tester/modules/utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bin/docker_detection_tester/modules/utils.py b/bin/docker_detection_tester/modules/utils.py
index 6671e16367..f4c850a39f 100644
--- a/bin/docker_detection_tester/modules/utils.py
+++ b/bin/docker_detection_tester/modules/utils.py
@@ -17,6 +17,7 @@ def download_file_from_http(url:str, destination_file:str, overwrite_file:bool=F
         with open(destination_file, "wb") as output:
             for piece in file_to_download.iter_content(chunk_size=chunk_size):
                 output.write(piece)
+        print("Done")
     except Exception as e:
         if verbose_print:
             print("FAILED")

From e818a0d1ed973f100124d815bb200fb084a5c1de Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Thu, 23 Jun 2022 15:35:56 -0700
Subject: [PATCH 35/38] Fixing print statement that was was not obeying the
 verbose argument.

---
 bin/docker_detection_tester/modules/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/docker_detection_tester/modules/utils.py b/bin/docker_detection_tester/modules/utils.py
index f4c850a39f..2d36239311 100644
--- a/bin/docker_detection_tester/modules/utils.py
+++ b/bin/docker_detection_tester/modules/utils.py
@@ -17,7 +17,8 @@ def download_file_from_http(url:str, destination_file:str, overwrite_file:bool=F
         with open(destination_file, "wb") as output:
             for piece in file_to_download.iter_content(chunk_size=chunk_size):
                 output.write(piece)
-        print("Done")
+        if verbose_print:
+            print("Done")
     except Exception as e:
         if verbose_print:
             print("FAILED")

From f75b406a2fc7687f78063750d85ac42626e357c7 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 28 Jun 2022 11:08:27 -0700
Subject: [PATCH 36/38] Suppress output of contentctl in CI/CD to test runtime
 constraints.

---
 .github/workflows/build-and-validate.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index 2732e4a7a7..de32ad1506 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -97,8 +97,8 @@ jobs:
       - name: content_ctl validate
         run: |
           source .venv/bin/activate
-          python contentctl.py -p . ${{ steps.vars.outputs.skip_enrichment_var }} validate -pr ESCU
-          python contentctl.py -p . ${{ steps.vars.outputs.skip_enrichment_var }} validate -pr SSA
+          python contentctl.py -p . ${{ steps.vars.outputs.skip_enrichment_var }} validate -pr ESCU > /dev/null
+          python contentctl.py -p . ${{ steps.vars.outputs.skip_enrichment_var }} validate -pr SSA > /dev/null
 
 
       #Now generate the documentation (uses Node)
@@ -120,8 +120,8 @@ jobs:
         run: |
           source .venv/bin/activate
           rm -rf dist/escu/default/data/ui/panels/*.xml
-          python contentctl.py --path . ${{ steps.vars.outputs.skip_enrichment_var }} generate --product ESCU --output dist/escu
-          python contentctl.py --path . ${{ steps.vars.outputs.skip_enrichment_var }} generate --product SSA --output dist/ssa
+          python contentctl.py --path . ${{ steps.vars.outputs.skip_enrichment_var }} generate --product ESCU --output dist/escu > /dev/null
+          python contentctl.py --path . ${{ steps.vars.outputs.skip_enrichment_var }} generate --product SSA --output dist/ssa > /dev/null
       
       - name: Copy lookups .mlmodel files
         run: |

From 27417d7cb6ce93d9af66c783cd8b98540571ce90 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 28 Jun 2022 11:10:46 -0700
Subject: [PATCH 37/38] Suppress more output for performance test

---
 .github/workflows/build-and-validate.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-and-validate.yml b/.github/workflows/build-and-validate.yml
index de32ad1506..475e1cd6e4 100644
--- a/.github/workflows/build-and-validate.yml
+++ b/.github/workflows/build-and-validate.yml
@@ -353,12 +353,12 @@ jobs:
       - name: Run doc-gen
         run: |
           source venv/bin/activate
-          python3 contentctl.py -p . docgen -o docs
+          python3 contentctl.py -p . docgen -o docs > /dev/null
 
       - name: Run reporting
         run: |
           source venv/bin/activate
-          python3 contentctl.py -p . reporting
+          python3 contentctl.py -p . reporting > /dev/null
 
       - name: Update github with new docs and package bits
         run: |

From 6cfa2014d2c4e6a08685959afe1a3e540089bc13 Mon Sep 17 00:00:00 2001
From: pyth0n1c <87383215+pyth0n1c@users.noreply.github.com>
Date: Tue, 28 Jun 2022 16:46:51 -0700
Subject: [PATCH 38/38] Remove ambiguity from the git checkout command used by
 the detection testing code.

---
 bin/docker_detection_tester/modules/github_service.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/bin/docker_detection_tester/modules/github_service.py b/bin/docker_detection_tester/modules/github_service.py
index c775c4be2f..da279c1e10 100644
--- a/bin/docker_detection_tester/modules/github_service.py
+++ b/bin/docker_detection_tester/modules/github_service.py
@@ -70,9 +70,13 @@ def __init__(self, security_content_branch: str, commit_hash: Union[str,None], P
 
         # No checking to see if the hash is to a commit inside of the branch - the user
         # has to do that by hand.
+        
+        # -- ensures that we check out the appropriate branch or commit hash.
+        # Without --, there can be ambiguity if a file/folder exists with the
+        # same name as the branch, causing the checkout to fail with error
         if commit_hash is not None:
             print("Checking out commit hash: [%s]" % (commit_hash))
-            self.security_content_repo_obj.git.checkout(commit_hash)
+            self.security_content_repo_obj.git.checkout(commit_hash, '--')
         else:
             #Even if we have fetched a PR, we still MUST check out the branch to
             # be able to do anything with it. Otherwise we won't have the files
@@ -80,7 +84,7 @@ def __init__(self, security_content_branch: str, commit_hash: Union[str,None], P
                   (security_content_branch), end='')
             sys.stdout.flush()
             self.security_content_repo_obj.git.checkout(
-                security_content_branch)
+                security_content_branch, '--')
             commit_hash = self.security_content_repo_obj.head.object.hexsha
             print("commit_hash %s" % (commit_hash))