From 07a1accbc57356750ef0fe0fbdeff867b51f54e8 Mon Sep 17 00:00:00 2001 From: nasbench <8741929+nasbench@users.noreply.github.com> Date: Wed, 26 Aug 2026 22:00:53 +0200 Subject: [PATCH 1/3] small fixes --- .github/workflows/detection-metadata.yml | 46 +++++++ ..._append_command_to_profile_config_file.yml | 4 +- ...n_network_traffic_during_package_build.yml | 2 +- pytest.ini | 4 + requirements.txt | 2 + tests/validation/test_detection_metadata.py | 119 ++++++++++++++++++ 6 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/detection-metadata.yml create mode 100644 pytest.ini create mode 100644 tests/validation/test_detection_metadata.py diff --git a/.github/workflows/detection-metadata.yml b/.github/workflows/detection-metadata.yml new file mode 100644 index 0000000000..0047422561 --- /dev/null +++ b/.github/workflows/detection-metadata.yml @@ -0,0 +1,46 @@ +name: Detection Metadata + +on: + pull_request: + paths: + - 'detections/**/*.yml' + - 'detections/**/*.yaml' + - 'tests/validation/**' + - 'pytest.ini' + - 'requirements.txt' + - '.github/workflows/detection-metadata.yml' + +jobs: + validate: + name: Validate changed detection metadata + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.11' + cache: pip + cache-dependency-path: requirements.txt + + - name: Install test dependencies + run: pip install -r requirements.txt + + - name: Select changed detections + shell: bash + run: | + git diff --name-only --diff-filter=ACMR "origin/${{ github.base_ref }}"...HEAD -- 'detections/**/*.yml' 'detections/**/*.yaml' > changed_detections.txt + { + echo 'DETECTION_PATHS<> "$GITHUB_ENV" + + - name: Validate metadata + run: | + pytest tests/validation -q --tb=short diff --git a/detections/endpoint/linux_possible_append_command_to_profile_config_file.yml b/detections/endpoint/linux_possible_append_command_to_profile_config_file.yml index e382b1d906..cd3a8eca4c 100644 --- a/detections/endpoint/linux_possible_append_command_to_profile_config_file.yml +++ b/detections/endpoint/linux_possible_append_command_to_profile_config_file.yml @@ -22,14 +22,14 @@ search: |- Processes.process = "*echo*" ( - Processes.process IN ("*/home/*", "*~/*") + Processes.process IN ("*/home/*", "*~/*", "*/etc/*") Processes.process IN ( "*/.bash_login*", "*/.bash_logout*", "*/.bash_profile*", "*/.bashrc*", "*/.profile*", - "*/etc/profile*" + "*/profile*" ) OR ( diff --git a/detections/endpoint/python_network_traffic_during_package_build.yml b/detections/endpoint/python_network_traffic_during_package_build.yml index eeb6f097b7..b5b36b69b2 100644 --- a/detections/endpoint/python_network_traffic_during_package_build.yml +++ b/detections/endpoint/python_network_traffic_during_package_build.yml @@ -46,7 +46,7 @@ search: |- by dest source process_id - | search process="* build_wheel*" + | search process="* build_wheel*" dest_ip=* | table firstTime lastTime parent_process_id parent_process_path parent_process_name parent_process diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000000..f707492fdc --- /dev/null +++ b/pytest.ini @@ -0,0 +1,4 @@ +[pytest] +testpaths = tests +addopts = -ra --tb=short +cache_dir = .pytest_cache diff --git a/requirements.txt b/requirements.txt index 4bbb0e2cf0..69c116cf75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ contentctl-ng==1.1.1 +pytest==8.4.2 +PyYAML==6.0.2 diff --git a/tests/validation/test_detection_metadata.py b/tests/validation/test_detection_metadata.py new file mode 100644 index 0000000000..69d096c8e7 --- /dev/null +++ b/tests/validation/test_detection_metadata.py @@ -0,0 +1,119 @@ +"""Fast, focused metadata checks for detection YAML files. + +Set DETECTION_PATHS to a newline-separated list of repository-relative paths to +validate only a subset (the CI workflow uses this for files changed in a PR). +Without it, every detection is checked. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Any + +import pytest +import yaml + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +DETECTIONS_DIRECTORY = REPOSITORY_ROOT / "detections" +THREAT_OBJECT_TYPES = frozenset({"TTP", "Anomaly"}) + + +@dataclass(frozen=True) +class Detection: + """The small metadata subset required by these validations.""" + + path: Path + title: str + type: str | None + fields: frozenset[str] + + +def _selected_paths() -> tuple[Path, ...]: + """Return requested files, or all detection YAML files in stable order.""" + configured_paths = os.environ.get("DETECTION_PATHS") + if configured_paths is None: + return tuple(sorted(DETECTIONS_DIRECTORY.rglob("*.y*ml"))) + + paths = [] + for value in configured_paths.splitlines(): + if not value.strip(): + continue + path = (REPOSITORY_ROOT / value).resolve() + if path.is_file() and DETECTIONS_DIRECTORY.resolve() in path.parents: + paths.append(path) + return tuple(sorted(set(paths))) + + +@lru_cache(maxsize=None) +def _load_detection(path: Path, mtime_ns: int, size: int) -> Detection: + """Parse each unchanged YAML file once per pytest process. + + File metadata is part of the cache key so an in-process edit cannot return + stale metadata. Both tests reuse the same cached Detection objects. + """ + try: + with path.open(encoding="utf-8") as stream: + document: Any = yaml.safe_load(stream) + except yaml.YAMLError as error: + relative_path = path.relative_to(REPOSITORY_ROOT) + pytest.fail(f"Invalid YAML in {relative_path}: {error}", pytrace=False) + + if not isinstance(document, dict): + pytest.fail( + f"{path.relative_to(REPOSITORY_ROOT)} must contain a YAML mapping.", + pytrace=False, + ) + + return Detection( + path=path, + title=str(document.get("name") or document.get("title") or ""), + type=document.get("type"), + fields=frozenset(document), + ) + + +@pytest.fixture(scope="session") +def detections() -> tuple[Detection, ...]: + """Load the selected corpus once; pytest reuses it across all checks.""" + return tuple( + _load_detection(path, path.stat().st_mtime_ns, path.stat().st_size) + for path in _selected_paths() + ) + + +def _failure_message(field: str, missing: list[Detection], requirement: str) -> str: + details = "\n".join( + f" - {detection.path.relative_to(REPOSITORY_ROOT)} " + f"[{detection.type or 'missing type'}] {detection.title}" + for detection in missing + ) + return ( + f"{len(missing)} detection(s) missing top-level `{field}`.\n" + f"Required for: {requirement}.\n\n" + f"{details}\n\n" + f"Fix: add a top-level `{field}:` field to each listed detection." + ) + + +def test_detections_include_references(detections: tuple[Detection, ...]) -> None: + """Every detection must declare references (an empty list is explicit and valid).""" + missing = [detection for detection in detections if "references" not in detection.fields] + assert not missing, _failure_message("references", missing, "all detection types") + + +def test_ttp_and_anomaly_detections_include_threat_objects( + detections: tuple[Detection, ...], +) -> None: + """TTP and Anomaly detections require threat_objects; Hunting is exempt.""" + missing = [ + detection + for detection in detections + if detection.type in THREAT_OBJECT_TYPES and "threat_objects" not in detection.fields + ] + assert not missing, _failure_message( + "threat_objects", missing, "TTP and Anomaly detections (Hunting is exempt)" + ) From 2f794645e21c3861f3e4c8621d119cf0df0f9ad4 Mon Sep 17 00:00:00 2001 From: nasbench <8741929+nasbench@users.noreply.github.com> Date: Wed, 26 Aug 2026 22:09:25 +0200 Subject: [PATCH 2/3] Revert "small fixes" This reverts commit 07a1accbc57356750ef0fe0fbdeff867b51f54e8. --- .github/workflows/detection-metadata.yml | 46 ------- ..._append_command_to_profile_config_file.yml | 4 +- ...n_network_traffic_during_package_build.yml | 2 +- pytest.ini | 4 - requirements.txt | 2 - tests/validation/test_detection_metadata.py | 119 ------------------ 6 files changed, 3 insertions(+), 174 deletions(-) delete mode 100644 .github/workflows/detection-metadata.yml delete mode 100644 pytest.ini delete mode 100644 tests/validation/test_detection_metadata.py diff --git a/.github/workflows/detection-metadata.yml b/.github/workflows/detection-metadata.yml deleted file mode 100644 index 0047422561..0000000000 --- a/.github/workflows/detection-metadata.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Detection Metadata - -on: - pull_request: - paths: - - 'detections/**/*.yml' - - 'detections/**/*.yaml' - - 'tests/validation/**' - - 'pytest.ini' - - 'requirements.txt' - - '.github/workflows/detection-metadata.yml' - -jobs: - validate: - name: Validate changed detection metadata - runs-on: ubuntu-latest - - steps: - - name: Check out code - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.11' - cache: pip - cache-dependency-path: requirements.txt - - - name: Install test dependencies - run: pip install -r requirements.txt - - - name: Select changed detections - shell: bash - run: | - git diff --name-only --diff-filter=ACMR "origin/${{ github.base_ref }}"...HEAD -- 'detections/**/*.yml' 'detections/**/*.yaml' > changed_detections.txt - { - echo 'DETECTION_PATHS<> "$GITHUB_ENV" - - - name: Validate metadata - run: | - pytest tests/validation -q --tb=short diff --git a/detections/endpoint/linux_possible_append_command_to_profile_config_file.yml b/detections/endpoint/linux_possible_append_command_to_profile_config_file.yml index cd3a8eca4c..e382b1d906 100644 --- a/detections/endpoint/linux_possible_append_command_to_profile_config_file.yml +++ b/detections/endpoint/linux_possible_append_command_to_profile_config_file.yml @@ -22,14 +22,14 @@ search: |- Processes.process = "*echo*" ( - Processes.process IN ("*/home/*", "*~/*", "*/etc/*") + Processes.process IN ("*/home/*", "*~/*") Processes.process IN ( "*/.bash_login*", "*/.bash_logout*", "*/.bash_profile*", "*/.bashrc*", "*/.profile*", - "*/profile*" + "*/etc/profile*" ) OR ( diff --git a/detections/endpoint/python_network_traffic_during_package_build.yml b/detections/endpoint/python_network_traffic_during_package_build.yml index b5b36b69b2..eeb6f097b7 100644 --- a/detections/endpoint/python_network_traffic_during_package_build.yml +++ b/detections/endpoint/python_network_traffic_during_package_build.yml @@ -46,7 +46,7 @@ search: |- by dest source process_id - | search process="* build_wheel*" dest_ip=* + | search process="* build_wheel*" | table firstTime lastTime parent_process_id parent_process_path parent_process_name parent_process diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index f707492fdc..0000000000 --- a/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -[pytest] -testpaths = tests -addopts = -ra --tb=short -cache_dir = .pytest_cache diff --git a/requirements.txt b/requirements.txt index 69c116cf75..4bbb0e2cf0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1 @@ contentctl-ng==1.1.1 -pytest==8.4.2 -PyYAML==6.0.2 diff --git a/tests/validation/test_detection_metadata.py b/tests/validation/test_detection_metadata.py deleted file mode 100644 index 69d096c8e7..0000000000 --- a/tests/validation/test_detection_metadata.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Fast, focused metadata checks for detection YAML files. - -Set DETECTION_PATHS to a newline-separated list of repository-relative paths to -validate only a subset (the CI workflow uses this for files changed in a PR). -Without it, every detection is checked. -""" - -from __future__ import annotations - -import os -from dataclasses import dataclass -from functools import lru_cache -from pathlib import Path -from typing import Any - -import pytest -import yaml - - -REPOSITORY_ROOT = Path(__file__).resolve().parents[2] -DETECTIONS_DIRECTORY = REPOSITORY_ROOT / "detections" -THREAT_OBJECT_TYPES = frozenset({"TTP", "Anomaly"}) - - -@dataclass(frozen=True) -class Detection: - """The small metadata subset required by these validations.""" - - path: Path - title: str - type: str | None - fields: frozenset[str] - - -def _selected_paths() -> tuple[Path, ...]: - """Return requested files, or all detection YAML files in stable order.""" - configured_paths = os.environ.get("DETECTION_PATHS") - if configured_paths is None: - return tuple(sorted(DETECTIONS_DIRECTORY.rglob("*.y*ml"))) - - paths = [] - for value in configured_paths.splitlines(): - if not value.strip(): - continue - path = (REPOSITORY_ROOT / value).resolve() - if path.is_file() and DETECTIONS_DIRECTORY.resolve() in path.parents: - paths.append(path) - return tuple(sorted(set(paths))) - - -@lru_cache(maxsize=None) -def _load_detection(path: Path, mtime_ns: int, size: int) -> Detection: - """Parse each unchanged YAML file once per pytest process. - - File metadata is part of the cache key so an in-process edit cannot return - stale metadata. Both tests reuse the same cached Detection objects. - """ - try: - with path.open(encoding="utf-8") as stream: - document: Any = yaml.safe_load(stream) - except yaml.YAMLError as error: - relative_path = path.relative_to(REPOSITORY_ROOT) - pytest.fail(f"Invalid YAML in {relative_path}: {error}", pytrace=False) - - if not isinstance(document, dict): - pytest.fail( - f"{path.relative_to(REPOSITORY_ROOT)} must contain a YAML mapping.", - pytrace=False, - ) - - return Detection( - path=path, - title=str(document.get("name") or document.get("title") or ""), - type=document.get("type"), - fields=frozenset(document), - ) - - -@pytest.fixture(scope="session") -def detections() -> tuple[Detection, ...]: - """Load the selected corpus once; pytest reuses it across all checks.""" - return tuple( - _load_detection(path, path.stat().st_mtime_ns, path.stat().st_size) - for path in _selected_paths() - ) - - -def _failure_message(field: str, missing: list[Detection], requirement: str) -> str: - details = "\n".join( - f" - {detection.path.relative_to(REPOSITORY_ROOT)} " - f"[{detection.type or 'missing type'}] {detection.title}" - for detection in missing - ) - return ( - f"{len(missing)} detection(s) missing top-level `{field}`.\n" - f"Required for: {requirement}.\n\n" - f"{details}\n\n" - f"Fix: add a top-level `{field}:` field to each listed detection." - ) - - -def test_detections_include_references(detections: tuple[Detection, ...]) -> None: - """Every detection must declare references (an empty list is explicit and valid).""" - missing = [detection for detection in detections if "references" not in detection.fields] - assert not missing, _failure_message("references", missing, "all detection types") - - -def test_ttp_and_anomaly_detections_include_threat_objects( - detections: tuple[Detection, ...], -) -> None: - """TTP and Anomaly detections require threat_objects; Hunting is exempt.""" - missing = [ - detection - for detection in detections - if detection.type in THREAT_OBJECT_TYPES and "threat_objects" not in detection.fields - ] - assert not missing, _failure_message( - "threat_objects", missing, "TTP and Anomaly detections (Hunting is exempt)" - ) From 181957598abe435f65fa89d74e10179a9e6f5c48 Mon Sep 17 00:00:00 2001 From: nasbench <8741929+nasbench@users.noreply.github.com> Date: Wed, 26 Aug 2026 22:10:14 +0200 Subject: [PATCH 3/3] fixes --- .../linux_possible_append_command_to_profile_config_file.yml | 4 ++-- .../endpoint/python_network_traffic_during_package_build.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/detections/endpoint/linux_possible_append_command_to_profile_config_file.yml b/detections/endpoint/linux_possible_append_command_to_profile_config_file.yml index e382b1d906..cd3a8eca4c 100644 --- a/detections/endpoint/linux_possible_append_command_to_profile_config_file.yml +++ b/detections/endpoint/linux_possible_append_command_to_profile_config_file.yml @@ -22,14 +22,14 @@ search: |- Processes.process = "*echo*" ( - Processes.process IN ("*/home/*", "*~/*") + Processes.process IN ("*/home/*", "*~/*", "*/etc/*") Processes.process IN ( "*/.bash_login*", "*/.bash_logout*", "*/.bash_profile*", "*/.bashrc*", "*/.profile*", - "*/etc/profile*" + "*/profile*" ) OR ( diff --git a/detections/endpoint/python_network_traffic_during_package_build.yml b/detections/endpoint/python_network_traffic_during_package_build.yml index eeb6f097b7..b5b36b69b2 100644 --- a/detections/endpoint/python_network_traffic_during_package_build.yml +++ b/detections/endpoint/python_network_traffic_during_package_build.yml @@ -46,7 +46,7 @@ search: |- by dest source process_id - | search process="* build_wheel*" + | search process="* build_wheel*" dest_ip=* | table firstTime lastTime parent_process_id parent_process_path parent_process_name parent_process