diff --git a/.agents/skills/adapter-registration-validation/tests/test_skill.py b/.agents/skills/adapter-registration-validation/tests/test_skill.py index ae49970..ced6167 100644 --- a/.agents/skills/adapter-registration-validation/tests/test_skill.py +++ b/.agents/skills/adapter-registration-validation/tests/test_skill.py @@ -3,7 +3,8 @@ from __future__ import annotations import unittest -from cops.adapters import ToolAdapterRegistry, AdapterInjectionError + +from cops.adapters import AdapterInjectionError, ToolAdapterRegistry class TestAdapterRegistrationSkill(unittest.TestCase): diff --git a/.agents/skills/agent-workboard/scripts/workboard.py b/.agents/skills/agent-workboard/scripts/workboard.py index 6d382e6..3e4e9ea 100644 --- a/.agents/skills/agent-workboard/scripts/workboard.py +++ b/.agents/skills/agent-workboard/scripts/workboard.py @@ -13,9 +13,9 @@ import subprocess import sys import uuid +from collections.abc import Iterable, Sequence from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple - +from typing import Any SCHEMA_VERSION = 1 LEASE_MINUTES = 30 @@ -37,14 +37,14 @@ class WorkboardError(RuntimeError): def utc_now() -> str: - return dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds") + return dt.datetime.now(dt.UTC).isoformat(timespec="seconds") def parse_time(value: str) -> dt.datetime: parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00")) if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=dt.timezone.utc) - return parsed.astimezone(dt.timezone.utc) + parsed = parsed.replace(tzinfo=dt.UTC) + return parsed.astimezone(dt.UTC) def json_text(value: Any) -> str: @@ -52,14 +52,14 @@ def json_text(value: Any) -> str: def new_id(prefix: str) -> str: - return "{}_{}".format(prefix, uuid.uuid4().hex) + return f"{prefix}_{uuid.uuid4().hex}" def canonical_path(path: str) -> Path: return Path(path).expanduser().resolve() -def run_git(cwd: Path, args: Sequence[str]) -> Optional[str]: +def run_git(cwd: Path, args: Sequence[str]) -> str | None: try: result = subprocess.run( ["git", "-C", str(cwd)] + list(args), @@ -73,7 +73,7 @@ def run_git(cwd: Path, args: Sequence[str]) -> Optional[str]: return result.stdout.strip() -def resolve_storage(cwd: str) -> Dict[str, str]: +def resolve_storage(cwd: str) -> dict[str, str]: project = canonical_path(cwd) common = run_git( project, ["rev-parse", "--path-format=absolute", "--git-common-dir"] @@ -103,7 +103,7 @@ def resolve_storage(cwd: str) -> Dict[str, str]: } -def resolve_database(cwd: str, explicit: Optional[str]) -> Tuple[Path, Dict[str, str]]: +def resolve_database(cwd: str, explicit: str | None) -> tuple[Path, dict[str, str]]: storage = resolve_storage(cwd) database = canonical_path(explicit) if explicit else Path(storage["database"]) storage["database"] = str(database) @@ -149,9 +149,7 @@ def migrate(connection: sqlite3.Connection) -> None: version = int(connection.execute("PRAGMA user_version").fetchone()[0]) if version > SCHEMA_VERSION: raise WorkboardError( - "database schema {} is newer than supported schema {}".format( - version, SCHEMA_VERSION - ) + f"database schema {version} is newer than supported schema {SCHEMA_VERSION}" ) if version < 1: connection.executescript( @@ -259,11 +257,11 @@ def record_event( connection: sqlite3.Connection, event_type: str, *, - run_id: Optional[str] = None, - task_id: Optional[str] = None, - session_id: Optional[str] = None, - agent_id: Optional[str] = None, - payload: Optional[Dict[str, Any]] = None, + run_id: str | None = None, + task_id: str | None = None, + session_id: str | None = None, + agent_id: str | None = None, + payload: dict[str, Any] | None = None, ) -> None: connection.execute( """ @@ -288,13 +286,13 @@ def fetch_task(connection: sqlite3.Connection, task_id: str) -> sqlite3.Row: "SELECT * FROM tasks WHERE task_id = ?", (task_id,) ).fetchone() if row is None: - raise WorkboardError("unknown task: {}".format(task_id)) + raise WorkboardError(f"unknown task: {task_id}") return row def incomplete_dependencies( connection: sqlite3.Connection, task_id: str -) -> List[sqlite3.Row]: +) -> list[sqlite3.Row]: return list( connection.execute( """ @@ -311,7 +309,7 @@ def incomplete_dependencies( ) -def refresh_ready_tasks(connection: sqlite3.Connection) -> List[str]: +def refresh_ready_tasks(connection: sqlite3.Connection) -> list[str]: rows = list( connection.execute( """ @@ -356,7 +354,7 @@ def write_handoff( task = fetch_task(connection, task_id) handoff_dir = database.parent / "handoffs" secure_directory(handoff_dir) - path = handoff_dir / "{}.md".format(task_id) + path = handoff_dir / f"{task_id}.md" inputs = json.loads(task["inputs_json"]) acceptance = json.loads(task["acceptance_json"]) validation = json.loads(task["validation_json"]) @@ -381,7 +379,7 @@ def write_handoff( ] def bullets(values: Iterable[Any], empty: str = "- None recorded") -> str: - lines = ["- {}".format(value) for value in values] + lines = [f"- {value}" for value in values] return "\n".join(lines) if lines else empty body = """# Task handoff: {title} @@ -505,9 +503,9 @@ def record_artifact_references( def command_init( connection: sqlite3.Connection, database: Path, - storage: Dict[str, str], + storage: dict[str, str], _args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: return { "database": str(database), "project_root": storage["project_root"], @@ -521,9 +519,9 @@ def command_init( def command_run_start( connection: sqlite3.Connection, database: Path, - storage: Dict[str, str], + storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: run_id = args.run_id or new_id("run") now = utc_now() try: @@ -543,7 +541,7 @@ def command_run_start( ), ) except sqlite3.IntegrityError as exc: - raise WorkboardError("cannot start run {}: {}".format(run_id, exc)) + raise WorkboardError(f"cannot start run {run_id}: {exc}") from exc record_event( connection, "run_started", @@ -555,8 +553,8 @@ def command_run_start( return {"database": str(database), "run_id": run_id, "status": "active"} -def load_contract(args: argparse.Namespace) -> Dict[str, Any]: - contract: Dict[str, Any] = {} +def load_contract(args: argparse.Namespace) -> dict[str, Any]: + contract: dict[str, Any] = {} if args.contract_file: contract = json.loads( Path(args.contract_file).expanduser().read_text(encoding="utf-8") @@ -613,15 +611,15 @@ def load_contract(args: argparse.Namespace) -> Dict[str, Any]: def command_task_add( connection: sqlite3.Connection, database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: contract = load_contract(args) run = connection.execute( "SELECT run_id FROM runs WHERE run_id = ?", (args.run_id,) ).fetchone() if run is None: - raise WorkboardError("unknown run: {}".format(args.run_id)) + raise WorkboardError(f"unknown run: {args.run_id}") task_id = args.task_id or new_id("task") dependencies = list(dict.fromkeys(contract["dependencies"])) now = utc_now() @@ -694,9 +692,9 @@ def command_task_add( def command_task_bind( connection: sqlite3.Connection, _database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: try: connection.execute("BEGIN IMMEDIATE") task = fetch_task(connection, args.task_id) @@ -735,16 +733,16 @@ def command_task_bind( def lease_deadline(minutes: int = LEASE_MINUTES) -> str: return ( - dt.datetime.now(dt.timezone.utc) + dt.timedelta(minutes=minutes) + dt.datetime.now(dt.UTC) + dt.timedelta(minutes=minutes) ).isoformat(timespec="seconds") def command_task_claim( connection: sqlite3.Connection, _database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: if args.lease_minutes <= 0: raise WorkboardError("--lease-minutes must be positive") try: @@ -819,9 +817,9 @@ def require_assignee(task: sqlite3.Row, agent_id: str) -> None: def command_task_heartbeat( connection: sqlite3.Connection, _database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: if args.lease_minutes <= 0: raise WorkboardError("--lease-minutes must be positive") try: @@ -862,9 +860,9 @@ def command_task_heartbeat( def command_task_block( connection: sqlite3.Connection, database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: try: connection.execute("BEGIN IMMEDIATE") task = fetch_task(connection, args.task_id) @@ -915,7 +913,7 @@ def command_task_block( } -def list_argument(values: Sequence[str], encoded: Optional[str]) -> List[str]: +def list_argument(values: Sequence[str], encoded: str | None) -> list[str]: if encoded is not None: parsed = json.loads(encoded) if not isinstance(parsed, list): @@ -927,9 +925,9 @@ def list_argument(values: Sequence[str], encoded: Optional[str]) -> List[str]: def command_task_complete( connection: sqlite3.Connection, database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: validation = list_argument(args.validation, args.validation_json) try: connection.execute("BEGIN IMMEDIATE") @@ -979,9 +977,9 @@ def command_task_complete( def command_task_cancel( connection: sqlite3.Connection, database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: try: connection.execute("BEGIN IMMEDIATE") task = fetch_task(connection, args.task_id) @@ -1019,7 +1017,7 @@ def command_task_cancel( } -def task_to_dict(row: sqlite3.Row) -> Dict[str, Any]: +def task_to_dict(row: sqlite3.Row) -> dict[str, Any]: result = dict(row) for field in ("inputs_json", "acceptance_json", "validation_json"): result[field[:-5]] = json.loads(result.pop(field)) @@ -1029,11 +1027,11 @@ def task_to_dict(row: sqlite3.Row) -> Dict[str, Any]: def command_status( connection: sqlite3.Connection, database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: clauses = [] - values: List[Any] = [] + values: list[Any] = [] if args.run_id: clauses.append("task.run_id = ?") values.append(args.run_id) @@ -1047,14 +1045,12 @@ def command_status( where = " WHERE " + " AND ".join(clauses) if clauses else "" rows = list( connection.execute( - """ + f""" SELECT task.* FROM tasks task - {} + {where} ORDER BY task.created_at, task.task_id - """.format( - where - ), + """, # noqa: S608 - fixed SQL clauses; values are bound parameters values, ) ) @@ -1078,9 +1074,9 @@ def command_status( def command_reconcile( connection: sqlite3.Connection, _database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], _args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: try: connection.execute("BEGIN IMMEDIATE") now = utc_now() @@ -1125,9 +1121,9 @@ def command_reconcile( def command_export( connection: sqlite3.Connection, _database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: artifact = connection.execute( """ SELECT path FROM artifacts @@ -1140,13 +1136,13 @@ def command_export( raise WorkboardError("task has no generated handoff") source = Path(artifact["path"]) if not source.is_file(): - raise WorkboardError("handoff is missing: {}".format(source)) + raise WorkboardError(f"handoff is missing: {source}") target = Path(args.to).expanduser() if target.exists() and target.is_dir(): target = target / source.name if target.exists() and not args.force: raise WorkboardError( - "export target exists; pass --force to replace it: {}".format(target) + f"export target exists; pass --force to replace it: {target}" ) target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(str(source), str(target)) @@ -1156,13 +1152,13 @@ def command_export( def command_gc( connection: sqlite3.Connection, database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: if args.older_than < 0: raise WorkboardError("--older-than must be non-negative") cutoff = ( - dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=args.older_than) + dt.datetime.now(dt.UTC) - dt.timedelta(days=args.older_than) ).isoformat(timespec="seconds") try: connection.execute("BEGIN IMMEDIATE") @@ -1252,7 +1248,7 @@ def compact_summary(connection: sqlite3.Connection, database: Path) -> str: """ ) ) - lines = ["Agent workboard: {}".format(database)] + lines = [f"Agent workboard: {database}"] if not rows: lines.append("No active tasks.") else: @@ -1286,7 +1282,7 @@ def hook_output(event: str, context: str) -> None: def session_upsert( - connection: sqlite3.Connection, payload: Dict[str, Any], status: str + connection: sqlite3.Connection, payload: dict[str, Any], status: str ) -> None: session_id = payload.get("session_id") if not session_id: @@ -1325,7 +1321,7 @@ def session_upsert( def task_for_agent( connection: sqlite3.Connection, agent_id: str -) -> Optional[sqlite3.Row]: +) -> sqlite3.Row | None: return connection.execute( """ SELECT * FROM tasks @@ -1451,12 +1447,12 @@ def hook_main(args: argparse.Namespace) -> int: ) connection.commit() else: - raise WorkboardError("unsupported hook event: {}".format(event)) + raise WorkboardError(f"unsupported hook event: {event}") finally: connection.close() return 0 except Exception as exc: - print("agent-workboard hook: {}".format(exc), file=sys.stderr) + print(f"agent-workboard hook: {exc}", file=sys.stderr) if "payload" in locals() and payload.get("hook_event_name") == "SubagentStop": print("{}") return 0 @@ -1577,7 +1573,7 @@ def build_parser() -> argparse.ArgumentParser: return parser -def print_result(result: Dict[str, Any], as_json: bool) -> None: +def print_result(result: dict[str, Any], as_json: bool) -> None: if as_json: print(json.dumps(result, indent=2, sort_keys=True)) return @@ -1592,12 +1588,12 @@ def print_result(result: Dict[str, Any], as_json: bool) -> None: return for key, value in result.items(): if isinstance(value, (dict, list)): - print("{}={}".format(key, json_text(value))) + print(f"{key}={json_text(value)}") else: - print("{}={}".format(key, value)) + print(f"{key}={value}") -def main(argv: Optional[Sequence[str]] = None) -> int: +def main(argv: Sequence[str] | None = None) -> int: os.umask(0o077) parser = build_parser() args = parser.parse_args(argv) @@ -1614,7 +1610,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: print_result(result, args.json) return 0 except (WorkboardError, json.JSONDecodeError, OSError, sqlite3.Error) as exc: - print("agent-workboard: {}".format(exc), file=sys.stderr) + print(f"agent-workboard: {exc}", file=sys.stderr) return 2 diff --git a/.agents/skills/credential-evidence-handling/tests/test_skill.py b/.agents/skills/credential-evidence-handling/tests/test_skill.py index ee31dcf..86a8806 100644 --- a/.agents/skills/credential-evidence-handling/tests/test_skill.py +++ b/.agents/skills/credential-evidence-handling/tests/test_skill.py @@ -2,11 +2,12 @@ from __future__ import annotations -import unittest -from cops.execution import StreamRedactor, EvidenceRecorder import tempfile +import unittest from pathlib import Path +from cops.execution import EvidenceRecorder, StreamRedactor + class TestCredentialEvidenceSkill(unittest.TestCase): def test_skill_redaction_flow(self): diff --git a/.agents/skills/engagement-contract-validation/tests/test_skill.py b/.agents/skills/engagement-contract-validation/tests/test_skill.py index 8de632e..decf1fc 100644 --- a/.agents/skills/engagement-contract-validation/tests/test_skill.py +++ b/.agents/skills/engagement-contract-validation/tests/test_skill.py @@ -1,11 +1,10 @@ """Tests for engagement-contract-validation skill.""" -from pathlib import Path import unittest +from pathlib import Path from cops.contracts import ( ContractError, - validate_contract, validate_transition, ) diff --git a/.agents/skills/engagement-intake-and-planning/tests/test_skill.py b/.agents/skills/engagement-intake-and-planning/tests/test_skill.py index 6a86fa3..9f4d316 100644 --- a/.agents/skills/engagement-intake-and-planning/tests/test_skill.py +++ b/.agents/skills/engagement-intake-and-planning/tests/test_skill.py @@ -2,14 +2,12 @@ from __future__ import annotations -from pathlib import Path import unittest from cops.catalog import ROOT from cops.engagement import ( build_action_plan, create_engagement_contract, - validate_engagement_intake, ) diff --git a/.agents/skills/execution-authorization-review/tests/test_skill.py b/.agents/skills/execution-authorization-review/tests/test_skill.py index ed4f722..9e4a307 100644 --- a/.agents/skills/execution-authorization-review/tests/test_skill.py +++ b/.agents/skills/execution-authorization-review/tests/test_skill.py @@ -3,14 +3,14 @@ from __future__ import annotations import json -from pathlib import Path import unittest +from pathlib import Path from cops.contracts.models import ActionPlan from cops.execution import ( + consume_execution_authorization, create_execution_authorization, verify_execution_authorization, - consume_execution_authorization, ) diff --git a/.agents/skills/execution-scope-enforcement/tests/test_skill.py b/.agents/skills/execution-scope-enforcement/tests/test_skill.py index 184c853..6441ca8 100644 --- a/.agents/skills/execution-scope-enforcement/tests/test_skill.py +++ b/.agents/skills/execution-scope-enforcement/tests/test_skill.py @@ -3,8 +3,8 @@ from __future__ import annotations import json -from pathlib import Path import unittest +from pathlib import Path from cops.execution import ScopeDefinition, ScopeGuard, ScopeViolationError diff --git a/.agents/skills/network-active-discovery/tests/test_skill.py b/.agents/skills/network-active-discovery/tests/test_skill.py index f07571b..8d7589a 100644 --- a/.agents/skills/network-active-discovery/tests/test_skill.py +++ b/.agents/skills/network-active-discovery/tests/test_skill.py @@ -1,18 +1,20 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-active-discovery contributor skill.""" from __future__ import annotations -from pathlib import Path import sys import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from cops.discovery import ( - ActiveScanSession, ActiveScanner, + ActiveScanSession, OfflineSyntheticDispatcher, ScanBudget, ScanVantage, diff --git a/.agents/skills/network-data-services/tests/test_skill.py b/.agents/skills/network-data-services/tests/test_skill.py index 6caef6f..6df2b60 100644 --- a/.agents/skills/network-data-services/tests/test_skill.py +++ b/.agents/skills/network-data-services/tests/test_skill.py @@ -1,32 +1,31 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-data-services contributor skill.""" from __future__ import annotations import argparse import json -from pathlib import Path import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from cops.discovery import ( - CleanupReceipt, DataAuthPrerequisite, DataExposureStatus, - DataPrivilegeCandidate, DataPrivilegeImpact, - DataServiceAssessment, - DataServiceCategory, DataServicesReport, - DataServiceType, OfflineSyntheticDataCollector, assess_data_services, ) -from cops.discovery.cli import command_data_discovery +from cops.discovery.cli import ( + command_data_discovery, +) class TestNetworkDataServicesSkill(unittest.TestCase): diff --git a/.agents/skills/network-developer-interfaces/tests/test_skill.py b/.agents/skills/network-developer-interfaces/tests/test_skill.py index b0e9db1..bc40ba0 100644 --- a/.agents/skills/network-developer-interfaces/tests/test_skill.py +++ b/.agents/skills/network-developer-interfaces/tests/test_skill.py @@ -1,33 +1,31 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-developer-interfaces contributor skill.""" from __future__ import annotations import argparse import json -from pathlib import Path import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from cops.discovery import ( - CleanupReceipt, DeveloperAuthPrerequisite, - DeveloperCategory, DeveloperExposureStatus, - DeveloperPrivilegeCandidate, - DeveloperPrivilegeImpact, - DeveloperServiceAssessment, DeveloperServicesReport, - DeveloperServiceType, ExecutionEffect, OfflineSyntheticDeveloperCollector, assess_developer_services, ) -from cops.discovery.cli import command_developer_discovery +from cops.discovery.cli import ( + command_developer_discovery, +) class TestNetworkDeveloperInterfacesSkill(unittest.TestCase): diff --git a/.agents/skills/network-infrastructure-services/tests/test_skill.py b/.agents/skills/network-infrastructure-services/tests/test_skill.py index 02fc5b7..485c355 100644 --- a/.agents/skills/network-infrastructure-services/tests/test_skill.py +++ b/.agents/skills/network-infrastructure-services/tests/test_skill.py @@ -1,10 +1,12 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-infrastructure-services contributor skill.""" from __future__ import annotations -from pathlib import Path import sys import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: @@ -13,7 +15,6 @@ from cops.discovery import ( AuthPrerequisite, IdentityAttackPathType, - InfraServiceAssessment, InfraServiceType, OfflineSyntheticInfraCollector, ServiceExposureStatus, diff --git a/.agents/skills/network-legacy-and-proxy-services/tests/test_skill.py b/.agents/skills/network-legacy-and-proxy-services/tests/test_skill.py index d2efddc..1aabd49 100644 --- a/.agents/skills/network-legacy-and-proxy-services/tests/test_skill.py +++ b/.agents/skills/network-legacy-and-proxy-services/tests/test_skill.py @@ -1,13 +1,15 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-legacy-and-proxy-services contributor skill.""" from __future__ import annotations import argparse import json -from pathlib import Path import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: @@ -21,13 +23,14 @@ LegacyExposureStatus, LegacyPrivilegeCandidate, LegacyPrivilegeImpact, - LegacyServiceAssessment, LegacyServicesReport, LegacyServiceType, OfflineSyntheticLegacyCollector, assess_legacy_services, ) -from cops.discovery.cli import command_legacy_discovery +from cops.discovery.cli import ( + command_legacy_discovery, +) class TestNetworkLegacyAndProxyServicesSkill(unittest.TestCase): diff --git a/.agents/skills/network-messaging-services/tests/test_skill.py b/.agents/skills/network-messaging-services/tests/test_skill.py index 2bdbe4e..f3e55d1 100644 --- a/.agents/skills/network-messaging-services/tests/test_skill.py +++ b/.agents/skills/network-messaging-services/tests/test_skill.py @@ -1,32 +1,30 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-messaging-services contributor skill.""" from __future__ import annotations import argparse import json -from pathlib import Path import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from cops.discovery import ( - CleanupReceipt, MessagingAuthPrerequisite, - MessagingCategory, MessagingExposureStatus, - MessagingPrivilegeCandidate, - MessagingPrivilegeImpact, - MessagingServiceAssessment, MessagingServicesReport, - MessagingServiceType, OfflineSyntheticMessagingCollector, assess_messaging_services, ) -from cops.discovery.cli import command_messaging_discovery +from cops.discovery.cli import ( + command_messaging_discovery, +) class TestNetworkMessagingServicesSkill(unittest.TestCase): diff --git a/.agents/skills/network-passive-discovery/tests/test_skill.py b/.agents/skills/network-passive-discovery/tests/test_skill.py index 0a4a174..e80de86 100644 --- a/.agents/skills/network-passive-discovery/tests/test_skill.py +++ b/.agents/skills/network-passive-discovery/tests/test_skill.py @@ -1,10 +1,12 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-passive-discovery contributor skill.""" from __future__ import annotations -from pathlib import Path import sys import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: diff --git a/.agents/skills/network-remote-and-file-services/tests/test_skill.py b/.agents/skills/network-remote-and-file-services/tests/test_skill.py index fbe8dbc..ed76fe8 100644 --- a/.agents/skills/network-remote-and-file-services/tests/test_skill.py +++ b/.agents/skills/network-remote-and-file-services/tests/test_skill.py @@ -1,32 +1,32 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-remote-and-file-services contributor skill.""" from __future__ import annotations import json -from pathlib import Path import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +import argparse + from cops.discovery import ( - CleanupReceipt, - HostPrivilegeCandidate, LateralMovementImpact, OfflineSyntheticRemoteCollector, RemoteAuthPrerequisite, RemoteExposureStatus, - RemoteServiceAssessment, - RemoteServiceCategory, RemoteServicesReport, - RemoteServiceType, assess_remote_services, ) -from cops.discovery.cli import command_remote_discovery -import argparse +from cops.discovery.cli import ( + command_remote_discovery, +) class TestNetworkRemoteAndFileServicesSkill(unittest.TestCase): diff --git a/.agents/skills/plugin-run-cost/scripts/run_cost.py b/.agents/skills/plugin-run-cost/scripts/run_cost.py index 58c1d4f..5c22ab0 100644 --- a/.agents/skills/plugin-run-cost/scripts/run_cost.py +++ b/.agents/skills/plugin-run-cost/scripts/run_cost.py @@ -3,17 +3,16 @@ from __future__ import annotations import argparse -from collections import Counter, defaultdict -from datetime import datetime, timezone -from decimal import Decimal, InvalidOperation import hashlib import importlib.util import json import math import os -from pathlib import Path import re import tempfile +from collections import Counter, defaultdict +from decimal import Decimal, InvalidOperation +from pathlib import Path ROOT = Path(__file__).resolve().parents[1] parser_spec = importlib.util.spec_from_file_location( @@ -324,7 +323,7 @@ def profile(repositories, wikis, threat_models, max_files=10000, max_bytes=20000 continue if not root.exists(): raise ValueError('profile input does not exist') - def walk(): + def walk(root=root): # Iterate directories without materializing their contents. The shared # budget counts roots, directories and excluded entries across all inputs. pending = [(root, False)] @@ -461,7 +460,7 @@ def compare(config): item = dict(name=safe_label(option['name']), qualified=option.get('qualified') is True, annual_capacity_hours=float(hours), at_hourly_rate={}) for rate in rates: - def recurring(a): + def recurring(a, rate=rate): return ((number(a['active_minutes'])/60*rate + number(a.get('variable_usd', 0)))*annual + number(a.get('monthly_maintenance_hours', 0))*rate*12 + number(a.get('monthly_maintenance_usd', 0))*12 + number(a.get('annual_license_usd', 0))) diff --git a/.agents/skills/plugin-run-cost/tests/test_cost.py b/.agents/skills/plugin-run-cost/tests/test_cost.py index 66fcc13..e6acbbb 100644 --- a/.agents/skills/plugin-run-cost/tests/test_cost.py +++ b/.agents/skills/plugin-run-cost/tests/test_cost.py @@ -1,8 +1,8 @@ import importlib.util import json -from pathlib import Path import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[1] spec = importlib.util.spec_from_file_location('run_cost', ROOT / 'scripts/run_cost.py') diff --git a/.agents/skills/scenario-laboratory-management/tests/test_skill.py b/.agents/skills/scenario-laboratory-management/tests/test_skill.py index 7b3b054..5ec6bc0 100644 --- a/.agents/skills/scenario-laboratory-management/tests/test_skill.py +++ b/.agents/skills/scenario-laboratory-management/tests/test_skill.py @@ -3,8 +3,8 @@ from __future__ import annotations import tempfile -from pathlib import Path import unittest +from pathlib import Path from cops.execution.store import ApprovalStore from cops.laboratory import ( diff --git a/.agents/skills/session-usage-audit/scripts/churn_git.py b/.agents/skills/session-usage-audit/scripts/churn_git.py index 991607c..e88c724 100644 --- a/.agents/skills/session-usage-audit/scripts/churn_git.py +++ b/.agents/skills/session-usage-audit/scripts/churn_git.py @@ -1,13 +1,13 @@ """Read-only Git inventory and immutable, first-parent history measurements.""" from __future__ import annotations -from collections import Counter -from datetime import datetime, timezone -from fnmatch import fnmatchcase import hashlib import os -from pathlib import Path import subprocess +from collections import Counter +from datetime import UTC, datetime +from fnmatch import fnmatchcase +from pathlib import Path CATEGORIES = ("production", "tests", "documentation", "generated_dependency", "unclassified") @@ -19,7 +19,7 @@ def git(path, *args, data=None, check=True): env.pop(name, None) result = subprocess.run(["git", "--no-optional-locks", "--no-replace-objects", "-C", str(path), "-c", "core.quotePath=false", *args], input=data, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + capture_output=True, env=env) if check and result.returncode: # Error bodies may contain private source; report only the operation. raise ValueError(f"Git {args[0]} failed in {path} (exit {result.returncode})") @@ -232,7 +232,7 @@ def changes(sha, parent): for line in decode(raw).splitlines(): ids, epoch = line.split("\t") fields = ids.split() - chain.append((fields[0], fields[1:], datetime.fromtimestamp(int(epoch), timezone.utc))) + chain.append((fields[0], fields[1:], datetime.fromtimestamp(int(epoch), UTC))) # Locate an immutable end then a contiguous ancestry range. Nonmonotonic # commit dates are diagnosed and disable range ratios/rework. before_end = [c for c in chain if c[2] < end] diff --git a/.agents/skills/session-usage-audit/scripts/churn_sessions.py b/.agents/skills/session-usage-audit/scripts/churn_sessions.py index 3ec9a9e..2cdbf82 100644 --- a/.agents/skills/session-usage-audit/scripts/churn_sessions.py +++ b/.agents/skills/session-usage-audit/scripts/churn_sessions.py @@ -1,12 +1,11 @@ """Static transcript extraction. Nothing read from a transcript is executed.""" from __future__ import annotations -from collections import Counter, defaultdict -from datetime import datetime import json -from pathlib import Path import re import shlex +from collections import Counter, defaultdict +from pathlib import Path import codex_token_usage as tokens from churn_git import metadata_repos, resolve_path @@ -490,7 +489,7 @@ def scan(paths, start, end, repos): for (rid, path, tree), edits in sorted(hotspots.items(), key=lambda kv: str(kv[0])): outcomes = Counter(e["outcome"] for e in edits) candidate_pairs = [] - for a, b in zip(edits, edits[1:]): + for a, b in zip(edits, edits[1:], strict=False): if a["operation"] == b["operation"] == "update" and a["_reverse"] == b["_forward"] and a["_forward"] != b["_forward"]: candidate_pairs.append({"kind": "inverse_patch_candidate", "confirmed": False, "evidence": [a["evidence"][0], b["evidence"][0]]}) diff --git a/.agents/skills/session-usage-audit/scripts/codex_churn_audit.py b/.agents/skills/session-usage-audit/scripts/codex_churn_audit.py index 5158825..aa4c791 100644 --- a/.agents/skills/session-usage-audit/scripts/codex_churn_audit.py +++ b/.agents/skills/session-usage-audit/scripts/codex_churn_audit.py @@ -3,14 +3,14 @@ from __future__ import annotations import argparse -from collections import Counter, defaultdict -from datetime import datetime, timedelta, timezone import hashlib import json import math import os -from pathlib import Path import sys +from collections import Counter, defaultdict +from datetime import UTC, datetime, timedelta +from pathlib import Path import codex_token_usage as tokens from churn_git import CATEGORIES, Classifier, contract_evidence, git, history, inventory @@ -121,7 +121,7 @@ def classify_edits(tasks, repos, config): rid = edit["repo_id"] ref = task["git_metadata"].get("commit_hash") if rid: - repo = next(r for r in repos if r["id"] == rid) + next(r for r in repos if r["id"] == rid) ref = ref if isinstance(ref, str) and len(ref) in (40, 64) and all(c in "0123456789abcdef" for c in ref.lower()) else "HEAD" groups[(rid, ref)].append(edit) else: @@ -151,7 +151,7 @@ def opportunities(tasks, repos, config): candidates.sort(key=lambda c: (c[0], c[1], (c[3] or {}).get("associated_usage", {}).get("observed_tokens", {}).get("uncached_input_tokens") or 0, c[2]["id"], (c[4] or c[5])["path"]), reverse=True) result, seen = [], set() - for strength, _, repo, task, hotspot, reversal in candidates: + for _strength, _, repo, task, hotspot, reversal in candidates: path = (hotspot or reversal)["path"] key = (repo["id"], path) if key in seen: @@ -451,7 +451,7 @@ def save_reports(report, output_dir): handle.write(body) except FileExistsError: if path.read_text() != body: - raise ValueError("existing report differs; refusing overwrite") + raise ValueError("existing report differs; refusing overwrite") from None result[suffix] = str(path) return result @@ -478,7 +478,7 @@ def main(argv=None): raise ValueError("--days must be positive and finite") if bool(args.start) != bool(args.end) or (args.start and args.days is not None): raise ValueError("use --start and --end together, or --days") - end = tokens.parse_timestamp(args.end) if args.end else datetime.now(timezone.utc) + end = tokens.parse_timestamp(args.end) if args.end else datetime.now(UTC) start = tokens.parse_timestamp(args.start) if args.start else end - timedelta(days=args.days or 7) if end <= start: raise ValueError("end must be after start") diff --git a/.agents/skills/session-usage-audit/scripts/codex_token_usage.py b/.agents/skills/session-usage-audit/scripts/codex_token_usage.py index abd296a..7cbd473 100644 --- a/.agents/skills/session-usage-audit/scripts/codex_token_usage.py +++ b/.agents/skills/session-usage-audit/scripts/codex_token_usage.py @@ -7,13 +7,13 @@ from __future__ import annotations import argparse -from collections import Counter, defaultdict -from datetime import datetime, timedelta, timezone import hashlib import json import os -from pathlib import Path import sys +from collections import Counter, defaultdict +from datetime import UTC, datetime, timedelta +from pathlib import Path FIELDS = ("input_tokens", "cached_input_tokens", "cache_write_input_tokens", "output_tokens", "reasoning_output_tokens", "total_tokens") @@ -24,8 +24,8 @@ def parse_timestamp(value): raise ValueError("timestamp must be a string") parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=timezone.utc) - return parsed.astimezone(timezone.utc) + parsed = parsed.replace(tzinfo=UTC) + return parsed.astimezone(UTC) def signature(value): @@ -382,7 +382,7 @@ def main(argv=None): raise ValueError("--top and --days must be positive and finite") if bool(args.start) != bool(args.end) or (args.start and args.days is not None): raise ValueError("use --start with --end, or --days, not both") - end = parse_timestamp(args.end) if args.end else datetime.now(timezone.utc) + end = parse_timestamp(args.end) if args.end else datetime.now(UTC) start = parse_timestamp(args.start) if args.start else end - timedelta(days=args.days or 7) if end <= start: raise ValueError("end must be after start") diff --git a/.agents/skills/session-usage-audit/tests/test_codex_churn_audit.py b/.agents/skills/session-usage-audit/tests/test_codex_churn_audit.py index a66c0c5..522df53 100644 --- a/.agents/skills/session-usage-audit/tests/test_codex_churn_audit.py +++ b/.agents/skills/session-usage-audit/tests/test_codex_churn_audit.py @@ -1,26 +1,26 @@ """Integration fixtures exercise evidence boundaries, not implementation mirrors.""" -from contextlib import redirect_stderr, redirect_stdout -from datetime import datetime, timezone import hashlib import io import json import os -from pathlib import Path import subprocess import sys import tempfile import unittest +from contextlib import redirect_stderr, redirect_stdout +from datetime import UTC, datetime +from pathlib import Path from unittest.mock import patch as mock_patch sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) -import codex_churn_audit as audit -import codex_token_usage as tokens import churn_git as cg import churn_sessions as cs +import codex_churn_audit as audit +import codex_token_usage as tokens -START = datetime(2026, 9, 1, tzinfo=timezone.utc) -END = datetime(2026, 9, 29, tzinfo=timezone.utc) -SECRET = "private source sentinel never exported" +START = datetime(2026, 9, 1, tzinfo=UTC) +END = datetime(2026, 9, 29, tzinfo=UTC) +SECRET = "private source sentinel never exported" # noqa: S105 - schema label or operation identifier, not a credential CONFIG = {"config_version": 1, "repositories": [], "classification": [ {"category": "tests", "patterns": ["tests/*"]}, {"category": "documentation", "patterns": ["*.md"]}, @@ -70,7 +70,7 @@ def g(self, *args, cwd=None, date=None): if date: env.update(GIT_AUTHOR_DATE=date, GIT_COMMITTER_DATE=date) result = subprocess.run(["git", "-C", str(cwd or self.repo), *args], env=env, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + capture_output=True, check=True) return result.stdout.decode().strip() def commit(self, files, date="2026-09-02T00:00:00Z", cwd=None): @@ -207,7 +207,7 @@ def test_historical_attributes_binary_and_root_commit(self): self.assertEqual(classifier.classify("code.py")[0], "tests") self.assertEqual(classifier.classify("code.py", self.initial)[0], "production") self.assertEqual(classifier.classify("asset.bin")[0], "generated_dependency") - r = cg.history(repo, datetime(2026, 8, 1, tzinfo=timezone.utc), END, CONFIG) + r = cg.history(repo, datetime(2026, 8, 1, tzinfo=UTC), END, CONFIG) self.assertTrue(r["primary"]["verified_contiguous_range"]) self.assertEqual(r["primary"]["change_volume"]["generated_dependency"]["binary_or_unknown_files"], 1) self.assertEqual(r["primary"]["change_volume"]["unclassified"]["added_lines"], 3) diff --git a/.agents/skills/session-usage-audit/tests/test_codex_token_usage.py b/.agents/skills/session-usage-audit/tests/test_codex_token_usage.py index 3344366..5eacd5c 100644 --- a/.agents/skills/session-usage-audit/tests/test_codex_token_usage.py +++ b/.agents/skills/session-usage-audit/tests/test_codex_token_usage.py @@ -1,9 +1,9 @@ """Accounting invariants; synthetic records contain no personal transcripts.""" import importlib.util import json -from pathlib import Path import tempfile import unittest +from pathlib import Path SPEC = importlib.util.spec_from_file_location("audit", Path(__file__).parents[1] / "scripts/codex_token_usage.py") audit = importlib.util.module_from_spec(SPEC) diff --git a/.agents/skills/worker-readiness-and-approval/tests/test_skill.py b/.agents/skills/worker-readiness-and-approval/tests/test_skill.py index 3ffef95..54caf60 100644 --- a/.agents/skills/worker-readiness-and-approval/tests/test_skill.py +++ b/.agents/skills/worker-readiness-and-approval/tests/test_skill.py @@ -4,8 +4,8 @@ import json import tempfile -from pathlib import Path import unittest +from pathlib import Path from cops.contracts.models import ActionPlan from cops.execution import ( diff --git a/.agents/skills/workflow-capability-diagnostics/tests/test_skill.py b/.agents/skills/workflow-capability-diagnostics/tests/test_skill.py index b5a7154..89a0ae0 100644 --- a/.agents/skills/workflow-capability-diagnostics/tests/test_skill.py +++ b/.agents/skills/workflow-capability-diagnostics/tests/test_skill.py @@ -1,10 +1,12 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for workflow-capability-diagnostics contributor skill.""" from __future__ import annotations -from pathlib import Path import sys import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: @@ -13,7 +15,6 @@ from cops.diagnostics import ( detect_system_platform, diagnose_host_tools, - diagnose_packages, diagnose_plugin_package, run_diagnostics, ) diff --git a/.claude/skills/adapter-registration-validation/tests/test_skill.py b/.claude/skills/adapter-registration-validation/tests/test_skill.py index ae49970..ced6167 100644 --- a/.claude/skills/adapter-registration-validation/tests/test_skill.py +++ b/.claude/skills/adapter-registration-validation/tests/test_skill.py @@ -3,7 +3,8 @@ from __future__ import annotations import unittest -from cops.adapters import ToolAdapterRegistry, AdapterInjectionError + +from cops.adapters import AdapterInjectionError, ToolAdapterRegistry class TestAdapterRegistrationSkill(unittest.TestCase): diff --git a/.claude/skills/agent-workboard/scripts/workboard.py b/.claude/skills/agent-workboard/scripts/workboard.py index 6d382e6..3e4e9ea 100644 --- a/.claude/skills/agent-workboard/scripts/workboard.py +++ b/.claude/skills/agent-workboard/scripts/workboard.py @@ -13,9 +13,9 @@ import subprocess import sys import uuid +from collections.abc import Iterable, Sequence from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple - +from typing import Any SCHEMA_VERSION = 1 LEASE_MINUTES = 30 @@ -37,14 +37,14 @@ class WorkboardError(RuntimeError): def utc_now() -> str: - return dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds") + return dt.datetime.now(dt.UTC).isoformat(timespec="seconds") def parse_time(value: str) -> dt.datetime: parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00")) if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=dt.timezone.utc) - return parsed.astimezone(dt.timezone.utc) + parsed = parsed.replace(tzinfo=dt.UTC) + return parsed.astimezone(dt.UTC) def json_text(value: Any) -> str: @@ -52,14 +52,14 @@ def json_text(value: Any) -> str: def new_id(prefix: str) -> str: - return "{}_{}".format(prefix, uuid.uuid4().hex) + return f"{prefix}_{uuid.uuid4().hex}" def canonical_path(path: str) -> Path: return Path(path).expanduser().resolve() -def run_git(cwd: Path, args: Sequence[str]) -> Optional[str]: +def run_git(cwd: Path, args: Sequence[str]) -> str | None: try: result = subprocess.run( ["git", "-C", str(cwd)] + list(args), @@ -73,7 +73,7 @@ def run_git(cwd: Path, args: Sequence[str]) -> Optional[str]: return result.stdout.strip() -def resolve_storage(cwd: str) -> Dict[str, str]: +def resolve_storage(cwd: str) -> dict[str, str]: project = canonical_path(cwd) common = run_git( project, ["rev-parse", "--path-format=absolute", "--git-common-dir"] @@ -103,7 +103,7 @@ def resolve_storage(cwd: str) -> Dict[str, str]: } -def resolve_database(cwd: str, explicit: Optional[str]) -> Tuple[Path, Dict[str, str]]: +def resolve_database(cwd: str, explicit: str | None) -> tuple[Path, dict[str, str]]: storage = resolve_storage(cwd) database = canonical_path(explicit) if explicit else Path(storage["database"]) storage["database"] = str(database) @@ -149,9 +149,7 @@ def migrate(connection: sqlite3.Connection) -> None: version = int(connection.execute("PRAGMA user_version").fetchone()[0]) if version > SCHEMA_VERSION: raise WorkboardError( - "database schema {} is newer than supported schema {}".format( - version, SCHEMA_VERSION - ) + f"database schema {version} is newer than supported schema {SCHEMA_VERSION}" ) if version < 1: connection.executescript( @@ -259,11 +257,11 @@ def record_event( connection: sqlite3.Connection, event_type: str, *, - run_id: Optional[str] = None, - task_id: Optional[str] = None, - session_id: Optional[str] = None, - agent_id: Optional[str] = None, - payload: Optional[Dict[str, Any]] = None, + run_id: str | None = None, + task_id: str | None = None, + session_id: str | None = None, + agent_id: str | None = None, + payload: dict[str, Any] | None = None, ) -> None: connection.execute( """ @@ -288,13 +286,13 @@ def fetch_task(connection: sqlite3.Connection, task_id: str) -> sqlite3.Row: "SELECT * FROM tasks WHERE task_id = ?", (task_id,) ).fetchone() if row is None: - raise WorkboardError("unknown task: {}".format(task_id)) + raise WorkboardError(f"unknown task: {task_id}") return row def incomplete_dependencies( connection: sqlite3.Connection, task_id: str -) -> List[sqlite3.Row]: +) -> list[sqlite3.Row]: return list( connection.execute( """ @@ -311,7 +309,7 @@ def incomplete_dependencies( ) -def refresh_ready_tasks(connection: sqlite3.Connection) -> List[str]: +def refresh_ready_tasks(connection: sqlite3.Connection) -> list[str]: rows = list( connection.execute( """ @@ -356,7 +354,7 @@ def write_handoff( task = fetch_task(connection, task_id) handoff_dir = database.parent / "handoffs" secure_directory(handoff_dir) - path = handoff_dir / "{}.md".format(task_id) + path = handoff_dir / f"{task_id}.md" inputs = json.loads(task["inputs_json"]) acceptance = json.loads(task["acceptance_json"]) validation = json.loads(task["validation_json"]) @@ -381,7 +379,7 @@ def write_handoff( ] def bullets(values: Iterable[Any], empty: str = "- None recorded") -> str: - lines = ["- {}".format(value) for value in values] + lines = [f"- {value}" for value in values] return "\n".join(lines) if lines else empty body = """# Task handoff: {title} @@ -505,9 +503,9 @@ def record_artifact_references( def command_init( connection: sqlite3.Connection, database: Path, - storage: Dict[str, str], + storage: dict[str, str], _args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: return { "database": str(database), "project_root": storage["project_root"], @@ -521,9 +519,9 @@ def command_init( def command_run_start( connection: sqlite3.Connection, database: Path, - storage: Dict[str, str], + storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: run_id = args.run_id or new_id("run") now = utc_now() try: @@ -543,7 +541,7 @@ def command_run_start( ), ) except sqlite3.IntegrityError as exc: - raise WorkboardError("cannot start run {}: {}".format(run_id, exc)) + raise WorkboardError(f"cannot start run {run_id}: {exc}") from exc record_event( connection, "run_started", @@ -555,8 +553,8 @@ def command_run_start( return {"database": str(database), "run_id": run_id, "status": "active"} -def load_contract(args: argparse.Namespace) -> Dict[str, Any]: - contract: Dict[str, Any] = {} +def load_contract(args: argparse.Namespace) -> dict[str, Any]: + contract: dict[str, Any] = {} if args.contract_file: contract = json.loads( Path(args.contract_file).expanduser().read_text(encoding="utf-8") @@ -613,15 +611,15 @@ def load_contract(args: argparse.Namespace) -> Dict[str, Any]: def command_task_add( connection: sqlite3.Connection, database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: contract = load_contract(args) run = connection.execute( "SELECT run_id FROM runs WHERE run_id = ?", (args.run_id,) ).fetchone() if run is None: - raise WorkboardError("unknown run: {}".format(args.run_id)) + raise WorkboardError(f"unknown run: {args.run_id}") task_id = args.task_id or new_id("task") dependencies = list(dict.fromkeys(contract["dependencies"])) now = utc_now() @@ -694,9 +692,9 @@ def command_task_add( def command_task_bind( connection: sqlite3.Connection, _database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: try: connection.execute("BEGIN IMMEDIATE") task = fetch_task(connection, args.task_id) @@ -735,16 +733,16 @@ def command_task_bind( def lease_deadline(minutes: int = LEASE_MINUTES) -> str: return ( - dt.datetime.now(dt.timezone.utc) + dt.timedelta(minutes=minutes) + dt.datetime.now(dt.UTC) + dt.timedelta(minutes=minutes) ).isoformat(timespec="seconds") def command_task_claim( connection: sqlite3.Connection, _database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: if args.lease_minutes <= 0: raise WorkboardError("--lease-minutes must be positive") try: @@ -819,9 +817,9 @@ def require_assignee(task: sqlite3.Row, agent_id: str) -> None: def command_task_heartbeat( connection: sqlite3.Connection, _database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: if args.lease_minutes <= 0: raise WorkboardError("--lease-minutes must be positive") try: @@ -862,9 +860,9 @@ def command_task_heartbeat( def command_task_block( connection: sqlite3.Connection, database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: try: connection.execute("BEGIN IMMEDIATE") task = fetch_task(connection, args.task_id) @@ -915,7 +913,7 @@ def command_task_block( } -def list_argument(values: Sequence[str], encoded: Optional[str]) -> List[str]: +def list_argument(values: Sequence[str], encoded: str | None) -> list[str]: if encoded is not None: parsed = json.loads(encoded) if not isinstance(parsed, list): @@ -927,9 +925,9 @@ def list_argument(values: Sequence[str], encoded: Optional[str]) -> List[str]: def command_task_complete( connection: sqlite3.Connection, database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: validation = list_argument(args.validation, args.validation_json) try: connection.execute("BEGIN IMMEDIATE") @@ -979,9 +977,9 @@ def command_task_complete( def command_task_cancel( connection: sqlite3.Connection, database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: try: connection.execute("BEGIN IMMEDIATE") task = fetch_task(connection, args.task_id) @@ -1019,7 +1017,7 @@ def command_task_cancel( } -def task_to_dict(row: sqlite3.Row) -> Dict[str, Any]: +def task_to_dict(row: sqlite3.Row) -> dict[str, Any]: result = dict(row) for field in ("inputs_json", "acceptance_json", "validation_json"): result[field[:-5]] = json.loads(result.pop(field)) @@ -1029,11 +1027,11 @@ def task_to_dict(row: sqlite3.Row) -> Dict[str, Any]: def command_status( connection: sqlite3.Connection, database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: clauses = [] - values: List[Any] = [] + values: list[Any] = [] if args.run_id: clauses.append("task.run_id = ?") values.append(args.run_id) @@ -1047,14 +1045,12 @@ def command_status( where = " WHERE " + " AND ".join(clauses) if clauses else "" rows = list( connection.execute( - """ + f""" SELECT task.* FROM tasks task - {} + {where} ORDER BY task.created_at, task.task_id - """.format( - where - ), + """, # noqa: S608 - fixed SQL clauses; values are bound parameters values, ) ) @@ -1078,9 +1074,9 @@ def command_status( def command_reconcile( connection: sqlite3.Connection, _database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], _args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: try: connection.execute("BEGIN IMMEDIATE") now = utc_now() @@ -1125,9 +1121,9 @@ def command_reconcile( def command_export( connection: sqlite3.Connection, _database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: artifact = connection.execute( """ SELECT path FROM artifacts @@ -1140,13 +1136,13 @@ def command_export( raise WorkboardError("task has no generated handoff") source = Path(artifact["path"]) if not source.is_file(): - raise WorkboardError("handoff is missing: {}".format(source)) + raise WorkboardError(f"handoff is missing: {source}") target = Path(args.to).expanduser() if target.exists() and target.is_dir(): target = target / source.name if target.exists() and not args.force: raise WorkboardError( - "export target exists; pass --force to replace it: {}".format(target) + f"export target exists; pass --force to replace it: {target}" ) target.parent.mkdir(parents=True, exist_ok=True) shutil.copy2(str(source), str(target)) @@ -1156,13 +1152,13 @@ def command_export( def command_gc( connection: sqlite3.Connection, database: Path, - _storage: Dict[str, str], + _storage: dict[str, str], args: argparse.Namespace, -) -> Dict[str, Any]: +) -> dict[str, Any]: if args.older_than < 0: raise WorkboardError("--older-than must be non-negative") cutoff = ( - dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=args.older_than) + dt.datetime.now(dt.UTC) - dt.timedelta(days=args.older_than) ).isoformat(timespec="seconds") try: connection.execute("BEGIN IMMEDIATE") @@ -1252,7 +1248,7 @@ def compact_summary(connection: sqlite3.Connection, database: Path) -> str: """ ) ) - lines = ["Agent workboard: {}".format(database)] + lines = [f"Agent workboard: {database}"] if not rows: lines.append("No active tasks.") else: @@ -1286,7 +1282,7 @@ def hook_output(event: str, context: str) -> None: def session_upsert( - connection: sqlite3.Connection, payload: Dict[str, Any], status: str + connection: sqlite3.Connection, payload: dict[str, Any], status: str ) -> None: session_id = payload.get("session_id") if not session_id: @@ -1325,7 +1321,7 @@ def session_upsert( def task_for_agent( connection: sqlite3.Connection, agent_id: str -) -> Optional[sqlite3.Row]: +) -> sqlite3.Row | None: return connection.execute( """ SELECT * FROM tasks @@ -1451,12 +1447,12 @@ def hook_main(args: argparse.Namespace) -> int: ) connection.commit() else: - raise WorkboardError("unsupported hook event: {}".format(event)) + raise WorkboardError(f"unsupported hook event: {event}") finally: connection.close() return 0 except Exception as exc: - print("agent-workboard hook: {}".format(exc), file=sys.stderr) + print(f"agent-workboard hook: {exc}", file=sys.stderr) if "payload" in locals() and payload.get("hook_event_name") == "SubagentStop": print("{}") return 0 @@ -1577,7 +1573,7 @@ def build_parser() -> argparse.ArgumentParser: return parser -def print_result(result: Dict[str, Any], as_json: bool) -> None: +def print_result(result: dict[str, Any], as_json: bool) -> None: if as_json: print(json.dumps(result, indent=2, sort_keys=True)) return @@ -1592,12 +1588,12 @@ def print_result(result: Dict[str, Any], as_json: bool) -> None: return for key, value in result.items(): if isinstance(value, (dict, list)): - print("{}={}".format(key, json_text(value))) + print(f"{key}={json_text(value)}") else: - print("{}={}".format(key, value)) + print(f"{key}={value}") -def main(argv: Optional[Sequence[str]] = None) -> int: +def main(argv: Sequence[str] | None = None) -> int: os.umask(0o077) parser = build_parser() args = parser.parse_args(argv) @@ -1614,7 +1610,7 @@ def main(argv: Optional[Sequence[str]] = None) -> int: print_result(result, args.json) return 0 except (WorkboardError, json.JSONDecodeError, OSError, sqlite3.Error) as exc: - print("agent-workboard: {}".format(exc), file=sys.stderr) + print(f"agent-workboard: {exc}", file=sys.stderr) return 2 diff --git a/.claude/skills/credential-evidence-handling/tests/test_skill.py b/.claude/skills/credential-evidence-handling/tests/test_skill.py index ee31dcf..86a8806 100644 --- a/.claude/skills/credential-evidence-handling/tests/test_skill.py +++ b/.claude/skills/credential-evidence-handling/tests/test_skill.py @@ -2,11 +2,12 @@ from __future__ import annotations -import unittest -from cops.execution import StreamRedactor, EvidenceRecorder import tempfile +import unittest from pathlib import Path +from cops.execution import EvidenceRecorder, StreamRedactor + class TestCredentialEvidenceSkill(unittest.TestCase): def test_skill_redaction_flow(self): diff --git a/.claude/skills/engagement-contract-validation/tests/test_skill.py b/.claude/skills/engagement-contract-validation/tests/test_skill.py index 8de632e..decf1fc 100644 --- a/.claude/skills/engagement-contract-validation/tests/test_skill.py +++ b/.claude/skills/engagement-contract-validation/tests/test_skill.py @@ -1,11 +1,10 @@ """Tests for engagement-contract-validation skill.""" -from pathlib import Path import unittest +from pathlib import Path from cops.contracts import ( ContractError, - validate_contract, validate_transition, ) diff --git a/.claude/skills/engagement-intake-and-planning/tests/test_skill.py b/.claude/skills/engagement-intake-and-planning/tests/test_skill.py index 6a86fa3..9f4d316 100644 --- a/.claude/skills/engagement-intake-and-planning/tests/test_skill.py +++ b/.claude/skills/engagement-intake-and-planning/tests/test_skill.py @@ -2,14 +2,12 @@ from __future__ import annotations -from pathlib import Path import unittest from cops.catalog import ROOT from cops.engagement import ( build_action_plan, create_engagement_contract, - validate_engagement_intake, ) diff --git a/.claude/skills/execution-authorization-review/tests/test_skill.py b/.claude/skills/execution-authorization-review/tests/test_skill.py index ed4f722..9e4a307 100644 --- a/.claude/skills/execution-authorization-review/tests/test_skill.py +++ b/.claude/skills/execution-authorization-review/tests/test_skill.py @@ -3,14 +3,14 @@ from __future__ import annotations import json -from pathlib import Path import unittest +from pathlib import Path from cops.contracts.models import ActionPlan from cops.execution import ( + consume_execution_authorization, create_execution_authorization, verify_execution_authorization, - consume_execution_authorization, ) diff --git a/.claude/skills/execution-scope-enforcement/tests/test_skill.py b/.claude/skills/execution-scope-enforcement/tests/test_skill.py index 184c853..6441ca8 100644 --- a/.claude/skills/execution-scope-enforcement/tests/test_skill.py +++ b/.claude/skills/execution-scope-enforcement/tests/test_skill.py @@ -3,8 +3,8 @@ from __future__ import annotations import json -from pathlib import Path import unittest +from pathlib import Path from cops.execution import ScopeDefinition, ScopeGuard, ScopeViolationError diff --git a/.claude/skills/network-active-discovery/tests/test_skill.py b/.claude/skills/network-active-discovery/tests/test_skill.py index f07571b..8d7589a 100644 --- a/.claude/skills/network-active-discovery/tests/test_skill.py +++ b/.claude/skills/network-active-discovery/tests/test_skill.py @@ -1,18 +1,20 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-active-discovery contributor skill.""" from __future__ import annotations -from pathlib import Path import sys import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from cops.discovery import ( - ActiveScanSession, ActiveScanner, + ActiveScanSession, OfflineSyntheticDispatcher, ScanBudget, ScanVantage, diff --git a/.claude/skills/network-data-services/tests/test_skill.py b/.claude/skills/network-data-services/tests/test_skill.py index 6caef6f..6df2b60 100644 --- a/.claude/skills/network-data-services/tests/test_skill.py +++ b/.claude/skills/network-data-services/tests/test_skill.py @@ -1,32 +1,31 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-data-services contributor skill.""" from __future__ import annotations import argparse import json -from pathlib import Path import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from cops.discovery import ( - CleanupReceipt, DataAuthPrerequisite, DataExposureStatus, - DataPrivilegeCandidate, DataPrivilegeImpact, - DataServiceAssessment, - DataServiceCategory, DataServicesReport, - DataServiceType, OfflineSyntheticDataCollector, assess_data_services, ) -from cops.discovery.cli import command_data_discovery +from cops.discovery.cli import ( + command_data_discovery, +) class TestNetworkDataServicesSkill(unittest.TestCase): diff --git a/.claude/skills/network-developer-interfaces/tests/test_skill.py b/.claude/skills/network-developer-interfaces/tests/test_skill.py index b0e9db1..bc40ba0 100644 --- a/.claude/skills/network-developer-interfaces/tests/test_skill.py +++ b/.claude/skills/network-developer-interfaces/tests/test_skill.py @@ -1,33 +1,31 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-developer-interfaces contributor skill.""" from __future__ import annotations import argparse import json -from pathlib import Path import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from cops.discovery import ( - CleanupReceipt, DeveloperAuthPrerequisite, - DeveloperCategory, DeveloperExposureStatus, - DeveloperPrivilegeCandidate, - DeveloperPrivilegeImpact, - DeveloperServiceAssessment, DeveloperServicesReport, - DeveloperServiceType, ExecutionEffect, OfflineSyntheticDeveloperCollector, assess_developer_services, ) -from cops.discovery.cli import command_developer_discovery +from cops.discovery.cli import ( + command_developer_discovery, +) class TestNetworkDeveloperInterfacesSkill(unittest.TestCase): diff --git a/.claude/skills/network-infrastructure-services/tests/test_skill.py b/.claude/skills/network-infrastructure-services/tests/test_skill.py index 02fc5b7..485c355 100644 --- a/.claude/skills/network-infrastructure-services/tests/test_skill.py +++ b/.claude/skills/network-infrastructure-services/tests/test_skill.py @@ -1,10 +1,12 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-infrastructure-services contributor skill.""" from __future__ import annotations -from pathlib import Path import sys import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: @@ -13,7 +15,6 @@ from cops.discovery import ( AuthPrerequisite, IdentityAttackPathType, - InfraServiceAssessment, InfraServiceType, OfflineSyntheticInfraCollector, ServiceExposureStatus, diff --git a/.claude/skills/network-legacy-and-proxy-services/tests/test_skill.py b/.claude/skills/network-legacy-and-proxy-services/tests/test_skill.py index d2efddc..1aabd49 100644 --- a/.claude/skills/network-legacy-and-proxy-services/tests/test_skill.py +++ b/.claude/skills/network-legacy-and-proxy-services/tests/test_skill.py @@ -1,13 +1,15 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-legacy-and-proxy-services contributor skill.""" from __future__ import annotations import argparse import json -from pathlib import Path import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: @@ -21,13 +23,14 @@ LegacyExposureStatus, LegacyPrivilegeCandidate, LegacyPrivilegeImpact, - LegacyServiceAssessment, LegacyServicesReport, LegacyServiceType, OfflineSyntheticLegacyCollector, assess_legacy_services, ) -from cops.discovery.cli import command_legacy_discovery +from cops.discovery.cli import ( + command_legacy_discovery, +) class TestNetworkLegacyAndProxyServicesSkill(unittest.TestCase): diff --git a/.claude/skills/network-messaging-services/tests/test_skill.py b/.claude/skills/network-messaging-services/tests/test_skill.py index 2bdbe4e..f3e55d1 100644 --- a/.claude/skills/network-messaging-services/tests/test_skill.py +++ b/.claude/skills/network-messaging-services/tests/test_skill.py @@ -1,32 +1,30 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-messaging-services contributor skill.""" from __future__ import annotations import argparse import json -from pathlib import Path import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from cops.discovery import ( - CleanupReceipt, MessagingAuthPrerequisite, - MessagingCategory, MessagingExposureStatus, - MessagingPrivilegeCandidate, - MessagingPrivilegeImpact, - MessagingServiceAssessment, MessagingServicesReport, - MessagingServiceType, OfflineSyntheticMessagingCollector, assess_messaging_services, ) -from cops.discovery.cli import command_messaging_discovery +from cops.discovery.cli import ( + command_messaging_discovery, +) class TestNetworkMessagingServicesSkill(unittest.TestCase): diff --git a/.claude/skills/network-passive-discovery/tests/test_skill.py b/.claude/skills/network-passive-discovery/tests/test_skill.py index 0a4a174..e80de86 100644 --- a/.claude/skills/network-passive-discovery/tests/test_skill.py +++ b/.claude/skills/network-passive-discovery/tests/test_skill.py @@ -1,10 +1,12 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-passive-discovery contributor skill.""" from __future__ import annotations -from pathlib import Path import sys import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: diff --git a/.claude/skills/network-remote-and-file-services/tests/test_skill.py b/.claude/skills/network-remote-and-file-services/tests/test_skill.py index fbe8dbc..ed76fe8 100644 --- a/.claude/skills/network-remote-and-file-services/tests/test_skill.py +++ b/.claude/skills/network-remote-and-file-services/tests/test_skill.py @@ -1,32 +1,32 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for network-remote-and-file-services contributor skill.""" from __future__ import annotations import json -from pathlib import Path import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +import argparse + from cops.discovery import ( - CleanupReceipt, - HostPrivilegeCandidate, LateralMovementImpact, OfflineSyntheticRemoteCollector, RemoteAuthPrerequisite, RemoteExposureStatus, - RemoteServiceAssessment, - RemoteServiceCategory, RemoteServicesReport, - RemoteServiceType, assess_remote_services, ) -from cops.discovery.cli import command_remote_discovery -import argparse +from cops.discovery.cli import ( + command_remote_discovery, +) class TestNetworkRemoteAndFileServicesSkill(unittest.TestCase): diff --git a/.claude/skills/plugin-run-cost/scripts/run_cost.py b/.claude/skills/plugin-run-cost/scripts/run_cost.py index 58c1d4f..5c22ab0 100644 --- a/.claude/skills/plugin-run-cost/scripts/run_cost.py +++ b/.claude/skills/plugin-run-cost/scripts/run_cost.py @@ -3,17 +3,16 @@ from __future__ import annotations import argparse -from collections import Counter, defaultdict -from datetime import datetime, timezone -from decimal import Decimal, InvalidOperation import hashlib import importlib.util import json import math import os -from pathlib import Path import re import tempfile +from collections import Counter, defaultdict +from decimal import Decimal, InvalidOperation +from pathlib import Path ROOT = Path(__file__).resolve().parents[1] parser_spec = importlib.util.spec_from_file_location( @@ -324,7 +323,7 @@ def profile(repositories, wikis, threat_models, max_files=10000, max_bytes=20000 continue if not root.exists(): raise ValueError('profile input does not exist') - def walk(): + def walk(root=root): # Iterate directories without materializing their contents. The shared # budget counts roots, directories and excluded entries across all inputs. pending = [(root, False)] @@ -461,7 +460,7 @@ def compare(config): item = dict(name=safe_label(option['name']), qualified=option.get('qualified') is True, annual_capacity_hours=float(hours), at_hourly_rate={}) for rate in rates: - def recurring(a): + def recurring(a, rate=rate): return ((number(a['active_minutes'])/60*rate + number(a.get('variable_usd', 0)))*annual + number(a.get('monthly_maintenance_hours', 0))*rate*12 + number(a.get('monthly_maintenance_usd', 0))*12 + number(a.get('annual_license_usd', 0))) diff --git a/.claude/skills/plugin-run-cost/tests/test_cost.py b/.claude/skills/plugin-run-cost/tests/test_cost.py index 66fcc13..e6acbbb 100644 --- a/.claude/skills/plugin-run-cost/tests/test_cost.py +++ b/.claude/skills/plugin-run-cost/tests/test_cost.py @@ -1,8 +1,8 @@ import importlib.util import json -from pathlib import Path import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[1] spec = importlib.util.spec_from_file_location('run_cost', ROOT / 'scripts/run_cost.py') diff --git a/.claude/skills/scenario-laboratory-management/tests/test_skill.py b/.claude/skills/scenario-laboratory-management/tests/test_skill.py index 7b3b054..5ec6bc0 100644 --- a/.claude/skills/scenario-laboratory-management/tests/test_skill.py +++ b/.claude/skills/scenario-laboratory-management/tests/test_skill.py @@ -3,8 +3,8 @@ from __future__ import annotations import tempfile -from pathlib import Path import unittest +from pathlib import Path from cops.execution.store import ApprovalStore from cops.laboratory import ( diff --git a/.claude/skills/session-usage-audit/scripts/churn_git.py b/.claude/skills/session-usage-audit/scripts/churn_git.py index 991607c..e88c724 100644 --- a/.claude/skills/session-usage-audit/scripts/churn_git.py +++ b/.claude/skills/session-usage-audit/scripts/churn_git.py @@ -1,13 +1,13 @@ """Read-only Git inventory and immutable, first-parent history measurements.""" from __future__ import annotations -from collections import Counter -from datetime import datetime, timezone -from fnmatch import fnmatchcase import hashlib import os -from pathlib import Path import subprocess +from collections import Counter +from datetime import UTC, datetime +from fnmatch import fnmatchcase +from pathlib import Path CATEGORIES = ("production", "tests", "documentation", "generated_dependency", "unclassified") @@ -19,7 +19,7 @@ def git(path, *args, data=None, check=True): env.pop(name, None) result = subprocess.run(["git", "--no-optional-locks", "--no-replace-objects", "-C", str(path), "-c", "core.quotePath=false", *args], input=data, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + capture_output=True, env=env) if check and result.returncode: # Error bodies may contain private source; report only the operation. raise ValueError(f"Git {args[0]} failed in {path} (exit {result.returncode})") @@ -232,7 +232,7 @@ def changes(sha, parent): for line in decode(raw).splitlines(): ids, epoch = line.split("\t") fields = ids.split() - chain.append((fields[0], fields[1:], datetime.fromtimestamp(int(epoch), timezone.utc))) + chain.append((fields[0], fields[1:], datetime.fromtimestamp(int(epoch), UTC))) # Locate an immutable end then a contiguous ancestry range. Nonmonotonic # commit dates are diagnosed and disable range ratios/rework. before_end = [c for c in chain if c[2] < end] diff --git a/.claude/skills/session-usage-audit/scripts/churn_sessions.py b/.claude/skills/session-usage-audit/scripts/churn_sessions.py index 3ec9a9e..2cdbf82 100644 --- a/.claude/skills/session-usage-audit/scripts/churn_sessions.py +++ b/.claude/skills/session-usage-audit/scripts/churn_sessions.py @@ -1,12 +1,11 @@ """Static transcript extraction. Nothing read from a transcript is executed.""" from __future__ import annotations -from collections import Counter, defaultdict -from datetime import datetime import json -from pathlib import Path import re import shlex +from collections import Counter, defaultdict +from pathlib import Path import codex_token_usage as tokens from churn_git import metadata_repos, resolve_path @@ -490,7 +489,7 @@ def scan(paths, start, end, repos): for (rid, path, tree), edits in sorted(hotspots.items(), key=lambda kv: str(kv[0])): outcomes = Counter(e["outcome"] for e in edits) candidate_pairs = [] - for a, b in zip(edits, edits[1:]): + for a, b in zip(edits, edits[1:], strict=False): if a["operation"] == b["operation"] == "update" and a["_reverse"] == b["_forward"] and a["_forward"] != b["_forward"]: candidate_pairs.append({"kind": "inverse_patch_candidate", "confirmed": False, "evidence": [a["evidence"][0], b["evidence"][0]]}) diff --git a/.claude/skills/session-usage-audit/scripts/codex_churn_audit.py b/.claude/skills/session-usage-audit/scripts/codex_churn_audit.py index 5158825..aa4c791 100644 --- a/.claude/skills/session-usage-audit/scripts/codex_churn_audit.py +++ b/.claude/skills/session-usage-audit/scripts/codex_churn_audit.py @@ -3,14 +3,14 @@ from __future__ import annotations import argparse -from collections import Counter, defaultdict -from datetime import datetime, timedelta, timezone import hashlib import json import math import os -from pathlib import Path import sys +from collections import Counter, defaultdict +from datetime import UTC, datetime, timedelta +from pathlib import Path import codex_token_usage as tokens from churn_git import CATEGORIES, Classifier, contract_evidence, git, history, inventory @@ -121,7 +121,7 @@ def classify_edits(tasks, repos, config): rid = edit["repo_id"] ref = task["git_metadata"].get("commit_hash") if rid: - repo = next(r for r in repos if r["id"] == rid) + next(r for r in repos if r["id"] == rid) ref = ref if isinstance(ref, str) and len(ref) in (40, 64) and all(c in "0123456789abcdef" for c in ref.lower()) else "HEAD" groups[(rid, ref)].append(edit) else: @@ -151,7 +151,7 @@ def opportunities(tasks, repos, config): candidates.sort(key=lambda c: (c[0], c[1], (c[3] or {}).get("associated_usage", {}).get("observed_tokens", {}).get("uncached_input_tokens") or 0, c[2]["id"], (c[4] or c[5])["path"]), reverse=True) result, seen = [], set() - for strength, _, repo, task, hotspot, reversal in candidates: + for _strength, _, repo, task, hotspot, reversal in candidates: path = (hotspot or reversal)["path"] key = (repo["id"], path) if key in seen: @@ -451,7 +451,7 @@ def save_reports(report, output_dir): handle.write(body) except FileExistsError: if path.read_text() != body: - raise ValueError("existing report differs; refusing overwrite") + raise ValueError("existing report differs; refusing overwrite") from None result[suffix] = str(path) return result @@ -478,7 +478,7 @@ def main(argv=None): raise ValueError("--days must be positive and finite") if bool(args.start) != bool(args.end) or (args.start and args.days is not None): raise ValueError("use --start and --end together, or --days") - end = tokens.parse_timestamp(args.end) if args.end else datetime.now(timezone.utc) + end = tokens.parse_timestamp(args.end) if args.end else datetime.now(UTC) start = tokens.parse_timestamp(args.start) if args.start else end - timedelta(days=args.days or 7) if end <= start: raise ValueError("end must be after start") diff --git a/.claude/skills/session-usage-audit/scripts/codex_token_usage.py b/.claude/skills/session-usage-audit/scripts/codex_token_usage.py index abd296a..7cbd473 100644 --- a/.claude/skills/session-usage-audit/scripts/codex_token_usage.py +++ b/.claude/skills/session-usage-audit/scripts/codex_token_usage.py @@ -7,13 +7,13 @@ from __future__ import annotations import argparse -from collections import Counter, defaultdict -from datetime import datetime, timedelta, timezone import hashlib import json import os -from pathlib import Path import sys +from collections import Counter, defaultdict +from datetime import UTC, datetime, timedelta +from pathlib import Path FIELDS = ("input_tokens", "cached_input_tokens", "cache_write_input_tokens", "output_tokens", "reasoning_output_tokens", "total_tokens") @@ -24,8 +24,8 @@ def parse_timestamp(value): raise ValueError("timestamp must be a string") parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=timezone.utc) - return parsed.astimezone(timezone.utc) + parsed = parsed.replace(tzinfo=UTC) + return parsed.astimezone(UTC) def signature(value): @@ -382,7 +382,7 @@ def main(argv=None): raise ValueError("--top and --days must be positive and finite") if bool(args.start) != bool(args.end) or (args.start and args.days is not None): raise ValueError("use --start with --end, or --days, not both") - end = parse_timestamp(args.end) if args.end else datetime.now(timezone.utc) + end = parse_timestamp(args.end) if args.end else datetime.now(UTC) start = parse_timestamp(args.start) if args.start else end - timedelta(days=args.days or 7) if end <= start: raise ValueError("end must be after start") diff --git a/.claude/skills/session-usage-audit/tests/test_codex_churn_audit.py b/.claude/skills/session-usage-audit/tests/test_codex_churn_audit.py index a66c0c5..522df53 100644 --- a/.claude/skills/session-usage-audit/tests/test_codex_churn_audit.py +++ b/.claude/skills/session-usage-audit/tests/test_codex_churn_audit.py @@ -1,26 +1,26 @@ """Integration fixtures exercise evidence boundaries, not implementation mirrors.""" -from contextlib import redirect_stderr, redirect_stdout -from datetime import datetime, timezone import hashlib import io import json import os -from pathlib import Path import subprocess import sys import tempfile import unittest +from contextlib import redirect_stderr, redirect_stdout +from datetime import UTC, datetime +from pathlib import Path from unittest.mock import patch as mock_patch sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) -import codex_churn_audit as audit -import codex_token_usage as tokens import churn_git as cg import churn_sessions as cs +import codex_churn_audit as audit +import codex_token_usage as tokens -START = datetime(2026, 9, 1, tzinfo=timezone.utc) -END = datetime(2026, 9, 29, tzinfo=timezone.utc) -SECRET = "private source sentinel never exported" +START = datetime(2026, 9, 1, tzinfo=UTC) +END = datetime(2026, 9, 29, tzinfo=UTC) +SECRET = "private source sentinel never exported" # noqa: S105 - schema label or operation identifier, not a credential CONFIG = {"config_version": 1, "repositories": [], "classification": [ {"category": "tests", "patterns": ["tests/*"]}, {"category": "documentation", "patterns": ["*.md"]}, @@ -70,7 +70,7 @@ def g(self, *args, cwd=None, date=None): if date: env.update(GIT_AUTHOR_DATE=date, GIT_COMMITTER_DATE=date) result = subprocess.run(["git", "-C", str(cwd or self.repo), *args], env=env, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) + capture_output=True, check=True) return result.stdout.decode().strip() def commit(self, files, date="2026-09-02T00:00:00Z", cwd=None): @@ -207,7 +207,7 @@ def test_historical_attributes_binary_and_root_commit(self): self.assertEqual(classifier.classify("code.py")[0], "tests") self.assertEqual(classifier.classify("code.py", self.initial)[0], "production") self.assertEqual(classifier.classify("asset.bin")[0], "generated_dependency") - r = cg.history(repo, datetime(2026, 8, 1, tzinfo=timezone.utc), END, CONFIG) + r = cg.history(repo, datetime(2026, 8, 1, tzinfo=UTC), END, CONFIG) self.assertTrue(r["primary"]["verified_contiguous_range"]) self.assertEqual(r["primary"]["change_volume"]["generated_dependency"]["binary_or_unknown_files"], 1) self.assertEqual(r["primary"]["change_volume"]["unclassified"]["added_lines"], 3) diff --git a/.claude/skills/session-usage-audit/tests/test_codex_token_usage.py b/.claude/skills/session-usage-audit/tests/test_codex_token_usage.py index 3344366..5eacd5c 100644 --- a/.claude/skills/session-usage-audit/tests/test_codex_token_usage.py +++ b/.claude/skills/session-usage-audit/tests/test_codex_token_usage.py @@ -1,9 +1,9 @@ """Accounting invariants; synthetic records contain no personal transcripts.""" import importlib.util import json -from pathlib import Path import tempfile import unittest +from pathlib import Path SPEC = importlib.util.spec_from_file_location("audit", Path(__file__).parents[1] / "scripts/codex_token_usage.py") audit = importlib.util.module_from_spec(SPEC) diff --git a/.claude/skills/worker-readiness-and-approval/tests/test_skill.py b/.claude/skills/worker-readiness-and-approval/tests/test_skill.py index 3ffef95..54caf60 100644 --- a/.claude/skills/worker-readiness-and-approval/tests/test_skill.py +++ b/.claude/skills/worker-readiness-and-approval/tests/test_skill.py @@ -4,8 +4,8 @@ import json import tempfile -from pathlib import Path import unittest +from pathlib import Path from cops.contracts.models import ActionPlan from cops.execution import ( diff --git a/.claude/skills/workflow-capability-diagnostics/tests/test_skill.py b/.claude/skills/workflow-capability-diagnostics/tests/test_skill.py index b5a7154..89a0ae0 100644 --- a/.claude/skills/workflow-capability-diagnostics/tests/test_skill.py +++ b/.claude/skills/workflow-capability-diagnostics/tests/test_skill.py @@ -1,10 +1,12 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Tests for workflow-capability-diagnostics contributor skill.""" from __future__ import annotations -from pathlib import Path import sys import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parents[4] if str(ROOT) not in sys.path: @@ -13,7 +15,6 @@ from cops.diagnostics import ( detect_system_platform, diagnose_host_tools, - diagnose_packages, diagnose_plugin_package, run_diagnostics, ) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 4552a35..0725cf2 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Initialize CodeQL uses: github/codeql-action/init@v3 diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index 403159c..430d817 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/security-and-quality.yml b/.github/workflows/security-and-quality.yml index 0767140..c4f5c48 100644 --- a/.github/workflows/security-and-quality.yml +++ b/.github/workflows/security-and-quality.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 0 @@ -44,7 +44,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: fetch-depth: 0 @@ -63,7 +63,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v5 @@ -88,7 +88,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/summary.yml b/.github/workflows/summary.yml index 1bc0b64..ffbe29c 100644 --- a/.github/workflows/summary.yml +++ b/.github/workflows/summary.yml @@ -28,7 +28,7 @@ jobs: - name: Comment with AI summary run: | - gh issue comment $ISSUE_NUMBER --body "$RESPONSE" + gh issue comment "$ISSUE_NUMBER" --body "$RESPONSE" env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} ISSUE_NUMBER: ${{ github.event.issue.number }} diff --git a/cops/__main__.py b/cops/__main__.py index 21b8bbf..827744c 100644 --- a/cops/__main__.py +++ b/cops/__main__.py @@ -4,6 +4,5 @@ from .cli import main - if __name__ == "__main__": raise SystemExit(main()) diff --git a/cops/adapters/registry.py b/cops/adapters/registry.py index bf77156..2d01985 100644 --- a/cops/adapters/registry.py +++ b/cops/adapters/registry.py @@ -6,14 +6,13 @@ from __future__ import annotations -from dataclasses import dataclass, field import ipaddress import json -from pathlib import Path import re -import shutil import sys -from typing import Any, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any class AdapterError(ValueError): @@ -83,13 +82,13 @@ def validate_and_format(self, value: Any) -> list[str]: ipaddress.ip_address(str_val) val_token = str_val except ValueError: - raise AdapterParameterError(f"Parameter '{self.name}' must be a valid IP address, got {str_val!r}") + raise AdapterParameterError(f"Parameter '{self.name}' must be a valid IP address, got {str_val!r}") from None elif self.param_type == "cidr": try: ipaddress.ip_network(str_val, strict=False) val_token = str_val except ValueError: - raise AdapterParameterError(f"Parameter '{self.name}' must be a valid CIDR network, got {str_val!r}") + raise AdapterParameterError(f"Parameter '{self.name}' must be a valid CIDR network, got {str_val!r}") from None else: val_token = str_val diff --git a/cops/authorization.py b/cops/authorization.py index ea126e2..c882dcc 100644 --- a/cops/authorization.py +++ b/cops/authorization.py @@ -12,12 +12,13 @@ import json import os import sys -from dataclasses import asdict, dataclass -from datetime import datetime, timedelta, timezone +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta from pathlib import Path -from typing import Any, Callable, Sequence +from typing import Any -from cops.evidence.canonical import canonical, digest, utc_now +from cops.evidence.canonical import digest class AuthorizationError(ValueError): @@ -88,7 +89,7 @@ def create_authorization_receipt( if approval_mode not in {"interactive_confirmation", "pre_signed_envelope"}: raise AuthorizationError(f"invalid approval_mode: {approval_mode}") - now = datetime.now(timezone.utc) + now = datetime.now(UTC) timestamp_str = now.isoformat(timespec="seconds").replace("+00:00", "Z") expiry_str = (now + timedelta(hours=valid_hours)).isoformat(timespec="seconds").replace("+00:00", "Z") @@ -104,7 +105,7 @@ def create_authorization_receipt( "approval_mode": approval_mode, } verification_hash = _compute_receipt_hash(payload) - receipt_id = hashlib.sha256(f"{verification_hash}:{timestamp_str}".encode("utf-8")).hexdigest()[:24] + receipt_id = hashlib.sha256(f"{verification_hash}:{timestamp_str}".encode()).hexdigest()[:24] return AuthorizationReceipt( schema_version="1.0", @@ -164,7 +165,7 @@ def validate_receipt_document(document: dict[str, Any]) -> AuthorizationReceipt: try: expiry = datetime.fromisoformat(document["authorized_until_utc"].replace("Z", "+00:00")) - now = datetime.now(timezone.utc) + now = datetime.now(UTC) if now > expiry: raise AuthorizationError(f"authorization receipt expired at {document['authorized_until_utc']}") except ValueError as err: diff --git a/cops/capabilities/auditor.py b/cops/capabilities/auditor.py index c0598a7..5cbabee 100644 --- a/cops/capabilities/auditor.py +++ b/cops/capabilities/auditor.py @@ -2,7 +2,6 @@ from __future__ import annotations -from datetime import datetime, timezone import json from pathlib import Path from typing import Any @@ -12,7 +11,6 @@ from .models import CapabilityEntry, CapabilityTruthError - ROOT: Path = Path(__file__).resolve().parents[2] diff --git a/cops/catalog.py b/cops/catalog.py index 86809c6..bff970a 100644 --- a/cops/catalog.py +++ b/cops/catalog.py @@ -8,7 +8,6 @@ from pathlib import Path from typing import Any - ROOT = Path(__file__).resolve().parents[1] diff --git a/cops/cli.py b/cops/cli.py index d73952f..6a31f55 100644 --- a/cops/cli.py +++ b/cops/cli.py @@ -10,8 +10,9 @@ import shutil import subprocess import sys +from collections.abc import Sequence from pathlib import Path -from typing import Any, Sequence +from typing import Any from .catalog import ROOT, CatalogError, PluginRecord, find_plugin, plugin_records, validate_declared_command from .coverage import ( @@ -98,7 +99,7 @@ def command_doctor(*, contributor: bool, root: Path = ROOT) -> int: missing: list[str] = [] print(f"COPS root: {root}") print(f"Python: {platform.python_version()} ({sys.executable})") - if sys.version_info < (3, 11): + if sys.version_info < (3, 11): # noqa: UP036 - diagnose unsupported Python runtimes missing.append("Python 3.11 or newer") try: records = validate_repository(root) @@ -460,8 +461,8 @@ def command_worker_store_list(args: argparse.Namespace) -> int: def command_worker_execute(args: argparse.Namespace) -> int: - from .contracts.models import ActionPlan, ExecutionAuthorization - from .execution import ApprovalStore, IsolatedWorker, WorkerConfig, ScopeGuard + from .contracts.models import ExecutionAuthorization + from .execution import ApprovalStore, IsolatedWorker, ScopeGuard, WorkerConfig plan_path = Path(args.plan) if not plan_path.is_file(): print(f"error: plan file not found: {plan_path}", file=sys.stderr) diff --git a/cops/connectors/__init__.py b/cops/connectors/__init__.py index db2861c..131dd6d 100644 --- a/cops/connectors/__init__.py +++ b/cops/connectors/__init__.py @@ -1,6 +1,6 @@ """Repository-owned, standard-library evidence connector SDK.""" -from .adapters.microsoft import GraphUsers, ResourceGraph from .adapters.m365 import GraphCollection +from .adapters.microsoft import GraphUsers, ResourceGraph from .checkpoint import Checkpoint from .interfaces import Adapter, CredentialProvider, Page, Request, Response, Result, Transport from .policy import Limits diff --git a/cops/connectors/adapters/m365.py b/cops/connectors/adapters/m365.py index 61f131f..eabdbbf 100644 --- a/cops/connectors/adapters/m365.py +++ b/cops/connectors/adapters/m365.py @@ -2,10 +2,10 @@ from urllib.parse import parse_qsl, urlencode from cops.evidence import EvidenceError + from ..interfaces import Page, Request from .microsoft import destination, guid, opaque - # Fields are intentionally narrower than the provider responses. The caller # cannot supply an arbitrary Graph path, projection, or OData expression. SOURCES = { diff --git a/cops/connectors/adapters/microsoft.py b/cops/connectors/adapters/microsoft.py index 51405a1..a26de2a 100644 --- a/cops/connectors/adapters/microsoft.py +++ b/cops/connectors/adapters/microsoft.py @@ -1,8 +1,10 @@ """Two reviewed, read-only projections; provider pagination stays private.""" import re from urllib.parse import parse_qsl, urlsplit + from cops.evidence import EvidenceError from cops.evidence.canonical import canonical + from ..interfaces import Page, Request UUID = re.compile(r'^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$') diff --git a/cops/connectors/checkpoint.py b/cops/connectors/checkpoint.py index 007ae21..4cacc9b 100644 --- a/cops/connectors/checkpoint.py +++ b/cops/connectors/checkpoint.py @@ -1,8 +1,9 @@ """Private SQLite page transactions and an exclusive acquisition lock.""" import os -from pathlib import Path import sqlite3 import stat +from pathlib import Path + from cops.evidence import EvidenceError, decode_json from cops.evidence.canonical import canonical diff --git a/cops/connectors/demo.py b/cops/connectors/demo.py index 2efeb94..1f2d7c0 100644 --- a/cops/connectors/demo.py +++ b/cops/connectors/demo.py @@ -4,6 +4,7 @@ from tempfile import TemporaryDirectory from cops.evidence import EvidenceError, build_envelope, canonical, report + from . import Checkpoint, GraphUsers, Limits, ResourceGraph, Response, collect TENANT = '00000000-0000-0000-0000-000000000001' diff --git a/cops/connectors/interfaces.py b/cops/connectors/interfaces.py index 63739c5..106604b 100644 --- a/cops/connectors/interfaces.py +++ b/cops/connectors/interfaces.py @@ -1,6 +1,7 @@ """Ephemeral transport values are deliberately excluded from representations.""" +from collections.abc import Iterable from dataclasses import dataclass, field -from typing import Iterable, Protocol +from typing import Protocol @dataclass(frozen=True) diff --git a/cops/connectors/policy.py b/cops/connectors/policy.py index 5f8ed95..5d566ba 100644 --- a/cops/connectors/policy.py +++ b/cops/connectors/policy.py @@ -1,6 +1,7 @@ """Finite, validated acquisition limits; resumes may only tighten them.""" -from dataclasses import asdict, dataclass import math +from dataclasses import asdict, dataclass + from cops.evidence import EvidenceError diff --git a/cops/connectors/runner.py b/cops/connectors/runner.py index b962c43..a9a305d 100644 --- a/cops/connectors/runner.py +++ b/cops/connectors/runner.py @@ -1,61 +1,116 @@ """One bounded lifecycle for reviewed adapters; no provider text enters diagnostics.""" -from datetime import datetime -from email.utils import parsedate_to_datetime + import re import time import uuid +from datetime import datetime +from email.utils import parsedate_to_datetime from cops.evidence import EvidenceError, build_envelope, decode_json, validate_envelope, validate_receipt from cops.evidence.canonical import canonical, digest, timestamp, utc_now -from .interfaces import Result, Response + +from .interfaces import Response, Result from .policy import Limits -REASONS = {'page_limit', 'attempt_limit', 'record_limit', 'byte_limit', 'storage_limit', - 'time_limit', 'response_limit', 'invalid_json', 'schema_drift', 'unsafe_destination', - 'cursor_cycle', 'cursor_expired', 'truncated_without_cursor', 'retry_continuation', - 'authentication', 'transport', 'throttled', 'request_timeout', 'checkpoint_invalid', 'projection_failed'} +REASONS = { + "page_limit", + "attempt_limit", + "record_limit", + "byte_limit", + "storage_limit", + "time_limit", + "response_limit", + "invalid_json", + "schema_drift", + "unsafe_destination", + "cursor_cycle", + "cursor_expired", + "truncated_without_cursor", + "retry_continuation", + "authentication", + "transport", + "throttled", + "request_timeout", + "checkpoint_invalid", + "projection_failed", +} def preview(adapter): """Build only the initial, credential-free request. No cursor or private response.""" request = adapter.request() adapter.validate_request(request) - return {'adapter': adapter.name, 'adapter_version': adapter.version, 'tenant': adapter.tenant, - 'scope': list(adapter.scope), 'method': request.method, 'url': request.url, - 'body': decode_json(request.body) if request.body else None, - 'headers': dict(request.headers)} + return { + "adapter": adapter.name, + "adapter_version": adapter.version, + "tenant": adapter.tenant, + "scope": list(adapter.scope), + "method": request.method, + "url": request.url, + "body": decode_json(request.body) if request.body else None, + "headers": dict(request.headers), + } def _receipt(state, now): - item = {key: state[key] for key in ('acquisition_id', 'generation', 'adapter', 'adapter_version', - 'tenant', 'scope', 'request_fingerprint', 'started_at', 'limits', 'consumed')} - finished_at = state['finished_at'] if state['status'] in ('complete', 'partial') else _finish_time(state, now) - item.update(schema_version='cops.acquisition/v1', finished_at=finished_at, - status=state['status'], reasons=list(state['reasons']), consistency='unknown') + item = { + key: state[key] + for key in ( + "acquisition_id", + "generation", + "adapter", + "adapter_version", + "tenant", + "scope", + "request_fingerprint", + "started_at", + "limits", + "consumed", + ) + } + finished_at = state["finished_at"] if state["status"] in ("complete", "partial") else _finish_time(state, now) + item.update( + schema_version="cops.acquisition/v1", + finished_at=finished_at, + status=state["status"], + reasons=list(state["reasons"]), + consistency="unknown", + ) return validate_receipt(item) def _finish_time(state, now): - return now if timestamp(now) >= timestamp(state['started_at']) else state['started_at'] + return now if timestamp(now) >= timestamp(state["started_at"]) else state["started_at"] def _delay(value, retry, now): if value is None: - return min(2 ** retry, 30) + return min(2**retry, 30) if not isinstance(value, str) or len(value) > 128: - raise EvidenceError('throttled') - if re.fullmatch(r'[0-9]{1,8}', value): + raise EvidenceError("throttled") + if re.fullmatch(r"[0-9]{1,8}", value): return float(value) try: date = parsedate_to_datetime(value) - if date.tzinfo is None: raise ValueError - return max(0, (date - datetime.fromisoformat(now.replace('Z', '+00:00'))).total_seconds()) + if date.tzinfo is None: + raise ValueError + return max(0, (date - datetime.fromisoformat(now.replace("Z", "+00:00"))).total_seconds()) except (ValueError, TypeError, OverflowError): - raise EvidenceError('throttled') from None + raise EvidenceError("throttled") from None -def collect(adapter, checkpoint, credentials, transport=None, *, limits=None, - clock=time.monotonic, sleep=time.sleep, now=utc_now, fault=lambda phase: None): +def collect( + adapter, + checkpoint, + credentials, + transport=None, + *, + limits=None, + clock=time.monotonic, + sleep=time.sleep, + now=utc_now, + fault=lambda phase: None, +): """Resume the same authorized query. Trusted callbacks must return promptly. Injected transports must enforce the supplied timeout and response byte cap. @@ -64,170 +119,235 @@ def collect(adapter, checkpoint, credentials, transport=None, *, limits=None, limits = limits or Limits() plan = preview(adapter) fingerprint = digest(plan) - binding = dict(adapter=adapter.name, adapter_version=adapter.version, tenant=adapter.tenant, - scope=list(adapter.scope), request_fingerprint=fingerprint) + binding = dict( + adapter=adapter.name, + adapter_version=adapter.version, + tenant=adapter.tenant, + scope=list(adapter.scope), + request_fingerprint=fingerprint, + ) state = checkpoint.load() if state is None: - state = dict(binding, acquisition_id=str(uuid.uuid4()), generation=1, started_at=now(), finished_at=None, - limits=limits.export(), consumed=dict(pages=0, attempts=0, records=0, duplicates=0, - bytes=0, storage_bytes=0, active_seconds=0), - cursor=None, status='unknown', reasons=[], refreshed=False, reservation=False, retry=0) + state = dict( + binding, + acquisition_id=str(uuid.uuid4()), + generation=1, + started_at=now(), + finished_at=None, + limits=limits.export(), + consumed=dict(pages=0, attempts=0, records=0, duplicates=0, bytes=0, storage_bytes=0, active_seconds=0), + cursor=None, + status="unknown", + reasons=[], + refreshed=False, + reservation=False, + retry=0, + ) checkpoint.save(state) else: try: if any(state[key] != value for key, value in binding.items()): - raise EvidenceError('resume_mismatch') - previous = Limits(**state['limits']) + raise EvidenceError("resume_mismatch") + previous = Limits(**state["limits"]) if any(value > previous.export()[key] for key, value in limits.export().items()): - raise EvidenceError('resume_limits') + raise EvidenceError("resume_limits") _receipt(state, now()) - counters = dict(pages='pages', attempts='attempts', records='records', bytes='total_bytes', - storage_bytes='storage_bytes', active_seconds='active_seconds') - if any(state['consumed'][key] > limits.export()[bound] for key, bound in counters.items()): - raise EvidenceError('resume_limits') + counters = dict( + pages="pages", + attempts="attempts", + records="records", + bytes="total_bytes", + storage_bytes="storage_bytes", + active_seconds="active_seconds", + ) + if any(state["consumed"][key] > limits.export()[bound] for key, bound in counters.items()): + raise EvidenceError("resume_limits") records = checkpoint.export() - if len(records) != state['consumed']['records'] or sum(len(canonical(record, max_bytes=previous.record_bytes, max_depth=previous.depth)) for record in records) != state['consumed']['storage_bytes']: - raise EvidenceError('checkpoint_invalid') - if type(state['retry']) is not int or not 0 <= state['retry'] <= previous.retries or type(state['refreshed']) is not bool or type(state['reservation']) is not bool: - raise EvidenceError('checkpoint_invalid') - if state['retry'] > limits.retries: - raise EvidenceError('resume_limits') + if ( + len(records) != state["consumed"]["records"] + or sum( + len(canonical(record, max_bytes=previous.record_bytes, max_depth=previous.depth)) + for record in records + ) + != state["consumed"]["storage_bytes"] + ): + raise EvidenceError("checkpoint_invalid") + if ( + type(state["retry"]) is not int + or not 0 <= state["retry"] <= previous.retries + or type(state["refreshed"]) is not bool + or type(state["reservation"]) is not bool + ): + raise EvidenceError("checkpoint_invalid") + if state["retry"] > limits.retries: + raise EvidenceError("resume_limits") for record in records: validate_envelope(record, max_bytes=previous.record_bytes, max_depth=previous.depth) - if record['acquisition_id'] != state['acquisition_id'] or record['source']['tenant'] != adapter.tenant or record['source']['scope'] != list(adapter.scope) or record['request_fingerprint'] != fingerprint: - raise EvidenceError('checkpoint_invalid') + if ( + record["acquisition_id"] != state["acquisition_id"] + or record["source"]["tenant"] != adapter.tenant + or record["source"]["scope"] != list(adapter.scope) + or record["request_fingerprint"] != fingerprint + ): + raise EvidenceError("checkpoint_invalid") try: canonical(record, max_bytes=limits.record_bytes, max_depth=limits.depth) except EvidenceError: - raise EvidenceError('resume_limits') from None - if state['cursor'] is not None: - adapter.validate_request(adapter.request(state['cursor'])) + raise EvidenceError("resume_limits") from None + if state["cursor"] is not None: + adapter.validate_request(adapter.request(state["cursor"])) except EvidenceError: raise except Exception: - raise EvidenceError('checkpoint_invalid') from None - state['generation'] += 1 - state['limits'] = limits.export() + raise EvidenceError("checkpoint_invalid") from None + state["generation"] += 1 + state["limits"] = limits.export() checkpoint.save(state) - if state['status'] in ('complete', 'partial'): + if state["status"] in ("complete", "partial"): return Result(checkpoint.export(), _receipt(state, now())) if transport is None: from .transport import HttpsTransport + transport = HttpsTransport(adapter.origin, adapter.path, adapter.method) - spent = state['consumed'] + spent = state["consumed"] def finish(reason=None): - state['status'] = 'partial' if reason else 'complete' - state['reasons'] = [reason] if reason else [] - state['finished_at'] = _finish_time(state, now()) + state["status"] = "partial" if reason else "complete" + state["reasons"] = [reason] if reason else [] + state["finished_at"] = _finish_time(state, now()) checkpoint.save(state) return Result(checkpoint.export(), _receipt(state, now())) while True: - for counter, maximum, reason in [('pages', limits.pages, 'page_limit'), - ('attempts', limits.attempts, 'attempt_limit'), ('records', limits.records, 'record_limit'), - ('bytes', limits.total_bytes, 'byte_limit'), ('storage_bytes', limits.storage_bytes, 'storage_limit'), - ('active_seconds', limits.active_seconds, 'time_limit')]: + for counter, maximum, reason in [ + ("pages", limits.pages, "page_limit"), + ("attempts", limits.attempts, "attempt_limit"), + ("records", limits.records, "record_limit"), + ("bytes", limits.total_bytes, "byte_limit"), + ("storage_bytes", limits.storage_bytes, "storage_limit"), + ("active_seconds", limits.active_seconds, "time_limit"), + ]: if spent[counter] >= maximum: return finish(reason) try: - request = adapter.request(state['cursor']) + request = adapter.request(state["cursor"]) adapter.validate_request(request) except Exception: - return finish('unsafe_destination') - retry, refresh = state['retry'], False + return finish("unsafe_destination") + retry, refresh = state["retry"], False while True: - if spent['attempts'] >= limits.attempts: return finish('attempt_limit') - byte_cap = min(limits.response_bytes, limits.total_bytes - spent['bytes']) - timeout = min(limits.request_seconds, limits.active_seconds - spent['active_seconds']) - if byte_cap <= 0: return finish('byte_limit') - if timeout <= 0: return finish('time_limit') + if spent["attempts"] >= limits.attempts: + return finish("attempt_limit") + byte_cap = min(limits.response_bytes, limits.total_bytes - spent["bytes"]) + timeout = min(limits.request_seconds, limits.active_seconds - spent["active_seconds"]) + if byte_cap <= 0: + return finish("byte_limit") + if timeout <= 0: + return finish("time_limit") # Validate destinations before every ephemeral credential lookup. try: adapter.validate_request(request) headers = credentials.headers(refresh=refresh) - if not isinstance(headers, dict) or set(headers) != {'Authorization'} or not isinstance(headers['Authorization'], str) or not headers['Authorization'].startswith('Bearer ') or any(c in headers['Authorization'] for c in '\r\n'): + if ( + not isinstance(headers, dict) + or set(headers) != {"Authorization"} + or not isinstance(headers["Authorization"], str) + or not headers["Authorization"].startswith("Bearer ") + or any(c in headers["Authorization"] for c in "\r\n") + ): raise ValueError except Exception: - return finish('authentication') + return finish("authentication") refresh = False - spent['attempts'] += 1 - spent['bytes'] += byte_cap - spent['active_seconds'] += timeout - state['reservation'] = True + spent["attempts"] += 1 + spent["bytes"] += byte_cap + spent["active_seconds"] += timeout + state["reservation"] = True checkpoint.save(state) - fault('after_reserve') + fault("after_reserve") began = clock() try: response = transport.send(request, headers, timeout=timeout, max_bytes=byte_cap) - if not isinstance(response, Response) or type(response.body) is not bytes or type(response.status) is not int: - raise EvidenceError('transport') + if ( + not isinstance(response, Response) + or type(response.body) is not bytes + or type(response.status) is not int + ): + raise EvidenceError("transport") if len(response.body) > byte_cap: - raise EvidenceError('response_limit') + raise EvidenceError("response_limit") except Exception as error: # Keep reserved bytes when actual transport consumption is unavailable. elapsed = max(0, clock() - began) - spent['active_seconds'] -= timeout - min(timeout, elapsed) - state['reservation'] = False + spent["active_seconds"] -= timeout - min(timeout, elapsed) + state["reservation"] = False checkpoint.save(state) - code = error.code if isinstance(error, EvidenceError) and error.code in REASONS else 'transport' + code = error.code if isinstance(error, EvidenceError) and error.code in REASONS else "transport" return finish(code) finally: # Credential values never enter the checkpoint, plan, receipt or result. headers = None elapsed = max(0, clock() - began) - spent['bytes'] -= byte_cap - len(response.body) + spent["bytes"] -= byte_cap - len(response.body) # Keep the active-time reservation until the page transaction commits. # An interruption during projection must not reset its processing cost. if response.status != 200 or elapsed >= timeout: - spent['active_seconds'] -= timeout - min(timeout, elapsed) - state['reservation'] = False + spent["active_seconds"] -= timeout - min(timeout, elapsed) + state["reservation"] = False checkpoint.save(state) - if elapsed >= timeout: return finish('request_timeout') - if response.status == 200: break - if response.status in (400, 404, 410) and state['cursor'] is not None: - return finish('cursor_expired') + if elapsed >= timeout: + return finish("request_timeout") + if response.status == 200: + break + if response.status in (400, 404, 410) and state["cursor"] is not None: + return finish("cursor_expired") if response.status == 401: - if state['refreshed']: return finish('authentication') + if state["refreshed"]: + return finish("authentication") # Persist before refresh so interruptions cannot grant more refreshes. - state['refreshed'] = True + state["refreshed"] = True checkpoint.save(state) refresh = True elif response.status not in (429, 500, 502, 503, 504): - return finish('transport') + return finish("transport") if retry >= limits.retries: - return finish('throttled' if response.status == 429 else 'authentication' if response.status == 401 else 'transport') + return finish( + "throttled" + if response.status == 429 + else "authentication" + if response.status == 401 + else "transport" + ) try: delay = 0 if response.status == 401 else _delay(response.retry_after, retry, now()) except EvidenceError as error: return finish(error.code) - if delay >= limits.active_seconds - spent['active_seconds']: - return finish('time_limit') + if delay >= limits.active_seconds - spent["active_seconds"]: + return finish("time_limit") # Reserve sleeps too: a crash must not reset backoff/time accounting. retry += 1 - state['retry'] = retry - spent['active_seconds'] += delay + state["retry"] = retry + spent["active_seconds"] += delay checkpoint.save(state) began_sleep = clock() try: sleep(delay) except Exception: - return finish('transport') + return finish("transport") oversleep = max(0, clock() - began_sleep - delay) - spent['active_seconds'] = min(limits.active_seconds, spent['active_seconds'] + oversleep) + spent["active_seconds"] = min(limits.active_seconds, spent["active_seconds"] + oversleep) checkpoint.save(state) parsed_at = clock() - def check_page_time(): + def check_page_time(elapsed=elapsed, parsed_at=parsed_at, timeout=timeout): actual = elapsed + max(0, clock() - parsed_at) - if spent['active_seconds'] - timeout + actual >= limits.active_seconds: - raise EvidenceError('time_limit') + if spent["active_seconds"] - timeout + actual >= limits.active_seconds: + raise EvidenceError("time_limit") - def settle_page_time(): + def settle_page_time(elapsed=elapsed, parsed_at=parsed_at, timeout=timeout): actual = elapsed + max(0, clock() - parsed_at) - spent['active_seconds'] = min(limits.active_seconds, - spent['active_seconds'] - timeout + actual) - state['reservation'] = False + spent["active_seconds"] = min(limits.active_seconds, spent["active_seconds"] - timeout + actual) + state["reservation"] = False try: check_page_time() @@ -247,56 +367,70 @@ def settle_page_time(): except StopIteration: break check_page_time() - record = build_envelope(acquisition_id=state['acquisition_id'], product=adapter.product, - api=adapter.api, tenant=adapter.tenant, scope=list(adapter.scope), - acquired_at=stamp, transformed_at=stamp, request_fingerprint=fingerprint, - page=spent['pages'] + 1, max_bytes=limits.record_bytes, max_depth=limits.depth, **projected) + record = build_envelope( + acquisition_id=state["acquisition_id"], + product=adapter.product, + api=adapter.api, + tenant=adapter.tenant, + scope=list(adapter.scope), + acquired_at=stamp, + transformed_at=stamp, + request_fingerprint=fingerprint, + page=spent["pages"] + 1, + max_bytes=limits.record_bytes, + max_depth=limits.depth, + **projected, + ) check_page_time() - duplicate = checkpoint.contains(record['record_id']) or record['record_id'] in seen + duplicate = checkpoint.contains(record["record_id"]) or record["record_id"] in seen check_page_time() if duplicate: duplicates += 1 continue size = len(canonical(record, max_bytes=limits.record_bytes, max_depth=limits.depth)) check_page_time() - if spent['records'] + len(pending) >= limits.records: - reason = 'record_limit'; break - if spent['storage_bytes'] + storage + size > limits.storage_bytes: - reason = 'storage_limit'; break - pending.append(record); seen.add(record['record_id']); storage += size + if spent["records"] + len(pending) >= limits.records: + reason = "record_limit" + break + if spent["storage_bytes"] + storage + size > limits.storage_bytes: + reason = "storage_limit" + break + pending.append(record) + seen.add(record["record_id"]) + storage += size check_page_time() cursor_hash = digest(page.cursor) if page.cursor is not None else None check_page_time() if cursor_hash and checkpoint.seen_cursor(cursor_hash): - reason, cursor_hash = 'cursor_cycle', None + reason, cursor_hash = "cursor_cycle", None check_page_time() except EvidenceError as error: - if error.code != 'time_limit': + if error.code != "time_limit": raise # Commit only the prefix accepted before this budget expired. - reason, cursor_hash = 'time_limit', None + reason, cursor_hash = "time_limit", None except EvidenceError as error: settle_page_time() - code = error.code if error.code in REASONS else 'projection_failed' + code = error.code if error.code in REASONS else "projection_failed" return finish(code) except Exception: settle_page_time() - return finish('schema_drift') + return finish("schema_drift") settle_page_time() - if spent['active_seconds'] >= limits.active_seconds: - reason = 'time_limit' - spent['pages'] += 1 - spent['records'] += len(pending) - spent['duplicates'] += duplicates - spent['storage_bytes'] += storage - state['cursor'] = page.cursor - state['retry'] = 0 - state['status'] = 'partial' if reason else 'complete' if page.cursor is None else 'unknown' - state['reasons'] = [reason] if reason else [] - if state['status'] != 'unknown': - state['finished_at'] = _finish_time(state, now()) - fault('before_commit') + if spent["active_seconds"] >= limits.active_seconds: + reason = "time_limit" + spent["pages"] += 1 + spent["records"] += len(pending) + spent["duplicates"] += duplicates + spent["storage_bytes"] += storage + state["cursor"] = page.cursor + state["retry"] = 0 + state["status"] = "partial" if reason else "complete" if page.cursor is None else "unknown" + state["reasons"] = [reason] if reason else [] + if state["status"] != "unknown": + state["finished_at"] = _finish_time(state, now()) + fault("before_commit") checkpoint.save(state, pending, cursor_hash) - fault('after_commit') - if state['status'] != 'unknown': + fault("after_commit") + if state["status"] != "unknown": return Result(checkpoint.export(), _receipt(state, now())) diff --git a/cops/connectors/transport.py b/cops/connectors/transport.py index 07cb7bd..ea59ae3 100644 --- a/cops/connectors/transport.py +++ b/cops/connectors/transport.py @@ -1,11 +1,14 @@ """TLS-only fixed-route HTTP transport, without redirects or decompression.""" + import http.client import math import socket import ssl import threading import time + from cops.evidence import EvidenceError + from .adapters.microsoft import destination from .interfaces import Response @@ -17,42 +20,48 @@ def __init__(self, origin, path, method): def send(self, request, headers, *, timeout, max_bytes): parts = destination(request.url, self.origin, self.path) if request.method != self.method: - raise EvidenceError('unsafe_destination') - if type(timeout) not in (int, float) or not math.isfinite(timeout) or not 0 < timeout <= 120 or type(max_bytes) is not int or not 0 < max_bytes <= 16777216: - raise EvidenceError('invalid_limit') + raise EvidenceError("unsafe_destination") + if ( + type(timeout) not in (int, float) + or not math.isfinite(timeout) + or not 0 < timeout <= 120 + or type(max_bytes) is not int + or not 0 < max_bytes <= 16777216 + ): + raise EvidenceError("invalid_limit") deadline = time.monotonic() + timeout done, cancelled = threading.Event(), threading.Event() outcome, active = {}, {} def cancel(): cancelled.set() - current_socket = active.get('socket') + current_socket = active.get("socket") if current_socket is not None: try: current_socket.shutdown(socket.SHUT_RDWR) except OSError: pass - connection = active.get('connection') + connection = active.get("connection") if connection is not None: connection.close() def remaining(): value = deadline - time.monotonic() if cancelled.is_set() or value <= 0: - raise EvidenceError('request_timeout') + raise EvidenceError("request_timeout") return value def exchange(): try: - outcome['response'] = self._exchange(request, headers, parts, max_bytes, remaining, active) + outcome["response"] = self._exchange(request, headers, parts, max_bytes, remaining, active) except EvidenceError as error: - outcome['error'] = error.code + outcome["error"] = error.code except TimeoutError: - outcome['error'] = 'request_timeout' + outcome["error"] = "request_timeout" except Exception: - outcome['error'] = 'transport' + outcome["error"] = "transport" finally: - connection = active.get('connection') + connection = active.get("connection") if connection is not None: connection.close() done.set() @@ -60,27 +69,27 @@ def exchange(): # A socket timeout alone does not bound DNS resolution or slow headers. # Cancellation prevents sending credentials after a delayed connect returns. # The daemon can outlive the caller while an OS resolver remains blocked. - threading.Thread(target=exchange, daemon=True, name='cops-evidence-http').start() + threading.Thread(target=exchange, daemon=True, name="cops-evidence-http").start() if not done.wait(max(0, deadline - time.monotonic())): cancel() - raise EvidenceError('request_timeout') - if 'error' in outcome: - raise EvidenceError(outcome['error']) - return outcome['response'] + raise EvidenceError("request_timeout") + if "error" in outcome: + raise EvidenceError(outcome["error"]) + return outcome["response"] def _exchange(self, request, headers, parts, max_bytes, remaining, active): connection = http.client.HTTPSConnection(self.origin, timeout=remaining(), context=ssl.create_default_context()) # Closing during cancellation must not let request() reopen the socket. connection.auto_open = 0 - active['connection'] = connection + active["connection"] = connection try: connection.connect() connection.sock.settimeout(remaining()) - active['socket'] = connection.sock + active["socket"] = connection.sock outgoing = dict(request.headers) outgoing.update(headers) - outgoing['Accept-Encoding'] = 'identity' - target = parts.path + ('?' + parts.query if parts.query else '') + outgoing["Accept-Encoding"] = "identity" + target = parts.path + ("?" + parts.query if parts.query else "") remaining() connection.request(request.method, target, body=request.body, headers=outgoing) if connection.sock is not None: @@ -89,24 +98,26 @@ def _exchange(self, request, headers, parts, max_bytes, remaining, active): remaining() if response.status != 200: # Provider error bodies are unnecessary for retry decisions. - return Response(response.status, b'', response.getheader('Retry-After')) - if response.getheader('Content-Encoding', 'identity').lower() != 'identity': - raise EvidenceError('response_limit') - length = response.getheader('Content-Length') + return Response(response.status, b"", response.getheader("Retry-After")) + if response.getheader("Content-Encoding", "identity").lower() != "identity": + raise EvidenceError("response_limit") + length = response.getheader("Content-Length") if length is not None and (not length.isdecimal() or int(length) > max_bytes): - raise EvidenceError('response_limit') + raise EvidenceError("response_limit") body = bytearray() while response.fp is not None: # The response can retain its socket after Connection: close. - active['socket'] = response.fp.raw._sock - active['socket'].settimeout(remaining()) + active["socket"] = response.fp.raw._sock + active["socket"].settimeout(remaining()) chunk = response.read1(min(65536, max_bytes - len(body) + 1)) - if not chunk: break + if not chunk: + break body.extend(chunk) - if len(body) > max_bytes: raise EvidenceError('response_limit') + if len(body) > max_bytes: + raise EvidenceError("response_limit") if length is not None and len(body) != int(length): - raise EvidenceError('transport') - retry_after = response.getheader('Retry-After') + raise EvidenceError("transport") + retry_after = response.getheader("Retry-After") return Response(response.status, bytes(body), retry_after) finally: connection.close() diff --git a/cops/contracts/__init__.py b/cops/contracts/__init__.py index c081fc5..844cc6d 100644 --- a/cops/contracts/__init__.py +++ b/cops/contracts/__init__.py @@ -13,11 +13,11 @@ ENGAGEMENT_TRANSITIONS, EXECUTION_AUTHORIZATION_STATES, EXECUTION_AUTHORIZATION_TRANSITIONS, + LABORATORY_ENVIRONMENT_STATES, + LABORATORY_ENVIRONMENT_TRANSITIONS, RUN_RESULT_STATUSES, SPECIALIST_HANDOFF_STATES, SPECIALIST_HANDOFF_TRANSITIONS, - LABORATORY_ENVIRONMENT_STATES, - LABORATORY_ENVIRONMENT_TRANSITIONS, ContractError, validate_transition, ) diff --git a/cops/coverage.py b/cops/coverage.py index 914fc2c..a6f1701 100644 --- a/cops/coverage.py +++ b/cops/coverage.py @@ -2,15 +2,14 @@ from __future__ import annotations -import copy -import hashlib import json import re import uuid -from dataclasses import asdict, dataclass -from datetime import datetime, timezone +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import UTC, datetime from pathlib import Path -from typing import Any, Sequence +from typing import Any ROOT = Path(__file__).resolve().parents[1] TECHNIQUE_ID_PATTERN = re.compile(r"^T\d{4}(\.\d{3})?$") @@ -396,7 +395,7 @@ def load_attack_coverage( validation_fixture=str(item["validation_fixture"]) if item.get("validation_fixture") else None, time_window=str(item["time_window"]) if item.get("time_window") else None, assumptions=tuple(str(a) for a in item.get("assumptions", [])), - known_limitations=tuple(str(l) for l in item["known_limitations"]), + known_limitations=tuple(str(limitation) for limitation in item["known_limitations"]), last_reviewed=str(item["last_reviewed"]), attck_version=str(item["attck_version"]), ) @@ -740,7 +739,7 @@ def generate_attack_flow( - 'azure-identity': Azure/Entra ID Identity Compromise to Role Assignment & Storage Exfiltration - 'm365-compromise': Microsoft 365 Phishing to Mailbox Forwarding & SharePoint Harvesting """ - now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + now_iso = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") if scenario_name == "azure-identity": flow_id = deterministic_uuid("attack-flow-azure-identity") diff --git a/cops/diagnostics/cli.py b/cops/diagnostics/cli.py index 9fa6463..8b36c82 100644 --- a/cops/diagnostics/cli.py +++ b/cops/diagnostics/cli.py @@ -6,6 +6,7 @@ import json import sys from pathlib import Path +from typing import Any from .runner import run_diagnostics diff --git a/cops/diagnostics/packages.py b/cops/diagnostics/packages.py index b63f55f..bc65246 100644 --- a/cops/diagnostics/packages.py +++ b/cops/diagnostics/packages.py @@ -4,7 +4,6 @@ import json from pathlib import Path -from typing import Any from .models import DiagnosticCheck, PackageDiagnostic diff --git a/cops/diagnostics/runner.py b/cops/diagnostics/runner.py index ef63e30..3bff8fc 100644 --- a/cops/diagnostics/runner.py +++ b/cops/diagnostics/runner.py @@ -8,7 +8,7 @@ from cops.capabilities.auditor import audit_capabilities from cops.evidence.canonical import utc_now -from .models import DiagnosticReport, PackageDiagnostic, SystemDiagnostic, ToolDiagnostic +from .models import DiagnosticReport, PackageDiagnostic from .packages import diagnose_packages, diagnose_plugin_package from .system import detect_system_platform, diagnose_host_tools diff --git a/cops/diagnostics/system.py b/cops/diagnostics/system.py index ded1055..3027ace 100644 --- a/cops/diagnostics/system.py +++ b/cops/diagnostics/system.py @@ -9,7 +9,7 @@ import sys from pathlib import Path -from cops.laboratory.matrix import TestedMatrix, parse_version_tuple, version_ge +from cops.laboratory.matrix import TestedMatrix, version_ge from .models import DiagnosticCheck, SystemDiagnostic, ToolDiagnostic @@ -74,7 +74,7 @@ def detect_system_platform(matrix: TestedMatrix | None = None) -> SystemDiagnost is_supported = False # Python version check (>= 3.11) - if sys.version_info >= (3, 11): + if sys.version_info >= (3, 11): # noqa: UP036 - diagnose unsupported Python runtimes checks.append(DiagnosticCheck("python_runtime", "passed", f"Python {py_ver} meets >= 3.11 requirement")) else: checks.append(DiagnosticCheck("python_runtime", "failed", f"Python {py_ver} is older than required 3.11")) diff --git a/cops/discovery/__init__.py b/cops/discovery/__init__.py index 2b7d9c8..0ff1562 100644 --- a/cops/discovery/__init__.py +++ b/cops/discovery/__init__.py @@ -27,6 +27,41 @@ compare_active_scans, ) from .cli import build_discovery_parser, command_discovery +from .data_collector import ( + DEFAULT_DATA_PORTS, + DataServicesCollector, + OfflineSyntheticDataCollector, + StandardSocketDataCollector, + assess_data_services, +) +from .data_models import ( + DataAuthPrerequisite, + DataExposureStatus, + DataPrivilegeCandidate, + DataPrivilegeImpact, + DataServiceAssessment, + DataServiceCategory, + DataServicesReport, + DataServiceType, +) +from .developer_collector import ( + DEFAULT_DEVELOPER_PORTS, + DeveloperServicesCollector, + OfflineSyntheticDeveloperCollector, + StandardSocketDeveloperCollector, + assess_developer_services, +) +from .developer_models import ( + DeveloperAuthPrerequisite, + DeveloperCategory, + DeveloperExposureStatus, + DeveloperPrivilegeCandidate, + DeveloperPrivilegeImpact, + DeveloperServiceAssessment, + DeveloperServicesReport, + DeveloperServiceType, + ExecutionEffect, +) from .fingerprinter import infer_service_fingerprint from .importer import import_masscan_json, import_nmap_xml from .infra_collector import ( @@ -45,7 +80,41 @@ InfraServiceType, ServiceExposureStatus, ) +from .legacy_collector import ( + DEFAULT_LEGACY_PORTS, + LegacyServicesCollector, + OfflineSyntheticLegacyCollector, + StandardSocketLegacyCollector, + assess_legacy_services, +) +from .legacy_models import ( + LegacyAuthPrerequisite, + LegacyCategory, + LegacyExposureStatus, + LegacyPrivilegeCandidate, + LegacyPrivilegeImpact, + LegacyServiceAssessment, + LegacyServicesReport, + LegacyServiceType, +) from .merger import merge_inventories, merge_two_assets +from .messaging_collector import ( + DEFAULT_MESSAGING_PORTS, + MessagingServicesCollector, + OfflineSyntheticMessagingCollector, + StandardSocketMessagingCollector, + assess_messaging_services, +) +from .messaging_models import ( + MessagingAuthPrerequisite, + MessagingCategory, + MessagingExposureStatus, + MessagingPrivilegeCandidate, + MessagingPrivilegeImpact, + MessagingServiceAssessment, + MessagingServicesReport, + MessagingServiceType, +) from .models import ( AssetType, DiscoveredAsset, @@ -62,23 +131,6 @@ normalize_ip_record, ) from .reconciler import reconcile_asset, reconcile_inventory -from .data_collector import ( - DEFAULT_DATA_PORTS, - DataServicesCollector, - OfflineSyntheticDataCollector, - StandardSocketDataCollector, - assess_data_services, -) -from .data_models import ( - DataAuthPrerequisite, - DataExposureStatus, - DataPrivilegeCandidate, - DataPrivilegeImpact, - DataServiceAssessment, - DataServiceCategory, - DataServicesReport, - DataServiceType, -) from .remote_collector import ( DEFAULT_REMOTE_PORTS, OfflineSyntheticRemoteCollector, @@ -98,62 +150,6 @@ RemoteServiceType, ) -from .messaging_collector import ( - DEFAULT_MESSAGING_PORTS, - MessagingServicesCollector, - OfflineSyntheticMessagingCollector, - StandardSocketMessagingCollector, - assess_messaging_services, -) -from .messaging_models import ( - MessagingAuthPrerequisite, - MessagingCategory, - MessagingExposureStatus, - MessagingPrivilegeCandidate, - MessagingPrivilegeImpact, - MessagingServiceAssessment, - MessagingServicesReport, - MessagingServiceType, -) - -from .developer_collector import ( - DEFAULT_DEVELOPER_PORTS, - DeveloperServicesCollector, - OfflineSyntheticDeveloperCollector, - StandardSocketDeveloperCollector, - assess_developer_services, -) -from .developer_models import ( - DeveloperAuthPrerequisite, - DeveloperCategory, - DeveloperExposureStatus, - DeveloperPrivilegeCandidate, - DeveloperPrivilegeImpact, - DeveloperServiceAssessment, - DeveloperServicesReport, - DeveloperServiceType, - ExecutionEffect, -) - -from .legacy_collector import ( - DEFAULT_LEGACY_PORTS, - LegacyServicesCollector, - OfflineSyntheticLegacyCollector, - StandardSocketLegacyCollector, - assess_legacy_services, -) -from .legacy_models import ( - LegacyAuthPrerequisite, - LegacyCategory, - LegacyExposureStatus, - LegacyPrivilegeCandidate, - LegacyPrivilegeImpact, - LegacyServiceAssessment, - LegacyServicesReport, - LegacyServiceType, -) - - __all__ = [ "ActiveScanError", "ActiveScanSession", diff --git a/cops/discovery/active_models.py b/cops/discovery/active_models.py index c5af48c..8d6c974 100644 --- a/cops/discovery/active_models.py +++ b/cops/discovery/active_models.py @@ -2,14 +2,14 @@ from __future__ import annotations +import json from dataclasses import asdict, dataclass, field from enum import Enum -import hashlib -import json from pathlib import Path from typing import Any from cops.evidence.canonical import canonical, utc_now + from .models import EvidenceProvenance diff --git a/cops/discovery/active_scanner.py b/cops/discovery/active_scanner.py index 8819df0..ad5c4c2 100644 --- a/cops/discovery/active_scanner.py +++ b/cops/discovery/active_scanner.py @@ -2,28 +2,26 @@ from __future__ import annotations -from abc import ABC, abstractmethod -from dataclasses import dataclass, field import hashlib import ipaddress import re import socket import ssl import time -from typing import Any, Callable +from abc import ABC, abstractmethod +from collections.abc import Callable +from typing import Any from cops.evidence.canonical import utc_now + from .active_models import ( ActiveScanSession, ActiveServiceAssessment, - ConfidenceLevel, ObservedConfiguration, ObservedTLS, PortState, Protocol, - ScanBudget, ScanDelta, - ScanVantage, ServiceReachability, TargetShiftQuarantine, ) @@ -34,6 +32,15 @@ class ActiveScanError(Exception): """Base error for active discovery operations.""" +def _create_tls_context() -> ssl.SSLContext: + """Create a client TLS context that cannot negotiate deprecated protocol versions.""" + context = ssl.create_default_context() + context.minimum_version = ssl.TLSVersion.TLSv1_2 + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + return context + + class ScopeViolationError(ActiveScanError): """Raised when active probe target is not permitted within approved boundaries.""" @@ -102,7 +109,7 @@ def dispatch_probe( self.side_effect_count += 1 probe_key = ActiveScanSession.make_probe_key(vantage, protocol, target_host, port) timestamp = utc_now() - probe_id = hashlib.sha256(f"{probe_key}:{timestamp}".encode("utf-8")).hexdigest()[:24] + probe_id = hashlib.sha256(f"{probe_key}:{timestamp}".encode()).hexdigest()[:24] self.dispatched_probes.append({ "probe_key": probe_key, @@ -223,7 +230,7 @@ def dispatch_probe( ) -> ActiveServiceAssessment: probe_key = ActiveScanSession.make_probe_key(vantage, protocol, target_host, port) timestamp = utc_now() - probe_id = hashlib.sha256(f"{probe_key}:{timestamp}".encode("utf-8")).hexdigest()[:24] + probe_id = hashlib.sha256(f"{probe_key}:{timestamp}".encode()).hexdigest()[:24] start_time = time.monotonic() sock = None @@ -239,20 +246,19 @@ def dispatch_probe( # If TLS port (443, 8443) or requested if port in (443, 8443): - ctx = ssl.create_default_context() - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE + ctx = _create_tls_context() try: with ctx.wrap_socket(sock, server_hostname=target_host) as ssock: cipher = ssock.cipher() proto_ver = ssock.version() - cert = ssock.getpeercert(binary_form=True) + ssock.getpeercert(binary_form=True) observed_tls = ObservedTLS( version=proto_ver, cipher_suite=cipher[0] if cipher else None, ) - except Exception: + except (OSError, ssl.SSLError): + # Failed TLS negotiation leaves observed_tls unset. pass observed_config = ObservedConfiguration( @@ -279,7 +285,7 @@ def dispatch_probe( latency_ms=latency_ms, timestamp_utc=timestamp, ) - except socket.timeout: + except TimeoutError: return ActiveServiceAssessment( probe_id=probe_id, target_host=target_host, @@ -313,7 +319,8 @@ def dispatch_probe( if sock: try: sock.close() - except Exception: + except OSError: + # Closing a failed probe must not replace its result. pass diff --git a/cops/discovery/cli.py b/cops/discovery/cli.py index ac04573..fe70bc0 100644 --- a/cops/discovery/cli.py +++ b/cops/discovery/cli.py @@ -5,14 +5,13 @@ import argparse import hashlib import json -from pathlib import Path import sys +from pathlib import Path from typing import Any from .active_models import ( ActiveScanSession, ScanBudget, - ScanVantage, ) from .active_scanner import ( ActiveScanner, @@ -20,18 +19,48 @@ StandardSocketDispatcher, compare_active_scans, ) +from .data_collector import ( + OfflineSyntheticDataCollector, + StandardSocketDataCollector, + assess_data_services, +) +from .data_models import ( + DataServicesReport, +) +from .developer_collector import ( + OfflineSyntheticDeveloperCollector, + StandardSocketDeveloperCollector, + assess_developer_services, +) +from .developer_models import ( + DeveloperServicesReport, +) from .importer import import_masscan_json, import_nmap_xml from .infra_collector import ( - DEFAULT_INFRA_PORTS, OfflineSyntheticInfraCollector, StandardSocketInfraCollector, assess_infrastructure_services, ) from .infra_models import ( InfraAssessmentReport, - ServiceExposureStatus, +) +from .legacy_collector import ( + OfflineSyntheticLegacyCollector, + StandardSocketLegacyCollector, + assess_legacy_services, +) +from .legacy_models import ( + LegacyServicesReport, ) from .merger import merge_inventories +from .messaging_collector import ( + OfflineSyntheticMessagingCollector, + StandardSocketMessagingCollector, + assess_messaging_services, +) +from .messaging_models import ( + MessagingServicesReport, +) from .models import DiscoveredAsset, DiscoveryInventory, EvidenceProvenance from .normalizers import ( normalize_certificate_record, @@ -42,59 +71,12 @@ ) from .reconciler import reconcile_inventory from .remote_collector import ( - DEFAULT_REMOTE_PORTS, OfflineSyntheticRemoteCollector, StandardSocketRemoteCollector, assess_remote_services, ) from .remote_models import ( RemoteServicesReport, - RemoteExposureStatus, - RemoteServiceCategory, -) -from .data_collector import ( - DEFAULT_DATA_PORTS, - OfflineSyntheticDataCollector, - StandardSocketDataCollector, - assess_data_services, -) -from .data_models import ( - DataServicesReport, - DataExposureStatus, - DataServiceCategory, -) -from .messaging_collector import ( - DEFAULT_MESSAGING_PORTS, - OfflineSyntheticMessagingCollector, - StandardSocketMessagingCollector, - assess_messaging_services, -) -from .messaging_models import ( - MessagingServicesReport, - MessagingExposureStatus, - MessagingCategory, -) -from .developer_collector import ( - DEFAULT_DEVELOPER_PORTS, - OfflineSyntheticDeveloperCollector, - StandardSocketDeveloperCollector, - assess_developer_services, -) -from .developer_models import ( - DeveloperServicesReport, - DeveloperExposureStatus, - DeveloperCategory, -) -from .legacy_collector import ( - DEFAULT_LEGACY_PORTS, - OfflineSyntheticLegacyCollector, - StandardSocketLegacyCollector, - assess_legacy_services, -) -from .legacy_models import ( - LegacyServicesReport, - LegacyExposureStatus, - LegacyCategory, ) ROOT: Path = Path(__file__).resolve().parents[2] @@ -423,7 +405,7 @@ def command_active_discovery(args: argparse.Namespace, root: Path | None = None) max_total_seconds=args.max_total_seconds, rate_limit_pps=args.rate_limit, ) - h = hashlib.sha256(f"{args.scope_ref}:{sorted(targets)}:{sorted(ports)}".encode("utf-8")).hexdigest()[:16] + h = hashlib.sha256(f"{args.scope_ref}:{sorted(targets)}:{sorted(ports)}".encode()).hexdigest()[:16] session = ActiveScanSession( session_id=f"active-{h}", scope_reference=args.scope_ref, diff --git a/cops/discovery/data_collector.py b/cops/discovery/data_collector.py index d8dd987..6483901 100644 --- a/cops/discovery/data_collector.py +++ b/cops/discovery/data_collector.py @@ -2,15 +2,12 @@ from __future__ import annotations -from abc import ABC, abstractmethod import hashlib import ipaddress -import json -from pathlib import Path import socket +from abc import ABC, abstractmethod from typing import Any -from cops.evidence.canonical import canonical, utc_now from .data_models import ( CleanupReceipt, DataAuthPrerequisite, @@ -23,7 +20,6 @@ DataServiceType, ) - DEFAULT_DATA_PORTS: dict[str, tuple[int, str, str]] = { # Relational Databases DataServiceType.MYSQL.value: (3306, "tcp", DataServiceCategory.RELATIONAL_DB.value), @@ -266,7 +262,7 @@ def probe_service( receipts: list[CleanupReceipt] = [] if canary_active and canary_artifact: - receipt_id = hashlib.sha256(f"clean:{target_host}:{service_type}:{canary_artifact}".encode("utf-8")).hexdigest()[:16] + receipt_id = hashlib.sha256(f"clean:{target_host}:{service_type}:{canary_artifact}".encode()).hexdigest()[:16] receipts.append( CleanupReceipt( receipt_id=f"rec-{receipt_id}", @@ -318,7 +314,7 @@ def probe_service( if service_type == DataServiceType.REDIS.value: if not auth_req or details.get("requirepass") is False: vulns.append("Redis instance accepts unauthenticated TCP commands") - c_id = hashlib.sha256(f"redis:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"redis:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DataPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -337,7 +333,7 @@ def probe_service( ) if details.get("config_set_enabled", True) and not auth_req: vulns.append("Redis CONFIG command accessible; potential arbitrary file write / code execution") - c_id = hashlib.sha256(f"redis_config:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"redis_config:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DataPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -358,7 +354,7 @@ def probe_service( elif service_type == DataServiceType.ELASTICSEARCH.value: if not auth_req or details.get("security_enabled") is False: vulns.append("Elasticsearch REST endpoint open without authentication") - c_id = hashlib.sha256(f"es:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"es:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DataPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -379,7 +375,7 @@ def probe_service( elif service_type == DataServiceType.MONGODB.value: if not auth_req or details.get("auth_enabled") is False: vulns.append("MongoDB instance running without authentication (--auth)") - c_id = hashlib.sha256(f"mongo:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"mongo:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DataPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -400,7 +396,7 @@ def probe_service( elif service_type == DataServiceType.MEMCACHED.value: if not auth_req or details.get("sasl_enabled") is False: vulns.append("Memcached daemon accepts unauthenticated slab dump requests") - c_id = hashlib.sha256(f"memcached:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"memcached:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DataPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -421,7 +417,7 @@ def probe_service( elif service_type == DataServiceType.MYSQL.value: if details.get("password_required") is False or not auth_req: vulns.append("MySQL server accessible with blank/unauthenticated root credentials") - c_id = hashlib.sha256(f"mysql:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"mysql:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DataPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -442,7 +438,7 @@ def probe_service( elif service_type == DataServiceType.POSTGRES.value: if details.get("auth_method") == "trust" or not auth_req: vulns.append("PostgreSQL pg_hba.conf configured with 'trust' authentication") - c_id = hashlib.sha256(f"postgres:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"postgres:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DataPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -463,7 +459,7 @@ def probe_service( elif service_type == DataServiceType.MSSQL.value: if details.get("blank_sa") is True or details.get("blank_sa_password") is True or not auth_req: vulns.append("Microsoft SQL Server configured with blank sa password or xp_cmdshell enabled") - c_id = hashlib.sha256(f"mssql:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"mssql:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DataPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -484,7 +480,7 @@ def probe_service( elif service_type == DataServiceType.ORACLE.value: if details.get("default_credentials") is True or not auth_req: vulns.append("Oracle database accessible with default administrative credentials (SYS/SYSTEM)") - c_id = hashlib.sha256(f"oracle:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"oracle:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DataPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -505,7 +501,7 @@ def probe_service( elif service_type == DataServiceType.COUCHDB.value: if details.get("admin_party") is True or not auth_req: vulns.append("CouchDB running in unauthenticated Admin Party mode") - c_id = hashlib.sha256(f"couchdb:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"couchdb:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DataPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -526,7 +522,7 @@ def probe_service( elif service_type == DataServiceType.CASSANDRA.value: if details.get("default_creds") is True or not auth_req: vulns.append("Cassandra cluster accessible using default superuser credentials (cassandra/cassandra)") - c_id = hashlib.sha256(f"cassandra:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"cassandra:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DataPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -547,7 +543,7 @@ def probe_service( elif service_type == DataServiceType.INFLUXDB.value: if not auth_req or details.get("auth_enabled") is False: vulns.append("InfluxDB HTTP API open without mandatory authentication") - c_id = hashlib.sha256(f"influx:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"influx:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DataPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -568,7 +564,7 @@ def probe_service( elif service_type == DataServiceType.KIBANA.value: if not auth_req or details.get("auth_enabled") is False: vulns.append("Kibana analytics dashboard exposed without user authentication") - c_id = hashlib.sha256(f"kibana:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"kibana:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DataPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -589,7 +585,7 @@ def probe_service( elif service_type == DataServiceType.SPLUNK.value: if details.get("default_creds") is True: vulns.append("Splunk management daemon accessible with default credentials (admin/changeme)") - c_id = hashlib.sha256(f"splunk:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"splunk:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DataPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -669,7 +665,7 @@ def probe_service( configuration_details={"socket_connected": True}, uncertainty_notes=["TCP connect succeeded; protocol credentials required to verify authentication"], ) - except (socket.timeout, ConnectionRefusedError, OSError) as err: + except (TimeoutError, ConnectionRefusedError, OSError) as err: return DataServiceAssessment( target_host=target_host, resolved_ip=resolved_ip, @@ -723,7 +719,7 @@ def assess_data_services( } report_id = hashlib.sha256( - f"{scope_ref}:{sorted(targets)}:{sorted(selected_services)}:{vantage}".encode("utf-8") + f"{scope_ref}:{sorted(targets)}:{sorted(selected_services)}:{vantage}".encode() ).hexdigest()[:16] for target in targets: diff --git a/cops/discovery/data_models.py b/cops/discovery/data_models.py index 106d794..fa21daa 100644 --- a/cops/discovery/data_models.py +++ b/cops/discovery/data_models.py @@ -2,14 +2,15 @@ from __future__ import annotations -from dataclasses import asdict, dataclass, field -from enum import Enum import hashlib import json +from dataclasses import asdict, dataclass, field +from enum import Enum from pathlib import Path from typing import Any from cops.evidence.canonical import canonical, utc_now + from .models import EvidenceProvenance @@ -63,9 +64,9 @@ class DataAuthPrerequisite(str, Enum): NONE = "none" ANONYMOUS = "anonymous" DEFAULT_CREDENTIALS = "default_credentials" - USER_PASSWORD = "user_password" + USER_PASSWORD = "user_password" # noqa: S105 - schema label or operation identifier, not a credential CLIENT_CERT = "client_cert" - TOKEN_OR_API_KEY = "token_or_api_key" + TOKEN_OR_API_KEY = "token_or_api_key" # noqa: S105 - schema label or operation identifier, not a credential KERBEROS = "kerberos" UNKNOWN = "unknown" @@ -103,7 +104,7 @@ class DataPrivilegeCandidate: def __post_init__(self) -> None: if not self.evidence_hash: - canonical_blob = f"{self.service_type}:{self.target_host}:{self.port}:{self.finding_type}:{self.auth_prerequisites}:{self.privilege_impact}".encode("utf-8") + canonical_blob = f"{self.service_type}:{self.target_host}:{self.port}:{self.finding_type}:{self.auth_prerequisites}:{self.privilege_impact}".encode() self.evidence_hash = hashlib.sha256(canonical_blob).hexdigest() if not self.remediation_guidance: self.remediation_guidance = self._default_remediation() @@ -168,7 +169,7 @@ class CleanupReceipt: def __post_init__(self) -> None: if not self.receipt_hash: - canonical_blob = f"{self.receipt_id}:{self.target_host}:{self.service_type}:{self.artifact_identifier}:{self.action_taken}".encode("utf-8") + canonical_blob = f"{self.receipt_id}:{self.target_host}:{self.service_type}:{self.artifact_identifier}:{self.action_taken}".encode() self.receipt_hash = hashlib.sha256(canonical_blob).hexdigest() def to_dict(self) -> dict[str, Any]: diff --git a/cops/discovery/developer_collector.py b/cops/discovery/developer_collector.py index a1faa25..e9a4e0f 100644 --- a/cops/discovery/developer_collector.py +++ b/cops/discovery/developer_collector.py @@ -2,15 +2,12 @@ from __future__ import annotations -from abc import ABC, abstractmethod import hashlib import ipaddress -import json -from pathlib import Path import socket +from abc import ABC, abstractmethod from typing import Any -from cops.evidence.canonical import canonical, utc_now from .developer_models import ( CleanupReceipt, DeveloperAuthPrerequisite, @@ -24,7 +21,6 @@ ExecutionEffect, ) - DEFAULT_DEVELOPER_PORTS: dict[str, tuple[int, str, str]] = { # Container & Orchestration Runtimes DeveloperServiceType.DOCKER.value: (2375, "tcp", DeveloperCategory.CONTAINER_ORCHESTRATION_RUNTIME.value), @@ -256,7 +252,7 @@ def probe_service( receipts: list[CleanupReceipt] = [] if canary_active and canary_artifact: - receipt_id = hashlib.sha256(f"clean:{target_host}:{service_type}:{canary_artifact}".encode("utf-8")).hexdigest()[:16] + receipt_id = hashlib.sha256(f"clean:{target_host}:{service_type}:{canary_artifact}".encode()).hexdigest()[:16] receipts.append( CleanupReceipt( receipt_id=f"rec-{receipt_id}", @@ -313,7 +309,7 @@ def probe_service( if service_type == DeveloperServiceType.DOCKER.value: if not auth_req or details.get("tls_verify") is False: vulns.append("Docker daemon TCP socket exposed without mutual TLS; root container breakout / RCE permitted") - c_id = hashlib.sha256(f"docker:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"docker:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DeveloperPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -337,7 +333,7 @@ def probe_service( elif service_type == DeveloperServiceType.DOCKER_REGISTRY.value: if not auth_req or details.get("auth_required") is False: vulns.append("Docker Registry HTTP API exposed without authentication; container image and secret leakage permitted") - c_id = hashlib.sha256(f"registry:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"registry:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DeveloperPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -360,7 +356,7 @@ def probe_service( elif service_type == DeveloperServiceType.RMI.value: if not auth_req or details.get("ssl_enabled") is False: vulns.append("Java Remote Method Invocation (RMI) registry exposed without authentication; remote class loading / RCE possible") - c_id = hashlib.sha256(f"rmi:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"rmi:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DeveloperPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -384,7 +380,7 @@ def probe_service( elif service_type == DeveloperServiceType.JDWP.value: if details.get("bind_localhost") is False or not auth_req: vulns.append("Java Debug Wire Protocol (JDWP) port exposed to network; arbitrary JVM bytecode execution permitted") - c_id = hashlib.sha256(f"jdwp:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"jdwp:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DeveloperPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -408,7 +404,7 @@ def probe_service( elif service_type == DeveloperServiceType.ERLANG_EPMD.value: if not auth_req or details.get("cookie_required") is False: vulns.append("Erlang Port Mapper Daemon (EPMD) exposed; unauthenticated node discovery and arbitrary command execution") - c_id = hashlib.sha256(f"epmd:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"epmd:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DeveloperPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -432,7 +428,7 @@ def probe_service( elif service_type == DeveloperServiceType.ADB.value: if not auth_req or details.get("rsa_key_enforced") is False: vulns.append("Android Debug Bridge (ADB) daemon exposed without RSA authorization; arbitrary shell access and app installation permitted") - c_id = hashlib.sha256(f"adb:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"adb:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DeveloperPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -456,7 +452,7 @@ def probe_service( elif service_type == DeveloperServiceType.DISTCC.value: if not auth_req or details.get("allow_cidr_enforced") is False: vulns.append("distcc distributed compiler daemon exposed without host filtering; arbitrary command execution (CVE-2004-2687)") - c_id = hashlib.sha256(f"distcc:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"distcc:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DeveloperPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -480,7 +476,7 @@ def probe_service( elif service_type == DeveloperServiceType.SVN.value: if details.get("anon_access_none") is False or not auth_req: vulns.append("Subversion repository daemon (svnserve) allows anonymous read access; source code repository exposure") - c_id = hashlib.sha256(f"svn:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"svn:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DeveloperPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -503,7 +499,7 @@ def probe_service( elif service_type == DeveloperServiceType.AJP.value: if details.get("secret_required") is False or not details.get("secret_configured"): vulns.append("Apache JServ Protocol (AJP13) exposed without secret; Ghostcat arbitrary file read / RCE (CVE-2020-1938)") - c_id = hashlib.sha256(f"ajp:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"ajp:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DeveloperPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -526,7 +522,7 @@ def probe_service( elif service_type == DeveloperServiceType.FASTCGI.value: if details.get("bind_localhost") is False or not auth_req: vulns.append("FastCGI (php-fpm) port exposed to external network; arbitrary PHP code execution / file inclusion") - c_id = hashlib.sha256(f"fastcgi:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"fastcgi:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( DeveloperPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -629,7 +625,7 @@ def probe_service( configuration_details={"socket_connected": True}, uncertainty_notes=["TCP connect succeeded; protocol-level handshake required to verify authentication"], ) - except (socket.timeout, ConnectionRefusedError, OSError) as err: + except (TimeoutError, ConnectionRefusedError, OSError) as err: return DeveloperServiceAssessment( target_host=target_host, resolved_ip=resolved_ip, @@ -673,7 +669,7 @@ def assess_developer_services( canary = canary_artifact or canary_id targets_str = ",".join(targets) - report_id = f"dev-rep-{hashlib.sha256(f'{scope_ref}:{vantage}:{targets_str}'.encode('utf-8')).hexdigest()[:16]}" + report_id = f"dev-rep-{hashlib.sha256(f'{scope_ref}:{vantage}:{targets_str}'.encode()).hexdigest()[:16]}" assessments: list[DeveloperServiceAssessment] = [] total_exposed = 0 diff --git a/cops/discovery/developer_models.py b/cops/discovery/developer_models.py index 9214ea0..8101707 100644 --- a/cops/discovery/developer_models.py +++ b/cops/discovery/developer_models.py @@ -2,14 +2,15 @@ from __future__ import annotations -from dataclasses import asdict, dataclass, field -from enum import Enum import hashlib import json +from dataclasses import asdict, dataclass, field +from enum import Enum from pathlib import Path from typing import Any -from cops.evidence.canonical import canonical, utc_now +from cops.evidence.canonical import utc_now + from .models import EvidenceProvenance @@ -60,9 +61,9 @@ class DeveloperAuthPrerequisite(str, Enum): NONE = "none" ANONYMOUS = "anonymous" DEFAULT_CREDENTIALS = "default_credentials" - USER_PASSWORD = "user_password" + USER_PASSWORD = "user_password" # noqa: S105 - schema label or operation identifier, not a credential CLIENT_CERT = "client_cert" - TOKEN_OR_API_KEY = "token_or_api_key" + TOKEN_OR_API_KEY = "token_or_api_key" # noqa: S105 - schema label or operation identifier, not a credential ERLANG_COOKIE = "erlang_cookie" UNKNOWN = "unknown" @@ -110,7 +111,7 @@ class DeveloperPrivilegeCandidate: def __post_init__(self) -> None: if not self.evidence_hash: - canonical_blob = f"{self.service_type}:{self.target_host}:{self.port}:{self.finding_type}:{self.auth_prerequisites}:{self.privilege_impact}:{self.execution_effect}".encode("utf-8") + canonical_blob = f"{self.service_type}:{self.target_host}:{self.port}:{self.finding_type}:{self.auth_prerequisites}:{self.privilege_impact}:{self.execution_effect}".encode() self.evidence_hash = hashlib.sha256(canonical_blob).hexdigest() if not self.remediation_guidance: self.remediation_guidance = self._default_remediation() @@ -173,7 +174,7 @@ class CleanupReceipt: def __post_init__(self) -> None: if not self.receipt_hash: - canonical_blob = f"{self.receipt_id}:{self.target_host}:{self.service_type}:{self.artifact_identifier}:{self.action_taken}".encode("utf-8") + canonical_blob = f"{self.receipt_id}:{self.target_host}:{self.service_type}:{self.artifact_identifier}:{self.action_taken}".encode() self.receipt_hash = hashlib.sha256(canonical_blob).hexdigest() def to_dict(self) -> dict[str, Any]: diff --git a/cops/discovery/fingerprinter.py b/cops/discovery/fingerprinter.py index 23caddc..b30d823 100644 --- a/cops/discovery/fingerprinter.py +++ b/cops/discovery/fingerprinter.py @@ -2,15 +2,13 @@ from __future__ import annotations -from datetime import datetime, timezone import re -from typing import Any +from datetime import UTC, datetime from .active_models import ( ConfidenceLevel, InferredFingerprint, ObservedConfiguration, - ObservedTLS, ) @@ -123,11 +121,12 @@ def infer_service_fingerprint( try: # Check certificate expiry if ISO or standard format exp_dt = datetime.fromisoformat(tls.valid_until.replace("Z", "+00:00")) - if exp_dt < datetime.now(timezone.utc): + if exp_dt < datetime.now(UTC): uncertainty_reasons.append("TLS certificate is expired; target service configuration may be unmaintained") confidence = ConfidenceLevel.UNCERTAIN.value - except Exception: - pass + except (TypeError, ValueError): + uncertainty_reasons.append("TLS certificate expiry date could not be parsed") + confidence = ConfidenceLevel.UNCERTAIN.value # Domain mismatch check (if target is a named hostname, not an IP) if not re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", target_host): diff --git a/cops/discovery/importer.py b/cops/discovery/importer.py index 3034ef6..267656e 100644 --- a/cops/discovery/importer.py +++ b/cops/discovery/importer.py @@ -4,11 +4,12 @@ import hashlib import json -from pathlib import Path -from typing import Any import xml.etree.ElementTree as ET +from pathlib import Path +from xml.parsers import expat from cops.evidence.canonical import utc_now + from .active_models import ( ActiveScanSession, ActiveServiceAssessment, @@ -17,7 +18,6 @@ ObservedConfiguration, ObservedTLS, PortState, - Protocol, ScanBudget, ScanVantage, ServiceReachability, @@ -64,7 +64,7 @@ def import_masscan_json( probe_key = ActiveScanSession.make_probe_key(vantage, proto, ip, port) completed_keys.add(probe_key) - probe_id = hashlib.sha256(f"{probe_key}:{file_hash}".encode("utf-8")).hexdigest()[:24] + probe_id = hashlib.sha256(f"{probe_key}:{file_hash}".encode()).hexdigest()[:24] port_state = PortState.OPEN.value if status == "open" else PortState.CLOSED.value reachability = ServiceReachability.REACHABLE.value if status == "open" else ServiceReachability.UNREACHABLE.value @@ -94,7 +94,7 @@ def import_masscan_json( ) ) - session_id = hashlib.sha256(f"{scope_ref}:{file_hash}".encode("utf-8")).hexdigest()[:16] + session_id = hashlib.sha256(f"{scope_ref}:{file_hash}".encode()).hexdigest()[:16] session = ActiveScanSession( session_id=session_id, scope_reference=scope_ref, @@ -121,7 +121,17 @@ def import_nmap_xml( """Import Nmap XML output (-oX) into an ActiveScanSession.""" path = Path(filepath) raw_content = path.read_text(encoding="utf-8") - root = ET.fromstring(raw_content) + # Nmap emits an ordinary DOCTYPE; entity declarations and external entities + # are never needed for scan data and must be rejected before tree expansion. + parser = expat.ParserCreate() + + def reject_entity(*_args): + raise ValueError("Nmap XML entity declarations and external entities are forbidden") + + parser.EntityDeclHandler = reject_entity + parser.ExternalEntityRefHandler = reject_entity + parser.Parse(raw_content, True) + root = ET.fromstring(raw_content) # noqa: S314 - entities rejected by Expat preflight above file_hash = hashlib.sha256(raw_content.encode("utf-8")).hexdigest() prov = EvidenceProvenance( @@ -232,7 +242,7 @@ def import_nmap_xml( probe_key = ActiveScanSession.make_probe_key(vantage, proto, target_host, port_id) completed_keys.add(probe_key) - probe_id = hashlib.sha256(f"{probe_key}:{file_hash}".encode("utf-8")).hexdigest()[:24] + probe_id = hashlib.sha256(f"{probe_key}:{file_hash}".encode()).hexdigest()[:24] assessments.append( ActiveServiceAssessment( @@ -251,7 +261,7 @@ def import_nmap_xml( ) ) - session_id = hashlib.sha256(f"{scope_ref}:{file_hash}".encode("utf-8")).hexdigest()[:16] + session_id = hashlib.sha256(f"{scope_ref}:{file_hash}".encode()).hexdigest()[:16] session = ActiveScanSession( session_id=session_id, scope_reference=scope_ref, diff --git a/cops/discovery/infra_collector.py b/cops/discovery/infra_collector.py index f7bac77..28c58ad 100644 --- a/cops/discovery/infra_collector.py +++ b/cops/discovery/infra_collector.py @@ -2,17 +2,12 @@ from __future__ import annotations -from abc import ABC, abstractmethod import hashlib -import ipaddress -import json -from pathlib import Path import re import socket -import time +from abc import ABC, abstractmethod from typing import Any -from cops.evidence.canonical import utc_now from .infra_models import ( AuthPrerequisite, IdentityAttackPathCandidate, @@ -22,7 +17,6 @@ InfraServiceType, ServiceExposureStatus, ) -from .models import EvidenceProvenance class InfraCollector(ABC): @@ -250,7 +244,7 @@ def probe_service( domain = details.get("defaultNamingContext") or ( details.get("naming_contexts", ["DC=corp,DC=internal"])[0] ) - candidate_id = hashlib.sha256(f"ldap:{target_host}:{port}:{domain}".encode("utf-8")).hexdigest()[:16] + candidate_id = hashlib.sha256(f"ldap:{target_host}:{port}:{domain}".encode()).hexdigest()[:16] candidates.append( IdentityAttackPathCandidate( candidate_id=f"cand-{candidate_id}", @@ -278,7 +272,7 @@ def probe_service( vulns.append("Kerberos pre-authentication disabled for one or more accounts") for acc in (preauth_disabled or ["vulnerable_account"]): principal = f"{acc}@{realm}" if "@" not in acc else acc - candidate_id = hashlib.sha256(f"krb:{target_host}:{port}:{principal}".encode("utf-8")).hexdigest()[:16] + candidate_id = hashlib.sha256(f"krb:{target_host}:{port}:{principal}".encode()).hexdigest()[:16] candidates.append( IdentityAttackPathCandidate( candidate_id=f"cand-{candidate_id}", @@ -305,7 +299,7 @@ def probe_service( status = ServiceExposureStatus.EXPOSED.value auth_prereq = AuthPrerequisite.NONE.value vulns.append(f"RPC Endpoint Mapper discloses {len(interfaces)} registered interfaces without authentication") - candidate_id = hashlib.sha256(f"rpc:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + candidate_id = hashlib.sha256(f"rpc:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( IdentityAttackPathCandidate( candidate_id=f"cand-{candidate_id}", @@ -329,7 +323,7 @@ def probe_service( status = ServiceExposureStatus.EXPOSED.value auth_prereq = AuthPrerequisite.DEFAULT_CREDENTIALS.value vulns.append(f"SNMP agent accessible via default community strings: {', '.join(default_comm)}") - candidate_id = hashlib.sha256(f"snmp:{target_host}:{port}:{default_comm[0]}".encode("utf-8")).hexdigest()[:16] + candidate_id = hashlib.sha256(f"snmp:{target_host}:{port}:{default_comm[0]}".encode()).hexdigest()[:16] candidates.append( IdentityAttackPathCandidate( candidate_id=f"cand-{candidate_id}", @@ -352,7 +346,7 @@ def probe_service( status = ServiceExposureStatus.MISCONFIGURED.value auth_prereq = AuthPrerequisite.NONE.value vulns.append("DNS resolver permits open recursion from external vantage") - candidate_id = hashlib.sha256(f"dns:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + candidate_id = hashlib.sha256(f"dns:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( IdentityAttackPathCandidate( candidate_id=f"cand-{candidate_id}", @@ -371,7 +365,7 @@ def probe_service( status = ServiceExposureStatus.MISCONFIGURED.value auth_prereq = AuthPrerequisite.NONE.value vulns.append("NTP daemon responds to Mode 6 / monlist enumeration queries") - candidate_id = hashlib.sha256(f"ntp:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + candidate_id = hashlib.sha256(f"ntp:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( IdentityAttackPathCandidate( candidate_id=f"cand-{candidate_id}", @@ -441,7 +435,7 @@ def probe_service( configuration_details={"socket_connected": True}, uncertainty_notes=["Service reachable via TCP; protocol-specific authentication check requires credentials"], ) - except (socket.timeout, ConnectionRefusedError, OSError) as err: + except (TimeoutError, ConnectionRefusedError, OSError) as err: return InfraServiceAssessment( target_host=target_host, resolved_ip=resolved_ip, @@ -497,7 +491,7 @@ def assess_infrastructure_services( "attack_path_candidates": 0, } - report_id = hashlib.sha256(f"{scope_ref}:{sorted(targets)}:{sorted(selected_services)}:{vantage}".encode("utf-8")).hexdigest()[:16] + report_id = hashlib.sha256(f"{scope_ref}:{sorted(targets)}:{sorted(selected_services)}:{vantage}".encode()).hexdigest()[:16] for target in targets: try: diff --git a/cops/discovery/infra_models.py b/cops/discovery/infra_models.py index e9123d5..972f360 100644 --- a/cops/discovery/infra_models.py +++ b/cops/discovery/infra_models.py @@ -2,14 +2,14 @@ from __future__ import annotations +import hashlib from dataclasses import asdict, dataclass, field from enum import Enum -import hashlib -import json from pathlib import Path from typing import Any from cops.evidence.canonical import canonical, utc_now + from .models import EvidenceProvenance @@ -80,7 +80,7 @@ class IdentityAttackPathCandidate: def __post_init__(self) -> None: if not self.evidence_hash: h = hashlib.sha256( - f"{self.candidate_id}:{self.target_host}:{self.port}:{self.attack_path_type}:{self.auth_prerequisites}".encode("utf-8") + f"{self.candidate_id}:{self.target_host}:{self.port}:{self.attack_path_type}:{self.auth_prerequisites}".encode() ).hexdigest() self.evidence_hash = h if not self.remediation_guidance: diff --git a/cops/discovery/legacy_collector.py b/cops/discovery/legacy_collector.py index 8bb5874..e6c4ea3 100644 --- a/cops/discovery/legacy_collector.py +++ b/cops/discovery/legacy_collector.py @@ -2,14 +2,12 @@ from __future__ import annotations -from abc import ABC, abstractmethod import hashlib import ipaddress -from pathlib import Path import socket +from abc import ABC, abstractmethod from typing import Any -from cops.evidence.canonical import utc_now from .legacy_models import ( CleanupReceipt, ExecutionEffect, @@ -23,7 +21,6 @@ LegacyServiceType, ) - DEFAULT_LEGACY_PORTS: dict[str, tuple[int, str, str]] = { # Enterprise Storage & Data Management LegacyServiceType.NDMP.value: (10000, "tcp", LegacyCategory.ENTERPRISE_STORAGE_MANAGEMENT.value), @@ -254,7 +251,7 @@ def probe_service( receipts: list[CleanupReceipt] = [] if canary_active and canary_artifact: - receipt_id = hashlib.sha256(f"clean:{target_host}:{service_type}:{canary_artifact}".encode("utf-8")).hexdigest()[:16] + receipt_id = hashlib.sha256(f"clean:{target_host}:{service_type}:{canary_artifact}".encode()).hexdigest()[:16] receipts.append( CleanupReceipt( receipt_id=f"rec-{receipt_id}", @@ -315,7 +312,7 @@ def probe_service( if service_type == LegacyServiceType.NDMP.value: if not auth_req or details.get("auth_required") is False: vulns.append("NDMP storage management interface exposed without authentication; tape/disk backup traversal permitted") - c_id = hashlib.sha256(f"ndmp:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"ndmp:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( LegacyPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -338,7 +335,7 @@ def probe_service( elif service_type == LegacyServiceType.ISCSI.value: if not auth_req or details.get("chap_enforced") is False: vulns.append("iSCSI storage target discovery permits unauthenticated SendTargets discovery and session attachment") - c_id = hashlib.sha256(f"iscsi:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"iscsi:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( LegacyPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -361,7 +358,7 @@ def probe_service( elif service_type == LegacyServiceType.IPMI.value: if details.get("cipher_zero") is True or details.get("cipher_zero_enabled") is True: vulns.append("IPMI 2.0 RMCP+ cipher suite 0 authentication bypass enabled; unrestricted BMC lights-out control") - c_id = hashlib.sha256(f"ipmi-c0:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"ipmi-c0:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( LegacyPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -383,7 +380,7 @@ def probe_service( can_change_state = True elif details.get("rakp_dumpable") is True or not auth_req: vulns.append("IPMI 2.0 RAKP HMAC-SHA1 password hashes retrievable via unauthenticated handshake request") - c_id = hashlib.sha256(f"ipmi-rakp:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"ipmi-rakp:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( LegacyPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -405,7 +402,7 @@ def probe_service( elif service_type == LegacyServiceType.CISCO_SMART_INSTALL.value: if not auth_req or details.get("smi_active") is True: vulns.append("Cisco Smart Install (SMI) active on TCP 4786 without authentication; arbitrary config download and RCE") - c_id = hashlib.sha256(f"smi:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"smi:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( LegacyPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -429,7 +426,7 @@ def probe_service( elif service_type == LegacyServiceType.TACACS.value: if not auth_req or details.get("single_connect") is True: vulns.append("TACACS+ AAA daemon exposed to network; unauthenticated handshake reveals legacy obfuscation key usage") - c_id = hashlib.sha256(f"tacacs:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"tacacs:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( LegacyPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -451,7 +448,7 @@ def probe_service( elif service_type == LegacyServiceType.IKE.value: if details.get("aggressive_mode") is True or not auth_req: vulns.append("IKEv1 Aggressive Mode enabled; responder returns pre-shared key (PSK) hash subject to offline cracking") - c_id = hashlib.sha256(f"ike:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"ike:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( LegacyPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -473,7 +470,7 @@ def probe_service( elif service_type == LegacyServiceType.PPTP.value: if details.get("mschapv2") is True or not auth_req: vulns.append("PPTP VPN service exposed using vulnerable MS-CHAPv2 authentication; credential hash cracking permitted") - c_id = hashlib.sha256(f"pptp:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"pptp:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( LegacyPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -498,7 +495,7 @@ def probe_service( proxy_egress_restricted = is_restricted if not auth_req or details.get("auth_required") is False or not is_restricted: vulns.append("SOCKS proxy exposed without authentication or egress restrictions; open network relaying permitted") - c_id = hashlib.sha256(f"socks:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"socks:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( LegacyPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -524,7 +521,7 @@ def probe_service( proxy_egress_restricted = is_restricted if details.get("open_proxy") is True or not is_restricted: vulns.append("Squid HTTP proxy allows open forward proxying without client subnet restriction; internal SSRF / pivoting permitted") - c_id = hashlib.sha256(f"squid:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"squid:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( LegacyPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -662,7 +659,7 @@ def probe_service( configuration_details={"socket_connected": True}, uncertainty_notes=["TCP connect succeeded; protocol-level handshake required to verify authentication"], ) - except (socket.timeout, ConnectionRefusedError, OSError) as err: + except (TimeoutError, ConnectionRefusedError, OSError) as err: return LegacyServiceAssessment( target_host=target_host, resolved_ip=resolved_ip, @@ -708,7 +705,7 @@ def assess_legacy_services( canary = canary_artifact or canary_id targets_str = ",".join(targets) - report_id = f"leg-rep-{hashlib.sha256(f'{scope_ref}:{vantage}:{targets_str}'.encode('utf-8')).hexdigest()[:16]}" + report_id = f"leg-rep-{hashlib.sha256(f'{scope_ref}:{vantage}:{targets_str}'.encode()).hexdigest()[:16]}" assessments: list[LegacyServiceAssessment] = [] total_exposed = 0 diff --git a/cops/discovery/legacy_models.py b/cops/discovery/legacy_models.py index 852207d..f000b6b 100644 --- a/cops/discovery/legacy_models.py +++ b/cops/discovery/legacy_models.py @@ -2,14 +2,15 @@ from __future__ import annotations -from dataclasses import asdict, dataclass, field -from enum import Enum import hashlib import json +from dataclasses import asdict, dataclass, field +from enum import Enum from pathlib import Path from typing import Any -from cops.evidence.canonical import canonical, utc_now +from cops.evidence.canonical import utc_now + from .models import EvidenceProvenance @@ -67,7 +68,7 @@ class LegacyAuthPrerequisite(str, Enum): SHARED_KEY = "shared_key" PSK = "psk" MSCHAPV2 = "mschapv2" - USER_PASSWORD = "user_password" + USER_PASSWORD = "user_password" # noqa: S105 - schema label or operation identifier, not a credential ACL_RESTRICTED = "acl_restricted" UNKNOWN = "unknown" @@ -116,7 +117,7 @@ class LegacyPrivilegeCandidate: def __post_init__(self) -> None: if not self.evidence_hash: - canonical_blob = f"{self.service_type}:{self.target_host}:{self.port}:{self.finding_type}:{self.auth_prerequisites}:{self.privilege_impact}:{self.execution_effect}".encode("utf-8") + canonical_blob = f"{self.service_type}:{self.target_host}:{self.port}:{self.finding_type}:{self.auth_prerequisites}:{self.privilege_impact}:{self.execution_effect}".encode() self.evidence_hash = hashlib.sha256(canonical_blob).hexdigest() if not self.remediation_guidance: self.remediation_guidance = self._default_remediation() @@ -179,7 +180,7 @@ class CleanupReceipt: def __post_init__(self) -> None: if not self.receipt_hash: - canonical_blob = f"{self.receipt_id}:{self.target_host}:{self.service_type}:{self.artifact_identifier}:{self.action_taken}".encode("utf-8") + canonical_blob = f"{self.receipt_id}:{self.target_host}:{self.service_type}:{self.artifact_identifier}:{self.action_taken}".encode() self.receipt_hash = hashlib.sha256(canonical_blob).hexdigest() def to_dict(self) -> dict[str, Any]: diff --git a/cops/discovery/merger.py b/cops/discovery/merger.py index 3ec2044..4310ebc 100644 --- a/cops/discovery/merger.py +++ b/cops/discovery/merger.py @@ -2,8 +2,7 @@ from __future__ import annotations -from datetime import datetime, timezone -from typing import Any +from datetime import UTC, datetime from .models import DiscoveredAsset, DiscoveryInventory, EvidenceProvenance @@ -102,7 +101,7 @@ def merge_inventories( }, } - now_iso = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + now_iso = datetime.now(UTC).isoformat().replace("+00:00", "Z") return DiscoveryInventory( timestamp=now_iso, assets=merged_assets, diff --git a/cops/discovery/messaging_collector.py b/cops/discovery/messaging_collector.py index b96ab56..fa09a00 100644 --- a/cops/discovery/messaging_collector.py +++ b/cops/discovery/messaging_collector.py @@ -2,15 +2,12 @@ from __future__ import annotations -from abc import ABC, abstractmethod import hashlib import ipaddress -import json -from pathlib import Path import socket +from abc import ABC, abstractmethod from typing import Any -from cops.evidence.canonical import canonical, utc_now from .messaging_models import ( CleanupReceipt, MessagingAuthPrerequisite, @@ -23,7 +20,6 @@ MessagingServiceType, ) - DEFAULT_MESSAGING_PORTS: dict[str, tuple[int, str, str]] = { # Mail Services MessagingServiceType.SMTP.value: (25, "tcp", MessagingCategory.MAIL_TRANSFER_RETRIEVAL.value), @@ -243,7 +239,7 @@ def probe_service( receipts: list[CleanupReceipt] = [] if canary_active and canary_artifact: - receipt_id = hashlib.sha256(f"clean:{target_host}:{service_type}:{canary_artifact}".encode("utf-8")).hexdigest()[:16] + receipt_id = hashlib.sha256(f"clean:{target_host}:{service_type}:{canary_artifact}".encode()).hexdigest()[:16] receipts.append( CleanupReceipt( receipt_id=f"rec-{receipt_id}", @@ -303,7 +299,7 @@ def probe_service( if service_type == MessagingServiceType.SMTP.value: if relay_permitted or details.get("relay_allowed") is True: vulns.append("SMTP open mail relay permitted; external recipients accepted without authentication") - c_id = hashlib.sha256(f"smtp_relay:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"smtp_relay:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( MessagingPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -322,7 +318,7 @@ def probe_service( ) if details.get("user_enumeration_enabled") is True or details.get("vrfy_supported") is True: vulns.append("SMTP VRFY/EXPN user enumeration supported; unauthenticated recipient verification enabled") - c_id = hashlib.sha256(f"smtp_enum:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"smtp_enum:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( MessagingPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -343,7 +339,7 @@ def probe_service( elif service_type == MessagingServiceType.POP3.value: if details.get("plaintext_auth_allowed") is True or not details.get("tls_enforced"): vulns.append("POP3 server permits plaintext USER/PASS authentication without transport layer security") - c_id = hashlib.sha256(f"pop3_plain:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"pop3_plain:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( MessagingPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -364,7 +360,7 @@ def probe_service( elif service_type == MessagingServiceType.IMAP.value: if details.get("anonymous_allowed") is True or not auth_req: vulns.append("IMAP service accepts anonymous login; unauthenticated mailbox traversal permitted") - c_id = hashlib.sha256(f"imap_anon:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"imap_anon:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( MessagingPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -385,7 +381,7 @@ def probe_service( elif service_type == MessagingServiceType.IRC.value: if details.get("unauthenticated_oper") is True or details.get("oper_password_required") is False: vulns.append("IRC server grants operator status without authentication or uses hardcoded oper credentials") - c_id = hashlib.sha256(f"irc_oper:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"irc_oper:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( MessagingPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -406,7 +402,7 @@ def probe_service( elif service_type == MessagingServiceType.RABBITMQ.value: if details.get("guest_enabled") is True: vulns.append("RabbitMQ broker retains default guest:guest credentials with administrative privileges") - c_id = hashlib.sha256(f"rabbit_guest:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"rabbit_guest:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( MessagingPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -425,7 +421,7 @@ def probe_service( ) if details.get("open_management") is True: vulns.append("RabbitMQ management HTTP API exposed without authentication") - c_id = hashlib.sha256(f"rabbit_mgmt:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"rabbit_mgmt:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( MessagingPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -446,7 +442,7 @@ def probe_service( elif service_type == MessagingServiceType.NATS.value: if details.get("auth_required") is False or not auth_req: vulns.append("NATS streaming broker accepts unauthenticated pub/sub client connections") - c_id = hashlib.sha256(f"nats_anon:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"nats_anon:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( MessagingPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -467,7 +463,7 @@ def probe_service( elif service_type == MessagingServiceType.IBMMQ.value: if details.get("blank_channel_enabled") is True or details.get("mcauser_enforced") is False: vulns.append("IBM MQ SVRCONN channel operates with blank MCAUSER; unauthenticated mqadmin privilege granted") - c_id = hashlib.sha256(f"ibmmq_mcauser:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"ibmmq_mcauser:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( MessagingPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -488,7 +484,7 @@ def probe_service( elif service_type == MessagingServiceType.KAFKA.value: if details.get("sasl_enabled") is False or not auth_req: vulns.append("Apache Kafka cluster accepts unauthenticated PLAINTEXT consumer and producer requests") - c_id = hashlib.sha256(f"kafka_anon:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"kafka_anon:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( MessagingPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -509,7 +505,7 @@ def probe_service( elif service_type == MessagingServiceType.MQTT.value: if details.get("allow_anonymous") is True or not auth_req: vulns.append("MQTT broker accepts anonymous pub/sub connections with wildcards on topic hierarchy") - c_id = hashlib.sha256(f"mqtt_anon:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"mqtt_anon:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( MessagingPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -594,7 +590,7 @@ def probe_service( configuration_details={"socket_connected": True}, uncertainty_notes=["TCP connect succeeded; protocol-level handshake required to verify authentication"], ) - except (socket.timeout, ConnectionRefusedError, OSError) as err: + except (TimeoutError, ConnectionRefusedError, OSError) as err: return MessagingServiceAssessment( target_host=target_host, resolved_ip=resolved_ip, @@ -633,7 +629,7 @@ def assess_messaging_services( canary = canary_artifact or canary_id targets_str = ",".join(targets) - report_id = f"msg-rep-{hashlib.sha256(f'{scope_ref}:{vantage}:{targets_str}'.encode('utf-8')).hexdigest()[:16]}" + report_id = f"msg-rep-{hashlib.sha256(f'{scope_ref}:{vantage}:{targets_str}'.encode()).hexdigest()[:16]}" assessments: list[MessagingServiceAssessment] = [] total_exposed = 0 diff --git a/cops/discovery/messaging_models.py b/cops/discovery/messaging_models.py index b565f6d..22c729f 100644 --- a/cops/discovery/messaging_models.py +++ b/cops/discovery/messaging_models.py @@ -2,14 +2,15 @@ from __future__ import annotations -from dataclasses import asdict, dataclass, field -from enum import Enum import hashlib import json +from dataclasses import asdict, dataclass, field +from enum import Enum from pathlib import Path from typing import Any -from cops.evidence.canonical import canonical, utc_now +from cops.evidence.canonical import utc_now + from .models import EvidenceProvenance @@ -56,9 +57,9 @@ class MessagingAuthPrerequisite(str, Enum): NONE = "none" ANONYMOUS = "anonymous" DEFAULT_CREDENTIALS = "default_credentials" - USER_PASSWORD = "user_password" + USER_PASSWORD = "user_password" # noqa: S105 - schema label or operation identifier, not a credential CLIENT_CERT = "client_cert" - TOKEN_OR_API_KEY = "token_or_api_key" + TOKEN_OR_API_KEY = "token_or_api_key" # noqa: S105 - schema label or operation identifier, not a credential SASL = "sasl" UNKNOWN = "unknown" @@ -96,7 +97,7 @@ class MessagingPrivilegeCandidate: def __post_init__(self) -> None: if not self.evidence_hash: - canonical_blob = f"{self.service_type}:{self.target_host}:{self.port}:{self.finding_type}:{self.auth_prerequisites}:{self.privilege_impact}".encode("utf-8") + canonical_blob = f"{self.service_type}:{self.target_host}:{self.port}:{self.finding_type}:{self.auth_prerequisites}:{self.privilege_impact}".encode() self.evidence_hash = hashlib.sha256(canonical_blob).hexdigest() if not self.remediation_guidance: self.remediation_guidance = self._default_remediation() @@ -159,7 +160,7 @@ class CleanupReceipt: def __post_init__(self) -> None: if not self.receipt_hash: - canonical_blob = f"{self.receipt_id}:{self.target_host}:{self.service_type}:{self.artifact_identifier}:{self.action_taken}".encode("utf-8") + canonical_blob = f"{self.receipt_id}:{self.target_host}:{self.service_type}:{self.artifact_identifier}:{self.action_taken}".encode() self.receipt_hash = hashlib.sha256(canonical_blob).hexdigest() def to_dict(self) -> dict[str, Any]: diff --git a/cops/discovery/normalizers.py b/cops/discovery/normalizers.py index f54851c..3cb3b97 100644 --- a/cops/discovery/normalizers.py +++ b/cops/discovery/normalizers.py @@ -17,7 +17,7 @@ def _compute_asset_id(asset_type: str, identifier: str) -> str: """Deterministic 24-character asset hash.""" - seed = f"{asset_type}:{identifier.strip().lower()}".encode("utf-8") + seed = f"{asset_type}:{identifier.strip().lower()}".encode() return hashlib.sha256(seed).hexdigest()[:24] diff --git a/cops/discovery/reconciler.py b/cops/discovery/reconciler.py index 414983c..a906dbc 100644 --- a/cops/discovery/reconciler.py +++ b/cops/discovery/reconciler.py @@ -2,8 +2,8 @@ from __future__ import annotations -from datetime import datetime, timezone import ipaddress +from datetime import UTC, datetime from typing import Any from .models import DiscoveredAsset, DiscoveryInventory @@ -63,7 +63,7 @@ def reconcile_asset(asset: DiscoveredAsset, scope: dict[str, Any]) -> Discovered if attrs.get("not_after"): try: not_after_dt = datetime.fromisoformat(str(attrs["not_after"]).replace("Z", "+00:00")) - if not_after_dt < datetime.now(timezone.utc): + if not_after_dt < datetime.now(UTC): quarantine_reasons.append("stale_record") except (ValueError, TypeError): pass @@ -163,7 +163,7 @@ def reconcile_inventory( }, } - now_iso = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + now_iso = datetime.now(UTC).isoformat().replace("+00:00", "Z") return DiscoveryInventory( timestamp=now_iso, assets=reconciled_assets, diff --git a/cops/discovery/remote_collector.py b/cops/discovery/remote_collector.py index eab3bc7..1d82d3f 100644 --- a/cops/discovery/remote_collector.py +++ b/cops/discovery/remote_collector.py @@ -2,26 +2,20 @@ from __future__ import annotations -from abc import ABC, abstractmethod import hashlib -import ipaddress -import json -from pathlib import Path import re import socket -import time +from abc import ABC, abstractmethod from typing import Any -from cops.evidence.canonical import utc_now -from .models import EvidenceProvenance from .remote_models import ( CleanupReceipt, HostPrivilegeCandidate, LateralMovementImpact, RemoteAuthPrerequisite, RemoteExposureStatus, - RemoteServiceCategory, RemoteServiceAssessment, + RemoteServiceCategory, RemoteServicesReport, RemoteServiceType, ) @@ -266,7 +260,7 @@ def probe_service( receipts: list[CleanupReceipt] = [] if canary_active and canary_artifact: - receipt_id = hashlib.sha256(f"clean:{target_host}:{service_type}:{canary_artifact}".encode("utf-8")).hexdigest()[:16] + receipt_id = hashlib.sha256(f"clean:{target_host}:{service_type}:{canary_artifact}".encode()).hexdigest()[:16] receipts.append( CleanupReceipt( receipt_id=f"rec-{receipt_id}", @@ -330,7 +324,7 @@ def probe_service( if has_smbv1: is_legacy = True vulns.append("Obsolete SMBv1/CIFS protocol enabled") - c_id = hashlib.sha256(f"smbv1:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"smbv1:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( HostPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -349,7 +343,7 @@ def probe_service( if signing_disabled: vulns.append("SMB packet signing not required; vulnerable to NTLM relay attacks") - c_id = hashlib.sha256(f"smbsign:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"smbsign:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( HostPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -368,7 +362,7 @@ def probe_service( if shares or not auth_req: vulns.append(f"Unauthenticated null/guest session permitted with {len(shares)} accessible shares") - c_id = hashlib.sha256(f"smbshare:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"smbshare:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( HostPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -388,7 +382,7 @@ def probe_service( elif service_type == RemoteServiceType.TELNET.value: is_legacy = True vulns.append("Unencrypted plaintext Telnet service accessible") - c_id = hashlib.sha256(f"telnet:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"telnet:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( HostPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -409,7 +403,7 @@ def probe_service( nla_disabled = details.get("nla_enabled") is False or details.get("nla_disabled") is True if nla_disabled: vulns.append("Remote Desktop exposes pre-authentication login screen without NLA enforcement") - c_id = hashlib.sha256(f"rdpnla:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"rdpnla:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( HostPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -429,7 +423,7 @@ def probe_service( elif service_type == RemoteServiceType.VNC.value: if not auth_req or details.get("auth_type") in ("none", "None", 1): vulns.append("VNC RFB service accessible without authentication") - c_id = hashlib.sha256(f"vnc:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"vnc:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( HostPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -449,7 +443,7 @@ def probe_service( elif service_type == RemoteServiceType.WINRM.value: if port == 5985 or details.get("https_enforced") is False: vulns.append("WinRM management endpoint exposed over unencrypted HTTP (5985)") - c_id = hashlib.sha256(f"winrm:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"winrm:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( HostPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -469,7 +463,7 @@ def probe_service( elif service_type == RemoteServiceType.X11.value: if not auth_req or details.get("auth_required") is False: vulns.append("X11 display server open to unauthenticated remote client connections") - c_id = hashlib.sha256(f"x11:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"x11:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( HostPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -492,7 +486,7 @@ def probe_service( if has_no_root_squash or exports: if has_no_root_squash: vulns.append("NFS export configured with 'no_root_squash' permitting root privilege escalation") - c_id = hashlib.sha256(f"nfs:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"nfs:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( HostPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -513,7 +507,7 @@ def probe_service( if details.get("anonymous_login") or not auth_req: is_legacy = True vulns.append("Unauthenticated anonymous FTP access permitted") - c_id = hashlib.sha256(f"ftp:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"ftp:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( HostPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -534,7 +528,7 @@ def probe_service( modules = details.get("modules", []) if not auth_req or modules: vulns.append(f"Rsync daemon exposes {len(modules)} modules without mandatory authentication") - c_id = hashlib.sha256(f"rsync:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"rsync:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( HostPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -554,7 +548,7 @@ def probe_service( elif service_type in (RemoteServiceType.LPD.value, RemoteServiceType.IPP.value, RemoteServiceType.RAW_PRINT.value): if not auth_req: vulns.append(f"Print service ({service_type.upper()}) accepts unauthenticated print job submissions") - c_id = hashlib.sha256(f"print:{target_host}:{port}".encode("utf-8")).hexdigest()[:16] + c_id = hashlib.sha256(f"print:{target_host}:{port}".encode()).hexdigest()[:16] candidates.append( HostPrivilegeCandidate( candidate_id=f"priv-{c_id}", @@ -629,7 +623,7 @@ def probe_service( configuration_details={"socket_connected": True}, uncertainty_notes=["TCP connect succeeded; protocol credentials required to verify authentication"], ) - except (socket.timeout, ConnectionRefusedError, OSError) as err: + except (TimeoutError, ConnectionRefusedError, OSError) as err: return RemoteServiceAssessment( target_host=target_host, resolved_ip=resolved_ip, @@ -680,7 +674,7 @@ def assess_remote_services( } report_id = hashlib.sha256( - f"{scope_ref}:{sorted(targets)}:{sorted(selected_services)}:{vantage}".encode("utf-8") + f"{scope_ref}:{sorted(targets)}:{sorted(selected_services)}:{vantage}".encode() ).hexdigest()[:16] for target in targets: diff --git a/cops/discovery/remote_models.py b/cops/discovery/remote_models.py index 0da24f8..7ad9cf8 100644 --- a/cops/discovery/remote_models.py +++ b/cops/discovery/remote_models.py @@ -2,14 +2,15 @@ from __future__ import annotations -from dataclasses import asdict, dataclass, field -from enum import Enum import hashlib import json +from dataclasses import asdict, dataclass, field +from enum import Enum from pathlib import Path from typing import Any from cops.evidence.canonical import canonical, utc_now + from .models import EvidenceProvenance @@ -62,7 +63,7 @@ class RemoteAuthPrerequisite(str, Enum): NONE = "none" ANONYMOUS = "anonymous" DEFAULT_CREDENTIALS = "default_credentials" - USER_PASSWORD = "user_password" + USER_PASSWORD = "user_password" # noqa: S105 - schema label or operation identifier, not a credential PUBLIC_KEY = "public_key" NLA_REQUIRED = "nla_required" KERBEROS = "kerberos" @@ -102,7 +103,7 @@ class HostPrivilegeCandidate: def __post_init__(self) -> None: if not self.evidence_hash: h = hashlib.sha256( - f"{self.candidate_id}:{self.target_host}:{self.port}:{self.service_type}:{self.finding_type}:{self.auth_prerequisites}".encode("utf-8") + f"{self.candidate_id}:{self.target_host}:{self.port}:{self.service_type}:{self.finding_type}:{self.auth_prerequisites}".encode() ).hexdigest() self.evidence_hash = h if not self.remediation_guidance: @@ -165,7 +166,7 @@ class CleanupReceipt: def __post_init__(self) -> None: if not self.receipt_hash: h = hashlib.sha256( - f"{self.receipt_id}:{self.target_host}:{self.service_type}:{self.artifact_type}:{self.artifact_identifier}:{self.action_taken}".encode("utf-8") + f"{self.receipt_id}:{self.target_host}:{self.service_type}:{self.artifact_type}:{self.artifact_identifier}:{self.action_taken}".encode() ).hexdigest() self.receipt_hash = h diff --git a/cops/evidence/__init__.py b/cops/evidence/__init__.py index 45cebcc..1c2cc0d 100644 --- a/cops/evidence/__init__.py +++ b/cops/evidence/__init__.py @@ -1,8 +1,8 @@ """Shared evidence contracts; portable consumers use the generated approved subset.""" +from .assessment import assess, report from .canonical import EvidenceError, canonical, decode_json from .contract import build_envelope from .validation import validate_envelope, validate_receipt -from .assessment import assess, report __all__ = ["EvidenceError", "canonical", "decode_json", "build_envelope", "validate_envelope", "validate_receipt", "assess", "report"] diff --git a/cops/evidence/assessment.py b/cops/evidence/assessment.py index e4f1af8..5f1f64c 100644 --- a/cops/evidence/assessment.py +++ b/cops/evidence/assessment.py @@ -1,5 +1,6 @@ """Provider-independent receipt and observation assessment; reports omit payloads.""" import math + from .canonical import EvidenceError, timestamp from .validation import validate_envelope, validate_receipt diff --git a/cops/evidence/canonical.py b/cops/evidence/canonical.py index 87a285f..22e079d 100644 --- a/cops/evidence/canonical.py +++ b/cops/evidence/canonical.py @@ -1,9 +1,9 @@ """Bounded canonical JSON; error messages never include untrusted values.""" -from datetime import datetime, timezone import hashlib import json import math import re +from datetime import UTC, datetime class EvidenceError(ValueError): @@ -24,7 +24,7 @@ def timestamp(value): def utc_now(): - return datetime.now(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z") def canonical(value, *, max_bytes=1024 * 1024, max_depth=32): diff --git a/cops/evidence/validation.py b/cops/evidence/validation.py index e482d02..650b870 100644 --- a/cops/evidence/validation.py +++ b/cops/evidence/validation.py @@ -1,7 +1,7 @@ """Validate the published, deliberately small JSON Schema vocabulary without deps.""" import json -from pathlib import Path import re +from pathlib import Path from .canonical import EvidenceError, canonical, digest, timestamp diff --git a/cops/execution/authorization.py b/cops/execution/authorization.py index ea0a51e..6f43987 100644 --- a/cops/execution/authorization.py +++ b/cops/execution/authorization.py @@ -7,20 +7,19 @@ from __future__ import annotations import getpass -import hashlib import json import os import sys import uuid import warnings -from datetime import datetime, timedelta, timezone -from pathlib import Path -from typing import Any, Callable, Sequence +from collections.abc import Callable +from datetime import UTC, datetime, timedelta +from typing import Any from cops.contracts.lifecycle import ContractError from cops.contracts.models import ActionPlan, ExecutionAuthorization from cops.contracts.validation import validate_contract -from cops.evidence.canonical import canonical, digest, timestamp, utc_now +from cops.evidence.canonical import digest, timestamp, utc_now class AuthorizationError(ValueError): @@ -85,7 +84,7 @@ def create_execution_authorization( f"cannot authorize action plan '{action_plan_model.plan_id}' with terminal status '{action_plan_model.status}'" ) - now = datetime.now(timezone.utc) + now = datetime.now(UTC) issued_at = now.isoformat(timespec="seconds").replace("+00:00", "Z") authorized_until_utc = (now + timedelta(hours=valid_hours)).isoformat(timespec="seconds").replace("+00:00", "Z") @@ -241,7 +240,7 @@ def verify_execution_authorization( ) # Verify expiration - now_dt = timestamp(current_time_iso) if current_time_iso else datetime.now(timezone.utc) + now_dt = timestamp(current_time_iso) if current_time_iso else datetime.now(UTC) auth_expiry = timestamp(auth_model.authorized_until_utc) if now_dt > auth_expiry: raise AuthorizationError( diff --git a/cops/execution/cleanup.py b/cops/execution/cleanup.py index 44fbc9a..8d38c61 100644 --- a/cops/execution/cleanup.py +++ b/cops/execution/cleanup.py @@ -12,9 +12,8 @@ import shutil import uuid from dataclasses import dataclass, field -from datetime import datetime, timezone from pathlib import Path -from typing import Any, Sequence +from typing import Any from cops.contracts.models import CleanupReceipt from cops.evidence.canonical import digest, utc_now diff --git a/cops/execution/evidence.py b/cops/execution/evidence.py index 035a0b2..ae6ed1c 100644 --- a/cops/execution/evidence.py +++ b/cops/execution/evidence.py @@ -6,14 +6,15 @@ from __future__ import annotations -from dataclasses import dataclass, field import hashlib import os -from pathlib import Path import re +from dataclasses import dataclass +from pathlib import Path from typing import Any from cops.evidence.canonical import digest, utc_now + from .redaction import StreamRedactor diff --git a/cops/execution/redaction.py b/cops/execution/redaction.py index 2e34223..c745165 100644 --- a/cops/execution/redaction.py +++ b/cops/execution/redaction.py @@ -7,8 +7,7 @@ from __future__ import annotations import re -from typing import Sequence - +from collections.abc import Sequence # Common credential and token regex patterns DEFAULT_SENSITIVE_PATTERNS: list[tuple[str, re.Pattern[str]]] = [ diff --git a/cops/execution/scope_guard.py b/cops/execution/scope_guard.py index aa787be..b842587 100644 --- a/cops/execution/scope_guard.py +++ b/cops/execution/scope_guard.py @@ -8,10 +8,10 @@ from __future__ import annotations import ipaddress -import re import socket +from collections.abc import Callable from dataclasses import dataclass, field -from typing import Any, Callable, Sequence +from typing import Any from urllib.parse import urlparse diff --git a/cops/execution/store.py b/cops/execution/store.py index ec1d90f..ce70c64 100644 --- a/cops/execution/store.py +++ b/cops/execution/store.py @@ -8,16 +8,11 @@ import os import sqlite3 -import sys -from dataclasses import dataclass -from datetime import datetime, timezone from pathlib import Path -from typing import Any, Sequence from cops.contracts.models import ExecutionAuthorization from cops.contracts.validation import validate_contract -from cops.evidence.canonical import canonical, digest, timestamp, utc_now -from .authorization import AuthorizationError, compute_authorization_signature +from cops.evidence.canonical import canonical, timestamp, utc_now class ApprovalStoreError(ValueError): @@ -304,7 +299,8 @@ def atomically_consume( if not committed: try: conn.execute("ROLLBACK") - except Exception: + except sqlite3.Error: + # Preserve the original transaction failure if rollback also fails. pass raise diff --git a/cops/execution/worker.py b/cops/execution/worker.py index 8c6746b..2ec0007 100644 --- a/cops/execution/worker.py +++ b/cops/execution/worker.py @@ -11,24 +11,23 @@ from __future__ import annotations -import json import os import platform import shutil import subprocess -import sys import tempfile import uuid from dataclasses import dataclass, field -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path -from typing import Any, Sequence +from typing import Any from cops.contracts.models import ActionPlan, ExecutionAuthorization, RunResult from cops.contracts.validation import validate_contract from cops.evidence.canonical import digest, utc_now -from .authorization import AuthorizationError, verify_execution_authorization -from .store import ApprovalStore, ApprovalStoreError + +from .authorization import verify_execution_authorization +from .store import ApprovalStore, ApprovalStoreConflictError class WorkerError(RuntimeError): @@ -113,13 +112,15 @@ def execute_plan( auth_model = ExecutionAuthorization.from_dict(authorization) try: self.store.store_authorization(auth_model) - except Exception: + except ApprovalStoreConflictError: + # Existing approvals are consumed atomically below, preserving replay checks. pass else: auth_model = authorization try: self.store.store_authorization(auth_model) - except Exception: + except ApprovalStoreConflictError: + # Existing approvals are consumed atomically below, preserving replay checks. pass # 3. Cryptographically verify authorization against plan and worker identity @@ -181,7 +182,7 @@ def execute_plan( max_duration_seconds = plan_model.limits.get("max_duration_seconds", self.config.max_wall_time_seconds) max_output_bytes = plan_model.limits.get("max_output_bytes", self.config.max_output_bytes) - start_dt = datetime.now(timezone.utc) + start_dt = datetime.now(UTC) clean_env = { "PATH": os.environ.get("PATH", "/usr/bin:/bin"), @@ -253,7 +254,7 @@ def execute_plan( break # Check overall plan duration limit - elapsed_seconds = (datetime.now(timezone.utc) - start_dt).total_seconds() + elapsed_seconds = (datetime.now(UTC) - start_dt).total_seconds() if elapsed_seconds >= max_duration_seconds: status = "partial" status_reason = f"operation exceeded action plan max_duration_seconds limit ({max_duration_seconds}s)" @@ -295,10 +296,10 @@ def execute_plan( try: if tool == "inert": # Simulated execution for testing - stdout_bytes = f"Inert step {step_id} executed successfully: {action}".encode("utf-8") + stdout_bytes = f"Inert step {step_id} executed successfully: {action}".encode() exit_code = 0 else: - from cops.adapters import ToolAdapterRegistry, AdapterError + from cops.adapters import AdapterError, ToolAdapterRegistry registry = ToolAdapterRegistry() if tool in registry.list_tools(): adapter = registry.get_adapter(tool) @@ -392,7 +393,8 @@ def execute_plan( if ephemeral and target_workspace.exists(): try: shutil.rmtree(target_workspace, ignore_errors=False) - except Exception: + except OSError: + # The existence check below records an unsuccessful cleanup. pass if target_workspace.exists(): cleanup_status = "failed" diff --git a/cops/mcp_validation.py b/cops/mcp_validation.py index 6dedaf5..fc89e68 100644 --- a/cops/mcp_validation.py +++ b/cops/mcp_validation.py @@ -8,7 +8,6 @@ from typing import Any from urllib.parse import urlsplit - MCP_SCHEMA = "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json" EXECUTABLE = re.compile(r"^[A-Za-z0-9_.+-]+$") HEADER = re.compile(r"^[!#$%&'*+.^_`|~0-9A-Za-z-]+$") diff --git a/cops/portable.py b/cops/portable.py index d3c30df..ccda15f 100644 --- a/cops/portable.py +++ b/cops/portable.py @@ -14,7 +14,6 @@ from .prerequisites import validate_prerequisites from .validation import ValidationError, _skill_frontmatter, validate_agent_plugin_manifest - # Every new root entry must be deliberately classified before packaging. PORTABLE_ENTRIES = { "plugin.json", "mcp.json", "skills", "scripts", "docs", "examples", "LICENSE", "README.md", diff --git a/cops/prerequisites.py b/cops/prerequisites.py index c1bc888..d53213a 100644 --- a/cops/prerequisites.py +++ b/cops/prerequisites.py @@ -6,8 +6,8 @@ import shutil import subprocess import sys -from typing import Any, Callable - +from collections.abc import Callable +from typing import Any MANAGERS = { "darwin": ("brew",), diff --git a/cops/routing/__init__.py b/cops/routing/__init__.py index 5f08b18..1cef5ee 100644 --- a/cops/routing/__init__.py +++ b/cops/routing/__init__.py @@ -2,9 +2,6 @@ from .catalog import get_specialist, load_specialists_registry from .classifier import route_request -from .models import RoutingDecision, SpecialistProfile, TriadExecutionPlan, TriadMember -from .triad import assemble_triad_plan - from .handoff import ( AuthorizationExpansionError, ConflictingEvidenceError, @@ -19,6 +16,8 @@ propose_specialist_handoff, review_with_skeptic, ) +from .models import RoutingDecision, SpecialistProfile, TriadExecutionPlan, TriadMember +from .triad import assemble_triad_plan __all__ = [ "SpecialistProfile", diff --git a/cops/routing/catalog.py b/cops/routing/catalog.py index 606f9e9..f4f432a 100644 --- a/cops/routing/catalog.py +++ b/cops/routing/catalog.py @@ -4,7 +4,6 @@ import json from pathlib import Path -from typing import Sequence from .models import SpecialistProfile diff --git a/cops/routing/classifier.py b/cops/routing/classifier.py index 86050bf..27ee0a9 100644 --- a/cops/routing/classifier.py +++ b/cops/routing/classifier.py @@ -4,7 +4,6 @@ import re from pathlib import Path -from typing import Sequence from .catalog import load_specialists_registry from .models import RoutingDecision, SpecialistProfile diff --git a/cops/routing/cli.py b/cops/routing/cli.py index 3634829..c2d34dd 100644 --- a/cops/routing/cli.py +++ b/cops/routing/cli.py @@ -5,10 +5,10 @@ import argparse import json import sys -from pathlib import Path -from typing import Sequence +from collections.abc import Sequence from cops.authorization import ensure_authorization + from .catalog import get_specialist, load_specialists_registry from .classifier import route_request diff --git a/cops/routing/triad.py b/cops/routing/triad.py index b287f39..6f6ffad 100644 --- a/cops/routing/triad.py +++ b/cops/routing/triad.py @@ -63,12 +63,12 @@ def assemble_triad_plan( handoff_steps = ( f"1. Task received: Router assigns Primary Specialist ({primary_profile.id}).", - f"2. Primary executes deterministic offline tooling and drafts initial findings/plan.", + "2. Primary executes deterministic offline tooling and drafts initial findings/plan.", f"3. Primary hands candidate artifact to Domain Skeptic ({skeptic_profile.id}) to critique assumptions.", - f"4. Skeptic reviews transitions; disputed claims are labeled 'candidate' or 'invalid'.", + "4. Skeptic reviews transitions; disputed claims are labeled 'candidate' or 'invalid'.", f"5. Primary and Skeptic submit reconciled package to Evidence Auditor ({auditor_profile.id}).", - f"6. Auditor verifies SHA-256 evidence envelopes and operator authorization receipt.", - f"7. Auditor seals and issues the final, tamper-evident deliverable.", + "6. Auditor verifies SHA-256 evidence envelopes and operator authorization receipt.", + "7. Auditor seals and issues the final, tamper-evident deliverable.", ) return TriadExecutionPlan( diff --git a/cops/scenarios/registry.py b/cops/scenarios/registry.py index 3f2a892..7538de3 100644 --- a/cops/scenarios/registry.py +++ b/cops/scenarios/registry.py @@ -6,7 +6,6 @@ from pathlib import Path from typing import Any -from cops.contracts.lifecycle import ContractError from cops.contracts.validation import validate_identifier from cops.evidence.canonical import EvidenceError, canonical from cops.evidence.validation import _check diff --git a/cops/validation.py b/cops/validation.py index 88abb83..c452cf8 100644 --- a/cops/validation.py +++ b/cops/validation.py @@ -10,7 +10,6 @@ from .catalog import CatalogError, PluginRecord, load_json, plugin_records, validate_declared_command from .prerequisites import validate_prerequisites - ROOT = Path(__file__).resolve().parents[1] AGENT_PLUGIN_SCHEMA = "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json" AGENT_PLUGIN_KEYS = { diff --git a/docs/_config.yml b/docs/_config.yml index d227a23..0cd308b 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -1,6 +1,9 @@ # Configures the published Jekyll documentation site for COPS. title: COPS Documentation -description: Copilot Operations Plugins for Security — Universal catalog of offline-first defensive cybersecurity plugins, 18 specialist AI agent profiles, and deterministic verification tools for GitHub Copilot, Claude Code, and Codex. +description: >- + Copilot Operations Plugins for Security — Universal catalog of offline-first defensive cybersecurity + plugins, 18 specialist AI agent profiles, and deterministic verification tools for GitHub Copilot, + Claude Code, and Codex. url: "https://sodejm.github.io" baseurl: "/copilot-operation-plugin-for-security" theme: jekyll-theme-cayman diff --git a/docs/contributing.md b/docs/contributing.md index 8f6e011..c363a23 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -76,6 +76,10 @@ make check PYTHON=.venv/bin/python make check-issue-coverage ``` +Python changes must also pass `ruff check .`; YAML changes must pass `yamllint -s .`, matching the hosted quality gate. Install these tools in the development environment before running them. After changing canonical skills or portable evidence sources, regenerate adapters with `make sync-agent-adapters` and the evidence bundle with `python scripts/agent/bundle_evidence.py`, then rerun validation. + +Security lint exceptions must describe the concrete reason at the narrowest applicable scope. Offline tests use synthetic credentials and noncryptographic seeded fuzzing; standalone scripts may set up the repository import path before imports. Public string-enum behavior is preserved rather than migrated solely to satisfy a style rule. Production authorization storage errors must propagate, and XML imports must reject entity declarations before parsing imported data. + ### Issue Documentation & Test Coverage Requirements Every branch addressing an issue or feature must include: 1. **Documentation**: Updated or newly created markdown files in `docs/`, `specs/`, or root guides. @@ -161,3 +165,5 @@ For air-gapped or restricted network development environments: make check-prerequisites make check PYTHON=.venv/bin/python ``` + +Workflow maintenance keeps checkout actions aligned with the current hosted runner runtime. The dependency pull requests update the remaining Python, CodeQL, and Pages actions; shell values used by issue summaries are quoted before passing them to the GitHub CLI. diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/__init__.py b/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/__init__.py index 45cebcc..1c2cc0d 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/__init__.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/__init__.py @@ -1,8 +1,8 @@ """Shared evidence contracts; portable consumers use the generated approved subset.""" +from .assessment import assess, report from .canonical import EvidenceError, canonical, decode_json from .contract import build_envelope from .validation import validate_envelope, validate_receipt -from .assessment import assess, report __all__ = ["EvidenceError", "canonical", "decode_json", "build_envelope", "validate_envelope", "validate_receipt", "assess", "report"] diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/assessment.py b/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/assessment.py index e4f1af8..5f1f64c 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/assessment.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/assessment.py @@ -1,5 +1,6 @@ """Provider-independent receipt and observation assessment; reports omit payloads.""" import math + from .canonical import EvidenceError, timestamp from .validation import validate_envelope, validate_receipt diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/canonical.py b/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/canonical.py index 87a285f..22e079d 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/canonical.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/canonical.py @@ -1,9 +1,9 @@ """Bounded canonical JSON; error messages never include untrusted values.""" -from datetime import datetime, timezone import hashlib import json import math import re +from datetime import UTC, datetime class EvidenceError(ValueError): @@ -24,7 +24,7 @@ def timestamp(value): def utc_now(): - return datetime.now(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + return datetime.now(UTC).isoformat(timespec="microseconds").replace("+00:00", "Z") def canonical(value, *, max_bytes=1024 * 1024, max_depth=32): diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/validation.py b/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/validation.py index e482d02..650b870 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/validation.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/cops/evidence/validation.py @@ -1,7 +1,7 @@ """Validate the published, deliberately small JSON Schema vocabulary without deps.""" import json -from pathlib import Path import re +from pathlib import Path from .canonical import EvidenceError, canonical, digest, timestamp diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/source-manifest.json b/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/source-manifest.json index c9ef32b..e9ca952 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/source-manifest.json +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/_runtime/source-manifest.json @@ -7,11 +7,11 @@ "sources": [ { "path": "cops/evidence/__init__.py", - "sha256": "f104b285bb1202f9b9447b2cde2b8ba8e1ea289a83213f28b21c5e658fd2e480" + "sha256": "a270427396430d20d22751e74c11cc4fa84e081a59eb3f4c68022f8d19be944f" }, { "path": "cops/evidence/canonical.py", - "sha256": "4f8a90a6738995313a2a0fc4723d15d062c18df3842d559d1a329594b68d2bae" + "sha256": "2aee3b4213ddf4668e76228ecacfc91bd8e0a7aac868414a4d308f62179f0753" }, { "path": "cops/evidence/contract.py", @@ -19,11 +19,11 @@ }, { "path": "cops/evidence/validation.py", - "sha256": "0fc330b7c7bef14153791fb0c0d2ba6d61da9c86848a6824314bf784ce80122a" + "sha256": "30b9369cca4872d014c48fe885e75578f1d995f3331243f304b790abd9641e44" }, { "path": "cops/evidence/assessment.py", - "sha256": "a4c70a6465d2bbaaafc4d1fdff71da7cd3739880e465916c36a03ff45eff7448" + "sha256": "643db0ceab7701993ee6084e48a7a4f13f9173fcb51a59708156894f0717b5c5" }, { "path": "catalog/schemas/evidence-envelope.schema.json", diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/collection.py b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/collection.py index 527c23c..e34506b 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/collection.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/collection.py @@ -1,8 +1,9 @@ """Pinned, read-only collection intentions. This module performs no HTTP requests.""" import json from urllib.parse import quote + from .input import fields, json_value, read_regular -from .model import AzureError, arm, stable, object_id +from .model import AzureError, arm, object_id, stable GRAPH = "MicrosoftGraph" ARM = "AzureResourceManager" diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/identity.py b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/identity.py index 3b532cb..6811f92 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/identity.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/identity.py @@ -1,6 +1,7 @@ """Direct membership witnesses and temporal/PIM decisions.""" -from .model import Decision + from .._runtime.cops.evidence.canonical import timestamp +from .model import Decision def find(graph, family, tenant, predicate): @@ -15,15 +16,15 @@ def membership(graph, tenant, principal): found = {principal.lower(): ([], [])} queue = [(principal.lower(), 0)] for child, depth in queue: - for row in graph.rows('group_members', tenant): + for row in graph.rows("group_members", tenant): graph.tick() - if row.data['id'].lower() != child or row.quality: + if row.data["id"].lower() != child or row.quality: continue - parent = row.data['groupId'].lower() + parent = row.data["groupId"].lower() if parent in found: continue - if depth >= graph.limits['hops']: - graph.partial.append('membership_depth_limit') + if depth >= graph.limits["hops"]: + graph.partial.append("membership_depth_limit") continue refs, cuts = found[child] found[parent] = (refs + row.evidence, cuts + [row.key]) @@ -34,90 +35,149 @@ def membership(graph, tenant, principal): def temporal(graph, row, scenario=None): p = row.properties if row.quality: - return Decision('unknown', row.quality, row.evidence, [row.key]) + return Decision("unknown", row.quality, row.evidence, [row.key]) now = timestamp(graph.as_of) - for name, sign in (('startDateTime', 1), ('endDateTime', -1)): + for name, sign in (("startDateTime", 1), ("endDateTime", -1)): if p.get(name): value = timestamp(p[name]) if (sign == 1 and value > now) or (sign == -1 and value <= now): - return Decision('denied', ['future_or_expired'], row.evidence, [row.key]) - if row.family != 'pim_eligible': - return Decision('allowed', [], row.evidence, [row.key]) + return Decision("denied", ["future_or_expired"], row.evidence, [row.key]) + if row.family != "pim_eligible": + return Decision("allowed", [], row.evidence, [row.key]) refs, cuts = list(row.evidence), [row.key] - policy_id = p.get('policyId') + policy_id = p.get("policyId") if not policy_id: - assignment = find(graph, 'pim_policy_assignments', row.tenant, lambda r: - not r.quality and r.data.get('plane') == row.data.get('plane') and r.properties.get('roleDefinitionId', '').lower() == p['roleDefinitionId'].lower() - and str(r.properties.get('scope', r.properties.get('scopeId'))).lower() == str(p.get('scope', p.get('directoryScopeId'))).lower()) + assignment = find( + graph, + "pim_policy_assignments", + row.tenant, + lambda r: ( + not r.quality + and r.data.get("plane") == row.data.get("plane") + and r.properties.get("roleDefinitionId", "").lower() == p["roleDefinitionId"].lower() + and str(r.properties.get("scope", r.properties.get("scopeId"))).lower() + == str(p.get("scope", p.get("directoryScopeId"))).lower() + ), + ) if assignment: - policy_id = assignment.properties.get('policyId') + policy_id = assignment.properties.get("policyId") refs += assignment.evidence cuts.append(assignment.key) - policy = find(graph, 'pim_policies', row.tenant, lambda r: r.data['id'].lower() == (policy_id or '').lower() and not r.quality and r.data.get('plane') == row.data.get('plane')) + policy = find( + graph, + "pim_policies", + row.tenant, + lambda r: ( + r.data["id"].lower() == (policy_id or "").lower() + and not r.quality + and r.data.get("plane") == row.data.get("plane") + ), + ) if policy is None: - return Decision('latent_eligibility', ['activation_policy_unknown'], refs, cuts) - requirements = policy.data.get('requirements', {}) - activation = (scenario or {}).get('activations', {}).get(row.data['id'], {}) - required = ('approval', 'mfa', 'authentication_context') - known = all(type(requirements.get(k)) is bool for k in required) and type(requirements.get('max_duration_minutes')) is int - fulfilled = (known and not requirements.get('extra_requirements', True) and type(activation.get('duration_minutes')) is int - and 0 < activation['duration_minutes'] <= requirements['max_duration_minutes'] - and all(not requirements[k] or activation.get(k) is True for k in required)) - return Decision('allowed' if fulfilled else 'latent_eligibility', - [] if fulfilled else ['activation_requirements_unfulfilled'], - refs + policy.evidence, cuts + [policy.key], - ['activation:' + row.key] if fulfilled else []) + return Decision("latent_eligibility", ["activation_policy_unknown"], refs, cuts) + requirements = policy.data.get("requirements", {}) + activation = (scenario or {}).get("activations", {}).get(row.data["id"], {}) + required = ("approval", "mfa", "authentication_context") + known = ( + all(type(requirements.get(k)) is bool for k in required) + and type(requirements.get("max_duration_minutes")) is int + ) + fulfilled = ( + known + and not requirements.get("extra_requirements", True) + and type(activation.get("duration_minutes")) is int + and 0 < activation["duration_minutes"] <= requirements["max_duration_minutes"] + and all(not requirements[k] or activation.get(k) is True for k in required) + ) + return Decision( + "allowed" if fulfilled else "latent_eligibility", + [] if fulfilled else ["activation_requirements_unfulfilled"], + refs + policy.evidence, + cuts + [policy.key], + ["activation:" + row.key] if fulfilled else [], + ) def directory(graph, tenant, principal, action, scope, scenario=None): principals = {principal.lower(): ([], [])} - for member in graph.rows('group_members', tenant): + for member in graph.rows("group_members", tenant): graph.tick() - if member.data['id'].lower() != principal.lower() or member.quality: + if member.data["id"].lower() != principal.lower() or member.quality: continue - group = find(graph, 'groups', tenant, lambda g: g.data['id'].lower() == member.data.get('groupId', '').lower()) - if group and not group.quality and group.data.get('isAssignableToRole') is True: - principals[group.data['id'].lower()] = (member.evidence + group.evidence, [member.key, group.key]) + group = find( + graph, + "groups", + tenant, + lambda g, member=member: g.data["id"].lower() == member.data.get("groupId", "").lower(), + ) + if group and not group.quality and group.data.get("isAssignableToRole") is True: + principals[group.data["id"].lower()] = (member.evidence + group.evidence, [member.key, group.key]) witnesses, unknown, latent = [], [], [] - for row in graph.rows('directory_assignments', tenant) + graph.rows('pim_eligible', tenant) + graph.rows('pim_active', tenant): + for row in ( + graph.rows("directory_assignments", tenant) + + graph.rows("pim_eligible", tenant) + + graph.rows("pim_active", tenant) + ): graph.tick() p = row.properties - if p.get('directoryScopeId') is None or p.get('principalId', '').lower() not in principals: + if p.get("directoryScopeId") is None or p.get("principalId", "").lower() not in principals: continue - assignment_scope = p['directoryScopeId'] + assignment_scope = p["directoryScopeId"] # Administrative units contain users, groups and devices, never application objects. - if (assignment_scope.lower().startswith('/administrativeunits/') - and action.lower().startswith(('microsoft.directory/applications/', - 'microsoft.directory/serviceprincipals/'))): + if assignment_scope.lower().startswith("/administrativeunits/") and action.lower().startswith( + ("microsoft.directory/applications/", "microsoft.directory/serviceprincipals/") + ): unknown += row.evidence continue scope_refs, scope_cuts = [], [] - if assignment_scope.lower() not in ('/', scope.lower()): - if not assignment_scope.lower().startswith('/administrativeunits/'): + if assignment_scope.lower() not in ("/", scope.lower()): + if not assignment_scope.lower().startswith("/administrativeunits/"): continue - unit = assignment_scope.split('/')[-1].lower() - member = find(graph, 'administrative_members', tenant, lambda r: not r.quality - and r.data['objectId'].lower() == unit and '/' + r.data['id'].lower() == scope.lower()) + unit = assignment_scope.split("/")[-1].lower() + member = find( + graph, + "administrative_members", + tenant, + lambda r, unit=unit: ( + not r.quality and r.data["objectId"].lower() == unit and "/" + r.data["id"].lower() == scope.lower() + ), + ) if not member: continue scope_refs, scope_cuts = member.evidence, [member.key] - definition = find(graph, 'directory_definitions', tenant, lambda r: r.data['id'].lower() == p['roleDefinitionId'].lower()) + definition = find( + graph, "directory_definitions", tenant, lambda r, p=p: r.data["id"].lower() == p["roleDefinitionId"].lower() + ) if not definition or definition.quality: unknown += row.evidence continue # Only exact actions and the universal wildcard have established semantics. - actions = [a.lower() for item in definition.data.get('rolePermissions', []) for a in item.get('allowedResourceActions', [])] - if action.lower() not in actions and '*' not in actions: - if any('*' in a for a in actions): + actions = [ + a.lower() + for item in definition.data.get("rolePermissions", []) + for a in item.get("allowedResourceActions", []) + ] + if action.lower() not in actions and "*" not in actions: + if any("*" in a for a in actions): unknown += definition.evidence continue valid = temporal(graph, row, scenario) - refs, cuts = principals[p['principalId'].lower()] - d = Decision(valid.state, valid.reasons, valid.evidence + definition.evidence + refs + scope_refs, - valid.cuts + [definition.key] + cuts + scope_cuts, valid.assumptions) - if d.state == 'allowed': witnesses.append(d) - elif d.state == 'latent_eligibility': latent.append(d) - elif d.state == 'unknown': unknown += d.evidence - if witnesses: return sorted(witnesses, key=lambda d: d.evidence)[0] - if latent: return latent[0] - return Decision('unknown', ['directory_permission_not_established'], unknown) + refs, cuts = principals[p["principalId"].lower()] + d = Decision( + valid.state, + valid.reasons, + valid.evidence + definition.evidence + refs + scope_refs, + valid.cuts + [definition.key] + cuts + scope_cuts, + valid.assumptions, + ) + if d.state == "allowed": + witnesses.append(d) + elif d.state == "latent_eligibility": + latent.append(d) + elif d.state == "unknown": + unknown += d.evidence + if witnesses: + return sorted(witnesses, key=lambda d: d.evidence)[0] + if latent: + return latent[0] + return Decision("unknown", ["directory_permission_not_established"], unknown) diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/input.py b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/input.py index 8d2f42c..78f1c18 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/input.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/input.py @@ -1,11 +1,13 @@ """Bounded descriptor-based bundle ingestion with validated SDK provenance.""" import hashlib -from pathlib import Path import re +from pathlib import Path + from .._runtime.cops.evidence import assess, validate_receipt -from .._runtime.cops.evidence.canonical import digest, EvidenceError -from ..ingestion import (DEFAULTS as INGEST_DEFAULTS, IngestError, Limits, RunBudget, - iter_jsonl, parse_json, read_regular as bounded_read) +from .._runtime.cops.evidence.canonical import EvidenceError, digest +from ..ingestion import DEFAULTS as INGEST_DEFAULTS +from ..ingestion import IngestError, Limits, RunBudget, iter_jsonl, parse_json +from ..ingestion import read_regular as bounded_read from .model import AzureError, Budget, Graph, arm, object_id from .normalize import FAMILIES, normalize diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/model.py b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/model.py index 3d0cecd..eb76082 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/model.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/model.py @@ -1,9 +1,11 @@ """Tenant-qualified observations, provenance, bounded work, and scope ancestry.""" from __future__ import annotations -from dataclasses import dataclass, field + import hashlib import os +from dataclasses import dataclass, field from pathlib import Path + from .._runtime.cops.evidence.canonical import canonical, timestamp @@ -196,7 +198,7 @@ def export(self): "role_definitions": "role_definition", "directory_definitions": "directory_role", "administrative_units": "administrative_unit", "federated_credentials": "federated_credential", "lighthouse": "delegation", "scope_parents": "scope"} - observed = {l["record_id"]: l["observed_at"] for l in self.ledger} + observed = {entry["record_id"]: entry["observed_at"] for entry in self.ledger} def node(identity, tenant, kind, row): if identity not in nodes: diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/normalize.py b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/normalize.py index 597a995..d126bbf 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/normalize.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/normalize.py @@ -1,46 +1,135 @@ """Allowlisted projections of documented Graph/ARM pages; never settings/secrets.""" + from .model import AzureError, arm -GRAPH_FAMILIES = {'users', 'groups', 'group_members', 'applications', 'service_principals', - 'owners', 'federated_credentials', 'directory_assignments', 'directory_definitions', - 'administrative_units', 'administrative_members', 'app_role_assignments', 'conditional_access', - 'authentication_strengths', 'directory_pim_eligible', 'directory_pim_active', - 'directory_pim_policies', 'directory_pim_policy_assignments'} -ARM_FAMILIES = {'resources', 'role_assignments', 'role_definitions', 'deny_assignments', - 'scope_parents', 'azure_pim_eligible', 'azure_pim_active', 'azure_pim_policies', - 'azure_pim_policy_assignments', 'lighthouse', 'lighthouse_assignments'} +GRAPH_FAMILIES = { + "users", + "groups", + "group_members", + "applications", + "service_principals", + "owners", + "federated_credentials", + "directory_assignments", + "directory_definitions", + "administrative_units", + "administrative_members", + "app_role_assignments", + "conditional_access", + "authentication_strengths", + "directory_pim_eligible", + "directory_pim_active", + "directory_pim_policies", + "directory_pim_policy_assignments", +} +ARM_FAMILIES = { + "resources", + "role_assignments", + "role_definitions", + "deny_assignments", + "scope_parents", + "azure_pim_eligible", + "azure_pim_active", + "azure_pim_policies", + "azure_pim_policy_assignments", + "lighthouse", + "lighthouse_assignments", +} FAMILIES = GRAPH_FAMILIES | ARM_FAMILIES # Nested projection uses the same explicit key allowlist. Credential values and # app settings are intentionally absent even when an export contains them. -FIELDS = {'id', 'appId', 'appOwnerOrganizationId', 'servicePrincipalType', 'isAssignableToRole', - 'principalId', 'principalType', 'roleDefinitionId', 'scope', 'directoryScopeId', 'appScopeId', - 'condition', 'conditionVersion', 'permissions', 'actions', 'notActions', 'dataActions', - 'notDataActions', 'assignableScopes', 'roleType', 'type', 'properties', 'identity', - 'tenantId', 'userAssignedIdentities', 'clientId', 'startDateTime', 'endDateTime', - 'rolePermissions', 'allowedResourceActions', 'issuer', 'subject', 'audiences', - 'parentId', 'parent', 'details', 'principals', 'excludePrincipals', - 'doNotApplyToChildScopes', 'enableRbacAuthorization', 'keyVaultPermissionModel', - 'kind', 'state', 'managedByTenantId', 'authorizations', 'eligibleAuthorizations', - 'delegatedRoleDefinitionIds', 'registrationDefinitionId', 'policyId', 'scopeId', 'scopeType', 'rules', - 'target', 'caller', 'level', 'operations', 'enabledRules', 'maximumDuration', - 'isApprovalRequired', 'approvalSettings', 'isEnabled', 'claimValue', - 'setting', 'rules', 'signInAudience', 'accessPolicies', 'objectId', 'resourceId', 'appRoleId', 'policyAssignmentId', - 'authenticationStrength', 'grantControls', 'builtInControls', 'conditions'} +FIELDS = { + "id", + "appId", + "appOwnerOrganizationId", + "servicePrincipalType", + "isAssignableToRole", + "principalId", + "principalType", + "roleDefinitionId", + "scope", + "directoryScopeId", + "appScopeId", + "condition", + "conditionVersion", + "permissions", + "actions", + "notActions", + "dataActions", + "notDataActions", + "assignableScopes", + "roleType", + "type", + "properties", + "identity", + "tenantId", + "userAssignedIdentities", + "clientId", + "startDateTime", + "endDateTime", + "rolePermissions", + "allowedResourceActions", + "issuer", + "subject", + "audiences", + "parentId", + "parent", + "details", + "principals", + "excludePrincipals", + "doNotApplyToChildScopes", + "enableRbacAuthorization", + "keyVaultPermissionModel", + "kind", + "state", + "managedByTenantId", + "authorizations", + "eligibleAuthorizations", + "delegatedRoleDefinitionIds", + "registrationDefinitionId", + "policyId", + "scopeId", + "scopeType", + "rules", + "target", + "caller", + "level", + "operations", + "enabledRules", + "maximumDuration", + "isApprovalRequired", + "approvalSettings", + "isEnabled", + "claimValue", + "setting", + "signInAudience", + "accessPolicies", + "objectId", + "resourceId", + "appRoleId", + "policyAssignmentId", + "authenticationStrength", + "grantControls", + "builtInControls", + "conditions", +} def project(value, key=None): if isinstance(value, dict): - if key == 'userAssignedIdentities': + if key == "userAssignedIdentities": return {arm(k): project(v) for k, v in value.items()} return {k: project(v, k) for k, v in value.items() if k in FIELDS and v is not None} if isinstance(value, list): - if key == 'accessPolicies': + if key == "accessPolicies": policies = [] for item in value: - if not isinstance(item, dict) or not isinstance(item.get('permissions'), dict): - raise AzureError('invalid_projection_field') + if not isinstance(item, dict) or not isinstance(item.get("permissions"), dict): + raise AzureError("invalid_projection_field") policy = project(item) - policy['permissions'] = {k: v for k, v in item['permissions'].items() if k in ('secrets', 'keys', 'certificates')} + policy["permissions"] = { + k: v for k, v in item["permissions"].items() if k in ("secrets", "keys", "certificates") + } policies.append(policy) return policies return [project(v, key) for v in value] @@ -49,160 +138,226 @@ def project(value, key=None): def policy_requirements(data): requirements = {} - for rule in data.get('rules', []): - target = rule.get('target', {}) - if target.get('caller') != 'EndUser' or target.get('level') != 'Assignment': + for rule in data.get("rules", []): + target = rule.get("target", {}) + if target.get("caller") != "EndUser" or target.get("level") != "Assignment": continue - rid = rule.get('id', '') - if rid.startswith('Enablement_'): - enabled = rule.get('enabledRules') - if isinstance(enabled, list) and set(enabled) <= {'MultiFactorAuthentication', 'Justification', 'Ticketing'}: - requirements['mfa'] = 'MultiFactorAuthentication' in enabled - requirements['extra_requirements'] = bool(set(enabled) - {'MultiFactorAuthentication'}) - if rid.startswith('Approval_'): - setting = rule.get('setting', rule.get('approvalSettings', {})) - if type(setting.get('isApprovalRequired')) is bool: - requirements['approval'] = setting['isApprovalRequired'] - if rid.startswith('AuthenticationContext_') and type(rule.get('isEnabled')) is bool: - requirements['authentication_context'] = rule['isEnabled'] - if rid.startswith('Expiration_'): + rid = rule.get("id", "") + if rid.startswith("Enablement_"): + enabled = rule.get("enabledRules") + if isinstance(enabled, list) and set(enabled) <= { + "MultiFactorAuthentication", + "Justification", + "Ticketing", + }: + requirements["mfa"] = "MultiFactorAuthentication" in enabled + requirements["extra_requirements"] = bool(set(enabled) - {"MultiFactorAuthentication"}) + if rid.startswith("Approval_"): + setting = rule.get("setting", rule.get("approvalSettings", {})) + if type(setting.get("isApprovalRequired")) is bool: + requirements["approval"] = setting["isApprovalRequired"] + if rid.startswith("AuthenticationContext_") and type(rule.get("isEnabled")) is bool: + requirements["authentication_context"] = rule["isEnabled"] + if rid.startswith("Expiration_"): import re - match = re.fullmatch(r'PT(?:(\d{1,4})H)?(?:(\d{1,4})M)?', rule.get('maximumDuration', '')) + + match = re.fullmatch(r"PT(?:(\d{1,4})H)?(?:(\d{1,4})M)?", rule.get("maximumDuration", "")) if match: - requirements['max_duration_minutes'] = int(match[1] or 0) * 60 + int(match[2] or 0) + requirements["max_duration_minutes"] = int(match[1] or 0) * 60 + int(match[2] or 0) return requirements def normalize(graph, source, envelope, quality): - payload = envelope['payload'] - family, tenant = source['family'], source['tenant'] - if not isinstance(payload, dict) or not isinstance(payload.get('value'), list): - raise AzureError('unsupported_page_shape') - context = source.get('context', {}) - for raw in payload['value']: + payload = envelope["payload"] + family, tenant = source["family"], source["tenant"] + if not isinstance(payload, dict) or not isinstance(payload.get("value"), list): + raise AzureError("unsupported_page_shape") + context = source.get("context", {}) + for raw in payload["value"]: graph.tick() if not isinstance(raw, dict): - raise AzureError('invalid_record_shape') + raise AzureError("invalid_record_shape") data = project(raw) - if not isinstance(data.get('id'), str): - raise AzureError('missing_object_id') - if family == 'group_members': - data['groupId'] = context['groupId'] - if family in ('owners', 'administrative_members', 'federated_credentials'): - data['objectId'] = context['objectId'] + if not isinstance(data.get("id"), str): + raise AzureError("missing_object_id") + if family == "group_members": + data["groupId"] = context["groupId"] + if family in ("owners", "administrative_members", "federated_credentials"): + data["objectId"] = context["objectId"] normalized = family - if '_pim_' in family: - data['plane'] = 'directory' if family.startswith('directory_') else 'azure' - if family.endswith('_pim_eligible'): - normalized = 'pim_eligible' - elif family.endswith('_pim_active'): - normalized = 'pim_active' - elif family.endswith('_pim_policy_assignments'): - normalized = 'pim_policy_assignments' - elif family.endswith('_pim_policies'): - normalized = 'pim_policies' + if "_pim_" in family: + data["plane"] = "directory" if family.startswith("directory_") else "azure" + if family.endswith("_pim_eligible"): + normalized = "pim_eligible" + elif family.endswith("_pim_active"): + normalized = "pim_active" + elif family.endswith("_pim_policy_assignments"): + normalized = "pim_policy_assignments" + elif family.endswith("_pim_policies"): + normalized = "pim_policies" # Preserve documented rules and derive only fully supported requirements. validate_projection(normalized, data) - data['requirements'] = policy_requirements(data.get('properties', data)) - if family == 'resources' and data.get('id', '').startswith('/'): - data['id'] = arm(data['id']) + data["requirements"] = policy_requirements(data.get("properties", data)) + if family == "resources" and data.get("id", "").startswith("/"): + data["id"] = arm(data["id"]) from .collection import RESOURCE_APIS - if RESOURCE_APIS.get(data.get('type', '').lower()) != source['api']: - raise AzureError('resource_api_mismatch') + + if RESOURCE_APIS.get(data.get("type", "").lower()) != source["api"]: + raise AzureError("resource_api_mismatch") validate_projection(normalized, data) - graph.add(normalized, tenant, data, envelope['record_id'], quality) + graph.add(normalized, tenant, data, envelope["record_id"], quality) return not quality def validate_projection(family, data): """Malformed nested authorization metadata cannot become a guessed grant.""" - p = data.get('properties', data) + p = data.get("properties", data) if not isinstance(p, dict): - raise AzureError('invalid_properties') + raise AzureError("invalid_properties") required = { - 'role_assignments': ('principalId', 'roleDefinitionId', 'scope'), - 'directory_assignments': ('principalId', 'roleDefinitionId', 'directoryScopeId'), - 'scope_parents': ('parentId',), - 'pim_eligible': ('principalId', 'roleDefinitionId'), - 'pim_active': ('principalId', 'roleDefinitionId'), + "role_assignments": ("principalId", "roleDefinitionId", "scope"), + "directory_assignments": ("principalId", "roleDefinitionId", "directoryScopeId"), + "scope_parents": ("parentId",), + "pim_eligible": ("principalId", "roleDefinitionId"), + "pim_active": ("principalId", "roleDefinitionId"), }.get(family, ()) if any(not isinstance(p.get(k), str) or not p[k] for k in required): - raise AzureError('missing_projection_field') - if family in ('pim_eligible', 'pim_active') and not (p.get('scope') or p.get('directoryScopeId')): - raise AzureError('missing_projection_field') - for key in ('permissions', 'principals', 'excludePrincipals', 'rolePermissions', 'rules', 'authorizations', 'eligibleAuthorizations', 'accessPolicies'): + raise AzureError("missing_projection_field") + if family in ("pim_eligible", "pim_active") and not (p.get("scope") or p.get("directoryScopeId")): + raise AzureError("missing_projection_field") + for key in ( + "permissions", + "principals", + "excludePrincipals", + "rolePermissions", + "rules", + "authorizations", + "eligibleAuthorizations", + "accessPolicies", + ): if key in p and (not isinstance(p[key], list) or any(not isinstance(v, dict) for v in p[key])): - raise AzureError('invalid_projection_field') - for item in p.get('permissions', []): - for key in ('actions', 'notActions', 'dataActions', 'notDataActions'): + raise AzureError("invalid_projection_field") + for item in p.get("permissions", []): + for key in ("actions", "notActions", "dataActions", "notDataActions"): if key in item and (not isinstance(item[key], list) or any(not isinstance(v, str) for v in item[key])): - raise AzureError('invalid_projection_field') - for item in p.get('rolePermissions', []): - if not isinstance(item.get('allowedResourceActions'), list) or any(not isinstance(v, str) for v in item['allowedResourceActions']): - raise AzureError('invalid_projection_field') - for key in ('scope', 'parentId'): + raise AzureError("invalid_projection_field") + for item in p.get("rolePermissions", []): + if not isinstance(item.get("allowedResourceActions"), list) or any( + not isinstance(v, str) for v in item["allowedResourceActions"] + ): + raise AzureError("invalid_projection_field") + for key in ("scope", "parentId"): if key in p: arm(p[key]) - if family == 'resources' and 'identity' in data: - identity = data['identity'] - if not isinstance(identity, dict) or ('principalId' in identity and not isinstance(identity['principalId'], str)) or not isinstance(identity.get('userAssignedIdentities', {}), dict): - raise AzureError('invalid_identity') - for key in ('startDateTime', 'endDateTime'): + if family == "resources" and "identity" in data: + identity = data["identity"] + if ( + not isinstance(identity, dict) + or ("principalId" in identity and not isinstance(identity["principalId"], str)) + or not isinstance(identity.get("userAssignedIdentities", {}), dict) + ): + raise AzureError("invalid_identity") + for key in ("startDateTime", "endDateTime"): if p.get(key): from .._runtime.cops.evidence.canonical import timestamp + try: timestamp(p[key]) except ValueError: - raise AzureError('invalid_validity') from None + raise AzureError("invalid_validity") from None - for key in ('assignableScopes', 'audiences'): + for key in ("assignableScopes", "audiences"): if key in p and (not isinstance(p[key], list) or any(not isinstance(v, str) or not v for v in p[key])): - raise AzureError('invalid_projection_field') - for scope in p.get('assignableScopes', []): + raise AzureError("invalid_projection_field") + for scope in p.get("assignableScopes", []): arm(scope) - for key in ('enableRbacAuthorization', 'doNotApplyToChildScopes', 'isAssignableToRole'): + for key in ("enableRbacAuthorization", "doNotApplyToChildScopes", "isAssignableToRole"): if key in p and type(p[key]) is not bool: - raise AzureError('invalid_projection_field') - for key in ('principals', 'excludePrincipals'): - if any(not isinstance(v.get('id'), str) or not v['id'] for v in p.get(key, [])): - raise AzureError('invalid_projection_field') - for rule in p.get('rules', []): - if not isinstance(rule.get('id'), str): - raise AzureError('invalid_projection_field') - for key in ('target', 'setting', 'approvalSettings'): + raise AzureError("invalid_projection_field") + for key in ("principals", "excludePrincipals"): + if any(not isinstance(v.get("id"), str) or not v["id"] for v in p.get(key, [])): + raise AzureError("invalid_projection_field") + for rule in p.get("rules", []): + if not isinstance(rule.get("id"), str): + raise AzureError("invalid_projection_field") + for key in ("target", "setting", "approvalSettings"): if key in rule and not isinstance(rule[key], dict): - raise AzureError('invalid_projection_field') - if 'enabledRules' in rule and (not isinstance(rule['enabledRules'], list) or any(not isinstance(v, str) for v in rule['enabledRules'])): - raise AzureError('invalid_projection_field') - if 'maximumDuration' in rule and not isinstance(rule['maximumDuration'], str): - raise AzureError('invalid_projection_field') - for policy in p.get('accessPolicies', []): - if not isinstance(policy.get('permissions'), dict): - raise AzureError('invalid_projection_field') - for values in policy['permissions'].values(): + raise AzureError("invalid_projection_field") + if "enabledRules" in rule and ( + not isinstance(rule["enabledRules"], list) or any(not isinstance(v, str) for v in rule["enabledRules"]) + ): + raise AzureError("invalid_projection_field") + if "maximumDuration" in rule and not isinstance(rule["maximumDuration"], str): + raise AzureError("invalid_projection_field") + for policy in p.get("accessPolicies", []): + if not isinstance(policy.get("permissions"), dict): + raise AzureError("invalid_projection_field") + for values in policy["permissions"].values(): if not isinstance(values, list) or any(not isinstance(v, str) for v in values): - raise AzureError('invalid_projection_field') - for key in ('tenantId', 'objectId'): + raise AzureError("invalid_projection_field") + for key in ("tenantId", "objectId"): if not isinstance(policy.get(key), str): - raise AzureError('invalid_projection_field') - for authorization in p.get('authorizations', []) + p.get('eligibleAuthorizations', []): - if any(not isinstance(authorization.get(k), str) or not authorization[k] for k in ('principalId', 'roleDefinitionId')): - raise AzureError('invalid_projection_field') - ids = authorization.get('delegatedRoleDefinitionIds', []) + raise AzureError("invalid_projection_field") + for authorization in p.get("authorizations", []) + p.get("eligibleAuthorizations", []): + if any( + not isinstance(authorization.get(k), str) or not authorization[k] + for k in ("principalId", "roleDefinitionId") + ): + raise AzureError("invalid_projection_field") + ids = authorization.get("delegatedRoleDefinitionIds", []) if not isinstance(ids, list) or any(not isinstance(v, str) for v in ids): - raise AzureError('invalid_projection_field') + raise AzureError("invalid_projection_field") + + strings = { + "id", + "appId", + "appOwnerOrganizationId", + "servicePrincipalType", + "principalId", + "principalType", + "roleDefinitionId", + "scope", + "scopeId", + "scopeType", + "directoryScopeId", + "appScopeId", + "condition", + "conditionVersion", + "roleType", + "type", + "tenantId", + "clientId", + "startDateTime", + "endDateTime", + "issuer", + "subject", + "parentId", + "kind", + "state", + "managedByTenantId", + "registrationDefinitionId", + "policyId", + "objectId", + "resourceId", + "appRoleId", + "policyAssignmentId", + "caller", + "level", + "maximumDuration", + "claimValue", + "signInAudience", + } - strings = {'id', 'appId', 'appOwnerOrganizationId', 'servicePrincipalType', 'principalId', 'principalType', - 'roleDefinitionId', 'scope', 'scopeId', 'scopeType', 'directoryScopeId', 'appScopeId', 'condition', 'conditionVersion', - 'roleType', 'type', 'tenantId', 'clientId', 'startDateTime', 'endDateTime', 'issuer', 'subject', 'parentId', - 'kind', 'state', 'managedByTenantId', 'registrationDefinitionId', 'policyId', 'objectId', 'resourceId', 'appRoleId', - 'policyAssignmentId', 'caller', 'level', 'maximumDuration', 'claimValue', 'signInAudience'} def validate(value): if isinstance(value, dict): for key, child in value.items(): - if key in strings and (not isinstance(child, str) or len(child) > 8192 or any(ord(c) < 32 for c in child)): - raise AzureError('invalid_projection_field') + if key in strings and ( + not isinstance(child, str) or len(child) > 8192 or any(ord(c) < 32 for c in child) + ): + raise AzureError("invalid_projection_field") validate(child) elif isinstance(value, list): - for child in value: validate(child) + for child in value: + validate(child) + validate(data) diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/paths.py b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/paths.py index 41e06db..fb94c3a 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/paths.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/paths.py @@ -1,7 +1,7 @@ """Deterministic bounded path traversal with explicit hypothetical steps.""" -from .model import Budget, stable, object_id +from .model import Budget, object_id, stable from .permissions import evaluate -from .rules import combine, transitions, step +from .rules import combine, step, transitions def classification(decision): diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/permissions.py b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/permissions.py index 167d42d..3478ebd 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/permissions.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/permissions.py @@ -1,7 +1,8 @@ """Conservative Azure RBAC decisions; exclusions apply within each role.""" import re -from .model import Decision, arm + from .identity import find, membership, temporal +from .model import Decision, arm # Resource Manager deployment clients require the complete deployment operation bundle. DEPLOYMENT_OPERATIONS = tuple("Microsoft.Resources/deployments/" + action for action in ( @@ -118,7 +119,7 @@ def evaluate(graph, tenant, principal, action, scope, *, plane="control", contex valid = temporal(graph, row, scenario) if valid.state == "denied": continue - definition = find(graph, "role_definitions", tenant, lambda r: arm(r.data["id"]) == arm(p["roleDefinitionId"])) + definition = find(graph, "role_definitions", tenant, lambda r, p=p: arm(r.data["id"]) == arm(p["roleDefinitionId"])) refs, cuts = principals[p["principalId"].lower()] if definition is None: unknown += row.evidence diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/report.py b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/report.py index 6425381..06c0050 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/report.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/report.py @@ -1,10 +1,10 @@ """Bounded reports and individually re-evaluated entitlement cuts.""" import html import os -from pathlib import Path + from .._runtime.cops.evidence.canonical import canonical from .input import load -from .model import AzureError, Budget, stable, absolute_parts +from .model import AzureError, Budget, absolute_parts, stable from .paths import search CUT_FAMILIES = {"role_assignments", "directory_assignments", "pim_active", "pim_eligible", diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/rules.py b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/rules.py index 33f5acf..aabf4f1 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/azure/rules.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/azure/rules.py @@ -1,7 +1,7 @@ """Pinned, bounded hypothetical transitions; no operations are executed.""" -from .model import Decision, arm, object_id, stable -from .permissions import DEPLOYMENT_OPERATIONS, evaluate, role_grant from .identity import directory, find, membership +from .model import Decision, arm, stable +from .permissions import DEPLOYMENT_OPERATIONS, evaluate, role_grant VERSION = 'azure-rules/v1' EXECUTION = { @@ -19,7 +19,7 @@ ['Microsoft.Logic/workflows/write', 'Microsoft.Logic/workflows/triggers/run/action'], ['logic_consumption', 'identity_action', 'invocation', 'network', 'token_endpoint']), } -SECRET = 'microsoft.keyvault/vaults/secrets/getsecret/action' +SECRET = 'microsoft.keyvault/vaults/secrets/getsecret/action' # noqa: S105 - schema label or operation identifier, not a credential OWNER = '8e3af657-a8ff-443c-a75c-2fe8c4bcb635' UAA = '18d7d88d-d35e-4fb5-a5c3-7773c20a72d9' @@ -59,7 +59,7 @@ def identity_targets(graph, row): targets.append((identity['principalId'], [observation(row)])) for rid in identity.get('userAssignedIdentities', {}): graph.tick() - uami = find(graph, 'resources', row.tenant, lambda r: arm(r.data['id']) == arm(rid)) + uami = find(graph, 'resources', row.tenant, lambda r, rid=rid: arm(r.data['id']) == arm(rid)) if uami and uami.properties.get('principalId'): targets.append((uami.properties['principalId'], [observation(row), observation(uami)])) return targets @@ -218,7 +218,7 @@ def transitions(graph, tenant, principal, target): for fic in graph.rows('federated_credentials', tenant): graph.tick() oid = fic.data.get('objectId', '') - obj = find(graph, 'resources', tenant, lambda r: r.data['id'].lower() == oid.lower()) or find(graph, 'applications', tenant, lambda r: r.data['id'].lower() == oid.lower()) + obj = find(graph, 'resources', tenant, lambda r, oid=oid: r.data['id'].lower() == oid.lower()) or find(graph, 'applications', tenant, lambda r, oid=oid: r.data['id'].lower() == oid.lower()) if not obj: continue if obj.family == 'resources' and obj.data.get('type', '').lower() != 'microsoft.managedidentity/userassignedidentities': @@ -273,7 +273,7 @@ def delegated(graph, tenant, principal, target): if aid not in principals: continue role_id = authorization.get('roleDefinitionId', '').split('/')[-1].lower() - role = find(graph, 'role_definitions', definition.tenant, lambda r: r.data['id'].split('/')[-1].lower() == role_id) + role = find(graph, 'role_definitions', definition.tenant, lambda r, role_id=role_id: r.data['id'].split('/')[-1].lower() == role_id) if not role: continue restricted = role.properties.get('roleType') != 'BuiltInRole' or role_id == OWNER or any( @@ -327,7 +327,7 @@ def delegated(graph, tenant, principal, target): for authorization, grant_ds in grants: for rid in authorization.get('delegatedRoleDefinitionIds', []): clone.tick() - role = find(clone, 'role_definitions', definition.tenant, lambda r: r.data['id'].split('/')[-1].lower() == rid.split('/')[-1].lower()) + role = find(clone, 'role_definitions', definition.tenant, lambda r, rid=rid: r.data['id'].split('/')[-1].lower() == rid.split('/')[-1].lower()) if not role or role.properties.get('roleType') != 'BuiltInRole' or role.data['id'].split('/')[-1].lower() in (OWNER, UAA): continue check = hypothetical_grant(clone, definition.tenant, mi, role, scope, target) diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/cli.py b/plugins/detection-hunting/attack-path-workbench/attackpath/cli.py index b630568..f36f299 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/cli.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/cli.py @@ -54,7 +54,8 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) if args.command in ("analyze-azure", "plan-azure-collection"): from .azure.model import AzureError - from .azure.report import analyze as analyze_azure, write_files + from .azure.report import analyze as analyze_azure + from .azure.report import write_files try: if args.command == "analyze-azure": result = analyze_azure(args.input, args.as_of, args.output, ingestion_overrides(args)) diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/core.py b/plugins/detection-hunting/attack-path-workbench/attackpath/core.py index 3226714..c7dc428 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/core.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/core.py @@ -2,17 +2,18 @@ from __future__ import annotations +import copy import hashlib import json -import copy from collections import defaultdict -from datetime import datetime, timezone +from datetime import UTC, datetime from fractions import Fraction from pathlib import Path from typing import Any from . import VERSION -from .ingestion import IngestError, Limits, RunBudget, parse_json as bounded_json, read_regular +from .ingestion import IngestError, Limits, RunBudget, read_regular +from .ingestion import parse_json as bounded_json from .search import SearchLimits, bounded_json_bytes @@ -67,7 +68,7 @@ def utc(value: Any, where: str) -> str: parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) except ValueError as exc: raise GateError(f"{where}: invalid UTC timestamp") from exc - if parsed.tzinfo != timezone.utc: + if parsed.tzinfo != UTC: raise GateError(f"{where}: expected UTC timestamp") return value @@ -377,7 +378,7 @@ def trace_paths(nodes: dict[str, dict[str, Any]], findings: list[dict[str, Any]] "deduplication_key": digest(signature), "scope": finding["scope"]} else: prior["evidence_refs"] = sorted(set(prior["evidence_refs"] + evidence)) - for existing_step, new_step in zip(prior["steps"], steps): + for existing_step, new_step in zip(prior["steps"], steps, strict=False): existing_step["supporting_evidence_refs"] = sorted(set( existing_step["supporting_evidence_refs"] + new_step["supporting_evidence_refs"])) continue @@ -654,7 +655,7 @@ def analyze(input_path: Path, limits: dict[str, int] | None = None, if any(ref not in evidence_ids for kind in ("nodes", "findings", "edges") for item in graph[kind] for ref in [item["evidence_ref"]]): raise GateError("G8: graph item lacks cited evidence") audit_report(report, sources) - for src, path, _ in sources: + for src, _path, _ in sources: try: current_hash = hashlib.sha256(read_regular(input_path.parent, src["path"], policy.file_bytes, budget, count_file=False).data).hexdigest() diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/ingestion.py b/plugins/detection-hunting/attack-path-workbench/attackpath/ingestion.py index c80014d..ab43c60 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/ingestion.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/ingestion.py @@ -1,11 +1,11 @@ """Shared fail-closed limits and descriptor-anchored reads for local evidence.""" from __future__ import annotations -from dataclasses import dataclass import json import os -from pathlib import Path import stat +from dataclasses import dataclass +from pathlib import Path from ._runtime.cops.evidence.canonical import EvidenceError, decode_json @@ -127,8 +127,9 @@ def read_regular(root, relative, limit, budget: RunBudget | None = None, *, coun raise IngestError("file_limit" if limit <= remaining else "total_byte_limit") chunks.append(chunk) after = os.fstat(fd) - identity = lambda item: (item.st_dev, item.st_ino, item.st_nlink, item.st_size, - item.st_mtime_ns, item.st_ctime_ns) + def identity(item): + return (item.st_dev, item.st_ino, item.st_nlink, item.st_size, + item.st_mtime_ns, item.st_ctime_ns) if identity(before) != identity(after) or size != after.st_size: raise IngestError("file_changed") if budget: diff --git a/plugins/detection-hunting/attack-path-workbench/attackpath/search.py b/plugins/detection-hunting/attack-path-workbench/attackpath/search.py index e87d3b3..b9f360f 100644 --- a/plugins/detection-hunting/attack-path-workbench/attackpath/search.py +++ b/plugins/detection-hunting/attack-path-workbench/attackpath/search.py @@ -16,7 +16,7 @@ class SearchLimits: report_bytes: int = 64 * 1024 * 1024 @classmethod - def from_values(cls, values: dict[str, int] | None = None) -> "SearchLimits": + def from_values(cls, values: dict[str, int] | None = None) -> SearchLimits: defaults = asdict(cls()) if values is None: values = {} diff --git a/plugins/detection-hunting/attack-path-workbench/scripts/attackpath.py b/plugins/detection-hunting/attack-path-workbench/scripts/attackpath.py index 0bb7bf6..d3ddcbc 100644 --- a/plugins/detection-hunting/attack-path-workbench/scripts/attackpath.py +++ b/plugins/detection-hunting/attack-path-workbench/scripts/attackpath.py @@ -9,6 +9,5 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from attackpath.cli import main # noqa: E402 - if __name__ == "__main__": raise SystemExit(main()) diff --git a/plugins/detection-hunting/attack-path-workbench/scripts/validate-package.py b/plugins/detection-hunting/attack-path-workbench/scripts/validate-package.py index 9f3f8c6..281ed6a 100644 --- a/plugins/detection-hunting/attack-path-workbench/scripts/validate-package.py +++ b/plugins/detection-hunting/attack-path-workbench/scripts/validate-package.py @@ -3,11 +3,10 @@ from __future__ import annotations import json -from pathlib import Path import subprocess import sys import tempfile - +from pathlib import Path PACKAGE = Path(__file__).resolve().parents[1] CLI = PACKAGE / "scripts/attackpath.py" diff --git a/plugins/detection-hunting/attack-path-workbench/tests/azure_fixtures.py b/plugins/detection-hunting/attack-path-workbench/tests/azure_fixtures.py index fe4d4e6..b70cc49 100644 --- a/plugins/detection-hunting/attack-path-workbench/tests/azure_fixtures.py +++ b/plugins/detection-hunting/attack-path-workbench/tests/azure_fixtures.py @@ -1,11 +1,13 @@ """Synthetic documented Graph/ARM pages wrapped in the canonical SDK contract.""" -from copy import deepcopy + import hashlib import json +from copy import deepcopy from pathlib import Path -from attackpath._runtime.cops.evidence.contract import build_envelope + from attackpath._runtime.cops.evidence.canonical import canonical -from attackpath.azure.collection import GRAPH_CATALOG, ARM_CATALOG, RESOURCE_APIS, source_contract +from attackpath._runtime.cops.evidence.contract import build_envelope +from attackpath.azure.collection import ARM_CATALOG, GRAPH_CATALOG, RESOURCE_APIS, source_contract NOW = "2026-09-28T12:00:00Z" S = "/subscriptions/sub-a" @@ -20,7 +22,11 @@ def write_bundle(root, graph, *, status="complete", observed_at=NOW, limits=None if family.startswith("pim_"): family = data.pop("plane") + "_" + family data.pop("requirements", None) - api = "v1.0" if family in {r[0] for r in GRAPH_CATALOG} else next((r[2] for r in ARM_CATALOG if r[0] == family), None) + api = ( + "v1.0" + if family in {r[0] for r in GRAPH_CATALOG} + else next((r[2] for r in ARM_CATALOG if r[0] == family), None) + ) if family == "resources": api = RESOURCE_APIS[data["type"].lower()] context = {} @@ -40,29 +46,80 @@ def write_bundle(root, graph, *, status="complete", observed_at=NOW, limits=None for index, ((family, tenant, api, pairs), values) in enumerate(sorted(groups.items())): sid = family + "-" + str(index) scopes = ["/"] if source_contract(family, api) == "MicrosoftGraph" else [S] - envelope = build_envelope(acquisition_id=sid, product=source_contract(family, api), api=api, - tenant=tenant, scope=scopes, identity=sid, locator=sid, payload={"value": values}, - acquired_at=NOW, transformed_at=NOW, observed_at=observed_at, - request_fingerprint="a"*64, page=1) + envelope = build_envelope( + acquisition_id=sid, + product=source_contract(family, api), + api=api, + tenant=tenant, + scope=scopes, + identity=sid, + locator=sid, + payload={"value": values}, + acquired_at=NOW, + transformed_at=NOW, + observed_at=observed_at, + request_fingerprint="a" * 64, + page=1, + ) raw = canonical(envelope) + b"\n" - receipt = {"schema_version":"cops.acquisition/v1", "acquisition_id":sid, "generation":1, - "adapter":"azure-synthetic-fixture", "adapter_version":"1", "tenant":tenant, "scope":scopes, - "request_fingerprint":"a"*64, "started_at":NOW, "finished_at":NOW, - "status":status, "reasons":[] if status == "complete" else ["page_limit"], "consistency":"unknown", - "limits":{"pages":10,"attempts":10,"records":10,"response_bytes":1048576, - "total_bytes":10485760,"record_bytes":1048576,"storage_bytes":10485760, - "depth":32,"request_seconds":10,"active_seconds":100,"retries":1}, - "consumed":{"pages":1,"attempts":1,"records":1,"duplicates":0,"bytes":len(raw), - "storage_bytes":len(raw),"active_seconds":1}} + receipt = { + "schema_version": "cops.acquisition/v1", + "acquisition_id": sid, + "generation": 1, + "adapter": "azure-synthetic-fixture", + "adapter_version": "1", + "tenant": tenant, + "scope": scopes, + "request_fingerprint": "a" * 64, + "started_at": NOW, + "finished_at": NOW, + "status": status, + "reasons": [] if status == "complete" else ["page_limit"], + "consistency": "unknown", + "limits": { + "pages": 10, + "attempts": 10, + "records": 10, + "response_bytes": 1048576, + "total_bytes": 10485760, + "record_bytes": 1048576, + "storage_bytes": 10485760, + "depth": 32, + "request_seconds": 10, + "active_seconds": 100, + "retries": 1, + }, + "consumed": { + "pages": 1, + "attempts": 1, + "records": 1, + "duplicates": 0, + "bytes": len(raw), + "storage_bytes": len(raw), + "active_seconds": 1, + }, + } rec = canonical(receipt) (root / (sid + ".jsonl")).write_bytes(raw) (root / (sid + ".receipt.json")).write_bytes(rec) - sources.append({"id":sid, "family":family,"api":api,"tenant":tenant,"scopes":scopes, - "path":sid+".jsonl","sha256":hashlib.sha256(raw).hexdigest(), - "receipt":sid+".receipt.json","receipt_sha256":hashlib.sha256(rec).hexdigest(), - "max_age_seconds":3600,"context":dict(pairs)}) - manifest = {"schema_version":"attackpath.azure.input/v1","sources":sources,"scenario":deepcopy(graph.scenario)} - if limits is not None: manifest["limits"] = limits + sources.append( + { + "id": sid, + "family": family, + "api": api, + "tenant": tenant, + "scopes": scopes, + "path": sid + ".jsonl", + "sha256": hashlib.sha256(raw).hexdigest(), + "receipt": sid + ".receipt.json", + "receipt_sha256": hashlib.sha256(rec).hexdigest(), + "max_age_seconds": 3600, + "context": dict(pairs), + } + ) + manifest = {"schema_version": "attackpath.azure.input/v1", "sources": sources, "scenario": deepcopy(graph.scenario)} + if limits is not None: + manifest["limits"] = limits path = root / "manifest.json" path.write_text(json.dumps(manifest)) return path diff --git a/plugins/detection-hunting/attack-path-workbench/tests/test_azure_contracts.py b/plugins/detection-hunting/attack-path-workbench/tests/test_azure_contracts.py index 4210e43..d537e95 100644 --- a/plugins/detection-hunting/attack-path-workbench/tests/test_azure_contracts.py +++ b/plugins/detection-hunting/attack-path-workbench/tests/test_azure_contracts.py @@ -1,11 +1,12 @@ """Malformed metadata fails closed and permission modes remain distinct.""" import unittest + +from attackpath.azure.identity import directory from attackpath.azure.input import scenario_contract from attackpath.azure.model import AzureError, Graph from attackpath.azure.normalize import validate_projection from attackpath.azure.permissions import evaluate -from attackpath.azure.identity import directory -from test_azure_paths import base, role, SECRET, VAULT, T, NOW +from test_azure_paths import NOW, SECRET, VAULT, T, base, role class ContractTests(unittest.TestCase): diff --git a/plugins/detection-hunting/attack-path-workbench/tests/test_azure_input.py b/plugins/detection-hunting/attack-path-workbench/tests/test_azure_input.py index 46b46ea..d4fb908 100644 --- a/plugins/detection-hunting/attack-path-workbench/tests/test_azure_input.py +++ b/plugins/detection-hunting/attack-path-workbench/tests/test_azure_input.py @@ -1,15 +1,16 @@ """Hostile local bundles fail before analysis; no payloads escape errors.""" -import hashlib import os -from pathlib import Path import tempfile import unittest -from attackpath.azure.input import read_regular, load +from pathlib import Path + +from attackpath.azure.input import load, read_regular from attackpath.azure.model import AzureError -from attackpath.ingestion import Limits, RunBudget, iter_jsonl, parse_json, IngestError +from attackpath.ingestion import IngestError, Limits, RunBudget, iter_jsonl, parse_json from azure_fixtures import NOW, write_bundle from test_azure_paths import base + class InputTests(unittest.TestCase): def test_descriptor_reader_rejects_links_traversal_and_nonregular(self): with tempfile.TemporaryDirectory() as tmp: diff --git a/plugins/detection-hunting/attack-path-workbench/tests/test_azure_paths.py b/plugins/detection-hunting/attack-path-workbench/tests/test_azure_paths.py index e6a5979..4883870 100644 --- a/plugins/detection-hunting/attack-path-workbench/tests/test_azure_paths.py +++ b/plugins/detection-hunting/attack-path-workbench/tests/test_azure_paths.py @@ -1,87 +1,175 @@ """Paired entitlement chains: exact rights and explicit runtime assumptions.""" + import unittest + from attackpath.azure.model import Graph from attackpath.azure.paths import search -T="tenant-a" -S="/subscriptions/sub-a" -NOW="2026-09-28T12:00:00Z" -SECRET="Microsoft.KeyVault/vaults/secrets/getSecret/action" -VAULT=S+"/resourceGroups/rg/providers/Microsoft.KeyVault/vaults/vault" +T = "tenant-a" +S = "/subscriptions/sub-a" +NOW = "2026-09-28T12:00:00Z" +SECRET = "Microsoft.KeyVault/vaults/secrets/getSecret/action" # noqa: S105 - schema label or operation identifier, not a credential +VAULT = S + "/resourceGroups/rg/providers/Microsoft.KeyVault/vaults/vault" + def role(g, principal, actions=(), data=(), scope=S, suffix="role", tenant=T): - rid=S+"/providers/Microsoft.Authorization/roleDefinitions/"+suffix - g.add("role_definitions",tenant,{"id":rid,"properties":{"permissions":[{"actions":list(actions),"notActions":[],"dataActions":list(data),"notDataActions":[]}],"assignableScopes":[S]}},suffix+"-definition") - g.add("role_assignments",tenant,{"id":suffix+"-assignment","properties":{"principalId":principal,"roleDefinitionId":rid,"scope":scope}},suffix+"-assignment") + rid = S + "/providers/Microsoft.Authorization/roleDefinitions/" + suffix + g.add( + "role_definitions", + tenant, + { + "id": rid, + "properties": { + "permissions": [ + {"actions": list(actions), "notActions": [], "dataActions": list(data), "notDataActions": []} + ], + "assignableScopes": [S], + }, + }, + suffix + "-definition", + ) + g.add( + "role_assignments", + tenant, + { + "id": suffix + "-assignment", + "properties": {"principalId": principal, "roleDefinitionId": rid, "scope": scope}, + }, + suffix + "-assignment", + ) return rid + def base(): - g=Graph(NOW) - for f in ("deny_assignments","role_assignments","group_members","azure_pim_eligible","azure_pim_active"): - g.cover(f,T,[S],True,f) - g.scenario={"controlled":[{"tenant":T,"id":"user"}],"targets":[{"tenant":T,"scope":VAULT,"action":SECRET,"plane":"data"}],"runtime":{}} - g.add("resources",T,{"id":VAULT,"type":"Microsoft.KeyVault/vaults","properties":{"enableRbacAuthorization":True}},"vault") + g = Graph(NOW) + for f in ("deny_assignments", "role_assignments", "group_members", "azure_pim_eligible", "azure_pim_active"): + g.cover(f, T, [S], True, f) + g.scenario = { + "controlled": [{"tenant": T, "id": "user"}], + "targets": [{"tenant": T, "scope": VAULT, "action": SECRET, "plane": "data"}], + "runtime": {}, + } + g.add( + "resources", + T, + {"id": VAULT, "type": "Microsoft.KeyVault/vaults", "properties": {"enableRbacAuthorization": True}}, + "vault", + ) return g + def execution(kind, actions, runtime): - g=base() - rid=S+"/resourceGroups/rg/providers/"+kind+"/workload" - obj={"id":rid,"type":kind,"identity":{"principalId":"mi"},"properties":{}} - if kind=="Microsoft.Web/sites":obj["kind"]="functionapp" - g.add("resources",T,obj,"workload") - role(g,"user",actions=actions) - role(g,"mi",data=[SECRET],scope=VAULT,suffix="downstream") - g.scenario["runtime"][rid.lower()]={k:True for k in runtime} - return g,rid + g = base() + rid = S + "/resourceGroups/rg/providers/" + kind + "/workload" + obj = {"id": rid, "type": kind, "identity": {"principalId": "mi"}, "properties": {}} + if kind == "Microsoft.Web/sites": + obj["kind"] = "functionapp" + g.add("resources", T, obj, "workload") + role(g, "user", actions=actions) + role(g, "mi", data=[SECRET], scope=VAULT, suffix="downstream") + g.scenario["runtime"][rid.lower()] = {k: True for k in runtime} + return g, rid + DEPLOYMENT_OPERATIONS = [ - "Microsoft.Resources/deployments/" + action for action in ( - "read", "write", "delete", "cancel/action", "validate/action", "whatIf/action", - "exportTemplate/action", "operations/read", "operationstatuses/read")] + "Microsoft.Resources/deployments/" + action + for action in ( + "read", + "write", + "delete", + "cancel/action", + "validate/action", + "whatIf/action", + "exportTemplate/action", + "operations/read", + "operationstatuses/read", + ) +] def deployment(): - g, rid = execution("Microsoft.Compute/virtualMachines", + g, rid = execution( + "Microsoft.Compute/virtualMachines", ["Microsoft.Resources/deployments/*", "Microsoft.Compute/virtualMachines/extensions/write"], - ["vm_agent", "network", "token_endpoint"]) + ["vm_agent", "network", "token_endpoint"], + ) g.rows("role_assignments")[0].properties["scope"] = S + "/resourceGroups/rg" return g, rid class ChainTests(unittest.TestCase): - def assert_pair(self,g,rule,break_it): - paths=search(g) - reached=[p for p in paths if p["classification"]=="modelled_reachable" and rule in [s["rule"] for s in p["steps"]]] - self.assertTrue(reached,rule) + def assert_pair(self, g, rule, break_it): + paths = search(g) + reached = [ + p for p in paths if p["classification"] == "modelled_reachable" and rule in [s["rule"] for s in p["steps"]] + ] + self.assertTrue(reached, rule) self.assertTrue(all(p["evidence"] for p in reached)) break_it(g) - paths=search(g) - self.assertFalse(any(p["classification"]=="modelled_reachable" and rule in [s["rule"] for s in p["steps"]] for p in paths),rule) + paths = search(g) + self.assertFalse( + any(p["classification"] == "modelled_reachable" and rule in [s["rule"] for s in p["steps"]] for p in paths), + rule, + ) def test_rbac_grant(self): - g=base();role(g,"user",actions=["Microsoft.Authorization/roleAssignments/write"]) - role(g,"other",data=[SECRET],scope=VAULT,suffix="target") - self.assert_pair(g,"rbac_grant",lambda g:g.rows("role_definitions")[0].properties["permissions"][0].update(actions=[])) + g = base() + role(g, "user", actions=["Microsoft.Authorization/roleAssignments/write"]) + role(g, "other", data=[SECRET], scope=VAULT, suffix="target") + self.assert_pair( + g, "rbac_grant", lambda g: g.rows("role_definitions")[0].properties["permissions"][0].update(actions=[]) + ) def test_application_credentials(self): - g=base() - g.add("applications",T,{"id":"app","appId":"client","signInAudience":"AzureADMyOrg"},"app") - g.add("service_principals",T,{"id":"sp","appId":"client","appOwnerOrganizationId":T,"servicePrincipalType":"Application"},"sp") - g.add("directory_definitions",T,{"id":"dr","rolePermissions":[{"allowedResourceActions":["microsoft.directory/applications/credentials/update"]}]},"dr") - g.add("directory_assignments",T,{"id":"da","principalId":"user","roleDefinitionId":"dr","directoryScopeId":"/"},"da") - g.scenario["credential_restrictions"]=[{"tenant":T,"object":"app","credential_update_allowed":True}] - role(g,"sp",data=[SECRET],scope=VAULT) - self.assert_pair(g,"application_credentials",lambda g:g.rows("service_principals")[0].data.update(appOwnerOrganizationId="other-tenant")) + g = base() + g.add("applications", T, {"id": "app", "appId": "client", "signInAudience": "AzureADMyOrg"}, "app") + g.add( + "service_principals", + T, + {"id": "sp", "appId": "client", "appOwnerOrganizationId": T, "servicePrincipalType": "Application"}, + "sp", + ) + g.add( + "directory_definitions", + T, + { + "id": "dr", + "rolePermissions": [ + {"allowedResourceActions": ["microsoft.directory/applications/credentials/update"]} + ], + }, + "dr", + ) + g.add( + "directory_assignments", + T, + {"id": "da", "principalId": "user", "roleDefinitionId": "dr", "directoryScopeId": "/"}, + "da", + ) + g.scenario["credential_restrictions"] = [{"tenant": T, "object": "app", "credential_update_allowed": True}] + role(g, "sp", data=[SECRET], scope=VAULT) + self.assert_pair( + g, + "application_credentials", + lambda g: g.rows("service_principals")[0].data.update(appOwnerOrganizationId="other-tenant"), + ) def test_vm_execution(self): - g,r=execution("Microsoft.Compute/virtualMachines",["Microsoft.Compute/virtualMachines/write","Microsoft.Compute/virtualMachines/extensions/write"],["vm_agent","network","token_endpoint"]) - self.assert_pair(g,"vm_execution",lambda g:g.scenario["runtime"][r.lower()].update(vm_agent=False)) + g, r = execution( + "Microsoft.Compute/virtualMachines", + ["Microsoft.Compute/virtualMachines/write", "Microsoft.Compute/virtualMachines/extensions/write"], + ["vm_agent", "network", "token_endpoint"], + ) + self.assert_pair(g, "vm_execution", lambda g: g.scenario["runtime"][r.lower()].update(vm_agent=False)) def test_arm_deployment_requires_underlying_resource_permission(self): g, rid = deployment() paths = search(g) - reached = [p for p in paths if p["classification"] == "modelled_reachable" and - "arm_deployment" in [s["rule"] for s in p["steps"]]] + reached = [ + p + for p in paths + if p["classification"] == "modelled_reachable" and "arm_deployment" in [s["rule"] for s in p["steps"]] + ] self.assertTrue(reached) steps = reached[0]["steps"] prepare = next(s for s in steps if s["rule"] == "arm_deployment") @@ -91,40 +179,74 @@ def test_arm_deployment_requires_underlying_resource_permission(self): required = {r for d in prepare["prerequisites"] for r in d["reasons"]} self.assertEqual({"required_operation:" + a for a in DEPLOYMENT_OPERATIONS}, required) # Deployment rights never confer the underlying extension right. - self.assert_pair(g, "arm_deployment", lambda g: - g.rows("role_definitions")[0].properties["permissions"][0].update( - actions=["Microsoft.Resources/deployments/*"])) + self.assert_pair( + g, + "arm_deployment", + lambda g: ( + g.rows("role_definitions")[0] + .properties["permissions"][0] + .update(actions=["Microsoft.Resources/deployments/*"]) + ), + ) def test_arm_deployment_checks_every_exclusion(self): for operation in DEPLOYMENT_OPERATIONS: with self.subTest(operation=operation): g, _ = deployment() - self.assert_pair(g, "arm_deployment", lambda g: - g.rows("role_definitions")[0].properties["permissions"][0].update(notActions=[operation])) + self.assert_pair( + g, + "arm_deployment", + lambda g, operation=operation: ( + g.rows("role_definitions")[0].properties["permissions"][0].update(notActions=[operation]) + ), + ) def test_arm_deployment_scope_deny_and_runtime(self): def wrong_scope(g, rid): g.rows("role_assignments")[0].properties["scope"] = S + "/resourceGroups/other" + def deny(g, rid): - g.add("deny_assignments", T, {"id": "deny-deployment", "properties": { - "scope": S + "/resourceGroups/rg", "principals": [{"id": "user"}], - "excludePrincipals": [], "permissions": [{"actions": ["Microsoft.Resources/deployments/write"], - "notActions": [], "dataActions": [], "notDataActions": []}]}}, "deny-deployment") + g.add( + "deny_assignments", + T, + { + "id": "deny-deployment", + "properties": { + "scope": S + "/resourceGroups/rg", + "principals": [{"id": "user"}], + "excludePrincipals": [], + "permissions": [ + { + "actions": ["Microsoft.Resources/deployments/write"], + "notActions": [], + "dataActions": [], + "notDataActions": [], + } + ], + }, + }, + "deny-deployment", + ) + def unavailable_runtime(g, rid): g.scenario["runtime"][rid.lower()]["vm_agent"] = False + for break_it in (wrong_scope, deny, unavailable_runtime): with self.subTest(case=break_it.__name__): g, rid = deployment() - self.assert_pair(g, "arm_deployment", lambda g: break_it(g, rid)) + self.assert_pair(g, "arm_deployment", lambda g, break_it=break_it, rid=rid: break_it(g, rid)) g, _ = deployment() g.scenario["runtime"] = {} paths = search(g) - self.assertTrue(any(p["classification"] == "conditional" and - "arm_deployment" in [s["rule"] for s in p["steps"]] for p in paths)) + self.assertTrue( + any( + p["classification"] == "conditional" and "arm_deployment" in [s["rule"] for s in p["steps"]] + for p in paths + ) + ) self.assertFalse(any(p["classification"] == "modelled_reachable" for p in paths)) g.coverage = [c for c in g.coverage if c["family"] != "deny_assignments"] - deployment_paths = [p for p in search(g) if - "arm_deployment" in [s["rule"] for s in p["steps"]]] + deployment_paths = [p for p in search(g) if "arm_deployment" in [s["rule"] for s in p["steps"]]] self.assertTrue(deployment_paths) self.assertTrue(all(p["classification"] == "unknown" for p in deployment_paths)) @@ -135,61 +257,193 @@ def test_arm_deployment_through_lighthouse(self): definition = g.rows("role_definitions")[0] definition.properties["roleType"] = "BuiltInRole" registration = S + "/providers/Microsoft.ManagedServices/registrationDefinitions/d" - g.add("lighthouse", T, {"id": registration, "properties": { - "managedByTenantId": "managing", "authorizations": [{ - "principalId": "operator", "roleDefinitionId": definition.data["id"]}]}}, "delegation") - g.add("lighthouse_assignments", T, {"id": S + "/providers/Microsoft.ManagedServices/registrationAssignments/a", - "properties": {"registrationDefinitionId": registration}}, "delegation-scope") + g.add( + "lighthouse", + T, + { + "id": registration, + "properties": { + "managedByTenantId": "managing", + "authorizations": [{"principalId": "operator", "roleDefinitionId": definition.data["id"]}], + }, + }, + "delegation", + ) + g.add( + "lighthouse_assignments", + T, + { + "id": S + "/providers/Microsoft.ManagedServices/registrationAssignments/a", + "properties": {"registrationDefinitionId": registration}, + }, + "delegation-scope", + ) g.scenario["controlled"] = [{"tenant": "managing", "id": "operator"}] - reached = [p for p in search(g) if p["classification"] == "modelled_reachable" and - "arm_deployment" in [s["rule"] for s in p["steps"]]] + reached = [ + p + for p in search(g) + if p["classification"] == "modelled_reachable" and "arm_deployment" in [s["rule"] for s in p["steps"]] + ] self.assertTrue(reached) self.assertTrue(all("lighthouse" in [s["rule"] for s in p["steps"]] for p in reached)) - self.assert_pair(g, "arm_deployment", lambda g: - definition.properties["permissions"][0].update(notActions=["Microsoft.Resources/deployments/validate/action"])) + self.assert_pair( + g, + "arm_deployment", + lambda g: definition.properties["permissions"][0].update( + notActions=["Microsoft.Resources/deployments/validate/action"] + ), + ) def test_automation_execution(self): - g,r=execution("Microsoft.Automation/automationAccounts",["Microsoft.Automation/automationAccounts/runbooks/draft/content/write","Microsoft.Automation/automationAccounts/runbooks/publish/action","Microsoft.Automation/automationAccounts/jobs/write"],["automation_sandbox","network","token_endpoint"]) - self.assert_pair(g,"automation_execution",lambda g:g.scenario["runtime"][r.lower()].update(automation_sandbox=False)) + g, r = execution( + "Microsoft.Automation/automationAccounts", + [ + "Microsoft.Automation/automationAccounts/runbooks/draft/content/write", + "Microsoft.Automation/automationAccounts/runbooks/publish/action", + "Microsoft.Automation/automationAccounts/jobs/write", + ], + ["automation_sandbox", "network", "token_endpoint"], + ) + self.assert_pair( + g, "automation_execution", lambda g: g.scenario["runtime"][r.lower()].update(automation_sandbox=False) + ) def test_functions_execution(self): - g,r=execution("Microsoft.Web/sites",["Microsoft.Web/sites/write","Microsoft.Web/sites/publish/action"],["function_deployment","invocation","network","token_endpoint"]) - self.assert_pair(g,"functions_execution",lambda g:g.scenario["runtime"][r.lower()].update(invocation=False)) + g, r = execution( + "Microsoft.Web/sites", + ["Microsoft.Web/sites/write", "Microsoft.Web/sites/publish/action"], + ["function_deployment", "invocation", "network", "token_endpoint"], + ) + self.assert_pair(g, "functions_execution", lambda g: g.scenario["runtime"][r.lower()].update(invocation=False)) def test_logic_execution(self): - g,r=execution("Microsoft.Logic/workflows",["Microsoft.Logic/workflows/write","Microsoft.Logic/workflows/triggers/run/action"],["logic_consumption","identity_action","invocation","network","token_endpoint"]) - self.assert_pair(g,"logic_execution",lambda g:g.scenario["runtime"][r.lower()].update(identity_action=False)) + g, r = execution( + "Microsoft.Logic/workflows", + ["Microsoft.Logic/workflows/write", "Microsoft.Logic/workflows/triggers/run/action"], + ["logic_consumption", "identity_action", "invocation", "network", "token_endpoint"], + ) + self.assert_pair(g, "logic_execution", lambda g: g.scenario["runtime"][r.lower()].update(identity_action=False)) def test_uami_attach(self): - g,r=execution("Microsoft.Compute/virtualMachines",["Microsoft.Compute/virtualMachines/write","Microsoft.Compute/virtualMachines/extensions/write","Microsoft.ManagedIdentity/userAssignedIdentities/assign/action"],["vm_agent","network","token_endpoint"]) + g, r = execution( + "Microsoft.Compute/virtualMachines", + [ + "Microsoft.Compute/virtualMachines/write", + "Microsoft.Compute/virtualMachines/extensions/write", + "Microsoft.ManagedIdentity/userAssignedIdentities/assign/action", + ], + ["vm_agent", "network", "token_endpoint"], + ) g.rows("resources")[1].data.pop("identity") - u=S+"/resourceGroups/rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/u" - g.add("resources",T,{"id":u,"type":"Microsoft.ManagedIdentity/userAssignedIdentities","properties":{"principalId":"mi"}},"uami") - self.assert_pair(g,"uami_attach",lambda g:g.rows("role_definitions")[0].properties["permissions"][0].update(actions=["Microsoft.Compute/virtualMachines/write","Microsoft.Compute/virtualMachines/extensions/write"])) + u = S + "/resourceGroups/rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/u" + g.add( + "resources", + T, + {"id": u, "type": "Microsoft.ManagedIdentity/userAssignedIdentities", "properties": {"principalId": "mi"}}, + "uami", + ) + self.assert_pair( + g, + "uami_attach", + lambda g: ( + g.rows("role_definitions")[0] + .properties["permissions"][0] + .update( + actions=[ + "Microsoft.Compute/virtualMachines/write", + "Microsoft.Compute/virtualMachines/extensions/write", + ] + ) + ), + ) def test_federation(self): - g=base();u=S+"/resourceGroups/rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/u" - g.add("resources",T,{"id":u,"type":"Microsoft.ManagedIdentity/userAssignedIdentities","properties":{"principalId":"mi"}},"uami") - g.add("federated_credentials",T,{"id":u+"/federatedIdentityCredentials/f","objectId":u,"issuer":"https://issuer.example","subject":"subject","audiences":["api://AzureADTokenExchange"]},"federation") - role(g,"user",actions=["Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials/write"]) - role(g,"mi",data=[SECRET],scope=VAULT,suffix="target") - g.scenario["assertions"]=[{"issuer":"https://issuer.example","subject":"subject","audience":"api://AzureADTokenExchange","controlled":True}] - self.assert_pair(g,"federation",lambda g:g.scenario["assertions"][0].update(subject="different")) + g = base() + u = S + "/resourceGroups/rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/u" + g.add( + "resources", + T, + {"id": u, "type": "Microsoft.ManagedIdentity/userAssignedIdentities", "properties": {"principalId": "mi"}}, + "uami", + ) + g.add( + "federated_credentials", + T, + { + "id": u + "/federatedIdentityCredentials/f", + "objectId": u, + "issuer": "https://issuer.example", + "subject": "subject", + "audiences": ["api://AzureADTokenExchange"], + }, + "federation", + ) + role(g, "user", actions=["Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials/write"]) + role(g, "mi", data=[SECRET], scope=VAULT, suffix="target") + g.scenario["assertions"] = [ + { + "issuer": "https://issuer.example", + "subject": "subject", + "audience": "api://AzureADTokenExchange", + "controlled": True, + } + ] + self.assert_pair(g, "federation", lambda g: g.scenario["assertions"][0].update(subject="different")) def test_lighthouse(self): - g,r=execution("Microsoft.Compute/virtualMachines",[],["vm_agent","network","token_endpoint"]) - g.objects=[x for x in g.objects if x.family!="role_assignments" or x.properties["principalId"]!="user"] - g.index["role_assignments"]=[x for x in g.index["role_assignments"] if x.properties["principalId"]!="user"] - rid=g.rows("role_definitions")[0].data["id"] - g.rows("role_definitions")[0].properties.update(roleType="BuiltInRole",permissions=[{"actions":["Microsoft.Compute/virtualMachines/write","Microsoft.Compute/virtualMachines/extensions/write"],"notActions":[],"dataActions":[],"notDataActions":[]}]) - g.add("lighthouse",T,{"id":S+"/providers/Microsoft.ManagedServices/registrationDefinitions/d","properties":{"managedByTenantId":"managing","authorizations":[{"principalId":"operator","roleDefinitionId":rid}],"eligibleAuthorizations":[{"principalId":"operator","roleDefinitionId":rid}]}},"delegation") - g.add("lighthouse_assignments",T,{"id":S+"/providers/Microsoft.ManagedServices/registrationAssignments/a","properties":{"registrationDefinitionId":g.rows("lighthouse")[0].data["id"]}},"delegation-scope") - g.scenario["controlled"]=[{"tenant":"managing","id":"operator"}] - self.assert_pair(g,"lighthouse",lambda g:g.rows("role_definitions")[0].properties.update(roleType="CustomRole")) + g, r = execution("Microsoft.Compute/virtualMachines", [], ["vm_agent", "network", "token_endpoint"]) + g.objects = [x for x in g.objects if x.family != "role_assignments" or x.properties["principalId"] != "user"] + g.index["role_assignments"] = [x for x in g.index["role_assignments"] if x.properties["principalId"] != "user"] + rid = g.rows("role_definitions")[0].data["id"] + g.rows("role_definitions")[0].properties.update( + roleType="BuiltInRole", + permissions=[ + { + "actions": [ + "Microsoft.Compute/virtualMachines/write", + "Microsoft.Compute/virtualMachines/extensions/write", + ], + "notActions": [], + "dataActions": [], + "notDataActions": [], + } + ], + ) + g.add( + "lighthouse", + T, + { + "id": S + "/providers/Microsoft.ManagedServices/registrationDefinitions/d", + "properties": { + "managedByTenantId": "managing", + "authorizations": [{"principalId": "operator", "roleDefinitionId": rid}], + "eligibleAuthorizations": [{"principalId": "operator", "roleDefinitionId": rid}], + }, + }, + "delegation", + ) + g.add( + "lighthouse_assignments", + T, + { + "id": S + "/providers/Microsoft.ManagedServices/registrationAssignments/a", + "properties": {"registrationDefinitionId": g.rows("lighthouse")[0].data["id"]}, + }, + "delegation-scope", + ) + g.scenario["controlled"] = [{"tenant": "managing", "id": "operator"}] + self.assert_pair( + g, "lighthouse", lambda g: g.rows("role_definitions")[0].properties.update(roleType="CustomRole") + ) def test_unknown_runtime_and_no_vault_write_secret_jump(self): - g,r=execution("Microsoft.Compute/virtualMachines",["Microsoft.Compute/virtualMachines/write","Microsoft.Compute/virtualMachines/extensions/write"],[]) - paths=search(g) - self.assertTrue(any(p["classification"]=="conditional" for p in paths)) - g=base();role(g,"user",actions=["Microsoft.KeyVault/vaults/write"]) - self.assertFalse(any(p["classification"]=="modelled_reachable" for p in search(g))) + g, r = execution( + "Microsoft.Compute/virtualMachines", + ["Microsoft.Compute/virtualMachines/write", "Microsoft.Compute/virtualMachines/extensions/write"], + [], + ) + paths = search(g) + self.assertTrue(any(p["classification"] == "conditional" for p in paths)) + g = base() + role(g, "user", actions=["Microsoft.KeyVault/vaults/write"]) + self.assertFalse(any(p["classification"] == "modelled_reachable" for p in search(g))) diff --git a/plugins/detection-hunting/attack-path-workbench/tests/test_azure_permissions.py b/plugins/detection-hunting/attack-path-workbench/tests/test_azure_permissions.py index 340925b..a09a437 100644 --- a/plugins/detection-hunting/attack-path-workbench/tests/test_azure_permissions.py +++ b/plugins/detection-hunting/attack-path-workbench/tests/test_azure_permissions.py @@ -1,6 +1,7 @@ """Authorization truth table encoded before the evaluator implementation.""" import copy import unittest + from attackpath.azure.model import Graph from attackpath.azure.permissions import evaluate diff --git a/plugins/detection-hunting/attack-path-workbench/tests/test_azure_schemas.py b/plugins/detection-hunting/attack-path-workbench/tests/test_azure_schemas.py index f7bc27a..643fcb4 100644 --- a/plugins/detection-hunting/attack-path-workbench/tests/test_azure_schemas.py +++ b/plugins/detection-hunting/attack-path-workbench/tests/test_azure_schemas.py @@ -1,13 +1,14 @@ """Validate actual CLI artifacts against the shipped Azure JSON Schemas.""" import copy import json -from pathlib import Path import tempfile import unittest -from jsonschema import Draft202012Validator, ValidationError +from pathlib import Path + from azure_fixtures import write_bundle -from test_azure_paths import base, role, NOW, SECRET, VAULT, T, S -from test_azure_sdk_cli import run_cli, PLUGIN +from jsonschema import Draft202012Validator, ValidationError +from test_azure_paths import NOW, SECRET, VAULT, S, T, base, role +from test_azure_sdk_cli import PLUGIN, run_cli def validate(document, name): diff --git a/plugins/detection-hunting/attack-path-workbench/tests/test_azure_sdk_cli.py b/plugins/detection-hunting/attack-path-workbench/tests/test_azure_sdk_cli.py index 67f7001..1556e3e 100644 --- a/plugins/detection-hunting/attack-path-workbench/tests/test_azure_sdk_cli.py +++ b/plugins/detection-hunting/attack-path-workbench/tests/test_azure_sdk_cli.py @@ -1,36 +1,73 @@ """Exercise the installed-style CLI with actual SDK evidence and no repository imports.""" + import hashlib import json import os -from pathlib import Path import stat import subprocess import sys import tempfile import unittest -from azure_fixtures import write_bundle -from test_azure_paths import base, role, execution, T, S, NOW, SECRET, VAULT +from pathlib import Path + from attackpath.azure.input import load from attackpath.azure.model import AzureError from attackpath.azure.report import build +from azure_fixtures import write_bundle +from test_azure_paths import NOW, SECRET, VAULT, S, T, base, execution, role PLUGIN = Path(__file__).resolve().parents[1] def run_cli(*args, script=PLUGIN / "scripts/attackpath.py"): env = dict(os.environ, PYTHONPATH="", PYTHONNOUSERSITE="1", PYTHONDONTWRITEBYTECODE="1") - return subprocess.run([sys.executable, str(script), *map(str, args)], env=env, - cwd=script.parent, capture_output=True, text=True, timeout=30) + return subprocess.run( + [sys.executable, str(script), *map(str, args)], + env=env, + cwd=script.parent, + capture_output=True, + text=True, + timeout=30, + ) class SDKCLI(unittest.TestCase): def test_execution_chains_through_sdk_and_cli(self): cases = [ - ("Microsoft.Compute/virtualMachines", ["Microsoft.Compute/virtualMachines/write", "Microsoft.Compute/virtualMachines/extensions/write"], ["vm_agent","network","token_endpoint"], "vm_execution"), - ("Microsoft.Compute/virtualMachines", ["Microsoft.Resources/deployments/*", "Microsoft.Compute/virtualMachines/extensions/write"], ["vm_agent","network","token_endpoint"], "arm_deployment"), - ("Microsoft.Automation/automationAccounts", ["Microsoft.Automation/automationAccounts/runbooks/draft/content/write", "Microsoft.Automation/automationAccounts/runbooks/publish/action", "Microsoft.Automation/automationAccounts/jobs/write"], ["automation_sandbox","network","token_endpoint"], "automation_execution"), - ("Microsoft.Web/sites", ["Microsoft.Web/sites/write", "Microsoft.Web/sites/publish/action"], ["function_deployment","invocation","network","token_endpoint"], "functions_execution"), - ("Microsoft.Logic/workflows", ["Microsoft.Logic/workflows/write", "Microsoft.Logic/workflows/triggers/run/action"], ["logic_consumption","identity_action","invocation","network","token_endpoint"], "logic_execution"), + ( + "Microsoft.Compute/virtualMachines", + ["Microsoft.Compute/virtualMachines/write", "Microsoft.Compute/virtualMachines/extensions/write"], + ["vm_agent", "network", "token_endpoint"], + "vm_execution", + ), + ( + "Microsoft.Compute/virtualMachines", + ["Microsoft.Resources/deployments/*", "Microsoft.Compute/virtualMachines/extensions/write"], + ["vm_agent", "network", "token_endpoint"], + "arm_deployment", + ), + ( + "Microsoft.Automation/automationAccounts", + [ + "Microsoft.Automation/automationAccounts/runbooks/draft/content/write", + "Microsoft.Automation/automationAccounts/runbooks/publish/action", + "Microsoft.Automation/automationAccounts/jobs/write", + ], + ["automation_sandbox", "network", "token_endpoint"], + "automation_execution", + ), + ( + "Microsoft.Web/sites", + ["Microsoft.Web/sites/write", "Microsoft.Web/sites/publish/action"], + ["function_deployment", "invocation", "network", "token_endpoint"], + "functions_execution", + ), + ( + "Microsoft.Logic/workflows", + ["Microsoft.Logic/workflows/write", "Microsoft.Logic/workflows/triggers/run/action"], + ["logic_consumption", "identity_action", "invocation", "network", "token_endpoint"], + "logic_execution", + ), ] for kind, actions, runtime, rule in cases: with self.subTest(rule=rule), tempfile.TemporaryDirectory() as tmp: @@ -40,13 +77,24 @@ def test_execution_chains_through_sdk_and_cli(self): result = run_cli("analyze-azure", "--input", manifest, "--as-of", NOW, "--output", output) self.assertEqual(0, result.returncode, result.stderr) report = json.loads((output / "report.json").read_text()) - paths = [p for p in report["paths"] if p["classification"] == "modelled_reachable" and rule in [s["rule"] for s in p["steps"]]] + paths = [ + p + for p in report["paths"] + if p["classification"] == "modelled_reachable" and rule in [s["rule"] for s in p["steps"]] + ] self.assertTrue(paths, report["paths"]) ledger = {r["record_id"] for r in report["evidence_ledger"]["records"]} self.assertTrue(all(set(p["evidence"]) <= ledger for p in paths)) self.assertIn("prerequisites", paths[0]["steps"][0]) self.assertTrue(paths[0]["assumptions"]) - self.assertTrue(all(set(d["evidence"]) <= ledger for p in report["paths"] for s in p["steps"] for d in s["prerequisites"])) + self.assertTrue( + all( + set(d["evidence"]) <= ledger + for p in report["paths"] + for s in p["steps"] + for d in s["prerequisites"] + ) + ) self.assertEqual(stat.S_IMODE(output.stat().st_mode), 0o700) marker = json.loads((output / "completion.json").read_text()) for name, digest in marker["files"].items(): @@ -56,63 +104,134 @@ def test_execution_chains_through_sdk_and_cli(self): g.scenario["runtime"][resource.lower()][runtime[0]] = False manifest = write_bundle(tmp, g) negative = build(load(manifest, NOW)) - self.assertFalse(any(p["classification"] == "modelled_reachable" and rule in [s["rule"] for s in p["steps"]] for p in negative["paths"])) + self.assertFalse( + any( + p["classification"] == "modelled_reachable" and rule in [s["rule"] for s in p["steps"]] + for p in negative["paths"] + ) + ) def test_federation_sdk_cli(self): g = base() u = S + "/resourceGroups/rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/u" - g.add("resources", T, {"id":u,"type":"Microsoft.ManagedIdentity/userAssignedIdentities","properties":{"principalId":"mi"}}, "uami") - g.add("federated_credentials", T, {"id":u+"/federatedIdentityCredentials/f","objectId":u,"issuer":"https://issuer.example","subject":"subject","audiences":["api://AzureADTokenExchange"]}, "fic") + g.add( + "resources", + T, + {"id": u, "type": "Microsoft.ManagedIdentity/userAssignedIdentities", "properties": {"principalId": "mi"}}, + "uami", + ) + g.add( + "federated_credentials", + T, + { + "id": u + "/federatedIdentityCredentials/f", + "objectId": u, + "issuer": "https://issuer.example", + "subject": "subject", + "audiences": ["api://AzureADTokenExchange"], + }, + "fic", + ) role(g, "user", actions=["Microsoft.ManagedIdentity/userAssignedIdentities/federatedIdentityCredentials/write"]) role(g, "mi", data=[SECRET], scope=VAULT, suffix="downstream") - g.scenario["assertions"] = [{"issuer":"https://issuer.example","subject":"subject","audience":"api://AzureADTokenExchange","controlled":True}] + g.scenario["assertions"] = [ + { + "issuer": "https://issuer.example", + "subject": "subject", + "audience": "api://AzureADTokenExchange", + "controlled": True, + } + ] with tempfile.TemporaryDirectory() as tmp: manifest = write_bundle(tmp, g) out = Path(tmp) / "report" result = run_cli("analyze-azure", "--input", manifest, "--as-of", NOW, "--output", out) self.assertEqual(0, result.returncode, result.stderr) report = json.loads((out / "report.json").read_text()) - self.assertTrue(any(p["classification"] == "modelled_reachable" and "federation" in [s["rule"] for s in p["steps"]] for p in report["paths"])) + self.assertTrue( + any( + p["classification"] == "modelled_reachable" and "federation" in [s["rule"] for s in p["steps"]] + for p in report["paths"] + ) + ) def test_incomplete_stale_and_tampered_sdk_evidence(self): - g = base(); role(g, "user", data=[SECRET], scope=VAULT) + g = base() + role(g, "user", data=[SECRET], scope=VAULT) with tempfile.TemporaryDirectory() as tmp: - for kwargs in ({"status":"partial"}, {"observed_at":"2026-09-27T12:00:00Z"}): + for kwargs in ({"status": "partial"}, {"observed_at": "2026-09-27T12:00:00Z"}): manifest = write_bundle(tmp, g, **kwargs) report = build(load(manifest, NOW)) self.assertFalse(any(p["classification"] == "modelled_reachable" for p in report["paths"])) self.assertTrue(any(r["quality"] for r in report["evidence_ledger"]["records"])) manifest = write_bundle(tmp, g) - data = json.loads(manifest.read_text()); data["sources"][0]["sha256"] = "0"*64 + data = json.loads(manifest.read_text()) + data["sources"][0]["sha256"] = "0" * 64 manifest.write_text(json.dumps(data)) - with self.assertRaisesRegex(AzureError, "integrity_mismatch"): load(manifest, NOW) - result = run_cli("analyze-azure", "--input", manifest, "--as-of", NOW, "--output", Path(tmp)/"bad") + with self.assertRaisesRegex(AzureError, "integrity_mismatch"): + load(manifest, NOW) + result = run_cli("analyze-azure", "--input", manifest, "--as-of", NOW, "--output", Path(tmp) / "bad") self.assertEqual(2, result.returncode) self.assertNotIn(tmp, result.stderr) - self.assertFalse((Path(tmp)/"bad/completion.json").exists()) + self.assertFalse((Path(tmp) / "bad/completion.json").exists()) def test_read_only_collection_plan(self): with tempfile.TemporaryDirectory() as tmp: - scope = Path(tmp)/"scope.json" - scope.write_text(json.dumps({"schema_version":"attackpath.azure.scope/v1", "tenants":[{"id":T,"scopes":[S],"groups":["group"],"applications":["app"],"service_principals":["sp"],"administrative_units":["au"],"resources":[VAULT],"user_assigned_identities":[S+"/resourceGroups/rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/u"],"management_groups":["mg"]}]})) - out = Path(tmp)/"plan" + scope = Path(tmp) / "scope.json" + scope.write_text( + json.dumps( + { + "schema_version": "attackpath.azure.scope/v1", + "tenants": [ + { + "id": T, + "scopes": [S], + "groups": ["group"], + "applications": ["app"], + "service_principals": ["sp"], + "administrative_units": ["au"], + "resources": [VAULT], + "user_assigned_identities": [ + S + + "/resourceGroups/rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/u" + ], + "management_groups": ["mg"], + } + ], + } + ) + ) + out = Path(tmp) / "plan" result = run_cli("plan-azure-collection", "--scope-file", scope, "--output", out) self.assertEqual(0, result.returncode, result.stderr) - plan = json.loads((out/"collection-plan.json").read_text()) + plan = json.loads((out / "collection-plan.json").read_text()) self.assertEqual(30, len({r["family"] for r in plan["requests"]})) - self.assertTrue(all((r["method"]=="GET" or (r["family"]=="resource_inventory" and r["method"]=="POST")) and r["reference"].startswith("https://learn.microsoft.com/") for r in plan["requests"])) + self.assertTrue( + all( + (r["method"] == "GET" or (r["family"] == "resource_inventory" and r["method"] == "POST")) + and r["reference"].startswith("https://learn.microsoft.com/") + for r in plan["requests"] + ) + ) self.assertTrue(all(r["permission"] and r["api"] and r["scopes"] for r in plan["requests"])) self.assertIn("cops.evidence/v1", plan["expected_evidence"]["envelope"]) def test_resource_graph_inventory_is_explicit_and_scope_bounded(self): from attackpath.azure.collection import plan, source_contract from attackpath.azure.model import AzureError + with tempfile.TemporaryDirectory() as tmp: - scope = Path(tmp)/"scope.json" + scope = Path(tmp) / "scope.json" rg = S + "/resourceGroups/rg" mg = "/providers/Microsoft.Management/managementGroups/mg" - scope.write_text(json.dumps({"schema_version":"attackpath.azure.scope/v1", "tenants":[ - {"id": T, "scopes":[rg, VAULT, mg, "/"], "management_groups":["ancestry-only"]}]})) + scope.write_text( + json.dumps( + { + "schema_version": "attackpath.azure.scope/v1", + "tenants": [{"id": T, "scopes": [rg, VAULT, mg, "/"], "management_groups": ["ancestry-only"]}], + } + ) + ) result = plan(scope) inventory = [r for r in result["requests"] if r["family"] == "resource_inventory"] self.assertEqual(3, len(inventory)) @@ -125,7 +244,7 @@ def test_resource_graph_inventory_is_explicit_and_scope_bounded(self): self.assertIn("Microsoft.ResourceGraph/resources/read", request["permission"]) self.assertIn("resource-type read", request["permission"]) self.assertIn("$skipToken", request["pagination"]) - self.assertEqual({"$top":1000, "resultFormat":"objectArray"}, request["body"]["options"]) + self.assertEqual({"$top": 1000, "resultFormat": "objectArray"}, request["body"]["options"]) query = request["body"]["query"] self.assertIn("project id, type", query) self.assertNotIn("ancestry-only", json.dumps(request)) @@ -142,9 +261,10 @@ def test_resource_graph_inventory_is_explicit_and_scope_bounded(self): source_contract("resource_inventory", "2024-04-01") def test_partial_graph_and_private_deterministic_reports(self): - g = base(); role(g, "user", data=[SECRET], scope=VAULT) + g = base() + role(g, "user", data=[SECRET], scope=VAULT) g.scenario["pseudonymize"] = True - g.scenario["business_impact"] = {VAULT.lower():"sensitive-owner-name"} + g.scenario["business_impact"] = {VAULT.lower(): "sensitive-owner-name"} with tempfile.TemporaryDirectory() as tmp: manifest = write_bundle(tmp, g) first, second = build(load(manifest, NOW)), build(load(manifest, NOW)) @@ -154,15 +274,17 @@ def test_partial_graph_and_private_deterministic_reports(self): self.assertNotIn(private, text) ids = {n["id"] for n in first["graph"]["nodes"]} self.assertTrue(all(e["source"] in ids and e["target"] in ids for e in first["graph"]["edges"])) - write_bundle(tmp, g, limits={"expansions":1}) + write_bundle(tmp, g, limits={"expansions": 1}) partial = build(load(manifest, NOW)) self.assertEqual("partial", partial["status"]) self.assertIn("expansion_limit", partial["partial_reasons"]) def test_execution_assumptions_do_not_expose_resource_identifiers(self): - g, resource = execution("Microsoft.Compute/virtualMachines", - ["Microsoft.Compute/virtualMachines/write", "Microsoft.Compute/virtualMachines/extensions/write"], - ["vm_agent", "network", "token_endpoint"]) + g, resource = execution( + "Microsoft.Compute/virtualMachines", + ["Microsoft.Compute/virtualMachines/write", "Microsoft.Compute/virtualMachines/extensions/write"], + ["vm_agent", "network", "token_endpoint"], + ) g.scenario["pseudonymize"] = True with tempfile.TemporaryDirectory() as tmp: report = build(load(write_bundle(tmp, g), NOW)) @@ -172,7 +294,8 @@ def test_execution_assumptions_do_not_expose_resource_identifiers(self): self.assertTrue(any(p["assumptions"] for p in report["paths"])) def test_alternate_entitlement_survives_single_removal(self): - g = base(); role(g, "user", data=[SECRET], scope=VAULT, suffix="a") + g = base() + role(g, "user", data=[SECRET], scope=VAULT, suffix="a") role(g, "user", data=[SECRET], scope=VAULT, suffix="b") with tempfile.TemporaryDirectory() as tmp: report = build(load(write_bundle(tmp, g), NOW)) @@ -184,12 +307,15 @@ def test_alternate_entitlement_survives_single_removal(self): self.assertFalse(report["remediation"]["global_minimum_claimed"]) def test_existing_and_symlink_output_rejected(self): - g = base(); role(g, "user", data=[SECRET], scope=VAULT) + g = base() + role(g, "user", data=[SECRET], scope=VAULT) with tempfile.TemporaryDirectory() as tmp: manifest = write_bundle(tmp, g) - real = Path(tmp)/"real"; real.mkdir() - alias = Path(tmp)/"alias"; alias.symlink_to(real) - for output in (real, alias/"new"): + real = Path(tmp) / "real" + real.mkdir() + alias = Path(tmp) / "alias" + alias.symlink_to(real) + for output in (real, alias / "new"): result = run_cli("analyze-azure", "--input", manifest, "--as-of", NOW, "--output", output) self.assertEqual(2, result.returncode) - self.assertFalse((real/"new").exists()) + self.assertFalse((real / "new").exists()) diff --git a/plugins/detection-hunting/attack-path-workbench/tests/test_ingestion.py b/plugins/detection-hunting/attack-path-workbench/tests/test_ingestion.py index f611cb3..ba43b45 100644 --- a/plugins/detection-hunting/attack-path-workbench/tests/test_ingestion.py +++ b/plugins/detection-hunting/attack-path-workbench/tests/test_ingestion.py @@ -1,14 +1,13 @@ """Boundary tests for local, untrusted evidence files.""" import json import os -from pathlib import Path import tempfile import unittest +from pathlib import Path from attackpath.core import GateError, analyze from attackpath.ingestion import IngestError, Limits, RunBudget, parse_json, read_regular - FIXTURE = Path(__file__).resolve().parents[1] / "fixtures" / "illustrative" diff --git a/plugins/detection-hunting/attack-path-workbench/tests/test_workbench.py b/plugins/detection-hunting/attack-path-workbench/tests/test_workbench.py index 69b93fa..27aa2f0 100644 --- a/plugins/detection-hunting/attack-path-workbench/tests/test_workbench.py +++ b/plugins/detection-hunting/attack-path-workbench/tests/test_workbench.py @@ -5,19 +5,27 @@ import copy import json import os -from pathlib import Path import stat import subprocess import sys import tempfile import unittest +from pathlib import Path -from attackpath.core import (GateError, analyze, audit_report, canonical, - file_hash, load_json, query_intent, rank_paths, - trace_paths, validate_input) +from attackpath.core import ( + GateError, + analyze, + audit_report, + canonical, + file_hash, + load_json, + query_intent, + rank_paths, + trace_paths, + validate_input, +) from attackpath.search import SearchLimits - FIXTURE = Path(__file__).resolve().parents[1] / "fixtures" / "illustrative" diff --git a/plugins/detection-hunting/detection-quality-workbench/detectionquality/cli.py b/plugins/detection-hunting/detection-quality-workbench/detectionquality/cli.py index 366d964..d57e03d 100644 --- a/plugins/detection-hunting/detection-quality-workbench/detectionquality/cli.py +++ b/plugins/detection-hunting/detection-quality-workbench/detectionquality/cli.py @@ -98,7 +98,7 @@ def main(argv: list[str] | None = None) -> int: rule_files = {p.stem: p for p in args.rules_dir.glob("*.json")} fixture_files = {p.stem: p for p in args.fixtures_dir.glob("*.json")} - print(f"Running Detection Quality Test Suite...") + print("Running Detection Quality Test Suite...") print(f"{'Rule ID':<35} {'Platform':<15} {'Precision':<12} {'Recall':<10} {'Status'}") print("-" * 85) diff --git a/plugins/detection-hunting/detection-quality-workbench/detectionquality/evaluator.py b/plugins/detection-hunting/detection-quality-workbench/detectionquality/evaluator.py index 0a320d2..b6f6d01 100644 --- a/plugins/detection-hunting/detection-quality-workbench/detectionquality/evaluator.py +++ b/plugins/detection-hunting/detection-quality-workbench/detectionquality/evaluator.py @@ -2,7 +2,6 @@ from __future__ import annotations -import re from typing import Any from .models import CaseResult, FixtureCase, FixtureSuite, Rule diff --git a/plugins/detection-hunting/detection-quality-workbench/scripts/run_demo.py b/plugins/detection-hunting/detection-quality-workbench/scripts/run_demo.py index 3372c6e..1098986 100644 --- a/plugins/detection-hunting/detection-quality-workbench/scripts/run_demo.py +++ b/plugins/detection-hunting/detection-quality-workbench/scripts/run_demo.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 #!/usr/bin/env python3 """Deterministic demo runner for Detection Quality Workbench.""" @@ -13,9 +15,16 @@ if str(PLUGIN_ROOT) not in sys.path: sys.path.insert(0, str(PLUGIN_ROOT)) -from detectionquality.evaluator import evaluate_rule_suite -from detectionquality.models import FixtureSuite, Rule -from detectionquality.reporting import render_markdown_report +from detectionquality.evaluator import ( + evaluate_rule_suite, +) +from detectionquality.models import ( + FixtureSuite, + Rule, +) +from detectionquality.reporting import ( + render_markdown_report, +) def main() -> int: diff --git a/plugins/detection-hunting/detection-quality-workbench/scripts/validate_package.py b/plugins/detection-hunting/detection-quality-workbench/scripts/validate_package.py index 8c974bd..27d7751 100644 --- a/plugins/detection-hunting/detection-quality-workbench/scripts/validate_package.py +++ b/plugins/detection-hunting/detection-quality-workbench/scripts/validate_package.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 #!/usr/bin/env python3 """Offline package gate validator for Detection Quality Workbench.""" @@ -13,8 +15,13 @@ if str(PLUGIN_ROOT) not in sys.path: sys.path.insert(0, str(PLUGIN_ROOT)) -from detectionquality.evaluator import evaluate_rule_suite -from detectionquality.models import FixtureSuite, Rule +from detectionquality.evaluator import ( + evaluate_rule_suite, +) +from detectionquality.models import ( + FixtureSuite, + Rule, +) def validate() -> int: diff --git a/plugins/detection-hunting/detection-quality-workbench/tests/test_workbench.py b/plugins/detection-hunting/detection-quality-workbench/tests/test_workbench.py index 8f8e86e..c3584eb 100644 --- a/plugins/detection-hunting/detection-quality-workbench/tests/test_workbench.py +++ b/plugins/detection-hunting/detection-quality-workbench/tests/test_workbench.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Comprehensive unit tests for Detection Quality Workbench.""" import json @@ -12,11 +14,26 @@ if str(PLUGIN_ROOT) not in sys.path: sys.path.insert(0, str(PLUGIN_ROOT)) -from detectionquality.cli import main as cli_main -from detectionquality.evaluator import evaluate_fixture_case, evaluate_rule_suite -from detectionquality.models import FixtureCase, FixtureSuite, QualityError, Rule -from detectionquality.reporting import render_json_report, render_markdown_report -from detectionquality.static_analysis import analyze_rule_dependencies +from detectionquality.cli import ( + main as cli_main, +) +from detectionquality.evaluator import ( + evaluate_fixture_case, + evaluate_rule_suite, +) +from detectionquality.models import ( + FixtureCase, + FixtureSuite, + QualityError, + Rule, +) +from detectionquality.reporting import ( + render_json_report, + render_markdown_report, +) +from detectionquality.static_analysis import ( + analyze_rule_dependencies, +) @pytest.fixture diff --git a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/adapters.py b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/adapters.py index 9685642..2331701 100644 --- a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/adapters.py +++ b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/adapters.py @@ -79,7 +79,7 @@ def _router_body(host: str, bundle_hash: str) -> bytes: an offline evidence state. Never claim tenant validation, production readiness, or detection effectiveness. This package has no live-service connector and must not receive credentials. -""".encode("utf-8") +""".encode() def _openai_manifest() -> dict[str, Any]: diff --git a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/catalog.py b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/catalog.py index 5862b68..383642f 100644 --- a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/catalog.py +++ b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/catalog.py @@ -10,10 +10,10 @@ import copy import datetime as dt from typing import Any +from urllib.parse import urlsplit from .paths import EVALUATIONS_DIR, FIXTURES_DIR, HUNTS_DIR, PROFILES_DIR, write_json - AS_OF = "2026-09-17" ATTACK_VERSION = "18" MICROSOFT_TABLE_ROOT = "https://learn.microsoft.com/en-us/azure/azure-monitor/reference/tables" @@ -440,10 +440,18 @@ def build_profiles() -> list[dict[str, Any]]: def _reference_record(url: str) -> dict[str, str]: - if "learn.microsoft.com" in url or "csrc.nist.gov" in url or "attack.mitre.org" in url: - kind = "authoritative" - else: - kind = "vendor_framework" + try: + parsed = urlsplit(url) + hostname = parsed.hostname + except ValueError: + hostname = None + parsed = None + authoritative_hosts = {"learn.microsoft.com", "csrc.nist.gov", "attack.mitre.org"} + kind = ( + "authoritative" + if parsed is not None and parsed.scheme == "https" and hostname in authoritative_hosts + else "vendor_framework" + ) return { "title": REFERENCE_TITLES.get(url, f"Microsoft schema reference: {url.rsplit('/', 1)[-1]}"), "url": url, @@ -937,7 +945,7 @@ def build_hunts() -> list[dict[str, Any]]: def _fixture_events(hunt_id: str, ordinal: int, stage_count: int) -> list[dict[str, Any]]: - start = dt.datetime(2026, 1, 1, tzinfo=dt.timezone.utc) + dt.timedelta(days=int(hunt_id[1:]), minutes=ordinal) + start = dt.datetime(2026, 1, 1, tzinfo=dt.UTC) + dt.timedelta(days=int(hunt_id[1:]), minutes=ordinal) events: list[dict[str, Any]] = [] for stage in range(1, stage_count + 1): event = { @@ -1090,7 +1098,7 @@ def _curated_variant( def build_curated_fixtures(hunt: dict[str, Any]) -> dict[str, Any]: scenarios: list[dict[str, Any]] = [] ordinal = 0 - for category, count, purpose, outcome in CURATED_CATEGORIES: + for category, count, purpose, _outcome in CURATED_CATEGORIES: for variant in range(1, count + 1): ordinal += 1 events, expected_matches, expected_outcome, controls = _curated_variant( diff --git a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/cli.py b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/cli.py index fd62f26..6fa0ecf 100644 --- a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/cli.py +++ b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/cli.py @@ -13,7 +13,7 @@ from .errors import ContentError, HuntWorkbenchError from .evaluator import run_target from .parameters import load_parameter_file -from .paths import load_bounded_json, load_json +from .paths import load_bounded_json from .rendering import compatibility_report, explain_hunt, list_hunts, render_hunt from .reports import release_report diff --git a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/contracts.py b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/contracts.py index 13e8336..7720bee 100644 --- a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/contracts.py +++ b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/contracts.py @@ -9,13 +9,13 @@ import re from collections import Counter +from collections.abc import Iterable from datetime import date -from typing import Any, Iterable +from typing import Any from .errors import ContentError from .paths import FIXTURES_DIR, HUNTS_DIR, PROFILES_DIR, load_json - EXPECTED_HUNT_IDS = tuple(f"H{number:02d}" for number in range(1, 13)) SURFACES = ( "sentinel_analytics", diff --git a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/errors.py b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/errors.py index 49a5102..98951ce 100644 --- a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/errors.py +++ b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/errors.py @@ -2,7 +2,6 @@ from __future__ import annotations - EXIT_OK = 0 EXIT_CONTENT = 2 EXIT_COMPATIBILITY = 3 diff --git a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/evaluator.py b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/evaluator.py index 05e366e..eba3915 100644 --- a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/evaluator.py +++ b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/evaluator.py @@ -11,16 +11,16 @@ import random import re from collections import Counter +from collections.abc import Callable from dataclasses import dataclass -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from pathlib import Path -from typing import Any, Callable +from typing import Any from .contracts import load_profiles, validate_hunt, validate_library from .errors import ContentError, TestFailure from .paths import FIXTURES_DIR, load_hunt, load_hunts, load_json - GENERATOR_VERSION = "1.0.0" DEFAULT_SEED = 20260916 EXPECTED_CURATED_PER_HUNT = 48 @@ -60,7 +60,7 @@ def _parse_time(value: Any) -> datetime | None: return None if parsed.tzinfo is None: return None - return parsed.astimezone(timezone.utc) + return parsed.astimezone(UTC) _WINDOW_PATTERN = re.compile(r"^(?P[1-9][0-9]*)(?P[smhd])$") @@ -182,7 +182,7 @@ def reference_match(hunt: dict[str, Any], scenario_input: dict[str, Any]) -> Ref if len(workspace_ids) != 1: return _fail("workspace_scope_collision", raw_counts, evaluated_counts) - for left_stage, right_stage in zip(stage_ids, stage_ids[1:]): + for left_stage, right_stage in zip(stage_ids, stage_ids[1:], strict=False): join_out = ordered[left_stage].get("join_out") join_in = ordered[right_stage].get("join_in") if not isinstance(join_out, str) or not join_out: @@ -192,7 +192,7 @@ def reference_match(hunt: dict[str, Any], scenario_input: dict[str, Any]) -> Ref if join_out != join_in: return _fail("join_key_mismatch", raw_counts, evaluated_counts) - if any(right < left for left, right in zip(times, times[1:])): + if any(right < left for left, right in zip(times, times[1:], strict=False)): return _fail("event_order_invalid", raw_counts, evaluated_counts) window = _parse_window(scenario_input.get("correlation_window")) if window is None: @@ -284,7 +284,7 @@ def _evaluate_curated_scenario( def _base_generated_input(hunt: dict[str, Any], index: int) -> dict[str, Any]: stage_ids = _stage_ids(hunt) - base_time = datetime(2026, 1, 1, tzinfo=timezone.utc) + timedelta(days=index % 28) + base_time = datetime(2026, 1, 1, tzinfo=UTC) + timedelta(days=index % 28) events: list[dict[str, Any]] = [] for position, stage_id in enumerate(stage_ids): event: dict[str, Any] = { @@ -500,7 +500,7 @@ def _op_unsupported_schema(data: dict[str, Any], _rng: random.Random) -> None: def _run_generated( hunt: dict[str, Any], seed: int ) -> tuple[int, list[str], dict[str, int]]: - rng = random.Random(seed) + rng = random.Random(seed) # noqa: S311 - seeded offline fuzz generation, not security randomness failures: list[str] = [] operation_counts: Counter[str] = Counter() for index in range(EXPECTED_GENERATED_PER_HUNT): diff --git a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/package_validation.py b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/package_validation.py index f47ebe3..d659891 100644 --- a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/package_validation.py +++ b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/package_validation.py @@ -12,7 +12,6 @@ from .paths import PACKAGE_ROOT, SKILLS_DIR, load_json from .reports import release_subject - _REQUIRED_DOCS = ( "LICENSE", "README.md", diff --git a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/parameters.py b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/parameters.py index 71ad1cc..9177954 100644 --- a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/parameters.py +++ b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/parameters.py @@ -12,8 +12,7 @@ from typing import Any from .errors import ContentError -from .paths import load_bounded_json, load_json - +from .paths import load_bounded_json _UUID = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$") _HOST = re.compile(r"^(?=.{1,253}$)(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?)(?:\.(?:[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?))*\.?$") @@ -50,7 +49,7 @@ def _utc(value: Any, label: str) -> str: raise ContentError(f"parameter {label} must be an ISO-8601 timestamp") from error if parsed.tzinfo is None or parsed.utcoffset() != dt.timedelta(0): raise ContentError(f"parameter {label} must include a UTC offset") - canonical = parsed.astimezone(dt.timezone.utc).isoformat().replace("+00:00", "Z") + canonical = parsed.astimezone(dt.UTC).isoformat().replace("+00:00", "Z") return f"datetime({canonical})" @@ -91,7 +90,7 @@ def _normalize_scalar(type_name: str, value: Any, definition: dict[str, Any]) -> if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password: raise ContentError(f"parameter {name} must be an HTTP(S) URL without user information") try: - parsed.port + _ = parsed.port except ValueError as error: raise ContentError(f"parameter {name} has an invalid port") from error hostname = parsed.hostname.rstrip(".") @@ -101,7 +100,7 @@ def _normalize_scalar(type_name: str, value: Any, definition: dict[str, Any]) -> ipaddress.ip_address(hostname) except ValueError: if not _HOST.fullmatch(hostname): - raise ContentError(f"parameter {name} must use a valid DNS name or IP address") + raise ContentError(f"parameter {name} must use a valid DNS name or IP address") from None return text if type_name == "file_hash": text = _string(value, name, 64) diff --git a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/paths.py b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/paths.py index 04451a6..cc2fa29 100644 --- a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/paths.py +++ b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/paths.py @@ -7,7 +7,6 @@ from pathlib import Path from typing import Any - PACKAGE_ROOT = Path(__file__).resolve().parent.parent REPOSITORY_ROOT = PACKAGE_ROOT.parent HUNTS_DIR = PACKAGE_ROOT / "hunts" diff --git a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/release_build.py b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/release_build.py index 7680e2b..b3b18d8 100644 --- a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/release_build.py +++ b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/release_build.py @@ -6,7 +6,7 @@ import re import stat import zipfile -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -18,7 +18,6 @@ from .rendering import compatibility_report from .reports import EXTERNAL_EVIDENCE_SCHEMA, INTEGRITY_EVIDENCE_SCHEMA, release_report, release_subject - ARCHIVE_NAME = "sentinel-hunt-workbench.zip" _SECRET_PATTERNS = { "private_key": re.compile(rb"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"), @@ -164,7 +163,7 @@ def build_release_artifacts(seed: int = 20260916) -> dict[str, Any]: }, } index: dict[str, dict[str, str]] = {} - completed_at = datetime.now(timezone.utc).isoformat() + completed_at = datetime.now(UTC).isoformat() for kind, payload in payloads.items(): payload_path, payload_hash = _evidence_file(f"{kind}.payload.json", payload) wrapper = { diff --git a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/reports.py b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/reports.py index 4e2b6b5..2a3cdfc 100644 --- a/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/reports.py +++ b/plugins/detection-hunting/sentinel-hunt-workbench/huntwb/reports.py @@ -13,10 +13,11 @@ import re import stat import zipfile +from collections.abc import Callable from datetime import datetime from pathlib import Path, PurePosixPath from statistics import median -from typing import Any, Callable +from typing import Any from .adapters import verify_adapters from .contracts import validate_library @@ -34,7 +35,6 @@ sha256_text, ) - REPORT_SCHEMA = "huntwb.release-report/v2" EXTERNAL_EVIDENCE_SCHEMA = "huntwb.external-evidence/v2" SUBJECT_SCHEMA = "huntwb.release-subject/v1" @@ -780,7 +780,7 @@ def _validate_release_archive( expected_names = [f"sentinel-hunt-workbench/{name}" for name in sorted(subject["artifacts"])] if [entry.filename for entry in entries] != expected_names: raise ValueError("archive inventory or order differs from the release subject") - for entry, digest in zip(entries, (subject["artifacts"][name] for name in sorted(subject["artifacts"]))): + for entry, digest in zip(entries, (subject["artifacts"][name] for name in sorted(subject["artifacts"])), strict=False): if ( entry.date_time != (1980, 1, 1, 0, 0, 0) or entry.compress_type != zipfile.ZIP_STORED diff --git a/plugins/detection-hunting/sentinel-hunt-workbench/tests/test_workbench.py b/plugins/detection-hunting/sentinel-hunt-workbench/tests/test_workbench.py index 7d863df..7fcf016 100644 --- a/plugins/detection-hunting/sentinel-hunt-workbench/tests/test_workbench.py +++ b/plugins/detection-hunting/sentinel-hunt-workbench/tests/test_workbench.py @@ -12,7 +12,8 @@ from unittest import mock from huntwb import reports -from huntwb.cli import main, _load_external_evidence +from huntwb.catalog import _reference_record +from huntwb.cli import _load_external_evidence, main from huntwb.errors import ContentError from huntwb.package_validation import validate_package from huntwb.parameters import load_parameter_file @@ -23,6 +24,16 @@ class WorkbenchTests(unittest.TestCase): + def test_authoritative_reference_requires_exact_hostname(self) -> None: + self.assertEqual(_reference_record("https://learn.microsoft.com/docs")['kind'], "authoritative") + for url in ( + "https://learn.microsoft.com.evil.example/docs", + "https://evil.example/learn.microsoft.com/docs", + "https://learn.microsoft.com@evil.example/docs", + ): + with self.subTest(url=url): + self.assertEqual(_reference_record(url)['kind'], "vendor_framework") + def test_package_inventory_and_adapters(self) -> None: report = validate_package() self.assertEqual(report["status"], "passed") @@ -46,6 +57,18 @@ def test_unknown_surface_is_rejected(self) -> None: with self.assertRaises(ContentError): render_hunt("H01", "unknown_surface", parameters) + def test_reference_classification_uses_parsed_https_host(self) -> None: + trusted = _reference_record("https://learn.microsoft.com/en-us/azure/sentinel/hunting") + self.assertEqual(trusted["kind"], "authoritative") + for url in ( + "https://untrusted.example/path?next=learn.microsoft.com", + "https://learn.microsoft.com.attacker.example/path", + "https://learn.microsoft.com@untrusted.example/path", + "http://learn.microsoft.com/path", + ): + with self.subTest(url=url): + self.assertEqual(_reference_record(url)["kind"], "vendor_framework") + def test_archive_is_reproducible_and_contains_subject(self) -> None: subject = release_subject() first = archive_bytes(subject) diff --git a/plugins/detection-hunting/soc-investigation-workbench/investigationwb/cli.py b/plugins/detection-hunting/soc-investigation-workbench/investigationwb/cli.py index dd518ad..da48c69 100644 --- a/plugins/detection-hunting/soc-investigation-workbench/investigationwb/cli.py +++ b/plugins/detection-hunting/soc-investigation-workbench/investigationwb/cli.py @@ -5,8 +5,8 @@ import argparse import json import os -from pathlib import Path import sys +from pathlib import Path from typing import Any from .engine import ContractError, digest, import_result, next_steps, report, revise, validate diff --git a/plugins/detection-hunting/soc-investigation-workbench/investigationwb/engine.py b/plugins/detection-hunting/soc-investigation-workbench/investigationwb/engine.py index f539d31..6d728be 100644 --- a/plugins/detection-hunting/soc-investigation-workbench/investigationwb/engine.py +++ b/plugins/detection-hunting/soc-investigation-workbench/investigationwb/engine.py @@ -2,12 +2,12 @@ from __future__ import annotations +import json +import re from copy import deepcopy from datetime import datetime from fractions import Fraction from hashlib import sha256 -import json -import re from typing import Any Document = dict[str, Any] diff --git a/plugins/detection-hunting/soc-investigation-workbench/investigationwb/files.py b/plugins/detection-hunting/soc-investigation-workbench/investigationwb/files.py index 42f49e5..b8f6d6e 100644 --- a/plugins/detection-hunting/soc-investigation-workbench/investigationwb/files.py +++ b/plugins/detection-hunting/soc-investigation-workbench/investigationwb/files.py @@ -1,9 +1,9 @@ """Bounded reads from regular files, checked on the opened descriptor.""" -from contextlib import contextmanager import os -from pathlib import Path import stat +from contextlib import contextmanager +from pathlib import Path from .engine import ContractError, require @@ -14,8 +14,8 @@ def _open_windows(path: Path, nofollow: bool) -> int: # Windows has no O_NOFOLLOW. Inspect the opened handle itself before adopting # it as a Python descriptor, so a final reparse point cannot redirect a read. import ctypes - from ctypes import wintypes import msvcrt + from ctypes import wintypes class AttributeTagInfo(ctypes.Structure): _fields_ = [("attributes", wintypes.DWORD), ("tag", wintypes.DWORD)] diff --git a/plugins/detection-hunting/soc-investigation-workbench/investigationwb/handoff.py b/plugins/detection-hunting/soc-investigation-workbench/investigationwb/handoff.py index 79141c8..df140cb 100644 --- a/plugins/detection-hunting/soc-investigation-workbench/investigationwb/handoff.py +++ b/plugins/detection-hunting/soc-investigation-workbench/investigationwb/handoff.py @@ -9,7 +9,6 @@ import json import os from pathlib import Path -from typing import Any from .engine import Document, digest, hypothesis_status, next_steps, validate diff --git a/plugins/detection-hunting/soc-investigation-workbench/investigationwb/intake.py b/plugins/detection-hunting/soc-investigation-workbench/investigationwb/intake.py index 14295c4..42b9674 100644 --- a/plugins/detection-hunting/soc-investigation-workbench/investigationwb/intake.py +++ b/plugins/detection-hunting/soc-investigation-workbench/investigationwb/intake.py @@ -8,15 +8,15 @@ from __future__ import annotations import csv -from datetime import datetime, timedelta, timezone -from hashlib import sha256 import io import json -from pathlib import Path import re +from datetime import UTC, datetime, timedelta, timezone +from hashlib import sha256 +from pathlib import Path from typing import Any -from .engine import ContractError, Document, alias, digest, fields, prose, require, utc, validate +from .engine import ContractError, Document, alias, require, utc, validate from .files import read_regular ID_PATTERN = re.compile(r"[a-z][a-z0-9_-]{0,63}\Z") @@ -60,7 +60,7 @@ def parse_timestamp(value: Any) -> str: # Try unix epoch in seconds try: epoch = float(cleaned) - dt = datetime.fromtimestamp(epoch, tz=timezone.utc) + dt = datetime.fromtimestamp(epoch, tz=UTC) return dt.strftime("%Y-%m-%dT%H:%M:%SZ") except (ValueError, OverflowError, OSError) as exc: raise ContractError(f"Unrecognized timestamp format: {value}") from exc @@ -68,7 +68,7 @@ def parse_timestamp(value: Any) -> str: base_time, offset = match.groups() dt = datetime.fromisoformat(base_time) if not offset or offset == "Z": - dt = dt.replace(tzinfo=timezone.utc) + dt = dt.replace(tzinfo=UTC) else: # Normalize offset format (e.g. +0500 -> +05:00) norm_offset = offset @@ -78,7 +78,7 @@ def parse_timestamp(value: Any) -> str: minutes = int(norm_offset[4:6]) sign = 1 if norm_offset[0] == "+" else -1 delta_tz = timezone(sign * timedelta(hours=hours, minutes=minutes)) - dt = dt.replace(tzinfo=delta_tz).astimezone(timezone.utc) + dt = dt.replace(tzinfo=delta_tz).astimezone(UTC) return dt.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -482,7 +482,7 @@ def ingest_sources( def get_or_create_entity(kind: str, raw_value: str) -> str: clean_val = raw_value.strip().lower() - key = sha256(f"{case_id}:{kind}:{clean_val}".encode("utf-8")).hexdigest() + key = sha256(f"{case_id}:{kind}:{clean_val}".encode()).hexdigest() if key in entity_key_to_alias: return entity_key_to_alias[key] @@ -555,7 +555,7 @@ def get_or_create_entity(kind: str, raw_value: str) -> str: assessments.append({ "hypothesis_id": "benign-explanation", "stance": "refutes", - "reason": f"Alert severity or failure code contradicts expected benign workflow.", + "reason": "Alert severity or failure code contradicts expected benign workflow.", }) else: assessments.append({ @@ -566,7 +566,7 @@ def get_or_create_entity(kind: str, raw_value: str) -> str: assessments.append({ "hypothesis_id": "malicious-activity", "stance": "refutes", - "reason": f"Clean completion or normal telemetry weakens compromise hypothesis.", + "reason": "Clean completion or normal telemetry weakens compromise hypothesis.", }) evidence_list.append({ diff --git a/plugins/detection-hunting/soc-investigation-workbench/investigationwb/m365.py b/plugins/detection-hunting/soc-investigation-workbench/investigationwb/m365.py index 66325e2..2c637f3 100644 --- a/plugins/detection-hunting/soc-investigation-workbench/investigationwb/m365.py +++ b/plugins/detection-hunting/soc-investigation-workbench/investigationwb/m365.py @@ -1,7 +1,7 @@ """Offline cross-source leads from validated, tenant-bound evidence envelopes.""" +import re from collections import defaultdict from hashlib import sha256 -import re from cops.evidence import EvidenceError, validate_envelope, validate_receipt diff --git a/plugins/detection-hunting/soc-investigation-workbench/investigationwb/vendor.py b/plugins/detection-hunting/soc-investigation-workbench/investigationwb/vendor.py index d7869f9..1ceecf5 100644 --- a/plugins/detection-hunting/soc-investigation-workbench/investigationwb/vendor.py +++ b/plugins/detection-hunting/soc-investigation-workbench/investigationwb/vendor.py @@ -2,18 +2,18 @@ from __future__ import annotations -from hashlib import sha256 import json import os -from pathlib import Path, PurePosixPath import re import shutil import stat import subprocess import tempfile +from hashlib import sha256 +from pathlib import Path, PurePosixPath from typing import Any -from .engine import ContractError, Document, HASH, digest, fields, next_steps, require, validate +from .engine import HASH, ContractError, Document, digest, fields, next_steps, require, validate from .files import open_regular, read_regular PACKAGE = Path(__file__).resolve().parent.parent diff --git a/plugins/detection-hunting/soc-investigation-workbench/scripts/investigate.py b/plugins/detection-hunting/soc-investigation-workbench/scripts/investigate.py index c987d5b..ee3815f 100644 --- a/plugins/detection-hunting/soc-investigation-workbench/scripts/investigate.py +++ b/plugins/detection-hunting/soc-investigation-workbench/scripts/investigate.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Run directly from any current directory without installation.""" -from pathlib import Path import sys +from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) diff --git a/plugins/detection-hunting/soc-investigation-workbench/scripts/m365-offline-demo.py b/plugins/detection-hunting/soc-investigation-workbench/scripts/m365-offline-demo.py index 1b6eae2..404050e 100644 --- a/plugins/detection-hunting/soc-investigation-workbench/scripts/m365-offline-demo.py +++ b/plugins/detection-hunting/soc-investigation-workbench/scripts/m365-offline-demo.py @@ -1,17 +1,18 @@ #!/usr/bin/env python3 """Synthetic Microsoft 365 collection and correlation without credentials.""" import json -from pathlib import Path import sys +from pathlib import Path from tempfile import TemporaryDirectory ROOT = Path(__file__).resolve().parents[4] sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from investigationwb.m365 import correlate # noqa: E402 + from cops.connectors import Checkpoint, GraphCollection, Response, collect, preview # noqa: E402 from cops.evidence import canonical # noqa: E402 -from investigationwb.m365 import correlate # noqa: E402 TENANT = '00000000-0000-0000-0000-000000000001' USER = '00000000-0000-0000-0000-000000000002' diff --git a/plugins/detection-hunting/soc-investigation-workbench/scripts/validate-package.py b/plugins/detection-hunting/soc-investigation-workbench/scripts/validate-package.py index ad6db6b..ae8b4ef 100644 --- a/plugins/detection-hunting/soc-investigation-workbench/scripts/validate-package.py +++ b/plugins/detection-hunting/soc-investigation-workbench/scripts/validate-package.py @@ -1,11 +1,13 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 #!/usr/bin/env python3 """Repository packaging checks; default gate requires a real vendor snapshot.""" import argparse import json -from pathlib import Path import re import sys +from pathlib import Path PACKAGE = Path(__file__).resolve().parents[1] @@ -22,7 +24,11 @@ def find_repository_root(package): sys.path.insert(0, str(PACKAGE)) from investigationwb.cli import read_json -from investigationwb.engine import ContractError, require, validate +from investigationwb.engine import ( + ContractError, + require, + validate, +) from investigationwb.vendor import verify diff --git a/plugins/detection-hunting/threat-intelligence-enrichment/scripts/run_demo.py b/plugins/detection-hunting/threat-intelligence-enrichment/scripts/run_demo.py index 7bd9ec1..35d14c6 100644 --- a/plugins/detection-hunting/threat-intelligence-enrichment/scripts/run_demo.py +++ b/plugins/detection-hunting/threat-intelligence-enrichment/scripts/run_demo.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Synthetic, no-network package demo.""" import sys from pathlib import Path diff --git a/plugins/detection-hunting/threat-intelligence-enrichment/tests/test_core.py b/plugins/detection-hunting/threat-intelligence-enrichment/tests/test_core.py index d3066f1..dbda1a2 100644 --- a/plugins/detection-hunting/threat-intelligence-enrichment/tests/test_core.py +++ b/plugins/detection-hunting/threat-intelligence-enrichment/tests/test_core.py @@ -1,6 +1,7 @@ import unittest -from threat_intel import Approval, Cache, Indicator, Source, enrich, normalize, conflicts +from threat_intel import Approval, Cache, Indicator, Source, conflicts, enrich, normalize + from cops.connectors.interfaces import Response diff --git a/plugins/detection-hunting/threat-intelligence-enrichment/threat_intel/__init__.py b/plugins/detection-hunting/threat-intelligence-enrichment/threat_intel/__init__.py index 2679693..558f997 100644 --- a/plugins/detection-hunting/threat-intelligence-enrichment/threat_intel/__init__.py +++ b/plugins/detection-hunting/threat-intelligence-enrichment/threat_intel/__init__.py @@ -1,5 +1,5 @@ """Read-only, source-specific threat intelligence enrichment.""" -from .core import (Approval, Cache, Indicator, Source, enrich, normalize, conflicts) +from .core import Approval, Cache, Indicator, Source, conflicts, enrich, normalize __all__ = ['Approval', 'Cache', 'Indicator', 'Source', 'enrich', 'normalize', 'conflicts'] diff --git a/plugins/detection-hunting/threat-intelligence-enrichment/threat_intel/core.py b/plugins/detection-hunting/threat-intelligence-enrichment/threat_intel/core.py index acdf5e1..af27c10 100644 --- a/plugins/detection-hunting/threat-intelligence-enrichment/threat_intel/core.py +++ b/plugins/detection-hunting/threat-intelligence-enrichment/threat_intel/core.py @@ -8,7 +8,7 @@ import re import time from dataclasses import dataclass, field -from datetime import datetime, timezone +from datetime import UTC, datetime from urllib.parse import quote, urlsplit from cops.connectors.interfaces import Request @@ -125,7 +125,7 @@ def put(self, key, result, now): def _stamp(seconds): - return datetime.fromtimestamp(seconds, timezone.utc).isoformat().replace('+00:00', 'Z') + return datetime.fromtimestamp(seconds, UTC).isoformat().replace('+00:00', 'Z') def _time(value): @@ -139,7 +139,7 @@ def _time(value): parsed = datetime.fromisoformat(value.replace('Z', '+00:00')) if parsed.tzinfo is None: return None - return parsed.astimezone(timezone.utc).isoformat().replace('+00:00', 'Z') + return parsed.astimezone(UTC).isoformat().replace('+00:00', 'Z') except ValueError: pass return None diff --git a/plugins/identity-access/entra-identity-workbench/entrawb/graph.py b/plugins/identity-access/entra-identity-workbench/entrawb/graph.py index ff96a12..d296432 100644 --- a/plugins/identity-access/entra-identity-workbench/entrawb/graph.py +++ b/plugins/identity-access/entra-identity-workbench/entrawb/graph.py @@ -4,7 +4,7 @@ from typing import Any -from .models import EntraError, IdentityEdge, IdentityGraph, IdentityNode +from .models import IdentityEdge, IdentityGraph, IdentityNode def build_identity_graph(manifest: dict[str, Any], sources: dict[str, Any]) -> IdentityGraph: diff --git a/plugins/identity-access/entra-identity-workbench/entrawb/reporting.py b/plugins/identity-access/entra-identity-workbench/entrawb/reporting.py index 1ff45fe..46066f3 100644 --- a/plugins/identity-access/entra-identity-workbench/entrawb/reporting.py +++ b/plugins/identity-access/entra-identity-workbench/entrawb/reporting.py @@ -2,7 +2,7 @@ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime from typing import Any from .models import IdentityGraph, ReviewHypothesis @@ -27,7 +27,7 @@ def build_report_summary(graph: IdentityGraph, hypotheses: list[ReviewHypothesis def render_json_report(graph: IdentityGraph, hypotheses: list[ReviewHypothesis]) -> dict[str, Any]: """Construct typed dictionary matching report.schema.json.""" summary = build_report_summary(graph, hypotheses) - now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + now_iso = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") return { "schema_version": "entra.identity-report/v1", "tenant_id": graph.tenant_id, diff --git a/plugins/identity-access/entra-identity-workbench/scripts/run_demo.py b/plugins/identity-access/entra-identity-workbench/scripts/run_demo.py index 6b89122..0234b93 100644 --- a/plugins/identity-access/entra-identity-workbench/scripts/run_demo.py +++ b/plugins/identity-access/entra-identity-workbench/scripts/run_demo.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 #!/usr/bin/env python3 """Deterministic demo runner for Entra Identity Workbench.""" @@ -13,10 +15,18 @@ if str(PLUGIN_ROOT) not in sys.path: sys.path.insert(0, str(PLUGIN_ROOT)) -from entrawb.analysis import analyze_identity_graph -from entrawb.graph import build_identity_graph -from entrawb.ingestion import ingest_tenant_export -from entrawb.reporting import render_markdown_report +from entrawb.analysis import ( + analyze_identity_graph, +) +from entrawb.graph import ( + build_identity_graph, +) +from entrawb.ingestion import ( + ingest_tenant_export, +) +from entrawb.reporting import ( + render_markdown_report, +) def main() -> int: @@ -25,7 +35,7 @@ def main() -> int: print(f"Error: Fixture manifest not found at {manifest_path}", file=sys.stderr) return 1 - print(f"=== Entra Identity Workbench Demo ===") + print("=== Entra Identity Workbench Demo ===") print(f"Loading tenant export from: {manifest_path.relative_to(REPO_ROOT)}") manifest, sources = ingest_tenant_export(manifest_path) diff --git a/plugins/identity-access/entra-identity-workbench/scripts/validate_package.py b/plugins/identity-access/entra-identity-workbench/scripts/validate_package.py index 774f86d..02043b1 100644 --- a/plugins/identity-access/entra-identity-workbench/scripts/validate_package.py +++ b/plugins/identity-access/entra-identity-workbench/scripts/validate_package.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 #!/usr/bin/env python3 """Offline package gate validator for Entra Identity Workbench.""" @@ -13,10 +15,19 @@ if str(PLUGIN_ROOT) not in sys.path: sys.path.insert(0, str(PLUGIN_ROOT)) -from entrawb.analysis import analyze_identity_graph -from entrawb.graph import build_identity_graph -from entrawb.ingestion import ingest_tenant_export -from entrawb.reporting import render_json_report, render_markdown_report +from entrawb.analysis import ( + analyze_identity_graph, +) +from entrawb.graph import ( + build_identity_graph, +) +from entrawb.ingestion import ( + ingest_tenant_export, +) +from entrawb.reporting import ( + render_json_report, + render_markdown_report, +) def validate() -> int: diff --git a/plugins/identity-access/entra-identity-workbench/tests/test_workbench.py b/plugins/identity-access/entra-identity-workbench/tests/test_workbench.py index 27b1da7..93aa319 100644 --- a/plugins/identity-access/entra-identity-workbench/tests/test_workbench.py +++ b/plugins/identity-access/entra-identity-workbench/tests/test_workbench.py @@ -1,7 +1,8 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Comprehensive unit tests for the Entra Identity Workbench.""" import json -import subprocess import sys from pathlib import Path @@ -13,11 +14,21 @@ if str(PLUGIN_ROOT) not in sys.path: sys.path.insert(0, str(PLUGIN_ROOT)) -from entrawb.analysis import analyze_identity_graph -from entrawb.graph import build_identity_graph -from entrawb.ingestion import ingest_tenant_export -from entrawb.models import EntraError, IdentityGraph, IdentityNode, IdentityEdge -from entrawb.reporting import render_json_report, render_markdown_report +from entrawb.analysis import ( + analyze_identity_graph, +) +from entrawb.graph import ( + build_identity_graph, +) +from entrawb.ingestion import ( + ingest_tenant_export, +) +from entrawb.models import ( + EntraError, + IdentityEdge, + IdentityGraph, + IdentityNode, +) @pytest.fixture diff --git a/plugins/incident-response/incident-response-sandbox/incident_response/core.py b/plugins/incident-response/incident-response-sandbox/incident_response/core.py index fe2bc6b..fe81f31 100644 --- a/plugins/incident-response/incident-response-sandbox/incident_response/core.py +++ b/plugins/incident-response/incident-response-sandbox/incident_response/core.py @@ -6,11 +6,10 @@ import json import os import tempfile -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Literal, TypedDict - Action = Literal["isolate-host", "revoke-session"] PLAN_FIELDS = {"schema", "action", "tenant", "target", "parameters", "expected_state", "desired_state", "expires_at", "nonce", "approver_assertion", "plan_hash"} @@ -60,7 +59,7 @@ def _expiry(value: object) -> datetime: raise ActionError("invalid expiry") from error if parsed.tzinfo is None: raise ActionError("expiry must include a timezone") - return parsed.astimezone(timezone.utc) + return parsed.astimezone(UTC) def validate_plan(plan: object, *, now: datetime | None = None) -> Plan: @@ -76,7 +75,7 @@ def validate_plan(plan: object, *, now: datetime | None = None) -> Plan: if not isinstance(plan["parameters"], dict) or plan["parameters"]: raise ActionError("parameters must be an empty object for fixture actions") expiry = _expiry(plan["expires_at"]) - if expiry <= (now or datetime.now(timezone.utc)): + if expiry <= (now or datetime.now(UTC)): raise ActionError("plan expired") supplied = plan["plan_hash"] if not isinstance(supplied, str) or supplied != _digest({k: v for k, v in plan.items() if k != "plan_hash"}): diff --git a/plugins/incident-response/incident-response-sandbox/tests/test_core.py b/plugins/incident-response/incident-response-sandbox/tests/test_core.py index acf29f2..812298f 100644 --- a/plugins/incident-response/incident-response-sandbox/tests/test_core.py +++ b/plugins/incident-response/incident-response-sandbox/tests/test_core.py @@ -4,7 +4,7 @@ import sys import unittest -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) @@ -21,7 +21,7 @@ def fixture() -> dict: def plan() -> dict: return build_plan(action="isolate-host", tenant="example-tenant", target="host-1", - expires_at=(datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(), + expires_at=(datetime.now(UTC) + timedelta(hours=1)).isoformat(), nonce="unique-1", approver_assertion="analyst-1") diff --git a/plugins/logging-telemetry/security-logging-advisor/scripts/test_plugin.py b/plugins/logging-telemetry/security-logging-advisor/scripts/test_plugin.py index 6da541a..e921408 100644 --- a/plugins/logging-telemetry/security-logging-advisor/scripts/test_plugin.py +++ b/plugins/logging-telemetry/security-logging-advisor/scripts/test_plugin.py @@ -4,12 +4,12 @@ Automated test suite for Security Logging Advisor plugin validation and scanning capabilities. """ -import os -import sys import json +import os import shutil -import tempfile import subprocess +import sys +import tempfile import unittest # Paths @@ -105,7 +105,7 @@ def test_detect_technology_and_cloud_from_hcl(self): def test_prevent_credential_leakage(self): """Scenario: Prevent credential leakage in scanner output""" - secret_val = "SuperSecretPassword123!" + secret_val = "SuperSecretPassword123!" # noqa: S105 - synthetic fixture for secret-detection regression env_content = f"db_password = '{secret_val}'\n" with open(os.path.join(self.temp_dir, "config.env"), "w") as f: f.write(env_content) diff --git a/plugins/logging-telemetry/security-logging-advisor/scripts/validate-plugin.py b/plugins/logging-telemetry/security-logging-advisor/scripts/validate-plugin.py index 904a146..14b0884 100755 --- a/plugins/logging-telemetry/security-logging-advisor/scripts/validate-plugin.py +++ b/plugins/logging-telemetry/security-logging-advisor/scripts/validate-plugin.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """Validate the Security Logging Advisor in a checkout or portable package.""" from __future__ import annotations + import json -from pathlib import Path import re +from pathlib import Path SOURCE_PACKAGE = Path("plugins/logging-telemetry/security-logging-advisor") REQUIRED_FILES = [ diff --git a/plugins/logging-telemetry/security-logging-advisor/skills/cve-reachability/scripts/reachability-report.py b/plugins/logging-telemetry/security-logging-advisor/skills/cve-reachability/scripts/reachability-report.py index 841ad62..b21e2d5 100644 --- a/plugins/logging-telemetry/security-logging-advisor/skills/cve-reachability/scripts/reachability-report.py +++ b/plugins/logging-telemetry/security-logging-advisor/skills/cve-reachability/scripts/reachability-report.py @@ -4,9 +4,9 @@ import hashlib import json import os -from pathlib import Path import stat import sys +from pathlib import Path LIMIT = 8 * 1024 * 1024 STATUSES = ('confirmed', 'likely', 'potential', 'not_reachable', 'unresolved') diff --git a/plugins/logging-telemetry/security-logging-advisor/skills/repository-context/scripts/collect-repository-context.py b/plugins/logging-telemetry/security-logging-advisor/skills/repository-context/scripts/collect-repository-context.py index 221b098..2314b3b 100755 --- a/plugins/logging-telemetry/security-logging-advisor/skills/repository-context/scripts/collect-repository-context.py +++ b/plugins/logging-telemetry/security-logging-advisor/skills/repository-context/scripts/collect-repository-context.py @@ -8,17 +8,18 @@ Outputs findings as a structured JSON object. """ -import os -import sys +import errno +import io import json +import os import re -import errno import stat -import io +import sys +from urllib.parse import urlsplit # Directory and file ignore patterns IGNORE_DIRS = { - ".git", "node_modules", "venv", ".venv", "dist", "build", "target", + ".git", "node_modules", "venv", ".venv", "dist", "build", "target", ".gemini", "__pycache__", ".pytest_cache", ".mypy_cache", ".idea" } @@ -41,11 +42,24 @@ SECRET_EXTENSIONS = CONTENT_EXTENSIONS - {".tfvars"} CONTENT_MANIFESTS = {"package.json", "requirements.txt", "Pipfile", "pyproject.toml", "pom.xml", "build.gradle"} + +def _is_arm_template(content): + try: + document = json.loads(content) + schema = document.get("$schema") if isinstance(document, dict) else None + if not isinstance(schema, str): + return False + parsed = urlsplit(schema) + return parsed.scheme == "https" and parsed.hostname == "schema.management.azure.com" + except (json.JSONDecodeError, ValueError): + return False + + def open_windows_regular_file(path): """Inspect a Windows handle before adopting it as a Python descriptor.""" import ctypes - from ctypes import wintypes import msvcrt + from ctypes import wintypes class AttributeTagInfo(ctypes.Structure): _fields_ = [("attributes", wintypes.DWORD), ("tag", wintypes.DWORD)] @@ -209,10 +223,10 @@ def scan_repository(root_dir, max_dirs=None, max_files=None): results["partial_scan_notice"] = f"File budget of {effective_max_files} reached; scan is partial." break files_seen += 1 - + file_path = os.path.join(root, file) rel_path = os.path.relpath(file_path, root_dir) - + try: discovered = os.lstat(file_path) except OSError: @@ -290,7 +304,7 @@ def scan_repository(root_dir, max_dirs=None, max_files=None): elif ext == ".scala": results["languages"]["Scala"] = results["languages"].get("Scala", 0) + 1 elif ext == ".json": - if content is not None and "schema.management.azure.com" in content[:1024]: + if content is not None and _is_arm_template(content): if "Azure ARM Template" not in results["iac_and_cloud"]: results["iac_and_cloud"].append("Azure ARM Template") @@ -321,34 +335,35 @@ def scan_repository(root_dir, max_dirs=None, max_files=None): if "oci" in dep: if "Oracle Cloud" not in results["iac_and_cloud"]: results["iac_and_cloud"].append("Oracle Cloud") - except Exception: - pass + except (ValueError, TypeError): + # Malformed manifests provide no dependable dependency evidence. + continue elif file == "requirements.txt" or file == "Pipfile" or file == "pyproject.toml": for line in io.StringIO(content, newline=None): - l = line.lower() - if "django" in l: + normalized_line = line.lower() + if "django" in normalized_line: results["frameworks_and_libraries"].append("Python Framework: Django") - if "flask" in l: + if "flask" in normalized_line: results["frameworks_and_libraries"].append("Python Framework: Flask") - if "fastapi" in l: + if "fastapi" in normalized_line: results["frameworks_and_libraries"].append("Python Framework: FastAPI") - if "sqlalchemy" in l or "psycopg2" in l or "pymongo" in l or "redis" in l: + if "sqlalchemy" in normalized_line or "psycopg2" in normalized_line or "pymongo" in normalized_line or "redis" in normalized_line: results["databases"].append("Python DB Client") - if "jwt" in l or "oauth" in l or "auth0" in l: + if "jwt" in normalized_line or "oauth" in normalized_line or "auth0" in normalized_line: results["identity_and_auth"].append("Python Auth Library") - if "structlog" in l: + if "structlog" in normalized_line: results["frameworks_and_libraries"].append("Logging Library: structlog") - if "boto3" in l or "aws" in l: + if "boto3" in normalized_line or "aws" in normalized_line: if "AWS" not in results["iac_and_cloud"]: results["iac_and_cloud"].append("AWS") - if "google-cloud" in l: + if "google-cloud" in normalized_line: if "GCP" not in results["iac_and_cloud"]: results["iac_and_cloud"].append("GCP") - if "azure" in l: + if "azure" in normalized_line: if "Azure" not in results["iac_and_cloud"]: results["iac_and_cloud"].append("Azure") - if "oci" in l: + if "oci" in normalized_line: if "Oracle Cloud" not in results["iac_and_cloud"]: results["iac_and_cloud"].append("Oracle Cloud") @@ -412,7 +427,7 @@ def main(): args = parser.parse_args() if not os.path.isdir(args.target_dir): - print(json.dumps({"error": f"Path '{args.target_dir}' is not a valid directory."}, indent=2)) + print(json.dumps({"error": "Target is not a valid directory."}, indent=2)) sys.exit(1) scan_data = scan_repository(args.target_dir, max_dirs=args.max_dirs, max_files=args.max_files) diff --git a/plugins/logging-telemetry/telemetry-proof-pack/proofpack/correlator.py b/plugins/logging-telemetry/telemetry-proof-pack/proofpack/correlator.py index d89685d..f3efed7 100644 --- a/plugins/logging-telemetry/telemetry-proof-pack/proofpack/correlator.py +++ b/plugins/logging-telemetry/telemetry-proof-pack/proofpack/correlator.py @@ -5,10 +5,9 @@ import hashlib import json from datetime import datetime -from pathlib import Path from typing import Any -from .models import PipelineProof, ProofError, RunManifest, StageObservation +from .models import RunManifest, StageObservation from .redaction import redact_dict diff --git a/plugins/logging-telemetry/telemetry-proof-pack/proofpack/reporting.py b/plugins/logging-telemetry/telemetry-proof-pack/proofpack/reporting.py index b54a81f..1f7939d 100644 --- a/plugins/logging-telemetry/telemetry-proof-pack/proofpack/reporting.py +++ b/plugins/logging-telemetry/telemetry-proof-pack/proofpack/reporting.py @@ -32,7 +32,7 @@ def render_markdown_report(report_data: dict[str, Any]) -> str: "", "> [!IMPORTANT]", f"> Traces synthetic test marker **`{marker}`** through every pipeline stage from source emission to alert creation.", - f"> Proves whether the event survived collection, Cribl stream routing, SIEM indexing, and scheduled detection.", + "> Proves whether the event survived collection, Cribl stream routing, SIEM indexing, and scheduled detection.", "", "## Pipeline Verification Summary", "", diff --git a/plugins/logging-telemetry/telemetry-proof-pack/scripts/run_demo.py b/plugins/logging-telemetry/telemetry-proof-pack/scripts/run_demo.py index 59d12b2..55aef88 100644 --- a/plugins/logging-telemetry/telemetry-proof-pack/scripts/run_demo.py +++ b/plugins/logging-telemetry/telemetry-proof-pack/scripts/run_demo.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 #!/usr/bin/env python3 """Deterministic demo runner for Telemetry Proof Pack.""" @@ -14,16 +16,20 @@ sys.path.insert(0, str(PLUGIN_ROOT)) from proofpack.cli import load_evidence_files -from proofpack.correlator import correlate_pipeline_evidence +from proofpack.correlator import ( + correlate_pipeline_evidence, +) from proofpack.models import RunManifest -from proofpack.reporting import render_markdown_report +from proofpack.reporting import ( + render_markdown_report, +) def run_scenario(name: str, directory: Path) -> None: - print(f"\n================================================================================") + print("\n================================================================================") print(f" SCENARIO: {name}") print(f" Location: {directory.relative_to(REPO_ROOT)}") - print(f"================================================================================\n") + print("================================================================================\n") manifest_file = directory / "manifest.json" if not manifest_file.is_file(): diff --git a/plugins/logging-telemetry/telemetry-proof-pack/scripts/validate_package.py b/plugins/logging-telemetry/telemetry-proof-pack/scripts/validate_package.py index 40b8fd5..c12d8dd 100644 --- a/plugins/logging-telemetry/telemetry-proof-pack/scripts/validate_package.py +++ b/plugins/logging-telemetry/telemetry-proof-pack/scripts/validate_package.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 #!/usr/bin/env python3 """Offline package gate validator for Telemetry Proof Pack.""" @@ -14,7 +16,9 @@ sys.path.insert(0, str(PLUGIN_ROOT)) from proofpack.cli import load_evidence_files -from proofpack.correlator import correlate_pipeline_evidence +from proofpack.correlator import ( + correlate_pipeline_evidence, +) from proofpack.models import RunManifest diff --git a/plugins/logging-telemetry/telemetry-proof-pack/tests/test_workbench.py b/plugins/logging-telemetry/telemetry-proof-pack/tests/test_workbench.py index 48d6583..75061e3 100644 --- a/plugins/logging-telemetry/telemetry-proof-pack/tests/test_workbench.py +++ b/plugins/logging-telemetry/telemetry-proof-pack/tests/test_workbench.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Comprehensive unit tests for Telemetry Proof Pack.""" import json @@ -11,11 +13,20 @@ if str(PLUGIN_ROOT) not in sys.path: sys.path.insert(0, str(PLUGIN_ROOT)) -from proofpack.cli import load_evidence_files, main as cli_main -from proofpack.correlator import correlate_pipeline_evidence -from proofpack.models import PipelineProof, ProofError, RunManifest, StageObservation -from proofpack.redaction import redact_dict, redact_text -from proofpack.reporting import render_json_report, render_markdown_report +from proofpack.cli import load_evidence_files +from proofpack.cli import main as cli_main +from proofpack.correlator import ( + correlate_pipeline_evidence, +) +from proofpack.models import ( + ProofError, + RunManifest, +) +from proofpack.redaction import redact_dict +from proofpack.reporting import ( + render_json_report, + render_markdown_report, +) @pytest.fixture @@ -135,7 +146,7 @@ def test_redaction_utility(): redacted = redact_dict(data) assert redacted["api_key"] == "[REDACTED]" assert "[REDACTED_TOKEN]" in redacted["auth_header"] - assert redacted["nested"]["password"] == "[REDACTED]" + assert redacted["nested"]["password"] == "[REDACTED]" # noqa: S105 - schema label or operation identifier, not a credential assert redacted["nested"]["marker"] == "SYN-TEST-MARKER-99" diff --git a/plugins/offensive-security/attack-surface-planner/attack_surface_planner/core.py b/plugins/offensive-security/attack-surface-planner/attack_surface_planner/core.py index 68c91da..a574ec0 100644 --- a/plugins/offensive-security/attack-surface-planner/attack_surface_planner/core.py +++ b/plugins/offensive-security/attack-surface-planner/attack_surface_planner/core.py @@ -9,13 +9,12 @@ import ipaddress import json import os -from pathlib import Path import re import stat -from datetime import datetime, timezone +from datetime import UTC, datetime +from pathlib import Path from urllib.parse import urlsplit - KINDS = ("azure_resource_graph", "entra", "dns_ct", "public_endpoint") MAX_MANIFEST_BYTES = 128 * 1024 MAX_SOURCE_BYTES = 4 * 1024 * 1024 @@ -51,9 +50,9 @@ def _utc(value: object, label: str) -> str: parsed = datetime.fromisoformat(raw.replace("Z", "+00:00")) except ValueError: _fail(f"{label} must be an ISO-8601 UTC timestamp") - if parsed.tzinfo is None or parsed.utcoffset() != timezone.utc.utcoffset(parsed): + if parsed.tzinfo is None or parsed.utcoffset() != UTC.utcoffset(parsed): _fail(f"{label} must be an ISO-8601 UTC timestamp") - return parsed.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + return parsed.astimezone(UTC).isoformat().replace("+00:00", "Z") def _list(value: object, label: str, *, maximum: int = 100) -> list: diff --git a/plugins/offensive-security/attack-surface-planner/docs/PLAYBOOK.md b/plugins/offensive-security/attack-surface-planner/docs/PLAYBOOK.md index 7b7d707..9b3b7e8 100644 --- a/plugins/offensive-security/attack-surface-planner/docs/PLAYBOOK.md +++ b/plugins/offensive-security/attack-surface-planner/docs/PLAYBOOK.md @@ -79,6 +79,7 @@ Invoke this planner under the following concrete triggers: 2. **Execute Resumable Probes**: - Execute bounded probes with checkpointing: `python3 -m cops discovery active scan ...`. - Resuming skips completed probes without repeating side effects: `python3 -m cops discovery active resume ...`. + - Live TLS handshakes require TLS 1.2 or newer; this dispatcher does not negotiate TLS 1.0 or 1.1. 3. **Calibrate Fingerprint Uncertainty**: - Distinguish observed configurations from inferred fingerprints with explicit confidence and visible uncertainty reasons. 4. **Enforce Boundary & DNS Rebind Defense**: diff --git a/plugins/offensive-security/attack-surface-planner/scripts/plan.py b/plugins/offensive-security/attack-surface-planner/scripts/plan.py index b84a329..cb9b624 100644 --- a/plugins/offensive-security/attack-surface-planner/scripts/plan.py +++ b/plugins/offensive-security/attack-surface-planner/scripts/plan.py @@ -4,8 +4,8 @@ from __future__ import annotations import argparse -from pathlib import Path import sys +from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from attack_surface_planner.core import GateError, analyze, canonical # noqa: E402 diff --git a/plugins/offensive-security/attack-surface-planner/scripts/validate-package.py b/plugins/offensive-security/attack-surface-planner/scripts/validate-package.py index 9ecfe9d..4b069f7 100644 --- a/plugins/offensive-security/attack-surface-planner/scripts/validate-package.py +++ b/plugins/offensive-security/attack-surface-planner/scripts/validate-package.py @@ -3,9 +3,9 @@ from __future__ import annotations import json -from pathlib import Path import subprocess import sys +from pathlib import Path PACKAGE = Path(__file__).resolve().parents[1] CLI = PACKAGE / "scripts/plan.py" diff --git a/plugins/offensive-security/attack-surface-planner/tests/test_planner.py b/plugins/offensive-security/attack-surface-planner/tests/test_planner.py index 3ce7346..0d7a4a3 100644 --- a/plugins/offensive-security/attack-surface-planner/tests/test_planner.py +++ b/plugins/offensive-security/attack-surface-planner/tests/test_planner.py @@ -4,12 +4,12 @@ import hashlib import json -from pathlib import Path import shutil import socket import sys import tempfile import unittest +from pathlib import Path from unittest.mock import patch PACKAGE = Path(__file__).resolve().parents[1] diff --git a/plugins/offensive-security/foundry-agent-harness/foundryharness/models.py b/plugins/offensive-security/foundry-agent-harness/foundryharness/models.py index 9018af8..8234bd8 100644 --- a/plugins/offensive-security/foundry-agent-harness/foundryharness/models.py +++ b/plugins/offensive-security/foundry-agent-harness/foundryharness/models.py @@ -2,7 +2,7 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any diff --git a/plugins/offensive-security/foundry-agent-harness/foundryharness/reporting.py b/plugins/offensive-security/foundry-agent-harness/foundryharness/reporting.py index 3893815..438c4dc 100644 --- a/plugins/offensive-security/foundry-agent-harness/foundryharness/reporting.py +++ b/plugins/offensive-security/foundry-agent-harness/foundryharness/reporting.py @@ -2,7 +2,7 @@ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime from typing import Any from .models import ExecutionEvent, Scenario @@ -22,8 +22,8 @@ def render_json_report( return { "schema_version": "foundry.review-report/v1", - "report_id": f"REPORT-FOUNDRY-{int(datetime.now(timezone.utc).timestamp())}", - "evaluated_at": datetime.now(timezone.utc).isoformat(), + "report_id": f"REPORT-FOUNDRY-{int(datetime.now(UTC).timestamp())}", + "evaluated_at": datetime.now(UTC).isoformat(), "target_agent": target_agent, "environment": environment, "summary": { diff --git a/plugins/offensive-security/foundry-agent-harness/foundryharness/simulator.py b/plugins/offensive-security/foundry-agent-harness/foundryharness/simulator.py index 922cfd1..d20174a 100644 --- a/plugins/offensive-security/foundry-agent-harness/foundryharness/simulator.py +++ b/plugins/offensive-security/foundry-agent-harness/foundryharness/simulator.py @@ -2,7 +2,6 @@ from __future__ import annotations -import hashlib import time from typing import Any @@ -21,7 +20,6 @@ def evaluate_scenario( trace_id = f"trace-{scenario.scenario_id.lower()}-{int(time.time())}" surface = scenario.attacker_controlled_surface injected = scenario.injected_artifact.lower() - legit = scenario.legitimate_task # Determine simulated agent behavior based on scenario surface & injection attempted_tool: dict[str, Any] | None = None diff --git a/plugins/offensive-security/foundry-agent-harness/scripts/run_demo.py b/plugins/offensive-security/foundry-agent-harness/scripts/run_demo.py index 5ebacb4..85b3a8c 100644 --- a/plugins/offensive-security/foundry-agent-harness/scripts/run_demo.py +++ b/plugins/offensive-security/foundry-agent-harness/scripts/run_demo.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 #!/usr/bin/env python3 """Deterministic demo runner for Foundry Agent Harness.""" @@ -14,10 +16,18 @@ sys.path.insert(0, str(PLUGIN_ROOT)) from foundryharness.cli import load_scenarios -from foundryharness.gate import IndependentAuthorizationGate -from foundryharness.mock_sandbox import MockSandbox -from foundryharness.reporting import render_markdown_report -from foundryharness.simulator import evaluate_scenario +from foundryharness.gate import ( + IndependentAuthorizationGate, +) +from foundryharness.mock_sandbox import ( + MockSandbox, +) +from foundryharness.reporting import ( + render_markdown_report, +) +from foundryharness.simulator import ( + evaluate_scenario, +) def main() -> int: diff --git a/plugins/offensive-security/foundry-agent-harness/scripts/validate_package.py b/plugins/offensive-security/foundry-agent-harness/scripts/validate_package.py index 8c47665..c33f6e0 100644 --- a/plugins/offensive-security/foundry-agent-harness/scripts/validate_package.py +++ b/plugins/offensive-security/foundry-agent-harness/scripts/validate_package.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 #!/usr/bin/env python3 """Offline package gate validator for Foundry Agent Harness.""" @@ -14,11 +16,20 @@ sys.path.insert(0, str(PLUGIN_ROOT)) from foundryharness.cli import load_scenarios -from foundryharness.gate import IndependentAuthorizationGate -from foundryharness.mock_sandbox import MockSandbox +from foundryharness.gate import ( + IndependentAuthorizationGate, +) +from foundryharness.mock_sandbox import ( + MockSandbox, +) from foundryharness.models import Scenario -from foundryharness.reporting import render_json_report, render_markdown_report -from foundryharness.simulator import evaluate_scenario +from foundryharness.reporting import ( + render_json_report, + render_markdown_report, +) +from foundryharness.simulator import ( + evaluate_scenario, +) def validate() -> int: diff --git a/plugins/offensive-security/foundry-agent-harness/tests/test_workbench.py b/plugins/offensive-security/foundry-agent-harness/tests/test_workbench.py index 7a60523..d165ae2 100644 --- a/plugins/offensive-security/foundry-agent-harness/tests/test_workbench.py +++ b/plugins/offensive-security/foundry-agent-harness/tests/test_workbench.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Comprehensive unit test suite for Foundry Agent Harness.""" import json @@ -12,13 +14,27 @@ if str(PLUGIN_ROOT) not in sys.path: sys.path.insert(0, str(PLUGIN_ROOT)) -from foundryharness.cli import load_scenarios, main as cli_main -from foundryharness.gate import IndependentAuthorizationGate -from foundryharness.mock_sandbox import MockSandbox -from foundryharness.models import HarnessError, Scenario -from foundryharness.redaction import redact_dict, redact_text -from foundryharness.reporting import render_json_report, render_markdown_report -from foundryharness.simulator import evaluate_scenario +from foundryharness.cli import load_scenarios +from foundryharness.cli import ( + main as cli_main, +) +from foundryharness.gate import ( + IndependentAuthorizationGate, +) +from foundryharness.mock_sandbox import ( + MockSandbox, +) +from foundryharness.models import ( + HarnessError, + Scenario, +) +from foundryharness.redaction import ( + redact_dict, + redact_text, +) +from foundryharness.simulator import ( + evaluate_scenario, +) @pytest.fixture @@ -206,7 +222,7 @@ def test_redaction_masks_sensitive_tokens(): d = {"api_key": "secret_key_123", "token": "ey12345.token.signature", "normal": "hello"} redacted_dict = redact_dict(d) assert redacted_dict["api_key"] == "[REDACTED]" - assert redacted_dict["token"] == "[REDACTED]" + assert redacted_dict["token"] == "[REDACTED]" # noqa: S105 - schema label or operation identifier, not a credential assert redacted_dict["normal"] == "hello" diff --git a/plugins/offensive-security/offensive-engagement-workbench/scripts/demo.py b/plugins/offensive-security/offensive-engagement-workbench/scripts/demo.py index 3c8b44f..cfc9c21 100644 --- a/plugins/offensive-security/offensive-engagement-workbench/scripts/demo.py +++ b/plugins/offensive-security/offensive-engagement-workbench/scripts/demo.py @@ -4,8 +4,8 @@ from __future__ import annotations import json -from pathlib import Path import sys +from pathlib import Path PACKAGE_ROOT = Path(__file__).resolve().parents[1] REPO_ROOT = Path(__file__).resolve().parents[4] diff --git a/plugins/offensive-security/offensive-engagement-workbench/scripts/validate-package.py b/plugins/offensive-security/offensive-engagement-workbench/scripts/validate-package.py index 710fa18..8663666 100644 --- a/plugins/offensive-security/offensive-engagement-workbench/scripts/validate-package.py +++ b/plugins/offensive-security/offensive-engagement-workbench/scripts/validate-package.py @@ -4,10 +4,10 @@ from __future__ import annotations import json -from pathlib import Path import subprocess import sys import unittest +from pathlib import Path PACKAGE = Path(__file__).resolve().parents[1] REPO_ROOT = Path(__file__).resolve().parents[4] diff --git a/plugins/offensive-security/offensive-engagement-workbench/tests/test_workbench.py b/plugins/offensive-security/offensive-engagement-workbench/tests/test_workbench.py index 9644341..bd09a10 100644 --- a/plugins/offensive-security/offensive-engagement-workbench/tests/test_workbench.py +++ b/plugins/offensive-security/offensive-engagement-workbench/tests/test_workbench.py @@ -3,9 +3,9 @@ from __future__ import annotations import json -from pathlib import Path import sys import unittest +from pathlib import Path PACKAGE_ROOT = Path(__file__).resolve().parents[1] REPO_ROOT = Path(__file__).resolve().parents[4] @@ -14,8 +14,8 @@ sys.path.insert(0, str(PACKAGE_ROOT)) from offensive_engagement_workbench.core import run_engagement_plan_workflow # noqa: E402 + from cops.engagement import ( # noqa: E402 - EngagementIntakeError, IncompatibleWindowError, IncompleteBudgetError, IncompleteLiveRequestError, diff --git a/plugins/vulnerability-management/exposure-triage-workbench/exposuretriage/evaluator.py b/plugins/vulnerability-management/exposure-triage-workbench/exposuretriage/evaluator.py index db7046b..502f532 100644 --- a/plugins/vulnerability-management/exposure-triage-workbench/exposuretriage/evaluator.py +++ b/plugins/vulnerability-management/exposure-triage-workbench/exposuretriage/evaluator.py @@ -3,13 +3,11 @@ from __future__ import annotations import re -from datetime import datetime, timezone -from typing import Any +from datetime import UTC, datetime from .models import ( AdvisoryFact, AssetFact, - ComponentFact, ReachabilityFact, TriageBundle, TriageQueueItem, @@ -69,7 +67,7 @@ def check_inventory_freshness(timestamp_str: str, max_age_days: int = 30) -> str try: ts = timestamp_str.replace("Z", "+00:00") dt = datetime.fromisoformat(ts) - now = datetime.now(timezone.utc) + now = datetime.now(UTC) diff = now - dt if diff.days > max_age_days: return "stale" @@ -211,13 +209,13 @@ def evaluate_triage_bundle(bundle: TriageBundle) -> list[TriageQueueItem]: elif exposure_route in ("isolated", "airgapped"): priority_tier = "P4_LOW" rationale_parts.append( - f"Confirmed vulnerable version deployed in isolated or air-gapped environment." + "Confirmed vulnerable version deployed in isolated or air-gapped environment." ) action = "Monitor environment isolation; patch during scheduled service update." else: priority_tier = "P3_MEDIUM" rationale_parts.append( - f"Confirmed vulnerable version deployed with unconfirmed route exposure." + "Confirmed vulnerable version deployed with unconfirmed route exposure." ) action = "Confirm network path and schedule remediation." diff --git a/plugins/vulnerability-management/exposure-triage-workbench/exposuretriage/reporting.py b/plugins/vulnerability-management/exposure-triage-workbench/exposuretriage/reporting.py index c2af27d..de46545 100644 --- a/plugins/vulnerability-management/exposure-triage-workbench/exposuretriage/reporting.py +++ b/plugins/vulnerability-management/exposure-triage-workbench/exposuretriage/reporting.py @@ -2,7 +2,7 @@ from __future__ import annotations -from datetime import datetime, timezone +from datetime import UTC, datetime from typing import Any from .models import TriageBundle, TriageQueueItem @@ -19,7 +19,7 @@ def render_json_report(bundle: TriageBundle, queue: list[TriageQueueItem]) -> di return { "schema_version": "cops.exposure-triage-report/v1", "report_id": f"REPORT-{bundle.bundle_id}", - "evaluated_at": datetime.now(timezone.utc).isoformat(), + "evaluated_at": datetime.now(UTC).isoformat(), "summary": { "total_items": len(queue), "p1_critical_count": p1, diff --git a/plugins/vulnerability-management/exposure-triage-workbench/scripts/run_demo.py b/plugins/vulnerability-management/exposure-triage-workbench/scripts/run_demo.py index 2e930c5..5ba6e0c 100644 --- a/plugins/vulnerability-management/exposure-triage-workbench/scripts/run_demo.py +++ b/plugins/vulnerability-management/exposure-triage-workbench/scripts/run_demo.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 #!/usr/bin/env python3 """Deterministic demo runner for Exposure Triage Workbench.""" @@ -12,9 +14,15 @@ if str(PLUGIN_ROOT) not in sys.path: sys.path.insert(0, str(PLUGIN_ROOT)) -from exposuretriage.evaluator import evaluate_triage_bundle -from exposuretriage.ingestion import ingest_triage_bundle -from exposuretriage.reporting import render_markdown_report +from exposuretriage.evaluator import ( + evaluate_triage_bundle, +) +from exposuretriage.ingestion import ( + ingest_triage_bundle, +) +from exposuretriage.reporting import ( + render_markdown_report, +) def main() -> int: diff --git a/plugins/vulnerability-management/exposure-triage-workbench/scripts/validate_package.py b/plugins/vulnerability-management/exposure-triage-workbench/scripts/validate_package.py index 69d70ce..e0a1d62 100644 --- a/plugins/vulnerability-management/exposure-triage-workbench/scripts/validate_package.py +++ b/plugins/vulnerability-management/exposure-triage-workbench/scripts/validate_package.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 #!/usr/bin/env python3 """Offline package gate validator for Exposure Triage Workbench.""" @@ -13,9 +15,16 @@ if str(PLUGIN_ROOT) not in sys.path: sys.path.insert(0, str(PLUGIN_ROOT)) -from exposuretriage.evaluator import evaluate_triage_bundle -from exposuretriage.ingestion import ingest_triage_bundle -from exposuretriage.reporting import render_json_report, render_markdown_report +from exposuretriage.evaluator import ( + evaluate_triage_bundle, +) +from exposuretriage.ingestion import ( + ingest_triage_bundle, +) +from exposuretriage.reporting import ( + render_json_report, + render_markdown_report, +) def validate() -> int: diff --git a/plugins/vulnerability-management/exposure-triage-workbench/tests/test_workbench.py b/plugins/vulnerability-management/exposure-triage-workbench/tests/test_workbench.py index 57843af..505d156 100644 --- a/plugins/vulnerability-management/exposure-triage-workbench/tests/test_workbench.py +++ b/plugins/vulnerability-management/exposure-triage-workbench/tests/test_workbench.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Comprehensive unit test suite for Exposure Triage Workbench.""" import json @@ -12,10 +14,14 @@ if str(PLUGIN_ROOT) not in sys.path: sys.path.insert(0, str(PLUGIN_ROOT)) -from exposuretriage.evaluator import check_version_match, evaluate_triage_bundle, parse_version_tuple -from exposuretriage.ingestion import ingest_triage_bundle -from exposuretriage.models import AdvisoryFact, AssetFact, ComponentFact, ReachabilityFact, TriageBundle, TriageError -from exposuretriage.reporting import render_json_report, render_markdown_report +from exposuretriage.evaluator import ( + check_version_match, + evaluate_triage_bundle, +) +from exposuretriage.ingestion import ( + ingest_triage_bundle, +) +from exposuretriage.models import TriageError @pytest.fixture diff --git a/pyproject.toml b/pyproject.toml index d8f26f8..4618d64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ select = [ "S", # flake8-bandit (security checks) ] ignore = [ + "UP042", # Preserve public str/Enum formatting semantics. "S101", # Use of assert (standard in tests and internal checks) "S603", # subprocess call - check for execution of untrusted input "S607", # start process with partial executable path @@ -50,3 +51,8 @@ ignore = [ "tests/**" = ["S", "B"] "specs/**" = ["S", "B"] "scripts/**" = ["S108"] + +".agents/skills/*/tests/**" = ["S", "B"] +".claude/skills/*/tests/**" = ["S", "B"] +"plugins/**/tests/**" = ["S", "B"] +"**/test_plugin.py" = ["S", "B"] diff --git a/scripts/_template_common.py b/scripts/_template_common.py index 906c203..5ccacc3 100644 --- a/scripts/_template_common.py +++ b/scripts/_template_common.py @@ -8,7 +8,6 @@ import shutil from pathlib import Path - TEMPLATE_MARKER = ".portable-agent-template" SKIP_NAMES = {".git", "LICENSE", "__pycache__", ".DS_Store"} DISTRIBUTION_IGNORE = shutil.ignore_patterns( diff --git a/scripts/agent/check.py b/scripts/agent/check.py index 6cdab5f..5aa924d 100755 --- a/scripts/agent/check.py +++ b/scripts/agent/check.py @@ -11,7 +11,6 @@ from check_prerequisites import check_prerequisites from repository_files import repository_files - ROOT = Path(__file__).resolve().parents[2] diff --git a/scripts/agent/check_issue_coverage.py b/scripts/agent/check_issue_coverage.py index 1740d34..74816df 100755 --- a/scripts/agent/check_issue_coverage.py +++ b/scripts/agent/check_issue_coverage.py @@ -7,7 +7,6 @@ from __future__ import annotations import argparse -import os import re import subprocess import sys diff --git a/scripts/agent/check_prerequisites.py b/scripts/agent/check_prerequisites.py index 4af4217..49feacd 100644 --- a/scripts/agent/check_prerequisites.py +++ b/scripts/agent/check_prerequisites.py @@ -50,7 +50,7 @@ def check_prerequisites(root: Path = ROOT) -> bool: print(f"Python {'.'.join(map(str, sys.version_info[:3]))}: {sys.executable} ({environment})", flush=True) errors = [] requirements = [] - if sys.version_info < (3, 11): + if sys.version_info < (3, 11): # noqa: UP036 - diagnose unsupported Python runtimes errors.append("Python 3.11 or newer is required for contributor checks") else: try: diff --git a/scripts/agent/doctor.py b/scripts/agent/doctor.py index 799849f..b015999 100755 --- a/scripts/agent/doctor.py +++ b/scripts/agent/doctor.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 #!/usr/bin/env python3 """Compatibility wrapper for the contributor environment check.""" @@ -6,7 +8,6 @@ import sys from pathlib import Path - ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT)) diff --git a/scripts/agent/install_hooks.py b/scripts/agent/install_hooks.py index 7ea95ec..6e69483 100755 --- a/scripts/agent/install_hooks.py +++ b/scripts/agent/install_hooks.py @@ -66,7 +66,7 @@ def install_hooks() -> int: pre_push_path.write_text(PRE_PUSH_HOOK, encoding="utf-8") current_mode = os.stat(pre_push_path).st_mode - os.chmod(pre_push_path, current_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + os.chmod(pre_push_path, current_mode | stat.S_IXUSR) print(f"✅ Successfully installed COPS pre-push hook: {pre_push_path}") return 0 diff --git a/scripts/agent/install_prerequisites.py b/scripts/agent/install_prerequisites.py index 6d3cf8e..e7a8f4a 100644 --- a/scripts/agent/install_prerequisites.py +++ b/scripts/agent/install_prerequisites.py @@ -13,7 +13,10 @@ from cops.catalog import load_json, plugin_records # noqa: E402 from cops.prerequisites import ( # noqa: E402 - PrerequisiteError, install_command, process_tools, validate_prerequisites, + PrerequisiteError, + install_command, + process_tools, + validate_prerequisites, ) diff --git a/scripts/agent/repository_files.py b/scripts/agent/repository_files.py index 278f22f..4f9e72b 100644 --- a/scripts/agent/repository_files.py +++ b/scripts/agent/repository_files.py @@ -6,7 +6,6 @@ import subprocess from pathlib import Path - ARCHIVE_EXCLUDES = { ".git", ".venv", "venv", "env", "ENV", "node_modules", "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", "build", "dist", ".tmp", "tmp", diff --git a/scripts/agent/sync_adapters.py b/scripts/agent/sync_adapters.py index 52520e1..f8a9101 100755 --- a/scripts/agent/sync_adapters.py +++ b/scripts/agent/sync_adapters.py @@ -13,7 +13,6 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from _template_common import DISTRIBUTION_IGNORE, TemplateError, synchronize_claude_skills # noqa: E402 - ROOT = Path(__file__).resolve().parents[2] diff --git a/scripts/agent/validate_contract.py b/scripts/agent/validate_contract.py index 4806125..d0bfee6 100755 --- a/scripts/agent/validate_contract.py +++ b/scripts/agent/validate_contract.py @@ -10,7 +10,6 @@ from repository_files import repository_files - ROOT = Path(__file__).resolve().parents[2] REQUIRED = ( "AGENTS.md", diff --git a/scripts/agent/validate_marketplace.py b/scripts/agent/validate_marketplace.py index 9e3c7c5..265f4b2 100644 --- a/scripts/agent/validate_marketplace.py +++ b/scripts/agent/validate_marketplace.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 #!/usr/bin/env python3 """Validate marketplace organization, host indexes, and evidence-backed findings.""" @@ -9,12 +11,13 @@ from pathlib import Path from typing import Any - ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT)) -from cops.validation import ValidationError, validate_repository - +from cops.validation import ( + ValidationError, + validate_repository, +) CLASSIFICATIONS = {"observation", "assessment", "unresolved"} VERIFICATIONS = {"verified", "partially-verified", "unverified", "contradicted"} diff --git a/scripts/audit_capabilities.py b/scripts/audit_capabilities.py index 68d6c87..133ba7a 100644 --- a/scripts/audit_capabilities.py +++ b/scripts/audit_capabilities.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 #!/usr/bin/env python3 """Audit, reconcile, and generate capability truth-in-advertising matrices for COPS.""" @@ -5,8 +7,8 @@ import argparse import json -from pathlib import Path import sys +from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) @@ -15,7 +17,6 @@ CapabilityTruthError, audit_capabilities, build_capability_registry, - generate_capability_matrix_markdown, ) diff --git a/specs/features/network_active_discovery.feature b/specs/features/network_active_discovery.feature index 32d1707..e5a0eb5 100644 --- a/specs/features/network_active_discovery.feature +++ b/specs/features/network_active_discovery.feature @@ -47,3 +47,8 @@ Feature: Network Active Discovery and Service Identification When the active scan delta comparison is evaluated Then closed or filtered ports are identified as remediated exposures And an exact remediation rate percentage is calculated + + Scenario: Excluding deprecated TLS versions from live probes + Given the standard socket TLS probe + When it creates a TLS client context + Then the minimum TLS version is TLS 1.2 diff --git a/specs/features/repository_scanning.feature b/specs/features/repository_scanning.feature index c1d84c7..9d3e6f3 100644 --- a/specs/features/repository_scanning.feature +++ b/specs/features/repository_scanning.feature @@ -25,3 +25,15 @@ Feature: Repository Security Scanning Then the JSON output "languages" must list "Bicep" And the JSON output "iac_and_cloud" must list "Azure Bicep" And the JSON output "iac_and_cloud" must list "AWS" + + Scenario: Detect ARM templates using the declared schema host + Given a workspace containing a file "template.json" + And "template.json" declares the schema URL "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#" + When the scanner script executes with target "." + Then the JSON output "iac_and_cloud" must list "Azure ARM Template" + + Scenario: Do not detect an ARM template from a schema-host substring + Given a workspace containing a file "template.json" + And "template.json" mentions "schema.management.azure.com" only in its description + When the scanner script executes with target "." + Then the JSON output "iac_and_cloud" must NOT list "Azure ARM Template" diff --git a/specs/features/sentinel_hunt_workbench.feature b/specs/features/sentinel_hunt_workbench.feature index 5c3636a..0606233 100644 --- a/specs/features/sentinel_hunt_workbench.feature +++ b/specs/features/sentinel_hunt_workbench.feature @@ -31,3 +31,8 @@ Feature: Offline-qualified Sentinel Hunt Workbench When I create a release qualification report Then the report is not offline qualified without two human approvals And the report marks cross-platform model evaluation as pending + + Scenario: Classify reference authority from its parsed HTTPS host + Given the Sentinel Hunt Workbench package + When I classify a reference URL that embeds an authoritative hostname + Then the reference is classified as a vendor framework diff --git a/tests/step_defs/test_attack_path_workbench.py b/tests/step_defs/test_attack_path_workbench.py index ab40bd3..77fbf04 100644 --- a/tests/step_defs/test_attack_path_workbench.py +++ b/tests/step_defs/test_attack_path_workbench.py @@ -5,23 +5,27 @@ import copy import hashlib import json -from pathlib import Path import sys +from pathlib import Path import pytest from pytest_bdd import given, scenarios, then, when - ROOT = Path(__file__).resolve().parents[2] PLUGIN = ROOT / "plugins/detection-hunting/attack-path-workbench" FIXTURE = PLUGIN / "fixtures/illustrative" sys.path.insert(0, str(PLUGIN)) from attackpath.core import ( # noqa: E402 - GateError, analyze, audit_report, canonical, collect_reviews, file_hash, - query_intent, validate_input, + GateError, + analyze, + audit_report, + canonical, + collect_reviews, + file_hash, + query_intent, + validate_input, ) - scenarios("../../specs/features/attack_path_workbench.feature") diff --git a/tests/step_defs/test_attack_surface_planner.py b/tests/step_defs/test_attack_surface_planner.py index b5ef30e..ef311f2 100644 --- a/tests/step_defs/test_attack_surface_planner.py +++ b/tests/step_defs/test_attack_surface_planner.py @@ -3,10 +3,10 @@ from __future__ import annotations import json -from pathlib import Path import shutil import socket import sys +from pathlib import Path from unittest.mock import patch import pytest diff --git a/tests/step_defs/test_azure_entitlements.py b/tests/step_defs/test_azure_entitlements.py index 17402c9..5e111c9 100644 --- a/tests/step_defs/test_azure_entitlements.py +++ b/tests/step_defs/test_azure_entitlements.py @@ -1,16 +1,29 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Executable acceptance scenarios for the offline Azure profile.""" import json -from pathlib import Path import sys +from pathlib import Path + import pytest -from pytest_bdd import given, when, then, scenarios +from pytest_bdd import given, scenarios, then, when ROOT = Path(__file__).resolve().parents[2] PLUGIN = ROOT / "plugins/detection-hunting/attack-path-workbench" sys.path.insert(0, str(PLUGIN)) sys.path.insert(0, str(PLUGIN / "tests")) from azure_fixtures import write_bundle -from test_azure_paths import base, role, execution, deployment, T, S, NOW, SECRET, VAULT +from test_azure_paths import ( + NOW, + SECRET, + VAULT, + S, + T, + base, + deployment, + execution, + role, +) from test_azure_sdk_cli import run_cli scenarios("../../specs/features/azure_entitlements.feature") diff --git a/tests/step_defs/test_capability_reconciliation.py b/tests/step_defs/test_capability_reconciliation.py index 364489e..d84be6e 100644 --- a/tests/step_defs/test_capability_reconciliation.py +++ b/tests/step_defs/test_capability_reconciliation.py @@ -5,6 +5,7 @@ import copy import json from pathlib import Path + import pytest from pytest_bdd import given, parsers, scenarios, then, when diff --git a/tests/step_defs/test_credential_evidence_steps.py b/tests/step_defs/test_credential_evidence_steps.py index 20ef796..0b97ca9 100644 --- a/tests/step_defs/test_credential_evidence_steps.py +++ b/tests/step_defs/test_credential_evidence_steps.py @@ -4,6 +4,7 @@ import tempfile from pathlib import Path + import pytest from pytest_bdd import given, parsers, scenarios, then, when diff --git a/tests/step_defs/test_cve_reachability.py b/tests/step_defs/test_cve_reachability.py index 02a13a7..673f433 100644 --- a/tests/step_defs/test_cve_reachability.py +++ b/tests/step_defs/test_cve_reachability.py @@ -2,14 +2,14 @@ import copy import importlib.util import json -from pathlib import Path import re import stat import subprocess import sys +from pathlib import Path import pytest -from pytest_bdd import given, when, then, scenarios, parsers +from pytest_bdd import given, parsers, scenarios, then, when ROOT = Path(__file__).resolve().parents[2] PACKAGE = ROOT / 'plugins/logging-telemetry/security-logging-advisor' diff --git a/tests/step_defs/test_engagement_contracts.py b/tests/step_defs/test_engagement_contracts.py index b973fde..398bf0e 100644 --- a/tests/step_defs/test_engagement_contracts.py +++ b/tests/step_defs/test_engagement_contracts.py @@ -4,15 +4,12 @@ import json from pathlib import Path + import pytest from pytest_bdd import given, parsers, scenarios, then, when from cops.contracts import ( - ActionPlan, ContractError, - Engagement, - Finding, - RunResult, evaluate_run_result, validate_contract, validate_transition, diff --git a/tests/step_defs/test_engagement_intake_planning_steps.py b/tests/step_defs/test_engagement_intake_planning_steps.py index 969b7f2..023267f 100644 --- a/tests/step_defs/test_engagement_intake_planning_steps.py +++ b/tests/step_defs/test_engagement_intake_planning_steps.py @@ -2,23 +2,15 @@ from __future__ import annotations -import copy -from pathlib import Path import pytest from pytest_bdd import given, parsers, scenarios, then, when from cops.catalog import ROOT -from cops.contracts.models import ActionPlan, Engagement +from cops.contracts.models import Engagement from cops.contracts.validation import validate_contract from cops.engagement import ( EngagementIntakeError, - IncompatibleWindowError, - IncompleteBudgetError, - IncompleteLiveRequestError, - MissingOwnerError, - ScopeAmbiguityError, build_action_plan, - create_engagement_contract, validate_engagement_intake, ) diff --git a/tests/step_defs/test_execution_authorization_steps.py b/tests/step_defs/test_execution_authorization_steps.py index 3e9ec54..380887c 100644 --- a/tests/step_defs/test_execution_authorization_steps.py +++ b/tests/step_defs/test_execution_authorization_steps.py @@ -4,10 +4,11 @@ import json from pathlib import Path + import pytest from pytest_bdd import given, parsers, scenarios, then, when -from cops.contracts.models import ActionPlan, ExecutionAuthorization +from cops.contracts.models import ActionPlan from cops.execution import ( AuthorizationError, LegacyReceiptDeprecationWarning, diff --git a/tests/step_defs/test_execution_recovery_cleanup_steps.py b/tests/step_defs/test_execution_recovery_cleanup_steps.py index d265247..ecdc0d5 100644 --- a/tests/step_defs/test_execution_recovery_cleanup_steps.py +++ b/tests/step_defs/test_execution_recovery_cleanup_steps.py @@ -6,6 +6,7 @@ import shutil import tempfile from pathlib import Path + import pytest from pytest_bdd import given, parsers, scenarios, then, when diff --git a/tests/step_defs/test_execution_scope_steps.py b/tests/step_defs/test_execution_scope_steps.py index 2099f4f..24649bc 100644 --- a/tests/step_defs/test_execution_scope_steps.py +++ b/tests/step_defs/test_execution_scope_steps.py @@ -3,6 +3,7 @@ from __future__ import annotations import ipaddress + import pytest from pytest_bdd import given, parsers, scenarios, then, when diff --git a/tests/step_defs/test_incident_response_sandbox.py b/tests/step_defs/test_incident_response_sandbox.py index d47aff2..9cf2946 100644 --- a/tests/step_defs/test_incident_response_sandbox.py +++ b/tests/step_defs/test_incident_response_sandbox.py @@ -2,9 +2,8 @@ from __future__ import annotations -import copy import sys -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta from pathlib import Path import pytest @@ -30,7 +29,7 @@ def setup(context): "host-2": {"action": "isolate-host", "state": "active"}}, "executions": []} context["plan"] = build_plan(action="isolate-host", tenant="example", target="host-1", - expires_at=(datetime.now(timezone.utc) + timedelta(hours=1)).isoformat(), + expires_at=(datetime.now(UTC) + timedelta(hours=1)).isoformat(), nonce="acceptance-1", approver_assertion="analyst") diff --git a/tests/step_defs/test_isolated_worker_steps.py b/tests/step_defs/test_isolated_worker_steps.py index 7c9671b..b4ae205 100644 --- a/tests/step_defs/test_isolated_worker_steps.py +++ b/tests/step_defs/test_isolated_worker_steps.py @@ -6,6 +6,7 @@ import tempfile import threading from pathlib import Path + import pytest from pytest_bdd import given, parsers, scenarios, then, when diff --git a/tests/step_defs/test_m365_investigation.py b/tests/step_defs/test_m365_investigation.py index 6f55ae6..ccc8bbd 100644 --- a/tests/step_defs/test_m365_investigation.py +++ b/tests/step_defs/test_m365_investigation.py @@ -1,6 +1,8 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Executable Microsoft 365 offline acquisition scenarios.""" -from pathlib import Path import sys +from pathlib import Path import pytest from pytest_bdd import given, scenarios, then, when @@ -9,10 +11,16 @@ sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT / 'plugins/detection-hunting/soc-investigation-workbench')) -from cops.connectors import Checkpoint, GraphCollection, Response, collect -from cops.evidence import canonical from investigationwb.m365 import correlate +from cops.connectors import ( + Checkpoint, + GraphCollection, + Response, + collect, +) +from cops.evidence import canonical + scenarios('../../specs/features/m365_investigation.feature') TENANT = '00000000-0000-0000-0000-000000000001' diff --git a/tests/step_defs/test_marketplace_contract.py b/tests/step_defs/test_marketplace_contract.py index 105192a..5f2be3c 100644 --- a/tests/step_defs/test_marketplace_contract.py +++ b/tests/step_defs/test_marketplace_contract.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 from __future__ import annotations import copy @@ -10,18 +12,36 @@ import pytest from pytest_bdd import given, parsers, scenarios, then, when - ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / "scripts" / "agent")) -from validate_marketplace import ValidationError, validate_finding_document, validate_marketplace import export_portable import install_prerequisites -from cops.prerequisites import PrerequisiteError, process_tools, validate_prerequisites -from cops.validation import ValidationError as PackageValidationError, validate_agent_plugin_manifest -from cops.portable import export_portable_package -from cops.mcp_validation import MCPValidationError, MCP_SCHEMA, validate_mcp_configuration - +from validate_marketplace import ( + ValidationError, + validate_finding_document, + validate_marketplace, +) + +from cops.mcp_validation import ( + MCP_SCHEMA, + MCPValidationError, + validate_mcp_configuration, +) +from cops.portable import ( + export_portable_package, +) +from cops.prerequisites import ( + PrerequisiteError, + process_tools, + validate_prerequisites, +) +from cops.validation import ( + ValidationError as PackageValidationError, +) +from cops.validation import ( + validate_agent_plugin_manifest, +) scenarios("../../specs/features/marketplace_portability.feature") diff --git a/tests/step_defs/test_network_active_discovery_steps.py b/tests/step_defs/test_network_active_discovery_steps.py index d024e3d..914fc42 100644 --- a/tests/step_defs/test_network_active_discovery_steps.py +++ b/tests/step_defs/test_network_active_discovery_steps.py @@ -1,19 +1,23 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Step definitions for Network Active Discovery BDD scenarios.""" from __future__ import annotations -from pathlib import Path +import ssl import sys +from pathlib import Path + import pytest -from pytest_bdd import given, parsers, scenarios, then, when +from pytest_bdd import given, scenarios, then, when ROOT = Path(__file__).resolve().parents[2] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from cops.discovery import ( - ActiveScanSession, ActiveScanner, + ActiveScanSession, ConfidenceLevel, ObservedConfiguration, ObservedTLS, @@ -25,6 +29,7 @@ compare_active_scans, infer_service_fingerprint, ) +from cops.discovery.active_scanner import StandardSocketDispatcher, _create_tls_context scenarios("../../specs/features/network_active_discovery.feature") @@ -41,6 +46,7 @@ def bdd_ctx(): "scanner": None, "completed": None, "delta": None, + "tls_context": None, "scope": { "domains": ["corp.internal"], "ip_ranges": ["198.51.100.0/24"], @@ -381,3 +387,19 @@ def then_remediated_ports_identified(bdd_ctx): def then_exact_remediation_rate(bdd_ctx): delta = bdd_ctx["delta"] assert delta.remediation_rate == 50.0 + + +@given("the standard socket TLS probe") +def given_standard_socket_tls_probe(bdd_ctx): + bdd_ctx["dispatcher"] = StandardSocketDispatcher() + + +@when("it creates a TLS client context") +def when_tls_context_created(bdd_ctx): + assert isinstance(bdd_ctx["dispatcher"], StandardSocketDispatcher) + bdd_ctx["tls_context"] = _create_tls_context() + + +@then("the minimum TLS version is TLS 1.2") +def then_minimum_tls_version(bdd_ctx): + assert bdd_ctx["tls_context"].minimum_version == ssl.TLSVersion.TLSv1_2 diff --git a/tests/step_defs/test_network_data_services_steps.py b/tests/step_defs/test_network_data_services_steps.py index 222e87c..afbf0af 100644 --- a/tests/step_defs/test_network_data_services_steps.py +++ b/tests/step_defs/test_network_data_services_steps.py @@ -1,9 +1,12 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Step definitions for Database, Cache, and Search Services BDD scenarios.""" from __future__ import annotations -from pathlib import Path import sys +from pathlib import Path + import pytest from pytest_bdd import given, parsers, scenarios, then, when @@ -12,15 +15,9 @@ sys.path.insert(0, str(ROOT)) from cops.discovery import ( - CleanupReceipt, DataAuthPrerequisite, DataExposureStatus, - DataPrivilegeCandidate, - DataPrivilegeImpact, - DataServiceAssessment, DataServiceCategory, - DataServicesReport, - DataServiceType, OfflineSyntheticDataCollector, assess_data_services, ) diff --git a/tests/step_defs/test_network_developer_services_steps.py b/tests/step_defs/test_network_developer_services_steps.py index e2c4b59..8d287c7 100644 --- a/tests/step_defs/test_network_developer_services_steps.py +++ b/tests/step_defs/test_network_developer_services_steps.py @@ -1,26 +1,24 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Step definitions for Developer and Runtime Interfaces BDD scenarios.""" from __future__ import annotations -from pathlib import Path import sys +from pathlib import Path + import pytest -from pytest_bdd import given, parsers, scenarios, then, when +from pytest_bdd import given, scenarios, then, when ROOT = Path(__file__).resolve().parents[2] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from cops.discovery import ( - CleanupReceipt, DeveloperAuthPrerequisite, DeveloperCategory, DeveloperExposureStatus, - DeveloperPrivilegeCandidate, DeveloperPrivilegeImpact, - DeveloperServiceAssessment, - DeveloperServicesReport, - DeveloperServiceType, ExecutionEffect, OfflineSyntheticDeveloperCollector, assess_developer_services, diff --git a/tests/step_defs/test_network_infrastructure_services_steps.py b/tests/step_defs/test_network_infrastructure_services_steps.py index 01f669b..33e4a48 100644 --- a/tests/step_defs/test_network_infrastructure_services_steps.py +++ b/tests/step_defs/test_network_infrastructure_services_steps.py @@ -1,9 +1,12 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Step definitions for Network Infrastructure and Identity-Facing Services BDD scenarios.""" from __future__ import annotations -from pathlib import Path import sys +from pathlib import Path + import pytest from pytest_bdd import given, parsers, scenarios, then, when @@ -13,11 +16,7 @@ from cops.discovery import ( AuthPrerequisite, - IdentityAttackPathCandidate, IdentityAttackPathType, - InfraAssessmentReport, - InfraServiceAssessment, - InfraServiceType, OfflineSyntheticInfraCollector, ServiceExposureStatus, assess_infrastructure_services, diff --git a/tests/step_defs/test_network_legacy_services_steps.py b/tests/step_defs/test_network_legacy_services_steps.py index 84364c6..d5841e9 100644 --- a/tests/step_defs/test_network_legacy_services_steps.py +++ b/tests/step_defs/test_network_legacy_services_steps.py @@ -1,9 +1,12 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Step definitions for Legacy Enterprise, Management, and Proxy Services BDD scenarios.""" from __future__ import annotations -from pathlib import Path import sys +from pathlib import Path + import pytest from pytest_bdd import given, parsers, scenarios, then, when @@ -12,16 +15,11 @@ sys.path.insert(0, str(ROOT)) from cops.discovery import ( - CleanupReceipt, ExecutionEffect, LegacyAuthPrerequisite, LegacyCategory, LegacyExposureStatus, - LegacyPrivilegeCandidate, LegacyPrivilegeImpact, - LegacyServiceAssessment, - LegacyServicesReport, - LegacyServiceType, OfflineSyntheticLegacyCollector, assess_legacy_services, ) diff --git a/tests/step_defs/test_network_messaging_services_steps.py b/tests/step_defs/test_network_messaging_services_steps.py index c61efa3..bca9dcd 100644 --- a/tests/step_defs/test_network_messaging_services_steps.py +++ b/tests/step_defs/test_network_messaging_services_steps.py @@ -1,9 +1,12 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Step definitions for Mail, Messaging, and Message Broker Services BDD scenarios.""" from __future__ import annotations -from pathlib import Path import sys +from pathlib import Path + import pytest from pytest_bdd import given, parsers, scenarios, then, when @@ -12,15 +15,9 @@ sys.path.insert(0, str(ROOT)) from cops.discovery import ( - CleanupReceipt, MessagingAuthPrerequisite, MessagingCategory, MessagingExposureStatus, - MessagingPrivilegeCandidate, - MessagingPrivilegeImpact, - MessagingServiceAssessment, - MessagingServicesReport, - MessagingServiceType, OfflineSyntheticMessagingCollector, assess_messaging_services, ) diff --git a/tests/step_defs/test_network_passive_discovery_steps.py b/tests/step_defs/test_network_passive_discovery_steps.py index a0abaf4..142d5ce 100644 --- a/tests/step_defs/test_network_passive_discovery_steps.py +++ b/tests/step_defs/test_network_passive_discovery_steps.py @@ -3,12 +3,12 @@ from __future__ import annotations from pathlib import Path + import pytest from pytest_bdd import given, parsers, scenarios, then, when from cops.discovery import ( DiscoveredAsset, - DiscoveryInventory, EvidenceProvenance, merge_inventories, normalize_certificate_record, diff --git a/tests/step_defs/test_network_remote_file_print_services_steps.py b/tests/step_defs/test_network_remote_file_print_services_steps.py index f4d427c..06f73bc 100644 --- a/tests/step_defs/test_network_remote_file_print_services_steps.py +++ b/tests/step_defs/test_network_remote_file_print_services_steps.py @@ -1,9 +1,12 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Step definitions for Network Remote Administration, File Sharing, and Printing Services BDD scenarios.""" from __future__ import annotations -from pathlib import Path import sys +from pathlib import Path + import pytest from pytest_bdd import given, parsers, scenarios, then, when @@ -12,16 +15,10 @@ sys.path.insert(0, str(ROOT)) from cops.discovery import ( - CleanupReceipt, - HostPrivilegeCandidate, - LateralMovementImpact, OfflineSyntheticRemoteCollector, RemoteAuthPrerequisite, RemoteExposureStatus, - RemoteServiceAssessment, RemoteServiceCategory, - RemoteServicesReport, - RemoteServiceType, assess_remote_services, ) diff --git a/tests/step_defs/test_pinned_scenario_registry.py b/tests/step_defs/test_pinned_scenario_registry.py index 75c204c..3e26e96 100644 --- a/tests/step_defs/test_pinned_scenario_registry.py +++ b/tests/step_defs/test_pinned_scenario_registry.py @@ -4,6 +4,7 @@ import json from pathlib import Path + import pytest from pytest_bdd import given, parsers, scenarios, then, when @@ -11,7 +12,6 @@ RegistryError, get_provenance_source, get_scenario, - list_provenance_sources, list_scenarios, load_provenance_registry, load_scenario_registry, diff --git a/tests/step_defs/test_plugin_run_cost.py b/tests/step_defs/test_plugin_run_cost.py index 9c27937..5c6dcee 100644 --- a/tests/step_defs/test_plugin_run_cost.py +++ b/tests/step_defs/test_plugin_run_cost.py @@ -1,10 +1,10 @@ """Executable acceptance tests using local synthetic evidence; no paid calls.""" import json -from pathlib import Path import subprocess import sys +from pathlib import Path -from pytest_bdd import given, when, then, scenarios +from pytest_bdd import given, scenarios, then, when ROOT = Path(__file__).resolve().parents[2] SKILL = ROOT / '.agents/skills/plugin-run-cost' diff --git a/tests/step_defs/test_plugin_validation.py b/tests/step_defs/test_plugin_validation.py index f240034..e615352 100644 --- a/tests/step_defs/test_plugin_validation.py +++ b/tests/step_defs/test_plugin_validation.py @@ -3,12 +3,12 @@ import json import os -from pathlib import Path import shutil import subprocess +from pathlib import Path -from pytest_bdd import given, parsers, scenarios, then, when import pytest +from pytest_bdd import given, parsers, scenarios, then, when ROOT = Path(__file__).resolve().parents[2] SOURCE_PACKAGE = ROOT / "plugins/logging-telemetry/security-logging-advisor" diff --git a/tests/step_defs/test_portable_operator_workflow.py b/tests/step_defs/test_portable_operator_workflow.py index 1b0b5bd..183324f 100644 --- a/tests/step_defs/test_portable_operator_workflow.py +++ b/tests/step_defs/test_portable_operator_workflow.py @@ -1,22 +1,29 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Executable acceptance scenarios for the catalog-driven operator workflow.""" from __future__ import annotations import json -from pathlib import Path import subprocess import sys +from pathlib import Path import pytest from pytest_bdd import given, scenarios, then, when - ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT)) -from cops.catalog import CatalogError, plugin_records, validate_declared_command -from cops.validation import ValidationError, generate_marketplaces - +from cops.catalog import ( + CatalogError, + plugin_records, + validate_declared_command, +) +from cops.validation import ( + ValidationError, + generate_marketplaces, +) scenarios("../../specs/features/portable_operator_workflow.feature") diff --git a/tests/step_defs/test_repository_conformance.py b/tests/step_defs/test_repository_conformance.py index 913f99c..b6184cc 100644 --- a/tests/step_defs/test_repository_conformance.py +++ b/tests/step_defs/test_repository_conformance.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Regression coverage for the adapted PARK validation gate.""" import subprocess diff --git a/tests/step_defs/test_repository_scanning.py b/tests/step_defs/test_repository_scanning.py index 0245cb7..7221fee 100644 --- a/tests/step_defs/test_repository_scanning.py +++ b/tests/step_defs/test_repository_scanning.py @@ -1,10 +1,10 @@ -import os -import sys +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 import json +import os import subprocess -import shutil -import tempfile -from pytest_bdd import scenarios, given, when, then, parsers + +from pytest_bdd import given, parsers, scenarios, then, when # Load all scenarios from the feature files scenarios('../../specs/features/repository_scanning.feature') @@ -12,6 +12,8 @@ # Fixture to hold context across steps import pytest + + @pytest.fixture def context(): state = {} @@ -47,6 +49,16 @@ def append_line(filename, line): with open(filename, 'a') as f: f.write(f'{line}\n') +@given(parsers.parse('"{filename}" declares the schema URL "{url}"')) +def declare_schema_url(filename, url): + with open(filename, 'w') as f: + json.dump({"$schema": url}, f) + +@given(parsers.parse('"{filename}" mentions "{hostname}" only in its description')) +def mention_schema_host_in_description(filename, hostname): + with open(filename, 'w') as f: + json.dump({"description": f"Reference text: {hostname}"}, f) + @given(parsers.parse('a workspace containing a Bicep file "{filename}"')) def create_bicep_file(filename): with open(filename, 'w') as f: @@ -126,14 +138,14 @@ def execute_scanner(context, target): # But this script runs from the pytest root, so we can pass the absolute path # Actually, we can just pass the path relative to the root dir which we know. # We will assume pytest runs from the root of the project. - + script_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../plugins/logging-telemetry/security-logging-advisor/skills/repository-context/scripts/collect-repository-context.py')) - + try: result = subprocess.run(['python3', script_path, target], capture_output=True, text=True, check=True, timeout=5) context['output'] = result.stdout context['error'] = False - + try: context['json'] = json.loads(result.stdout) except json.JSONDecodeError: @@ -156,6 +168,11 @@ def json_output_must_list(context, key, value): else: assert value in context['json'][key] +@then(parsers.parse('the JSON output "{key}" must NOT list "{value}"')) +def json_output_must_not_list(context, key, value): + assert context['json'] is not None + assert value not in context['json'][key] + @then(parsers.parse('the JSON output "{key}" must list a finding for "{filename}"')) def json_output_must_list_finding(context, key, filename): assert context['json'] is not None diff --git a/tests/step_defs/test_sentinel_hunt_workbench.py b/tests/step_defs/test_sentinel_hunt_workbench.py index 1dd1a05..1a5698f 100644 --- a/tests/step_defs/test_sentinel_hunt_workbench.py +++ b/tests/step_defs/test_sentinel_hunt_workbench.py @@ -3,17 +3,19 @@ from __future__ import annotations import json -from pathlib import Path import subprocess import sys +from pathlib import Path import pytest from pytest_bdd import given, parsers, scenarios, then, when - ROOT = Path(__file__).resolve().parents[2] PLUGIN = ROOT / "plugins/detection-hunting/sentinel-hunt-workbench" CLI = PLUGIN / "scripts/huntwb.py" +if str(PLUGIN) not in sys.path: + sys.path.insert(0, str(PLUGIN)) +from huntwb.catalog import _reference_record # noqa: E402 - package path is bootstrapped above scenarios("../../specs/features/sentinel_hunt_workbench.feature") @@ -66,6 +68,18 @@ def sentinel_package(context): assert CLI.is_file() +@when("I classify a reference URL that embeds an authoritative hostname") +def classify_embedded_authoritative_hostname(context): + context["reference"] = _reference_record( + "https://untrusted.example/path?next=learn.microsoft.com" + ) + + +@then("the reference is classified as a vendor framework") +def reference_is_vendor_framework(context): + assert context["reference"]["kind"] == "vendor_framework" + + @when("I validate the Sentinel hunt library") def validate_library(context): context["result"] = cli("validate", "library") diff --git a/tests/step_defs/test_shared_evidence_acquisition.py b/tests/step_defs/test_shared_evidence_acquisition.py index 4255e66..9df8567 100644 --- a/tests/step_defs/test_shared_evidence_acquisition.py +++ b/tests/step_defs/test_shared_evidence_acquisition.py @@ -1,10 +1,17 @@ """Executable shared-consumer scenarios using synthetic adapters only.""" import pytest -from pytest_bdd import given, when, then, scenarios, parsers +from pytest_bdd import given, parsers, scenarios, then, when from cops.connectors import Checkpoint, Limits, collect, preview -from cops.connectors.demo import (AS_OF, FixtureCredentials, FixtureInterruption, - FixtureTransport, interrupt_after_commit, offline_fixture, stale_example) +from cops.connectors.demo import ( + AS_OF, + FixtureCredentials, + FixtureInterruption, + FixtureTransport, + interrupt_after_commit, + offline_fixture, + stale_example, +) from cops.evidence import report scenarios('../../specs/features/shared_evidence_acquisition.feature') diff --git a/tests/step_defs/test_soc_investigation.py b/tests/step_defs/test_soc_investigation.py index 6c0251c..3c059b4 100644 --- a/tests/step_defs/test_soc_investigation.py +++ b/tests/step_defs/test_soc_investigation.py @@ -1,16 +1,18 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Executable acceptance scenarios for the case planner, not hunt qualification.""" -from copy import deepcopy -from hashlib import sha256 import json import os -from pathlib import Path import runpy import shlex import shutil import stat import subprocess import sys +from copy import deepcopy +from hashlib import sha256 +from pathlib import Path from types import SimpleNamespace import pytest @@ -20,10 +22,28 @@ PLUGIN = ROOT / "plugins/detection-hunting/soc-investigation-workbench" sys.path.insert(0, str(PLUGIN)) -from investigationwb.engine import ContractError, digest, import_result, next_steps, report, revise, validate from investigationwb.cli import read_json -from investigationwb.files import MAX_BYTES, read_regular -from investigationwb.vendor import UPSTREAM, handoff, inventory, sync, validate_lock, verify +from investigationwb.engine import ( + ContractError, + digest, + import_result, + next_steps, + report, + revise, + validate, +) +from investigationwb.files import ( + MAX_BYTES, + read_regular, +) +from investigationwb.vendor import ( + UPSTREAM, + handoff, + inventory, + sync, + validate_lock, + verify, +) scenarios("../../specs/features/soc_investigation.feature") @@ -164,7 +184,8 @@ def ranking(state): @then("only ready independent steps within budget are proposed") def eligible(state): - ids = lambda value: [c["step_id"] for c in value["candidates"]] + def ids(value): + return [c["step_id"] for c in value["candidates"]] assert ids(state["before"]) == ["signin", "spray"] assert ids(state["after"]) == ["consent", "spray"] assert ids(state["limited"]) == ["signin"] diff --git a/tests/step_defs/test_specialist_workflow_handoff_steps.py b/tests/step_defs/test_specialist_workflow_handoff_steps.py index 9c2188d..d6727da 100644 --- a/tests/step_defs/test_specialist_workflow_handoff_steps.py +++ b/tests/step_defs/test_specialist_workflow_handoff_steps.py @@ -2,22 +2,11 @@ from __future__ import annotations -import copy -import json -from pathlib import Path import pytest from pytest_bdd import given, parsers, scenarios, then, when -from cops.catalog import ROOT -from cops.contracts.models import ActionPlan, Engagement, SpecialistHandoff from cops.contracts.validation import validate_contract from cops.routing import ( - AuthorizationExpansionError, - ConflictingEvidenceError, - HandoffError, - InvalidHandoffResultError, - MaterialPlanModifiedError, - MissingCapabilityError, accept_specialist_handoff, audit_and_approve_handoff, execute_triad_handoff_workflow, diff --git a/tests/step_defs/test_threat_intelligence_enrichment.py b/tests/step_defs/test_threat_intelligence_enrichment.py index ca90beb..46d50d6 100644 --- a/tests/step_defs/test_threat_intelligence_enrichment.py +++ b/tests/step_defs/test_threat_intelligence_enrichment.py @@ -10,7 +10,8 @@ PACKAGE = Path(__file__).resolve().parents[2] / "plugins/detection-hunting/threat-intelligence-enrichment" sys.path.insert(0, str(PACKAGE)) -from threat_intel import Approval, Cache, Indicator, Source, enrich, normalize, conflicts # noqa: E402 +from threat_intel import Approval, Cache, Indicator, Source, conflicts, enrich, normalize # noqa: E402 + from cops.connectors.interfaces import Response # noqa: E402 scenarios("../../specs/features/threat_intelligence_enrichment.feature") diff --git a/tests/step_defs/test_workflow_capability_diagnostics_steps.py b/tests/step_defs/test_workflow_capability_diagnostics_steps.py index fbcc5aa..676140d 100644 --- a/tests/step_defs/test_workflow_capability_diagnostics_steps.py +++ b/tests/step_defs/test_workflow_capability_diagnostics_steps.py @@ -3,6 +3,7 @@ from __future__ import annotations from pathlib import Path + import pytest from pytest_bdd import given, parsers, scenarios, then, when diff --git a/tests/test_active_discovery.py b/tests/test_active_discovery.py index fee16b7..e5936a4 100644 --- a/tests/test_active_discovery.py +++ b/tests/test_active_discovery.py @@ -2,24 +2,22 @@ from __future__ import annotations +import argparse import json -from pathlib import Path +import ssl import tempfile -import time import unittest +from pathlib import Path from cops.discovery import ( - ActiveScanSession, ActiveScanner, - ActiveServiceAssessment, + ActiveScanSession, ConfidenceLevel, ObservedConfiguration, ObservedTLS, OfflineSyntheticDispatcher, PortState, - Protocol, ScanBudget, - ScanDelta, ScanVantage, ServiceReachability, compare_active_scans, @@ -27,11 +25,15 @@ import_nmap_xml, infer_service_fingerprint, ) +from cops.discovery.active_scanner import _create_tls_context from cops.discovery.cli import command_active_discovery -import argparse class TestActiveDiscoveryModelsAndFingerprinting(unittest.TestCase): + def test_live_tls_probe_requires_tls_12_or_newer(self): + context = _create_tls_context() + self.assertEqual(context.minimum_version, ssl.TLSVersion.TLSv1_2) + def test_ssh_fingerprint_high_confidence(self): obs = ObservedConfiguration(raw_banner="SSH-2.0-OpenSSH_8.9p1 Ubuntu-3ubuntu0.6\r\n") fp = infer_service_fingerprint("ssh.corp.internal", 22, "tcp", obs) @@ -90,6 +92,14 @@ def test_expired_tls_certificate_uncertainty(self): fp = infer_service_fingerprint("target.corp.internal", 443, "tcp", obs) self.assertTrue(any("TLS certificate is expired" in r for r in fp.uncertainty_reasons)) + def test_invalid_or_naive_tls_expiry_records_uncertainty(self): + for expiry in ("invalid", "2028-01-01T00:00:00"): + with self.subTest(expiry=expiry): + tls = ObservedTLS(subject_cn="target.example.com", sans=["target.example.com"], valid_until=expiry) + fp = infer_service_fingerprint("target.example.com", 443, "tcp", ObservedConfiguration(tls=tls)) + self.assertEqual(fp.confidence, ConfidenceLevel.UNCERTAIN.value) + self.assertTrue(any("expiry date could not be parsed" in reason for reason in fp.uncertainty_reasons)) + def test_generic_open_port_without_banner_is_uncertain(self): obs = ObservedConfiguration() fp = infer_service_fingerprint("198.51.100.10", 9999, "tcp", obs) @@ -447,3 +457,12 @@ def test_cli_plan_scan_resume_diff_pipeline(self): if __name__ == "__main__": unittest.main() + + +def test_nmap_import_rejects_declared_entities(tmp_path): + from cops.discovery.importer import import_nmap_xml + + scan = tmp_path / "entities.xml" + scan.write_text(']>&expanded;') + with unittest.TestCase().assertRaisesRegex(ValueError, "entity declarations"): + import_nmap_xml(scan) diff --git a/tests/test_adapter_registry.py b/tests/test_adapter_registry.py index c465e34..785b4cc 100644 --- a/tests/test_adapter_registry.py +++ b/tests/test_adapter_registry.py @@ -1,6 +1,7 @@ """Unit tests for declarative tool adapter registry.""" import pytest + from cops.adapters import ( AdapterError, AdapterInjectionError, diff --git a/tests/test_attack_coverage.py b/tests/test_attack_coverage.py index b0fcae6..d361685 100644 --- a/tests/test_attack_coverage.py +++ b/tests/test_attack_coverage.py @@ -10,8 +10,6 @@ from cops.cli import main from cops.coverage import ( - CoverageError, - CoverageMapping, evaluate_coverage_gaps, generate_attack_flow, generate_coverage_matrix, diff --git a/tests/test_azure_sdk_bundle.py b/tests/test_azure_sdk_bundle.py index 1515b77..a6d262c 100644 --- a/tests/test_azure_sdk_bundle.py +++ b/tests/test_azure_sdk_bundle.py @@ -19,12 +19,13 @@ def test_generated_evidence_subset_and_drift(tmp_path): def test_exported_azure_cli_runs_without_repository_imports(tmp_path): import json import sys + from cops.portable import export_portable_package plugin = ROOT / "plugins/detection-hunting/attack-path-workbench" sys.path.insert(0, str(plugin)) sys.path.insert(0, str(plugin / "tests")) from azure_fixtures import write_bundle - from test_azure_paths import execution, NOW + from test_azure_paths import NOW, execution from test_azure_sdk_cli import run_cli exported = tmp_path / "export" export_portable_package(plugin, exported) diff --git a/tests/test_capability_diagnostics.py b/tests/test_capability_diagnostics.py index ad285fe..717aff0 100644 --- a/tests/test_capability_diagnostics.py +++ b/tests/test_capability_diagnostics.py @@ -4,14 +4,9 @@ import json from pathlib import Path -import pytest from cops.diagnostics import ( - DiagnosticCheck, DiagnosticReport, - PackageDiagnostic, - SystemDiagnostic, - ToolDiagnostic, detect_system_platform, diagnose_host_tools, diagnose_packages, diff --git a/tests/test_capability_truth.py b/tests/test_capability_truth.py index 3c41036..689a959 100644 --- a/tests/test_capability_truth.py +++ b/tests/test_capability_truth.py @@ -5,6 +5,7 @@ import copy import json from pathlib import Path + import pytest from cops.capabilities import ( diff --git a/tests/test_check_prerequisites.py b/tests/test_check_prerequisites.py index a5c210d..00e5d42 100644 --- a/tests/test_check_prerequisites.py +++ b/tests/test_check_prerequisites.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Offline regression tests for the contributor prerequisite preflight.""" import importlib.metadata diff --git a/tests/test_cleanup_recovery.py b/tests/test_cleanup_recovery.py index d5ab6cc..9e71a9a 100644 --- a/tests/test_cleanup_recovery.py +++ b/tests/test_cleanup_recovery.py @@ -3,10 +3,10 @@ from __future__ import annotations import json -import os import shutil import tempfile from pathlib import Path + import pytest from cops.contracts.models import ActionPlan, CleanupReceipt @@ -15,7 +15,6 @@ ApprovalStore, ApprovalStoreConflictError, CleanupManager, - CleanupOwnershipError, IsolatedWorker, SideEffectLedger, WorkerConfig, diff --git a/tests/test_connector_sdk.py b/tests/test_connector_sdk.py index 42a3f6a..18186ea 100644 --- a/tests/test_connector_sdk.py +++ b/tests/test_connector_sdk.py @@ -1,318 +1,406 @@ """Fault-injection tests for the common acquisition runner (no tenant access).""" + import json -from pathlib import Path import sys +from pathlib import Path import pytest sys.path.insert(0, str(Path(__file__).resolve().parents[1])) -from cops.connectors import Checkpoint, GraphUsers, ResourceGraph, Limits, Response, collect, preview +from cops.connectors import Checkpoint, GraphUsers, Limits, ResourceGraph, Response, collect, preview from cops.evidence import EvidenceError, validate_receipt -TENANT = '00000000-0000-0000-0000-000000000001' -SUB = '00000000-0000-0000-0000-000000000002' -USER1 = '00000000-0000-0000-0000-000000000003' -USER2 = '00000000-0000-0000-0000-000000000004' -CANARY = 'CANARY_AUTH_OR_UNREVIEWED_DATA' +TENANT = "00000000-0000-0000-0000-000000000001" +SUB = "00000000-0000-0000-0000-000000000002" +USER1 = "00000000-0000-0000-0000-000000000003" +USER2 = "00000000-0000-0000-0000-000000000004" +CANARY = "CANARY_AUTH_OR_UNREVIEWED_DATA" -def fixture(adapter='graph'): - if adapter == 'graph': +def fixture(adapter="graph"): + if adapter == "graph": return GraphUsers(TENANT), [ - {'value': [{'id': USER1, 'userType': 'Member', 'displayName': CANARY}], - '@odata.nextLink': 'https://graph.microsoft.com/v1.0/users?$skiptoken=opaque'}, - {'value': [{'id': USER2, 'userType': 'Guest'}]}] + { + "value": [{"id": USER1, "userType": "Member", "displayName": CANARY}], + "@odata.nextLink": "https://graph.microsoft.com/v1.0/users?$skiptoken=opaque", + }, + {"value": [{"id": USER2, "userType": "Guest"}]}, + ] return ResourceGraph(TENANT, [SUB]), [ - {'data': [{'id': f'/subscriptions/{SUB}/resourceGroups/demo/providers/microsoft.compute/virtualMachines/a', - 'type': 'microsoft.compute/virtualmachines', 'location': 'eastus', 'tags': {'secret': CANARY}}], - 'count': 1, 'totalRecords': 2, 'resultTruncated': 'true', '$skipToken': 'opaque'}, - {'data': [{'id': f'/subscriptions/{SUB}/resourceGroups/demo/providers/microsoft.compute/virtualMachines/b', - 'type': 'microsoft.compute/virtualmachines', 'location': 'eastus'}], - 'count': 1, 'totalRecords': 2, 'resultTruncated': 'false'}] + { + "data": [ + { + "id": f"/subscriptions/{SUB}/resourceGroups/demo/providers/microsoft.compute/virtualMachines/a", + "type": "microsoft.compute/virtualmachines", + "location": "eastus", + "tags": {"secret": CANARY}, + } + ], + "count": 1, + "totalRecords": 2, + "resultTruncated": "true", + "$skipToken": "opaque", + }, + { + "data": [ + { + "id": f"/subscriptions/{SUB}/resourceGroups/demo/providers/microsoft.compute/virtualMachines/b", + "type": "microsoft.compute/virtualmachines", + "location": "eastus", + } + ], + "count": 1, + "totalRecords": 2, + "resultTruncated": "false", + }, + ] class Credentials: - def __init__(self): self.refreshes = [] + def __init__(self): + self.refreshes = [] + def headers(self, *, refresh=False): self.refreshes.append(refresh) - return {'Authorization': 'Bearer ' + CANARY} + return {"Authorization": "Bearer " + CANARY} class Transport: - def __init__(self, pages): self.pages, self.requests = list(pages), [] + def __init__(self, pages): + self.pages, self.requests = list(pages), [] + def send(self, request, headers, *, timeout, max_bytes): self.requests.append(request) page = self.pages.pop(0) - if isinstance(page, BaseException): raise page - if isinstance(page, Response): return page + if isinstance(page, BaseException): + raise page + if isinstance(page, Response): + return page return Response(200, json.dumps(page).encode()) -class Crash(BaseException): pass +class Crash(BaseException): + pass def run(tmp_path, pages, adapter=None, limits=None, **kw): - with Checkpoint(tmp_path / 'private') as checkpoint: - return collect(adapter or fixture()[0], checkpoint, Credentials(), Transport(pages), - limits=limits or Limits(), **kw) + with Checkpoint(tmp_path / "private") as checkpoint: + return collect( + adapter or fixture()[0], checkpoint, Credentials(), Transport(pages), limits=limits or Limits(), **kw + ) -@pytest.mark.parametrize('name', ['graph', 'arg']) +@pytest.mark.parametrize("name", ["graph", "arg"]) def test_resume_after_page_commit(tmp_path, name): adapter, pages = fixture(name) with pytest.raises(Crash): - run(tmp_path, pages, adapter, fault=lambda phase: (_ for _ in ()).throw(Crash()) if phase == 'after_commit' else None) + run( + tmp_path, + pages, + adapter, + fault=lambda phase: (_ for _ in ()).throw(Crash()) if phase == "after_commit" else None, + ) result = run(tmp_path, [pages[1]], adapter) assert len(result.records) == 2 - assert result.receipt['status'] == 'complete' + assert result.receipt["status"] == "complete" validate_receipt(result.receipt) assert CANARY not in json.dumps(result.records) - assert result.records[0]['observed']['at'] is None + assert result.records[0]["observed"]["at"] is None def test_limits_stop_and_preserve_accepted_records(tmp_path): adapter, pages = fixture() result = run(tmp_path, pages, adapter, Limits(records=1)) assert len(result.records) == 1 - assert result.receipt['reasons'] == ['record_limit'] - assert result.receipt['status'] == 'partial' + assert result.receipt["reasons"] == ["record_limit"] + assert result.receipt["status"] == "partial" -@pytest.mark.parametrize('partial', [False, True]) +@pytest.mark.parametrize("partial", [False, True]) def test_terminal_resume_preserves_collection_interval(tmp_path, partial): adapter, pages = fixture() limits = Limits(records=1) if partial else Limits() - first = run(tmp_path, pages, adapter, limits, now=lambda: '2026-09-28T00:00:00Z') + first = run(tmp_path, pages, adapter, limits, now=lambda: "2026-09-28T00:00:00Z") credentials, transport = Credentials(), Transport([]) - with Checkpoint(tmp_path / 'private') as checkpoint: - resumed = collect(adapter, checkpoint, credentials, transport, limits=limits, - now=lambda: '2026-09-29T00:00:00Z') - assert resumed.receipt['finished_at'] == first.receipt['finished_at'] - assert resumed.receipt['started_at'] == first.receipt['started_at'] - assert resumed.receipt['status'] == ('partial' if partial else 'complete') - assert resumed.receipt['generation'] == 2 + with Checkpoint(tmp_path / "private") as checkpoint: + resumed = collect( + adapter, checkpoint, credentials, transport, limits=limits, now=lambda: "2026-09-29T00:00:00Z" + ) + assert resumed.receipt["finished_at"] == first.receipt["finished_at"] + assert resumed.receipt["started_at"] == first.receipt["started_at"] + assert resumed.receipt["status"] == ("partial" if partial else "complete") + assert resumed.receipt["generation"] == 2 assert credentials.refreshes == transport.requests == [] def test_terminal_commit_persists_finish_before_interruption(tmp_path): adapter, pages = fixture() with pytest.raises(Crash): - run(tmp_path, [pages[1]], adapter, now=lambda: '2026-09-28T00:00:00Z', - fault=lambda phase: (_ for _ in ()).throw(Crash()) if phase == 'after_commit' else None) - resumed = run(tmp_path, [], adapter, now=lambda: '2026-09-29T00:00:00Z') - assert resumed.receipt['finished_at'] == '2026-09-28T00:00:00Z' - - -@pytest.mark.parametrize('name', ['graph', 'arg']) + run( + tmp_path, + [pages[1]], + adapter, + now=lambda: "2026-09-28T00:00:00Z", + fault=lambda phase: (_ for _ in ()).throw(Crash()) if phase == "after_commit" else None, + ) + resumed = run(tmp_path, [], adapter, now=lambda: "2026-09-29T00:00:00Z") + assert resumed.receipt["finished_at"] == "2026-09-28T00:00:00Z" + + +@pytest.mark.parametrize("name", ["graph", "arg"]) def test_duplicate_processing_stops_at_active_time_limit(tmp_path, monkeypatch, name): import cops.connectors.runner as runner + adapter, pages = fixture(name) - key = 'value' if name == 'graph' else 'data' + key = "value" if name == "graph" else "data" pages[0][key] *= 100 - if name == 'arg': - pages[0]['count'] = pages[0]['totalRecords'] = 100 + if name == "arg": + pages[0]["count"] = pages[0]["totalRecords"] = 100 current, calls = [0.0], [] original = runner.build_envelope def processing_cost(**kwargs): - calls.append(kwargs['identity']) + calls.append(kwargs["identity"]) current[0] += 0.4 return original(**kwargs) - monkeypatch.setattr(runner, 'build_envelope', processing_cost) - result = run(tmp_path, pages, adapter, Limits(active_seconds=1, request_seconds=1), - clock=lambda: current[0]) + monkeypatch.setattr(runner, "build_envelope", processing_cost) + result = run(tmp_path, pages, adapter, Limits(active_seconds=1, request_seconds=1), clock=lambda: current[0]) assert len(calls) <= 3 assert len(result.records) == 1 - assert result.receipt['reasons'] == ['time_limit'] - assert result.receipt['consumed']['active_seconds'] == 1 + assert result.receipt["reasons"] == ["time_limit"] + assert result.receipt["consumed"]["active_seconds"] == 1 -@pytest.mark.parametrize('name', ['graph', 'arg']) +@pytest.mark.parametrize("name", ["graph", "arg"]) def test_adapter_projection_stops_at_active_time_limit(tmp_path, monkeypatch, name): import cops.connectors.adapters.microsoft as microsoft + adapter, pages = fixture(name) - key = 'value' if name == 'graph' else 'data' + key = "value" if name == "graph" else "data" pages[0][key] *= 100 - if name == 'arg': - pages[0]['count'] = pages[0]['totalRecords'] = 100 + if name == "arg": + pages[0]["count"] = pages[0]["totalRecords"] = 100 current, projections = [0.0], [] - if name == 'graph': + if name == "graph": original = microsoft.UUID + def match(value): projections.append(value) current[0] += 0.4 return original.fullmatch(value) + from types import SimpleNamespace - monkeypatch.setattr(microsoft, 'UUID', SimpleNamespace(fullmatch=match)) + + monkeypatch.setattr(microsoft, "UUID", SimpleNamespace(fullmatch=match)) else: original = microsoft.re.fullmatch + def match(pattern, value, *args, **kwargs): - if pattern.startswith('/subscriptions/'): + if pattern.startswith("/subscriptions/"): projections.append(value) current[0] += 0.4 return original(pattern, value, *args, **kwargs) - monkeypatch.setattr(microsoft.re, 'fullmatch', match) - result = run(tmp_path, pages, adapter, Limits(active_seconds=1, request_seconds=1), - clock=lambda: current[0]) + + monkeypatch.setattr(microsoft.re, "fullmatch", match) + result = run(tmp_path, pages, adapter, Limits(active_seconds=1, request_seconds=1), clock=lambda: current[0]) assert len(projections) <= 3 assert len(result.records) == 1 - assert result.receipt['reasons'] == ['time_limit'] + assert result.receipt["reasons"] == ["time_limit"] def test_auth_throttle_duplicates_and_secret_boundary(tmp_path, capsys): adapter, pages = fixture() - pages[1]['value'].insert(0, pages[0]['value'][0]) + pages[1]["value"].insert(0, pages[0]["value"][0]) credentials = Credentials() - transport = Transport([pages[0], Response(401, CANARY.encode()), Response(429, CANARY.encode(), '0'), pages[1]]) - with Checkpoint(tmp_path / 'private') as cp: + transport = Transport([pages[0], Response(401, CANARY.encode()), Response(429, CANARY.encode(), "0"), pages[1]]) + with Checkpoint(tmp_path / "private") as cp: result = collect(adapter, cp, credentials, transport, sleep=lambda seconds: None) - assert result.receipt['status'] == 'complete' - assert result.receipt['consumed']['duplicates'] == 1 + assert result.receipt["status"] == "complete" + assert result.receipt["consumed"]["duplicates"] == 1 assert credentials.refreshes.count(True) == 1 - assert result.receipt['consumed']['attempts'] == 4 + assert result.receipt["consumed"]["attempts"] == 4 assert CANARY not in repr(result) + json.dumps(result.receipt) + str(capsys.readouterr()) -@pytest.mark.parametrize('failure,code', [({'unexpected': []}, 'schema_drift'), - (Response(401, CANARY.encode()), 'authentication'), - (ValueError(CANARY), 'transport')]) +@pytest.mark.parametrize( + "failure,code", + [ + ({"unexpected": []}, "schema_drift"), + (Response(401, CANARY.encode()), "authentication"), + (ValueError(CANARY), "transport"), + ], +) def test_partial_errors_are_safe(tmp_path, failure, code): _, pages = fixture() result = run(tmp_path, [pages[0], failure, failure]) - assert result.receipt['status'] == 'partial' - assert code in result.receipt['reasons'] + assert result.receipt["status"] == "partial" + assert code in result.receipt["reasons"] assert len(result.records) == 1 assert CANARY not in json.dumps(result.receipt) -@pytest.mark.parametrize('link', ['http://graph.microsoft.com/v1.0/users', - 'https://evil.invalid/v1.0/users', 'https://graph.microsoft.com:444/v1.0/users', - 'https://graph.microsoft.com/v1.0/users?access_token=secret', - 'https://graph.microsoft.com/v1.0/users/../../me', - 'https://graph.microsoft.com/v1.0/users#$skiptoken=x']) +@pytest.mark.parametrize( + "link", + [ + "http://graph.microsoft.com/v1.0/users", + "https://evil.invalid/v1.0/users", + "https://graph.microsoft.com:444/v1.0/users", + "https://graph.microsoft.com/v1.0/users?access_token=secret", + "https://graph.microsoft.com/v1.0/users/../../me", + "https://graph.microsoft.com/v1.0/users#$skiptoken=x", + ], +) def test_hostile_continuation_never_gets_credentials(tmp_path, link): adapter, pages = fixture() - pages[0]['@odata.nextLink'] = link + pages[0]["@odata.nextLink"] = link credentials, transport = Credentials(), Transport(pages) - with Checkpoint(tmp_path / 'private') as cp: + with Checkpoint(tmp_path / "private") as cp: result = collect(adapter, cp, credentials, transport) - assert result.receipt['reasons'] == ['unsafe_destination'] + assert result.receipt["reasons"] == ["unsafe_destination"] assert len(credentials.refreshes) == 1 assert len(transport.requests) == 1 def test_preview_has_no_effects(): plan = preview(fixture()[0]) - assert plan['method'] == 'GET' - assert 'Authorization' not in json.dumps(plan) + assert plan["method"] == "GET" + assert "Authorization" not in json.dumps(plan) def test_crash_reservation_and_resume_binding(tmp_path): adapter, pages = fixture() with pytest.raises(Crash): - run(tmp_path, pages, adapter, fault=lambda phase: (_ for _ in ()).throw(Crash()) if phase == 'after_reserve' else None) - with pytest.raises(EvidenceError, match='resume_mismatch'): + run( + tmp_path, + pages, + adapter, + fault=lambda phase: (_ for _ in ()).throw(Crash()) if phase == "after_reserve" else None, + ) + with pytest.raises(EvidenceError, match="resume_mismatch"): run(tmp_path, pages, GraphUsers(SUB)) - with pytest.raises(EvidenceError, match='resume_limits'): + with pytest.raises(EvidenceError, match="resume_limits"): run(tmp_path, pages, adapter, Limits(pages=101)) result = run(tmp_path, pages, adapter) - assert result.receipt['consumed']['attempts'] == 3 - assert result.receipt['consumed']['bytes'] >= Limits().response_bytes + assert result.receipt["consumed"]["attempts"] == 3 + assert result.receipt["consumed"]["bytes"] >= Limits().response_bytes def test_private_checkpoint_rejects_symlink_and_permissions(tmp_path): - target = tmp_path / 'target'; target.mkdir() - link = tmp_path / 'link'; link.symlink_to(target) - with pytest.raises(EvidenceError, match='private_storage'): + target = tmp_path / "target" + target.mkdir() + link = tmp_path / "link" + link.symlink_to(target) + with pytest.raises(EvidenceError, match="private_storage"): Checkpoint(link) target.chmod(0o755) - with pytest.raises(EvidenceError, match='private_storage'): + with pytest.raises(EvidenceError, match="private_storage"): Checkpoint(target) -@pytest.mark.parametrize('limits,reason', [(Limits(pages=1), 'page_limit'), - (Limits(attempts=1), 'attempt_limit'), (Limits(total_bytes=1), 'response_limit'), - (Limits(response_bytes=1), 'response_limit'), (Limits(storage_bytes=1), 'storage_limit'), - (Limits(record_bytes=1), 'projection_failed')]) +@pytest.mark.parametrize( + "limits,reason", + [ + (Limits(pages=1), "page_limit"), + (Limits(attempts=1), "attempt_limit"), + (Limits(total_bytes=1), "response_limit"), + (Limits(response_bytes=1), "response_limit"), + (Limits(storage_bytes=1), "storage_limit"), + (Limits(record_bytes=1), "projection_failed"), + ], +) def test_budgets_are_enforced(tmp_path, limits, reason): result = run(tmp_path, fixture()[1], limits=limits) - assert result.receipt['status'] == 'partial' - assert result.receipt['reasons'] == [reason] + assert result.receipt["status"] == "partial" + assert result.receipt["reasons"] == [reason] validate_receipt(result.receipt) -@pytest.mark.parametrize('failure,reason', [(Response(200, b'{broken'), 'invalid_json'), - (Response(410, b'PRIVATE'), 'cursor_expired'), - (Response(302, b'PRIVATE'), 'transport'), - (Response(429, b'', 'invalid'), 'throttled')]) +@pytest.mark.parametrize( + "failure,reason", + [ + (Response(200, b"{broken"), "invalid_json"), + (Response(410, b"PRIVATE"), "cursor_expired"), + (Response(302, b"PRIVATE"), "transport"), + (Response(429, b"", "invalid"), "throttled"), + ], +) def test_later_failure_keeps_first_page(tmp_path, failure, reason): result = run(tmp_path, [fixture()[1][0], failure]) - assert result.receipt['reasons'] == [reason] + assert result.receipt["reasons"] == [reason] assert len(result.records) == 1 def test_cycles_and_retry_continuations_are_partial(tmp_path): adapter, pages = fixture() - pages[1]['@odata.nextLink'] = pages[0]['@odata.nextLink'] + pages[1]["@odata.nextLink"] = pages[0]["@odata.nextLink"] result = run(tmp_path, pages) - assert result.receipt['reasons'] == ['cursor_cycle'] + assert result.receipt["reasons"] == ["cursor_cycle"] assert len(result.records) == 2 - other = tmp_path / 'other'; other.mkdir() - result = run(other, [Response(429, b'', '0'), pages[0]], sleep=lambda seconds: None) - assert result.receipt['reasons'] == ['retry_continuation'] + other = tmp_path / "other" + other.mkdir() + result = run(other, [Response(429, b"", "0"), pages[0]], sleep=lambda seconds: None) + assert result.receipt["reasons"] == ["retry_continuation"] def test_arg_truncation_without_cursor_is_partial(tmp_path): - adapter, pages = fixture('arg') - del pages[0]['$skipToken'] + adapter, pages = fixture("arg") + del pages[0]["$skipToken"] result = run(tmp_path, pages, adapter) - assert result.receipt['reasons'] == ['truncated_without_cursor'] + assert result.receipt["reasons"] == ["truncated_without_cursor"] assert len(result.records) == 1 def test_before_commit_crash_replays_page_once(tmp_path): adapter, pages = fixture() with pytest.raises(Crash): - run(tmp_path, pages, fault=lambda phase: (_ for _ in ()).throw(Crash()) if phase == 'before_commit' else None) + run(tmp_path, pages, fault=lambda phase: (_ for _ in ()).throw(Crash()) if phase == "before_commit" else None) result = run(tmp_path, pages) assert len(result.records) == 2 - assert result.receipt['consumed']['attempts'] == 3 - assert result.receipt['consumed']['pages'] == 2 - assert result.receipt['consumed']['active_seconds'] >= Limits().request_seconds + assert result.receipt["consumed"]["attempts"] == 3 + assert result.receipt["consumed"]["pages"] == 2 + assert result.receipt["consumed"]["active_seconds"] >= Limits().request_seconds def test_retry_budget_survives_interruption(tmp_path): - adapter, pages = fixture('arg') + adapter, pages = fixture("arg") with pytest.raises(Crash): - run(tmp_path, [Response(429, b'', '0')], adapter, Limits(retries=1), - sleep=lambda seconds: (_ for _ in ()).throw(Crash())) - result = run(tmp_path, [Response(429, b'', '0'), pages[0], pages[1]], adapter, - Limits(retries=1), sleep=lambda seconds: None) - assert result.receipt['reasons'] == ['throttled'] - assert result.receipt['consumed']['attempts'] == 2 + run( + tmp_path, + [Response(429, b"", "0")], + adapter, + Limits(retries=1), + sleep=lambda seconds: (_ for _ in ()).throw(Crash()), + ) + result = run( + tmp_path, [Response(429, b"", "0"), pages[0], pages[1]], adapter, Limits(retries=1), sleep=lambda seconds: None + ) + assert result.receipt["reasons"] == ["throttled"] + assert result.receipt["consumed"]["attempts"] == 2 def test_resume_rejects_budget_below_spend_and_corrupt_counts(tmp_path): adapter, pages = fixture() with pytest.raises(Crash): - run(tmp_path, pages, fault=lambda phase: (_ for _ in ()).throw(Crash()) if phase == 'after_commit' else None) - with pytest.raises(EvidenceError, match='resume_limits'): + run(tmp_path, pages, fault=lambda phase: (_ for _ in ()).throw(Crash()) if phase == "after_commit" else None) + with pytest.raises(EvidenceError, match="resume_limits"): run(tmp_path, pages, limits=Limits(total_bytes=1)) - with Checkpoint(tmp_path / 'private') as cp: - state = cp.load(); state['consumed']['records'] += 1; cp.save(state) - with pytest.raises(EvidenceError, match='checkpoint_invalid'): + with Checkpoint(tmp_path / "private") as cp: + state = cp.load() + state["consumed"]["records"] += 1 + cp.save(state) + with pytest.raises(EvidenceError, match="checkpoint_invalid"): run(tmp_path, pages) -@pytest.mark.parametrize('limits', [Limits(record_bytes=1), Limits(depth=1)]) +@pytest.mark.parametrize("limits", [Limits(record_bytes=1), Limits(depth=1)]) def test_resume_rejects_bounds_that_cannot_hold_committed_records(tmp_path, limits): adapter, pages = fixture() with pytest.raises(Crash): - run(tmp_path, pages, fault=lambda phase: (_ for _ in ()).throw(Crash()) if phase == 'after_commit' else None) + run(tmp_path, pages, fault=lambda phase: (_ for _ in ()).throw(Crash()) if phase == "after_commit" else None) credentials, transport = Credentials(), Transport(pages[1:]) - with Checkpoint(tmp_path / 'private') as checkpoint: - with pytest.raises(EvidenceError, match='resume_limits'): + with Checkpoint(tmp_path / "private") as checkpoint: + with pytest.raises(EvidenceError, match="resume_limits"): collect(adapter, checkpoint, credentials, transport, limits=limits) assert credentials.refreshes == [] assert transport.requests == [] @@ -320,20 +408,29 @@ def test_resume_rejects_bounds_that_cannot_hold_committed_records(tmp_path, limi def test_timeout_and_sleep_use_active_budget(tmp_path): instant = [0] + class Slow(Transport): def send(self, *args, **kwargs): instant[0] += 2 return super().send(*args, **kwargs) - with Checkpoint(tmp_path / 'private') as cp: - result = collect(fixture()[0], cp, Credentials(), Slow(fixture()[1]), - limits=Limits(request_seconds=1), clock=lambda: instant[0]) - assert result.receipt['reasons'] == ['request_timeout'] - other = tmp_path / 'other'; other.mkdir() - result = run(other, [Response(429, b'', '300')]) - assert result.receipt['reasons'] == ['time_limit'] + + with Checkpoint(tmp_path / "private") as cp: + result = collect( + fixture()[0], + cp, + Credentials(), + Slow(fixture()[1]), + limits=Limits(request_seconds=1), + clock=lambda: instant[0], + ) + assert result.receipt["reasons"] == ["request_timeout"] + other = tmp_path / "other" + other.mkdir() + result = run(other, [Response(429, b"", "300")]) + assert result.receipt["reasons"] == ["time_limit"] def test_checkpoint_and_export_never_persist_auth_or_unselected_data(tmp_path): result = run(tmp_path, fixture()[1]) - assert CANARY.encode() not in (tmp_path / 'private' / 'checkpoint.sqlite3').read_bytes() + assert CANARY.encode() not in (tmp_path / "private" / "checkpoint.sqlite3").read_bytes() assert CANARY not in json.dumps(result.records) + json.dumps(result.receipt) diff --git a/tests/test_contracts.py b/tests/test_contracts.py index ddf4a77..a63b4dc 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -4,6 +4,7 @@ import json from pathlib import Path + import pytest from cops.contracts import ( @@ -12,12 +13,10 @@ Engagement, Finding, RunResult, - Scenario, build_action_plan_digest, evaluate_run_result, validate_contract, validate_identifier, - validate_transition, ) FIXTURES_DIR = Path(__file__).resolve().parents[1] / "cops" / "contracts" / "fixtures" diff --git a/tests/test_credential_evidence.py b/tests/test_credential_evidence.py index 5f90aff..5609362 100644 --- a/tests/test_credential_evidence.py +++ b/tests/test_credential_evidence.py @@ -2,7 +2,6 @@ import tempfile from pathlib import Path -import pytest from cops.execution.evidence import EvidenceRecorder from cops.execution.redaction import StreamRedactor diff --git a/tests/test_data_services.py b/tests/test_data_services.py index 84dca77..dc28164 100644 --- a/tests/test_data_services.py +++ b/tests/test_data_services.py @@ -1,14 +1,15 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Unit, contract, and CLI tests for database, cache, and search/analytics services assessment.""" from __future__ import annotations import argparse -import io import json -from pathlib import Path import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parent.parent if str(ROOT) not in sys.path: @@ -16,21 +17,21 @@ from cops.discovery import ( CleanupReceipt, - DEFAULT_DATA_PORTS, DataAuthPrerequisite, DataExposureStatus, DataPrivilegeCandidate, DataPrivilegeImpact, DataServiceAssessment, DataServiceCategory, - DataServicesCollector, DataServicesReport, DataServiceType, OfflineSyntheticDataCollector, StandardSocketDataCollector, assess_data_services, ) -from cops.discovery.cli import command_data_discovery +from cops.discovery.cli import ( + command_data_discovery, +) class TestDataModelsAndSerialization(unittest.TestCase): diff --git a/tests/test_developer_services.py b/tests/test_developer_services.py index a056197..c62749c 100644 --- a/tests/test_developer_services.py +++ b/tests/test_developer_services.py @@ -1,16 +1,16 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Unit, contract, and CLI tests for developer and runtime interfaces assessment.""" from __future__ import annotations import argparse -import io import json -from pathlib import Path -import socket import sys import tempfile import unittest -from unittest.mock import patch, MagicMock +from pathlib import Path +from unittest.mock import MagicMock, patch ROOT = Path(__file__).resolve().parent.parent if str(ROOT) not in sys.path: @@ -18,22 +18,20 @@ from cops.discovery import ( CleanupReceipt, - DEFAULT_DEVELOPER_PORTS, DeveloperAuthPrerequisite, - DeveloperCategory, DeveloperExposureStatus, DeveloperPrivilegeCandidate, DeveloperPrivilegeImpact, DeveloperServiceAssessment, - DeveloperServicesCollector, DeveloperServicesReport, - DeveloperServiceType, ExecutionEffect, OfflineSyntheticDeveloperCollector, StandardSocketDeveloperCollector, assess_developer_services, ) -from cops.discovery.cli import command_developer_discovery +from cops.discovery.cli import ( + command_developer_discovery, +) class TestDeveloperModelsAndSerialization(unittest.TestCase): diff --git a/tests/test_engagement_planning.py b/tests/test_engagement_planning.py index e6aaedd..89e1f8f 100644 --- a/tests/test_engagement_planning.py +++ b/tests/test_engagement_planning.py @@ -6,12 +6,11 @@ import copy import io import json -from pathlib import Path import sys import unittest from cops.catalog import ROOT -from cops.contracts.models import ActionPlan, Engagement +from cops.contracts.models import Engagement from cops.contracts.validation import validate_contract from cops.engagement import ( EngagementIntakeError, diff --git a/tests/test_evidence_contract.py b/tests/test_evidence_contract.py index aa4bbf8..75a064c 100644 --- a/tests/test_evidence_contract.py +++ b/tests/test_evidence_contract.py @@ -1,7 +1,7 @@ """Behavior-first tests for shared provenance and integrity.""" import json -from pathlib import Path import sys +from pathlib import Path import pytest diff --git a/tests/test_evidence_transport.py b/tests/test_evidence_transport.py index 1ab5df4..4c00f57 100644 --- a/tests/test_evidence_transport.py +++ b/tests/test_evidence_transport.py @@ -1,10 +1,11 @@ """Exercise deadlines and response caps without sockets or tenant credentials.""" -from pathlib import Path + import http.client import io import sys import threading import time +from pathlib import Path from types import SimpleNamespace import pytest @@ -21,42 +22,57 @@ def __init__(self, *args, **kwargs): self.sent = False self.closed = threading.Event() - def connect(self): pass - def request(self, *args, **kwargs): self.sent = True + def connect(self): + pass + + def request(self, *args, **kwargs): + self.sent = True + def getresponse(self): return self.response - def close(self): self.closed.set() + + def close(self): + self.closed.set() class Reply: status = 200 - def __init__(self, data=b'{}', headers=None): + + def __init__(self, data=b"{}", headers=None): self.data, self.headers = data, headers or {} self.fp = SimpleNamespace(raw=SimpleNamespace(_sock=SimpleNamespace(settimeout=lambda seconds: None))) - def getheader(self, key, default=None): return self.headers.get(key, default) + + def getheader(self, key, default=None): + return self.headers.get(key, default) + def read1(self, size): chunk, self.data = self.data[:size], self.data[size:] return chunk def dispatch(monkeypatch, connection, *, cap=20, timeout=1): - monkeypatch.setattr('cops.connectors.transport.http.client.HTTPSConnection', lambda *a, **k: connection) - return HttpsTransport('example.invalid', '/read', 'GET').send( - Request('GET', 'https://example.invalid/read'), {'Authorization': 'Bearer TEST_ONLY'}, - timeout=timeout, max_bytes=cap) + monkeypatch.setattr("cops.connectors.transport.http.client.HTTPSConnection", lambda *a, **k: connection) + return HttpsTransport("example.invalid", "/read", "GET").send( + Request("GET", "https://example.invalid/read"), + {"Authorization": "Bearer TEST_ONLY"}, + timeout=timeout, + max_bytes=cap, + ) def test_deadline_includes_connect_and_prevents_late_credentials(monkeypatch): connection = Connection() released, done = threading.Event(), threading.Event() + def stalled_connect(): released.wait(2) done.set() + connection.connect = stalled_connect connection.response = Reply() began = time.monotonic() try: - with pytest.raises(EvidenceError, match='request_timeout'): + with pytest.raises(EvidenceError, match="request_timeout"): dispatch(monkeypatch, connection, timeout=0.05) assert time.monotonic() - began < 0.5 finally: @@ -67,15 +83,17 @@ def stalled_connect(): def test_deadline_includes_headers(monkeypatch): connection = Connection() + def stalled_headers(): connection.closed.wait(2) - raise TimeoutError('PRIVATE_RESPONSE') + raise TimeoutError("PRIVATE_RESPONSE") + connection.getresponse = stalled_headers began = time.monotonic() - with pytest.raises(EvidenceError, match='request_timeout') as error: + with pytest.raises(EvidenceError, match="request_timeout") as error: dispatch(monkeypatch, connection, timeout=0.05) assert time.monotonic() - began < 0.5 - assert 'PRIVATE_RESPONSE' not in str(error.value) + assert "PRIVATE_RESPONSE" not in str(error.value) def test_cancelled_connection_cannot_reopen_before_sending_headers(monkeypatch): @@ -87,8 +105,9 @@ class Client(http.client.HTTPSConnection): def connect(self): self.connections += 1 - self.sock = SimpleNamespace(settimeout=lambda seconds: None, - shutdown=lambda how: None, close=lambda: None, sendall=sent.append) + self.sock = SimpleNamespace( + settimeout=lambda seconds: None, shutdown=lambda how: None, close=lambda: None, sendall=sent.append + ) def request(self, *args, **kwargs): entered.set() @@ -100,11 +119,11 @@ def request(self, *args, **kwargs): finished.set() def getresponse(self): - raise EvidenceError('transport') + raise EvidenceError("transport") - connection = Client('example.invalid') + connection = Client("example.invalid") try: - with pytest.raises(EvidenceError, match='request_timeout'): + with pytest.raises(EvidenceError, match="request_timeout"): dispatch(monkeypatch, connection, timeout=0.1) assert entered.is_set() finally: @@ -114,31 +133,44 @@ def getresponse(self): assert sent == [] -@pytest.mark.parametrize('reply', [Reply(b'x' * 21), Reply(headers={'Content-Length': '21'}), - Reply(headers={'Content-Encoding': 'gzip'}), - Reply(headers={'Content-Length': 'invalid'})]) +@pytest.mark.parametrize( + "reply", + [ + Reply(b"x" * 21), + Reply(headers={"Content-Length": "21"}), + Reply(headers={"Content-Encoding": "gzip"}), + Reply(headers={"Content-Length": "invalid"}), + ], +) def test_reject_oversize_or_encoded_responses(monkeypatch, reply): - connection = Connection(); connection.response = reply - with pytest.raises(EvidenceError, match='response_limit'): + connection = Connection() + connection.response = reply + with pytest.raises(EvidenceError, match="response_limit"): dispatch(monkeypatch, connection) assert connection.closed.is_set() def test_exact_route_is_checked_before_connection(monkeypatch): - def forbidden(*args, **kwargs): raise AssertionError('network call') - monkeypatch.setattr('cops.connectors.transport.http.client.HTTPSConnection', forbidden) - with pytest.raises(EvidenceError, match='unsafe_destination'): - HttpsTransport('example.invalid', '/read', 'GET').send( - Request('GET', 'https://other.invalid/read'), {}, timeout=1, max_bytes=20) - - -@pytest.mark.parametrize('framing, wire_body, expected', [ - (b'Content-Length: 2', b'{}', b'{}'), - (b'Content-Length: 0', b'', b''), - (b'Transfer-Encoding: chunked', b'2\r\n{}\r\n0\r\n\r\n', b'{}'), -]) + def forbidden(*args, **kwargs): + raise AssertionError("network call") + + monkeypatch.setattr("cops.connectors.transport.http.client.HTTPSConnection", forbidden) + with pytest.raises(EvidenceError, match="unsafe_destination"): + HttpsTransport("example.invalid", "/read", "GET").send( + Request("GET", "https://other.invalid/read"), {}, timeout=1, max_bytes=20 + ) + + +@pytest.mark.parametrize( + "framing, wire_body, expected", + [ + (b"Content-Length: 2", b"{}", b"{}"), + (b"Content-Length: 0", b"", b""), + (b"Transfer-Encoding: chunked", b"2\r\n{}\r\n0\r\n\r\n", b"{}"), + ], +) def test_http_response_can_close_its_reader_at_end_of_body(monkeypatch, framing, wire_body, expected): - raw = io.BytesIO(b'HTTP/1.1 200 OK\r\n' + framing + b'\r\n\r\n' + wire_body) + raw = io.BytesIO(b"HTTP/1.1 200 OK\r\n" + framing + b"\r\n\r\n" + wire_body) raw._sock = SimpleNamespace(settimeout=lambda seconds: None) response = http.client.HTTPResponse(SimpleNamespace(makefile=lambda mode: io.BufferedReader(raw))) response.begin() @@ -150,6 +182,6 @@ def test_http_response_can_close_its_reader_at_end_of_body(monkeypatch, framing, def test_incomplete_fixed_length_body_is_not_accepted(monkeypatch): connection = Connection() - connection.response = Reply(b'{}', {'Content-Length': '3'}) - with pytest.raises(EvidenceError, match='transport'): + connection.response = Reply(b"{}", {"Content-Length": "3"}) + with pytest.raises(EvidenceError, match="transport"): dispatch(monkeypatch, connection) diff --git a/tests/test_execution_authorization.py b/tests/test_execution_authorization.py index ad85185..19c9ff5 100644 --- a/tests/test_execution_authorization.py +++ b/tests/test_execution_authorization.py @@ -5,15 +5,14 @@ import io import json from pathlib import Path + import pytest -from cops.contracts.lifecycle import ContractError from cops.contracts.models import ActionPlan, ExecutionAuthorization from cops.contracts.validation import validate_contract from cops.execution import ( AuthorizationDeniedError, AuthorizationError, - AuthorizationRequiredError, LegacyReceiptDeprecationWarning, compute_authorization_signature, consume_execution_authorization, diff --git a/tests/test_infrastructure_services.py b/tests/test_infrastructure_services.py index c784643..387efec 100644 --- a/tests/test_infrastructure_services.py +++ b/tests/test_infrastructure_services.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Unit, contract, and CLI tests for infrastructure and identity-facing services assessment.""" from __future__ import annotations @@ -5,10 +7,10 @@ import argparse import io import json -from pathlib import Path import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parent.parent if str(ROOT) not in sys.path: @@ -19,15 +21,15 @@ IdentityAttackPathCandidate, IdentityAttackPathType, InfraAssessmentReport, - InfraCollector, InfraServiceAssessment, InfraServiceType, OfflineSyntheticInfraCollector, ServiceExposureStatus, - StandardSocketInfraCollector, assess_infrastructure_services, ) -from cops.discovery.cli import command_infra_discovery +from cops.discovery.cli import ( + command_infra_discovery, +) class TestInfrastructureModelsAndSerialization(unittest.TestCase): @@ -108,7 +110,7 @@ def test_infra_assessment_report_save_and_properties(self): out_file = Path(tmpdir) / "report.json" report.save(out_file) self.assertTrue(out_file.exists()) - with open(out_file, "r", encoding="utf-8") as f: + with open(out_file, encoding="utf-8") as f: loaded = json.load(f) self.assertEqual(loaded["report_id"], "infra-test-001") @@ -336,7 +338,7 @@ def test_cli_assess_and_candidates_and_inspect(self): self.assertEqual(rc_cand, 0) self.assertTrue(cand_path.exists()) - with open(cand_path, "r", encoding="utf-8") as f: + with open(cand_path, encoding="utf-8") as f: cands_data = json.load(f) self.assertIsInstance(cands_data, list) self.assertGreater(len(cands_data), 0) diff --git a/tests/test_isolated_worker.py b/tests/test_isolated_worker.py index 874c5d2..d42ae51 100644 --- a/tests/test_isolated_worker.py +++ b/tests/test_isolated_worker.py @@ -3,10 +3,9 @@ from __future__ import annotations import json -import shutil -import tempfile import threading from pathlib import Path + import pytest from cops.contracts.models import ActionPlan @@ -14,10 +13,8 @@ ApprovalStore, ApprovalStoreConflictError, ApprovalStoreNotFoundError, - AuthorizationError, IsolatedWorker, WorkerConfig, - WorkerIsolationError, create_execution_authorization, ) @@ -191,3 +188,17 @@ def test_isolated_worker_rejects_unauthorized_tool(temp_store, sample_plan): assert not result.is_successful() assert result.status == "failed" assert "not in worker allowed tools" in result.status_details["reason"] + + +@pytest.mark.parametrize("as_dict", [False, True]) +def test_authorization_storage_failure_stops_execution(temp_store, sample_plan, monkeypatch, as_dict): + auth = create_execution_authorization(sample_plan, operator="test-operator", valid_hours=1) + worker = IsolatedWorker(WorkerConfig(worker_id="storage-test"), store=temp_store) + + def fail_storage(_authorization): + raise OSError("approval storage unavailable") + + monkeypatch.setattr(temp_store, "store_authorization", fail_storage) + with pytest.raises(OSError, match="approval storage unavailable"): + worker.execute_plan(sample_plan, authorization=auth.to_dict() if as_dict else auth) + assert temp_store.list_approvals() == [] diff --git a/tests/test_issue_coverage.py b/tests/test_issue_coverage.py index 0f85033..18e90ae 100644 --- a/tests/test_issue_coverage.py +++ b/tests/test_issue_coverage.py @@ -4,8 +4,6 @@ import sys from pathlib import Path -import pytest - from scripts.agent.check_issue_coverage import ( check_exemptions, evaluate_coverage, diff --git a/tests/test_legacy_services.py b/tests/test_legacy_services.py index b62191d..1bf7b89 100644 --- a/tests/test_legacy_services.py +++ b/tests/test_legacy_services.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Unit, contract, and CLI tests for legacy enterprise, management, and proxy services assessment.""" from __future__ import annotations @@ -5,20 +7,19 @@ import argparse import io import json -from pathlib import Path -import socket import sys import tempfile import unittest -from unittest.mock import patch, MagicMock +from pathlib import Path +from unittest.mock import MagicMock, patch ROOT = Path(__file__).resolve().parent.parent if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from cops.discovery import ( - CleanupReceipt, DEFAULT_LEGACY_PORTS, + CleanupReceipt, ExecutionEffect, LegacyAuthPrerequisite, LegacyCategory, @@ -26,14 +27,15 @@ LegacyPrivilegeCandidate, LegacyPrivilegeImpact, LegacyServiceAssessment, - LegacyServicesCollector, LegacyServicesReport, LegacyServiceType, OfflineSyntheticLegacyCollector, StandardSocketLegacyCollector, assess_legacy_services, ) -from cops.discovery.cli import command_legacy_discovery +from cops.discovery.cli import ( + command_legacy_discovery, +) class TestLegacyModelsAndSerialization(unittest.TestCase): @@ -436,7 +438,7 @@ def test_udp_socket_probe_success(self, mock_socket_cls, mock_resolve): @patch("socket.create_connection") def test_tcp_socket_timeout_inaccessible(self, mock_connect, mock_resolve): mock_resolve.return_value = "198.51.100.99" - mock_connect.side_effect = socket.timeout("timed out") + mock_connect.side_effect = TimeoutError("timed out") collector = StandardSocketLegacyCollector() ass = collector.probe_service( diff --git a/tests/test_m365_connectors.py b/tests/test_m365_connectors.py index 303a56c..bb235d3 100644 --- a/tests/test_m365_connectors.py +++ b/tests/test_m365_connectors.py @@ -7,10 +7,11 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1])) sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'plugins/detection-hunting/soc-investigation-workbench')) +from investigationwb.m365 import correlate + from cops.connectors import Checkpoint, GraphCollection, Response, collect from cops.connectors.adapters.m365 import SOURCES from cops.evidence import EvidenceError, canonical -from investigationwb.m365 import correlate TENANT = '00000000-0000-0000-0000-000000000001' USER = '00000000-0000-0000-0000-000000000002' diff --git a/tests/test_messaging_services.py b/tests/test_messaging_services.py index 8a4544a..aba4d6a 100644 --- a/tests/test_messaging_services.py +++ b/tests/test_messaging_services.py @@ -1,14 +1,15 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Unit, contract, and CLI tests for mail, chat, and message broker services assessment.""" from __future__ import annotations import argparse -import io import json -from pathlib import Path import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parent.parent if str(ROOT) not in sys.path: @@ -16,21 +17,20 @@ from cops.discovery import ( CleanupReceipt, - DEFAULT_MESSAGING_PORTS, MessagingAuthPrerequisite, MessagingCategory, MessagingExposureStatus, MessagingPrivilegeCandidate, MessagingPrivilegeImpact, MessagingServiceAssessment, - MessagingServicesCollector, MessagingServicesReport, MessagingServiceType, OfflineSyntheticMessagingCollector, - StandardSocketMessagingCollector, assess_messaging_services, ) -from cops.discovery.cli import command_messaging_discovery +from cops.discovery.cli import ( + command_messaging_discovery, +) class TestMessagingModelsAndSerialization(unittest.TestCase): diff --git a/tests/test_passive_discovery.py b/tests/test_passive_discovery.py index 0828de2..392737b 100644 --- a/tests/test_passive_discovery.py +++ b/tests/test_passive_discovery.py @@ -4,14 +4,13 @@ import json from pathlib import Path + import pytest from cops.discovery import ( DiscoveredAsset, - DiscoveryInventory, EvidenceProvenance, command_discovery, - merge_inventories, merge_two_assets, normalize_certificate_record, normalize_cloud_export, diff --git a/tests/test_prepare_pages.py b/tests/test_prepare_pages.py index 77c69a2..25942bb 100644 --- a/tests/test_prepare_pages.py +++ b/tests/test_prepare_pages.py @@ -4,7 +4,6 @@ import importlib.util from pathlib import Path -import pytest ROOT = Path(__file__).resolve().parents[1] SCRIPT_PATH = ROOT / "scripts" / "agent" / "prepare_pages.py" diff --git a/tests/test_remote_file_print_services.py b/tests/test_remote_file_print_services.py index 58bb62f..f4912f3 100644 --- a/tests/test_remote_file_print_services.py +++ b/tests/test_remote_file_print_services.py @@ -1,3 +1,5 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Unit, contract, and CLI tests for remote administration, file sharing, and printing services assessment.""" from __future__ import annotations @@ -5,18 +7,18 @@ import argparse import io import json -from pathlib import Path import sys import tempfile import unittest +from pathlib import Path ROOT = Path(__file__).resolve().parent.parent if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from cops.discovery import ( - CleanupReceipt, DEFAULT_REMOTE_PORTS, + CleanupReceipt, HostPrivilegeCandidate, LateralMovementImpact, OfflineSyntheticRemoteCollector, @@ -24,13 +26,14 @@ RemoteExposureStatus, RemoteServiceAssessment, RemoteServiceCategory, - RemoteServicesCollector, RemoteServicesReport, RemoteServiceType, StandardSocketRemoteCollector, assess_remote_services, ) -from cops.discovery.cli import command_remote_discovery +from cops.discovery.cli import ( + command_remote_discovery, +) class TestRemoteModelsAndSerialization(unittest.TestCase): diff --git a/tests/test_repository_context_reader.py b/tests/test_repository_context_reader.py index 6da3694..a8165fe 100644 --- a/tests/test_repository_context_reader.py +++ b/tests/test_repository_context_reader.py @@ -7,10 +7,9 @@ import subprocess import sys import tempfile - -import pytest from pathlib import Path +import pytest SCRIPT = Path(__file__).resolve().parents[1] / "plugins/logging-telemetry/security-logging-advisor/skills/repository-context/scripts/collect-repository-context.py" spec = importlib.util.spec_from_file_location("repository_context_scanner", SCRIPT) @@ -18,6 +17,17 @@ spec.loader.exec_module(scanner) +@pytest.mark.parametrize("schema_url,expected", [ + ("https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", True), + ("https://schema.management.azure.com.evil.example/template.json", False), + ("https://evil.example/schema.management.azure.com/template.json", False), +]) +def test_arm_classification_requires_schema_hostname(tmp_path, schema_url, expected): + (tmp_path / "template.json").write_text(json.dumps({"$schema": schema_url})) + result = scanner.scan_repository(tmp_path) + assert ("Azure ARM Template" in result["iac_and_cloud"]) is expected + + def test_replaced_file_is_not_read_after_discovery(tmp_path, monkeypatch): target = tmp_path / "config.env" replacement = tmp_path / "replacement.env" @@ -190,6 +200,30 @@ def test_ordinary_findings_and_universal_newlines_are_preserved(tmp_path): assert value not in json.dumps(result) +def test_arm_template_detection_uses_the_declared_schema_host(tmp_path): + template = tmp_path / "template.json" + template.write_text(json.dumps({ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "description": "A synthetic ARM template", + })) + + result = scanner.scan_repository(tmp_path) + + assert "Azure ARM Template" in result["iac_and_cloud"] + + +def test_arm_template_detection_rejects_schema_host_substrings(tmp_path): + template = tmp_path / "template.json" + template.write_text(json.dumps({ + "$schema": "https://schema.management.azure.com.attacker.example/schemas/template.json", + "description": "schema.management.azure.com", + })) + + result = scanner.scan_repository(tmp_path) + + assert "Azure ARM Template" not in result["iac_and_cloud"] + + @pytest.mark.skipif(os.name == "nt", reason="POSIX sockets") def test_socket_is_skipped(): # Keep the pathname within the smaller AF_UNIX limit, even with long temp roots. diff --git a/tests/test_routing.py b/tests/test_routing.py index 337cda1..526ddad 100644 --- a/tests/test_routing.py +++ b/tests/test_routing.py @@ -5,8 +5,9 @@ import io import json from pathlib import Path -import pytest + import jsonschema +import pytest from cops.authorization import ( AuthorizationDeniedError, diff --git a/tests/test_scenario_registry.py b/tests/test_scenario_registry.py index b646166..cd7b7d2 100644 --- a/tests/test_scenario_registry.py +++ b/tests/test_scenario_registry.py @@ -2,9 +2,9 @@ from __future__ import annotations -import io import json from pathlib import Path + import pytest from cops.cli import ( diff --git a/tests/test_scope_guard.py b/tests/test_scope_guard.py index de290ab..5d31ae2 100644 --- a/tests/test_scope_guard.py +++ b/tests/test_scope_guard.py @@ -3,6 +3,7 @@ from __future__ import annotations import ipaddress + import pytest from cops.execution import ScopeDefinition, ScopeGuard, ScopeViolationError diff --git a/tests/test_soc_intake_handoff.py b/tests/test_soc_intake_handoff.py index c7fd63b..5d3752e 100644 --- a/tests/test_soc_intake_handoff.py +++ b/tests/test_soc_intake_handoff.py @@ -1,14 +1,14 @@ +# Repository path setup precedes standalone entry point imports. +# ruff: noqa: E402 """Unit and integration tests for SOC evidence intake and case handoff workflow (Issue #15).""" from __future__ import annotations -import json import os -from pathlib import Path +import sys import tempfile import unittest - -import sys +from pathlib import Path ROOT = Path(__file__).resolve().parents[1] PLUGIN_DIR = ROOT / "plugins/detection-hunting/soc-investigation-workbench"