diff --git a/.github/workflows/repowise.yml b/.github/workflows/repowise.yml index 938c4b642..7ee5cd04a 100644 --- a/.github/workflows/repowise.yml +++ b/.github/workflows/repowise.yml @@ -18,7 +18,7 @@ jobs: code-health: name: RepoWise code health runs-on: [self-hosted, codex, Linux] - timeout-minutes: 30 + timeout-minutes: 40 env: REPOWISE_VENV_NAME: repowise-venv-${{ github.run_id }}-${{ github.run_attempt }}-code-health REPOWISE_REPORT_NAME: repowise-report-${{ github.run_id }}-${{ github.run_attempt }}-code-health @@ -52,54 +52,22 @@ jobs: "$RUNNER_TEMP/$REPOWISE_VENV_NAME/bin/python" -m pip install \ --disable-pip-version-check \ --no-cache-dir \ - 'repowise==0.45.0' + 'repowise==0.58.0' - name: Generate RepoWise reports env: REPOWISE_SKIP_EDITOR_SETUP: '1' run: | set -euo pipefail report_dir="$RUNNER_TEMP/$REPOWISE_REPORT_NAME" - repowise_bin="$RUNNER_TEMP/$REPOWISE_VENV_NAME/bin/repowise" - mkdir -p "$report_dir" - analyzed_sha="$(git rev-parse HEAD)" - printf '%s\n' "$analyzed_sha" > "$report_dir/revision.txt" + python_bin="$RUNNER_TEMP/$REPOWISE_VENV_NAME/bin/python" cleanup() { rm -rf .repowise .mcp.json rm -f .vscode/mcp.json .vscode/extensions.json rmdir .vscode 2>/dev/null || true } trap cleanup EXIT - "$repowise_bin" init \ - --no-prose --no-claude-md --no-codex --no-editor-setup \ - --no-onboarding --no-distill-hook --no-workspace . > "$report_dir/index.log" - "$repowise_bin" health --no-workspace --refactoring-targets --format md . | tee "$report_dir/health.md" - "$repowise_bin" health --no-workspace --refactoring-targets --format json . > "$report_dir/health.json" - advisory_scan() { - local name="$1" - shift - if (ulimit -v 2097152; timeout --kill-after=10s 90s "$@") > "$report_dir/$name.json"; then - printf '%s: complete\n' "$name" >> "$report_dir/advisory-status.txt" - else - local status="$?" - rm -f "$report_dir/$name.json" - printf '%s: incomplete (exit %s)\n' "$name" "$status" >> "$report_dir/advisory-status.txt" - echo "::warning::RepoWise $name advisory scan incomplete (exit $status); completed health reports remain available." - fi - } - advisory_scan dead-code "$repowise_bin" dead-code --safe-only --no-workspace --format json . - echo 'security scope: full reachable history (manual dispatch)' >> "$report_dir/advisory-status.txt" - advisory_scan security "$repowise_bin" security scan --history --path . --format json - { - echo '## RepoWise code health' - echo - echo 'Advisory reports are retained as workflow artifacts for this commit.' - echo - printf 'Analyzed revision: `%s`\n\n' "$analyzed_sha" - sed -n '1,100p' "$report_dir/health.md" - echo - echo '### Advisory scan status' - cat "$report_dir/advisory-status.txt" - } >> "$GITHUB_STEP_SUMMARY" + "$python_bin" -m unittest discover -s scripts/tests -p 'test_repowise*.py' -v + "$python_bin" scripts/repowise_reports.py --report-dir "$report_dir" - name: Upload RepoWise reports if: ${{ always() }} uses: actions/upload-artifact@v6 diff --git a/.gitignore b/.gitignore index 9a83bca90..eb924bdb6 100644 --- a/.gitignore +++ b/.gitignore @@ -147,3 +147,6 @@ opencode.json # Repository-local scratch output, including sandboxed Jest transform caches .tmp/ + +# Python analysis tooling +__pycache__/ diff --git a/docs/development/testing.mdx b/docs/development/testing.mdx index e0b9e9dd1..01a16d9f1 100644 --- a/docs/development/testing.mdx +++ b/docs/development/testing.mdx @@ -111,17 +111,37 @@ reviewed repository branch, and launch it. The CLI equivalent for development is `gh workflow run repowise.yml --ref develop`. The workflow already exists on the default branch, as required for [manual dispatch](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manually-run-a-workflow). -Open the run's Job Summary for the analyzed commit and scan status. Download -the `repowise-code-health` artifact for the index log, Markdown/JSON health -reports, `revision.txt` and `advisory-status.txt`; artifacts are retained for -14 days. History security scans cover the complete reachable history of that -revision. Dead-code and security scans each have a 90-second, 2 GiB limit -inside the 30-minute job budget. An incomplete scan produces a warning and an -incomplete status while preserving completed health reports. A failed health -step fails the run; available files are still uploaded. Inspect the failure -before retrying. A newer manual run for the same ref cancels the older one; -cancellation or runner loss can prevent artifact publication. No manual run -blocks the automatic CI path. +Open the run's Job Summary for the analyzed commit, phase status, elapsed time +and peak RSS. Download `repowise-code-health` (14 days) for health Markdown/JSON, +safe-only dead-code JSON, the security JSON aggregate, index/phase logs, +`revision.txt`, `scope.json`, `timings.json` and `history-resources.json`. + +The pinned RepoWise 0.58.0 builds one fresh standard index and exports its stored +health/dead-code results. Index git analytics retain a 500-commit limit; security +separately covers **all reachable history**, including removed secrets and generated +bundles, using RepoWise's supported source extensions, secret patterns and +allowlists. Two spawned workers read one blob at a time each, with at most two +tasks/results in flight and results consumed in upstream order. Worker failures +propagate. The adapter avoids +computing code-smell findings discarded by history mode. This remains an advisory +signal scanner; an aggregate with no findings is not proof that no secrets exist. + +Indexing has a 900-second/4 GiB virtual-memory cap. Each health export and dead-code +has a 60-second/2 GiB cap; security has a 1200-second/2 GiB coordinator cap and +a 1 GiB cap per worker, inside the +40-minute job. Limits apply per process; reported peak RSS is the largest process, +not aggregate simultaneous memory. These budgets accommodate the roughly 9 GB of +historical JavaScript without retaining all of it in memory. Two available CPU +cores are recommended for the bounded parallel history scan. + +An interrupted scan, Git error, stale/failed index or invalid report marks analysis +incomplete and fails this optional run. Completed reports and diagnostics still +upload; invalid JSON is removed. Inspect the named phase log, fix the cause, then +rerun. A newer manual run for the same ref cancels the older one; cancellation or +runner loss can prevent publication. No manual run blocks automatic CI. Upgrading +the pin requires the Python compatibility fixtures, which run before every manual +analysis. They compare secret detection with upstream and verify deleted secrets, +other refs, bounded reads and failures. Manual RepoWise runs use self-hosted Ubuntu `codex` runners. `CI Check` uses that pool for repository-owned PRs and pushes; fork PRs and all merge groups diff --git a/docs/security-operations/security/self-hosted-runners.mdx b/docs/security-operations/security/self-hosted-runners.mdx index 3130652bd..ec0aec513 100644 --- a/docs/security-operations/security/self-hosted-runners.mdx +++ b/docs/security-operations/security/self-hosted-runners.mdx @@ -46,7 +46,10 @@ Publishing currently requires a hosted runner. Their other jobs continue on self-hosted runners. RepoWise obtains a complete Python 3.12 distribution via `actions/setup-python`, creates its virtual environment and reports in run-specific temporary directories, and removes them after artifact upload. -Its runner needs a writable Actions tool cache; the workflow does not install +Its runner needs a writable Actions tool cache and GNU `/usr/bin/time` for phase +RSS evidence. The isolated Python suite verifies the pinned history adapter before +analysis; phase time/virtual-memory caps and a 40-minute job bound the run. +The workflow does not install Ubuntu system packages. RepoWise runs only through manual dispatch on `[self-hosted, codex, Linux]`, using the reviewed repository ref selected by the maintainer. It is optional and must not be required by branch protections or diff --git a/scripts/repowise_history.py b/scripts/repowise_history.py new file mode 100644 index 000000000..9fb0007af --- /dev/null +++ b/scripts/repowise_history.py @@ -0,0 +1,248 @@ +"""Bounded full-history adapter for the explicitly pinned RepoWise version. + +Keep RepoWise's traversal, provenance, allowlists, deduplication and detectors. +Replace only its eager blob dictionary and work discarded by secrets-only mode. +Private interfaces are deliberate: upgrades require the compatibility fixtures. +""" + +from __future__ import annotations + +import atexit +from collections import deque +from concurrent.futures import ProcessPoolExecutor +from importlib.metadata import version +import multiprocessing +from pathlib import Path +import subprocess + +REPOWISE_VERSION = "0.58.0" +HISTORY_WORKERS = 2 + + +class BlobReader: + """One request/response at a time; never retain previous blob contents.""" + + def __init__(self, repo: Path): + self.process = subprocess.Popen( + ["git", "-C", str(repo), "cat-file", "--batch"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + ) + self.count = 0 + self.bytes_read = 0 + self.largest_blob = 0 + atexit.register(self.close) + + def get(self, sha: str, default: str = "") -> str: + # Object names come from Git, never accept a batch protocol injection. + if len(sha) not in (40, 64) or any(c not in "0123456789abcdef" for c in sha): + raise ValueError("Invalid blob object name") + self.process.stdin.write((sha + "\n").encode("ascii")) + self.process.stdin.flush() + header = self.process.stdout.readline(256).split() + if len(header) != 3 or header[0].decode("ascii") != sha or header[1] != b"blob": + raise RuntimeError("Missing or malformed Git blob response") + size = int(header[2]) + if size < 0: + raise RuntimeError("Negative Git blob size") + content = self.process.stdout.read(size) + if len(content) != size or self.process.stdout.read(1) != b"\n": + raise RuntimeError("Truncated Git blob response") + self.count += 1 + self.bytes_read += size + self.largest_blob = max(self.largest_blob, size) + if self.count % 1000 == 0: + import sys + print(f"History: {self.count} source blobs, {self.bytes_read} bytes read", file=sys.stderr, flush=True) + return content.decode("utf-8", errors="replace") + + def close(self): + if self.process.poll() is None: + self.process.terminate() + try: + self.process.wait(timeout=2) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait() + for stream in (self.process.stdin, self.process.stdout): + if stream: + stream.close() + + +class SecretPrefilter: + # Necessary literals in every line-oriented SECRET_KINDS regex in 0.58.0. + # False positives are allowed; authoritative matching stays in RepoWise. + # PEM is a separate spanning pass and deliberately never gated here. + literals = ( + "password", "apikey", "api_key", "secret", "token", "accesskey", "access_key", + "akia", "asia", "gho_", "ghp_", "ghr_", "ghs_", "ghu_", "github_pat_", + "xox", "aiza", "sk_", "rk_", "glpat-", "gldt-", "glrt-", "glptt-", + "gloas-", "glimt-", "glcbt-", "gr1348941", "azdo", "://", + ) + + def search(self, text): + # Python IGNORECASE also accepts dotless i, long s and dotted I. + lowered = text.lower().replace("ı", "i").replace("ſ", "s").replace("\u0307", "") + return any(literal in lowered for literal in self.literals) + + +def configure_secret_scan(): + from repowise.core.analysis.security import scanner + from repowise.core.analysis.security.patterns import SECRET_KINDS + + scanner._PATTERNS = [p for p in scanner._PATTERNS if p[1] in SECRET_KINDS] + scanner._SPANNING_PATTERNS = [p for p in scanner._SPANNING_PATTERNS if p[1] in SECRET_KINDS] + scanner._ANY_PATTERN = SecretPrefilter() + # History's secrets_only=True discards all call findings; don't compute them. + scanner._call_findings = lambda *args: [] + + +def initialize_worker(repo): + import resource + import sys + from multiprocessing.util import Finalize + + # Two workers have a deliberately smaller cap than the coordinating CLI. + if sys.platform == "linux": + resource.setrlimit(resource.RLIMIT_AS, (1024 * 1024**2,) * 2) + configure_secret_scan() + global worker_reader + worker_reader = BlobReader(Path(repo)) + Finalize(None, worker_reader.close, exitpriority=10) + + +def scan_blob(sha, path): + from repowise.core.analysis.security.scanner import scan_source + + before = worker_reader.bytes_read + content = worker_reader.get(sha) + findings = scan_source(path, content) + # Upstream only needs source again for findings' fingerprints/material. + # Negative results carry no large source string across the process pipe. + return findings, content if findings else "", worker_reader.bytes_read - before + + +class PrefetchedBlobs: + """At most two tasks/results resident; consume in upstream history order.""" + + def __init__(self, repo, shas, paths): + self.pool = ProcessPoolExecutor( + max_workers=HISTORY_WORKERS, mp_context=multiprocessing.get_context("spawn"), + initializer=initialize_worker, initargs=(str(repo),), + ) + self.remaining = iter(shas) + self.paths = paths + self.pending = deque() + self.findings = [] + self.count = self.bytes_read = self.largest_blob = 0 + for _ in range(HISTORY_WORKERS): + self.enqueue() + + def enqueue(self): + sha = next(self.remaining, None) + if sha: + self.pending.append((sha, self.pool.submit(scan_blob, sha, self.paths[sha]))) + + def get(self, sha, default=""): + expected, future = self.pending.popleft() + if expected != sha: + raise RuntimeError("History traversal order changed; adapter compatibility failure") + self.findings, content, size = future.result() + self.count += 1 + self.bytes_read += size + self.largest_blob = max(self.largest_blob, size) + self.enqueue() + if self.count % 1000 == 0: + import sys + print(f"History: {self.count} source blobs, {self.bytes_read} bytes scanned", file=sys.stderr, flush=True) + return content + + def close(self): + self.pool.shutdown(wait=True, cancel_futures=True) + + +class PrefetchedScanner: + def __init__(self, reader, delegate): + self.reader = reader + self.delegate = delegate + + async def scan_file(self, path, content, symbols): + return self.reader.findings + + async def persist(self, *args, **kwargs): + return await self.delegate.persist(*args, **kwargs) + + +def bounded_scanner_class(original, readers): + class BoundedHistoryScanner(original): + def _unique_blobs(self, *args): + self.blob_paths = super()._unique_blobs(*args) + return self.blob_paths + + def _read_blobs_batch(self, repo_path, blob_shas): + reader = PrefetchedBlobs(repo_path, blob_shas, self.blob_paths) + readers.append(reader) + self._scanner = PrefetchedScanner(reader, self._scanner) + return reader + + return BoundedHistoryScanner + + +def strict_git(repo_path, args, *, timeout=30.0): + # Upstream's best-effort helper returns an empty string on Git failures, + # which could otherwise turn a failed history enumeration into a clean scan. + return subprocess.run( + ["git", "-C", str(repo_path), *args], check=True, timeout=timeout, + capture_output=True, text=True, encoding="utf-8", errors="replace", + ).stdout + + +def fail_on_persistence_error(): + import logging + + class PersistenceFailure(logging.Handler): + def emit(self, record): + if record.getMessage().startswith("security_finding_persist_failed"): + raise RuntimeError("History finding persistence failed") + + logger = logging.getLogger("repowise.core.analysis.security_scan") + # Override the CLI's ancestor ERROR level: upstream logs swallowed writes + # at WARNING. This handler raises before a diagnostic can pollute JSON. + logger.setLevel(logging.WARNING) + handler = PersistenceFailure() + logger.addHandler(handler) + return handler + + +def main(): + import json + import os + + if version("repowise") != REPOWISE_VERSION: + raise RuntimeError("Unsupported RepoWise version; rerun compatibility fixtures before upgrading") + from repowise.core.analysis import history_scan + from repowise.cli.main import cli + + readers = [] + fail_on_persistence_error() + history_scan._run_git = strict_git + history_scan.HistorySecurityScanner = bounded_scanner_class(history_scan.HistorySecurityScanner, readers) + try: + cli(["security", "scan", "--history", "--path", ".", "--format", "json"]) + finally: + for reader in readers: + reader.close() + metrics = os.environ.get("REPOWISE_HISTORY_METRICS") + if metrics: + Path(metrics).write_text(json.dumps({ + "blobs_read": sum(r.count for r in readers), + "bytes_read": sum(r.bytes_read for r in readers), + "largest_blob_bytes": max((r.largest_blob for r in readers), default=0), + "reader_cache_blobs": 0, + "max_blob_requests_in_flight": HISTORY_WORKERS, + "workers": HISTORY_WORKERS, + "worker_virtual_memory_mib": 1024, + }) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/repowise_reports.py b/scripts/repowise_reports.py new file mode 100644 index 000000000..e3cb0c8e8 --- /dev/null +++ b/scripts/repowise_reports.py @@ -0,0 +1,172 @@ +"""Manual RepoWise orchestration. Stdlib only; no application/runtime dependency.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import resource +import signal +import subprocess +import sys +import time + +from repowise_history import REPOWISE_VERSION + +# Maximum wall seconds and virtual MiB per process. Sum fits the 40-minute job, +# leaving setup, tests, publication and cleanup time. No caller-controlled flags. +BUDGETS = {"index": (900, 4096), "health-md": (60, 2048), + "health-json": (60, 2048), "dead-code": (60, 2048), "security": (1200, 2048)} + + +def validate_index(path): + events = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + summaries = [e for e in events if e.get("event") == "summary"] + dead = [e for e in events if e.get("phase") == "dead_code" and e.get("state") == "done"] + health = [e for e in events if e.get("phase") == "health" and e.get("state") == "done"] + if not summaries or summaries[-1].get("status") != "ready" or not dead or not health: + raise ValueError("Index did not confirm ready status and completed dead-code analysis") + if any(e.get("state") in ("failed", "error") for e in events): + raise ValueError("Index contains an incomplete phase") + state = json.loads(Path(".repowise/state.json").read_text()) + head = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + if state.get("last_sync_commit") != head: + raise ValueError("Index revision does not match HEAD") + + +def validate_json(path, kind): + value = json.loads(path.read_text()) + if kind == "dead-code": + if not isinstance(value, list) or not all(isinstance(f, dict) and "kind" in f for f in value): + raise ValueError("Invalid dead-code report") + if "Read from the index" not in (path.parent / "dead-code.stderr.log").read_text(): + raise ValueError("Dead-code did not reuse the index") + return value + if not isinstance(value, dict): + raise ValueError("Expected a JSON object") + required = {"health-json": ("targets", "refactoring_plans"), + "security": ("commits_scanned", "files_scanned", "blobs_scanned", "by_kind", "by_severity")}[kind] + if not all(k in value for k in required): + raise ValueError("Incomplete report schema") + if kind == "health-json" and value.get("source") != "index": + raise ValueError("Health report did not reuse the index") + if kind == "health-json": + head = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + if value.get("analyzed_commit") != head: + raise ValueError("Stored health revision does not match HEAD") + if kind == "security" and (value["commits_scanned"] <= 0 or value["files_scanned"] <= 0): + raise ValueError("History scan did not cover any commits or source blobs") + if kind == "security": + expected = int(subprocess.check_output(["git", "rev-list", "--all", "--count"], text=True)) + measured = json.loads((path.parent / "history-resources.json").read_text()) + if value["commits_scanned"] != expected or measured["blobs_read"] != value["files_scanned"]: + raise ValueError("History coverage does not match reachable commits and requested source blobs") + return value + + +def run_stage(name, command, output, report_dir, validator=None): + seconds, memory_mib = BUDGETS[name] + started = time.monotonic() + measurement = report_dir / (name + ".resources.txt") + result = {"status": "incomplete", "timeout_seconds": seconds, "virtual_memory_mib": memory_mib} + + def limits(): + resource.setrlimit(resource.RLIMIT_AS, (memory_mib * 1024**2,) * 2) + + try: + with output.open("w") as out, (report_dir / (name + ".stderr.log")).open("w") as err: + process = subprocess.Popen( + ["/usr/bin/time", "-f", "%e %M", "-o", str(measurement), *command], + stdout=out, stderr=err, start_new_session=True, preexec_fn=limits, + ) + try: + code = process.wait(timeout=seconds) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + raise RuntimeError(f"Timed out after {seconds}s") + if code: + raise RuntimeError(f"Command exited {code}; inspect {name}.stderr.log") + if validator: + validator(output) + elif not output.stat().st_size: + raise ValueError("Empty report") + result["status"] = "complete" + except Exception as error: + result["error"] = str(error) + # Keep diagnostic/index logs; never publish malformed JSON as a report. + if output.suffix == ".json": + output.unlink(missing_ok=True) + print(f"::warning::RepoWise {name} incomplete: {error}", flush=True) + finally: + result["wall_seconds"] = round(time.monotonic() - started, 3) + if measurement.exists(): + fields = (measurement.read_text().splitlines() or [""])[-1].split() + if len(fields) == 2 and fields[1].isdigit(): + result["peak_rss_kib"] = int(fields[1]) + print(f"RepoWise {name}: {result['status']} ({result['wall_seconds']}s)", flush=True) + return result + + +def write_summary(report_dir, sha, results): + lines = ["## RepoWise code health", "", f"Analyzed revision: `{sha}`", "", + "Security scope: full reachable history, RepoWise secret patterns; generated bundles included.", + "Health and safe-only dead-code: stored results from this revision's completed index.", "", + "| Phase | Status | Seconds | Peak RSS (MiB) |", "|---|---|---:|---:|"] + for name, result in results.items(): + rss = result.get("peak_rss_kib") + lines.append(f"| {name} | {result['status']} | {result.get('wall_seconds', '—')} | {round(rss / 1024, 1) if rss is not None else '—'} |") + incomplete = any(r["status"] != "complete" for r in results.values()) + lines += ["", ("Analysis incomplete. Inspect phase logs in the artifact, fix the cause, then rerun." + if incomplete else "Analysis complete. Findings are advisory; download the reports to review them."), + "Artifact: `repowise-code-health` (14 days), including scope, timings, logs and completed reports.", ""] + text = "\n".join(lines) + (report_dir / "summary.md").write_text(text) + (report_dir / "timings.json").write_text(json.dumps(results, indent=2) + "\n") + (report_dir / "advisory-status.txt").write_text("\n".join(f"{n}: {r['status']}" for n, r in results.items()) + "\n") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary: + summary.write(text) + return incomplete + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--report-dir", required=True, type=Path) + args = parser.parse_args() + report_dir = args.report_dir.resolve() + report_dir.mkdir(parents=True, exist_ok=True) + sha = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + (report_dir / "revision.txt").write_text(sha + "\n") + refs = subprocess.check_output(["git", "for-each-ref", "--format=%(objectname) %(refname)"], text=True).splitlines() + (report_dir / "scope.json").write_text(json.dumps({ + "revision": sha, "refs": refs, "repowise_version": REPOWISE_VERSION, + "security": "full reachable history (--all), RepoWise source extensions and secret patterns; upstream allowlists apply", + "health": "all indexed origins, cached refactoring targets", "dead_code": "cached safe-only findings", + "index_commit_limit": 500, "history_commit_limit": None, "logical_cpus": os.cpu_count(), + }, indent=2) + "\n") + os.environ.update(OPENBLAS_NUM_THREADS="1", OMP_NUM_THREADS="1", REPOWISE_SKIP_EDITOR_SETUP="1", REPOWISE_NO_SAVE_KEY="1", + REPOWISE_HISTORY_METRICS=str(report_dir / "history-resources.json")) + binary = str(Path(sys.executable).with_name("repowise")) + commands = { + "index": ([binary, "init", "--no-prose", "--no-claude-md", "--no-codex", "--no-editor-setup", + "--no-onboarding", "--no-distill-hook", "--no-workspace", "--no-seed", "--no-save-key", + "--commit-limit", "500", "--progress", "json", "."], "index.log", validate_index), + "health-md": ([binary, "health", "--no-workspace", "--refactoring-targets", "--scope", "all", "--format", "md", "."], "health.md", None), + "health-json": ([binary, "health", "--no-workspace", "--refactoring-targets", "--scope", "all", "--format", "json", "."], "health.json", lambda p: validate_json(p, "health-json")), + "dead-code": ([binary, "dead-code", "--safe-only", "--no-workspace", "--format", "json", "."], "dead-code.json", lambda p: validate_json(p, "dead-code")), + "security": ([sys.executable, str(Path(__file__).with_name("repowise_history.py"))], "security.json", lambda p: validate_json(p, "security")), + } + results = {} + for name, (command, filename, validator) in commands.items(): + # Exporting cached empty data after a failed index would be misleading. + if name != "security" and name != "index" and results["index"]["status"] != "complete": + results[name] = {"status": "incomplete", "error": "Index did not complete"} + continue + results[name] = run_stage(name, command, report_dir / filename, report_dir, validator) + return int(write_summary(report_dir, sha, results)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_repowise.py b/scripts/tests/test_repowise.py new file mode 100644 index 000000000..a41a400ff --- /dev/null +++ b/scripts/tests/test_repowise.py @@ -0,0 +1,355 @@ +"""Pinned-package compatibility and resource/failure regressions for manual CI.""" + +import asyncio +from contextlib import contextmanager +from importlib.metadata import version +import io +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import Mock, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import repowise_history as history +import repowise_reports as reports +from repowise.core.analysis import history_scan +from repowise.core.analysis.security import scanner +from repowise.core.analysis.security.patterns import SECRET_KINDS + + +@contextmanager +def optimized(): + names = ("_PATTERNS", "_SPANNING_PATTERNS", "_ANY_PATTERN", "_call_findings") + saved = {name: getattr(scanner, name) for name in names} + try: + history.configure_secret_scan() + yield + finally: + for name, value in saved.items(): + setattr(scanner, name, value) + + +class SecretCompatibility(unittest.TestCase): + def compare(self, kind, text): + expected = [f for f in scanner.scan_source("src/code.ts", text) if f["kind"] in SECRET_KINDS] + self.assertIn(kind, {f["kind"] for f in expected}) + with optimized(): + self.assertEqual(expected, scanner.scan_source("src/code.ts", text)) + + def test_pin(self): + self.assertEqual(history.REPOWISE_VERSION, version("repowise")) + + def test_password(self): + self.compare("hardcoded_password", 'password = "Q9vLm2rX7pNs4kT8"') + + def test_secret(self): + self.compare("hardcoded_secret", 'API_KEY = "V8mK2qZ9fT4nY7jP"') + + def test_unicode_case(self): + self.compare("hardcoded_secret", 'APı_KEY = "V8mK2qZ9fT4nY7jP"') + + def test_aws(self): + self.compare("aws_access_key", 'const x = "AK' + 'IA' + 'QWERTYUIOPASDFGH"') + + def test_github(self): + self.compare("github_token", 'const x = "gh' + 'p_' + 'Q2wE3rT4yU5iO6pA7sD8fG9hJ0kL1zX2cV3b' + '"') + + def test_slack(self): + self.compare("slack_token", 'const x = "xo' + 'xb-' + '123456789012-987654321012-A7bC9dE2fG4hI6jK"') + + def test_google(self): + self.compare("google_api_key", 'const x = "AI' + 'za' + 'Q2wE3rT4yU5iO6pA7sD8fG9hJ0kL1zX2cV3"') + + def test_stripe(self): + self.compare("stripe_key", 'const x = "sk_' + 'live_' + 'Q2wE3rT4yU5iO6pA7sD8fG9h"') + + def test_gitlab(self): + self.compare("gitlab_token", 'const x = "gl' + 'pat-' + 'Q2wE3rT4yU5iO6pA7sD8fG9h"') + + def test_azure(self): + self.compare("azure_devops_pat", 'const x = "' + ('Q2wE3rT4yU5iO6pA7sD8fG9hJ0kL1zX2cV3bN4m' * 3)[:76] + 'AZDOq7W2"') + + def test_git_url(self): + self.compare("git_url_credentials", 'const x = "https://user:Q9vLm2rX7pNs4kT8@github.com/org/repo.git"') + + def test_multiline_pem(self): + self.compare("private_key_pem", '-----BEGIN PRIVATE KEY-----\n' + 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASC' * 3 + '\n-----END PRIVATE KEY-----') + + def test_placeholders_and_suppression(self): + text = 'api_key = "example"; eval("code");\n// repowise:ignore\npassword = "${ENV}"' + expected = [f for f in scanner.scan_source("tests/a.ts", text) if f["kind"] in SECRET_KINDS] + with optimized(): + self.assertEqual(expected, scanner.scan_source("tests/a.ts", text)) + + +class BlobProtocol(unittest.TestCase): + def reader(self, response): + process = Mock(stdin=io.BytesIO(), stdout=io.BytesIO(response)) + process.poll.return_value = 0 + with patch.object(history.subprocess, "Popen", return_value=process), patch.object(history.atexit, "register"): + return history.BlobReader(Path(".")) + + def test_sequential_blobs_no_cache(self): + sha = "a" * 40 + reader = self.reader(f"{sha} blob 3\none\n{sha} blob 3\ntwo\n".encode()) + self.assertEqual("one", reader.get(sha)) + self.assertEqual("two", reader.get(sha)) + self.assertEqual((2, 6, 3), (reader.count, reader.bytes_read, reader.largest_blob)) + self.assertNotIn("contents", reader.__dict__) + + def test_missing_blob_fails(self): + reader = self.reader(b"a missing\n") + with self.assertRaises(RuntimeError): + reader.get("a" * 40) + + def test_truncated_blob_fails(self): + reader = self.reader(("a" * 40 + " blob 100\nshort").encode()) + with self.assertRaises(RuntimeError): + reader.get("a" * 40) + + def test_protocol_injection_rejected(self): + with self.assertRaises(ValueError): + self.reader(b"").get("HEAD\nHEAD") + + def test_git_error_is_not_empty_history(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaises(subprocess.CalledProcessError): + history.strict_git(Path(directory), ["rev-list", "--all"]) + + +class HistoryIntegration(unittest.TestCase): + def test_deleted_and_other_ref_secrets_preserve_upstream_provenance(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + def git(*args): + return subprocess.check_output(["git", "-C", directory, *args], stderr=subprocess.DEVNULL, text=True).strip() + git("init", "-b", "main") + git("config", "user.name", "Fixture") + git("config", "user.email", "fixture@example.invalid") + (root / "deleted.ts").write_text('password = "Q9vLm2rX7pNs4kT8"') + git("add", ".") + git("-c", "core.hooksPath=/dev/null", "commit", "-qm", "introduce") + introduction = git("rev-parse", "HEAD") + git("rm", "deleted.ts") + git("-c", "core.hooksPath=/dev/null", "commit", "-qm", "remove") + git("checkout", "-b", "other") + (root / "other.ts").write_text('secret = "J4fT8nP2qW6rY9kZ"') + git("add", ".") + git("-c", "core.hooksPath=/dev/null", "commit", "-qm", "other branch") + git("checkout", "main") + + class Sink: + def __init__(self): self.saved = [] + async def scan_file(self, path, content, symbols): return scanner.scan_source(path, content, symbols) + async def persist(self, path, findings, **provenance): + self.saved.append((path, findings, provenance)) + return len(findings) + + async def scan(bounded): + readers = [] + cls = history.bounded_scanner_class(history_scan.HistorySecurityScanner, readers) if bounded else history_scan.HistorySecurityScanner + instance = cls(None, "fixture") + sink = Sink() + instance._scanner = sink + try: + with patch.object(history_scan, "_run_git", history.strict_git): + summary = await instance.scan_history(root) + return summary, sink.saved + finally: + for reader in readers: reader.close() + + expected = asyncio.run(scan(False)) + with optimized(): actual = asyncio.run(scan(True)) + self.assertEqual(expected, actual) + self.assertEqual(3, actual[0].commits_scanned) + self.assertEqual(2, actual[0].findings_inserted) + deleted = next(f for f in actual[1] if f[0] == "deleted.ts") + self.assertEqual(introduction, deleted[2]["commit_sha"]) + + +class BoundedPrefetch(unittest.TestCase): + def reader(self, count=5): + pool = Mock() + def submit(function, sha, path): + future = Mock() + future.result.return_value = ([], "", 3) + return future + pool.submit.side_effect = submit + with patch.object(history, "ProcessPoolExecutor", return_value=pool): + reader = history.PrefetchedBlobs(Path("."), [str(i) for i in range(count)], {str(i): "a.ts" for i in range(count)}) + return reader, pool + + def test_queue_has_fixed_width(self): + reader, pool = self.reader() + self.assertEqual(2, pool.submit.call_count) + for i in range(5): + reader.get(str(i)) + self.assertLessEqual(len(reader.pending), 2) + self.assertEqual((5, 15), (reader.count, reader.bytes_read)) + reader.close() + pool.shutdown.assert_called_once_with(wait=True, cancel_futures=True) + + def test_traversal_order_mismatch_fails(self): + reader, _ = self.reader() + with self.assertRaises(RuntimeError): reader.get("other") + + def test_worker_failure_propagates(self): + reader, _ = self.reader() + reader.pending[0][1].result.side_effect = RuntimeError("worker failed") + with self.assertRaisesRegex(RuntimeError, "worker failed"): reader.get("0") + self.assertEqual(0, reader.count) + + def test_prefetched_results_preserve_delegate_persistence(self): + from unittest.mock import AsyncMock + reader = Mock(findings=[{"kind": "hardcoded_secret"}]) + delegate = Mock(); delegate.persist = AsyncMock(return_value=1) + instance = history.PrefetchedScanner(reader, delegate) + self.assertEqual(reader.findings, asyncio.run(instance.scan_file("a.ts", "", []))) + self.assertEqual(1, asyncio.run(instance.persist("a.ts", reader.findings, commit_sha="abc"))) + delegate.persist.assert_awaited_once_with("a.ts", reader.findings, commit_sha="abc") + + +class ReportFailures(unittest.TestCase): + def setUp(self): + # Failure fixtures must not publish fake warnings or a synthetic partial + # Job Summary when this suite runs inside the real manual workflow. + output = patch("sys.stdout", new_callable=io.StringIO) + output.start() + self.addCleanup(output.stop) + environment = patch.dict(reports.os.environ, {"GITHUB_STEP_SUMMARY": ""}) + environment.start() + self.addCleanup(environment.stop) + + def test_swallowed_persistence_failure_becomes_fatal(self): + from repowise.core.analysis.security.store import SecurityScanner, logger + from unittest.mock import AsyncMock + previous = logger.level + handler = history.fail_on_persistence_error() + session = Mock(); session.execute = AsyncMock(side_effect=RuntimeError("broken database")) + try: + instance = SecurityScanner(session, "fixture") + with self.assertRaisesRegex(RuntimeError, "persistence failed"): + asyncio.run(instance.persist("a.ts", [{"kind": "hardcoded_secret", "severity": "high"}])) + finally: + logger.removeHandler(handler) + logger.setLevel(previous) + + def test_budgets_fit_job_with_publication_margin(self): + self.assertLessEqual(sum(v[0] for v in reports.BUDGETS.values()), 38 * 60) + self.assertEqual((1200, 2048), reports.BUDGETS["security"]) + self.assertEqual((60, 2048), reports.BUDGETS["dead-code"]) + + def test_success_measures_rss_and_publishes_valid_json(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); p = root / "dead-code.json" + def start(command, **kwargs): + kwargs["stdout"].write("[]") + kwargs["stdout"].flush() + kwargs["stderr"].write("Read from the index") + kwargs["stderr"].flush() + (root / "dead-code.resources.txt").write_text("1.2 2048\n") + process = Mock(); process.wait.return_value = 0 + return process + with patch.object(reports.subprocess, "Popen", side_effect=start): + result = reports.run_stage("dead-code", ["fixture"], p, root, lambda p: reports.validate_json(p, "dead-code")) + self.assertEqual("complete", result["status"]) + self.assertEqual(2048, result["peak_rss_kib"]) + self.assertTrue(p.exists()) + + def test_dead_code_live_fallback_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); p = root / "dead-code.json" + p.write_text('[]'); (root / "dead-code.stderr.log").write_text("Computing live") + with self.assertRaisesRegex(ValueError, "reuse the index"): + reports.validate_json(p, "dead-code") + + def test_stale_health_revision_rejected(self): + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / "health.json" + p.write_text('{"targets":[],"refactoring_plans":[],"source":"index","analyzed_commit":"old"}') + with patch.object(reports.subprocess, "check_output", return_value="current"): + with self.assertRaisesRegex(ValueError, "revision"): + reports.validate_json(p, "health-json") + + def test_command_failure_is_incomplete(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); process = Mock(); process.wait.return_value = 1 + with patch.object(reports.subprocess, "Popen", return_value=process): + result = reports.run_stage("security", ["fixture"], root / "security.json", root) + self.assertEqual("incomplete", result["status"]) + self.assertIn("exited 1", result["error"]) + + def test_security_coverage_mismatch_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); p = root / "security.json" + p.write_text(json.dumps(dict(commits_scanned=3, files_scanned=2, blobs_scanned=3, by_kind={}, by_severity={}))) + (root / "history-resources.json").write_text('{"blobs_read":1}') + with patch.object(reports.subprocess, "check_output", return_value="3"): + with self.assertRaises(ValueError): reports.validate_json(p, "security") + + def test_live_health_report_rejected(self): + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / "health.json" + p.write_text('{"targets":[],"refactoring_plans":[],"source":"live"}') + with self.assertRaises(ValueError): reports.validate_json(p, "health-json") + + def test_dead_code_report_schema_rejected(self): + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / "dead-code.json"; p.write_text('{}') + with self.assertRaises(ValueError): reports.validate_json(p, "dead-code") + + def test_index_requires_completed_dead_code(self): + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / "index.log" + p.write_text(json.dumps({"event": "summary", "status": "ready"}) + "\n") + with self.assertRaises(ValueError): reports.validate_index(p) + + def test_index_rejects_degraded(self): + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / "index.log" + p.write_text('{"phase":"dead_code","state":"done"}\n{"event":"summary","status":"partial"}\n') + with self.assertRaises(ValueError): reports.validate_index(p) + + def test_malformed_json_deleted_and_failure_reported(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); p = root / "security.json" + process = Mock(pid=1234); process.wait.return_value = 0 + with patch.object(reports.subprocess, "Popen", return_value=process): + result = reports.run_stage("security", ["fixture"], p, root, lambda p: reports.validate_json(p, "security")) + self.assertEqual("incomplete", result["status"]) + self.assertFalse(p.exists()) + + def test_timeout_kills_process_group(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); process = Mock(pid=1234) + process.wait.side_effect = [subprocess.TimeoutExpired("fixture", 1), 0] + with patch.object(reports.subprocess, "Popen", return_value=process), patch.object(reports.os, "killpg") as kill: + result = reports.run_stage("dead-code", ["fixture"], root / "dead-code.json", root) + kill.assert_called_once_with(1234, reports.signal.SIGKILL) + self.assertEqual("incomplete", result["status"]) + + def test_empty_security_coverage_rejected(self): + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / "security.json" + p.write_text(json.dumps(dict(commits_scanned=0, files_scanned=0, blobs_scanned=0, by_kind={}, by_severity={}))) + with self.assertRaises(ValueError): reports.validate_json(p, "security") + + def test_partial_summary_retains_complete_phases(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + incomplete = reports.write_summary(root, "a" * 40, { + "health-md": {"status": "complete", "wall_seconds": 1, "peak_rss_kib": 1024}, + "security": {"status": "incomplete", "wall_seconds": 1200}, + }) + self.assertTrue(incomplete) + text = (root / "summary.md").read_text() + self.assertIn("Analysis incomplete", text) + self.assertIn("| health-md | complete |", text) + self.assertIn("full reachable history", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate-workflow-contract.cjs b/scripts/validate-workflow-contract.cjs index 06f2e5447..51eb5fb60 100644 --- a/scripts/validate-workflow-contract.cjs +++ b/scripts/validate-workflow-contract.cjs @@ -259,19 +259,15 @@ function assertRepoWiseTemporaryCleanup(relativeFile, job) { throw new Error(`${relativeFile} must select complete Python, isolate RepoWise files, and remove them after artifact upload.`); } const reports = (job.steps ?? []).find(step => step?.name === 'Generate RepoWise reports'); - if (!reports?.run?.includes('advisory_scan security "$repowise_bin" security scan --history --path . --format json') - || !reports.run.includes('security scope: full reachable history (manual dispatch)') - || !reports.run.includes('ulimit -v 2097152; timeout --kill-after=10s 90s') - || !reports.run.includes('incomplete (exit %s)') - || !reports.run.includes('health --no-workspace --refactoring-targets --format md') - || !reports.run.includes('health --no-workspace --refactoring-targets --format json') - || !reports.run.includes('analyzed_sha="$(git rev-parse HEAD)"') - || !reports.run.includes('printf \'%s\\n\' "$analyzed_sha" > "$report_dir/revision.txt"') - || !reports.run.includes('printf \'Analyzed revision: `%s`\\n\\n\' "$analyzed_sha"') - || !reports.run.includes('>> "$GITHUB_STEP_SUMMARY"') + const install = (job.steps ?? []).find(step => step?.name === 'Install RepoWise'); + if (!install?.run?.includes("'repowise==0.58.0'") + || !reports?.run?.includes('"$python_bin" scripts/repowise_reports.py --report-dir "$report_dir"') + || !reports.run.includes('"$python_bin" -m unittest discover -s scripts/tests -p \'test_repowise*.py\' -v') + || !reports.run.includes('trap cleanup EXIT') + || reports.run.replace('rmdir .vscode 2>/dev/null || true', '').includes('|| true') || upload.if !== '${{ always() }}' || upload.with?.['retention-days'] !== 14) { - throw new Error(`${relativeFile} must preserve bounded manual history scans, health reports, revision evidence, summary and unconditional artifacts.`); + throw new Error(`${relativeFile} must preserve pinned bounded reports, compatibility tests, cleanup and unconditional artifacts.`); } } @@ -283,7 +279,7 @@ function assertManualRepoWiseWorkflow(file, workflow) { } const job = workflow.jobs?.['code-health']; const checkout = (job?.steps ?? []).find(step => step?.uses === CHECKOUT_ACTION); - if (job?.['timeout-minutes'] !== 30 + if (job?.['timeout-minutes'] !== 40 || JSON.stringify(workflow.permissions) !== JSON.stringify({ contents: 'read' }) || job.permissions !== undefined || checkout?.with?.['fetch-depth'] !== 0 diff --git a/specs/CATALOG.md b/specs/CATALOG.md index 1cabdc39b..e78248ac6 100644 --- a/specs/CATALOG.md +++ b/specs/CATALOG.md @@ -12,7 +12,7 @@ debt or convert unknown historic intent into a design decision. |---|---|---|---|---| | `github-communication-experience` | Implemented | English-default, localized, semantic, bounded, and idempotent product messages across GitHub and repository-aware operator surfaces | [Semantic GitHub communication and repository localization](./semantic-github-publication-and-notification.md) + 3 companion | 277 paths · 2026-10-20 | | `release-orchestration` | Implemented | Release and hotfix promotion, publication, bounded merge readiness recovery, and receipt-verified reconciliation without republication | [Configurable production-first release orchestration](./configurable-release-orchestration.md) + 2 companion | 59 paths · 2026-10-16 | -| `merge-queue-readiness` | Implemented | Fail-closed validation of required checks and merge-group workflow support; optional manual-only RepoWise analysis with independent CI and verified required-check migration | [Merge queue readiness and effective target rules](./merge-queue-readiness.md) | 24 paths · 2026-10-21 | +| `merge-queue-readiness` | Implemented | Fail-closed validation of required checks and merge-group workflow support; optional manual-only RepoWise analysis with independent CI and verified required-check migration; cached RepoWise exports and bounded full-history secret processing with truthful completeness and resource evidence | [Merge queue readiness and effective target rules](./merge-queue-readiness.md) | 29 paths · 2026-10-21 | | `bugbot-review-state-reconciliation` | Implemented | Reconcile review snapshots, findings, threads, comments, and check conclusions | [Bugbot review-state reconciliation](./bugbot-review-state-reconciliation.md) | 59 paths · 2026-10-11 | | `execution-lifecycle` | Implemented | Shared GitHub Action lifecycle from event admission through durable user-facing results | [Execution admission, queueing, routing, and result publication](./execution-admission-queue-and-publication.md) + 3 companion | 97 paths · 2026-10-14 | | `architecture-quality-hardening` | Implemented | Close verified concurrency, error-contract, context-coupling, fan-out, setup/doctor, and provider-policy risks in dependency order | [Architecture quality and scalability hardening](./architecture-quality-and-scalability-hardening.md) + 1 companion | 72 paths · 2026-10-13 | @@ -63,9 +63,9 @@ debt or convert unknown historic intent into a design decision. - Specifications: [`specs/merge-queue-readiness.md`](./merge-queue-readiness.md) - Workflows: [`.github/workflows/ci_check.yml`](../.github/workflows/ci_check.yml) · [`.github/workflows/repowise.yml`](../.github/workflows/repowise.yml) · [`.github/workflows/copilot_pull_request_merge_queue.yml`](../.github/workflows/copilot_pull_request_merge_queue.yml) · [`setup/workflows/copilot_pull_request_merge_queue.yml`](../setup/workflows/copilot_pull_request_merge_queue.yml) · [`.github/workflows/copilot_deployment_orchestration.yml`](../.github/workflows/copilot_deployment_orchestration.yml) - Entrypoints: [`src/application/usecases/setup/merge_queue_readiness_use_case.ts`](../src/application/usecases/setup/merge_queue_readiness_use_case.ts) -- Core code: [`scripts/coverage-budgets.json`](../scripts/coverage-budgets.json) · [`src/domain/merge_queue_readiness.ts`](../src/domain/merge_queue_readiness.ts) · [`src/application/policies/merge_queue_message_catalog.ts`](../src/application/policies/merge_queue_message_catalog.ts) · [`src/application/policies/deployment_message_catalog.ts`](../src/application/policies/deployment_message_catalog.ts) · [`src/application/policies/deployment_plan_policy.ts`](../src/application/policies/deployment_plan_policy.ts) · [`src/application/policies/setup_doctor_message_catalog.ts`](../src/application/policies/setup_doctor_message_catalog.ts) · [`src/application/usecases/setup/merge_queue_readiness_use_case.ts`](../src/application/usecases/setup/merge_queue_readiness_use_case.ts) · [`src/data/repository/deployment/github_target_merge_capabilities_inspector.ts`](../src/data/repository/deployment/github_target_merge_capabilities_inspector.ts) · [`src/infrastructure/github/octokit_checks_adapters.ts`](../src/infrastructure/github/octokit_checks_adapters.ts) · [`scripts/validate-workflow-contract.cjs`](../scripts/validate-workflow-contract.cjs) -- Tests: [`src/domain/__tests__/merge_queue_readiness.test.ts`](../src/domain/__tests__/merge_queue_readiness.test.ts) · [`src/application/policies/__tests__/deployment_plan_policy.test.ts`](../src/application/policies/__tests__/deployment_plan_policy.test.ts) · [`src/application/policies/__tests__/setup_doctor_message_catalog.test.ts`](../src/application/policies/__tests__/setup_doctor_message_catalog.test.ts) · [`src/application/usecases/setup/__tests__/merge_queue_readiness_use_case.test.ts`](../src/application/usecases/setup/__tests__/merge_queue_readiness_use_case.test.ts) · [`src/data/repository/deployment/__tests__/github_deployment_adapters.test.ts`](../src/data/repository/deployment/__tests__/github_deployment_adapters.test.ts) · [`src/tooling/__tests__/validate_workflow_contract.test.ts`](../src/tooling/__tests__/validate_workflow_contract.test.ts) -- User documentation: [`docs/issues/deployment-orchestration.mdx`](../docs/issues/deployment-orchestration.mdx) · [`docs/security-operations/operations/verification.mdx`](../docs/security-operations/operations/verification.mdx) +- Core code: [`scripts/coverage-budgets.json`](../scripts/coverage-budgets.json) · [`src/domain/merge_queue_readiness.ts`](../src/domain/merge_queue_readiness.ts) · [`src/application/policies/merge_queue_message_catalog.ts`](../src/application/policies/merge_queue_message_catalog.ts) · [`src/application/policies/deployment_message_catalog.ts`](../src/application/policies/deployment_message_catalog.ts) · [`src/application/policies/deployment_plan_policy.ts`](../src/application/policies/deployment_plan_policy.ts) · [`src/application/policies/setup_doctor_message_catalog.ts`](../src/application/policies/setup_doctor_message_catalog.ts) · [`src/application/usecases/setup/merge_queue_readiness_use_case.ts`](../src/application/usecases/setup/merge_queue_readiness_use_case.ts) · [`src/data/repository/deployment/github_target_merge_capabilities_inspector.ts`](../src/data/repository/deployment/github_target_merge_capabilities_inspector.ts) · [`src/infrastructure/github/octokit_checks_adapters.ts`](../src/infrastructure/github/octokit_checks_adapters.ts) · [`scripts/validate-workflow-contract.cjs`](../scripts/validate-workflow-contract.cjs) · [`scripts/repowise_history.py`](../scripts/repowise_history.py) · [`scripts/repowise_reports.py`](../scripts/repowise_reports.py) +- Tests: [`src/domain/__tests__/merge_queue_readiness.test.ts`](../src/domain/__tests__/merge_queue_readiness.test.ts) · [`src/application/policies/__tests__/deployment_plan_policy.test.ts`](../src/application/policies/__tests__/deployment_plan_policy.test.ts) · [`src/application/policies/__tests__/setup_doctor_message_catalog.test.ts`](../src/application/policies/__tests__/setup_doctor_message_catalog.test.ts) · [`src/application/usecases/setup/__tests__/merge_queue_readiness_use_case.test.ts`](../src/application/usecases/setup/__tests__/merge_queue_readiness_use_case.test.ts) · [`src/data/repository/deployment/__tests__/github_deployment_adapters.test.ts`](../src/data/repository/deployment/__tests__/github_deployment_adapters.test.ts) · [`src/tooling/__tests__/validate_workflow_contract.test.ts`](../src/tooling/__tests__/validate_workflow_contract.test.ts) · [`scripts/tests/test_repowise.py`](../scripts/tests/test_repowise.py) +- User documentation: [`docs/issues/deployment-orchestration.mdx`](../docs/issues/deployment-orchestration.mdx) · [`docs/security-operations/operations/verification.mdx`](../docs/security-operations/operations/verification.mdx) · [`docs/development/testing.mdx`](../docs/development/testing.mdx) · [`docs/security-operations/security/self-hosted-runners.mdx`](../docs/security-operations/security/self-hosted-runners.mdx) ### `bugbot-review-state-reconciliation` — Bugbot review-state reconciliation diff --git a/specs/catalog.json b/specs/catalog.json index 5aa5def10..7bb958ab6 100644 --- a/specs/catalog.json +++ b/specs/catalog.json @@ -388,7 +388,7 @@ "id": "merge-queue-readiness", "title": "Merge queue readiness and effective target rules", "status": "implemented", - "scope": "Fail-closed validation of required checks and merge-group workflow support; optional manual-only RepoWise analysis with independent CI and verified required-check migration", + "scope": "Fail-closed validation of required checks and merge-group workflow support; optional manual-only RepoWise analysis with independent CI and verified required-check migration; cached RepoWise exports and bounded full-history secret processing with truthful completeness and resource evidence", "owner": "Copilot maintainers", "lastVerified": "2026-10-21", "specs": [ @@ -414,7 +414,9 @@ "src/application/usecases/setup/merge_queue_readiness_use_case.ts", "src/data/repository/deployment/github_target_merge_capabilities_inspector.ts", "src/infrastructure/github/octokit_checks_adapters.ts", - "scripts/validate-workflow-contract.cjs" + "scripts/validate-workflow-contract.cjs", + "scripts/repowise_history.py", + "scripts/repowise_reports.py" ], "tests": [ "src/domain/__tests__/merge_queue_readiness.test.ts", @@ -422,11 +424,14 @@ "src/application/policies/__tests__/setup_doctor_message_catalog.test.ts", "src/application/usecases/setup/__tests__/merge_queue_readiness_use_case.test.ts", "src/data/repository/deployment/__tests__/github_deployment_adapters.test.ts", - "src/tooling/__tests__/validate_workflow_contract.test.ts" + "src/tooling/__tests__/validate_workflow_contract.test.ts", + "scripts/tests/test_repowise.py" ], "documentation": [ "docs/issues/deployment-orchestration.mdx", - "docs/security-operations/operations/verification.mdx" + "docs/security-operations/operations/verification.mdx", + "docs/development/testing.mdx", + "docs/security-operations/security/self-hosted-runners.mdx" ] }, { diff --git a/specs/local-web-setup-assistant.md b/specs/local-web-setup-assistant.md index 72f8a0840..4b04c5fa2 100644 --- a/specs/local-web-setup-assistant.md +++ b/specs/local-web-setup-assistant.md @@ -2451,9 +2451,10 @@ now run only under `workflow_dispatch` on Ubuntu `codex` with the native selecte repository ref; it MUST NOT run on PR, push or merge-group events or be required for them. The retained CI routing cases still prove the fork/merge-group trust boundary. The [manual-only migration contract](./merge-queue-readiness.md#manual-only-repowise-migration-2026-10-21-issue-639) -defines the 24-case regression floor, live required-check evidence, immutable -revision reporting, unchanged scan budgets and cleanup, operator journey, -failure/retry behavior and final-revision manual acceptance. This is a workflow +and its [bounded-analysis follow-up](./merge-queue-readiness.md#bounded-manual-repowise-analysis-2026-10-21-issue-644) +define the 60-case regression floor, live required-check evidence, immutable +revision reporting, measured phase budgets and cleanup, operator journey, +failure/retry behavior and complete final-revision manual acceptance. This is a workflow scheduling change with no setup-session, PAT, web UX or new configuration change. The first [RepoWise run on `e197cb0c`](https://github.com/vypdev/copilot/actions/runs/37289899480) diff --git a/specs/merge-queue-readiness.md b/specs/merge-queue-readiness.md index 5cdc3f774..aa52a8ca8 100644 --- a/specs/merge-queue-readiness.md +++ b/specs/merge-queue-readiness.md @@ -193,10 +193,10 @@ already exists on the default branch. CLI example: `gh workflow run repowise.yml --ref develop`. The workflow owns event admission, Ubuntu `codex` runner selection, read-only -contents permission and a 30-minute job budget. Checkout MUST retain the selected -revision and full history without credentials. Each advisory subprocess retains -its 90-second and 2 GiB limits; manual security scanning covers full reachable -history. Per-ref `repowise-` concurrency cancels an older manual run for the same +contents permission and originally a 30-minute job budget. Checkout MUST retain the selected +revision and full history without credentials. The original advisory subprocesses used +90-second and 2 GiB limits; issue #644 supersedes these budgets below. Manual +security scanning covers full reachable history. Per-ref `repowise-` concurrency cancels an older manual run for the same ref without sharing CI's group. Python environments and reports remain isolated by run/attempt, with unconditional upload followed by cleanup. No domain, use case, port, provider DTO or composition change is needed: the repository workflow @@ -239,6 +239,119 @@ Graphify update. Completion requires a real manual run on the final revision and review of its summary/artifact; a queued run or static test is insufficient. Catalog evidence and the runner appendix in the setup-assistant SDD MUST agree. +### Bounded manual RepoWise analysis (2026-10-21, issue #644) + +Status: implemented contract; native final-revision evidence is required before +completion. Owner: Copilot maintainers. No open design decisions. This section +supersedes the original budgets, incomplete-status handling and regression count +above; admission, permissions, checkout, isolation and retention stay in force. + +The [previous manual run](https://github.com/vypdev/copilot/actions/runs/39573977241) +spent 20m37s in the job: indexing took 8m6s, health ran twice, dead-code exceeded +90 seconds and full-history security raised `MemoryError` under 2 GiB. Historical +JavaScript blobs alone total approximately 9.2 GB. Upstream +[RepoWise 0.58.0](https://github.com/repowise-dev/repowise/releases/tag/v0.58.0) +serves stored health/dead-code, but its history reader still materializes all +blobs. Upgrading alone does not resolve the memory failure. A local single-worker +streaming trial covered 22,975 source blobs / 9,515,793,415 bytes in 1232 seconds; +this CPU measurement motivates the fixed two-worker queue rather than merely +extending the history deadline. Native final-head metrics remain the release gate. + +```text +Manual selected ref -> one fresh standard index -> cached health MD/JSON + safe dead-code + -> full-history secret scan, two bounded analysis workers + -> validated reports + scope/timings -> artifact + summary +``` + +Text equivalent: index once, export stored results, scan full history with bounded +blob residency, validate completeness, and publish evidence. No application +runtime, setup session, domain model, provider port or public configuration changes +are needed. The workflow owns execution and cleanup; `repowise_reports.py` owns +phase sequencing, resource limits, validation and summary presentation; +`repowise_history.py` adapts pinned RepoWise/Git interfaces. Neither imports the +TypeScript application. YAML mutation tests enforce wiring; Python fixtures enforce +the private-package adapter and output semantics. + +Fixed, non-configurable execution budgets: + +| Phase | Wall limit | Virtual memory per process | Behavior | +|---|---:|---:|---| +| Fresh standard index | 900s | 4096 MiB | No prose/editor/key/hook setup or worktree seeding; 500 commits for git analytics | +| Each cached health format | 60s | 2048 MiB | All indexed origins, refactoring targets, no recomputation | +| Cached safe-only dead-code | 60s | 2048 MiB | Requires successful dead-code phase and matching index SHA | +| Full-history security | 1200s | 2048 MiB coordinator; 1024 MiB per worker | Two workers, all reachable refs/commits, supported source extensions, RepoWise secret kinds and upstream allowlists | +| Complete job | 40 minutes | Phase limits above | Leaves two minutes beyond summed phase ceilings for setup/publication | + +The 500-commit analytics setting preserves the former index default and MUST NOT +limit history security. Bundles remain included. The history adapter MUST retain +upstream provenance, secret kinds, deduplication, line suppression, path allowlists +and multiline PEM detection. It MUST fail on Git enumeration/read/persistence +errors. At most two tasks/results are in flight, with one request/response per worker. +The reader retains no blob cache; results are consumed in upstream history order. +Memory scales with metadata plus current blob processing, not total historical +content. Existing upstream metadata tables are still materialized. A single giant +blob can exceed the process cap; that is incomplete, never silently skipped. + +The adapter uses two spawned processes with a fixed-width prefetch queue. +Negative results do not copy source text back to the coordinator; positive results +retain complete text for unchanged fingerprint/material handling. Worker failures +propagate, and timeout kills the entire process group, including workers and Git. +The adapter omits call/code-smell analysis that upstream history mode discards, +and uses a conservative literal prefilter before the unchanged secret detectors. +Fixtures MUST compare every supported secret kind, Unicode keyword matching, +multiline PEM and removed/other-ref secrets against upstream behavior. The +version is pinned exactly; any upgrade requires compatibility fixtures before +publication. No remote cache or stale index is accepted. JSON health identifies +`source: index`; index events MUST confirm ready status plus completed health and +dead-code analysis at HEAD. Security MUST match the reachable commit count and +requested/read source-blob count. Malformed JSON is removed instead of published +as a complete report. + +The artifact MUST contain `revision.txt`, `scope.json` (SHA, ref snapshot, tool +version and explicit scopes), `timings.json`, `history-resources.json`, per-phase +logs/resource records, `summary.md`, advisory status and all completed reports. +RSS is GNU time's largest-process peak, not a sum of simultaneous processes; +virtual-memory limits apply to each process. Security output is RepoWise's JSON +aggregate, not a guarantee that no secrets exist. The local database is removed +and findings/secret material are not added to the artifact. + +The English summary shows status first by phase, SHA, seconds and peak RSS with +artifact/retry instructions. Example rows: `dead-code | complete | 2.1 | 96.0` and +`security | incomplete | 1200 | —`. Completed analysis says "Analysis complete. +Findings are advisory"; partial analysis says "Analysis incomplete. Inspect phase +logs in the artifact, fix the cause, then rerun." Incomplete scans now fail this +optional job while preserving completed reports. Index failure blocks cached +exports but still attempts the independent history scan. Timeout kills the process +group. Cancellation/runner loss can still prevent publication; retry manually. +No comments, locale settings or automatic retries are introduced. Native GitHub +queued/running/canceled states and artifact links remain the navigation; textual +statuses and plain tables work on narrow screens and in light/dark mode. + +Regression floor: **60 distinct cases**, comprising the retained **21** manual +workflow admission/permission/isolation/wiring/report mutations and **39** Python +cases: 14 detector/version compatibility, 5 blob/Git protocol/error cases, +1 deleted-secret/other-ref/provenance integration, 15 resource/report/failure cases, +and 4 bounded-prefetch/order/worker-failure/delegation cases. Cases use synthetic credentials, temporary repositories and mocked clocks/ +processes for timeouts; they do not need live services or real waits. The Python +suite runs in the pinned Python 3.12 manual environment before analysis. Repository +Jest coverage thresholds remain in force; the standalone Python adapter uses +behavioral compatibility/failure evidence rather than a new percentage threshold. + +| Requirement | Acceptance/evidence | Documentation | +|---|---|---| +| Cached results belong to completed current index | Progress/state/JSON guards, fixtures and final-head artifact | Testing guide | +| Full history completes without bulk contents | All-kind differential fixtures, deleted/other-ref provenance, native counts/RSS/bytes/timing | Testing + runner guides | +| Failures remain truthful and bounded | Protocol, malformed data, exit, timeout/group kill, partial summary fixtures | Retry instructions | +| Optional workflow remains isolated | Parsed YAML regressions, CI and manual final-head run | Manual dispatch guide | + +Roll out only through the linked PR. Rollback restores the former pin/workflow +but also restores known incomplete scans; it must not restore automatic triggers +or required checks. Definition of Done: the numeric test budget passes; workflow, +documentation and specification validators pass; catalog evidence and Graphify +are current; native final-head analysis produces **complete dead-code and security** +with measured time/memory and inspected summary/artifact; CI passes independently. +A queued run, green partial job, or merely relaxed limits is insufficient. + ### Other merge-queue evidence from the original implementation - Copilot's own required PR context is produced for merge groups by the dedicated `copilot_pull_request_merge_queue.yml` workflow. Keeping that diff --git a/src/tooling/__tests__/validate_workflow_contract.test.ts b/src/tooling/__tests__/validate_workflow_contract.test.ts index 2a6329c37..2da3fc728 100644 --- a/src/tooling/__tests__/validate_workflow_contract.test.ts +++ b/src/tooling/__tests__/validate_workflow_contract.test.ts @@ -1424,10 +1424,12 @@ describe('workflow contract validator', () => { const steps = workflow.jobs['code-health'].steps; const reports = steps.find((step: { name?: string }) => step.name === 'Generate RepoWise reports'); const upload = steps.find((step: { name?: string }) => step.name === 'Upload RepoWise reports'); - expect(reports.run).toContain('health --no-workspace --refactoring-targets --format md'); - expect(reports.run).toContain('health --no-workspace --refactoring-targets --format json'); - expect(reports.run).toContain('ulimit -v 2097152; timeout --kill-after=10s 90s'); - expect(reports.run).toContain('incomplete (exit %s)'); + expect(reports.run).toContain('"$python_bin" scripts/repowise_reports.py --report-dir "$report_dir"'); + expect(reports.run).toContain('"$python_bin" -m unittest discover'); + const adapter = readFileSync(path.join(process.cwd(), 'scripts', 'repowise_reports.py'), 'utf8'); + expect(adapter).toContain('resource.RLIMIT_AS'); + expect(adapter).toContain('os.killpg(process.pid, signal.SIGKILL)'); + expect(adapter).toContain('write_summary(report_dir, sha, results)'); expect(upload.if).toBe('${{ always() }}'); }); @@ -1476,22 +1478,21 @@ describe('workflow contract validator', () => { expect(() => assertRunner(file, workflow)).toThrow('must use runs-on self-hosted, codex, Linux'); }); - it.each(['full-history', 'scan-budget', 'incomplete-status', 'revision-file', 'summary', 'upload', 'retention', 'health-md', 'health-json'])( + it.each(['pin', 'script', 'compatibility', 'cleanup', 'upload', 'retention'])( 'rejects loss of RepoWise manual report evidence: %s', boundary => { const { file, workflow } = loadMutationWorkflow('.github/workflows', 'repowise.yml'); expect(() => validateWorkflow(file, workflow)).not.toThrow(); const job = workflow.jobs['code-health']; const reports = job.steps.find((step: { name?: string }) => step.name === 'Generate RepoWise reports'); + const install = job.steps.find((step: { name?: string }) => step.name === 'Install RepoWise'); const upload = job.steps.find((step: { name?: string }) => step.name === 'Upload RepoWise reports'); - if (boundary === 'full-history') reports.run = reports.run.replace('--history --path .', '--history --since HEAD --path .'); - else if (boundary === 'scan-budget') reports.run = reports.run.replace('ulimit -v 2097152;', ''); - else if (boundary === 'incomplete-status') reports.run = reports.run.replace('incomplete (exit %s)', 'complete'); - else if (boundary === 'revision-file') reports.run = reports.run.replace('> "$report_dir/revision.txt"', '> /dev/null'); - else if (boundary === 'summary') reports.run = reports.run.replace('>> "$GITHUB_STEP_SUMMARY"', '> /dev/null'); + if (boundary === 'pin') install.run = install.run.replace('repowise==0.58.0', 'repowise'); + else if (boundary === 'script') reports.run = reports.run.replace('scripts/repowise_reports.py', 'other.py'); + else if (boundary === 'compatibility') reports.run = reports.run.replace('-m unittest discover', '-m other'); + else if (boundary === 'cleanup') reports.run = reports.run.replace('trap cleanup EXIT', ''); else if (boundary === 'upload') delete upload.if; - else if (boundary === 'retention') upload.with['retention-days'] = 1; - else reports.run = reports.run.replace(`health --no-workspace --refactoring-targets --format ${boundary === 'health-md' ? 'md' : 'json'}`, 'status'); - expect(() => validateWorkflow(file, workflow)).toThrow('bounded manual history scans, health reports, revision evidence'); + else upload.with['retention-days'] = 1; + expect(() => validateWorkflow(file, workflow)).toThrow('pinned bounded reports, compatibility tests'); }, );