From 55a75494f75f8b3a328f073c47685aa9c84c7f51 Mon Sep 17 00:00:00 2001 From: Efra Espada Date: Wed, 21 Oct 2026 11:56:55 +0200 Subject: [PATCH 1/5] chore: bound manual RepoWise history and reuse indexed analysis --- .github/workflows/repowise.yml | 42 +-- .gitignore | 3 + docs/development/testing.mdx | 38 ++- .../security/self-hosted-runners.mdx | 5 +- scripts/repowise_history.py | 160 ++++++++++ scripts/repowise_reports.py | 166 ++++++++++ scripts/tests/test_repowise.py | 287 ++++++++++++++++++ scripts/validate-workflow-contract.cjs | 20 +- specs/CATALOG.md | 8 +- specs/catalog.json | 13 +- specs/local-web-setup-assistant.md | 7 +- specs/merge-queue-readiness.md | 113 ++++++- .../validate_workflow_contract.test.ts | 27 +- 13 files changed, 800 insertions(+), 89 deletions(-) create mode 100644 scripts/repowise_history.py create mode 100644 scripts/repowise_reports.py create mode 100644 scripts/tests/test_repowise.py diff --git a/.github/workflows/repowise.yml b/.github/workflows/repowise.yml index 938c4b642..7ee5cd04a 100644 --- a/.github/workflows/repowise.yml +++ b/.github/workflows/repowise.yml @@ -18,7 +18,7 @@ jobs: code-health: name: RepoWise code health runs-on: [self-hosted, codex, Linux] - timeout-minutes: 30 + timeout-minutes: 40 env: REPOWISE_VENV_NAME: repowise-venv-${{ github.run_id }}-${{ github.run_attempt }}-code-health REPOWISE_REPORT_NAME: repowise-report-${{ github.run_id }}-${{ github.run_attempt }}-code-health @@ -52,54 +52,22 @@ jobs: "$RUNNER_TEMP/$REPOWISE_VENV_NAME/bin/python" -m pip install \ --disable-pip-version-check \ --no-cache-dir \ - 'repowise==0.45.0' + 'repowise==0.58.0' - name: Generate RepoWise reports env: REPOWISE_SKIP_EDITOR_SETUP: '1' run: | set -euo pipefail report_dir="$RUNNER_TEMP/$REPOWISE_REPORT_NAME" - repowise_bin="$RUNNER_TEMP/$REPOWISE_VENV_NAME/bin/repowise" - mkdir -p "$report_dir" - analyzed_sha="$(git rev-parse HEAD)" - printf '%s\n' "$analyzed_sha" > "$report_dir/revision.txt" + python_bin="$RUNNER_TEMP/$REPOWISE_VENV_NAME/bin/python" cleanup() { rm -rf .repowise .mcp.json rm -f .vscode/mcp.json .vscode/extensions.json rmdir .vscode 2>/dev/null || true } trap cleanup EXIT - "$repowise_bin" init \ - --no-prose --no-claude-md --no-codex --no-editor-setup \ - --no-onboarding --no-distill-hook --no-workspace . > "$report_dir/index.log" - "$repowise_bin" health --no-workspace --refactoring-targets --format md . | tee "$report_dir/health.md" - "$repowise_bin" health --no-workspace --refactoring-targets --format json . > "$report_dir/health.json" - advisory_scan() { - local name="$1" - shift - if (ulimit -v 2097152; timeout --kill-after=10s 90s "$@") > "$report_dir/$name.json"; then - printf '%s: complete\n' "$name" >> "$report_dir/advisory-status.txt" - else - local status="$?" - rm -f "$report_dir/$name.json" - printf '%s: incomplete (exit %s)\n' "$name" "$status" >> "$report_dir/advisory-status.txt" - echo "::warning::RepoWise $name advisory scan incomplete (exit $status); completed health reports remain available." - fi - } - advisory_scan dead-code "$repowise_bin" dead-code --safe-only --no-workspace --format json . - echo 'security scope: full reachable history (manual dispatch)' >> "$report_dir/advisory-status.txt" - advisory_scan security "$repowise_bin" security scan --history --path . --format json - { - echo '## RepoWise code health' - echo - echo 'Advisory reports are retained as workflow artifacts for this commit.' - echo - printf 'Analyzed revision: `%s`\n\n' "$analyzed_sha" - sed -n '1,100p' "$report_dir/health.md" - echo - echo '### Advisory scan status' - cat "$report_dir/advisory-status.txt" - } >> "$GITHUB_STEP_SUMMARY" + "$python_bin" -m unittest discover -s scripts/tests -p 'test_repowise*.py' -v + "$python_bin" scripts/repowise_reports.py --report-dir "$report_dir" - name: Upload RepoWise reports if: ${{ always() }} uses: actions/upload-artifact@v6 diff --git a/.gitignore b/.gitignore index 9a83bca90..eb924bdb6 100644 --- a/.gitignore +++ b/.gitignore @@ -147,3 +147,6 @@ opencode.json # Repository-local scratch output, including sandboxed Jest transform caches .tmp/ + +# Python analysis tooling +__pycache__/ diff --git a/docs/development/testing.mdx b/docs/development/testing.mdx index e0b9e9dd1..9c952976e 100644 --- a/docs/development/testing.mdx +++ b/docs/development/testing.mdx @@ -111,17 +111,33 @@ reviewed repository branch, and launch it. The CLI equivalent for development is `gh workflow run repowise.yml --ref develop`. The workflow already exists on the default branch, as required for [manual dispatch](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manually-run-a-workflow). -Open the run's Job Summary for the analyzed commit and scan status. Download -the `repowise-code-health` artifact for the index log, Markdown/JSON health -reports, `revision.txt` and `advisory-status.txt`; artifacts are retained for -14 days. History security scans cover the complete reachable history of that -revision. Dead-code and security scans each have a 90-second, 2 GiB limit -inside the 30-minute job budget. An incomplete scan produces a warning and an -incomplete status while preserving completed health reports. A failed health -step fails the run; available files are still uploaded. Inspect the failure -before retrying. A newer manual run for the same ref cancels the older one; -cancellation or runner loss can prevent artifact publication. No manual run -blocks the automatic CI path. +Open the run's Job Summary for the analyzed commit, phase status, elapsed time +and peak RSS. Download `repowise-code-health` (14 days) for health Markdown/JSON, +safe-only dead-code JSON, the security JSON aggregate, index/phase logs, +`revision.txt`, `scope.json`, `timings.json` and `history-resources.json`. + +The pinned RepoWise 0.58.0 builds one fresh standard index and exports its stored +health/dead-code results. Index git analytics retain a 500-commit limit; security +separately covers **all reachable history**, including removed secrets and generated +bundles, using RepoWise's supported source extensions, secret patterns and +allowlists. Historical blob contents are read one at a time. The adapter avoids +computing code-smell findings discarded by history mode. This remains an advisory +signal scanner; an aggregate with no findings is not proof that no secrets exist. + +Indexing has a 900-second/4 GiB virtual-memory cap. Each health export and dead-code +has a 60-second/2 GiB cap; security has a 1200-second/2 GiB cap, inside the +40-minute job. Limits apply per process; reported peak RSS is the largest process, +not aggregate simultaneous memory. These budgets accommodate the roughly 9 GB of +historical JavaScript without retaining all of it in memory. + +An interrupted scan, Git error, stale/failed index or invalid report marks analysis +incomplete and fails this optional run. Completed reports and diagnostics still +upload; invalid JSON is removed. Inspect the named phase log, fix the cause, then +rerun. A newer manual run for the same ref cancels the older one; cancellation or +runner loss can prevent publication. No manual run blocks automatic CI. Upgrading +the pin requires the Python compatibility fixtures, which run before every manual +analysis. They compare secret detection with upstream and verify deleted secrets, +other refs, bounded reads and failures. Manual RepoWise runs use self-hosted Ubuntu `codex` runners. `CI Check` uses that pool for repository-owned PRs and pushes; fork PRs and all merge groups diff --git a/docs/security-operations/security/self-hosted-runners.mdx b/docs/security-operations/security/self-hosted-runners.mdx index 3130652bd..ec0aec513 100644 --- a/docs/security-operations/security/self-hosted-runners.mdx +++ b/docs/security-operations/security/self-hosted-runners.mdx @@ -46,7 +46,10 @@ Publishing currently requires a hosted runner. Their other jobs continue on self-hosted runners. RepoWise obtains a complete Python 3.12 distribution via `actions/setup-python`, creates its virtual environment and reports in run-specific temporary directories, and removes them after artifact upload. -Its runner needs a writable Actions tool cache; the workflow does not install +Its runner needs a writable Actions tool cache and GNU `/usr/bin/time` for phase +RSS evidence. The isolated Python suite verifies the pinned history adapter before +analysis; phase time/virtual-memory caps and a 40-minute job bound the run. +The workflow does not install Ubuntu system packages. RepoWise runs only through manual dispatch on `[self-hosted, codex, Linux]`, using the reviewed repository ref selected by the maintainer. It is optional and must not be required by branch protections or diff --git a/scripts/repowise_history.py b/scripts/repowise_history.py new file mode 100644 index 000000000..f05bd85c0 --- /dev/null +++ b/scripts/repowise_history.py @@ -0,0 +1,160 @@ +"""Bounded full-history adapter for the explicitly pinned RepoWise version. + +Keep RepoWise's traversal, provenance, allowlists, deduplication and detectors. +Replace only its eager blob dictionary and work discarded by secrets-only mode. +Private interfaces are deliberate: upgrades require the compatibility fixtures. +""" + +from __future__ import annotations + +import atexit +from importlib.metadata import version +from pathlib import Path +import subprocess + +REPOWISE_VERSION = "0.58.0" + + +class BlobReader: + """One request/response at a time; never retain previous blob contents.""" + + def __init__(self, repo: Path): + self.process = subprocess.Popen( + ["git", "-C", str(repo), "cat-file", "--batch"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + ) + self.count = 0 + self.bytes_read = 0 + self.largest_blob = 0 + atexit.register(self.close) + + def get(self, sha: str, default: str = "") -> str: + # Object names come from Git, never accept a batch protocol injection. + if len(sha) not in (40, 64) or any(c not in "0123456789abcdef" for c in sha): + raise ValueError("Invalid blob object name") + self.process.stdin.write((sha + "\n").encode("ascii")) + self.process.stdin.flush() + header = self.process.stdout.readline(256).split() + if len(header) != 3 or header[0].decode("ascii") != sha or header[1] != b"blob": + raise RuntimeError("Missing or malformed Git blob response") + size = int(header[2]) + if size < 0: + raise RuntimeError("Negative Git blob size") + content = self.process.stdout.read(size) + if len(content) != size or self.process.stdout.read(1) != b"\n": + raise RuntimeError("Truncated Git blob response") + self.count += 1 + self.bytes_read += size + self.largest_blob = max(self.largest_blob, size) + if self.count % 1000 == 0: + import sys + print(f"History: {self.count} source blobs, {self.bytes_read} bytes read", file=sys.stderr, flush=True) + return content.decode("utf-8", errors="replace") + + def close(self): + if self.process.poll() is None: + self.process.terminate() + try: + self.process.wait(timeout=2) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait() + for stream in (self.process.stdin, self.process.stdout): + if stream: + stream.close() + + +class SecretPrefilter: + # Necessary literals in every line-oriented SECRET_KINDS regex in 0.58.0. + # False positives are allowed; authoritative matching stays in RepoWise. + # PEM is a separate spanning pass and deliberately never gated here. + literals = ( + "password", "apikey", "api_key", "secret", "token", "accesskey", "access_key", + "akia", "asia", "gho_", "ghp_", "ghr_", "ghs_", "ghu_", "github_pat_", + "xox", "aiza", "sk_", "rk_", "glpat-", "gldt-", "glrt-", "glptt-", + "gloas-", "glimt-", "glcbt-", "gr1348941", "azdo", "://", + ) + + def search(self, text): + # Python IGNORECASE also accepts dotless i, long s and dotted I. + lowered = text.lower().replace("ı", "i").replace("ſ", "s").replace("\u0307", "") + return any(literal in lowered for literal in self.literals) + + +def configure_secret_scan(): + from repowise.core.analysis.security import scanner + from repowise.core.analysis.security.patterns import SECRET_KINDS + + scanner._PATTERNS = [p for p in scanner._PATTERNS if p[1] in SECRET_KINDS] + scanner._SPANNING_PATTERNS = [p for p in scanner._SPANNING_PATTERNS if p[1] in SECRET_KINDS] + scanner._ANY_PATTERN = SecretPrefilter() + # History's secrets_only=True discards all call findings; don't compute them. + scanner._call_findings = lambda *args: [] + + +def strict_git(repo_path, args, *, timeout=30.0): + # Upstream's best-effort helper returns an empty string on Git failures, + # which could otherwise turn a failed history enumeration into a clean scan. + return subprocess.run( + ["git", "-C", str(repo_path), *args], check=True, timeout=timeout, + capture_output=True, text=True, encoding="utf-8", errors="replace", + ).stdout + + +def fail_on_persistence_error(): + import logging + + class PersistenceFailure(logging.Handler): + def emit(self, record): + if record.getMessage().startswith("security_finding_persist_failed"): + raise RuntimeError("History finding persistence failed") + + logger = logging.getLogger("repowise.core.analysis.security_scan") + # Override the CLI's ancestor ERROR level: upstream logs swallowed writes + # at WARNING. This handler raises before a diagnostic can pollute JSON. + logger.setLevel(logging.WARNING) + handler = PersistenceFailure() + logger.addHandler(handler) + return handler + + +def main(): + import json + import os + + if version("repowise") != REPOWISE_VERSION: + raise RuntimeError("Unsupported RepoWise version; rerun compatibility fixtures before upgrading") + from repowise.core.analysis import history_scan + from repowise.cli.main import cli + + readers = [] + original = history_scan.HistorySecurityScanner + + class BoundedHistoryScanner(original): + def _read_blobs_batch(self, repo_path, blob_shas): + reader = BlobReader(repo_path) + readers.append(reader) + return reader + + fail_on_persistence_error() + history_scan._run_git = strict_git + history_scan.HistorySecurityScanner = BoundedHistoryScanner + configure_secret_scan() + try: + cli(["security", "scan", "--history", "--path", ".", "--format", "json"]) + finally: + for reader in readers: + reader.close() + metrics = os.environ.get("REPOWISE_HISTORY_METRICS") + if metrics: + Path(metrics).write_text(json.dumps({ + "blobs_read": sum(r.count for r in readers), + "bytes_read": sum(r.bytes_read for r in readers), + "largest_blob_bytes": max((r.largest_blob for r in readers), default=0), + "reader_cache_blobs": 0, + "max_blob_requests_in_flight": 1, + }) + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/repowise_reports.py b/scripts/repowise_reports.py new file mode 100644 index 000000000..55d50a797 --- /dev/null +++ b/scripts/repowise_reports.py @@ -0,0 +1,166 @@ +"""Manual RepoWise orchestration. Stdlib only; no application/runtime dependency.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import resource +import signal +import subprocess +import sys +import time + +from repowise_history import REPOWISE_VERSION + +# Maximum wall seconds and virtual MiB per process. Sum fits the 40-minute job, +# leaving setup, tests, publication and cleanup time. No caller-controlled flags. +BUDGETS = {"index": (900, 4096), "health-md": (60, 2048), + "health-json": (60, 2048), "dead-code": (60, 2048), "security": (1200, 2048)} + + +def validate_index(path): + events = [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + summaries = [e for e in events if e.get("event") == "summary"] + dead = [e for e in events if e.get("phase") == "dead_code" and e.get("state") == "done"] + health = [e for e in events if e.get("phase") == "health" and e.get("state") == "done"] + if not summaries or summaries[-1].get("status") != "ready" or not dead or not health: + raise ValueError("Index did not confirm ready status and completed dead-code analysis") + if any(e.get("state") in ("failed", "error") for e in events): + raise ValueError("Index contains an incomplete phase") + state = json.loads(Path(".repowise/state.json").read_text()) + head = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + if state.get("last_sync_commit") != head: + raise ValueError("Index revision does not match HEAD") + + +def validate_json(path, kind): + value = json.loads(path.read_text()) + if kind == "dead-code": + if not isinstance(value, list) or not all(isinstance(f, dict) and "kind" in f for f in value): + raise ValueError("Invalid dead-code report") + return value + if not isinstance(value, dict): + raise ValueError("Expected a JSON object") + required = {"health-json": ("targets", "refactoring_plans"), + "security": ("commits_scanned", "files_scanned", "blobs_scanned", "by_kind", "by_severity")}[kind] + if not all(k in value for k in required): + raise ValueError("Incomplete report schema") + if kind == "health-json" and value.get("source") != "index": + raise ValueError("Health report did not reuse the index") + if kind == "security" and (value["commits_scanned"] <= 0 or value["files_scanned"] <= 0): + raise ValueError("History scan did not cover any commits or source blobs") + if kind == "security": + expected = int(subprocess.check_output(["git", "rev-list", "--all", "--count"], text=True)) + measured = json.loads((path.parent / "history-resources.json").read_text()) + if value["commits_scanned"] != expected or measured["blobs_read"] != value["files_scanned"]: + raise ValueError("History coverage does not match reachable commits and requested source blobs") + return value + + +def run_stage(name, command, output, report_dir, validator=None): + seconds, memory_mib = BUDGETS[name] + started = time.monotonic() + measurement = report_dir / (name + ".resources.txt") + result = {"status": "incomplete", "timeout_seconds": seconds, "virtual_memory_mib": memory_mib} + + def limits(): + resource.setrlimit(resource.RLIMIT_AS, (memory_mib * 1024**2,) * 2) + + try: + with output.open("w") as out, (report_dir / (name + ".stderr.log")).open("w") as err: + process = subprocess.Popen( + ["/usr/bin/time", "-f", "%e %M", "-o", str(measurement), *command], + stdout=out, stderr=err, start_new_session=True, preexec_fn=limits, + ) + try: + code = process.wait(timeout=seconds) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + raise RuntimeError(f"Timed out after {seconds}s") + if code: + raise RuntimeError(f"Command exited {code}; inspect {name}.stderr.log") + if validator: + validator(output) + elif not output.stat().st_size: + raise ValueError("Empty report") + result["status"] = "complete" + except Exception as error: + result["error"] = str(error) + # Keep diagnostic/index logs; never publish malformed JSON as a report. + if output.suffix == ".json": + output.unlink(missing_ok=True) + print(f"::warning::RepoWise {name} incomplete: {error}", flush=True) + finally: + result["wall_seconds"] = round(time.monotonic() - started, 3) + if measurement.exists(): + fields = (measurement.read_text().splitlines() or [""])[-1].split() + if len(fields) == 2 and fields[1].isdigit(): + result["peak_rss_kib"] = int(fields[1]) + print(f"RepoWise {name}: {result['status']} ({result['wall_seconds']}s)", flush=True) + return result + + +def write_summary(report_dir, sha, results): + lines = ["## RepoWise code health", "", f"Analyzed revision: `{sha}`", "", + "Security scope: full reachable history, RepoWise secret patterns; generated bundles included.", + "Health and safe-only dead-code: stored results from this revision's completed index.", "", + "| Phase | Status | Seconds | Peak RSS (MiB) |", "|---|---|---:|---:|"] + for name, result in results.items(): + rss = result.get("peak_rss_kib") + lines.append(f"| {name} | {result['status']} | {result.get('wall_seconds', '—')} | {round(rss / 1024, 1) if rss is not None else '—'} |") + incomplete = any(r["status"] != "complete" for r in results.values()) + lines += ["", ("Analysis incomplete. Inspect phase logs in the artifact, fix the cause, then rerun." + if incomplete else "Analysis complete. Findings are advisory; download the reports to review them."), + "Artifact: `repowise-code-health` (14 days), including scope, timings, logs and completed reports.", ""] + text = "\n".join(lines) + (report_dir / "summary.md").write_text(text) + (report_dir / "timings.json").write_text(json.dumps(results, indent=2) + "\n") + (report_dir / "advisory-status.txt").write_text("\n".join(f"{n}: {r['status']}" for n, r in results.items()) + "\n") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary: + summary.write(text) + return incomplete + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--report-dir", required=True, type=Path) + args = parser.parse_args() + report_dir = args.report_dir.resolve() + report_dir.mkdir(parents=True, exist_ok=True) + sha = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + (report_dir / "revision.txt").write_text(sha + "\n") + refs = subprocess.check_output(["git", "for-each-ref", "--format=%(objectname) %(refname)"], text=True).splitlines() + (report_dir / "scope.json").write_text(json.dumps({ + "revision": sha, "refs": refs, "repowise_version": REPOWISE_VERSION, + "security": "full reachable history (--all), RepoWise source extensions and secret patterns; upstream allowlists apply", + "health": "all indexed origins, cached refactoring targets", "dead_code": "cached safe-only findings", + "index_commit_limit": 500, "history_commit_limit": None, + }, indent=2) + "\n") + os.environ.update(OPENBLAS_NUM_THREADS="1", OMP_NUM_THREADS="1", REPOWISE_SKIP_EDITOR_SETUP="1", REPOWISE_NO_SAVE_KEY="1", + REPOWISE_HISTORY_METRICS=str(report_dir / "history-resources.json")) + binary = str(Path(sys.executable).with_name("repowise")) + commands = { + "index": ([binary, "init", "--no-prose", "--no-claude-md", "--no-codex", "--no-editor-setup", + "--no-onboarding", "--no-distill-hook", "--no-workspace", "--no-seed", "--no-save-key", + "--commit-limit", "500", "--progress", "json", "."], "index.log", validate_index), + "health-md": ([binary, "health", "--no-workspace", "--refactoring-targets", "--scope", "all", "--format", "md", "."], "health.md", None), + "health-json": ([binary, "health", "--no-workspace", "--refactoring-targets", "--scope", "all", "--format", "json", "."], "health.json", lambda p: validate_json(p, "health-json")), + "dead-code": ([binary, "dead-code", "--safe-only", "--no-workspace", "--format", "json", "."], "dead-code.json", lambda p: validate_json(p, "dead-code")), + "security": ([sys.executable, str(Path(__file__).with_name("repowise_history.py"))], "security.json", lambda p: validate_json(p, "security")), + } + results = {} + for name, (command, filename, validator) in commands.items(): + # Exporting cached empty data after a failed index would be misleading. + if name != "security" and name != "index" and results["index"]["status"] != "complete": + results[name] = {"status": "incomplete", "error": "Index did not complete"} + continue + results[name] = run_stage(name, command, report_dir / filename, report_dir, validator) + return int(write_summary(report_dir, sha, results)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_repowise.py b/scripts/tests/test_repowise.py new file mode 100644 index 000000000..926c631ca --- /dev/null +++ b/scripts/tests/test_repowise.py @@ -0,0 +1,287 @@ +"""Pinned-package compatibility and resource/failure regressions for manual CI.""" + +import asyncio +from contextlib import contextmanager +from importlib.metadata import version +import io +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import Mock, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import repowise_history as history +import repowise_reports as reports +from repowise.core.analysis import history_scan +from repowise.core.analysis.security import scanner +from repowise.core.analysis.security.patterns import SECRET_KINDS + + +@contextmanager +def optimized(): + names = ("_PATTERNS", "_SPANNING_PATTERNS", "_ANY_PATTERN", "_call_findings") + saved = {name: getattr(scanner, name) for name in names} + try: + history.configure_secret_scan() + yield + finally: + for name, value in saved.items(): + setattr(scanner, name, value) + + +class SecretCompatibility(unittest.TestCase): + def compare(self, kind, text): + expected = [f for f in scanner.scan_source("src/code.ts", text) if f["kind"] in SECRET_KINDS] + self.assertIn(kind, {f["kind"] for f in expected}) + with optimized(): + self.assertEqual(expected, scanner.scan_source("src/code.ts", text)) + + def test_pin(self): + self.assertEqual(history.REPOWISE_VERSION, version("repowise")) + + def test_password(self): + self.compare("hardcoded_password", 'password = "Q9vLm2rX7pNs4kT8"') + + def test_secret(self): + self.compare("hardcoded_secret", 'API_KEY = "V8mK2qZ9fT4nY7jP"') + + def test_unicode_case(self): + self.compare("hardcoded_secret", 'APı_KEY = "V8mK2qZ9fT4nY7jP"') + + def test_aws(self): + self.compare("aws_access_key", 'const x = "AK' + 'IA' + 'QWERTYUIOPASDFGH"') + + def test_github(self): + self.compare("github_token", 'const x = "gh' + 'p_' + 'Q2wE3rT4yU5iO6pA7sD8fG9hJ0kL1zX2cV3b' + '"') + + def test_slack(self): + self.compare("slack_token", 'const x = "xo' + 'xb-' + '123456789012-987654321012-A7bC9dE2fG4hI6jK"') + + def test_google(self): + self.compare("google_api_key", 'const x = "AI' + 'za' + 'Q2wE3rT4yU5iO6pA7sD8fG9hJ0kL1zX2cV3"') + + def test_stripe(self): + self.compare("stripe_key", 'const x = "sk_' + 'live_' + 'Q2wE3rT4yU5iO6pA7sD8fG9h"') + + def test_gitlab(self): + self.compare("gitlab_token", 'const x = "gl' + 'pat-' + 'Q2wE3rT4yU5iO6pA7sD8fG9h"') + + def test_azure(self): + self.compare("azure_devops_pat", 'const x = "' + ('Q2wE3rT4yU5iO6pA7sD8fG9hJ0kL1zX2cV3bN4m' * 3)[:76] + 'AZDOq7W2"') + + def test_git_url(self): + self.compare("git_url_credentials", 'const x = "https://user:Q9vLm2rX7pNs4kT8@github.com/org/repo.git"') + + def test_multiline_pem(self): + self.compare("private_key_pem", '-----BEGIN PRIVATE KEY-----\n' + 'MIIEvQIBADANBgkqhkiG9w0BAQEFAASC' * 3 + '\n-----END PRIVATE KEY-----') + + def test_placeholders_and_suppression(self): + text = 'api_key = "example"; eval("code");\n// repowise:ignore\npassword = "${ENV}"' + expected = [f for f in scanner.scan_source("tests/a.ts", text) if f["kind"] in SECRET_KINDS] + with optimized(): + self.assertEqual(expected, scanner.scan_source("tests/a.ts", text)) + + +class BlobProtocol(unittest.TestCase): + def reader(self, response): + process = Mock(stdin=io.BytesIO(), stdout=io.BytesIO(response)) + process.poll.return_value = 0 + with patch.object(history.subprocess, "Popen", return_value=process), patch.object(history.atexit, "register"): + return history.BlobReader(Path(".")) + + def test_sequential_blobs_no_cache(self): + sha = "a" * 40 + reader = self.reader(f"{sha} blob 3\none\n{sha} blob 3\ntwo\n".encode()) + self.assertEqual("one", reader.get(sha)) + self.assertEqual("two", reader.get(sha)) + self.assertEqual((2, 6, 3), (reader.count, reader.bytes_read, reader.largest_blob)) + self.assertNotIn("contents", reader.__dict__) + + def test_missing_blob_fails(self): + reader = self.reader(b"a missing\n") + with self.assertRaises(RuntimeError): + reader.get("a" * 40) + + def test_truncated_blob_fails(self): + reader = self.reader(("a" * 40 + " blob 100\nshort").encode()) + with self.assertRaises(RuntimeError): + reader.get("a" * 40) + + def test_protocol_injection_rejected(self): + with self.assertRaises(ValueError): + self.reader(b"").get("HEAD\nHEAD") + + def test_git_error_is_not_empty_history(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaises(subprocess.CalledProcessError): + history.strict_git(Path(directory), ["rev-list", "--all"]) + + +class HistoryIntegration(unittest.TestCase): + def test_deleted_and_other_ref_secrets_preserve_upstream_provenance(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + def git(*args): + return subprocess.check_output(["git", "-C", directory, *args], stderr=subprocess.DEVNULL, text=True).strip() + git("init", "-b", "main") + git("config", "user.name", "Fixture") + git("config", "user.email", "fixture@example.invalid") + (root / "deleted.ts").write_text('password = "Q9vLm2rX7pNs4kT8"') + git("add", ".") + git("-c", "core.hooksPath=/dev/null", "commit", "-qm", "introduce") + introduction = git("rev-parse", "HEAD") + git("rm", "deleted.ts") + git("-c", "core.hooksPath=/dev/null", "commit", "-qm", "remove") + git("checkout", "-b", "other") + (root / "other.ts").write_text('secret = "J4fT8nP2qW6rY9kZ"') + git("add", ".") + git("-c", "core.hooksPath=/dev/null", "commit", "-qm", "other branch") + git("checkout", "main") + + class Sink: + def __init__(self): self.saved = [] + async def scan_file(self, path, content, symbols): return scanner.scan_source(path, content, symbols) + async def persist(self, path, findings, **provenance): + self.saved.append((path, findings, provenance)) + return len(findings) + + async def scan(bounded): + instance = history_scan.HistorySecurityScanner(None, "fixture") + sink = Sink() + instance._scanner = sink + reader = history.BlobReader(root) if bounded else None + if bounded: + instance._read_blobs_batch = lambda *_: reader + try: + with patch.object(history_scan, "_run_git", history.strict_git): + summary = await instance.scan_history(root) + return summary, sink.saved + finally: + if reader: reader.close() + + expected = asyncio.run(scan(False)) + with optimized(): actual = asyncio.run(scan(True)) + self.assertEqual(expected, actual) + self.assertEqual(3, actual[0].commits_scanned) + self.assertEqual(2, actual[0].findings_inserted) + deleted = next(f for f in actual[1] if f[0] == "deleted.ts") + self.assertEqual(introduction, deleted[2]["commit_sha"]) + + +class ReportFailures(unittest.TestCase): + def test_swallowed_persistence_failure_becomes_fatal(self): + from repowise.core.analysis.security.store import SecurityScanner, logger + from unittest.mock import AsyncMock + previous = logger.level + handler = history.fail_on_persistence_error() + session = Mock(); session.execute = AsyncMock(side_effect=RuntimeError("broken database")) + try: + instance = SecurityScanner(session, "fixture") + with self.assertRaisesRegex(RuntimeError, "persistence failed"): + asyncio.run(instance.persist("a.ts", [{"kind": "hardcoded_secret", "severity": "high"}])) + finally: + logger.removeHandler(handler) + logger.setLevel(previous) + + def test_budgets_fit_job_with_publication_margin(self): + self.assertLessEqual(sum(v[0] for v in reports.BUDGETS.values()), 38 * 60) + self.assertEqual((1200, 2048), reports.BUDGETS["security"]) + self.assertEqual((60, 2048), reports.BUDGETS["dead-code"]) + + def test_success_measures_rss_and_publishes_valid_json(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); p = root / "dead-code.json" + def start(command, **kwargs): + kwargs["stdout"].write("[]") + kwargs["stdout"].flush() + (root / "dead-code.resources.txt").write_text("1.2 2048\n") + process = Mock(); process.wait.return_value = 0 + return process + with patch.object(reports.subprocess, "Popen", side_effect=start): + result = reports.run_stage("dead-code", ["fixture"], p, root, lambda p: reports.validate_json(p, "dead-code")) + self.assertEqual("complete", result["status"]) + self.assertEqual(2048, result["peak_rss_kib"]) + self.assertTrue(p.exists()) + + def test_command_failure_is_incomplete(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); process = Mock(); process.wait.return_value = 1 + with patch.object(reports.subprocess, "Popen", return_value=process): + result = reports.run_stage("security", ["fixture"], root / "security.json", root) + self.assertEqual("incomplete", result["status"]) + self.assertIn("exited 1", result["error"]) + + def test_security_coverage_mismatch_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); p = root / "security.json" + p.write_text(json.dumps(dict(commits_scanned=3, files_scanned=2, blobs_scanned=3, by_kind={}, by_severity={}))) + (root / "history-resources.json").write_text('{"blobs_read":1}') + with patch.object(reports.subprocess, "check_output", return_value="3"): + with self.assertRaises(ValueError): reports.validate_json(p, "security") + + def test_live_health_report_rejected(self): + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / "health.json" + p.write_text('{"targets":[],"refactoring_plans":[],"source":"live"}') + with self.assertRaises(ValueError): reports.validate_json(p, "health-json") + + def test_dead_code_report_schema_rejected(self): + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / "dead-code.json"; p.write_text('{}') + with self.assertRaises(ValueError): reports.validate_json(p, "dead-code") + + def test_index_requires_completed_dead_code(self): + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / "index.log" + p.write_text(json.dumps({"event": "summary", "status": "ready"}) + "\n") + with self.assertRaises(ValueError): reports.validate_index(p) + + def test_index_rejects_degraded(self): + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / "index.log" + p.write_text('{"phase":"dead_code","state":"done"}\n{"event":"summary","status":"partial"}\n') + with self.assertRaises(ValueError): reports.validate_index(p) + + def test_malformed_json_deleted_and_failure_reported(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); p = root / "security.json" + process = Mock(pid=1234); process.wait.return_value = 0 + with patch.object(reports.subprocess, "Popen", return_value=process): + result = reports.run_stage("security", ["fixture"], p, root, lambda p: reports.validate_json(p, "security")) + self.assertEqual("incomplete", result["status"]) + self.assertFalse(p.exists()) + + def test_timeout_kills_process_group(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); process = Mock(pid=1234) + process.wait.side_effect = [subprocess.TimeoutExpired("fixture", 1), 0] + with patch.object(reports.subprocess, "Popen", return_value=process), patch.object(reports.os, "killpg") as kill: + result = reports.run_stage("dead-code", ["fixture"], root / "dead-code.json", root) + kill.assert_called_once_with(1234, reports.signal.SIGKILL) + self.assertEqual("incomplete", result["status"]) + + def test_empty_security_coverage_rejected(self): + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / "security.json" + p.write_text(json.dumps(dict(commits_scanned=0, files_scanned=0, blobs_scanned=0, by_kind={}, by_severity={}))) + with self.assertRaises(ValueError): reports.validate_json(p, "security") + + def test_partial_summary_retains_complete_phases(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + incomplete = reports.write_summary(root, "a" * 40, { + "health-md": {"status": "complete", "wall_seconds": 1, "peak_rss_kib": 1024}, + "security": {"status": "incomplete", "wall_seconds": 1200}, + }) + self.assertTrue(incomplete) + text = (root / "summary.md").read_text() + self.assertIn("Analysis incomplete", text) + self.assertIn("| health-md | complete |", text) + self.assertIn("full reachable history", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate-workflow-contract.cjs b/scripts/validate-workflow-contract.cjs index 06f2e5447..51eb5fb60 100644 --- a/scripts/validate-workflow-contract.cjs +++ b/scripts/validate-workflow-contract.cjs @@ -259,19 +259,15 @@ function assertRepoWiseTemporaryCleanup(relativeFile, job) { throw new Error(`${relativeFile} must select complete Python, isolate RepoWise files, and remove them after artifact upload.`); } const reports = (job.steps ?? []).find(step => step?.name === 'Generate RepoWise reports'); - if (!reports?.run?.includes('advisory_scan security "$repowise_bin" security scan --history --path . --format json') - || !reports.run.includes('security scope: full reachable history (manual dispatch)') - || !reports.run.includes('ulimit -v 2097152; timeout --kill-after=10s 90s') - || !reports.run.includes('incomplete (exit %s)') - || !reports.run.includes('health --no-workspace --refactoring-targets --format md') - || !reports.run.includes('health --no-workspace --refactoring-targets --format json') - || !reports.run.includes('analyzed_sha="$(git rev-parse HEAD)"') - || !reports.run.includes('printf \'%s\\n\' "$analyzed_sha" > "$report_dir/revision.txt"') - || !reports.run.includes('printf \'Analyzed revision: `%s`\\n\\n\' "$analyzed_sha"') - || !reports.run.includes('>> "$GITHUB_STEP_SUMMARY"') + const install = (job.steps ?? []).find(step => step?.name === 'Install RepoWise'); + if (!install?.run?.includes("'repowise==0.58.0'") + || !reports?.run?.includes('"$python_bin" scripts/repowise_reports.py --report-dir "$report_dir"') + || !reports.run.includes('"$python_bin" -m unittest discover -s scripts/tests -p \'test_repowise*.py\' -v') + || !reports.run.includes('trap cleanup EXIT') + || reports.run.replace('rmdir .vscode 2>/dev/null || true', '').includes('|| true') || upload.if !== '${{ always() }}' || upload.with?.['retention-days'] !== 14) { - throw new Error(`${relativeFile} must preserve bounded manual history scans, health reports, revision evidence, summary and unconditional artifacts.`); + throw new Error(`${relativeFile} must preserve pinned bounded reports, compatibility tests, cleanup and unconditional artifacts.`); } } @@ -283,7 +279,7 @@ function assertManualRepoWiseWorkflow(file, workflow) { } const job = workflow.jobs?.['code-health']; const checkout = (job?.steps ?? []).find(step => step?.uses === CHECKOUT_ACTION); - if (job?.['timeout-minutes'] !== 30 + if (job?.['timeout-minutes'] !== 40 || JSON.stringify(workflow.permissions) !== JSON.stringify({ contents: 'read' }) || job.permissions !== undefined || checkout?.with?.['fetch-depth'] !== 0 diff --git a/specs/CATALOG.md b/specs/CATALOG.md index 1cabdc39b..e78248ac6 100644 --- a/specs/CATALOG.md +++ b/specs/CATALOG.md @@ -12,7 +12,7 @@ debt or convert unknown historic intent into a design decision. |---|---|---|---|---| | `github-communication-experience` | Implemented | English-default, localized, semantic, bounded, and idempotent product messages across GitHub and repository-aware operator surfaces | [Semantic GitHub communication and repository localization](./semantic-github-publication-and-notification.md) + 3 companion | 277 paths · 2026-10-20 | | `release-orchestration` | Implemented | Release and hotfix promotion, publication, bounded merge readiness recovery, and receipt-verified reconciliation without republication | [Configurable production-first release orchestration](./configurable-release-orchestration.md) + 2 companion | 59 paths · 2026-10-16 | -| `merge-queue-readiness` | Implemented | Fail-closed validation of required checks and merge-group workflow support; optional manual-only RepoWise analysis with independent CI and verified required-check migration | [Merge queue readiness and effective target rules](./merge-queue-readiness.md) | 24 paths · 2026-10-21 | +| `merge-queue-readiness` | Implemented | Fail-closed validation of required checks and merge-group workflow support; optional manual-only RepoWise analysis with independent CI and verified required-check migration; cached RepoWise exports and bounded full-history secret processing with truthful completeness and resource evidence | [Merge queue readiness and effective target rules](./merge-queue-readiness.md) | 29 paths · 2026-10-21 | | `bugbot-review-state-reconciliation` | Implemented | Reconcile review snapshots, findings, threads, comments, and check conclusions | [Bugbot review-state reconciliation](./bugbot-review-state-reconciliation.md) | 59 paths · 2026-10-11 | | `execution-lifecycle` | Implemented | Shared GitHub Action lifecycle from event admission through durable user-facing results | [Execution admission, queueing, routing, and result publication](./execution-admission-queue-and-publication.md) + 3 companion | 97 paths · 2026-10-14 | | `architecture-quality-hardening` | Implemented | Close verified concurrency, error-contract, context-coupling, fan-out, setup/doctor, and provider-policy risks in dependency order | [Architecture quality and scalability hardening](./architecture-quality-and-scalability-hardening.md) + 1 companion | 72 paths · 2026-10-13 | @@ -63,9 +63,9 @@ debt or convert unknown historic intent into a design decision. - Specifications: [`specs/merge-queue-readiness.md`](./merge-queue-readiness.md) - Workflows: [`.github/workflows/ci_check.yml`](../.github/workflows/ci_check.yml) · [`.github/workflows/repowise.yml`](../.github/workflows/repowise.yml) · [`.github/workflows/copilot_pull_request_merge_queue.yml`](../.github/workflows/copilot_pull_request_merge_queue.yml) · [`setup/workflows/copilot_pull_request_merge_queue.yml`](../setup/workflows/copilot_pull_request_merge_queue.yml) · [`.github/workflows/copilot_deployment_orchestration.yml`](../.github/workflows/copilot_deployment_orchestration.yml) - Entrypoints: [`src/application/usecases/setup/merge_queue_readiness_use_case.ts`](../src/application/usecases/setup/merge_queue_readiness_use_case.ts) -- Core code: [`scripts/coverage-budgets.json`](../scripts/coverage-budgets.json) · [`src/domain/merge_queue_readiness.ts`](../src/domain/merge_queue_readiness.ts) · [`src/application/policies/merge_queue_message_catalog.ts`](../src/application/policies/merge_queue_message_catalog.ts) · [`src/application/policies/deployment_message_catalog.ts`](../src/application/policies/deployment_message_catalog.ts) · [`src/application/policies/deployment_plan_policy.ts`](../src/application/policies/deployment_plan_policy.ts) · [`src/application/policies/setup_doctor_message_catalog.ts`](../src/application/policies/setup_doctor_message_catalog.ts) · [`src/application/usecases/setup/merge_queue_readiness_use_case.ts`](../src/application/usecases/setup/merge_queue_readiness_use_case.ts) · [`src/data/repository/deployment/github_target_merge_capabilities_inspector.ts`](../src/data/repository/deployment/github_target_merge_capabilities_inspector.ts) · [`src/infrastructure/github/octokit_checks_adapters.ts`](../src/infrastructure/github/octokit_checks_adapters.ts) · [`scripts/validate-workflow-contract.cjs`](../scripts/validate-workflow-contract.cjs) -- Tests: [`src/domain/__tests__/merge_queue_readiness.test.ts`](../src/domain/__tests__/merge_queue_readiness.test.ts) · [`src/application/policies/__tests__/deployment_plan_policy.test.ts`](../src/application/policies/__tests__/deployment_plan_policy.test.ts) · [`src/application/policies/__tests__/setup_doctor_message_catalog.test.ts`](../src/application/policies/__tests__/setup_doctor_message_catalog.test.ts) · [`src/application/usecases/setup/__tests__/merge_queue_readiness_use_case.test.ts`](../src/application/usecases/setup/__tests__/merge_queue_readiness_use_case.test.ts) · [`src/data/repository/deployment/__tests__/github_deployment_adapters.test.ts`](../src/data/repository/deployment/__tests__/github_deployment_adapters.test.ts) · [`src/tooling/__tests__/validate_workflow_contract.test.ts`](../src/tooling/__tests__/validate_workflow_contract.test.ts) -- User documentation: [`docs/issues/deployment-orchestration.mdx`](../docs/issues/deployment-orchestration.mdx) · [`docs/security-operations/operations/verification.mdx`](../docs/security-operations/operations/verification.mdx) +- Core code: [`scripts/coverage-budgets.json`](../scripts/coverage-budgets.json) · [`src/domain/merge_queue_readiness.ts`](../src/domain/merge_queue_readiness.ts) · [`src/application/policies/merge_queue_message_catalog.ts`](../src/application/policies/merge_queue_message_catalog.ts) · [`src/application/policies/deployment_message_catalog.ts`](../src/application/policies/deployment_message_catalog.ts) · [`src/application/policies/deployment_plan_policy.ts`](../src/application/policies/deployment_plan_policy.ts) · [`src/application/policies/setup_doctor_message_catalog.ts`](../src/application/policies/setup_doctor_message_catalog.ts) · [`src/application/usecases/setup/merge_queue_readiness_use_case.ts`](../src/application/usecases/setup/merge_queue_readiness_use_case.ts) · [`src/data/repository/deployment/github_target_merge_capabilities_inspector.ts`](../src/data/repository/deployment/github_target_merge_capabilities_inspector.ts) · [`src/infrastructure/github/octokit_checks_adapters.ts`](../src/infrastructure/github/octokit_checks_adapters.ts) · [`scripts/validate-workflow-contract.cjs`](../scripts/validate-workflow-contract.cjs) · [`scripts/repowise_history.py`](../scripts/repowise_history.py) · [`scripts/repowise_reports.py`](../scripts/repowise_reports.py) +- Tests: [`src/domain/__tests__/merge_queue_readiness.test.ts`](../src/domain/__tests__/merge_queue_readiness.test.ts) · [`src/application/policies/__tests__/deployment_plan_policy.test.ts`](../src/application/policies/__tests__/deployment_plan_policy.test.ts) · [`src/application/policies/__tests__/setup_doctor_message_catalog.test.ts`](../src/application/policies/__tests__/setup_doctor_message_catalog.test.ts) · [`src/application/usecases/setup/__tests__/merge_queue_readiness_use_case.test.ts`](../src/application/usecases/setup/__tests__/merge_queue_readiness_use_case.test.ts) · [`src/data/repository/deployment/__tests__/github_deployment_adapters.test.ts`](../src/data/repository/deployment/__tests__/github_deployment_adapters.test.ts) · [`src/tooling/__tests__/validate_workflow_contract.test.ts`](../src/tooling/__tests__/validate_workflow_contract.test.ts) · [`scripts/tests/test_repowise.py`](../scripts/tests/test_repowise.py) +- User documentation: [`docs/issues/deployment-orchestration.mdx`](../docs/issues/deployment-orchestration.mdx) · [`docs/security-operations/operations/verification.mdx`](../docs/security-operations/operations/verification.mdx) · [`docs/development/testing.mdx`](../docs/development/testing.mdx) · [`docs/security-operations/security/self-hosted-runners.mdx`](../docs/security-operations/security/self-hosted-runners.mdx) ### `bugbot-review-state-reconciliation` — Bugbot review-state reconciliation diff --git a/specs/catalog.json b/specs/catalog.json index 5aa5def10..7bb958ab6 100644 --- a/specs/catalog.json +++ b/specs/catalog.json @@ -388,7 +388,7 @@ "id": "merge-queue-readiness", "title": "Merge queue readiness and effective target rules", "status": "implemented", - "scope": "Fail-closed validation of required checks and merge-group workflow support; optional manual-only RepoWise analysis with independent CI and verified required-check migration", + "scope": "Fail-closed validation of required checks and merge-group workflow support; optional manual-only RepoWise analysis with independent CI and verified required-check migration; cached RepoWise exports and bounded full-history secret processing with truthful completeness and resource evidence", "owner": "Copilot maintainers", "lastVerified": "2026-10-21", "specs": [ @@ -414,7 +414,9 @@ "src/application/usecases/setup/merge_queue_readiness_use_case.ts", "src/data/repository/deployment/github_target_merge_capabilities_inspector.ts", "src/infrastructure/github/octokit_checks_adapters.ts", - "scripts/validate-workflow-contract.cjs" + "scripts/validate-workflow-contract.cjs", + "scripts/repowise_history.py", + "scripts/repowise_reports.py" ], "tests": [ "src/domain/__tests__/merge_queue_readiness.test.ts", @@ -422,11 +424,14 @@ "src/application/policies/__tests__/setup_doctor_message_catalog.test.ts", "src/application/usecases/setup/__tests__/merge_queue_readiness_use_case.test.ts", "src/data/repository/deployment/__tests__/github_deployment_adapters.test.ts", - "src/tooling/__tests__/validate_workflow_contract.test.ts" + "src/tooling/__tests__/validate_workflow_contract.test.ts", + "scripts/tests/test_repowise.py" ], "documentation": [ "docs/issues/deployment-orchestration.mdx", - "docs/security-operations/operations/verification.mdx" + "docs/security-operations/operations/verification.mdx", + "docs/development/testing.mdx", + "docs/security-operations/security/self-hosted-runners.mdx" ] }, { diff --git a/specs/local-web-setup-assistant.md b/specs/local-web-setup-assistant.md index 72f8a0840..b52c967be 100644 --- a/specs/local-web-setup-assistant.md +++ b/specs/local-web-setup-assistant.md @@ -2451,9 +2451,10 @@ now run only under `workflow_dispatch` on Ubuntu `codex` with the native selecte repository ref; it MUST NOT run on PR, push or merge-group events or be required for them. The retained CI routing cases still prove the fork/merge-group trust boundary. The [manual-only migration contract](./merge-queue-readiness.md#manual-only-repowise-migration-2026-10-21-issue-639) -defines the 24-case regression floor, live required-check evidence, immutable -revision reporting, unchanged scan budgets and cleanup, operator journey, -failure/retry behavior and final-revision manual acceptance. This is a workflow +and its [bounded-analysis follow-up](./merge-queue-readiness.md#bounded-manual-repowise-analysis-2026-10-21-issue-644) +define the 54-case regression floor, live required-check evidence, immutable +revision reporting, measured phase budgets and cleanup, operator journey, +failure/retry behavior and complete final-revision manual acceptance. This is a workflow scheduling change with no setup-session, PAT, web UX or new configuration change. The first [RepoWise run on `e197cb0c`](https://github.com/vypdev/copilot/actions/runs/37289899480) diff --git a/specs/merge-queue-readiness.md b/specs/merge-queue-readiness.md index 5cdc3f774..5e8c2b113 100644 --- a/specs/merge-queue-readiness.md +++ b/specs/merge-queue-readiness.md @@ -193,10 +193,10 @@ already exists on the default branch. CLI example: `gh workflow run repowise.yml --ref develop`. The workflow owns event admission, Ubuntu `codex` runner selection, read-only -contents permission and a 30-minute job budget. Checkout MUST retain the selected -revision and full history without credentials. Each advisory subprocess retains -its 90-second and 2 GiB limits; manual security scanning covers full reachable -history. Per-ref `repowise-` concurrency cancels an older manual run for the same +contents permission and originally a 30-minute job budget. Checkout MUST retain the selected +revision and full history without credentials. The original advisory subprocesses used +90-second and 2 GiB limits; issue #644 supersedes these budgets below. Manual +security scanning covers full reachable history. Per-ref `repowise-` concurrency cancels an older manual run for the same ref without sharing CI's group. Python environments and reports remain isolated by run/attempt, with unconditional upload followed by cleanup. No domain, use case, port, provider DTO or composition change is needed: the repository workflow @@ -239,6 +239,111 @@ Graphify update. Completion requires a real manual run on the final revision and review of its summary/artifact; a queued run or static test is insufficient. Catalog evidence and the runner appendix in the setup-assistant SDD MUST agree. +### Bounded manual RepoWise analysis (2026-10-21, issue #644) + +Status: implemented contract; native final-revision evidence is required before +completion. Owner: Copilot maintainers. No open design decisions. This section +supersedes the original budgets, incomplete-status handling and regression count +above; admission, permissions, checkout, isolation and retention stay in force. + +The [previous manual run](https://github.com/vypdev/copilot/actions/runs/39573977241) +spent 20m37s in the job: indexing took 8m6s, health ran twice, dead-code exceeded +90 seconds and full-history security raised `MemoryError` under 2 GiB. Historical +JavaScript blobs alone total approximately 9.2 GB. Upstream +[RepoWise 0.58.0](https://github.com/repowise-dev/repowise/releases/tag/v0.58.0) +serves stored health/dead-code, but its history reader still materializes all +blobs. Upgrading alone does not resolve the memory failure. + +```text +Manual selected ref -> one fresh standard index -> cached health MD/JSON + safe dead-code + -> full-history secret scan, one blob request at a time + -> validated reports + scope/timings -> artifact + summary +``` + +Text equivalent: index once, export stored results, scan full history with bounded +blob residency, validate completeness, and publish evidence. No application +runtime, setup session, domain model, provider port or public configuration changes +are needed. The workflow owns execution and cleanup; `repowise_reports.py` owns +phase sequencing, resource limits, validation and summary presentation; +`repowise_history.py` adapts pinned RepoWise/Git interfaces. Neither imports the +TypeScript application. YAML mutation tests enforce wiring; Python fixtures enforce +the private-package adapter and output semantics. + +Fixed, non-configurable execution budgets: + +| Phase | Wall limit | Virtual memory per process | Behavior | +|---|---:|---:|---| +| Fresh standard index | 900s | 4096 MiB | No prose/editor/key/hook setup or worktree seeding; 500 commits for git analytics | +| Each cached health format | 60s | 2048 MiB | All indexed origins, refactoring targets, no recomputation | +| Cached safe-only dead-code | 60s | 2048 MiB | Requires successful dead-code phase and matching index SHA | +| Full-history security | 1200s | 2048 MiB | All reachable refs/commits, supported source extensions, RepoWise secret kinds and upstream allowlists | +| Complete job | 40 minutes | Phase limits above | Leaves two minutes beyond summed phase ceilings for setup/publication | + +The 500-commit analytics setting preserves the former index default and MUST NOT +limit history security. Bundles remain included. The history adapter MUST retain +upstream provenance, secret kinds, deduplication, line suppression, path allowlists +and multiline PEM detection. It MUST fail on Git enumeration/read/persistence +errors. One request/response is in flight and the reader retains no blob cache; +memory scales with metadata plus current blob processing, not total historical +content. Existing upstream metadata tables are still materialized. A single giant +blob can exceed the process cap; that is incomplete, never silently skipped. + +The adapter omits call/code-smell analysis that upstream history mode discards, +and uses a conservative literal prefilter before the unchanged secret detectors. +Fixtures MUST compare every supported secret kind, Unicode keyword matching, +multiline PEM and removed/other-ref secrets against upstream behavior. The +version is pinned exactly; any upgrade requires compatibility fixtures before +publication. No remote cache or stale index is accepted. JSON health identifies +`source: index`; index events MUST confirm ready status plus completed health and +dead-code analysis at HEAD. Security MUST match the reachable commit count and +requested/read source-blob count. Malformed JSON is removed instead of published +as a complete report. + +The artifact MUST contain `revision.txt`, `scope.json` (SHA, ref snapshot, tool +version and explicit scopes), `timings.json`, `history-resources.json`, per-phase +logs/resource records, `summary.md`, advisory status and all completed reports. +RSS is GNU time's largest-process peak, not a sum of simultaneous processes; +virtual-memory limits apply to each process. Security output is RepoWise's JSON +aggregate, not a guarantee that no secrets exist. The local database is removed +and findings/secret material are not added to the artifact. + +The English summary shows status first by phase, SHA, seconds and peak RSS with +artifact/retry instructions. Example rows: `dead-code | complete | 2.1 | 96.0` and +`security | incomplete | 1200 | —`. Completed analysis says "Analysis complete. +Findings are advisory"; partial analysis says "Analysis incomplete. Inspect phase +logs in the artifact, fix the cause, then rerun." Incomplete scans now fail this +optional job while preserving completed reports. Index failure blocks cached +exports but still attempts the independent history scan. Timeout kills the process +group. Cancellation/runner loss can still prevent publication; retry manually. +No comments, locale settings or automatic retries are introduced. Native GitHub +queued/running/canceled states and artifact links remain the navigation; textual +statuses and plain tables work on narrow screens and in light/dark mode. + +Regression floor: **54 distinct cases**, comprising the retained **21** manual +workflow admission/permission/isolation/wiring/report mutations and **33** Python +cases: 14 detector/version compatibility, 5 blob/Git protocol/error cases, +1 deleted-secret/other-ref/provenance integration and 13 resource/report/failure +cases. Cases use synthetic credentials, temporary repositories and mocked clocks/ +processes for timeouts; they do not need live services or real waits. The Python +suite runs in the pinned Python 3.12 manual environment before analysis. Repository +Jest coverage thresholds remain in force; the standalone Python adapter uses +behavioral compatibility/failure evidence rather than a new percentage threshold. + +| Requirement | Acceptance/evidence | Documentation | +|---|---|---| +| Cached results belong to completed current index | Progress/state/JSON guards, fixtures and final-head artifact | Testing guide | +| Full history completes without bulk contents | All-kind differential fixtures, deleted/other-ref provenance, native counts/RSS/bytes/timing | Testing + runner guides | +| Failures remain truthful and bounded | Protocol, malformed data, exit, timeout/group kill, partial summary fixtures | Retry instructions | +| Optional workflow remains isolated | Parsed YAML regressions, CI and manual final-head run | Manual dispatch guide | + +Roll out only through the linked PR. Rollback restores the former pin/workflow +but also restores known incomplete scans; it must not restore automatic triggers +or required checks. Definition of Done: the numeric test budget passes; workflow, +documentation and specification validators pass; catalog evidence and Graphify +are current; native final-head analysis produces **complete dead-code and security** +with measured time/memory and inspected summary/artifact; CI passes independently. +A queued run, green partial job, or merely relaxed limits is insufficient. + ### Other merge-queue evidence from the original implementation - Copilot's own required PR context is produced for merge groups by the dedicated `copilot_pull_request_merge_queue.yml` workflow. Keeping that diff --git a/src/tooling/__tests__/validate_workflow_contract.test.ts b/src/tooling/__tests__/validate_workflow_contract.test.ts index 2a6329c37..2da3fc728 100644 --- a/src/tooling/__tests__/validate_workflow_contract.test.ts +++ b/src/tooling/__tests__/validate_workflow_contract.test.ts @@ -1424,10 +1424,12 @@ describe('workflow contract validator', () => { const steps = workflow.jobs['code-health'].steps; const reports = steps.find((step: { name?: string }) => step.name === 'Generate RepoWise reports'); const upload = steps.find((step: { name?: string }) => step.name === 'Upload RepoWise reports'); - expect(reports.run).toContain('health --no-workspace --refactoring-targets --format md'); - expect(reports.run).toContain('health --no-workspace --refactoring-targets --format json'); - expect(reports.run).toContain('ulimit -v 2097152; timeout --kill-after=10s 90s'); - expect(reports.run).toContain('incomplete (exit %s)'); + expect(reports.run).toContain('"$python_bin" scripts/repowise_reports.py --report-dir "$report_dir"'); + expect(reports.run).toContain('"$python_bin" -m unittest discover'); + const adapter = readFileSync(path.join(process.cwd(), 'scripts', 'repowise_reports.py'), 'utf8'); + expect(adapter).toContain('resource.RLIMIT_AS'); + expect(adapter).toContain('os.killpg(process.pid, signal.SIGKILL)'); + expect(adapter).toContain('write_summary(report_dir, sha, results)'); expect(upload.if).toBe('${{ always() }}'); }); @@ -1476,22 +1478,21 @@ describe('workflow contract validator', () => { expect(() => assertRunner(file, workflow)).toThrow('must use runs-on self-hosted, codex, Linux'); }); - it.each(['full-history', 'scan-budget', 'incomplete-status', 'revision-file', 'summary', 'upload', 'retention', 'health-md', 'health-json'])( + it.each(['pin', 'script', 'compatibility', 'cleanup', 'upload', 'retention'])( 'rejects loss of RepoWise manual report evidence: %s', boundary => { const { file, workflow } = loadMutationWorkflow('.github/workflows', 'repowise.yml'); expect(() => validateWorkflow(file, workflow)).not.toThrow(); const job = workflow.jobs['code-health']; const reports = job.steps.find((step: { name?: string }) => step.name === 'Generate RepoWise reports'); + const install = job.steps.find((step: { name?: string }) => step.name === 'Install RepoWise'); const upload = job.steps.find((step: { name?: string }) => step.name === 'Upload RepoWise reports'); - if (boundary === 'full-history') reports.run = reports.run.replace('--history --path .', '--history --since HEAD --path .'); - else if (boundary === 'scan-budget') reports.run = reports.run.replace('ulimit -v 2097152;', ''); - else if (boundary === 'incomplete-status') reports.run = reports.run.replace('incomplete (exit %s)', 'complete'); - else if (boundary === 'revision-file') reports.run = reports.run.replace('> "$report_dir/revision.txt"', '> /dev/null'); - else if (boundary === 'summary') reports.run = reports.run.replace('>> "$GITHUB_STEP_SUMMARY"', '> /dev/null'); + if (boundary === 'pin') install.run = install.run.replace('repowise==0.58.0', 'repowise'); + else if (boundary === 'script') reports.run = reports.run.replace('scripts/repowise_reports.py', 'other.py'); + else if (boundary === 'compatibility') reports.run = reports.run.replace('-m unittest discover', '-m other'); + else if (boundary === 'cleanup') reports.run = reports.run.replace('trap cleanup EXIT', ''); else if (boundary === 'upload') delete upload.if; - else if (boundary === 'retention') upload.with['retention-days'] = 1; - else reports.run = reports.run.replace(`health --no-workspace --refactoring-targets --format ${boundary === 'health-md' ? 'md' : 'json'}`, 'status'); - expect(() => validateWorkflow(file, workflow)).toThrow('bounded manual history scans, health reports, revision evidence'); + else upload.with['retention-days'] = 1; + expect(() => validateWorkflow(file, workflow)).toThrow('pinned bounded reports, compatibility tests'); }, ); From 3031f2d23fc41f824110dfbe626b931e59119cbf Mon Sep 17 00:00:00 2001 From: Efra Espada Date: Wed, 21 Oct 2026 12:00:01 +0200 Subject: [PATCH 2/5] test: isolate RepoWise failure fixtures from live summaries --- scripts/tests/test_repowise.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/tests/test_repowise.py b/scripts/tests/test_repowise.py index 926c631ca..7da0391a5 100644 --- a/scripts/tests/test_repowise.py +++ b/scripts/tests/test_repowise.py @@ -172,6 +172,16 @@ async def scan(bounded): class ReportFailures(unittest.TestCase): + def setUp(self): + # Failure fixtures must not publish fake warnings or a synthetic partial + # Job Summary when this suite runs inside the real manual workflow. + output = patch("sys.stdout", new_callable=io.StringIO) + output.start() + self.addCleanup(output.stop) + environment = patch.dict(reports.os.environ, {"GITHUB_STEP_SUMMARY": ""}) + environment.start() + self.addCleanup(environment.stop) + def test_swallowed_persistence_failure_becomes_fatal(self): from repowise.core.analysis.security.store import SecurityScanner, logger from unittest.mock import AsyncMock From de6ae7bc04eba24ea2adaf51261bcef1e66be229 Mon Sep 17 00:00:00 2001 From: Efra Espada Date: Wed, 21 Oct 2026 12:01:32 +0200 Subject: [PATCH 3/5] test: enforce current cached RepoWise report provenance --- scripts/repowise_reports.py | 6 ++++++ scripts/tests/test_repowise.py | 17 +++++++++++++++++ specs/local-web-setup-assistant.md | 2 +- specs/merge-queue-readiness.md | 6 +++--- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/scripts/repowise_reports.py b/scripts/repowise_reports.py index 55d50a797..a0dcd53a2 100644 --- a/scripts/repowise_reports.py +++ b/scripts/repowise_reports.py @@ -40,6 +40,8 @@ def validate_json(path, kind): if kind == "dead-code": if not isinstance(value, list) or not all(isinstance(f, dict) and "kind" in f for f in value): raise ValueError("Invalid dead-code report") + if "Read from the index" not in (path.parent / "dead-code.stderr.log").read_text(): + raise ValueError("Dead-code did not reuse the index") return value if not isinstance(value, dict): raise ValueError("Expected a JSON object") @@ -49,6 +51,10 @@ def validate_json(path, kind): raise ValueError("Incomplete report schema") if kind == "health-json" and value.get("source") != "index": raise ValueError("Health report did not reuse the index") + if kind == "health-json": + head = subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip() + if value.get("analyzed_commit") != head: + raise ValueError("Stored health revision does not match HEAD") if kind == "security" and (value["commits_scanned"] <= 0 or value["files_scanned"] <= 0): raise ValueError("History scan did not cover any commits or source blobs") if kind == "security": diff --git a/scripts/tests/test_repowise.py b/scripts/tests/test_repowise.py index 7da0391a5..d0c50a9e0 100644 --- a/scripts/tests/test_repowise.py +++ b/scripts/tests/test_repowise.py @@ -207,6 +207,8 @@ def test_success_measures_rss_and_publishes_valid_json(self): def start(command, **kwargs): kwargs["stdout"].write("[]") kwargs["stdout"].flush() + kwargs["stderr"].write("Read from the index") + kwargs["stderr"].flush() (root / "dead-code.resources.txt").write_text("1.2 2048\n") process = Mock(); process.wait.return_value = 0 return process @@ -216,6 +218,21 @@ def start(command, **kwargs): self.assertEqual(2048, result["peak_rss_kib"]) self.assertTrue(p.exists()) + def test_dead_code_live_fallback_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); p = root / "dead-code.json" + p.write_text('[]'); (root / "dead-code.stderr.log").write_text("Computing live") + with self.assertRaisesRegex(ValueError, "reuse the index"): + reports.validate_json(p, "dead-code") + + def test_stale_health_revision_rejected(self): + with tempfile.TemporaryDirectory() as directory: + p = Path(directory) / "health.json" + p.write_text('{"targets":[],"refactoring_plans":[],"source":"index","analyzed_commit":"old"}') + with patch.object(reports.subprocess, "check_output", return_value="current"): + with self.assertRaisesRegex(ValueError, "revision"): + reports.validate_json(p, "health-json") + def test_command_failure_is_incomplete(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory); process = Mock(); process.wait.return_value = 1 diff --git a/specs/local-web-setup-assistant.md b/specs/local-web-setup-assistant.md index b52c967be..20222a865 100644 --- a/specs/local-web-setup-assistant.md +++ b/specs/local-web-setup-assistant.md @@ -2452,7 +2452,7 @@ repository ref; it MUST NOT run on PR, push or merge-group events or be required for them. The retained CI routing cases still prove the fork/merge-group trust boundary. The [manual-only migration contract](./merge-queue-readiness.md#manual-only-repowise-migration-2026-10-21-issue-639) and its [bounded-analysis follow-up](./merge-queue-readiness.md#bounded-manual-repowise-analysis-2026-10-21-issue-644) -define the 54-case regression floor, live required-check evidence, immutable +define the 56-case regression floor, live required-check evidence, immutable revision reporting, measured phase budgets and cleanup, operator journey, failure/retry behavior and complete final-revision manual acceptance. This is a workflow scheduling change with no setup-session, PAT, web UX or new configuration change. diff --git a/specs/merge-queue-readiness.md b/specs/merge-queue-readiness.md index 5e8c2b113..8a3c491d1 100644 --- a/specs/merge-queue-readiness.md +++ b/specs/merge-queue-readiness.md @@ -319,10 +319,10 @@ No comments, locale settings or automatic retries are introduced. Native GitHub queued/running/canceled states and artifact links remain the navigation; textual statuses and plain tables work on narrow screens and in light/dark mode. -Regression floor: **54 distinct cases**, comprising the retained **21** manual -workflow admission/permission/isolation/wiring/report mutations and **33** Python +Regression floor: **56 distinct cases**, comprising the retained **21** manual +workflow admission/permission/isolation/wiring/report mutations and **35** Python cases: 14 detector/version compatibility, 5 blob/Git protocol/error cases, -1 deleted-secret/other-ref/provenance integration and 13 resource/report/failure +1 deleted-secret/other-ref/provenance integration and 15 resource/report/failure cases. Cases use synthetic credentials, temporary repositories and mocked clocks/ processes for timeouts; they do not need live services or real waits. The Python suite runs in the pinned Python 3.12 manual environment before analysis. Repository From defc7b98b0255f14a73ab352fcba7be011fab840 Mon Sep 17 00:00:00 2001 From: Efra Espada Date: Wed, 21 Oct 2026 12:15:58 +0200 Subject: [PATCH 4/5] perf: bound parallel RepoWise history scanning to two workers --- docs/development/testing.mdx | 8 ++- scripts/repowise_history.py | 110 ++++++++++++++++++++++++++--- scripts/repowise_reports.py | 2 +- scripts/tests/test_repowise.py | 51 +++++++++++-- specs/local-web-setup-assistant.md | 2 +- specs/merge-queue-readiness.md | 22 ++++-- 6 files changed, 167 insertions(+), 28 deletions(-) diff --git a/docs/development/testing.mdx b/docs/development/testing.mdx index 9c952976e..8e202eb47 100644 --- a/docs/development/testing.mdx +++ b/docs/development/testing.mdx @@ -120,15 +120,17 @@ The pinned RepoWise 0.58.0 builds one fresh standard index and exports its store health/dead-code results. Index git analytics retain a 500-commit limit; security separately covers **all reachable history**, including removed secrets and generated bundles, using RepoWise's supported source extensions, secret patterns and -allowlists. Historical blob contents are read one at a time. The adapter avoids +allowlists. Two spawned workers read one blob at a time each, with at most two tasks/results +in flight and results consumed in upstream order. Worker failures propagate. The adapter avoids computing code-smell findings discarded by history mode. This remains an advisory signal scanner; an aggregate with no findings is not proof that no secrets exist. Indexing has a 900-second/4 GiB virtual-memory cap. Each health export and dead-code -has a 60-second/2 GiB cap; security has a 1200-second/2 GiB cap, inside the +has a 60-second/2 GiB cap; security has a 1200-second/2 GiB coordinator cap and a 1 GiB cap per worker, inside the 40-minute job. Limits apply per process; reported peak RSS is the largest process, not aggregate simultaneous memory. These budgets accommodate the roughly 9 GB of -historical JavaScript without retaining all of it in memory. +historical JavaScript without retaining all of it in memory. Two available CPU cores are recommended +for the bounded parallel history scan. An interrupted scan, Git error, stale/failed index or invalid report marks analysis incomplete and fails this optional run. Completed reports and diagnostics still diff --git a/scripts/repowise_history.py b/scripts/repowise_history.py index f05bd85c0..9fb0007af 100644 --- a/scripts/repowise_history.py +++ b/scripts/repowise_history.py @@ -8,11 +8,15 @@ from __future__ import annotations import atexit +from collections import deque +from concurrent.futures import ProcessPoolExecutor from importlib.metadata import version +import multiprocessing from pathlib import Path import subprocess REPOWISE_VERSION = "0.58.0" +HISTORY_WORKERS = 2 class BlobReader: @@ -92,6 +96,97 @@ def configure_secret_scan(): scanner._call_findings = lambda *args: [] +def initialize_worker(repo): + import resource + import sys + from multiprocessing.util import Finalize + + # Two workers have a deliberately smaller cap than the coordinating CLI. + if sys.platform == "linux": + resource.setrlimit(resource.RLIMIT_AS, (1024 * 1024**2,) * 2) + configure_secret_scan() + global worker_reader + worker_reader = BlobReader(Path(repo)) + Finalize(None, worker_reader.close, exitpriority=10) + + +def scan_blob(sha, path): + from repowise.core.analysis.security.scanner import scan_source + + before = worker_reader.bytes_read + content = worker_reader.get(sha) + findings = scan_source(path, content) + # Upstream only needs source again for findings' fingerprints/material. + # Negative results carry no large source string across the process pipe. + return findings, content if findings else "", worker_reader.bytes_read - before + + +class PrefetchedBlobs: + """At most two tasks/results resident; consume in upstream history order.""" + + def __init__(self, repo, shas, paths): + self.pool = ProcessPoolExecutor( + max_workers=HISTORY_WORKERS, mp_context=multiprocessing.get_context("spawn"), + initializer=initialize_worker, initargs=(str(repo),), + ) + self.remaining = iter(shas) + self.paths = paths + self.pending = deque() + self.findings = [] + self.count = self.bytes_read = self.largest_blob = 0 + for _ in range(HISTORY_WORKERS): + self.enqueue() + + def enqueue(self): + sha = next(self.remaining, None) + if sha: + self.pending.append((sha, self.pool.submit(scan_blob, sha, self.paths[sha]))) + + def get(self, sha, default=""): + expected, future = self.pending.popleft() + if expected != sha: + raise RuntimeError("History traversal order changed; adapter compatibility failure") + self.findings, content, size = future.result() + self.count += 1 + self.bytes_read += size + self.largest_blob = max(self.largest_blob, size) + self.enqueue() + if self.count % 1000 == 0: + import sys + print(f"History: {self.count} source blobs, {self.bytes_read} bytes scanned", file=sys.stderr, flush=True) + return content + + def close(self): + self.pool.shutdown(wait=True, cancel_futures=True) + + +class PrefetchedScanner: + def __init__(self, reader, delegate): + self.reader = reader + self.delegate = delegate + + async def scan_file(self, path, content, symbols): + return self.reader.findings + + async def persist(self, *args, **kwargs): + return await self.delegate.persist(*args, **kwargs) + + +def bounded_scanner_class(original, readers): + class BoundedHistoryScanner(original): + def _unique_blobs(self, *args): + self.blob_paths = super()._unique_blobs(*args) + return self.blob_paths + + def _read_blobs_batch(self, repo_path, blob_shas): + reader = PrefetchedBlobs(repo_path, blob_shas, self.blob_paths) + readers.append(reader) + self._scanner = PrefetchedScanner(reader, self._scanner) + return reader + + return BoundedHistoryScanner + + def strict_git(repo_path, args, *, timeout=30.0): # Upstream's best-effort helper returns an empty string on Git failures, # which could otherwise turn a failed history enumeration into a clean scan. @@ -128,18 +223,9 @@ def main(): from repowise.cli.main import cli readers = [] - original = history_scan.HistorySecurityScanner - - class BoundedHistoryScanner(original): - def _read_blobs_batch(self, repo_path, blob_shas): - reader = BlobReader(repo_path) - readers.append(reader) - return reader - fail_on_persistence_error() history_scan._run_git = strict_git - history_scan.HistorySecurityScanner = BoundedHistoryScanner - configure_secret_scan() + history_scan.HistorySecurityScanner = bounded_scanner_class(history_scan.HistorySecurityScanner, readers) try: cli(["security", "scan", "--history", "--path", ".", "--format", "json"]) finally: @@ -152,7 +238,9 @@ def _read_blobs_batch(self, repo_path, blob_shas): "bytes_read": sum(r.bytes_read for r in readers), "largest_blob_bytes": max((r.largest_blob for r in readers), default=0), "reader_cache_blobs": 0, - "max_blob_requests_in_flight": 1, + "max_blob_requests_in_flight": HISTORY_WORKERS, + "workers": HISTORY_WORKERS, + "worker_virtual_memory_mib": 1024, }) + "\n") diff --git a/scripts/repowise_reports.py b/scripts/repowise_reports.py index a0dcd53a2..e3cb0c8e8 100644 --- a/scripts/repowise_reports.py +++ b/scripts/repowise_reports.py @@ -144,7 +144,7 @@ def main(): "revision": sha, "refs": refs, "repowise_version": REPOWISE_VERSION, "security": "full reachable history (--all), RepoWise source extensions and secret patterns; upstream allowlists apply", "health": "all indexed origins, cached refactoring targets", "dead_code": "cached safe-only findings", - "index_commit_limit": 500, "history_commit_limit": None, + "index_commit_limit": 500, "history_commit_limit": None, "logical_cpus": os.cpu_count(), }, indent=2) + "\n") os.environ.update(OPENBLAS_NUM_THREADS="1", OMP_NUM_THREADS="1", REPOWISE_SKIP_EDITOR_SETUP="1", REPOWISE_NO_SAVE_KEY="1", REPOWISE_HISTORY_METRICS=str(report_dir / "history-resources.json")) diff --git a/scripts/tests/test_repowise.py b/scripts/tests/test_repowise.py index d0c50a9e0..a41a400ff 100644 --- a/scripts/tests/test_repowise.py +++ b/scripts/tests/test_repowise.py @@ -149,18 +149,17 @@ async def persist(self, path, findings, **provenance): return len(findings) async def scan(bounded): - instance = history_scan.HistorySecurityScanner(None, "fixture") + readers = [] + cls = history.bounded_scanner_class(history_scan.HistorySecurityScanner, readers) if bounded else history_scan.HistorySecurityScanner + instance = cls(None, "fixture") sink = Sink() instance._scanner = sink - reader = history.BlobReader(root) if bounded else None - if bounded: - instance._read_blobs_batch = lambda *_: reader try: with patch.object(history_scan, "_run_git", history.strict_git): summary = await instance.scan_history(root) return summary, sink.saved finally: - if reader: reader.close() + for reader in readers: reader.close() expected = asyncio.run(scan(False)) with optimized(): actual = asyncio.run(scan(True)) @@ -171,6 +170,48 @@ async def scan(bounded): self.assertEqual(introduction, deleted[2]["commit_sha"]) +class BoundedPrefetch(unittest.TestCase): + def reader(self, count=5): + pool = Mock() + def submit(function, sha, path): + future = Mock() + future.result.return_value = ([], "", 3) + return future + pool.submit.side_effect = submit + with patch.object(history, "ProcessPoolExecutor", return_value=pool): + reader = history.PrefetchedBlobs(Path("."), [str(i) for i in range(count)], {str(i): "a.ts" for i in range(count)}) + return reader, pool + + def test_queue_has_fixed_width(self): + reader, pool = self.reader() + self.assertEqual(2, pool.submit.call_count) + for i in range(5): + reader.get(str(i)) + self.assertLessEqual(len(reader.pending), 2) + self.assertEqual((5, 15), (reader.count, reader.bytes_read)) + reader.close() + pool.shutdown.assert_called_once_with(wait=True, cancel_futures=True) + + def test_traversal_order_mismatch_fails(self): + reader, _ = self.reader() + with self.assertRaises(RuntimeError): reader.get("other") + + def test_worker_failure_propagates(self): + reader, _ = self.reader() + reader.pending[0][1].result.side_effect = RuntimeError("worker failed") + with self.assertRaisesRegex(RuntimeError, "worker failed"): reader.get("0") + self.assertEqual(0, reader.count) + + def test_prefetched_results_preserve_delegate_persistence(self): + from unittest.mock import AsyncMock + reader = Mock(findings=[{"kind": "hardcoded_secret"}]) + delegate = Mock(); delegate.persist = AsyncMock(return_value=1) + instance = history.PrefetchedScanner(reader, delegate) + self.assertEqual(reader.findings, asyncio.run(instance.scan_file("a.ts", "", []))) + self.assertEqual(1, asyncio.run(instance.persist("a.ts", reader.findings, commit_sha="abc"))) + delegate.persist.assert_awaited_once_with("a.ts", reader.findings, commit_sha="abc") + + class ReportFailures(unittest.TestCase): def setUp(self): # Failure fixtures must not publish fake warnings or a synthetic partial diff --git a/specs/local-web-setup-assistant.md b/specs/local-web-setup-assistant.md index 20222a865..4b04c5fa2 100644 --- a/specs/local-web-setup-assistant.md +++ b/specs/local-web-setup-assistant.md @@ -2452,7 +2452,7 @@ repository ref; it MUST NOT run on PR, push or merge-group events or be required for them. The retained CI routing cases still prove the fork/merge-group trust boundary. The [manual-only migration contract](./merge-queue-readiness.md#manual-only-repowise-migration-2026-10-21-issue-639) and its [bounded-analysis follow-up](./merge-queue-readiness.md#bounded-manual-repowise-analysis-2026-10-21-issue-644) -define the 56-case regression floor, live required-check evidence, immutable +define the 60-case regression floor, live required-check evidence, immutable revision reporting, measured phase budgets and cleanup, operator journey, failure/retry behavior and complete final-revision manual acceptance. This is a workflow scheduling change with no setup-session, PAT, web UX or new configuration change. diff --git a/specs/merge-queue-readiness.md b/specs/merge-queue-readiness.md index 8a3c491d1..7a8a4545b 100644 --- a/specs/merge-queue-readiness.md +++ b/specs/merge-queue-readiness.md @@ -252,11 +252,14 @@ spent 20m37s in the job: indexing took 8m6s, health ran twice, dead-code exceede JavaScript blobs alone total approximately 9.2 GB. Upstream [RepoWise 0.58.0](https://github.com/repowise-dev/repowise/releases/tag/v0.58.0) serves stored health/dead-code, but its history reader still materializes all -blobs. Upgrading alone does not resolve the memory failure. +blobs. Upgrading alone does not resolve the memory failure. A local single-worker +streaming trial covered 22,975 source blobs / 9,515,793,415 bytes in 1232 seconds; +this CPU measurement motivates the fixed two-worker queue rather than merely +extending the history deadline. Native final-head metrics remain the release gate. ```text Manual selected ref -> one fresh standard index -> cached health MD/JSON + safe dead-code - -> full-history secret scan, one blob request at a time + -> full-history secret scan, two bounded analysis workers -> validated reports + scope/timings -> artifact + summary ``` @@ -276,18 +279,23 @@ Fixed, non-configurable execution budgets: | Fresh standard index | 900s | 4096 MiB | No prose/editor/key/hook setup or worktree seeding; 500 commits for git analytics | | Each cached health format | 60s | 2048 MiB | All indexed origins, refactoring targets, no recomputation | | Cached safe-only dead-code | 60s | 2048 MiB | Requires successful dead-code phase and matching index SHA | -| Full-history security | 1200s | 2048 MiB | All reachable refs/commits, supported source extensions, RepoWise secret kinds and upstream allowlists | +| Full-history security | 1200s | 2048 MiB coordinator; 1024 MiB per worker | Two workers, all reachable refs/commits, supported source extensions, RepoWise secret kinds and upstream allowlists | | Complete job | 40 minutes | Phase limits above | Leaves two minutes beyond summed phase ceilings for setup/publication | The 500-commit analytics setting preserves the former index default and MUST NOT limit history security. Bundles remain included. The history adapter MUST retain upstream provenance, secret kinds, deduplication, line suppression, path allowlists and multiline PEM detection. It MUST fail on Git enumeration/read/persistence -errors. One request/response is in flight and the reader retains no blob cache; +errors. At most two tasks/results are in flight, with one request/response per worker. +The reader retains no blob cache; results are consumed in upstream history order. memory scales with metadata plus current blob processing, not total historical content. Existing upstream metadata tables are still materialized. A single giant blob can exceed the process cap; that is incomplete, never silently skipped. +The adapter uses two spawned processes with a fixed-width prefetch queue. +Negative results do not copy source text back to the coordinator; positive results +retain complete text for unchanged fingerprint/material handling. Worker failures +propagate, and timeout kills the entire process group, including workers and Git. The adapter omits call/code-smell analysis that upstream history mode discards, and uses a conservative literal prefilter before the unchanged secret detectors. Fixtures MUST compare every supported secret kind, Unicode keyword matching, @@ -319,10 +327,10 @@ No comments, locale settings or automatic retries are introduced. Native GitHub queued/running/canceled states and artifact links remain the navigation; textual statuses and plain tables work on narrow screens and in light/dark mode. -Regression floor: **56 distinct cases**, comprising the retained **21** manual -workflow admission/permission/isolation/wiring/report mutations and **35** Python +Regression floor: **60 distinct cases**, comprising the retained **21** manual +workflow admission/permission/isolation/wiring/report mutations and **39** Python cases: 14 detector/version compatibility, 5 blob/Git protocol/error cases, -1 deleted-secret/other-ref/provenance integration and 15 resource/report/failure +1 deleted-secret/other-ref/provenance integration 15 resource/report/failure cases and 4 bounded-prefetch/order/worker-failure/delegation cases. Cases use synthetic credentials, temporary repositories and mocked clocks/ processes for timeouts; they do not need live services or real waits. The Python suite runs in the pinned Python 3.12 manual environment before analysis. Repository From 6a6dc1227ab1dea54b930b996fe8b7e25a13c3d2 Mon Sep 17 00:00:00 2001 From: Efra Espada Date: Wed, 21 Oct 2026 12:16:40 +0200 Subject: [PATCH 5/5] docs: clarify bounded RepoWise worker budgets --- docs/development/testing.mdx | 12 +++++++----- specs/merge-queue-readiness.md | 6 +++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/development/testing.mdx b/docs/development/testing.mdx index 8e202eb47..01a16d9f1 100644 --- a/docs/development/testing.mdx +++ b/docs/development/testing.mdx @@ -120,17 +120,19 @@ The pinned RepoWise 0.58.0 builds one fresh standard index and exports its store health/dead-code results. Index git analytics retain a 500-commit limit; security separately covers **all reachable history**, including removed secrets and generated bundles, using RepoWise's supported source extensions, secret patterns and -allowlists. Two spawned workers read one blob at a time each, with at most two tasks/results -in flight and results consumed in upstream order. Worker failures propagate. The adapter avoids +allowlists. Two spawned workers read one blob at a time each, with at most two +tasks/results in flight and results consumed in upstream order. Worker failures +propagate. The adapter avoids computing code-smell findings discarded by history mode. This remains an advisory signal scanner; an aggregate with no findings is not proof that no secrets exist. Indexing has a 900-second/4 GiB virtual-memory cap. Each health export and dead-code -has a 60-second/2 GiB cap; security has a 1200-second/2 GiB coordinator cap and a 1 GiB cap per worker, inside the +has a 60-second/2 GiB cap; security has a 1200-second/2 GiB coordinator cap and +a 1 GiB cap per worker, inside the 40-minute job. Limits apply per process; reported peak RSS is the largest process, not aggregate simultaneous memory. These budgets accommodate the roughly 9 GB of -historical JavaScript without retaining all of it in memory. Two available CPU cores are recommended -for the bounded parallel history scan. +historical JavaScript without retaining all of it in memory. Two available CPU +cores are recommended for the bounded parallel history scan. An interrupted scan, Git error, stale/failed index or invalid report marks analysis incomplete and fails this optional run. Completed reports and diagnostics still diff --git a/specs/merge-queue-readiness.md b/specs/merge-queue-readiness.md index 7a8a4545b..aa52a8ca8 100644 --- a/specs/merge-queue-readiness.md +++ b/specs/merge-queue-readiness.md @@ -288,7 +288,7 @@ upstream provenance, secret kinds, deduplication, line suppression, path allowli and multiline PEM detection. It MUST fail on Git enumeration/read/persistence errors. At most two tasks/results are in flight, with one request/response per worker. The reader retains no blob cache; results are consumed in upstream history order. -memory scales with metadata plus current blob processing, not total historical +Memory scales with metadata plus current blob processing, not total historical content. Existing upstream metadata tables are still materialized. A single giant blob can exceed the process cap; that is incomplete, never silently skipped. @@ -330,8 +330,8 @@ statuses and plain tables work on narrow screens and in light/dark mode. Regression floor: **60 distinct cases**, comprising the retained **21** manual workflow admission/permission/isolation/wiring/report mutations and **39** Python cases: 14 detector/version compatibility, 5 blob/Git protocol/error cases, -1 deleted-secret/other-ref/provenance integration 15 resource/report/failure cases and 4 bounded-prefetch/order/worker-failure/delegation -cases. Cases use synthetic credentials, temporary repositories and mocked clocks/ +1 deleted-secret/other-ref/provenance integration, 15 resource/report/failure cases, +and 4 bounded-prefetch/order/worker-failure/delegation cases. Cases use synthetic credentials, temporary repositories and mocked clocks/ processes for timeouts; they do not need live services or real waits. The Python suite runs in the pinned Python 3.12 manual environment before analysis. Repository Jest coverage thresholds remain in force; the standalone Python adapter uses