chore: complete bounded manual RepoWise analysis

.github/workflows/repowise.yml

@@ -18,7 +18,7 @@ jobs:
   code-health:
     name: RepoWise code health
     runs-on: [self-hosted, codex, Linux]
-    timeout-minutes: 30
+    timeout-minutes: 40
     env:
       REPOWISE_VENV_NAME: repowise-venv-${{ github.run_id }}-${{ github.run_attempt }}-code-health
       REPOWISE_REPORT_NAME: repowise-report-${{ github.run_id }}-${{ github.run_attempt }}-code-health
@@ -52,54 +52,22 @@ jobs:
           "$RUNNER_TEMP/$REPOWISE_VENV_NAME/bin/python" -m pip install \
             --disable-pip-version-check \
             --no-cache-dir \
-            'repowise==0.45.0'
+            'repowise==0.58.0'
       - name: Generate RepoWise reports
         env:
           REPOWISE_SKIP_EDITOR_SETUP: '1'
         run: |
           set -euo pipefail
           report_dir="$RUNNER_TEMP/$REPOWISE_REPORT_NAME"
-          repowise_bin="$RUNNER_TEMP/$REPOWISE_VENV_NAME/bin/repowise"
-          mkdir -p "$report_dir"
-          analyzed_sha="$(git rev-parse HEAD)"
-          printf '%s\n' "$analyzed_sha" > "$report_dir/revision.txt"
+          python_bin="$RUNNER_TEMP/$REPOWISE_VENV_NAME/bin/python"
           cleanup() {
             rm -rf .repowise .mcp.json
             rm -f .vscode/mcp.json .vscode/extensions.json
             rmdir .vscode 2>/dev/null || true
           }
           trap cleanup EXIT
-          "$repowise_bin" init \
-            --no-prose --no-claude-md --no-codex --no-editor-setup \
-            --no-onboarding --no-distill-hook --no-workspace . > "$report_dir/index.log"
-          "$repowise_bin" health --no-workspace --refactoring-targets --format md . | tee "$report_dir/health.md"
-          "$repowise_bin" health --no-workspace --refactoring-targets --format json . > "$report_dir/health.json"
-          advisory_scan() {
-            local name="$1"
-            shift
-            if (ulimit -v 2097152; timeout --kill-after=10s 90s "$@") > "$report_dir/$name.json"; then
-              printf '%s: complete\n' "$name" >> "$report_dir/advisory-status.txt"
-            else
-              local status="$?"
-              rm -f "$report_dir/$name.json"
-              printf '%s: incomplete (exit %s)\n' "$name" "$status" >> "$report_dir/advisory-status.txt"
-              echo "::warning::RepoWise $name advisory scan incomplete (exit $status); completed health reports remain available."
-            fi
-          }
-          advisory_scan dead-code "$repowise_bin" dead-code --safe-only --no-workspace --format json .
-          echo 'security scope: full reachable history (manual dispatch)' >> "$report_dir/advisory-status.txt"
-          advisory_scan security "$repowise_bin" security scan --history --path . --format json
-          {
-            echo '## RepoWise code health'
-            echo
-            echo 'Advisory reports are retained as workflow artifacts for this commit.'
-            echo
-            printf 'Analyzed revision: `%s`\n\n' "$analyzed_sha"
-            sed -n '1,100p' "$report_dir/health.md"
-            echo
-            echo '### Advisory scan status'
-            cat "$report_dir/advisory-status.txt"
-          } >> "$GITHUB_STEP_SUMMARY"
+          "$python_bin" -m unittest discover -s scripts/tests -p 'test_repowise*.py' -v
+          "$python_bin" scripts/repowise_reports.py --report-dir "$report_dir"
       - name: Upload RepoWise reports
         if: ${{ always() }}
         uses: actions/upload-artifact@v6

.gitignore

@@ -147,3 +147,6 @@ opencode.json
 
 # Repository-local scratch output, including sandboxed Jest transform caches
 .tmp/
+
+# Python analysis tooling
+__pycache__/

docs/development/testing.mdx

@@ -111,17 +111,37 @@ reviewed repository branch, and launch it. The CLI equivalent for development is
 `gh workflow run repowise.yml --ref develop`. The workflow already exists on the
 default branch, as required for [manual dispatch](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manually-run-a-workflow).
 
-Open the run's Job Summary for the analyzed commit and scan status. Download
-the `repowise-code-health` artifact for the index log, Markdown/JSON health
-reports, `revision.txt` and `advisory-status.txt`; artifacts are retained for
-14 days. History security scans cover the complete reachable history of that
-revision. Dead-code and security scans each have a 90-second, 2 GiB limit
-inside the 30-minute job budget. An incomplete scan produces a warning and an
-incomplete status while preserving completed health reports. A failed health
-step fails the run; available files are still uploaded. Inspect the failure
-before retrying. A newer manual run for the same ref cancels the older one;
-cancellation or runner loss can prevent artifact publication. No manual run
-blocks the automatic CI path.
+Open the run's Job Summary for the analyzed commit, phase status, elapsed time
+and peak RSS. Download `repowise-code-health` (14 days) for health Markdown/JSON,
+safe-only dead-code JSON, the security JSON aggregate, index/phase logs,
+`revision.txt`, `scope.json`, `timings.json` and `history-resources.json`.
+
+The pinned RepoWise 0.58.0 builds one fresh standard index and exports its stored
+health/dead-code results. Index git analytics retain a 500-commit limit; security
+separately covers **all reachable history**, including removed secrets and generated
+bundles, using RepoWise's supported source extensions, secret patterns and
+allowlists. Two spawned workers read one blob at a time each, with at most two
+tasks/results in flight and results consumed in upstream order. Worker failures
+propagate. The adapter avoids
+computing code-smell findings discarded by history mode. This remains an advisory
+signal scanner; an aggregate with no findings is not proof that no secrets exist.
+
+Indexing has a 900-second/4 GiB virtual-memory cap. Each health export and dead-code
+has a 60-second/2 GiB cap; security has a 1200-second/2 GiB coordinator cap and
+a 1 GiB cap per worker, inside the
+40-minute job. Limits apply per process; reported peak RSS is the largest process,
+not aggregate simultaneous memory. These budgets accommodate the roughly 9 GB of
+historical JavaScript without retaining all of it in memory. Two available CPU
+cores are recommended for the bounded parallel history scan.
+
+An interrupted scan, Git error, stale/failed index or invalid report marks analysis
+incomplete and fails this optional run. Completed reports and diagnostics still
+upload; invalid JSON is removed. Inspect the named phase log, fix the cause, then
+rerun. A newer manual run for the same ref cancels the older one; cancellation or
+runner loss can prevent publication. No manual run blocks automatic CI. Upgrading
+the pin requires the Python compatibility fixtures, which run before every manual
+analysis. They compare secret detection with upstream and verify deleted secrets,
+other refs, bounded reads and failures.
 
 Manual RepoWise runs use self-hosted Ubuntu `codex` runners. `CI Check` uses
 that pool for repository-owned PRs and pushes; fork PRs and all merge groups

docs/security-operations/security/self-hosted-runners.mdx

@@ -46,7 +46,10 @@ Publishing currently requires a hosted runner. Their other jobs continue on
 self-hosted runners. RepoWise obtains a complete Python 3.12 distribution via
 `actions/setup-python`, creates its virtual environment and reports in
 run-specific temporary directories, and removes them after artifact upload.
-Its runner needs a writable Actions tool cache; the workflow does not install
+Its runner needs a writable Actions tool cache and GNU `/usr/bin/time` for phase
+RSS evidence. The isolated Python suite verifies the pinned history adapter before
+analysis; phase time/virtual-memory caps and a 40-minute job bound the run.
+The workflow does not install
 Ubuntu system packages. RepoWise runs only through manual dispatch on
 `[self-hosted, codex, Linux]`, using the reviewed repository ref selected by the
 maintainer. It is optional and must not be required by branch protections or

scripts/repowise_history.py

@@ -0,0 +1,248 @@
+"""Bounded full-history adapter for the explicitly pinned RepoWise version.
+
+Keep RepoWise's traversal, provenance, allowlists, deduplication and detectors.
+Replace only its eager blob dictionary and work discarded by secrets-only mode.
+Private interfaces are deliberate: upgrades require the compatibility fixtures.
+"""
+
+from __future__ import annotations
+
+import atexit
+from collections import deque
+from concurrent.futures import ProcessPoolExecutor
+from importlib.metadata import version
+import multiprocessing
+from pathlib import Path
+import subprocess
+
+REPOWISE_VERSION = "0.58.0"
+HISTORY_WORKERS = 2
+
+
+class BlobReader:
+    """One request/response at a time; never retain previous blob contents."""
+
+    def __init__(self, repo: Path):
+        self.process = subprocess.Popen(
+            ["git", "-C", str(repo), "cat-file", "--batch"],
+            stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+        )
+        self.count = 0
+        self.bytes_read = 0
+        self.largest_blob = 0
+        atexit.register(self.close)
+
+    def get(self, sha: str, default: str = "") -> str:
+        # Object names come from Git, never accept a batch protocol injection.
+        if len(sha) not in (40, 64) or any(c not in "0123456789abcdef" for c in sha):
+            raise ValueError("Invalid blob object name")
+        self.process.stdin.write((sha + "\n").encode("ascii"))
+        self.process.stdin.flush()
+        header = self.process.stdout.readline(256).split()
+        if len(header) != 3 or header[0].decode("ascii") != sha or header[1] != b"blob":
+            raise RuntimeError("Missing or malformed Git blob response")
+        size = int(header[2])
+        if size < 0:
+            raise RuntimeError("Negative Git blob size")
+        content = self.process.stdout.read(size)
+        if len(content) != size or self.process.stdout.read(1) != b"\n":
+            raise RuntimeError("Truncated Git blob response")
+        self.count += 1
+        self.bytes_read += size
+        self.largest_blob = max(self.largest_blob, size)
+        if self.count % 1000 == 0:
+            import sys
+            print(f"History: {self.count} source blobs, {self.bytes_read} bytes read", file=sys.stderr, flush=True)
+        return content.decode("utf-8", errors="replace")
+
+    def close(self):
+        if self.process.poll() is None:
+            self.process.terminate()
+            try:
+                self.process.wait(timeout=2)
+            except subprocess.TimeoutExpired:
+                self.process.kill()
+                self.process.wait()
+        for stream in (self.process.stdin, self.process.stdout):
+            if stream:
+                stream.close()
+
+
+class SecretPrefilter:
+    # Necessary literals in every line-oriented SECRET_KINDS regex in 0.58.0.
+    # False positives are allowed; authoritative matching stays in RepoWise.
+    # PEM is a separate spanning pass and deliberately never gated here.
+    literals = (
+        "password", "apikey", "api_key", "secret", "token", "accesskey", "access_key",
+        "akia", "asia", "gho_", "ghp_", "ghr_", "ghs_", "ghu_", "github_pat_",
+        "xox", "aiza", "sk_", "rk_", "glpat-", "gldt-", "glrt-", "glptt-",
+        "gloas-", "glimt-", "glcbt-", "gr1348941", "azdo", "://",
+    )
+
+    def search(self, text):
+        # Python IGNORECASE also accepts dotless i, long s and dotted I.
+        lowered = text.lower().replace("ı", "i").replace("ſ", "s").replace("\u0307", "")
+        return any(literal in lowered for literal in self.literals)
+
+
+def configure_secret_scan():
+    from repowise.core.analysis.security import scanner
+    from repowise.core.analysis.security.patterns import SECRET_KINDS
+
+    scanner._PATTERNS = [p for p in scanner._PATTERNS if p[1] in SECRET_KINDS]
+    scanner._SPANNING_PATTERNS = [p for p in scanner._SPANNING_PATTERNS if p[1] in SECRET_KINDS]
+    scanner._ANY_PATTERN = SecretPrefilter()
+    # History's secrets_only=True discards all call findings; don't compute them.
+    scanner._call_findings = lambda *args: []
+
+
+def initialize_worker(repo):
+    import resource
+    import sys
+    from multiprocessing.util import Finalize
+
+    # Two workers have a deliberately smaller cap than the coordinating CLI.
+    if sys.platform == "linux":
+        resource.setrlimit(resource.RLIMIT_AS, (1024 * 1024**2,) * 2)
+    configure_secret_scan()
+    global worker_reader
+    worker_reader = BlobReader(Path(repo))
+    Finalize(None, worker_reader.close, exitpriority=10)
+
+
+def scan_blob(sha, path):
+    from repowise.core.analysis.security.scanner import scan_source
+
+    before = worker_reader.bytes_read
+    content = worker_reader.get(sha)
+    findings = scan_source(path, content)
+    # Upstream only needs source again for findings' fingerprints/material.
+    # Negative results carry no large source string across the process pipe.
+    return findings, content if findings else "", worker_reader.bytes_read - before
+
+
+class PrefetchedBlobs:
+    """At most two tasks/results resident; consume in upstream history order."""
+
+    def __init__(self, repo, shas, paths):
+        self.pool = ProcessPoolExecutor(
+            max_workers=HISTORY_WORKERS, mp_context=multiprocessing.get_context("spawn"),
+            initializer=initialize_worker, initargs=(str(repo),),
+        )
+        self.remaining = iter(shas)
+        self.paths = paths
+        self.pending = deque()
+        self.findings = []
+        self.count = self.bytes_read = self.largest_blob = 0
+        for _ in range(HISTORY_WORKERS):
+            self.enqueue()
+
+    def enqueue(self):
+        sha = next(self.remaining, None)
+        if sha:
+            self.pending.append((sha, self.pool.submit(scan_blob, sha, self.paths[sha])))
+
+    def get(self, sha, default=""):
+        expected, future = self.pending.popleft()
+        if expected != sha:
+            raise RuntimeError("History traversal order changed; adapter compatibility failure")
+        self.findings, content, size = future.result()
+        self.count += 1
+        self.bytes_read += size
+        self.largest_blob = max(self.largest_blob, size)
+        self.enqueue()
+        if self.count % 1000 == 0:
+            import sys
+            print(f"History: {self.count} source blobs, {self.bytes_read} bytes scanned", file=sys.stderr, flush=True)
+        return content
+
+    def close(self):
+        self.pool.shutdown(wait=True, cancel_futures=True)
+
+
+class PrefetchedScanner:
+    def __init__(self, reader, delegate):
+        self.reader = reader
+        self.delegate = delegate
+
+    async def scan_file(self, path, content, symbols):
+        return self.reader.findings
+
+    async def persist(self, *args, **kwargs):
+        return await self.delegate.persist(*args, **kwargs)
+
+
+def bounded_scanner_class(original, readers):
+    class BoundedHistoryScanner(original):
+        def _unique_blobs(self, *args):
+            self.blob_paths = super()._unique_blobs(*args)
+            return self.blob_paths
+
+        def _read_blobs_batch(self, repo_path, blob_shas):
+            reader = PrefetchedBlobs(repo_path, blob_shas, self.blob_paths)
+            readers.append(reader)
+            self._scanner = PrefetchedScanner(reader, self._scanner)
+            return reader
+
+    return BoundedHistoryScanner
+
+
+def strict_git(repo_path, args, *, timeout=30.0):
+    # Upstream's best-effort helper returns an empty string on Git failures,
+    # which could otherwise turn a failed history enumeration into a clean scan.
+    return subprocess.run(
+        ["git", "-C", str(repo_path), *args], check=True, timeout=timeout,
+        capture_output=True, text=True, encoding="utf-8", errors="replace",
+    ).stdout
+
+
+def fail_on_persistence_error():
+    import logging
+
+    class PersistenceFailure(logging.Handler):
+        def emit(self, record):
+            if record.getMessage().startswith("security_finding_persist_failed"):
+                raise RuntimeError("History finding persistence failed")
+
+    logger = logging.getLogger("repowise.core.analysis.security_scan")
+    # Override the CLI's ancestor ERROR level: upstream logs swallowed writes
+    # at WARNING. This handler raises before a diagnostic can pollute JSON.
+    logger.setLevel(logging.WARNING)
+    handler = PersistenceFailure()
+    logger.addHandler(handler)
+    return handler
+
+
+def main():
+    import json
+    import os
+
+    if version("repowise") != REPOWISE_VERSION:
+        raise RuntimeError("Unsupported RepoWise version; rerun compatibility fixtures before upgrading")
+    from repowise.core.analysis import history_scan
+    from repowise.cli.main import cli
+
+    readers = []
+    fail_on_persistence_error()
+    history_scan._run_git = strict_git
+    history_scan.HistorySecurityScanner = bounded_scanner_class(history_scan.HistorySecurityScanner, readers)
+    try:
+        cli(["security", "scan", "--history", "--path", ".", "--format", "json"])
+    finally:
+        for reader in readers:
+            reader.close()
+        metrics = os.environ.get("REPOWISE_HISTORY_METRICS")
+        if metrics:
+            Path(metrics).write_text(json.dumps({
+                "blobs_read": sum(r.count for r in readers),
+                "bytes_read": sum(r.bytes_read for r in readers),
+                "largest_blob_bytes": max((r.largest_blob for r in readers), default=0),
+                "reader_cache_blobs": 0,
+                "max_blob_requests_in_flight": HISTORY_WORKERS,
+                "workers": HISTORY_WORKERS,
+                "worker_virtual_memory_mib": 1024,
+            }) + "\n")
+
+
+if __name__ == "__main__":
+    main()