diff --git a/.github/workflows/fix-drift.yml b/.github/workflows/fix-drift.yml index de9456c0..2179a6bd 100644 --- a/.github/workflows/fix-drift.yml +++ b/.github/workflows/fix-drift.yml @@ -1,6 +1,13 @@ name: Fix Drift on: workflow_dispatch: + # C3: the deprecation detector fires on its OWN schedule, independent of the + # drift-failure gate below — a model family vanishing from a live `/models` + # listing does not, by itself, red the Drift Tests workflow (a vanished + # family simply stops appearing in `live`), so it would never auto-fire under + # the workflow_run gate alone. + schedule: + - cron: "10 6 * * *" # Daily 6:10am UTC (offset from test-drift.yml's 6:00am) workflow_run: workflows: ["Drift Tests"] types: [completed] @@ -14,9 +21,23 @@ permissions: contents: read jobs: - fix: + # C3: this job used to run an autonomous coding-agent subprocess to rewrite + # WHATEVER drift the collector found however it saw fit, then gate the + # resulting diff behind a large anti-cheat verdict function before opening a + # PR. Both the agent invocation and the verdict-gating module have been + # DELETED entirely — see `self-healing-drift-plan.md` / + # `drift-auto-adjust-architecture-spec.md`: general (non-model-churn) drift + # is no longer auto-fixed by anything. It is caught by the daily + # `Drift Tests` workflow (which alerts on its own) and left for a human to + # fix normally; the automation boundary that remains is exactly the + # DETERMINISTIC, zero-agent model-family sync below (`drift-sync.ts` / + # `drift-sync-check.ts`), which only ever performs a mechanical, data-only + # edit to `model-registry.ts` or drops a needs-human dedup note file — never + # arbitrary code generation — so there is no adversarial diff left to police. + sync: if: >- github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule' || github.event.workflow_run.conclusion == 'failure' runs-on: ubuntu-latest timeout-minutes: 30 @@ -25,7 +46,6 @@ jobs: pull-requests: write checks: read statuses: read - # issues: write # removed — no longer creating issues on drift failure steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -49,29 +69,37 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile - # Step 0a: Clone ag-ui repo for AG-UI schema drift detection + # Clone ag-ui: drift-sync-check.ts's "clean re-collect" gate (gate 3, see + # drift-sync-check.ts) shells out the FULL drift-report-collector, which + # includes the AG-UI schema drift leg — that leg reads this checkout. - name: Clone ag-ui repo run: git clone --depth 1 https://github.com/ag-ui-protocol/ag-ui.git ../ag-ui - # Step 0b: Configure git identity and create fix branch + # Configure git identity and create a fix branch. drift-sync.ts commits + # its own mechanical edit (or needs-human note file) directly onto this + # branch; this step only prepares the branch to commit onto. - name: Configure git + id: gitcfg env: RUN_ID: ${{ github.run_id }} run: | git config user.name "aimock-drift-bot" git config user.email "drift-bot@copilotkit.ai" + # Record the pre-sync HEAD (the commit the workflow ran on) BEFORE the + # fix branch is created. drift-sync.ts commits its own edit/note onto + # this branch; the needs-human persist step below compares HEAD against + # this base to tell a real new note (a commit to push) from a re-fire + # of an already-persisted note (nothing to push). + echo "base_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" git checkout -B "fix/drift-$(date +%Y-%m-%d)-${RUN_ID}" - # Provision a local Ollama daemon so the OLLAMA_HOST-gated live drift leg - # (src/__tests__/drift/ollama.drift.ts) is detected here too. Ollama needs - # NO API key — the "credential" is just a running daemon. Pull the smallest - # chat+generate-capable model (qwen2:0.5b, ~350MB); OLLAMA_MODEL points the - # leg at it. COST: install + model pull adds ~2-4 min to the fix run. - - name: Provision Ollama daemon (live drift leg) + # Provision a local Ollama daemon so drift-sync-check's clean-re-collect + # gate (which shells out the FULL collector, exercising every live leg) + # behaves identically to test-drift.yml's own daily run rather than + # spuriously quarantining/erroring on the Ollama leg for a missing daemon. + - name: Provision Ollama daemon (drift-sync-check re-collect gate) run: | set -euo pipefail - # Download the installer to disk first, then execute it — avoids piping - # a remote, mutable script straight into a shell (no `curl | sh`). curl -fsSL https://ollama.com/install.sh -o "${RUNNER_TEMP}/ollama-install.sh" sh "${RUNNER_TEMP}/ollama-install.sh" ollama serve > /tmp/ollama-serve.log 2>&1 & @@ -83,316 +111,190 @@ jobs: done ollama pull qwen2:0.5b - # Step 1: Detect drift and produce report. + # Step 1: run the deterministic (zero-LLM) model-family sync. # - # FIX #F3 (round-4) — write the report OUTSIDE the repo checkout, to - # $RUNNER_TEMP. A collector output left in the repo cwd shows up as an - # untracked file in `git status --porcelain`, which the drift-success - # predicate scores as an UNSANCTIONED_CHANGE (fail-closed) and would break - # the happy path. $RUNNER_TEMP is outside the checkout, so neither the - # predicate's changed-file scan nor createPr's staging ever sees it. (The - # .gitignore also covers `drift-report*.json` as belt-and-suspenders.) - - name: Collect drift report - id: detect + # drift-sync.ts fetches each provider's live `/models` listing directly + # (no drift-report.json input needed), diffs it against the frozen + # `model-registry.ts` classification, and: + # - a zero-reference deprecated family -> mechanical, comment-marked + # removal, gated behind drift-sync-check.ts (allowlist + checksum-pin + # re-assert + clean re-collect) BEFORE the edit is kept, then commits. + # - a still-referenced deprecation, or a genuinely new/unclassified + # family -> NEVER auto-edited; drops a family-keyed dedup note file + # under drift-proposals/ and reports needs-human (job fails RED below + # so a human sees it — no PR spam on re-fire, since the note already + # exists on subsequent runs). + # No autonomous coding agent, no subprocess spawn, no free-form code + # generation anywhere in this path (see drift-sync.ts module doc). + - name: Run deterministic drift sync + id: sync env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - # Un-gates the Ollama live drift leg against the daemon provisioned above. + # The drift-sync-check re-collect gate exercises every live leg + # (including Ollama), so it needs the same env as the daily run. OLLAMA_HOST: 127.0.0.1:11434 OLLAMA_MODEL: qwen2:0.5b FAL_KEY: ${{ secrets.FAL_KEY }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }} - PRE_FIX_REPORT: ${{ runner.temp }}/drift-report.json + SYNC_LOG: ${{ runner.temp }}/drift-sync.log run: | set +e - npx tsx scripts/drift-report-collector.ts --out "$PRE_FIX_REPORT" - EXIT_CODE=$? + npx tsx scripts/drift-sync.ts 2>&1 | tee "${SYNC_LOG}" + SYNC_EXIT=${PIPESTATUS[0]} set -e - echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT" - if [ "$EXIT_CODE" -eq 2 ]; then - : # critical drift found, continue - elif [ "$EXIT_CODE" -eq 5 ]; then - : # quarantine: collector encountered unparseable output, continue without aborting - elif [ "$EXIT_CODE" -ne 0 ]; then - # A Step-1 collector crash is a DISTINCT failure cause from an - # auto-fix step failure. Record it as an output BEFORE aborting so - # the dedicated collector-crash alert (not the "auto-fix STEP - # failed" alert) fires and names the real cause. - echo "collector_crashed=true" >> "$GITHUB_OUTPUT" - echo "::error::Drift collector (Step 1) crashed with exit code $EXIT_CODE — this is a collector failure, NOT an auto-fix step failure" - exit $EXIT_CODE - fi - - # Always upload the report as an artifact (from $RUNNER_TEMP — see Step 1). - - name: Upload drift report - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: drift-report - path: ${{ runner.temp }}/drift-report.json - if-no-files-found: warn - retention-days: 30 - - # Step 2: Exit if no critical drift - - name: Check for critical diffs - id: check - env: - DETECT_EXIT_CODE: ${{ steps.detect.outputs.exit_code }} - run: | - set -euo pipefail - if [ "$DETECT_EXIT_CODE" = "2" ]; then - echo "skip=false" >> "$GITHUB_OUTPUT" - echo "quarantine=false" >> "$GITHUB_OUTPUT" - echo "Critical drift detected" - elif [ "$DETECT_EXIT_CODE" = "5" ]; then - # Unparseable / quarantine: NOT a silent skip — a human must look. - echo "skip=true" >> "$GITHUB_OUTPUT" - echo "quarantine=true" >> "$GITHUB_OUTPUT" - echo "::warning::Drift collector returned quarantine (exit 5) — unparseable output, routing to human alert" - else - echo "skip=true" >> "$GITHUB_OUTPUT" - echo "quarantine=false" >> "$GITHUB_OUTPUT" - echo "No critical drift detected (exit code: $DETECT_EXIT_CODE) — skipping fix" - fi - - # Step 2b: PIN the PRE-fix report OUTSIDE the repo before the autofix LLM - # runs (INTEGRITY GUARANTEE — CR round-3 F-A). - # - # The drift-success predicate derives its ALLOWLIST's sanctioned-target set - # (union of every entry's builderFile/typesFile) from the PRE-fix report. The - # autofix step below grants the Claude Code subprocess unrestricted Write/Edit - # scoped to the repo checkout (its cwd), so a repo-committed `drift-report.json` - # is LLM-writable: a malicious/confused fix could overwrite it to name an - # arbitrary fixture as a builderFile and thereby expand the allowlist. - # - # Pinning the report to $RUNNER_TEMP — which is OUTSIDE the repo checkout and - # therefore outside the autofix tool's writable scope — makes the sanctioned - # set immutable across the autofix. Both the "Assert" and "Create PR" steps - # read `--report` from this pinned copy, NEVER the in-repo file the LLM can - # touch. The POST-fix report is separately re-generated after autofix (Step - # 4c) and so needs no pin. Runs AFTER "Check for critical diffs" (so it is - # gated by the same skip guard) and BEFORE "Auto-fix drift" (so the pin is - # captured before the LLM can touch the on-disk report). - - name: Pin pre-fix drift report (integrity) - if: steps.check.outputs.skip != 'true' - env: - PRE_FIX_REPORT: ${{ runner.temp }}/drift-report.json - PINNED_REPORT: ${{ runner.temp }}/drift-report.pinned.json - run: | - set -euo pipefail - cp "$PRE_FIX_REPORT" "$PINNED_REPORT" - echo "Pinned pre-fix drift report to $PINNED_REPORT (outside the LLM-writable repo checkout)" - - # Step 3: Invoke Claude Code to fix - # - # --report reads the PINNED pre-fix report (Step 2b), NOT the default - # in-repo `drift-report.json`. The collector writes its report to - # $RUNNER_TEMP (Step 1, FIX #F3), which is OUTSIDE the repo checkout, so - # the repo-root `drift-report.json` that fix-drift.ts defaults to does not - # exist — without this flag readDriftReport throws "Drift report not - # found" and the fixer never runs. Reading the pinned copy (same source - # the Assert and Create PR steps read) also keeps the fixer's view of the - # drift consistent with the downstream integrity gate. - - name: Auto-fix drift - id: autofix - if: steps.check.outputs.skip != 'true' - continue-on-error: true - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - FAL_KEY: ${{ secrets.FAL_KEY }} - PINNED_REPORT: ${{ runner.temp }}/drift-report.pinned.json - run: npx tsx scripts/fix-drift.ts --report "${PINNED_REPORT}" + REASON="$(grep '^reason=' "${SYNC_LOG}" | tail -n1)" + REASON="${REASON#reason=}" + # Stable, date-independent identity of this run's changeset (the sorted + # set of applied + deferred family outcomes, hashed — see + # computeChangesetKey in drift-sync.ts). BOTH PR-open paths dedup on + # this so a daily re-fire of the same drift never opens a second PR, + # in EVERY run shape — including the mixed run that commits a registry + # edit but NO new note file (where a note-path-only key is empty). + CHANGESET_KEY="$(grep '^changeset-key=' "${SYNC_LOG}" | tail -n1)" + CHANGESET_KEY="${CHANGESET_KEY#changeset-key=}" + { + echo "exit_code=${SYNC_EXIT}" + echo "reason=${REASON}" + echo "changeset_key=${CHANGESET_KEY}" + } >> "$GITHUB_OUTPUT" + echo "drift-sync exited ${SYNC_EXIT} (reason=${REASON}, changeset_key=${CHANGESET_KEY})" - # Upload Claude Code output for debugging - - name: Upload Claude Code logs + - name: Upload drift-sync log if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: claude-code-output - path: claude-code-output.log + name: drift-sync-log + path: ${{ runner.temp }}/drift-sync.log if-no-files-found: warn retention-days: 30 - # Step 4: Verify fix independently — only when autofix actually succeeded. - # `autofix` has `continue-on-error: true`, so without these explicit guards - # `success()` would remain true on autofix failure and we'd verify+ship a - # broken state. - - name: Verify conformance - if: steps.autofix.outcome == 'success' && steps.check.outputs.skip != 'true' - run: pnpm test - - # `pnpm test:drift` is a FAST fail here, but it is NO LONGER the - # authoritative signal — it is the same suite the fixer was told to make - # pass, so a fixture-relaxation cheat sails through it. The authoritative - # signal is the fresh re-collect + drift-success predicate below. - - name: Verify drift resolved - if: steps.autofix.outcome == 'success' && steps.check.outputs.skip != 'true' - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - # Keep the Ollama live leg un-gated so post-fix verification exercises it - # (the daemon provisioned earlier in this job is still running). - OLLAMA_HOST: 127.0.0.1:11434 - OLLAMA_MODEL: qwen2:0.5b - FAL_KEY: ${{ secrets.FAL_KEY }} - COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} - ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }} - run: pnpm test:drift - - # Step 4c: Re-collect drift AUTHORITATIVELY against the LIVE SDK, writing - # to a DISTINCT path so the pre-fix report is preserved for classification. - # This is a fresh collector invocation (same capture pattern as Step 1) — - # its exit code is the authoritative "is the drift gone?" signal that the - # predicate scores. A cheat that relaxed the SDK-shape fixture still - # reports clean HERE (its own SDK leg reads the relaxed fixture), which is - # exactly why the predicate ALSO requires a real production change. - # - # FIX #F3 (round-4) — write the post-fix report to $RUNNER_TEMP, OUTSIDE - # the repo checkout, for the same reason as Step 1: a post-fix report left - # in the repo cwd is an untracked file that the predicate's changed-file - # scan would score as UNSANCTIONED_CHANGE, breaking the happy path. Writing - # it outside the checkout (and out of the autofix LLM's writable scope) - # keeps it out of `git status --porcelain` entirely. - - name: Re-collect drift (authoritative) - id: recollect - if: steps.autofix.outcome == 'success' && steps.check.outputs.skip != 'true' + # Step 2: defense-in-depth re-assertion of drift-sync-check.ts against the + # now-committed state, ONLY when the sync actually applied a mechanical + # edit (reason=ok-applied). drift-sync.ts already gates its OWN edit + # behind this exact check before committing (reverting on failure), so + # this step re-verifies the committed result the same way the old + # workflow re-asserted its (then-agent-authored) diff via a verdict + # function before opening a PR — mirrored here with the trivial, + # deterministic replacement instead. + - name: Assert drift-sync-check (defense-in-depth) + id: assert + if: steps.sync.outputs.reason == 'ok-applied' env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - # Keep the Ollama live leg un-gated so the authoritative re-collect - # exercises it (the daemon provisioned earlier in this job is running). OLLAMA_HOST: 127.0.0.1:11434 OLLAMA_MODEL: qwen2:0.5b FAL_KEY: ${{ secrets.FAL_KEY }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }} - POST_FIX_REPORT: ${{ runner.temp }}/drift-report.post-fix.json + ASSERT_LOG: ${{ runner.temp }}/drift-sync-check.log run: | set +e - npx tsx scripts/drift-report-collector.ts --out "$POST_FIX_REPORT" - POST_FIX_EXIT=$? + npx tsx scripts/drift-sync-check.ts 2>&1 | tee "${ASSERT_LOG}" + ASSERT_EXIT=${PIPESTATUS[0]} set -e - echo "post_fix_exit=$POST_FIX_EXIT" >> "$GITHUB_OUTPUT" - echo "Post-fix collector exited with code $POST_FIX_EXIT" + REASON="$(grep '^reason=' "${ASSERT_LOG}" | tail -n1)" + REASON="${REASON#reason=}" + echo "reason=${REASON}" >> "$GITHUB_OUTPUT" + if [ "$ASSERT_EXIT" -ne 0 ]; then + echo "::error::drift-sync-check refused (exit ${ASSERT_EXIT}, reason=${REASON}) — not opening a PR" + exit "$ASSERT_EXIT" + fi + echo "drift-sync-check: sync verified data-only, pins intact, clean re-collect" - - name: Upload post-fix drift report + - name: Upload drift-sync-check log if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: drift-report-post-fix - path: ${{ runner.temp }}/drift-report.post-fix.json + name: drift-sync-check-log + path: ${{ runner.temp }}/drift-sync-check.log if-no-files-found: warn retention-days: 30 - # Step 4d: Assert the drift was TRULY resolved (not a fixture relaxation). - # The predicate scores three independent signals — post-fix collector - # clean, a real production mock-builder change, and no comparison-leg - # relaxation — and exits non-zero (with a distinct reason) on a cheat. - # A non-zero exit blocks the pipeline: no PR is opened, so nothing merges. - - name: Assert drift truly resolved - id: assert - if: steps.autofix.outcome == 'success' && steps.check.outputs.skip != 'true' - run: | - # Run the predicate to a log file so we capture its REAL exit code - # directly (a `VAR="$(pipeline)"` assignment would report the - # assignment's own status via PIPESTATUS, never the predicate's). - # - # --report reads the PINNED pre-fix report (Step 1b), NOT the in-repo - # drift-report.json the autofix LLM could have overwritten — the - # allowlist's sanctioned-target set is derived from it, so it must be - # the untamperable copy (CR round-3 F-A). - # Write the predicate log OUTSIDE the repo checkout ($RUNNER_TEMP): - # the predicate scans `git status --porcelain` in this same working - # directory, so a predicate.log written into cwd would appear as an - # untracked file and be scored as UNSANCTIONED_CHANGE, breaking the - # happy path (FIX #F3). - set +e - npx tsx scripts/drift-success-predicate.ts \ - --report "${PINNED_REPORT}" \ - --post-fix-report "${POST_FIX_REPORT}" \ - --post-fix-exit "${POST_FIX_EXIT}" 2>&1 | tee "${PREDICATE_LOG}" - PRED_EXIT=${PIPESTATUS[0]} - set -e - REASON="$(grep '^reason=' "${PREDICATE_LOG}" | tail -n1)" - REASON="${REASON#reason=}" - echo "reason=${REASON}" >> "$GITHUB_OUTPUT" - if [ "$PRED_EXIT" -ne 0 ]; then - echo "::error::Drift-success predicate refused (exit ${PRED_EXIT}, reason=${REASON}) — not a real drift fix, blocking PR" - exit "$PRED_EXIT" - fi - echo "Drift-success predicate: drift truly resolved" - env: - POST_FIX_EXIT: ${{ steps.recollect.outputs.post_fix_exit }} - POST_FIX_REPORT: ${{ runner.temp }}/drift-report.post-fix.json - PINNED_REPORT: ${{ runner.temp }}/drift-report.pinned.json - PREDICATE_LOG: ${{ runner.temp }}/predicate.log - - # Inject git credentials only when needed for push (persist-credentials: false above) + # Inject git credentials only when needed for push (persist-credentials: + # false above). BOTH push paths need it: the ok-applied mechanical-edit PR + # AND the needs-human note-persist PR below. - name: Configure git for push - if: steps.autofix.outcome == 'success' && success() && steps.check.outputs.skip != 'true' + if: >- + success() && + (steps.sync.outputs.reason == 'ok-applied' || + steps.sync.outputs.reason == 'needs-human') run: git config --local url."https://x-access-token:${TOKEN}@github.com/".insteadOf "https://github.com/" env: TOKEN: ${{ steps.app-token.outputs.token }} - # Step 5: Create PR on success - # - # Select the PR by HEAD-SHA match (not `gh pr list ... .[0]`) so a stale - # or unrelated open PR on the same head branch cannot be mistaken for the - # one we just pushed. - - name: Create PR + # Step 3: push the branch drift-sync committed onto and open a data-only + # PR. Select the PR by HEAD-SHA match so a stale/unrelated open PR on the + # same head branch is never mistaken for the one just pushed. + - name: Push branch + create PR id: pr - if: steps.autofix.outcome == 'success' && success() && steps.check.outputs.skip != 'true' + if: steps.sync.outputs.reason == 'ok-applied' && success() env: GH_TOKEN: ${{ steps.app-token.outputs.token }} - POST_FIX_EXIT: ${{ steps.recollect.outputs.post_fix_exit }} - POST_FIX_REPORT: ${{ runner.temp }}/drift-report.post-fix.json - PINNED_REPORT: ${{ runner.temp }}/drift-report.pinned.json - CREATE_PR_LOG: ${{ runner.temp }}/create-pr.log + CHANGESET_KEY: ${{ steps.sync.outputs.changeset_key }} run: | set -euo pipefail - # Defense-in-depth: the workflow already asserted the predicate in the - # "Assert drift truly resolved" step, but fix-drift.ts runs it AGAIN - # (via --post-fix-report/--post-fix-exit) as an in-script gate BEFORE - # any git add/commit, so a cheat opens no PR even if the step guard is - # ever bypassed. This opens the PR and STOPS — a human reviews and - # merges (see the "NO AUTO-MERGE" comment below); fix-drift.ts records - # the predicate verdict in the PR body for that reviewer. - # - # --report reads the PINNED pre-fix report (Step 1b), NOT the in-repo - # drift-report.json — same integrity guarantee as the Assert step: the - # in-script predicate's sanctioned-target set must come from the copy - # the autofix LLM could not overwrite (CR round-3 F-A). - # Capture the script's own exit code + emitted `reason=` line so a - # fail-closed exit (e.g. version-bump-failed) is NAMED in the failure - # alert rather than reported blank. Write the log OUTSIDE the repo - # checkout ($RUNNER_TEMP) so it is never scored by any git scan. - set +e - npx tsx scripts/fix-drift.ts --create-pr \ - --report "${PINNED_REPORT}" \ - --post-fix-report "${POST_FIX_REPORT}" \ - --post-fix-exit "${POST_FIX_EXIT}" 2>&1 | tee "${CREATE_PR_LOG}" - PR_EXIT=${PIPESTATUS[0]} - set -e - if [ "$PR_EXIT" -ne 0 ]; then - PR_REASON="$(grep '^reason=' "${CREATE_PR_LOG}" | tail -n1)" - PR_REASON="${PR_REASON#reason=}" - echo "::error::Create-PR step failed (exit ${PR_EXIT}, reason=${PR_REASON:-unknown})" - echo "reason=${PR_REASON}" >> "$GITHUB_OUTPUT" - exit "$PR_EXIT" + # De-dup (idempotent across daily re-fires): the mechanical edit is + # never auto-merged, so an un-merged drift is re-detected every cron + # run and would otherwise open a brand-new PR each day. Skip when an + # open PR already carries this run's stable changeset marker (keyed on + # the date-independent changeset key, NOT the date-stamped branch name + # or diff bytes). An applied edit deserves exactly ONE open PR, + # re-findable across runs. + if [ -n "${CHANGESET_KEY:-}" ]; then + OPEN_PRS="$(gh pr list --state open --json number,body 2>/dev/null || echo '[]')" + DUP="$(printf '%s' "$OPEN_PRS" | jq -r --arg m "drift-changeset: ${CHANGESET_KEY}" \ + 'map(select(.body | contains($m))) | .[0].number // empty')" + if [ -n "$DUP" ]; then + echo "ok-applied: changeset ${CHANGESET_KEY} already proposed in open PR #${DUP} — not opening a duplicate" + exit 0 + fi fi + BRANCH="$(git rev-parse --abbrev-ref HEAD)" + git push -u origin "$BRANCH" HEAD_SHA="$(git rev-parse HEAD)" - echo "Pushed head SHA: $HEAD_SHA" - # Find the open PR whose headRefOid matches the exact SHA we pushed. + echo "Pushed branch ${BRANCH} at ${HEAD_SHA}" + + PR_BODY_FILE="${RUNNER_TEMP}/drift-sync-pr-body.md" + { + echo "## Summary" + echo "" + echo "Auto-generated, DATA-ONLY model-family sync (zero-LLM)." + echo "" + # Stable changeset marker — the dedup guard above matches open PRs on + # this so a daily re-fire of the same drift never opens a second PR. + echo "" + echo "" + echo "> **Needs human review + merge.** This PR was opened by the" + echo "> deterministic drift-sync pipeline after \`drift-sync-check\`" + echo "> passed (changed-file allowlist, checksum-pin re-assert, clean" + echo "> re-collect). It is NOT auto-merged — review CI, the diff, and" + echo "> merge." + echo "" + echo "### drift-sync-check verdict" + echo "" + echo '```' + cat "${RUNNER_TEMP}/drift-sync-check.log" 2>/dev/null || echo "(see drift-sync-log / drift-sync-check-log artifacts)" + echo '```' + echo "" + echo "### drift-sync outcome" + echo "" + echo '```' + cat "${RUNNER_TEMP}/drift-sync.log" 2>/dev/null || echo "(see drift-sync-log artifact)" + echo '```' + } > "$PR_BODY_FILE" + + gh pr create \ + --title "fix(drift-sync): mechanical model-family sync ($(date +%Y-%m-%d))" \ + --assignee jpr5 \ + --body-file "$PR_BODY_FILE" + PR_URL="" PR_NUMBER="" for attempt in $(seq 1 6); do @@ -408,7 +310,6 @@ jobs: done if [ -z "$PR_URL" ]; then echo "::error::No open PR found whose head matches pushed SHA $HEAD_SHA" - echo "reason=no-pr-match" >> "$GITHUB_OUTPUT" exit 1 fi echo "Matched PR #$PR_NUMBER: $PR_URL" @@ -418,187 +319,278 @@ jobs: echo "head_sha=$HEAD_SHA" } >> "$GITHUB_OUTPUT" - # NO AUTO-MERGE FOR THE DRIFT PATH (WS-2, round-4 — user-approved design). - # - # The drift-success predicate is a strong AUTO-FILTER, NOT a provable merge - # gate. Its authoritative "is the drift gone?" signal comes from the in- - # workflow RE-COLLECT (Step 4c), which is NOT independent of the fix: the - # collector's SDK/expected leg reads the very fixtures the autofix touched, - # so a run that relaxed an input leg reports clean here too. The allowlist + - # always-block-on-leg-edit rules make that clean signal trustworthy enough - # to FILTER OUT cheats and never-fixed drift, but they cannot PROVE the mock - # was genuinely fixed (that requires an independent SDK leg — WS-2b, out of - # scope). Given that fundamental residual, the drift path opens a PR and - # STOPS: a HUMAN reviews CI + the committed diff + the predicate verdict - # (recorded in the PR body) and merges. The merge gate is the human plus the - # branch ruleset's one-approval requirement — NOT an unattended in-workflow - # merge command. The predicate having already filtered the PR down to - # genuine-looking fixes means the human review is a confirmation step, not a - # from-scratch triage. + # G#2: PERSIST the needs-human note (the Bucket-B human touchpoint). + # drift-sync.ts has ALREADY committed the note file(s) (and, in a mixed + # run, its already-gated mechanical registry edit) onto the fix branch via + # commitSyncChanges — but that commit lives only in the runner's working + # tree. The ok-applied "Push branch + create PR" step above fires ONLY on + # reason == 'ok-applied', so on a needs-human run nothing was pushed and + # the note was DISCARDED with the runner: the human had no note to review, + # and the two-run "Decision: include -> next run applies" protocol could + # never trigger (the next run finds no persisted note). This step closes + # that gap: it pushes a DISTINCT branch and opens a needs-human PR so the + # note lands in the repo. NO AUTO-MERGE — a human sets `Decision: include` + # (or closes to reject) and merges; the next drift-sync run reads the + # approved note from main and applies the mechanical edit. # - # Accepted residuals under this backstop (documented, not fixed): - # • F6 — working-tree-vs-committed TOCTOU: the predicate scores the working - # tree; the committed diff is a gated subset. Mitigated by human review - # of the COMMITTED diff and CI running on the pushed SHA. - # • WS-2b — no independent SDK leg: the re-collect is not independent of the - # fix. Mitigated by the human confirming the diff is a real mock change. - - # Alert: the Step-1 drift collector crashed (exit code other than - # 0/2/5). This is a DISTINCT cause from an auto-fix step failure — the - # fixer never ran because we could not even produce a drift report — so it - # gets its own message instead of the misleading "auto-fix STEP failed". - - name: Alert on collector crash - if: always() && steps.detect.outputs.collector_crashed == 'true' + # Re-fire discipline (no PR spam): three layers, idempotent in EVERY run + # shape. (1) If this run produced NO new commit (drift-sync.ts's + # ensureProposalNote already deduped the note against main), there is + # nothing to push. (2) PRIMARY: if an open PR already carries this run's + # stable, date-independent CHANGESET KEY marker, skip — this covers the + # MIXED run whose committed diff is a registry edit with NO new note file + # (where the per-note check below is empty and would otherwise fall + # through to an unconditional push + PR every daily cron run). (3) + # SECONDARY: if an open PR already proposes one of this run's note files + # (matched by the per-note body marker), skip. The Slack alert below still + # fires either way. + - name: Persist needs-human note + open PR + id: needs_human_pr + if: steps.sync.outputs.reason == 'needs-human' && success() env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - REPO: ${{ github.repository }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} + BASE_SHA: ${{ steps.gitcfg.outputs.base_sha }} RUN_ID: ${{ github.run_id }} - COLLECTOR_EXIT: ${{ steps.detect.outputs.exit_code }} + CHANGESET_KEY: ${{ steps.sync.outputs.changeset_key }} run: | set -euo pipefail - echo "::error::Drift collector (Step 1) crashed (exit ${COLLECTOR_EXIT}) — no drift report produced, auto-fix never ran" - if [ -z "${SLACK_WEBHOOK:-}" ]; then - echo "::error::SLACK_WEBHOOK not set — cannot send collector-crash alert" - else - MSG="💥❌ *Drift collector crashed* (Step 1, exit ${COLLECTOR_EXIT}) — no drift report was produced and no auto-fix was attempted. This is a collector failure, not an auto-fix failure. Manual intervention needed.\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" - PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" - curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ - -H "Content-Type: application/json" \ - -d "$PAYLOAD" + HEAD_SHA="$(git rev-parse HEAD)" + # Re-fire of an already-persisted note: no new commit -> already reachable. + if [ "$HEAD_SHA" = "$BASE_SHA" ]; then + echo "needs-human: no new commit (note already persisted in the repo) — nothing to push" + exit 0 fi - exit 1 - # Alert: the autofix step itself failed. Because that step is - # `continue-on-error: true`, on failure the job would otherwise sail on - # green with no signal. Detect outcome != 'success' (and not the skip - # case), alert distinctly, then fail the job. - # NOTE: also require the `check` step to have RUN (skip output is one of - # 'true'/'false', never empty). Without that guard this step would ALSO - # fire on an EARLY-infra failure (checkout/token/pnpm/clone/git-config — - # all before `detect`/`check`), mislabelling it "auto-fix STEP failed" - # when the fixer never ran. The end-of-job catch-all owns that window. - - name: Alert on autofix step failure - if: >- - always() && steps.check.outputs.skip == 'false' && - steps.autofix.outcome != 'success' && - steps.detect.outputs.collector_crashed != 'true' - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - REPO: ${{ github.repository }} - RUN_ID: ${{ github.run_id }} - AUTOFIX_OUTCOME: ${{ steps.autofix.outcome }} - run: | - set -euo pipefail - echo "::error::Auto-fix step outcome was '${AUTOFIX_OUTCOME}' (not success) — failing the job" - if [ -z "${SLACK_WEBHOOK:-}" ]; then - echo "::error::SLACK_WEBHOOK not set — cannot send autofix-failure alert" - else - MSG="🛠️❌ *Drift auto-fix STEP failed* (outcome: ${AUTOFIX_OUTCOME}) — the fixer errored before producing a PR. Manual intervention needed.\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" - PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" - curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ - -H "Content-Type: application/json" \ - -d "$PAYLOAD" + OPEN_PRS="$(gh pr list --state open --json number,body 2>/dev/null || echo '[]')" + + # PRIMARY de-dup (idempotent across daily re-fires in EVERY shape, + # INCLUDING the mixed run whose committed diff is a registry edit with + # NO new drift-proposals/* note). Keyed on the stable, date-independent + # changeset key — which is non-empty here even when no note file is in + # the diff, so this guard fires where the per-note loop below cannot. + # Without it, a mixed run re-detects the same un-merged registry edit + # every daily cron run and opens a brand-new near-identical PR each + # time (unbounded PR-spam). Matched by the "drift-changeset: " + # body marker this step writes below. + if [ -n "${CHANGESET_KEY:-}" ]; then + DUP="$(printf '%s' "$OPEN_PRS" | jq -r --arg m "drift-changeset: ${CHANGESET_KEY}" \ + 'map(select(.body | contains($m))) | .[0].number // empty')" + if [ -n "$DUP" ]; then + echo "needs-human: changeset ${CHANGESET_KEY} already proposed in open PR #${DUP} — not opening a duplicate" + exit 0 + fi fi - exit 1 - # Alert: drift collector returned quarantine (exit 5) — unparseable - # output that a human must triage. Routed here instead of the silent - # no-drift skip path. - - name: Alert on drift quarantine - if: always() && steps.check.outputs.quarantine == 'true' + # The note file(s) drift-sync.ts committed this run. + mapfile -t COMMITTED < <(git diff --name-only "$BASE_SHA" HEAD) + NOTES=() + for f in "${COMMITTED[@]}"; do + case "$f" in drift-proposals/*) NOTES+=("$f") ;; esac + done + + # SECONDARY de-dup (retained): never open a SECOND PR for a note a + # still-open PR already proposes (matched by the per-note + # "drift-proposal-note: " body marker this step writes below). + for note in "${NOTES[@]:-}"; do + [ -n "$note" ] || continue + DUP="$(printf '%s' "$OPEN_PRS" | jq -r --arg m "drift-proposal-note: ${note}" \ + 'map(select(.body | contains($m))) | .[0].number // empty')" + if [ -n "$DUP" ]; then + echo "needs-human: note ${note} already proposed in open PR #${DUP} — not opening a duplicate" + exit 0 + fi + done + + # Distinct branch (never collides with the ok-applied fix/drift-* branch). + BRANCH="drift-needs-human/$(date +%Y-%m-%d)-${RUN_ID}" + git checkout -B "$BRANCH" + git push -u origin "$BRANCH" + echo "Pushed needs-human branch ${BRANCH} at ${HEAD_SHA}" + + PR_BODY_FILE="${RUNNER_TEMP}/drift-needs-human-pr-body.md" + { + echo "## Needs a human decision (drift-sync)" + echo "" + echo "The deterministic, zero-LLM drift-sync found a model-family change it must" + echo "NOT auto-apply (a genuinely new/unclassified family, a still-referenced" + echo "deprecation, or a registry structural mismatch). It wrote the note file(s)" + echo "below and opened this PR so the decision is REACHABLE in the repo." + echo "" + echo "> **Not auto-merged — a human decides.** For a *new-family* note: set the" + echo "> note's decision line to \`Decision: include\` (to classify it) or delete the" + echo "> note / close this PR (to reject), then **merge this PR**. The NEXT drift-sync" + echo "> run reads the approved note from \`main\` and mechanically applies the registry" + echo "> edit (still zero-LLM — a human-authored marker, not generated code), opening a" + echo "> separate data-only PR. That two-run hand-off is how the loop closes." + echo "" + echo "### Note file(s) in this PR" + echo "" + # PRIMARY dedup marker — the changeset-key guard above matches open + # PRs on this, so a daily re-fire of the same drift (in ANY shape, + # including a mixed run that carries no NEW note file) never opens a + # second PR. + echo "" + for note in "${NOTES[@]:-}"; do + [ -n "$note" ] || continue + echo "- \`${note}\`" + echo "" + done + echo "" + echo "### drift-sync outcome" + echo "" + echo '```' + cat "${RUNNER_TEMP}/drift-sync.log" 2>/dev/null || echo "(see drift-sync-log artifact)" + echo '```' + } > "$PR_BODY_FILE" + + gh pr create \ + --title "chore(drift-sync): needs-human model-family decision ($(date +%Y-%m-%d))" \ + --assignee jpr5 \ + --body-file "$PR_BODY_FILE" + + PR_URL="" + for attempt in $(seq 1 6); do + MATCH="$(gh pr list --state open --json url,headRefOid \ + --jq ".[] | select(.headRefOid == \"$HEAD_SHA\") | .url" 2>/dev/null || true)" + if [ -n "$MATCH" ]; then + PR_URL="$(printf '%s' "$MATCH" | head -n1)" + break + fi + echo "needs-human PR for head SHA not indexed yet (attempt $attempt/6), waiting 10s..." + sleep 10 + done + if [ -z "$PR_URL" ]; then + echo "::error::No open needs-human PR found whose head matches pushed SHA $HEAD_SHA" + exit 1 + fi + echo "Opened needs-human PR: $PR_URL" + echo "url=$PR_URL" >> "$GITHUB_OUTPUT" + + # NO AUTO-MERGE FOR THE DRIFT-SYNC PATH (Phase 0/1 — see + # drift-auto-adjust-architecture-spec.md §6.1). Through Phase 1 the sync + # opens a PR and STOPS: a HUMAN reviews CI + the committed diff and + # merges. Auto-merge is an explicit, invariant-gated exception introduced + # only later (Phase 4, opt-in) for the single Tier-A2 class (zero-reference + # deprecation) — out of scope for this change. drift-sync-check is an + # AUTO-FILTER, NOT a provable merge gate (its clean re-collect is not + # independent of the edit), so human review remains the merge gate here, + # unchanged in spirit from the pre-C3 verdict-gated path. + + # Alert: the sync found a genuinely-new family or a still-referenced + # deprecation — the two irreducible human decisions (see + # self-healing-drift-plan.md "Bucket B"). The needs-human note was + # persisted by the step above (a distinct PR was opened, or the note was + # already in the repo / already proposed in an open PR — no PR spam). This + # alert fires the job RED so a human sees it. It is SKIPPED when the + # persist step itself FAILED (steps.needs_human_pr.outcome == 'failure') — + # that window is a tooling fault handled by the distinct gate-failure alert + # below, so the two never double-fire. + - name: Alert on needs-human decision + if: >- + always() && + steps.sync.outputs.reason == 'needs-human' && + steps.needs_human_pr.outcome != 'failure' env: SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} REPO: ${{ github.repository }} RUN_ID: ${{ github.run_id }} + PR_URL: ${{ steps.needs_human_pr.outputs.url }} run: | set -euo pipefail - # A missing webhook must NOT swallow a quarantine — it is a real - # needs-human event. Emit a VISIBLE ::error:: and FAIL the step (same - # discipline as the autofix-failure alert), never a silent exit 0. + echo "::error::drift-sync found a genuinely-new model family or a still-referenced deprecation — needs a human decision (see the needs-human PR / drift-proposals/*.md)" if [ -z "${SLACK_WEBHOOK:-}" ]; then - echo "::error::Drift quarantine (exit 5) — a human must inspect the drift report — AND SLACK_WEBHOOK is not set, so no alert could be sent. See run https://github.com/${REPO}/actions/runs/${RUN_ID}" + echo "::error::SLACK_WEBHOOK not set — cannot send needs-human alert" exit 1 fi - MSG="⚠️ *Drift quarantine — needs human* — the drift collector produced unparseable output (exit 5). No auto-fix attempted; a human must inspect the drift report artifact.\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + if [ -n "${PR_URL:-}" ]; then + WHERE="Review the note in the PR, set \`Decision: include\` (or close to reject), and merge: ${PR_URL}" + else + WHERE="The note is already in the repo (already persisted / already proposed in an open PR). See the drift-sync-log artifact and \`drift-proposals/\`." + fi + MSG="🧭 *Drift sync — needs a human decision* — a new/unclassified model family or a still-referenced deprecated family was found. ${WHERE}\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ -H "Content-Type: application/json" \ -d "$PAYLOAD" - # Quarantine is a needs-human outcome — FAIL the job so a human - # watching CI status (not just Slack) sees it (matching the - # collector-crash / autofix-failure alerts which also exit 1). The - # other alert steps are mutually exclusive with quarantine (fix-failure - # requires skip != 'true'; catch-all requires skip == ''; both false on - # quarantine where skip == 'true'), so this does not double-alert. exit 1 - # Slack notification on success. The drift path NEVER auto-merges (see the - # "NO AUTO-MERGE" comment above): a passing predicate opens a PR that a - # human must review and merge. The message says exactly that — it never - # claims "merged to main". - - name: Notify Slack on fix success - if: success() && steps.pr.outputs.url != '' + # Alert: covers every failure window once the sync itself has committed + # a mechanical edit (reason == 'ok-applied'), OR drift-sync's own + # internal gate reverted the edit before commit (reason == + # 'gate-failed'). This is deliberately WIDE — `failure()` in the + # ok-applied branch, not narrowly `steps.assert.outcome == 'failure'` — + # because the defense-in-depth Assert step can SUCCEED and a LATER step + # (Push branch + create PR: rejected push, `gh pr create` error, or the + # head-SHA PR-match polling loop exhausting its attempts) can still fail + # the job. With the narrower check, that window left reason == + # 'ok-applied' (non-empty, so the early-infra catch-all below stays + # silent) and steps.assert.outcome == 'success' (so this alert itself + # stayed silent too) — a silent failure on an unattended daily cron. + # Either way (gate-failed, assert failure, or a later step failure) this + # is a genuine tooling fault, NOT a needs-human product decision, so it + # gets its own distinct alert whose message names the step that failed. + # + # G#2: this ALSO covers the needs-human persist step's OWN failure + # (steps.needs_human_pr.outcome == 'failure' — a rejected push, a + # `gh pr create` error, or its head-SHA PR-match polling loop exhausting + # its attempts). That is a tooling fault, not a product decision, so it + # belongs here rather than in the needs-human alert (which is skipped on a + # persist failure, so the two never double-fire). + - name: Alert on drift-sync-check gate failure + if: >- + always() && + (steps.sync.outputs.reason == 'gate-failed' || + (steps.sync.outputs.reason == 'ok-applied' && failure()) || + (steps.sync.outputs.reason == 'needs-human' && steps.needs_human_pr.outcome == 'failure')) env: SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - PR_URL: ${{ steps.pr.outputs.url }} REPO: ${{ github.repository }} RUN_ID: ${{ github.run_id }} + SYNC_REASON: ${{ steps.sync.outputs.reason }} + ASSERT_OUTCOME: ${{ steps.assert.outcome }} + PR_OUTCOME: ${{ steps.pr.outcome }} + NEEDS_HUMAN_PR_OUTCOME: ${{ steps.needs_human_pr.outcome }} run: | set -euo pipefail + if [ "${SYNC_REASON}" = "gate-failed" ]; then + FAILING_STEP="Run deterministic drift sync (drift-sync.ts's own internal drift-sync-check gate reverted the mechanical edit before commit)" + elif [ "${NEEDS_HUMAN_PR_OUTCOME}" = "failure" ]; then + FAILING_STEP="Persist needs-human note + open PR (push rejected, gh pr create error, or PR-match polling exhausted — the needs-human note may not have been persisted)" + elif [ "${ASSERT_OUTCOME}" = "failure" ]; then + FAILING_STEP="Assert drift-sync-check (defense-in-depth)" + elif [ "${PR_OUTCOME}" = "failure" ]; then + FAILING_STEP="Push branch + create PR" + else + FAILING_STEP="an unidentified step after the ok-applied sync" + fi + echo "::error::drift-sync-check gate failure window — failing step: ${FAILING_STEP} — this is a tooling fault, not a product decision" if [ -z "${SLACK_WEBHOOK:-}" ]; then - echo "::error::SLACK_WEBHOOK not set — cannot send fix-success alert" - exit 0 + echo "::error::SLACK_WEBHOOK not set — cannot send gate-failure alert" + exit 1 fi - MSG="✅ *Drift-fix PR opened — needs human review + merge*\nThe drift-success predicate passed (verdict in the PR body); a human must review CI + the diff and merge.\nPR: ${PR_URL}\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + MSG="🔒❌ *Drift sync — gate refused* — failing step: ${FAILING_STEP}. This needs engineering triage, not a product decision.\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ -H "Content-Type: application/json" \ -d "$PAYLOAD" + exit 1 - # Slack notification on failure. A missing SLACK_WEBHOOK is a VISIBLE - # ::error:: annotation (not a silent exit 0). The drift path has no merge - # step, so the failure reasons are the PR-step reason and the drift-success - # predicate's reason (the "Assert drift truly resolved" step). - - name: Notify Slack on fix failure - if: failure() && steps.check.outputs.skip != 'true' && steps.autofix.outcome == 'success' + # Slack notification on success. Never claims the change auto-merged — + # see the NO AUTO-MERGE comment above. + - name: Notify Slack on sync success + if: success() && steps.pr.outputs.url != '' env: SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + PR_URL: ${{ steps.pr.outputs.url }} REPO: ${{ github.repository }} RUN_ID: ${{ github.run_id }} - PR_REASON: ${{ steps.pr.outputs.reason }} - ASSERT_REASON: ${{ steps.assert.outputs.reason }} run: | set -euo pipefail - # The PR-step reason takes precedence, then the drift-success - # predicate's reason (the "Assert drift truly resolved" step) so a - # blocked cheat is named rather than reported blank. The predicate - # reasons below (WS-6 needs-human tie-in) name the attempted-cheat / - # not-actually-fixed cause explicitly. - REASON="${PR_REASON:-${ASSERT_REASON:-}}" - case "$REASON" in - no-pr-match) DETAIL=" (no open PR matched the pushed head SHA)";; - # Drift-success predicate reasons — NEEDS HUMAN. The LOUD cheat - # reasons (comparison-leg-only / suppression-suspected) mean the - # auto-fix tried to game the drift detector rather than fix the mock. - comparison-leg-only) DETAIL=" (🚨 NEEDS HUMAN: auto-fix changed ONLY comparison-leg fixtures — a fixture-relaxation cheat, NOT a real mock fix. See run logs for the offending files)";; - suppression-suspected) DETAIL=" (🚨 NEEDS HUMAN: auto-fix edited the drift allowlist or a *.drift.ts assertion — silencing the detector, never a valid fix. See run logs)";; - unsanctioned-change) DETAIL=" (🚨 NEEDS HUMAN: auto-fix changed a file NOT on the sanctioned allowlist — deps/config/manifests, unrelated tests, or an unnamed fixture that could game the collector. See run logs for the offending files)";; - no-production-change) DETAIL=" (NEEDS HUMAN: auto-fix changed zero production mock-builder files — nothing shippable)";; - still-dirty) DETAIL=" (NEEDS HUMAN: post-fix collector still reports critical drift — the fix did not resolve it)";; - quarantine-after-fix) DETAIL=" (NEEDS HUMAN: post-fix collector returned quarantine — unparseable/untrusted output after the fix)";; - collector-infra) DETAIL=" (NEEDS HUMAN: post-fix collector infra failure — cannot trust a clean signal)";; - production-change-off-target) DETAIL=" (NEEDS HUMAN: production change did not touch any file the drift report named as a fix target — may be a legit shared-helper fix)";; - version-bump-failed) DETAIL=" (NEEDS HUMAN: the version bump / CHANGELOG step failed — refused to open an UNVERSIONED PR that would merge a fix which never publishes a release. See run logs)";; - post-fix-parse-error) DETAIL=" (NEEDS HUMAN: the post-fix collector result could not be parsed/read — empty/non-integer --post-fix-exit (e.g. a skipped recollect) or an unreadable post-fix report. Failed closed: no PR opened. See run logs)";; - git-push-failed) DETAIL=" (NEEDS HUMAN: a git checkout/add/commit/push failed while opening the PR — failed closed, no PR opened. See run logs)";; - config-error) DETAIL=" (drift-success predicate CONFIG error — missing/unreadable report or bad args; the gate itself needs attention)";; - *) DETAIL="";; - esac if [ -z "${SLACK_WEBHOOK:-}" ]; then - echo "::error::Drift auto-fix failed${DETAIL} AND SLACK_WEBHOOK is not set — no alert could be sent. See run https://github.com/${REPO}/actions/runs/${RUN_ID}" + echo "::error::SLACK_WEBHOOK not set — cannot send sync-success alert" exit 0 fi - MSG="❌ *Drift auto-fix failed*${DETAIL} — manual intervention needed\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + MSG="✅ *Drift-sync PR opened — needs human review + merge*\nA data-only, zero-LLM model-family sync passed drift-sync-check; a human must review CI + the diff and merge.\nPR: ${PR_URL}\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ -H "Content-Type: application/json" \ @@ -606,41 +598,26 @@ jobs: # End-of-job CATCH-ALL for ANY job failure the specific alerts above did # NOT cover — chiefly the EARLY-infra window (checkout / mint-app-token / - # pnpm install / clone ag-ui / git config) that runs BEFORE `detect`/ - # `check` set their outputs. Those early steps leave collector_crashed, - # quarantine, and check.outputs.skip all EMPTY, so none of the four - # specific alerts fire and the job would otherwise die red with ZERO Slack - # signal. This step is UNCONDITIONAL on the earlier step outputs (only - # `failure()` + the anti-double-alert guard below), so it fires whenever - # nothing else did. - # - # Anti-double-alert: the four specific alerts fire iff collector_crashed - # or quarantine is set, OR the fixer actually ran (check.outputs.skip is - # 'true'/'false' — never empty — once `check` executed). So this catch-all - # only fires when NONE of those hold: collector_crashed != 'true' AND - # quarantine != 'true' AND check.outputs.skip is empty (check never ran). - # That is exactly the early-infra window, and it never overlaps with the - # specific alerts. + # pnpm install / clone ag-ui / git config / ollama provisioning) that runs + # BEFORE the `sync` step sets its outputs. That window leaves + # `steps.sync.outputs.reason` empty, so none of the three specific alerts + # fire and the job would otherwise die red with ZERO Slack signal. This + # step is UNCONDITIONAL on the earlier step outputs (only `failure()` + + # the anti-double-alert guard below), so it fires whenever nothing else + # did. - name: Alert on early-infra failure (catch-all) - if: >- - failure() && steps.detect.outputs.collector_crashed != 'true' && - steps.check.outputs.quarantine != 'true' && - steps.check.outputs.skip == '' + if: failure() && steps.sync.outputs.reason == '' env: SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} REPO: ${{ github.repository }} RUN_ID: ${{ github.run_id }} run: | set -euo pipefail - # This is an INFRA/SETUP failure — the fixer never ran (checkout / - # token mint / pnpm install / ag-ui clone / git config failed), so it - # is distinct from a drift-fix failure. A missing webhook must NOT - # swallow it: emit a VISIBLE ::error:: and fail the step. if [ -z "${SLACK_WEBHOOK:-}" ]; then - echo "::error::Drift-fix job failed during INFRA/SETUP (before the collector ran) AND SLACK_WEBHOOK is not set — no alert could be sent. See run https://github.com/${REPO}/actions/runs/${RUN_ID}" + echo "::error::Drift-sync job failed during INFRA/SETUP (before drift-sync ran) AND SLACK_WEBHOOK is not set — no alert could be sent. See run https://github.com/${REPO}/actions/runs/${RUN_ID}" exit 1 fi - MSG="🧱❌ *Drift-fix job — INFRA/SETUP failure* — the job failed during setup (checkout / token mint / pnpm install / ag-ui clone / git config) BEFORE the drift collector ran; this is not a drift-fix failure. Manual intervention needed.\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + MSG="🧱❌ *Drift-sync job — INFRA/SETUP failure* — the job failed during setup (checkout / token mint / pnpm install / ag-ui clone / git config / ollama provisioning) BEFORE drift-sync ran; this is not a sync-gate failure. Manual intervention needed.\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ -H "Content-Type: application/json" \ diff --git a/CLAUDE.md b/CLAUDE.md index 440a9893..2ab92c48 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,10 +36,13 @@ entire repo, not just staged files. ## Drift Remediation -Automated drift remediation lives in `scripts/`: +There is no LLM/agent in the remediation loop. Automated remediation is +deterministic and scoped to model-family churn only; general drift is caught by +the daily drift test and fixed by a human. Lives in `scripts/`: - `scripts/drift-report-collector.ts` — runs drift tests, produces `drift-report.json` -- `scripts/fix-drift.ts` — reads drift report, invokes Claude Code to fix builders, creates PR or issue +- `scripts/drift-sync.ts` — zero-LLM model-family sync (mechanical registry edit or needs-human note file) + reusable git/PR plumbing +- `scripts/drift-sync-check.ts` — the deterministic gate (changed-file allowlist, checksum-pin re-assert, clean re-collect) replacing remediation-diff review See `DRIFT.md` for full documentation and `.github/workflows/fix-drift.yml` for the CI workflow. diff --git a/DRIFT.md b/DRIFT.md index ab4c6af0..264e32d7 100644 --- a/DRIFT.md +++ b/DRIFT.md @@ -180,26 +180,33 @@ See `.github/workflows/test-drift.yml`. ## Automated Drift Remediation -When the daily drift test detects critical diffs on the `main` branch, the `fix-drift.yml` workflow runs automatically: - -1. **Collect** — `scripts/drift-report-collector.ts` runs drift tests and produces a structured `drift-report.json` -2. **Fix** — `scripts/fix-drift.ts` (default mode) constructs a prompt from the report and invokes Claude Code to fix the builders -3. **Verify** — Independent `pnpm test` and `pnpm test:drift` steps confirm the fix works -4. **PR** — `scripts/fix-drift.ts --create-pr` stages and commits the changes, bumps the version, and opens a pull request -5. **Issue** (on failure) — `scripts/fix-drift.ts --create-issue` opens a GitHub issue with the drift report and Claude Code output - -Steps 2 and 4/5 are separate invocations of `fix-drift.ts` with different modes. +There is no LLM/agent in the remediation loop. General (non-model-churn) drift +is **not** auto-fixed by anything — it is caught by the daily drift test (which +alerts on its own; see above) and fixed by a human like any other bug. The only +automated remediation is the deterministic, zero-LLM **model-family sync**, +which handles exactly one class of drift: a provider adding or retiring a +model family. The `fix-drift.yml` workflow runs it on `workflow_dispatch`, a +daily **scheduled cron** (independent of drift-test failure — a retired model +family does not, by itself, fail the drift tests), and on a failed `Drift +Tests` run (an opportunistic attempt in case the failure was model churn): + +1. **Sync** — `scripts/drift-sync.ts` fetches each provider's live `/models` listing directly and diffs it against the frozen classification in `src/__tests__/drift/model-registry.ts`: + - a classified family absent from live listings with **zero remaining aimock references** → a mechanical, comment-marked removal + - a still-referenced deprecated family, or a genuinely new/unclassified family → **never** auto-edited; a family-keyed dedup note file is written under `drift-proposals/` and the run is routed to a human (no PR spam on re-fire) +2. **Gate** — `scripts/drift-sync-check.ts` re-verifies any mechanical edit before (inside `drift-sync.ts`) and after (workflow defense-in-depth) it is kept: a changed-file allowlist (only `model-registry.ts` data literals + `drift-proposals/` notes), a checksum-pin re-assert over the frozen classification logic, and a clean re-collect +3. **PR** — the workflow always opens a pull request that a human reviews + merges (no auto-merge). There are two distinct PR classes: + - **`ok-applied`** — a successful mechanical registry edit. Pushed onto the `fix/drift-*` branch `drift-sync.ts` committed onto; a human reviews CI + the diff and merges. + - **`needs-human`** — a routed decision. `drift-sync.ts` commits the `drift-proposals/` note file(s), and the workflow pushes a **distinct `drift-needs-human/*` branch** and opens a PR so the note lands in the repo (the job also goes RED + Slack-alerts so the decision is seen). The PR is **never auto-merged**. To approve a _new-family_ note, set its `Decision: include` line and **merge the PR**; the **next** drift-sync run reads the approved note from `main` and applies the mechanical registry edit (an `ok-applied` PR). That two-run hand-off is how the loop closes. + + **Re-fires never spam a second PR — idempotent in every run shape.** Because a drift-sync PR is never auto-merged, an un-merged drift is re-detected on every daily cron run. Both PR classes therefore dedup on a **stable changeset key**: `drift-sync.ts` emits a date-independent `changeset-key` (a hash of the sorted set of applied + deferred family outcomes, independent of the date-stamped comment text and the run-id branch name), and each PR body carries a `` marker. Before opening a PR, the workflow skips if an open PR already carries that marker. This covers the **mixed run** — a mechanical removal of one family committed the same run a _different_ family is deferred to a human (its note already on `main`) — whose committed diff is a registry edit with **no new note file**: a note-path-only key would be empty there and let a new PR open every day. A run that produces no new commit at all (note already on `main`, nothing applied) pushes nothing; and the older per-note `drift-proposal-note: ` body marker is retained as a secondary guard. ### Artifacts -Both workflows upload artifacts: - -- `drift-report.json` — structured drift data (retained 30 days) -- `claude-code-output.log` — Claude Code's reasoning and tool calls (fix workflow only) +- `drift-report.json` (test-drift.yml) / `drift-sync-log`, `drift-sync-check-log` (fix-drift.yml) — structured/plaintext run output (retained 30 days) ### Manual trigger -The fix workflow also supports `workflow_dispatch` for manual runs. +The sync workflow also supports `workflow_dispatch` for manual runs. ## Cost diff --git a/scripts/drift-report-collector.ts b/scripts/drift-report-collector.ts index a78807c3..81226a20 100644 --- a/scripts/drift-report-collector.ts +++ b/scripts/drift-report-collector.ts @@ -633,6 +633,28 @@ export function extractRawLocation(msg: string): string { ); } +/** + * F1 — PER-LEG (single-message) infra classification. This is the primitive + * that keeps ONE leg's failure from batch-poisoning its siblings: each + * unparseable failure is judged ALONE, so an unparseable leg can never flip a + * genuine infra leg into the quarantine lane (nor a benign infra leg mask a + * genuinely-unparseable sibling). + * + * A message is benign infra iff it carries POSITIVE infra evidence AND shows no + * drift-like signal, both judged on the SAME stack-stripped text (A3 — the two + * scans must never disagree because they saw different inputs). A bare + * AssertionError with no infra token is NOT infra (returns false → the caller + * quarantines it for review), preserving the CLASS 1 "unrecognized ⇒ never a + * false all-clear" invariant at the per-leg grain. + */ +export function classifySingleUnparseableAsInfra(message: string): boolean { + // Normalize ONCE; both scans consume the identical normalized text (A3). + const normalized = stripStackFrames(message); + const hasInfraEvidence = INFRA_INDICATORS.some((re) => re.test(normalized)); + const isDriftLike = DRIFT_LIKE_INDICATORS.some((re) => re.test(normalized)); + return hasInfraEvidence && !isDriftLike; +} + export function classifyUnparseableAsInfra(unparseableMessages: string[]): boolean { // CLASS 1 — fail-loud on absent evidence. No messages means NO positive infra // evidence, so this is NOT a benign "all clear". `[].every(...)` is vacuously @@ -641,17 +663,13 @@ export function classifyUnparseableAsInfra(unparseableMessages: string[]): boole // false all-clear. if (unparseableMessages.length === 0) return false; - // Normalize ONCE per message; both scans consume the identical normalized - // text (A3 — the two scans must never disagree due to differing inputs). - const normalized = unparseableMessages.map(stripStackFrames); - - // Infra requires POSITIVE evidence on EVERY message AND zero drift signal on - // ALL of them. If any message lacks an infra indicator, or any message looks - // drift-like, we do NOT swallow — the caller throws (exit 1, investigate). - const allInfraErrors = normalized.every((msg) => INFRA_INDICATORS.some((re) => re.test(msg))); - const anyDriftLike = normalized.some((msg) => DRIFT_LIKE_INDICATORS.some((re) => re.test(msg))); - - return allInfraErrors && !anyDriftLike; + // Batch semantics are exactly the conjunction of the per-leg predicate: the + // whole set is benign infra iff EVERY message is individually benign infra + // (every message has an infra indicator AND none is drift-like). Kept for the + // back-compat batch callers/tests; the collector's swallow-vs-quarantine + // decision below is now made PER LEG via classifySingleUnparseableAsInfra so a + // mixed batch no longer drags benign infra legs into quarantine. + return unparseableMessages.every(classifySingleUnparseableAsInfra); } /** @@ -927,21 +945,33 @@ export function collectDriftEntries(results: VitestJsonResult): CollectResult { console.warn(` Unparseable failure message (first 300 chars): ${msg.slice(0, 300)}`); } - if (classifyUnparseableAsInfra(unparseableMessages)) { + // F1: classify EACH unparseable failure on its own so one leg's genuinely- + // unparseable output never batch-poisons a sibling that failed on benign + // infra. Previously a single all-or-nothing classifyUnparseableAsInfra call + // over the whole batch meant one non-infra leg flipped the ENTIRE batch to + // "not infra" and dragged every benign infra sibling into quarantine (exit + // 5) — poisoning the shared base report for every downstream PR. Now an infra + // leg is swallowed on its own merits and only the genuinely-unparseable + // leg(s) are quarantined for review. A1.3: quarantine (exit 5), never a + // fail-loud crash (exit 1) — exit 1 is reserved for genuine collector bugs. + const infraCount = unparseableFailures.filter((f) => + classifySingleUnparseableAsInfra(f.message), + ).length; + const quarantinedLegs = unparseableFailures.filter( + (f) => !classifySingleUnparseableAsInfra(f.message), + ); + if (infraCount > 0) { console.warn( - `WARNING: ${unparseable} test failure(s) appear to be API/infrastructure errors ` + - `(not drift reports). Continuing with 0 drift entries.`, + `WARNING: ${infraCount} test failure(s) appear to be API/infrastructure errors ` + + `(not drift reports) — swallowed as benign infra.`, ); - } else { - // A1.3: genuine-but-unparseable drift is no longer a fail-loud crash (exit - // 1). Each such failure is quarantined (exit 5) so it surfaces for human - // review without being silently swallowed as a green. Exit 1 is now - // reserved for genuine collector bugs (unhandled exceptions). + } + if (quarantinedLegs.length > 0) { console.warn( - `WARNING: ${unparseable} test failure(s) could not be parsed as drift reports — ` + + `WARNING: ${quarantinedLegs.length} test failure(s) could not be parsed as drift reports — ` + `quarantined for review (exit 5).`, ); - for (const f of unparseableFailures) { + for (const f of quarantinedLegs) { quarantine.push({ provider: "unknown", testName: f.testName, diff --git a/scripts/drift-slack-summary.ts b/scripts/drift-slack-summary.ts index 6b705477..3d3d3609 100644 --- a/scripts/drift-slack-summary.ts +++ b/scripts/drift-slack-summary.ts @@ -26,7 +26,7 @@ import { appendFileSync, existsSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { readDriftReport } from "./fix-drift.js"; +import { readDriftReport } from "./drift-sync.js"; import type { DriftEntry, DriftReport, DriftSeverity } from "./drift-types.js"; // --------------------------------------------------------------------------- diff --git a/scripts/drift-success-predicate.ts b/scripts/drift-success-predicate.ts deleted file mode 100644 index 02a30672..00000000 --- a/scripts/drift-success-predicate.ts +++ /dev/null @@ -1,916 +0,0 @@ -/// - -/** - * Drift-Success Predicate (WS-2) - * - * A pure predicate — plus a thin CLI wrapper — that decides whether an - * auto-fix run ACTUALLY resolved API drift, versus merely GAMING the drift - * detector by relaxing one of the comparison legs (the SDK-shape fixture, the - * triangulation schema/allowlist, the real-API harness, or a `*.drift.ts` - * assertion). - * - * ALLOWLIST MODEL (round-2 CR — replaces the earlier denylist). A denylist of - * "known gameable legs" leaks: any editable collector input NOT on the list - * (package.json/lockfile pinning a vendored SDK, a tsconfig, an imported - * sub-fixture, an unknown path, a drift-dir `*.test.ts`) could accompany a token - * on-target production edit and reach `resolved:true`. So the predicate INVERTS - * to an allowlist: a fix is RESOLVED only when EVERY changed file is on the - * allowlist, which is (a) PRODUCTION SOURCE — `src/**` that is NOT under - * `src/__tests__/` and is not a config/manifest — PLUS (b) a fixture file - * EXPLICITLY NAMED for a drift entry in the report (`entry.builderFile` / - * `entry.typesFile`, e.g. a canary model-registry.ts the collector sanctioned). - * ANYTHING else blocks: any other `src/__tests__/**` file (legs, `*.drift.ts`, - * `*.test.ts`, providers/ws-providers/schema/sdk-shapes), package.json / - * lockfiles / manifests / config, and any unrecognized path. All paths are - * CANONICALIZED (strip `./`, collapse `//`, resolve `.`/`..`, reject repo-root - * escapes) before classification so a spelling variant cannot sneak a leg past - * the matcher. - * - * The hole this closes: the drift tests are three-way triangulations - * (SDK vs real API vs mock). The SDK leg is literally the repo fixture - * `src/__tests__/drift/sdk-shapes.ts`. Deleting a field from that fixture makes - * the SDK leg null for that path, so the "critical" branch cannot fire and the - * collector reports clean (exit 0) WITHOUT any change to the mock builder. - * Re-running the collector alone therefore cannot detect the cheat — its own - * SDK leg reads the relaxed fixture. The old guard in fix-drift.ts - * (`builderFiles.length === 0 && testFiles.length === 0`) ACCEPTS such a run - * because `testFiles` is non-empty. - * - * The predicate requires THREE independent signals for `resolved:true`: - * 1. AUTHORITATIVE — the post-fix collector re-run is clean (exit 0 AND - * criticalCount 0). - * 2. PRODUCTION CHANGE — at least one PRODUCTION mock-builder file changed - * (`src/**` excluding `src/__tests__/`). A relaxation NEVER changes one. - * 3. NO GAMEABLE-LEG EDIT AT ALL — the changed set touches NO gameable leg. - * HARDENED (fix #1): a gameable-leg edit ALWAYS blocks, INDEPENDENT of how - * many production files also changed. A legitimate auto-remediation updates - * the mock BUILDER to match the SDK; it never edits a comparison/SDK/harness - * leg (those change only on a deliberate human vendored-SDK bump). This - * closes the WS-2b hybrid cheat (relax a leg + one trivial on-target - * production edit) that the old "only check legs when productionFiles===0" - * logic passed straight to auto-merge. Schema/allowlist and `*.drift.ts` - * assertion edits always map to SUPPRESSION_SUSPECTED (actively silencing - * the detector); other gameable legs (sdk-shapes, harness incl the - * dual-classified voice-models.ts) map to SUPPRESSION_SUSPECTED when paired - * with a production change and COMPARISON_LEG_ONLY when standalone. - * - * Additionally, the production change should intersect the report's SANCTIONED - * target set (`union(entry.builderFile, entry.typesFile≠null)`); an off-target - * production change is a WARNING that still blocks (a shared helper MAY be the - * real fix, so it is distinct from an outright cheat). An EMPTY sanctioned set is - * fail-closed (fix #3): with no named target we cannot verify the change landed - * where the drift lives, so it routes to human rather than rubber-stamps. - * - * CLI exit codes (mirrors drift-report-collector.ts's distinct-code discipline): - * 0 — RESOLVED - * 10 — NO_PRODUCTION_CHANGE - * 11 — COMPARISON_LEG_ONLY (leg edit with NO production change) - * 12 — SUPPRESSION_SUSPECTED (schema/*.drift.ts edit, OR any gameable - * leg edited ALONGSIDE a production change) - * 13 — STILL_DIRTY (post-fix collector exit 2, or exit 0 with - * criticalCount>0) - * 14 — QUARANTINE_AFTER_FIX (post-fix collector exit 5 — checked BEFORE - * criticalCount, so exit 5 wins) - * 15 — COLLECTOR_INFRA (post-fix collector exit 1 — likewise wins - * over criticalCount) - * 16 — PRODUCTION_CHANGE_OFF_TARGET (off-target OR zero sanctioned targets; - * WARNING, still blocks) - * 17 — UNSANCTIONED_CHANGE (a changed file is NOT on the allowlist — - * package.json/lockfile/config/unknown path/ - * non-drift or drift `*.test.ts`; a real fix - * touches only production source + report- - * named fixture targets) - * 18 — VERSION_BUMP_FAILED (NOT scored by the predicate — surfaced by - * fix-drift.ts's createPr when the mandatory - * version bump / CHANGELOG step fails, so an - * unversioned fix that never publishes is - * never opened as a PR; fail-closed to human) - * 2 — CONFIG_ERROR (missing/unreadable report, bad args, a - * --changed-file list that disagrees w/ git, - * a path escaping the repo root, or a - * malformed post-fix report that cannot be - * scored) - * - * Usage: - * npx tsx scripts/drift-success-predicate.ts \ - * --report drift-report.json \ - * --post-fix-report drift-report.post-fix.json \ - * --post-fix-exit \ - * [--changed-file src/helpers.ts ...] - * - * When no --changed-file args are supplied the CLI derives the changed set from - * `git status --porcelain` (mirrors fix-drift.ts:getChangedFiles()). When a - * --changed-file list IS supplied it is cross-checked against git and rejected - * (CONFIG_ERROR) on any mismatch (fix #4 — a leg-omitting list must not blind - * the predicate). - */ - -import { execSync } from "node:child_process"; -import { existsSync, readFileSync } from "node:fs"; -import { resolve } from "node:path"; -import { fileURLToPath } from "node:url"; - -import type { DriftReport } from "./drift-types.js"; - -// --------------------------------------------------------------------------- -// Public types -// --------------------------------------------------------------------------- - -export enum PredicateReason { - RESOLVED = "resolved", - NO_PRODUCTION_CHANGE = "no-production-change", - COMPARISON_LEG_ONLY = "comparison-leg-only", - SUPPRESSION_SUSPECTED = "suppression-suspected", - UNSANCTIONED_CHANGE = "unsanctioned-change", - STILL_DIRTY = "still-dirty", - QUARANTINE_AFTER_FIX = "quarantine-after-fix", - COLLECTOR_INFRA = "collector-infra", - PRODUCTION_CHANGE_OFF_TARGET = "production-change-off-target", - CONFIG_ERROR = "config-error", - /** - * The version bump / CHANGELOG step failed while opening a drift-fix PR. A - * release ALWAYS accompanies an auto-remediation; without it the PR would - * merge a fix that never publishes (silent value loss). This is a hard, - * fail-closed reason surfaced by fix-drift.ts's createPr — never produced by - * the predicate's own scoring — routed to human review like any other - * needs-human reason. - */ - VERSION_BUMP_FAILED = "version-bump-failed", - /** - * The `--post-fix-*` arguments were present but could not be parsed/read - * while opening a drift-fix PR (e.g. an empty/non-integer `--post-fix-exit` - * from a skipped recollect, or an unreadable post-fix report). fix-drift.ts - * already fails CLOSED on this (no PR), but the throw historically reached the - * top-level catch with a BLANK `reason=`; this names the cause so the Slack - * alert is not blank. Surfaced by fix-drift.ts's main(), never the predicate. - */ - POST_FIX_PARSE_ERROR = "post-fix-parse-error", - /** - * A git operation (checkout / add / commit / push) failed while opening a - * drift-fix PR. This fails CLOSED (no PR is opened — the push never completed, - * so no partial/unversioned PR ships) but historically alerted with a BLANK - * `reason=`; this names the cause. Surfaced by fix-drift.ts's createPr, never - * the predicate. - */ - GIT_PUSH_FAILED = "git-push-failed", -} - -/** Stable exit code for each reason (see module header). */ -export const REASON_EXIT_CODE: Record = { - [PredicateReason.RESOLVED]: 0, - [PredicateReason.NO_PRODUCTION_CHANGE]: 10, - [PredicateReason.COMPARISON_LEG_ONLY]: 11, - [PredicateReason.SUPPRESSION_SUSPECTED]: 12, - [PredicateReason.UNSANCTIONED_CHANGE]: 17, - [PredicateReason.STILL_DIRTY]: 13, - [PredicateReason.QUARANTINE_AFTER_FIX]: 14, - [PredicateReason.COLLECTOR_INFRA]: 15, - [PredicateReason.PRODUCTION_CHANGE_OFF_TARGET]: 16, - [PredicateReason.CONFIG_ERROR]: 2, - [PredicateReason.VERSION_BUMP_FAILED]: 18, - [PredicateReason.POST_FIX_PARSE_ERROR]: 19, - [PredicateReason.GIT_PUSH_FAILED]: 20, -}; - -export interface PredicateInputs { - /** Changed-file paths from getChangedFiles() (git porcelain). */ - changedFiles: string[]; - /** The ORIGINAL pre-fix drift report (source of sanctioned fix targets). */ - report: DriftReport; - /** Exit code of the re-run collector (0 clean / 2 dirty / 5 quarantine / 1 infra). */ - postFixCollectorExit: number; - /** - * criticalCount parsed from the re-run report (belt-and-suspenders vs exit 0). - * - * FIX #7 — INDEPENDENCE CAVEAT: this signal (and the collector exit code) is - * derived from the SAME fixtures the fixer was told to make pass, so it is - * NOT independent of a fixture-relaxation cheat — a run that relaxed the SDK - * leg reports clean here too. It is therefore only trustworthy BECAUSE the - * ALLOWLIST gate now requires EVERY changed file to be production source or a - * report-named fixture target (see isAllowlisted / the UNSANCTIONED_CHANGE + - * SUPPRESSION_SUSPECTED + COMPARISON_LEG_ONLY branches). Any leg/fixture/config - * edit that could relax the collector's own inputs is NOT allowlisted and - * blocks, regardless of production files. That default-deny rule is - * load-bearing: it is what makes a clean post-fix signal mean "the mock was - * really fixed" rather than "an input the collector reads was relaxed". - * - * FIX #4 (assess) — the criticalCount and the post-fix exit code are both - * scored from the AUTHORITATIVE in-workflow re-collect output - * (drift-report.post-fix.json, written by the "Re-collect drift" step), NOT a - * repo-committed file the fixer could forge: neither the pre-fix report nor the - * post-fix report is committed to the repo (see .github/workflows/fix-drift.yml - * — both are produced by collector invocations in-workflow). The re-collect - * runs AFTER autofix and OVERWRITES anything written to that path during the - * fix, so the predicate scores a freshly-generated report, not attacker - * content. - */ - postFixCriticalCount: number; -} - -export interface PredicateResult { - resolved: boolean; - reason: PredicateReason; - /** Human-readable one-liner for Slack / PR body. */ - detail: string; - /** The subset of changedFiles that triggered a block (for LOUD alerts). */ - offendingFiles: string[]; -} - -/** Thrown for malformed CLI args / unreadable inputs / bad paths — maps to exit 2. */ -export class PredicateConfigError extends Error {} - -// --------------------------------------------------------------------------- -// File classification (see spec §2) -// --------------------------------------------------------------------------- - -/** - * Canonicalize a git-reported path to a stable repo-relative POSIX form BEFORE - * classification, so an equivalent-but-non-identical spelling of a leg cannot - * sneak past the exact-string matchers (round-2 CR F1 / slot-1 / slot-2): - * - strip a leading `./` - * - collapse doubled slashes (`//` → `/`) - * - resolve `.` segments and interior `..` segments - * - FAIL CLOSED (PredicateConfigError → exit 2) on any path that escapes the - * repo root (a leading `..` after resolution) or is absolute — such a path - * was never a legitimate in-repo change and must not be silently reclassified. - */ -export function canonicalizePath(file: string): string { - if (file.startsWith("/")) { - throw new PredicateConfigError(`Refusing to classify an absolute path: ${file}`); - } - const rawSegments = file.split("/"); - const out: string[] = []; - for (const seg of rawSegments) { - if (seg === "" || seg === ".") continue; // drop empty (//, leading ./) and `.` - if (seg === "..") { - if (out.length === 0) { - throw new PredicateConfigError(`Path escapes the repo root (fail-closed): ${file}`); - } - out.pop(); - continue; - } - out.push(seg); - } - return out.join("/"); -} - -/** - * The triangulation SCHEMA file. Editing it (esp. its ALLOWLISTED_PATHS set) - * silences diffs globally — a human-reviewed artifact, never a valid fix. - */ -const SCHEMA_FILE = "src/__tests__/drift/schema.ts"; - -/** - * The SDK-shape fixture — the SDK leg of the three-way compare and the primary - * cheat surface (relaxing it makes the critical branch unreachable). - */ -const SDK_SHAPES_FILE = "src/__tests__/drift/sdk-shapes.ts"; - -/** - * The real-API call harness files. Weakening these could elicit a smaller real - * shape (shrinking the real leg), making a diff disappear without a mock change. - */ -const HARNESS_FILES: ReadonlySet = new Set([ - "src/__tests__/drift/providers.ts", - "src/__tests__/drift/ws-providers.ts", - "src/__tests__/drift/helpers.ts", - "src/__tests__/drift/voice-models.ts", -]); - -/** - * LEGITIMATE-FIXTURE-THAT-IS-THE-FIX-TARGET: drift fixtures under - * `src/__tests__/drift/` that ARE the correct fix target for certain drifts - * (the known-models canary routes fixes to these model-list files). These are - * NOT gameable comparison legs — adding a newly-shipped model id here is a - * legit fix, not a relaxation. They are allowed as accompanying changes and are - * never counted as a comparison-leg cheat. - * - * FIX #2 — `voice-models.ts` is deliberately NOT listed here: it is also a - * real-API HARNESS file, and block-classification wins over legit-accept - * (fail-closed precedence). It is classified as a gameable leg in isGameableLeg. - */ -const LEGIT_FIXTURE_TARGETS: ReadonlySet = new Set([ - "src/__tests__/drift/model-registry.ts", - "src/__tests__/drift/model-family.ts", -]); - -/** - * True when `file` is a PRODUCTION mock-builder source file: under `src/` but - * NOT under `src/__tests__/`. This matches fix-drift.ts's existing - * `builderFiles` predicate exactly. - */ -export function isProductionFile(file: string): boolean { - return file.startsWith("src/") && !file.startsWith("src/__tests__/"); -} - -/** - * True when `file` is a `*.drift.ts` test file whose assertions could be - * loosened to make a diff disappear (e.g. `expect(...).toEqual([])`). - */ -export function isDriftTestFile(file: string): boolean { - return file.startsWith("src/__tests__/drift/") && file.endsWith(".drift.ts"); -} - -/** - * GAMEABLE-LEG (the block set). Editing ANY of these can erase a drift diff - * WITHOUT changing the mock output — either by relaxing a comparison leg (the - * SDK-shape fixture, the schema/allowlist, the real-API harness) or by loosening - * a `*.drift.ts` assertion. Every one of these is a HUMAN-REVIEWED artifact: a - * legitimate auto-remediation updates the mock BUILDER to match the SDK, and the - * leg only changes on a deliberate human vendored-SDK bump. So any leg edit — - * ALONE OR ACCOMPANIED BY A PRODUCTION CHANGE — is fail-closed to needs-human - * (SUPPRESSION_SUSPECTED). This closes the WS-2b hybrid cheat (relax a leg + - * one trivial on-target production edit), which the old "only check legs when - * productionFiles===0" logic let through to auto-merge. - * - * FIX #2 — CLASSIFICATION PRECEDENCE: a file that is BOTH a harness leg AND a - * legit fixture target (e.g. `voice-models.ts`) is treated as gameable (block). - * Block-classification wins over legit-accept; fail-closed. Only PURE legit - * fixture targets (model-registry.ts / model-family.ts), which are NOT harness - * legs, are excluded — a canary model-list fix routes to those plus its - * production builder and is not blocked. - */ -export function isGameableLeg(file: string): boolean { - // Block-set membership is checked FIRST (fix #2 precedence): a file that is - // BOTH a harness leg AND a legit fixture target (voice-models.ts) matches - // HARNESS_FILES here and blocks, before the legit-target set is ever consulted. - if (file === SDK_SHAPES_FILE) return true; - if (file === SCHEMA_FILE) return true; - if (HARNESS_FILES.has(file)) return true; - if (isDriftTestFile(file)) return true; - // PURE legit fixture targets (not also a harness leg) are explicitly - // non-gameable — a canary model-list fix routes here plus its production - // builder and must not be blocked. - if (LEGIT_FIXTURE_TARGETS.has(file)) return false; - // Everything else (production files, unrelated paths) is non-gameable. - return false; -} - -/** - * SUPPRESSION surface (the NARROW always-SUPPRESSION_SUSPECTED subset): the - * triangulation schema/allowlist and `*.drift.ts` assertion files. Editing these - * ACTIVELY SILENCES the detector (allowlist growth, loosened assertion) and is - * never a valid fix — so it ALWAYS maps to SUPPRESSION_SUSPECTED, standalone or - * alongside a production change. (The broader gameable-leg set below — sdk-shapes, - * harness — also always blocks per fix #1, but maps to COMPARISON_LEG_ONLY when - * standalone and SUPPRESSION_SUSPECTED only when paired with a production change.) - */ -export function isSuppressionSurface(file: string): boolean { - return file === SCHEMA_FILE || isDriftTestFile(file); -} - -/** - * GAMEABLE-COMPARISON-LEG (retained for API compatibility / classification - * unit coverage). Same full set as isGameableLeg — every leg edit blocks. - */ -export function isComparisonLeg(file: string): boolean { - return isGameableLeg(file); -} - -/** - * Derive the SANCTIONED fix-target set from the pre-fix report: - * `union(entry.builderFile, entry.typesFile≠null)`. These are the files the - * collector itself named as the correct place to fix each drift. - */ -export function sanctionedTargets(report: DriftReport): Set { - const targets = new Set(); - for (const entry of report.entries) { - if (entry.builderFile) targets.add(canonicalizePath(entry.builderFile)); - if (entry.typesFile) targets.add(canonicalizePath(entry.typesFile)); - } - return targets; -} - -/** - * ALLOWLIST membership (round-2 CR F1/F2/F3 — the inversion). A changed file is - * SANCTIONED for an auto-fix run when it is either: - * (a) PRODUCTION SOURCE — `src/**` that is NOT under `src/__tests__/` (a mock - * builder / type file, the legitimate fix target); OR - * (b) a fixture file EXPLICITLY NAMED for a drift entry in the report - * (`entry.builderFile` / `entry.typesFile`, e.g. a canary model-registry.ts - * the collector itself sanctioned as the fix target for this run). - * - * EVERYTHING else is NOT allowlisted and blocks: any other `src/__tests__/**` - * file (comparison legs, `*.drift.ts`, `*.test.ts`, providers/schema/sdk-shapes), - * `package.json` / lockfiles / manifests / config, and any unrecognized path. - * `file` MUST already be canonicalized; `sanctioned` is the canonicalized - * sanctioned-target set (see sanctionedTargets). - */ -export function isAllowlisted(file: string, sanctioned: ReadonlySet): boolean { - if (isProductionFile(file)) return true; - if (sanctioned.has(file)) return true; - return false; -} - -// --------------------------------------------------------------------------- -// The predicate (see spec §3) -// --------------------------------------------------------------------------- - -export function evaluateDriftResolved(i: PredicateInputs): PredicateResult { - const { report, postFixCollectorExit, postFixCriticalCount } = i; - // Canonicalize every changed path BEFORE classification so a spelling variant - // (`./src/...`, `src//...`, `.`/`..` segments) of a leg cannot slip past the - // exact-string matchers (round-2 CR F1). An absolute path or a repo-root escape - // throws PredicateConfigError from canonicalizePath → CONFIG_ERROR at the CLI. - const changedFiles = i.changedFiles.map(canonicalizePath); - - // ---- Signal 1: AUTHORITATIVE — collector clean on re-run. ------------- - // Checked FIRST: a dirty/quarantine/infra collector makes any file-set moot. - // - // FIX #6 — the collector-STATE classification (exit 2/5/1) is checked BEFORE - // the belt-and-suspenders criticalCount>0 branch. A quarantine (exit 5) or an - // infra failure (exit 1) that ALSO happens to carry a parseable criticalCount>0 - // is a quarantine/infra event, NOT a plain STILL_DIRTY — it gets its own - // distinct reason so the Slack alert names the real cause. STILL_DIRTY is - // reserved for exit 2, or a clean-looking exit 0 that nonetheless reports - // criticalCount>0 (report/exit disagreement). - if (postFixCollectorExit === 5) { - return { - resolved: false, - reason: PredicateReason.QUARANTINE_AFTER_FIX, - detail: - "Post-fix drift collector returned quarantine (exit 5) — unparseable/untrusted output after the fix. Needs human review.", - offendingFiles: [], - }; - } - if (postFixCollectorExit === 1) { - return { - resolved: false, - reason: PredicateReason.COLLECTOR_INFRA, - detail: - "Post-fix drift collector returned infra failure (exit 1) — AG-UI skipped or the collector crashed. Cannot trust a clean signal.", - offendingFiles: [], - }; - } - if (postFixCollectorExit === 2 || postFixCriticalCount > 0) { - return { - resolved: false, - reason: PredicateReason.STILL_DIRTY, - detail: - "Post-fix drift collector still reports critical drift " + - `(exit ${postFixCollectorExit}, criticalCount ${postFixCriticalCount}). The fix did not resolve the drift.`, - offendingFiles: [], - }; - } - if (postFixCollectorExit !== 0) { - // Any other non-zero exit is an unrecognized collector state — fail closed. - return { - resolved: false, - reason: PredicateReason.COLLECTOR_INFRA, - detail: `Post-fix drift collector returned an unexpected exit code (${postFixCollectorExit}). Failing closed.`, - offendingFiles: [], - }; - } - - // ---- Classify the changed-file set. ----------------------------------- - const targets = sanctionedTargets(report); - const productionFiles = changedFiles.filter(isProductionFile); - const gameableLegFiles = changedFiles.filter(isGameableLeg); - const suppressionFiles = changedFiles.filter(isSuppressionSurface); - - // ---- Signal 3a (suppression surface): ALWAYS block, standalone or paired. - - // Editing the schema/allowlist or a *.drift.ts assertion actively SILENCES the - // detector — never a valid fix, even alongside a production change. Distinct - // reason so the workflow's Slack alert names "silenced the detector". - if (suppressionFiles.length > 0) { - return { - resolved: false, - reason: PredicateReason.SUPPRESSION_SUSPECTED, - detail: - "Fix edited the triangulation schema/allowlist or a *.drift.ts assertion " + - `(${suppressionFiles.join(", ")}) — silencing the drift detector is never a valid fix. Needs human review.`, - offendingFiles: suppressionFiles, - }; - } - - // ---- Signal 3b (gameable leg): ALWAYS block, independent of production. - - // FIX #1 (HEADLINE) + FIX #2 — a legitimate auto-remediation updates the mock - // BUILDER to match the SDK; it NEVER edits a comparison/SDK/harness leg (those - // change only on a deliberate human vendored-SDK bump). So the presence of ANY - // gameable-leg file blocks REGARDLESS of how many production files also - // changed. This closes the WS-2b hybrid cheat (relax sdk-shapes.ts + one - // trivial on-target production edit) that the old "only check legs when - // productionFiles===0" logic passed straight to auto-merge. voice-models.ts is - // dual-classified (harness + legit target) and blocks here (fix #2 precedence). - // - // Reason split (both distinct, both NEEDS-HUMAN in the workflow): - // • leg edit WITH a production change → SUPPRESSION_SUSPECTED — the WS-2b - // hybrid, the dangerous auto-merge vector. - // • leg edit with NO production change → COMPARISON_LEG_ONLY — a pure - // relaxation, no mock fix even attempted. - if (gameableLegFiles.length > 0) { - if (productionFiles.length > 0) { - return { - resolved: false, - reason: PredicateReason.SUPPRESSION_SUSPECTED, - detail: - "Fix edited a gameable comparison/SDK/harness leg " + - `(${gameableLegFiles.join(", ")}) ALONGSIDE a production change — relaxing a leg is never a valid ` + - "drift fix (a real fix updates the mock builder, not the leg). The WS-2b hybrid cheat. Needs human review.", - offendingFiles: gameableLegFiles, - }; - } - return { - resolved: false, - reason: PredicateReason.COMPARISON_LEG_ONLY, - detail: - "Fix changed ONLY comparison/SDK/harness-leg files " + - `(${gameableLegFiles.join(", ")}) with no production mock-builder change — ` + - "this relaxes the drift detector instead of fixing the mock. The exact cheat this gate blocks.", - offendingFiles: gameableLegFiles, - }; - } - - // ---- ALLOWLIST GATE (round-2 CR F1/F2/F3 — the inversion): EVERY changed file - // must be on the allowlist. The suppression + gameable-leg branches above give - // the KNOWN gameable legs their own LOUD, specific reasons; this gate catches - // EVERYTHING ELSE that is not sanctioned — package.json / lockfiles / manifests - // / config, a drift-dir `*.test.ts`, a non-drift `__tests__` file, an imported - // sub-fixture the report did not name, or any unrecognized path. A denylist of - // "known legs" leaks these; an allowlist does not (default-deny). This is what - // closes the in-diff vectors (F3) and the drift-dir `.test.ts` gap. - const unsanctionedFiles = changedFiles.filter((f) => !isAllowlisted(f, targets)); - if (unsanctionedFiles.length > 0) { - return { - resolved: false, - reason: PredicateReason.UNSANCTIONED_CHANGE, - detail: - "Fix changed files that are NOT on the sanctioned allowlist " + - `(${unsanctionedFiles.join(", ")}) — a real drift fix touches only production mock-builder ` + - "source and the fixture targets the report named. Any other change (deps/config/manifests, " + - "unrelated tests, unnamed fixtures) could game the collector. Fail-closed, needs human review.", - offendingFiles: unsanctionedFiles, - }; - } - - // ---- Signal 2: at least one PRODUCTION mock-builder change is present. -- - // The module invariant (Signal 2 in the header) is that a genuine drift fix - // ALWAYS changes at least one production mock-builder file (`src/**` excluding - // `src/__tests__/`) — that is the only place the mock output is produced. A run - // that changed ZERO production files cannot be a real fix, even if it edited a - // report-named fixture target (the canary model-registry.ts case): a - // fixture-target-only change (e.g. adding a model id to the model-list fixture - // with NO production/builder change) is NOT independently verifiable — the - // re-collect's clean signal is derived from the same fixture the change touched, - // so a clean post-fix report there means only "the fixture agrees with itself", - // not "the mock was fixed". Fix #F2 (round-4): require >=1 production change for - // RESOLVED, unconditionally. A fixture-target-only diff is routed to - // needs-human (NO_PRODUCTION_CHANGE) rather than auto-resolved — matching the - // docstring invariant, which the earlier `onTargetFiles`-satisfies-it logic - // violated (it let a canary fixture-only edit reach resolved:true). - const onTargetFiles = changedFiles.filter((f) => targets.has(f)); - if (productionFiles.length === 0) { - return { - resolved: false, - reason: PredicateReason.NO_PRODUCTION_CHANGE, - detail: - "Fix changed zero production mock-builder files — a real drift fix always updates the " + - "production mock builder (src/** outside src/__tests__/). A fixture-target-only change " + - "(e.g. a model-list fixture edit with no builder change) is not independently verifiable " + - "(the re-collect reads the same fixture) and is routed to human review, not auto-resolved.", - offendingFiles: [], - }; - } - - // ---- Signal 3 (on-target): the change must land on a file the report named as - // a fix target. A production change to an UNNAMED file (a shared helper) MAY be - // a legitimate fix, so it WARNS and blocks (distinct from a cheat). - // - // FIX #3 — an EMPTY sanctioned-target set is fail-closed, not a free pass. An - // empty target set means the report could not name where to fix the drift — - // route to human rather than rubber-stamp an unverifiable change. - if (targets.size === 0) { - return { - resolved: false, - reason: PredicateReason.PRODUCTION_CHANGE_OFF_TARGET, - detail: - "Drift report named ZERO sanctioned fix targets (no builderFile/typesFile) — cannot verify the " + - `change (${productionFiles.join(", ")}) landed where the drift lives. Fail-closed, needs human review.`, - offendingFiles: productionFiles, - }; - } - if (onTargetFiles.length === 0) { - return { - resolved: false, - reason: PredicateReason.PRODUCTION_CHANGE_OFF_TARGET, - detail: - "Change did not touch any file the drift report named as a fix target " + - `(changed: ${productionFiles.join(", ")}; sanctioned: ${[...targets].join(", ")}). ` + - "May be a legitimate shared-helper fix — needs human review.", - offendingFiles: productionFiles, - }; - } - - // ---- All signals satisfied. ------------------------------------------- - return { - resolved: true, - reason: PredicateReason.RESOLVED, - detail: - "Drift genuinely resolved: post-fix collector clean, " + - `the change landed on a report-named fix target (${onTargetFiles.join(", ")}), and every ` + - "changed file is on the sanctioned allowlist (production source + report-named fixture targets only).", - offendingFiles: [], - }; -} - -// --------------------------------------------------------------------------- -// CLI wrapper -// --------------------------------------------------------------------------- - -interface CliArgs { - reportPath: string; - postFixReportPath: string; - postFixExit: number; - /** Explicit changed files, or null to derive from git. */ - changedFiles: string[] | null; -} - -/** Parse argv (without node/script) into CliArgs. Throws PredicateConfigError. */ -export function parseCliArgs(argv: string[]): CliArgs { - let reportPath: string | null = null; - let postFixReportPath: string | null = null; - let postFixExit: number | null = null; - const changedFiles: string[] = []; - let sawChangedFlag = false; - - for (let idx = 0; idx < argv.length; idx++) { - const arg = argv[idx]; - const next = argv[idx + 1]; - switch (arg) { - case "--report": - if (!next) throw new PredicateConfigError("--report requires a path argument"); - reportPath = next; - idx++; - break; - case "--post-fix-report": - if (!next) throw new PredicateConfigError("--post-fix-report requires a path argument"); - postFixReportPath = next; - idx++; - break; - case "--post-fix-exit": { - if (next === undefined) - throw new PredicateConfigError("--post-fix-exit requires a numeric argument"); - // FIX #6 — fail CLOSED on an empty/whitespace value. `Number("")` and - // `Number(" ")` are both 0, which Number.isInteger accepts, so a missing - // recollect output (`--post-fix-exit ""`) would masquerade as a clean - // exit 0 and be trusted as authoritative-clean. Reject it explicitly. - if (next.trim() === "") { - throw new PredicateConfigError( - "--post-fix-exit is empty/whitespace — a missing collector exit code must fail closed, not be treated as clean exit 0", - ); - } - const parsed = Number(next); - if (!Number.isInteger(parsed)) { - throw new PredicateConfigError(`--post-fix-exit must be an integer, got "${next}"`); - } - postFixExit = parsed; - idx++; - break; - } - case "--changed-file": - if (!next) throw new PredicateConfigError("--changed-file requires a path argument"); - sawChangedFlag = true; - changedFiles.push(next); - idx++; - break; - default: - throw new PredicateConfigError(`Unknown argument: ${arg}`); - } - } - - if (!reportPath) throw new PredicateConfigError("--report is required"); - if (!postFixReportPath) throw new PredicateConfigError("--post-fix-report is required"); - if (postFixExit === null) throw new PredicateConfigError("--post-fix-exit is required"); - - return { - reportPath, - postFixReportPath, - postFixExit, - changedFiles: sawChangedFlag ? changedFiles : null, - }; -} - -/** Minimal drift-report read + shape validation. Throws PredicateConfigError. */ -export function readReport(path: string): DriftReport { - if (!existsSync(path)) { - throw new PredicateConfigError(`Drift report not found at ${path}`); - } - let parsed: unknown; - try { - parsed = JSON.parse(readFileSync(path, "utf-8")); - } catch (err: unknown) { - throw new PredicateConfigError( - `Drift report at ${path} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`, - ); - } - if ( - !parsed || - typeof parsed !== "object" || - !Array.isArray((parsed as Record).entries) - ) { - throw new PredicateConfigError( - `Drift report at ${path} has invalid structure: expected { entries: [...] }`, - ); - } - // FIX #6 — align this (previously loose) validator with the stricter - // fix-drift.ts:readDriftReport: a report missing/carrying a non-string - // `timestamp` is a corrupt/truncated collector run and must fail-closed rather - // than be silently trusted as a clean "drift is gone" signal. NOTE: an EMPTY - // `entries: []` is intentionally ACCEPTED — that is exactly what the collector - // emits when no drift remains (the legitimate clean signal); the trust anchor - // for "clean" is the collector EXIT CODE plus fix #1's always-block-on-leg-edit - // rule, not a non-empty entries array. - if (typeof (parsed as Record).timestamp !== "string") { - throw new PredicateConfigError( - `Drift report at ${path} is missing a string "timestamp" — corrupt/truncated report, failing closed`, - ); - } - - // F3 — ENTRY-LEVEL validation, aligned with fix-drift.ts:readDriftReport. The - // predicate reads `entry.builderFile` / `entry.typesFile` (sanctionedTargets) - // and `entry.diffs` (countCriticalDiffs); a structurally-valid report whose - // entries are malformed at those fields would otherwise throw a bare TypeError - // deep inside classification, caught only by the outer runCli try/catch and - // surfaced as an UNNAMED config-error. Validate here so every malformed shape - // fails-closed with a DISTINCT, named PredicateConfigError → CONFIG_ERROR. - const entries = (parsed as { entries: unknown[] }).entries; - for (let i = 0; i < entries.length; i++) { - const entry = entries[i] as Record | null | undefined; - if (!entry || typeof entry !== "object") { - throw new PredicateConfigError(`Drift report at ${path} entry[${i}] is not an object`); - } - if (typeof entry.builderFile !== "string" || entry.builderFile === "") { - throw new PredicateConfigError( - `Drift report at ${path} entry[${i}] has a missing/empty "builderFile" — cannot derive the sanctioned target set, failing closed`, - ); - } - if (entry.typesFile !== null && typeof entry.typesFile !== "string") { - throw new PredicateConfigError( - `Drift report at ${path} entry[${i}] "typesFile" must be a string or null, failing closed`, - ); - } - if (!Array.isArray(entry.diffs)) { - throw new PredicateConfigError( - `Drift report at ${path} entry[${i}] is missing a "diffs" array — cannot score criticalCount, failing closed`, - ); - } - } - return parsed as DriftReport; -} - -/** - * Count critical diffs in a (post-fix) report. Belt-and-suspenders against a - * collector exit code that disagrees with the report contents. - */ -export function countCriticalDiffs(report: DriftReport): number { - return report.entries.reduce( - (sum, e) => sum + e.diffs.filter((d) => d.severity === "critical").length, - 0, - ); -} - -/** - * Parse a `git status --porcelain` line into a file path. Handles quoted paths - * and rename notation. Kept in sync with fix-drift.ts:parsePorcelainLine. - */ -export function parsePorcelainLine(line: string): string { - let path = line.slice(3).trim(); - const arrowIdx = path.indexOf(" -> "); - if (arrowIdx !== -1) path = path.slice(arrowIdx + 4); - if (path.startsWith('"') && path.endsWith('"')) path = path.slice(1, -1); - return path; -} - -/** - * Changed files from `git status --porcelain`. `-c core.quotePath=false` keeps - * non-ASCII paths verbatim (UTF-8) rather than C-quoted/octal-escaped, so a leg - * path with a special character is not mangled into a non-matching spelling - * before classification (round-2 CR slot-1/slot-2 path finding). - */ -export function gitChangedFiles(): string[] { - const out = execSync("git -c core.quotePath=false status --porcelain", { - encoding: "utf-8", - }).trimEnd(); - return out.split("\n").filter(Boolean).map(parsePorcelainLine); -} - -/** - * FIX #4 — AUTHORITATIVE changed-file set. The git working tree is the single - * source of truth for what actually changed. An explicit `--changed-file` list - * is only a hint, and a supplied list that OMITS a relaxed leg would BLIND the - * predicate (the exact WS-2b vector, from the other side). So when a list is - * supplied we cross-check it against git as a set and fail-closed - * (PredicateConfigError → exit 2) on ANY disagreement — a missing file (leg - * hidden) OR an extra file (phantom) both mean the caller's view diverges from - * ground truth and the verdict cannot be trusted. When no list is supplied we - * use the git set directly. - */ -export function crossCheckChangedFiles(explicit: string[] | null, git: string[]): string[] { - if (explicit === null) return git; - const gitSet = new Set(git); - const explicitSet = new Set(explicit); - const missing = git.filter((f) => !explicitSet.has(f)); // git has it, list omits it - const extra = explicit.filter((f) => !gitSet.has(f)); // list has it, git does not - if (missing.length > 0 || extra.length > 0) { - throw new PredicateConfigError( - "--changed-file list disagrees with the git working tree (fail-closed): " + - `${missing.length > 0 ? `omitted by the list: ${missing.join(", ")}; ` : ""}` + - `${extra.length > 0 ? `not present in git: ${extra.join(", ")}` : ""}`.trim(), - ); - } - return explicit; -} - -/** - * Run the predicate from CLI args. Returns the process exit code and prints - * `detail` (and offending files for LOUD reasons) to stdout/stderr. - */ -export function runCli(argv: string[]): number { - let args: CliArgs; - try { - args = parseCliArgs(argv); - } catch (err: unknown) { - const msg = err instanceof Error ? err.message : String(err); - console.error(`CONFIG_ERROR: ${msg}`); - console.log(`reason=${PredicateReason.CONFIG_ERROR}`); - return REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]; - } - - let report: DriftReport; - let postFixReport: DriftReport; - try { - report = readReport(args.reportPath); - postFixReport = readReport(args.postFixReportPath); - } catch (err: unknown) { - const msg = err instanceof Error ? err.message : String(err); - console.error(`CONFIG_ERROR: ${msg}`); - console.log(`reason=${PredicateReason.CONFIG_ERROR}`); - return REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]; - } - - // FIX #4 — always derive the AUTHORITATIVE changed set from git, and - // cross-check any supplied --changed-file list against it (fail-closed on - // mismatch) so a leg-omitting list cannot blind the predicate. - let changedFiles: string[]; - try { - changedFiles = crossCheckChangedFiles(args.changedFiles, gitChangedFiles()); - } catch (err: unknown) { - const msg = err instanceof Error ? err.message : String(err); - console.error(`CONFIG_ERROR: ${msg}`); - console.log(`reason=${PredicateReason.CONFIG_ERROR}`); - return REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]; - } - - // FIX #8 — a post-fix report that structurally passes readReport (has a - // string timestamp + an entries array) can still be malformed at the entry - // level (e.g. an entry missing its `diffs` array), which would make - // countCriticalDiffs throw a bare TypeError. evaluateDriftResolved can also - // throw PredicateConfigError from canonicalizePath (a repo-root-escaping or - // absolute changed-file path). Catch both here and map to a NAMED CONFIG_ERROR - // so the human gets a named cause instead of an uncaught stacktrace with an - // empty reason= line. - let verdict: PredicateResult; - try { - verdict = evaluateDriftResolved({ - changedFiles, - report, - postFixCollectorExit: args.postFixExit, - postFixCriticalCount: countCriticalDiffs(postFixReport), - }); - } catch (err: unknown) { - const msg = err instanceof Error ? err.message : String(err); - console.error(`CONFIG_ERROR: unable to score the drift reports: ${msg}`); - console.log(`reason=${PredicateReason.CONFIG_ERROR}`); - return REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]; - } - - if (verdict.resolved) { - console.log(verdict.detail); - } else { - console.error(`DRIFT NOT RESOLVED [${verdict.reason}]: ${verdict.detail}`); - if (verdict.offendingFiles.length > 0) { - console.error(`Offending files: ${verdict.offendingFiles.join(", ")}`); - } - } - // Emit a machine-readable reason line for the workflow to capture. - console.log(`reason=${verdict.reason}`); - return REASON_EXIT_CODE[verdict.reason]; -} - -// --------------------------------------------------------------------------- -// Entry-point guard (mirrors drift-report-collector.ts:isDirectRun) -// --------------------------------------------------------------------------- - -function isDirectRun(): boolean { - const entry = process.argv[1]; - if (!entry) return false; - try { - return fileURLToPath(import.meta.url) === resolve(entry); - } catch { - return false; - } -} - -if (isDirectRun()) { - process.exit(runCli(process.argv.slice(2))); -} diff --git a/scripts/drift-sync-check.ts b/scripts/drift-sync-check.ts new file mode 100644 index 00000000..80768d61 --- /dev/null +++ b/scripts/drift-sync-check.ts @@ -0,0 +1,330 @@ +/// + +/** + * Drift Sync Check — the deterministic REPLACEMENT for the 916-line LLM + * anti-cheat predicate (`drift-success-predicate.ts`, spec §3/§6). + * + * `drift-sync.ts` (C2) never freewrites a fix — it only ever performs one of + * two deterministic, data-only edits: (a) a zero-reference deprecation + * removal in `model-registry.ts`, or (b) drop a needs-human dedup note file + * under `drift-proposals/`. Because the SYNC path can no longer produce an + * arbitrary diff, verifying it is "real" no longer needs adversarial-intent + * modeling or TS-diff parsing (the predicate's whole reason for being 916 + * lines) — it only needs three trivial, mechanical assertions: + * + * 1. CHANGED-FILE ALLOWLIST — every file the sync touched is either the + * model-registry DATA file or a `drift-proposals/` note file. Anything + * else (detector source, predicate, test harness, *.drift.ts, schema, + * sdk-shapes, CI workflow, ...) fails closed. + * 2. CHECKSUM-PIN RE-ASSERT — P0's `logic-pin.test.ts` must still be green + * after the sync. Being on the allowlist above does NOT exempt + * `model-registry.ts` from this: a sync that mutated a frozen surface + * inside that file (familySet, NON_MODEL_TOKENS, PREVIEW_FAMILY, ...) + * is caught here even though the file itself was "allowed" to change. + * 3. CLEAN RE-COLLECT — a fresh drift-report-collector run reports zero + * residual critical diffs, so a sync that claims to resolve drift but + * didn't is never waved through. + * + * No LLM, no model call, no heuristic scoring — every one of the three gates + * above is a plain data check. A sync that fails any of them is NOT resolved + * and no PR opens (mirrors the predicate's fail-closed contract, spec §3). + * + * C5 only ADDS this script + its test. Wiring it into `fix-drift.yml` in place + * of the "Assert drift truly resolved" step, and deleting + * `drift-success-predicate.ts`, is C3's job. + */ + +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { resolve } from "node:path"; + +import { getChangedFiles } from "./drift-sync.js"; +import type { DriftReport } from "./drift-types.js"; + +// --------------------------------------------------------------------------- +// Reasons / exit codes +// --------------------------------------------------------------------------- + +export enum SyncCheckReason { + OK = "ok", + OFF_ALLOWLIST_CHANGE = "off-allowlist-change", + PIN_CHECK_FAILED = "pin-check-failed", + RESIDUAL_CRITICAL_DRIFT = "residual-critical-drift", + CONFIG_ERROR = "config-error", +} + +export const REASON_EXIT_CODE: Record = { + [SyncCheckReason.OK]: 0, + [SyncCheckReason.OFF_ALLOWLIST_CHANGE]: 20, + [SyncCheckReason.PIN_CHECK_FAILED]: 21, + [SyncCheckReason.RESIDUAL_CRITICAL_DRIFT]: 22, + [SyncCheckReason.CONFIG_ERROR]: 2, +}; + +/** Fail-closed config error (missing report, unreadable output, etc). */ +export class SyncCheckConfigError extends Error {} + +// --------------------------------------------------------------------------- +// (1) Changed-file allowlist — DATA surfaces only. +// --------------------------------------------------------------------------- + +/** + * The ONLY file a sync may edit directly: the model-registry DATA file + * (`includeFamilies`/`excludeFamilies` literal entries). It also hosts P0's + * frozen logic surfaces — see the pin re-assert in gate (2), which still + * blocks an edit here that touches one of those surfaces. + */ +const ALLOWED_EXACT_FILES: ReadonlySet = new Set(["src/__tests__/drift/model-registry.ts"]); + +/** + * Needs-human dedup note files (C2) live under this prefix — never a code + * file, always a plain artifact recording a genuinely-new family alert. + */ +const ALLOWED_PREFIXES: readonly string[] = ["drift-proposals/"]; + +/** True when `file` is on the sync's data-only allowlist. */ +export function isAllowedSyncFile(file: string): boolean { + if (ALLOWED_EXACT_FILES.has(file)) return true; + return ALLOWED_PREFIXES.some((prefix) => file.startsWith(prefix)); +} + +/** Return the subset of `changedFiles` that is NOT on the allowlist. */ +export function checkChangedFileAllowlist(changedFiles: string[]): string[] { + return changedFiles.filter((file) => !isAllowedSyncFile(file)); +} + +// --------------------------------------------------------------------------- +// (2) Checksum-pin re-assert. +// --------------------------------------------------------------------------- + +/** The exact test file P0 froze the classification logic in. Single source of truth. */ +const LOGIC_PIN_TEST = "src/__tests__/drift/logic-pin.test.ts"; + +export interface CommandResult { + status: number; + output: string; +} + +/** Run `file args...`, capturing stdout+stderr and the real exit status (never throws). */ +export function runCommand(file: string, args: string[]): CommandResult { + try { + const output = execFileSync(file, args, { + encoding: "utf-8", + stdio: ["ignore", "pipe", "pipe"], + }); + return { status: 0, output }; + } catch (err: unknown) { + const e = err as { status?: number | null; stdout?: string; stderr?: string }; + const output = [e.stdout, e.stderr].filter(Boolean).join("\n"); + return { status: e.status ?? 1, output }; + } +} + +export interface PinCheckResult { + ok: boolean; + output: string; +} + +/** + * Re-assert P0's checksum freeze after a sync by spawning vitest directly + * against `logic-pin.test.ts` — the SAME test file, not a re-implementation + * of its hashing, so there is exactly one source of truth for "frozen". + */ +export function runPinCheck( + runner: (file: string, args: string[]) => CommandResult = runCommand, +): PinCheckResult { + const { status, output } = runner("pnpm", ["exec", "vitest", "run", LOGIC_PIN_TEST]); + return { ok: status === 0, output }; +} + +// --------------------------------------------------------------------------- +// (3) Clean re-collect. +// --------------------------------------------------------------------------- + +const DEFAULT_RECOLLECT_OUT = "drift-report.sync-check.json"; + +/** Count `severity === "critical"` diffs across every entry of a report. */ +export function countCriticalDiffs(report: DriftReport): number { + return report.entries.reduce( + (sum, entry) => sum + entry.diffs.filter((d) => d.severity === "critical").length, + 0, + ); +} + +/** + * Run a fresh `drift-report-collector.ts` pass and read back its report. Throws + * `SyncCheckConfigError` (fail-closed) if the collector did not produce a + * readable report — a missing/corrupt post-sync report must never be treated + * as an implicit "clean". + */ +export function recollect( + runner: (file: string, args: string[]) => CommandResult = runCommand, + outPath: string = resolve(DEFAULT_RECOLLECT_OUT), +): DriftReport { + runner("npx", ["tsx", "scripts/drift-report-collector.ts", "--out", outPath]); + if (!existsSync(outPath)) { + throw new SyncCheckConfigError(`Clean re-collect did not produce a report at ${outPath}`); + } + let parsed: unknown; + try { + parsed = JSON.parse(readFileSync(outPath, "utf-8")); + } catch (err: unknown) { + throw new SyncCheckConfigError( + `Post-sync report at ${outPath} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`, + ); + } + if ( + !parsed || + typeof parsed !== "object" || + !Array.isArray((parsed as Record).entries) + ) { + throw new SyncCheckConfigError( + `Post-sync report at ${outPath} has invalid structure: expected { entries: [...] }`, + ); + } + return parsed as DriftReport; +} + +// --------------------------------------------------------------------------- +// The check (composition of gates 1-3). +// --------------------------------------------------------------------------- + +export interface SyncCheckDeps { + getChangedFiles: () => string[]; + runPinCheck: () => PinCheckResult; + recollect: () => DriftReport; +} + +export interface EvaluateSyncCheckOptions { + /** + * Run gate-1 (allowlist) + gate-2 (pin) but SKIP gate-3 (the live + * re-collect). Used by the sync core for a run that applied a mechanical + * registry edit but ALSO deferred a family to a human: a fresh collector run + * would still (correctly) report that deferred family as residual critical + * drift, so gate-3 is not a meaningful full-resolution check for such a run + * and — left on — would wrongly revert the valid mechanical edit. Gate-1 and + * gate-2 remain in force: the edit is still proven data-only with the frozen + * classification logic intact. + */ + skipRecollect?: boolean; +} + +export interface SyncCheckVerdict { + ok: boolean; + reason: SyncCheckReason; + detail: string; + offendingFiles: string[]; +} + +/** + * Evaluate the three deterministic gates in order, short-circuiting on the + * first failure (fail-closed — no gate is skipped when an earlier one could + * have already caught the problem, but there is no reason to pay for a live + * re-collect once the changed-file/pin gates already refused). + */ +export function evaluateSyncCheck( + deps: SyncCheckDeps, + opts: EvaluateSyncCheckOptions = {}, +): SyncCheckVerdict { + const changedFiles = deps.getChangedFiles(); + const offendingFiles = checkChangedFileAllowlist(changedFiles); + if (offendingFiles.length > 0) { + return { + ok: false, + reason: SyncCheckReason.OFF_ALLOWLIST_CHANGE, + detail: `Sync touched file(s) outside the data-only allowlist: ${offendingFiles.join(", ")}`, + offendingFiles, + }; + } + + const pin = deps.runPinCheck(); + if (!pin.ok) { + return { + ok: false, + reason: SyncCheckReason.PIN_CHECK_FAILED, + detail: `Frozen classification-logic checksum pin failed after sync — a pinned rule moved:\n${pin.output}`, + offendingFiles: [], + }; + } + + // gate-3 (live re-collect) is skipped for a run that ALSO deferred a family + // to a human — see EvaluateSyncCheckOptions.skipRecollect. + if (opts.skipRecollect) { + return { + ok: true, + reason: SyncCheckReason.OK, + detail: + "drift-sync-check passed: changed files are data-only, classification pins intact " + + "(live re-collect skipped — this run also deferred a family to a human)", + offendingFiles: [], + }; + } + + const report = deps.recollect(); + const criticalCount = countCriticalDiffs(report); + if (criticalCount > 0) { + return { + ok: false, + reason: SyncCheckReason.RESIDUAL_CRITICAL_DRIFT, + detail: `Clean re-collect after sync still reports ${criticalCount} critical diff(s) — sync did not resolve the drift`, + offendingFiles: [], + }; + } + + return { + ok: true, + reason: SyncCheckReason.OK, + detail: + "drift-sync-check passed: changed files are data-only, classification pins intact, clean re-collect", + offendingFiles: [], + }; +} + +// --------------------------------------------------------------------------- +// CLI +// --------------------------------------------------------------------------- + +const REAL_DEPS: SyncCheckDeps = { + getChangedFiles, + runPinCheck: () => runPinCheck(), + recollect: () => recollect(), +}; + +/** Run the check against real deps, printing a machine-readable `reason=` line. */ +export function runCli(deps: SyncCheckDeps = REAL_DEPS): number { + let verdict: SyncCheckVerdict; + try { + verdict = evaluateSyncCheck(deps); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + console.error(`CONFIG_ERROR: ${msg}`); + console.log(`reason=${SyncCheckReason.CONFIG_ERROR}`); + return REASON_EXIT_CODE[SyncCheckReason.CONFIG_ERROR]; + } + + if (verdict.ok) { + console.log(verdict.detail); + } else { + console.error(`DRIFT SYNC NOT RESOLVED [${verdict.reason}]: ${verdict.detail}`); + if (verdict.offendingFiles.length > 0) { + console.error(`Offending files: ${verdict.offendingFiles.join(", ")}`); + } + } + console.log(`reason=${verdict.reason}`); + return REASON_EXIT_CODE[verdict.reason]; +} + +function isDirectRun(): boolean { + const entry = process.argv[1]; + if (!entry) return false; + try { + return fileURLToPath(import.meta.url) === resolve(entry); + } catch { + return false; + } +} + +if (isDirectRun()) { + process.exit(runCli()); +} diff --git a/scripts/drift-sync.ts b/scripts/drift-sync.ts new file mode 100644 index 00000000..8699dbd2 --- /dev/null +++ b/scripts/drift-sync.ts @@ -0,0 +1,1413 @@ +/// + +/** + * Drift Sync — reusable git / branch / commit / PR plumbing, PLUS the + * deterministic (zero-LLM) model-family sync core. + * + * This module holds the provider-agnostic, LLM-agnostic building blocks the + * drift-remediation pipeline uses to read a report, shape a fix into a PR, and + * (below) mechanically sync model-family churn: shell/exec helpers, drift-report + * reading/validation, changed-file parsing, version-bump + CHANGELOG authoring, + * PR-body construction, and the gated commit-file partition. + * + * C3 (delete-freewriter-predicate-rewire): these functions were originally + * extracted VERBATIM from `scripts/fix-drift.ts` by C1 (behavior-preserving + * move). `fix-drift.ts` and `scripts/drift-success-predicate.ts` — the LLM + * freewriter invocation and its 916-line anti-cheat predicate — have since been + * DELETED entirely (there is no arbitrary/free-form code generation left in the + * drift-remediation pipeline to police), so `readDriftReport` and + * `isProductionFile` (previously re-exported/imported from those two modules + * respectively) now live here permanently as this module's own exports. + */ + +import { execSync, execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFileSync, writeFileSync, existsSync, mkdirSync, rmSync } from "node:fs"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +import ts from "typescript"; + +import { evaluateSyncCheck, runPinCheck, recollect } from "./drift-sync-check.js"; +import type { DriftReport, DriftSeverity } from "./drift-types.js"; + +import { normalizeModelFamily } from "../src/__tests__/drift/model-family.js"; +import { + includeFamilies, + isClassifiedFamily, + NON_MODEL_TOKENS, +} from "../src/__tests__/drift/model-registry.js"; +import { isFamilyStillReferenced } from "../src/__tests__/drift/deprecation-detector.js"; +import { + InfraError, + isInfraSkip, + listOpenAIModels, + listAnthropicModels, + listGeminiModels, +} from "../src/__tests__/drift/providers.js"; + +// --------------------------------------------------------------------------- +// Drift report reading + validation (moved from the deleted fix-drift.ts). +// --------------------------------------------------------------------------- + +const VALID_SEVERITIES: ReadonlySet = new Set(["critical", "warning", "info"]); + +/** + * Read + structurally validate a `drift-report.json` (produced by + * `drift-report-collector.ts`). Still consumed by `drift-slack-summary.ts` (the + * `test-drift.yml` "Summarize drift for Slack" step), independent of the + * drift-remediation path. + */ +export function readDriftReport(path: string): DriftReport { + if (!existsSync(path)) { + throw new Error(`Drift report not found at ${path}`); + } + const raw = readFileSync(path, "utf-8"); + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (err: unknown) { + throw new Error( + `Drift report at ${path} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`, + ); + } + if ( + !parsed || + typeof parsed !== "object" || + !Array.isArray((parsed as Record).entries) + ) { + throw new Error(`Drift report at ${path} has invalid structure: expected { entries: [...] }`); + } + if (typeof (parsed as Record).timestamp !== "string") { + throw new Error('Drift report missing "timestamp" field'); + } + const report = parsed as DriftReport; + + // Validate individual entry fields to catch malformed reports early + for (let i = 0; i < report.entries.length; i++) { + const entry = report.entries[i]; + if (!entry || typeof entry.provider !== "string" || !entry.provider) { + throw new Error(`Drift report entry[${i}] missing required "provider" field`); + } + if (!entry.builderFile || typeof entry.builderFile !== "string") { + throw new Error(`Drift report entry[${i}] (${entry.provider}) missing "builderFile"`); + } + if ( + !Array.isArray(entry.builderFunctions) || + entry.builderFunctions.length === 0 || + !entry.builderFunctions.every((f: unknown) => typeof f === "string") + ) { + throw new Error( + `Drift report entry[${i}] (${entry.provider}) "builderFunctions" must be non-empty string array`, + ); + } + if (!entry.scenario || typeof entry.scenario !== "string") { + throw new Error(`Drift report entry[${i}] (${entry.provider}) missing "scenario"`); + } + if (!entry.sdkShapesFile || typeof entry.sdkShapesFile !== "string") { + throw new Error(`Drift report entry[${i}] (${entry.provider}) missing "sdkShapesFile"`); + } + if (entry.typesFile !== null && typeof entry.typesFile !== "string") { + throw new Error( + `Drift report entry[${i}] (${entry.provider}) "typesFile" must be string or null`, + ); + } + if (!Array.isArray(entry.diffs)) { + throw new Error(`Drift report entry[${i}] (${entry.provider}) missing "diffs" array`); + } + for (let j = 0; j < entry.diffs.length; j++) { + const diff = entry.diffs[j]; + if (!diff.path || typeof diff.path !== "string") { + throw new Error(`Drift report entry[${i}].diffs[${j}]: missing "path"`); + } + if (!diff.issue || typeof diff.issue !== "string") { + throw new Error(`Drift report entry[${i}].diffs[${j}]: missing "issue"`); + } + if (typeof diff.expected !== "string") { + throw new Error(`Drift report entry[${i}].diffs[${j}]: missing "expected"`); + } + if (typeof diff.real !== "string") { + throw new Error(`Drift report entry[${i}].diffs[${j}]: missing "real"`); + } + if (typeof diff.mock !== "string") { + throw new Error(`Drift report entry[${i}].diffs[${j}]: missing "mock"`); + } + if (!VALID_SEVERITIES.has(diff.severity)) { + throw new Error( + `Drift report entry[${i}].diffs[${j}]: invalid severity "${diff.severity}" — expected one of: ${[...VALID_SEVERITIES].join(", ")}`, + ); + } + } + } + + return report; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +export function todayStamp(): string { + return new Date().toISOString().slice(0, 10); +} + +/** GitHub hard limit on PR/issue body length. */ +export const GH_BODY_MAX = 65536; +/** Safety margin below the hard limit. */ +export const GH_BODY_SAFE_MAX = 60000; + +/** + * Truncate `body` to at most `max` characters. When truncation occurs, the + * HEAD of the body (summary/diffs) is preserved and the tail is replaced with a + * marker. The full detail is always available as a workflow artifact. + */ +export function truncateBody(body: string, max: number = GH_BODY_SAFE_MAX): string { + // Never exceed the hard GitHub limit, even if a caller passes a larger max. + const effectiveMax = Math.min(max, GH_BODY_MAX); + if (body.length <= effectiveMax) return body; + const marker = + "\n\n---\n" + + "_Body truncated to fit GitHub's 65536-character limit. " + + "Full drift report is attached as the `drift-report` workflow artifact._\n"; + // When the budget can't even fit the marker, hard-cut instead of overflowing. + if (effectiveMax <= marker.length) return body.slice(0, effectiveMax); + return body.slice(0, effectiveMax - marker.length) + marker; +} + +/** + * Format an exec error into a human-readable Error object. + * Includes exit status, signal, and stderr when available. + * Logs stderr to console.error as a side effect when present. + */ +function formatExecError(cmd: string, err: unknown): Error { + const e = err as { status?: number; signal?: string; stderr?: string | Buffer }; + const detail = [ + e.status !== undefined ? `exit ${e.status}` : null, + e.signal ? `signal ${e.signal}` : null, + e.stderr ? String(e.stderr).trim() : null, + ] + .filter(Boolean) + .join(", "); + const msg = `Command failed: ${cmd}${detail ? ` (${detail})` : ""}`; + if (e.stderr) console.error(msg); + return new Error(msg); +} + +/** + * Run a shell command and return its trimmed stdout. + * + * WARNING: This function passes the command string directly to a shell. + * NEVER call it with interpolated values — use execFileSafe() for commands + * with dynamic arguments. + */ +export function exec(cmd: string): string { + try { + return execSync(cmd, { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trimEnd(); + } catch (err: unknown) { + throw formatExecError(cmd, err); + } +} + +/** + * Run a command safely without shell interpolation. + * Use this for all commands with dynamic arguments. + */ +export function execFileSafe(file: string, args: string[]): void { + try { + execFileSync(file, args, { stdio: "inherit" }); + } catch (err: unknown) { + throw formatExecError(`${file} ${args.join(" ")}`, err); + } +} + +/** + * Map builder source files to the corresponding section names in the + * write-fixtures skill documentation. Used to flag which skill sections + * may need updating when a drift fix changes a builder's output format. + */ +export const BUILDER_TO_SKILL_SECTION: Record = { + "src/responses.ts": "Responses API", + "src/messages.ts": "Claude Messages", + "src/gemini.ts": "Gemini", + "src/bedrock.ts": "Bedrock", + "src/bedrock-converse.ts": "Bedrock", + "src/embeddings.ts": "Embeddings", + "src/ollama.ts": "Ollama", + "src/cohere.ts": "Cohere", + "src/ws-realtime.ts": "OpenAI Realtime WebSocket", + "src/ws-responses.ts": "OpenAI Responses WebSocket", + "src/ws-gemini-live.ts": "Gemini Live WebSocket", + "src/helpers.ts": "OpenAI Chat Completions", + "src/gemini-interactions.ts": "Gemini Interactions", + "src/agui-types.ts": "AG-UI Events", + "src/agui-handler.ts": "AG-UI Events", +}; + +/** + * Given a list of changed file paths, return the unique skill section names + * that correspond to modified builder files. Returns an empty array when + * no builder files map to a known skill section. + */ +export function affectedSkillSections(changedFiles: string[]): string[] { + const sections = new Set(); + for (const file of changedFiles) { + const section = BUILDER_TO_SKILL_SECTION[file]; + if (section) sections.add(section); + } + return [...sections].sort(); +} + +export function readFileIfExists(path: string): string | null { + if (!existsSync(path)) return null; + return readFileSync(path, "utf-8"); +} + +// --------------------------------------------------------------------------- +// Changed-file parsing +// --------------------------------------------------------------------------- + +/** + * Parse a single line from `git status --porcelain` output into a file path. + * Handles quoted paths (special characters) and rename notation (old -> new). + */ +export function parsePorcelainLine(line: string): string { + let path = line.slice(3).trim(); + // Handle renames first: "old -> new" → take the new path + const arrowIdx = path.indexOf(" -> "); + if (arrowIdx !== -1) { + path = path.slice(arrowIdx + 4); + } + // Then strip quotes (git quotes paths with special characters) + if (path.startsWith('"') && path.endsWith('"')) { + path = path.slice(1, -1); + } + return path; +} + +/** + * Return the list of changed files from `git status --porcelain`. + */ +export function getChangedFiles(): string[] { + return exec("git status --porcelain").split("\n").filter(Boolean).map(parsePorcelainLine); +} + +// --------------------------------------------------------------------------- +// Version bump + CHANGELOG +// --------------------------------------------------------------------------- + +export function patchBumpVersion(): string { + const pkgPath = resolve("package.json"); + const pkg = JSON.parse(readFileSync(pkgPath, "utf-8")) as { + version: string; + description?: string; + [key: string]: unknown; + }; + const parts = pkg.version.split(".").map(Number); + if (parts.length !== 3 || parts.some(isNaN)) { + throw new Error(`Cannot patch-bump non-standard version: ${pkg.version}`); + } + parts[2] += 1; + const newVersion = parts.join("."); + pkg.version = newVersion; + + // Sync description with README subtitle + syncDescriptionFromReadme(pkg); + + writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + "\n", "utf-8"); + return newVersion; +} + +/** Keep package.json description in sync with the README subtitle. */ +function syncDescriptionFromReadme(pkg: { description?: string; [key: string]: unknown }): void { + const readmePath = resolve("README.md"); + try { + const readme = readFileSync(readmePath, "utf-8"); + // The description is the first non-empty, non-heading, non-badge, non-video line + const lines = readme.split("\n"); + for (const line of lines) { + const trimmed = line.trim(); + if ( + !trimmed || + trimmed.startsWith("#") || + trimmed.startsWith("[![") || + trimmed.startsWith("![") || + trimmed.startsWith("[") || + trimmed.startsWith("http") + ) { + continue; + } + // Found the subtitle — strip markdown formatting + const clean = trimmed.replace(/[*_`]/g, "").replace(/\s+/g, " ").trim(); + if (clean && clean !== pkg.description) { + pkg.description = clean; + } + break; + } + } catch (err: unknown) { + if ((err as NodeJS.ErrnoException).code !== "ENOENT") { + console.warn("Could not sync description from README:", err); + } + } +} + +export function addChangelogEntry(report: DriftReport, version: string): void { + const changelogPath = resolve("CHANGELOG.md"); + const existing = readFileIfExists(changelogPath) ?? ""; + + const providerSummaries = report.entries.map((entry) => { + const fields = entry.diffs.map((d) => d.path).join(", "); + return `- ${entry.provider} (${entry.scenario}): ${fields}`; + }); + + const newEntry = [ + `## ${version}`, + "", + "### Patch Changes", + "", + "- Auto-remediate API drift:", + ...providerSummaries.map((s) => ` ${s}`), + "", + ].join("\n"); + + // Insert after the title line (any line starting with "# ") + const titleMatch = existing.match(/^# .+\n/); + if (titleMatch) { + const titleLine = titleMatch[0]; + const rest = existing.slice(titleLine.length); + writeFileSync(changelogPath, titleLine + "\n" + newEntry + rest, "utf-8"); + } else { + writeFileSync(changelogPath, newEntry + "\n" + existing, "utf-8"); + } +} + +// --------------------------------------------------------------------------- +// PR body construction +// --------------------------------------------------------------------------- + +export function buildPrBody( + report: DriftReport, + changedFiles?: string[], + verdictDetail?: string, +): string { + const providers: string[] = []; + const diffs: string[] = []; + + for (const entry of report.entries) { + providers.push(`- ${entry.provider}: ${entry.scenario}`); + for (const diff of entry.diffs) { + diffs.push(`- \`${diff.path}\`: ${diff.issue}`); + } + } + + const reportJson = JSON.stringify(report, null, 2); + + const sections: string[] = [ + "## Summary", + "", + "Auto-generated drift remediation.", + "", + // Human-approval backstop (WS-2): this PR was auto-FILTERED by the + // drift-success predicate but is NOT auto-merged. A human must review CI + + // this diff + the verdict below and merge. The predicate is a strong filter, + // not a provable merge gate (the re-collect is not independent of the fix — + // WS-2b), so the merge decision stays with a human. + "> **Needs human review + merge.** This drift-fix PR was opened by the", + "> automated pipeline after the drift-success predicate passed. It is NOT", + "> auto-merged — review CI, the diff, and the verdict below, then merge.", + "", + "### Drift-success predicate verdict", + "", + verdictDetail ? `RESOLVED — ${verdictDetail}` : "RESOLVED.", + "", + "### Providers affected", + ...providers, + "", + "### Diffs fixed", + ...diffs, + "", + ]; + + // Flag skill sections that may need review based on which builders changed + const skillSections = changedFiles ? affectedSkillSections(changedFiles) : []; + if (skillSections.length > 0) { + sections.push( + "### Skill documentation", + "", + `The following write-fixtures skill sections may need review after these builder changes:`, + ...skillSections.map((s) => `- ${s}`), + "", + ); + } + + sections.push( + "## Drift Report", + "", + "
", + "Full drift report JSON", + "", + "```json", + reportJson, + "```", + "", + "
", + ); + + return truncateBody(sections.join("\n")); +} + +// --------------------------------------------------------------------------- +// Gated commit-file partition +// --------------------------------------------------------------------------- + +/** + * True when `file` is a PRODUCTION mock-builder source file: under `src/` but + * NOT under `src/__tests__/`. Moved here from the deleted + * `drift-success-predicate.ts` — `gatedCommitFiles` below is its only consumer. + */ +export function isProductionFile(file: string): boolean { + return file.startsWith("src/") && !file.startsWith("src/__tests__/"); +} + +/** + * The GATED commit groups for a resolved fix. Given the canonicalized + * changed-file set and a sanctioned-target set (e.g. `union(entry.builderFile, + * entry.typesFile≠null)` from a drift report), partition into the ONLY groups + * a caller is permitted to stage: + * - `builderFiles` — production mock-builder source (always allowlisted). + * - `testFiles` — ONLY report-named fixture targets under src/__tests__/ + * (any other test file is unclassified and must never be + * staged). + * `stragglers` is every canonicalized changed file that is NOT in one of those + * gated groups — a caller must never `git add` it. + */ +export function gatedCommitFiles( + changedFiles: string[], + sanctioned: ReadonlySet, +): { + builderFiles: string[]; + testFiles: string[]; + stragglers: string[]; +} { + const builderFiles = changedFiles.filter(isProductionFile); + const testFiles = changedFiles.filter((f) => f.startsWith("src/__tests__/") && sanctioned.has(f)); + const gated = new Set([...builderFiles, ...testFiles]); + const stragglers = changedFiles.filter((f) => !gated.has(f)); + return { builderFiles, testFiles, stragglers }; +} + +// ============================================================================= +// C2: deterministic sync core — the DATA-only, ZERO-LLM replacement for the +// freewriter's DECISION role on the model-churn (add/deprecate) leg. +// +// This mechanically applies live `/models` churn to the frozen registry +// (`src/__tests__/drift/model-registry.ts`) with no model call and no +// free-form code generation: +// +// - DEPRECATION (classified − live, via a mirror of C4's +// `detectDeprecatedFamilies`): a family aimock mocks that a healthy live +// listing no longer contains. +// * zero-reference (nothing in aimock's own source still names it) → +// mechanical, comment-marked removal from `includeFamilies`. +// * still-referenced → NEVER auto-removed. Routed to a human via a +// family-keyed dedup note file under `drift-proposals/`. +// - ADDITION (a genuinely new, UNCLASSIFIED family — matches no include, +// exclude, `-preview`, or Gemma rule): NEVER auto-classified. Routed to a +// human via the same dedup note-file mechanism. Only once a human edits +// that note's `Decision:` line to `include` does the NEXT sync run +// mechanically add the family literal — still zero-LLM (a plain text +// marker a human wrote is not code generation), and still never silent +// (the registry is never touched without an explicit, reviewed decision +// recorded in the diff). +// +// Both mechanical edits are gated behind C5's `drift-sync-check` (the +// allowlist + pin re-assert + clean re-collect) before anything is kept; a +// failing gate reverts the edit rather than leaving a broken write behind. +// +// NOTE ON WHY THIS DOES NOT IMPORT `models.drift.ts` DIRECTLY: that module +// (like every `*.drift.ts` file) imports `{ describe, it, expect }` from +// "vitest", and merely EVALUATING that import outside an active vitest worker +// throws ("Vitest failed to access its internal state") — confirmed +// empirically; this is exactly why `drift-report-collector.ts` shells out to +// `npx vitest run` for the `*.drift.ts` suites instead of importing them. +// Since this script runs as a plain CI step (`npx tsx scripts/drift-sync.ts`), +// it cannot import `models.drift.ts`. The two pure predicates below +// (`detectDeprecatedFamiliesForSync`, `unclassifiedFamiliesForSync`) mirror +// C4's `detectDeprecatedFamilies`/`unclassifiedFamilies` byte-for-byte against +// the SAME underlying data/logic modules (`model-registry.ts`, +// `model-family.ts`, `deprecation-detector.ts` — none of which import +// vitest), so the two call sites cannot silently diverge in RESULT even +// though they are textually separate: P0's checksum pin freezes the +// `isClassifiedFamily`/`normalizeModelFamily` primitives both copies compose, +// and `models.drift.ts`'s own vitest suite exercises its copy directly. +// ============================================================================= + +export type Provider = "openai" | "anthropic" | "gemini"; + +// --------------------------------------------------------------------------- +// Mirrored classification predicates (see module doc above). +// --------------------------------------------------------------------------- + +export interface DeprecationCandidate { + provider: Provider; + family: string; + stillReferenced: boolean; +} + +export type DeprecationCheckResult = + | { status: "skipped"; reason: string } + | { status: "checked"; candidates: DeprecationCandidate[] }; + +/** Mirror of `models.drift.ts`'s `detectDeprecatedFamilies` — see module doc. */ +export function detectDeprecatedFamiliesForSync( + liveModelIds: string[], + provider: Provider, + opts: { + isReferenced?: (family: string, provider: Provider) => boolean; + minListingSize?: number; + } = {}, +): DeprecationCheckResult { + const classified = includeFamilies[provider]; + const floor = opts.minListingSize ?? classified.size; + + if (liveModelIds.length === 0 || liveModelIds.length < floor) { + return { + status: "skipped", + reason: + `live /models listing too short to trust for ${provider} ` + + `(${liveModelIds.length} raw id(s), need >= ${floor} — the number of ` + + `families aimock mocks for this provider) — never mass-removing off a ` + + `truncated or empty listing`, + }; + } + + const liveFamilies = new Set(liveModelIds.map((id) => normalizeModelFamily(id, provider))); + const missing = [...classified].filter((family) => !liveFamilies.has(family)).sort(); + const isReferenced = opts.isReferenced ?? isFamilyStillReferenced; + + return { + status: "checked", + candidates: missing.map((family) => ({ + provider, + family, + stillReferenced: isReferenced(family, provider), + })), + }; +} + +/** Mirror of `models.drift.ts`'s `unclassifiedFamilies` — see module doc. */ +export function unclassifiedFamiliesForSync(modelIds: string[], provider: Provider): string[] { + const unclassified = new Set(); + for (const id of modelIds) { + const family = normalizeModelFamily(id, provider); + if (isClassifiedFamily(family, provider)) continue; + if (NON_MODEL_TOKENS.has(family) || NON_MODEL_TOKENS.has(id)) continue; + unclassified.add(family); + } + return [...unclassified].sort(); +} + +// --------------------------------------------------------------------------- +// Needs-human dedup note files (`drift-proposals/`). +// --------------------------------------------------------------------------- + +/** Must match `scripts/drift-sync-check.ts`'s `ALLOWED_PREFIXES`. */ +export const DRIFT_PROPOSALS_DIR = "drift-proposals"; + +export type ProposalKind = + | "new-family" + | "still-referenced-deprecation" + | "registry-structural-mismatch"; +export type ProposalDecision = "pending" | "include"; + +/** Family-keyed dedup path — re-firing the same alert always resolves to the SAME path. */ +export function proposalNoteRelPath( + provider: Provider, + family: string, + kind: ProposalKind, +): string { + const slug = family.replace(/[^a-z0-9.-]+/gi, "-"); + const kindSlug = + kind === "new-family" + ? "new-family" + : kind === "registry-structural-mismatch" + ? "structural-mismatch" + : "deprecated-referenced"; + return `${DRIFT_PROPOSALS_DIR}/${provider}-${slug}-${kindSlug}.md`; +} + +/** Parse the note's `Decision:` line. Defaults to "pending" (fail-closed — never infers approval). */ +export function parseProposalDecision(noteText: string): ProposalDecision { + const m = noteText.match(/^Decision:\s*(\S+)/m); + return m && m[1].toLowerCase() === "include" ? "include" : "pending"; +} + +export function renderProposalNote( + provider: Provider, + family: string, + kind: ProposalKind, + detail: string, + stamp: string, +): string { + const title = + kind === "new-family" + ? "New / unclassified model family" + : kind === "registry-structural-mismatch" + ? "Registry structural mismatch — mechanical edit could not be applied" + : "Deprecated-but-still-referenced model family"; + const lines = [ + `# ${title}: ${family}`, + "", + `Provider: ${provider}`, + `Detected: ${stamp}`, + "Status: NEEDS HUMAN REVIEW", + "", + detail, + "", + ]; + if (kind === "new-family") { + lines.push( + "## Decision", + "", + "Decision: pending", + "", + ); + } + return lines.join("\n"); +} + +// --------------------------------------------------------------------------- +// Mechanical registry edits — AST-LOCATED (via the real TypeScript parser, not +// a hand-rolled regex/lexer scan) then applied as a single-line text splice. +// The parser is used only to unambiguously find the exact line of the exact +// string-literal element inside `includeFamilies[provider]` / +// `excludeFamilies[provider]`'s array literal (or the array's closing-bracket +// line, for an insert) — the mutation itself is a trivial whole-line +// replace/insert, never a partial-token or multi-line reformat, so it cannot +// silently mangle an adjacent grouping comment or a sibling entry. +// --------------------------------------------------------------------------- + +export const MODEL_REGISTRY_REL_PATH = "src/__tests__/drift/model-registry.ts"; + +type RegistrySetName = "includeFamilies" | "excludeFamilies"; + +interface FamilySetLocation { + /** family literal text -> 0-based source line index of that literal's own line. */ + elementLines: Map; + /** 0-based source line index of the array's closing `]` line. */ + arrayEndLine: number; + /** Indentation captured from an existing element line (fallback for inserts into an empty array). */ + elementIndent: string; +} + +/** Locate `exportName[provider]`'s array literal inside `model-registry.ts` source text. */ +function locateFamilySetArray( + sourceText: string, + exportName: RegistrySetName, + provider: Provider, +): FamilySetLocation | null { + const sf = ts.createSourceFile("model-registry.ts", sourceText, ts.ScriptTarget.Latest, true); + let target: ts.ArrayLiteralExpression | undefined; + + for (const stmt of sf.statements) { + if (!ts.isVariableStatement(stmt)) continue; + for (const decl of stmt.declarationList.declarations) { + if (!ts.isIdentifier(decl.name) || decl.name.text !== exportName) continue; + if (!decl.initializer || !ts.isObjectLiteralExpression(decl.initializer)) continue; + for (const prop of decl.initializer.properties) { + if (!ts.isPropertyAssignment(prop)) continue; + const key = ts.isIdentifier(prop.name) + ? prop.name.text + : ts.isStringLiteral(prop.name) + ? prop.name.text + : null; + if (key !== provider) continue; + const init = prop.initializer; + if ( + ts.isCallExpression(init) && + init.arguments.length >= 2 && + ts.isArrayLiteralExpression(init.arguments[1]) + ) { + target = init.arguments[1]; + } + } + } + } + if (!target) return null; + + const elementLines = new Map(); + let elementIndent = " "; + for (const el of target.elements) { + if (ts.isStringLiteral(el)) { + const { line, character } = sf.getLineAndCharacterOfPosition(el.getStart(sf)); + elementLines.set(el.text, line); + elementIndent = " ".repeat(character); + } + } + const { line: arrayEndLine } = sf.getLineAndCharacterOfPosition(target.getEnd()); + return { elementLines, arrayEndLine, elementIndent }; +} + +export interface RegistryEditResult { + changed: boolean; + text: string; + detail: string; + /** + * True when the AST locator could NOT find the target array literal in + * `model-registry.ts` (structural mismatch). This is distinct from a benign + * no-op (family already-absent for a remove / already-present for an add): + * a locator miss means a real add/remove could not be applied and MUST be + * routed to a human — never collapsed into a silent, clean no-op (G#1). + */ + locatorMiss?: boolean; +} + +/** Comment-marked removal of `family` from `exportName[provider]`. Never touches any other line. */ +export function removeFamilyLiteralInSource( + sourceText: string, + exportName: RegistrySetName, + provider: Provider, + family: string, + reasonComment: string, +): RegistryEditResult { + const loc = locateFamilySetArray(sourceText, exportName, provider); + if (!loc) { + return { + changed: false, + text: sourceText, + detail: `could not locate ${exportName}.${provider} array in model-registry.ts (structural mismatch — routing to human)`, + locatorMiss: true, + }; + } + const lineIdx = loc.elementLines.get(family); + if (lineIdx === undefined) { + return { + changed: false, + text: sourceText, + detail: `"${family}" is not present in ${exportName}.${provider} — nothing to remove`, + }; + } + const lines = sourceText.split("\n"); + const indentMatch = lines[lineIdx].match(/^(\s*)/); + const indent = indentMatch ? indentMatch[1] : loc.elementIndent; + lines[lineIdx] = `${indent}// ${reasonComment}`; + return { + changed: true, + text: lines.join("\n"), + detail: `removed "${family}" from ${exportName}.${provider} (comment-marked)`, + }; +} + +/** Mechanical, comment-marked addition of `family` to `exportName[provider]`. */ +export function addFamilyLiteralInSource( + sourceText: string, + exportName: RegistrySetName, + provider: Provider, + family: string, + reasonComment: string, +): RegistryEditResult { + const loc = locateFamilySetArray(sourceText, exportName, provider); + if (!loc) { + return { + changed: false, + text: sourceText, + detail: `could not locate ${exportName}.${provider} array in model-registry.ts (structural mismatch — routing to human)`, + locatorMiss: true, + }; + } + if (loc.elementLines.has(family)) { + return { + changed: false, + text: sourceText, + detail: `"${family}" is already present in ${exportName}.${provider}`, + }; + } + const lines = sourceText.split("\n"); + const newLine = `${loc.elementIndent}"${family}", // ${reasonComment}`; + lines.splice(loc.arrayEndLine, 0, newLine); + return { + changed: true, + text: lines.join("\n"), + detail: `added "${family}" to ${exportName}.${provider} (comment-marked)`, + }; +} + +// --------------------------------------------------------------------------- +// Orchestration — pure over injected deps (fully testable with no real fs/git/network I/O). +// --------------------------------------------------------------------------- + +export enum SyncCoreReason { + OK_NO_CHURN = "ok-no-churn", + OK_APPLIED = "ok-applied", + NEEDS_HUMAN = "needs-human", + GATE_FAILED = "gate-failed", +} + +export interface ProviderChurnInput { + provider: Provider; + /** Live `/models` ids, or `null` if the live check was skipped (no key / infra error). */ + liveModelIds: string[] | null; + skipReason?: string; +} + +export interface SyncCheckResultLike { + ok: boolean; + reason: string; + detail: string; +} + +export interface SyncCoreDeps { + isReferenced?: (family: string, provider: Provider) => boolean; + readRegistrySource: () => string; + writeRegistrySource: (text: string) => void; + readProposalNote: (relPath: string) => string | null; + writeProposalNote: (relPath: string, text: string) => void; + /** + * Run C5's drift-sync-check gate. `opts.skipRecollect` runs gate-1 + * (allowlist) + gate-2 (pin) but SKIPS gate-3 (the live re-collect) — used + * when this run applied a mechanical edit AND also deferred a family to a + * human, so a fresh collector run would still (correctly) see that deferred + * family as residual critical drift. + */ + runSyncCheck: (opts?: { skipRecollect?: boolean }) => SyncCheckResultLike; + /** Revert every file in `relPaths` (e.g. `git checkout -- `) after a failed gate. */ + revertFiles: (relPaths: string[]) => void; + now?: () => Date; +} + +export type FamilyAction = + | "removed" + | "added" + | "needs-human-new-family" + | "needs-human-still-referenced" + | "needs-human-structural-mismatch" + | "no-op"; + +export interface FamilyOutcome { + provider: Provider; + family: string; + action: FamilyAction; + detail: string; +} + +export interface SyncCoreOutcome { + ok: boolean; + reason: SyncCoreReason; + detail: string; + outcomes: FamilyOutcome[]; + skipped: { provider: Provider; reason: string }[]; +} + +/** Read-or-create a dedup note (write only on first sighting — re-fire never spams a duplicate). */ +function ensureProposalNote( + deps: SyncCoreDeps, + path: string, + render: () => string, + touched: Set, +): string | null { + const existing = deps.readProposalNote(path); + if (existing !== null) return existing; + deps.writeProposalNote(path, render()); + touched.add(path); + return null; +} + +/** + * The C2 sync core. Mechanically applies model churn (deprecation + + * genuinely-new-family) across every provider input to the frozen registry, + * gated behind C5's `drift-sync-check` before any edit is kept. NEVER invokes + * an LLM and NEVER generates free-form code — every mutation is one of the + * two mechanical text edits above, or a note file. + */ +export function runDriftSyncCore( + inputs: ProviderChurnInput[], + deps: SyncCoreDeps, +): SyncCoreOutcome { + const now = deps.now ?? (() => new Date()); + const stamp = now().toISOString().slice(0, 10); + + const outcomes: FamilyOutcome[] = []; + const skipped: { provider: Provider; reason: string }[] = []; + const touchedFiles = new Set(); + + let registrySource = deps.readRegistrySource(); + let registryChanged = false; + + for (const input of inputs) { + if (input.liveModelIds === null) { + skipped.push({ + provider: input.provider, + reason: input.skipReason ?? "live listing unavailable", + }); + continue; + } + + // --- Deprecation half: classified − live (C4's algorithm, mirrored). --- + const dep = detectDeprecatedFamiliesForSync(input.liveModelIds, input.provider, { + isReferenced: deps.isReferenced, + }); + if (dep.status === "skipped") { + skipped.push({ provider: input.provider, reason: dep.reason }); + } else { + for (const cand of dep.candidates) { + if (!cand.stillReferenced) { + const edit = removeFamilyLiteralInSource( + registrySource, + "includeFamilies", + cand.provider, + cand.family, + `REMOVED ${stamp} (drift-sync): "${cand.family}" no longer in live /models, zero-reference`, + ); + if (edit.changed) { + registrySource = edit.text; + registryChanged = true; + outcomes.push({ + provider: cand.provider, + family: cand.family, + action: "removed", + detail: edit.detail, + }); + } else if (edit.locatorMiss) { + // G#1: the AST locator could not find includeFamilies[provider] in + // model-registry.ts. A real deprecation could not be applied — this + // must route to a human, NEVER collapse into a silent clean no-op. + const smPath = proposalNoteRelPath( + cand.provider, + cand.family, + "registry-structural-mismatch", + ); + ensureProposalNote( + deps, + smPath, + () => + renderProposalNote( + cand.provider, + cand.family, + "registry-structural-mismatch", + `A zero-reference deprecation was detected for "${cand.family}" but drift-sync ` + + `could not locate the includeFamilies.${cand.provider} array literal in ` + + `${MODEL_REGISTRY_REL_PATH} — the registry's structure changed. A human must ` + + `apply the removal (or fix the locator).`, + stamp, + ), + touchedFiles, + ); + outcomes.push({ + provider: cand.provider, + family: cand.family, + action: "needs-human-structural-mismatch", + detail: `${edit.detail} (${smPath})`, + }); + } else { + outcomes.push({ + provider: cand.provider, + family: cand.family, + action: "no-op", + detail: edit.detail, + }); + } + } else { + const notePath = proposalNoteRelPath( + cand.provider, + cand.family, + "still-referenced-deprecation", + ); + ensureProposalNote( + deps, + notePath, + () => + renderProposalNote( + cand.provider, + cand.family, + "still-referenced-deprecation", + "This family no longer appears in the live /models listing, but aimock's " + + "own source still references it (builders, DEFAULT_MODELS, or fixtures). " + + "drift-sync never silently removes a still-referenced family.", + stamp, + ), + touchedFiles, + ); + outcomes.push({ + provider: cand.provider, + family: cand.family, + action: "needs-human-still-referenced", + detail: `"${cand.family}" is deprecated but still referenced in source — routed to human (${notePath})`, + }); + } + } + } + + // --- Addition half: genuinely new / unclassified family. --- + for (const family of unclassifiedFamiliesForSync(input.liveModelIds, input.provider)) { + const notePath = proposalNoteRelPath(input.provider, family, "new-family"); + const existing = ensureProposalNote( + deps, + notePath, + () => + renderProposalNote( + input.provider, + family, + "new-family", + "This model family appeared in a live /models listing but matches no " + + "classification rule (include, exclude, -preview, gemma). drift-sync never " + + "silently classifies a new family.", + stamp, + ), + touchedFiles, + ); + const decision = existing !== null ? parseProposalDecision(existing) : "pending"; + if (decision === "include") { + const edit = addFamilyLiteralInSource( + registrySource, + "includeFamilies", + input.provider, + family, + `ADDED ${stamp} (drift-sync): approved via ${notePath}`, + ); + if (edit.changed) { + registrySource = edit.text; + registryChanged = true; + outcomes.push({ + provider: input.provider, + family, + action: "added", + detail: edit.detail, + }); + } else if (edit.locatorMiss) { + // G#1: a human-approved add could not be applied because the AST + // locator could not find includeFamilies[provider]. Route to a human + // rather than reporting a silent clean no-op. + const smPath = proposalNoteRelPath( + input.provider, + family, + "registry-structural-mismatch", + ); + ensureProposalNote( + deps, + smPath, + () => + renderProposalNote( + input.provider, + family, + "registry-structural-mismatch", + `An approved addition of "${family}" could not be applied: drift-sync could not ` + + `locate the includeFamilies.${input.provider} array literal in ` + + `${MODEL_REGISTRY_REL_PATH} — the registry's structure changed. A human must ` + + `apply the addition (or fix the locator).`, + stamp, + ), + touchedFiles, + ); + outcomes.push({ + provider: input.provider, + family, + action: "needs-human-structural-mismatch", + detail: `${edit.detail} (${smPath})`, + }); + } else { + outcomes.push({ + provider: input.provider, + family, + action: "no-op", + detail: edit.detail, + }); + } + } else { + outcomes.push({ + provider: input.provider, + family, + action: "needs-human-new-family", + detail: `"${family}" is unclassified — routed to human (${notePath})`, + }); + } + } + } + + if (registryChanged) { + touchedFiles.add(MODEL_REGISTRY_REL_PATH); + } + + const anyNeedsHuman = outcomes.some((o) => o.action.startsWith("needs-human-")); + + if (touchedFiles.size === 0) { + return { + ok: !anyNeedsHuman, + reason: anyNeedsHuman ? SyncCoreReason.NEEDS_HUMAN : SyncCoreReason.OK_NO_CHURN, + detail: anyNeedsHuman + ? "no new mechanical edit this run — one or more families still need human review" + : "no model churn detected — nothing to sync", + outcomes, + skipped, + }; + } + + // D-M1: a NOTE-ONLY run (a fresh needs-human note, but NO registry edit) must + // NOT be gated behind the live re-collect. Gate-3 re-runs the collector, + // which STILL sees the un-actioned family this run just routed to a human as + // residual critical drift, and would revert the note it just wrote — + // defeating the whole route-to-human protocol (this is the MOST COMMON case: + // a genuinely new family). There is no registry mutation to re-verify, so + // keep the note and report NEEDS_HUMAN directly, without any gate call. + if (!registryChanged) { + return { + ok: false, + reason: SyncCoreReason.NEEDS_HUMAN, + detail: + "needs-human note(s) written this run — routed to human without a live re-collect " + + "(no registry edit to re-verify)", + outcomes, + skipped, + }; + } + + // A mechanical registry edit WAS applied — persist it and gate it. Gate-1 + // (allowlist) and gate-2 (pin) always apply: they cheaply prove the edit + // stayed on the data-only surface and left the frozen classification logic + // intact. Gate-3 (the live re-collect) only makes sense when this run CLAIMS + // to have fully resolved the drift — i.e. no family was simultaneously + // deferred to a human. In a mixed run (a valid removal PLUS a family routed + // to a human), the re-collect would (correctly) still see that deferred + // family as residual drift and would wrongly revert the valid edit (D-M1, + // mixed-run leg), so skip gate-3 and report NEEDS_HUMAN with the edit kept. + deps.writeRegistrySource(registrySource); + const verdict = deps.runSyncCheck({ skipRecollect: anyNeedsHuman }); + if (!verdict.ok) { + deps.revertFiles([...touchedFiles]); + return { + ok: false, + reason: SyncCoreReason.GATE_FAILED, + detail: `drift-sync-check rejected the sync [${verdict.reason}]: ${verdict.detail} — reverted`, + outcomes, + skipped, + }; + } + + return { + ok: !anyNeedsHuman, + reason: anyNeedsHuman ? SyncCoreReason.NEEDS_HUMAN : SyncCoreReason.OK_APPLIED, + detail: anyNeedsHuman + ? "mechanical sync applied (drift-sync-check allowlist + pin passed); one or more " + + "families still need human review" + : "mechanical sync applied and drift-sync-check passed", + outcomes, + skipped, + }; +} + +// --------------------------------------------------------------------------- +// CLI — real deps (live fetch, real fs, real git, C5's real gate). NO LLM, +// no `@anthropic-ai/claude-code` spawn, no free-form code generation anywhere +// in this path. +// --------------------------------------------------------------------------- + +const REGISTRY_ABS_PATH = resolve(MODEL_REGISTRY_REL_PATH); + +const LIVE_MODEL_ENV_KEY: Record = { + openai: "OPENAI_API_KEY", + anthropic: "ANTHROPIC_API_KEY", + gemini: "GOOGLE_API_KEY", +}; + +const LIVE_MODEL_LISTERS: Record Promise> = { + openai: listOpenAIModels, + anthropic: listAnthropicModels, + gemini: listGeminiModels, +}; + +/** Fetch one provider's live `/models` ids, or an honest skip (no key / infra error). */ +export async function fetchProviderChurnInput(provider: Provider): Promise { + const envKey = LIVE_MODEL_ENV_KEY[provider]; + const apiKey = process.env[envKey]; + if (!apiKey) { + return { + provider, + liveModelIds: null, + skipReason: `${envKey} not set — skipping live sync for ${provider}`, + }; + } + try { + const liveModelIds = await LIVE_MODEL_LISTERS[provider](apiKey); + return { provider, liveModelIds }; + } catch (err: unknown) { + if (err instanceof InfraError && isInfraSkip(err.status)) { + return { + provider, + liveModelIds: null, + skipReason: `infra error (status ${err.status}) fetching live /models for ${provider} — never mass-removing off a failed listing`, + }; + } + throw err; + } +} + +/** True when `relPath` is tracked by git (`git ls-files --error-unmatch` exits 0). */ +function isTrackedByGit(relPath: string): boolean { + try { + execFileSync("git", ["ls-files", "--error-unmatch", "--", relPath], { + stdio: ["ignore", "ignore", "ignore"], + }); + return true; + } catch { + return false; + } +} + +/** + * Revert every file in `relPaths` to its pre-sync state after a failed gate. + * + * D-M2: `git checkout -- ` THROWS on a freshly-written UNTRACKED note file + * ("did not match any file(s) known to git") — and when the set mixes tracked + * and untracked paths, a single `git checkout --` of all of them reverts + * NOTHING and throws uncaught, exiting the sync 1. So partition the set: + * `git checkout --` restores tracked files (e.g. the registry edit), and each + * untracked file (a note git never knew) is simply deleted — the correct + * "revert" for a file that did not exist before this run. Never throws on the + * untracked case. + */ +export function revertSyncFiles(relPaths: string[]): void { + if (relPaths.length === 0) return; + const tracked: string[] = []; + const untracked: string[] = []; + for (const p of relPaths) { + (isTrackedByGit(p) ? tracked : untracked).push(p); + } + if (tracked.length > 0) { + execFileSafe("git", ["checkout", "--", ...tracked]); + } + for (const p of untracked) { + rmSync(resolve(p), { force: true }); + } +} + +const REAL_SYNC_CORE_DEPS: SyncCoreDeps = { + readRegistrySource: () => readFileSync(REGISTRY_ABS_PATH, "utf-8"), + writeRegistrySource: (text: string) => writeFileSync(REGISTRY_ABS_PATH, text, "utf-8"), + readProposalNote: (relPath: string) => { + const abs = resolve(relPath); + return existsSync(abs) ? readFileSync(abs, "utf-8") : null; + }, + writeProposalNote: (relPath: string, text: string) => { + const abs = resolve(relPath); + mkdirSync(dirname(abs), { recursive: true }); + writeFileSync(abs, text, "utf-8"); + }, + runSyncCheck: (opts) => { + const verdict = evaluateSyncCheck( + { + getChangedFiles, + runPinCheck: () => runPinCheck(), + recollect: () => recollect(), + }, + { skipRecollect: opts?.skipRecollect }, + ); + return { ok: verdict.ok, reason: verdict.reason, detail: verdict.detail }; + }, + revertFiles: revertSyncFiles, +}; + +/** + * A stable, date-independent identity of a run's changeset, used by the CI + * workflow (`fix-drift.yml`) to de-duplicate PRs across daily re-fires in + * EVERY run shape. + * + * The key is derived from the SORTED set of every non-`no-op` family outcome + * (`:/`) — the mechanical registry edits AND the + * families deferred to a human alike — so it is byte-identical on every re-fire + * of the same underlying drift (same families, same actions), independent of + * the date-stamped comment text inside the registry edit or the CI run id in + * the branch name. + * + * WHY not key on the committed note-file paths alone (the workflow's older + * approach): the D-M1 "mixed run" (a mechanical registry removal of family X + * committed the SAME run a *different* family Y is deferred to a human, Y's + * note already sitting on `main` from a prior run) commits ONLY the registry + * edit — no `drift-proposals/*` file — so a note-path key is EMPTY and the + * dedup is bypassed, re-opening a near-identical PR every daily cron run + * (unbounded PR-spam). The outcome-derived key is non-empty here (it carries + * both `removed:openai/X` and `needs-human-…:gemini/Y`) and identical on every + * re-fire, so the workflow can find the already-open PR and skip. + * + * A 16-hex-char SHA-256 prefix is used as the marker token: fixed-length, so + * two distinct changesets can never be substring-confused in the PR-body + * `contains()` match. Returns `""` when nothing was applied or deferred. + */ +export function computeChangesetKey(outcome: SyncCoreOutcome): string { + const entries = outcome.outcomes + .filter((o) => o.action !== "no-op") + .map((o) => `${o.action}:${o.provider}/${o.family}`) + .sort(); + if (entries.length === 0) return ""; + return createHash("sha256").update(entries.join("\n")).digest("hex").slice(0, 16); +} + +/** Stage + commit exactly the sync core's own touched files (never a catch-all `git add`). */ +function commitSyncChanges(outcome: SyncCoreOutcome): boolean { + const changed = getChangedFiles().filter( + (f) => f === MODEL_REGISTRY_REL_PATH || f.startsWith(`${DRIFT_PROPOSALS_DIR}/`), + ); + if (changed.length === 0) return false; + const applied = outcome.outcomes.filter((o) => o.action === "removed" || o.action === "added"); + const summary = + applied.length > 0 + ? applied.map((o) => `${o.action} ${o.provider}/${o.family}`).join(", ") + : "needs-human note file(s)"; + execFileSafe("git", ["add", ...changed]); + execFileSafe("git", [ + "commit", + "-m", + `fix(drift-sync): mechanical model-family sync (${summary})`, + ]); + return true; +} + +/** Run the full CLI: fetch every provider's live listing, sync, commit. Never invokes an LLM. */ +export async function runDriftSyncCli( + providers: Provider[] = ["openai", "anthropic", "gemini"], +): Promise { + const inputs = await Promise.all(providers.map(fetchProviderChurnInput)); + const outcome = runDriftSyncCore(inputs, REAL_SYNC_CORE_DEPS); + + console.log(outcome.detail); + for (const o of outcome.outcomes) { + console.log(` [${o.action}] ${o.provider}/${o.family}: ${o.detail}`); + } + for (const s of outcome.skipped) { + console.log(` [skipped] ${s.provider}: ${s.reason}`); + } + + if ( + outcome.reason === SyncCoreReason.OK_APPLIED || + outcome.outcomes.some((o) => o.action.startsWith("needs-human-")) + ) { + commitSyncChanges(outcome); + } + + console.log(`reason=${outcome.reason}`); + // Stable, date-independent identity of this run's changeset — the workflow + // greps this to de-dup PRs across daily re-fires in EVERY shape (including + // the mixed run that commits a registry edit but no new note file, where a + // note-path-only key would be empty). See computeChangesetKey. + console.log(`changeset-key=${computeChangesetKey(outcome)}`); + return outcome.ok ? 0 : 1; +} + +function isDirectRun(): boolean { + const entry = process.argv[1]; + if (!entry) return false; + try { + return fileURLToPath(import.meta.url) === resolve(entry); + } catch { + return false; + } +} + +if (isDirectRun()) { + runDriftSyncCli() + .then((code) => process.exit(code)) + .catch((err: unknown) => { + console.error("drift-sync fatal error:", err instanceof Error ? err.message : err); + process.exit(1); + }); +} diff --git a/scripts/drift-types.ts b/scripts/drift-types.ts index c1246aab..448663ab 100644 --- a/scripts/drift-types.ts +++ b/scripts/drift-types.ts @@ -1,7 +1,7 @@ /** * Shared types for the drift remediation pipeline. * - * Used by both drift-report-collector.ts and fix-drift.ts. + * Used by both drift-report-collector.ts and drift-sync.ts. */ /** diff --git a/scripts/fix-drift.ts b/scripts/fix-drift.ts deleted file mode 100644 index 40af57bd..00000000 --- a/scripts/fix-drift.ts +++ /dev/null @@ -1,1229 +0,0 @@ -/// - -/** - * Drift Fix Orchestrator - * - * Reads a drift-report.json (produced by drift-report-collector.ts), constructs - * a structured prompt, and invokes Claude Code CLI to auto-fix the drift. - * - * Modes: - * Default: npx tsx scripts/fix-drift.ts - * PR mode: npx tsx scripts/fix-drift.ts --create-pr - * Issue mode: npx tsx scripts/fix-drift.ts --create-issue - * - * Exit codes: - * 0 — success (or issue created successfully in --create-issue mode) - * 1 — failure - * 2 — critical drift found (drift collector) - * 4 — no source files changed (--create-pr mode, nothing to commit) - * 3 — unhandled error (e.g. bad arguments, missing report, git/gh command failure) - * 124 — Claude Code timed out (default mode) - * In default mode, the exit code is passed through from Claude Code. - * - * In --create-pr mode, the drift-success predicate (drift-success-predicate.ts) - * gates PR creation BEFORE any git add/commit. When it rejects the fix (e.g. a - * fixture-relaxation cheat rather than a real mock change), createPr exits with - * the predicate's reason code instead of opening a PR: - * 10 — NO_PRODUCTION_CHANGE (zero production mock-builder files changed) - * 11 — COMPARISON_LEG_ONLY (only comparison-leg files changed — the cheat) - * 12 — SUPPRESSION_SUSPECTED (allowlist / *.drift.ts assertion edited) - * 13 — STILL_DIRTY (post-fix collector still reports critical drift) - * 14 — QUARANTINE_AFTER_FIX (post-fix collector returned quarantine) - * 15 — COLLECTOR_INFRA (post-fix collector infra failure, OR the - * MANDATORY post-fix args were not supplied) - * 16 — PRODUCTION_CHANGE_OFF_TARGET (production change not in report's target set) - * 17 — UNSANCTIONED_CHANGE (a changed file is not on the allowlist) - * 18 — VERSION_BUMP_FAILED (version bump / CHANGELOG step failed) - * 19 — POST_FIX_PARSE_ERROR (unparseable --post-fix-exit / -report) - * 20 — GIT_PUSH_FAILED (git checkout/add/commit/push failed) - * The legacy exit 4 (no source files changed) is subsumed by 10/11. The - * drift-success predicate is MANDATORY in --create-pr mode: BOTH - * --post-fix-report and --post-fix-exit are required (there is no legacy - * no-post-fix fallback — a missing pair fails closed to COLLECTOR_INFRA rather - * than opening a PR). The real workflow also supplies --report pointing at the - * PINNED pre-fix report (see .github/workflows/fix-drift.yml) so the - * sanctioned-target set cannot be forged by the autofix LLM. - */ - -import { spawn, execSync, execFileSync } from "node:child_process"; -import { readFileSync, writeFileSync, existsSync, unlinkSync } from "node:fs"; -import { resolve } from "node:path"; -import { fileURLToPath } from "node:url"; - -import { - evaluateDriftResolved, - readReport as readPostFixReport, - countCriticalDiffs, - canonicalizePath, - gitChangedFiles, - isProductionFile, - sanctionedTargets, - REASON_EXIT_CODE, - PredicateReason, -} from "./drift-success-predicate.js"; -import type { DriftReport, DriftSeverity } from "./drift-types.js"; - -// --------------------------------------------------------------------------- -// Constants -// --------------------------------------------------------------------------- - -/** 30-minute hard ceiling for the Claude Code subprocess */ -const CLAUDE_TIMEOUT_MS = 30 * 60 * 1000; - -/** Grace period between SIGTERM and SIGKILL */ -const KILL_GRACE_MS = 10_000; - -const VALID_SEVERITIES: ReadonlySet = new Set(["critical", "warning", "info"]); - -/** - * Map builder source files to the corresponding section names in the - * write-fixtures skill documentation. Used to flag which skill sections - * may need updating when a drift fix changes a builder's output format. - */ -export const BUILDER_TO_SKILL_SECTION: Record = { - "src/responses.ts": "Responses API", - "src/messages.ts": "Claude Messages", - "src/gemini.ts": "Gemini", - "src/bedrock.ts": "Bedrock", - "src/bedrock-converse.ts": "Bedrock", - "src/embeddings.ts": "Embeddings", - "src/ollama.ts": "Ollama", - "src/cohere.ts": "Cohere", - "src/ws-realtime.ts": "OpenAI Realtime WebSocket", - "src/ws-responses.ts": "OpenAI Responses WebSocket", - "src/ws-gemini-live.ts": "Gemini Live WebSocket", - "src/helpers.ts": "OpenAI Chat Completions", - "src/gemini-interactions.ts": "Gemini Interactions", - "src/agui-types.ts": "AG-UI Events", - "src/agui-handler.ts": "AG-UI Events", -}; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -export function todayStamp(): string { - return new Date().toISOString().slice(0, 10); -} - -/** GitHub hard limit on PR/issue body length. */ -export const GH_BODY_MAX = 65536; -/** Safety margin below the hard limit. */ -export const GH_BODY_SAFE_MAX = 60000; - -/** - * Truncate `body` to at most `max` characters. When truncation occurs, the - * HEAD of the body (summary/diffs) is preserved and the tail is replaced with a - * marker. The full detail is always available as a workflow artifact. - */ -export function truncateBody(body: string, max: number = GH_BODY_SAFE_MAX): string { - // Never exceed the hard GitHub limit, even if a caller passes a larger max. - const effectiveMax = Math.min(max, GH_BODY_MAX); - if (body.length <= effectiveMax) return body; - const marker = - "\n\n---\n" + - "_Body truncated to fit GitHub's 65536-character limit. " + - "Full drift report is attached as the `drift-report` workflow artifact._\n"; - // When the budget can't even fit the marker, hard-cut instead of overflowing. - if (effectiveMax <= marker.length) return body.slice(0, effectiveMax); - return body.slice(0, effectiveMax - marker.length) + marker; -} - -/** - * Format an exec error into a human-readable Error object. - * Includes exit status, signal, and stderr when available. - * Logs stderr to console.error as a side effect when present. - */ -function formatExecError(cmd: string, err: unknown): Error { - const e = err as { status?: number; signal?: string; stderr?: string | Buffer }; - const detail = [ - e.status !== undefined ? `exit ${e.status}` : null, - e.signal ? `signal ${e.signal}` : null, - e.stderr ? String(e.stderr).trim() : null, - ] - .filter(Boolean) - .join(", "); - const msg = `Command failed: ${cmd}${detail ? ` (${detail})` : ""}`; - if (e.stderr) console.error(msg); - return new Error(msg); -} - -/** - * Run a shell command and return its trimmed stdout. - * - * WARNING: This function passes the command string directly to a shell. - * NEVER call it with interpolated values — use execFileSafe() for commands - * with dynamic arguments. - */ -function exec(cmd: string): string { - try { - return execSync(cmd, { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trimEnd(); - } catch (err: unknown) { - throw formatExecError(cmd, err); - } -} - -/** - * Run a command safely without shell interpolation. - * Use this for all commands with dynamic arguments. - */ -export function execFileSafe(file: string, args: string[]): void { - try { - execFileSync(file, args, { stdio: "inherit" }); - } catch (err: unknown) { - throw formatExecError(`${file} ${args.join(" ")}`, err); - } -} - -/** - * Given a list of changed file paths, return the unique skill section names - * that correspond to modified builder files. Returns an empty array when - * no builder files map to a known skill section. - */ -export function affectedSkillSections(changedFiles: string[]): string[] { - const sections = new Set(); - for (const file of changedFiles) { - const section = BUILDER_TO_SKILL_SECTION[file]; - if (section) sections.add(section); - } - return [...sections].sort(); -} - -export function readFileIfExists(path: string): string | null { - if (!existsSync(path)) return null; - return readFileSync(path, "utf-8"); -} - -export function readDriftReport(path: string): DriftReport { - if (!existsSync(path)) { - throw new Error(`Drift report not found at ${path}`); - } - const raw = readFileSync(path, "utf-8"); - let parsed: unknown; - try { - parsed = JSON.parse(raw); - } catch (err: unknown) { - throw new Error( - `Drift report at ${path} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`, - ); - } - if ( - !parsed || - typeof parsed !== "object" || - !Array.isArray((parsed as Record).entries) - ) { - throw new Error(`Drift report at ${path} has invalid structure: expected { entries: [...] }`); - } - if (typeof (parsed as Record).timestamp !== "string") { - throw new Error('Drift report missing "timestamp" field'); - } - const report = parsed as DriftReport; - - // Validate individual entry fields to catch malformed reports early - for (let i = 0; i < report.entries.length; i++) { - const entry = report.entries[i]; - if (!entry || typeof entry.provider !== "string" || !entry.provider) { - throw new Error(`Drift report entry[${i}] missing required "provider" field`); - } - if (!entry.builderFile || typeof entry.builderFile !== "string") { - throw new Error(`Drift report entry[${i}] (${entry.provider}) missing "builderFile"`); - } - if ( - !Array.isArray(entry.builderFunctions) || - entry.builderFunctions.length === 0 || - !entry.builderFunctions.every((f: unknown) => typeof f === "string") - ) { - throw new Error( - `Drift report entry[${i}] (${entry.provider}) "builderFunctions" must be non-empty string array`, - ); - } - if (!entry.scenario || typeof entry.scenario !== "string") { - throw new Error(`Drift report entry[${i}] (${entry.provider}) missing "scenario"`); - } - if (!entry.sdkShapesFile || typeof entry.sdkShapesFile !== "string") { - throw new Error(`Drift report entry[${i}] (${entry.provider}) missing "sdkShapesFile"`); - } - if (entry.typesFile !== null && typeof entry.typesFile !== "string") { - throw new Error( - `Drift report entry[${i}] (${entry.provider}) "typesFile" must be string or null`, - ); - } - if (!Array.isArray(entry.diffs)) { - throw new Error(`Drift report entry[${i}] (${entry.provider}) missing "diffs" array`); - } - for (let j = 0; j < entry.diffs.length; j++) { - const diff = entry.diffs[j]; - if (!diff.path || typeof diff.path !== "string") { - throw new Error(`Drift report entry[${i}].diffs[${j}]: missing "path"`); - } - if (!diff.issue || typeof diff.issue !== "string") { - throw new Error(`Drift report entry[${i}].diffs[${j}]: missing "issue"`); - } - if (typeof diff.expected !== "string") { - throw new Error(`Drift report entry[${i}].diffs[${j}]: missing "expected"`); - } - if (typeof diff.real !== "string") { - throw new Error(`Drift report entry[${i}].diffs[${j}]: missing "real"`); - } - if (typeof diff.mock !== "string") { - throw new Error(`Drift report entry[${i}].diffs[${j}]: missing "mock"`); - } - if (!VALID_SEVERITIES.has(diff.severity)) { - throw new Error( - `Drift report entry[${i}].diffs[${j}]: invalid severity "${diff.severity}" — expected one of: ${[...VALID_SEVERITIES].join(", ")}`, - ); - } - } - } - - return report; -} - -// --------------------------------------------------------------------------- -// Prompt construction -// --------------------------------------------------------------------------- - -export function buildPrompt(report: DriftReport): string { - const lines: string[] = []; - - lines.push("You are fixing API drift in the aimock mock server."); - lines.push(""); - lines.push("## Workflow"); - lines.push(""); - lines.push("Follow this exact workflow for each drift fix:"); - lines.push(""); - lines.push("1. RED: Confirm the drift test currently fails by running:"); - lines.push(' pnpm test:drift 2>&1 | grep -A5 "DRIFT"'); - lines.push(""); - lines.push("2. Fix the builder function to add/modify the field matching the real API shape."); - lines.push(" Also fix the corresponding builder for the same provider (e.g., if non-streaming"); - lines.push(" text drifted, also fix non-streaming tool call since they share the same message"); - lines.push(" structure)."); - lines.push(""); - lines.push("3. If the builder file uses TypeScript interfaces from src/types.ts, update those."); - lines.push(""); - lines.push("4. Update the SDK shape in src/__tests__/drift/sdk-shapes.ts if the corresponding"); - lines.push(" shape function doesn't include the new field."); - lines.push(""); - lines.push("5. GREEN: Run pnpm test to verify conformance tests pass."); - lines.push(""); - lines.push("6. Run pnpm test:drift to verify drift is resolved."); - lines.push(""); - lines.push("7. Run npx prettier --write on all changed files."); - lines.push(""); - lines.push("8. REFACTOR: Review your changes for unnecessary complexity."); - lines.push(""); - lines.push("## Drift Entries"); - lines.push(""); - - for (let i = 0; i < report.entries.length; i++) { - const entry = report.entries[i]; - lines.push(`DRIFT ${i + 1}: ${entry.provider} — ${entry.scenario}`); - lines.push(` File: ${entry.builderFile}`); - lines.push(` Functions: ${entry.builderFunctions.join(", ")}`); - lines.push(` Types file: ${entry.typesFile ?? "N/A"}`); - lines.push(` SDK shapes: ${entry.sdkShapesFile}`); - lines.push(" Diffs:"); - for (const diff of entry.diffs) { - lines.push(` - [${diff.severity}] ${diff.issue}`); - lines.push(` Path: ${diff.path}`); - lines.push(` SDK type: ${diff.expected}`); - lines.push(` Real API: ${diff.real}`); - lines.push(` Mock: ${diff.mock}`); - } - lines.push(""); - } - - // Add AG-UI specific guidance if any AG-UI entries exist - const hasAgUiDrift = report.entries.some((e) => e.provider === "AG-UI"); - if (hasAgUiDrift) { - lines.push("## AG-UI Schema Drift"); - lines.push(""); - lines.push("For AG-UI drift entries, the fix target is `src/agui-types.ts`."); - lines.push( - "Compare against the canonical source at `../ag-ui/sdks/typescript/packages/core/src/events.ts`.", - ); - lines.push(""); - lines.push("- Add missing event types to the `AGUIEventType` union type."); - lines.push("- Add missing fields to the corresponding `AGUI*Event` interfaces."); - lines.push("- Fix optionality mismatches (required vs optional) to match canonical schemas."); - lines.push( - "- Also update any builder functions in `src/agui-handler.ts` that construct these events.", - ); - lines.push(""); - } - - lines.push("## After all fixes"); - lines.push(""); - lines.push("1. Run the full test suite: pnpm test"); - lines.push("2. Run drift verification: pnpm test:drift"); - lines.push("3. Format: npx prettier --write src/ src/__tests__/"); - lines.push("4. Lint: npx eslint src/ src/__tests__/ --fix"); - - return lines.join("\n"); -} - -// --------------------------------------------------------------------------- -// Claude Code invocation (default mode) -// --------------------------------------------------------------------------- - -/** - * Kill an entire process GROUP by its leader pid. - * - * `spawn(..., { detached: true })` makes the child a group leader whose group - * id equals its pid, so `process.kill(-pid, signal)` signals the child AND all - * of its descendants (e.g. the `npx` wrapper's `@anthropic-ai/claude-code` - * grandchild). Signalling the child pid alone (`child.kill()`) reaches only the - * `npx` wrapper and leaves a wedged grandchild alive to burn the job budget. - * - * ESRCH (group already gone) is a benign "nothing left to kill" and is swallowed - * to `false`. EPERM is DIFFERENT and must NOT be treated as benign: `kill(2)` - * returns EPERM when the target process(es) EXIST but the caller lacks - * permission to signal them — e.g. a grandchild that changed credentials (a - * setuid postinstall) or was re-parented under a remapped container user. Such a - * process can be STILL ALIVE and STILL BURNING the job budget — exactly the - * WS-4 leak this guards against. So on EPERM we log a VISIBLE warning (never - * silently claim success) and attempt a single-PID fallback (`kill(pid)` rather - * than the whole group) in case only the group-leader escaped our permission. - * The job's `timeout-minutes: 30` ceiling remains the ultimate backstop. - * - * Returns true if a group signal was delivered, false if the group was already - * gone (ESRCH) or is present-but-unkillable (EPERM after the fallback attempt). - */ -export function killProcessGroup(pid: number, signal: NodeJS.Signals): boolean { - try { - // Negative pid targets the whole process group led by `pid`. - process.kill(-pid, signal); - return true; - } catch (err) { - const code = (err as NodeJS.ErrnoException).code; - if (code === "ESRCH") { - // The group has already fully exited — nothing left to kill. - return false; - } - if (code === "EPERM") { - // NOT a benign "nothing to kill": the group (or part of it) exists but is - // unkillable by us. Surface it loudly and try a single-PID fallback in - // case only the leader escaped our permission. - console.error( - `WARNING: EPERM signalling process group ${pid} with ${signal} — the group may ` + - "still be ALIVE and unkillable by us (re-credentialed / re-parented child). " + - "Attempting single-PID fallback; the 30-min job ceiling is the final backstop.", - ); - try { - process.kill(pid, signal); - return true; - } catch (fallbackErr) { - const fallbackCode = (fallbackErr as NodeJS.ErrnoException).code; - if (fallbackCode === "ESRCH") { - return false; - } - console.error( - `WARNING: single-PID fallback kill of ${pid} with ${signal} also failed ` + - `(${fallbackCode ?? "unknown"}) — process may leak until the job timeout.`, - ); - return false; - } - } - throw err; - } -} - -/** - * Escalating timeout kill for a detached subprocess: deliver SIGTERM to the - * whole GROUP, then after a grace period escalate to SIGKILL on the GROUP — - * but ONLY if the process has NOT already exited. - * - * The has-exited signal is the caller-supplied `hasExited()` predicate, which - * MUST be backed by the real `close` event (not `child.killed`). Node sets - * `child.killed = true` the instant a signal is DELIVERED, long before the - * process actually exits, so a `!child.killed` guard makes the SIGKILL - * escalation dead code — the original WS-4 defect. Gating on a real exit flag - * is what makes the escalation actually fire against a process that ignores - * SIGTERM. - * - * Returns the grace timer so the caller can cancel it from its `close` handler - * (a clean early exit must not leave a pending SIGKILL escalation queued). - */ -export function scheduleEscalatingKill( - pid: number, - hasExited: () => boolean, - graceMs: number = KILL_GRACE_MS, -): NodeJS.Timeout { - killProcessGroup(pid, "SIGTERM"); - return setTimeout(() => { - if (!hasExited()) { - console.error("Process group did not exit after SIGTERM. Sending SIGKILL to the group..."); - killProcessGroup(pid, "SIGKILL"); - } - }, graceMs); -} - -export function invokeClaudeCode(prompt: string): Promise { - return new Promise((done, reject) => { - const args = [ - "@anthropic-ai/claude-code", - "--print", - "--verbose", - "-p", - prompt, - "--allowedTools", - [ - "Read", - "Edit", - "Write", - "Glob", - "Grep", - "Bash(pnpm test)", - "Bash(pnpm test:drift)", - "Bash(pnpm test:drift *)", - "Bash(npx prettier *)", - "Bash(npx eslint *)", - "Bash(git diff *)", - "Bash(git status *)", - "Bash(git log *)", - ].join(","), - "--max-turns", - "50", - ]; - - // `detached: true` puts the child in its OWN process group (gpid === pid), - // so a timeout can signal the WHOLE group — the `npx` wrapper AND its - // `@anthropic-ai/claude-code` grandchild — via `process.kill(-pid, …)`. - // Without it, killing the child pid reaches only the wrapper and a wedged - // grandchild survives to burn the 30-min job budget. - const child = spawn("npx", args, { - stdio: ["inherit", "pipe", "pipe"], - detached: true, - }); - - const logChunks: Buffer[] = []; - let killGraceTimer: NodeJS.Timeout | undefined; - let timedOut = false; - // REAL has-exited flag, flipped by the `close` handler. The SIGKILL - // escalation is gated on THIS, never `child.killed` (which is true the - // instant SIGTERM is delivered, making the escalation dead code). - let exited = false; - - const killTimer = setTimeout(() => { - timedOut = true; - console.error( - `Claude Code timed out after ${CLAUDE_TIMEOUT_MS / 60000} minutes. ` + - "Sending SIGTERM to the process group...", - ); - // child.pid can be undefined if the spawn failed; the `error` handler - // covers that path, so only escalate when we have a real group leader. - if (typeof child.pid === "number") { - killGraceTimer = scheduleEscalatingKill(child.pid, () => exited); - } - }, CLAUDE_TIMEOUT_MS); - - child.on("error", (err) => { - clearTimeout(killTimer); - console.error("Failed to spawn Claude Code process:", err.message); - try { - writeFileSync("claude-code-output.log", `Spawn error: ${err.message}\n`, "utf-8"); - } catch (writeErr) { - console.error( - "Failed to write claude-code-output.log:", - writeErr instanceof Error ? writeErr.message : writeErr, - ); - } - reject(err); - }); - - // Wire the stream + close handlers inside a guard so a SYNCHRONOUS throw - // here (e.g. `child.stdout` is null on a spawn edge case, making - // `child.stdout.on(...)` a TypeError) cannot strand the already-armed - // `killTimer`. Without this, such a throw rejects the Promise via the - // executor's synchronous-throw semantics BUT the 30-min timer stays live and - // would later group-kill a possibly-reused PID. Clear the timer, then reject. - try { - if (!child.stdout || !child.stderr) { - throw new Error( - "Claude Code child has no stdout/stderr pipe (spawn produced null streams)", - ); - } - child.stdout.on("data", (chunk: Buffer) => { - process.stdout.write(chunk); - logChunks.push(chunk); - }); - - child.stderr.on("data", (chunk: Buffer) => { - process.stderr.write(chunk); - logChunks.push(chunk); - }); - } catch (setupErr) { - clearTimeout(killTimer); - reject(setupErr instanceof Error ? setupErr : new Error(String(setupErr))); - return; - } - - child.on("close", (code, signal) => { - // Mark real exit BEFORE clearing timers so any in-flight grace timer that - // fires in the same tick sees `exited === true` and skips the SIGKILL. - exited = true; - clearTimeout(killTimer); - if (killGraceTimer) clearTimeout(killGraceTimer); - const logContent = Buffer.concat(logChunks).toString("utf-8"); - try { - writeFileSync("claude-code-output.log", logContent, "utf-8"); - } catch (writeErr) { - console.error( - "Failed to write claude-code-output.log:", - writeErr instanceof Error ? writeErr.message : writeErr, - ); - } - if (code === null && signal) { - console.error(`Claude Code process killed by signal: ${signal}`); - } - done(timedOut ? 124 : (code ?? 1)); - }); - }); -} - -// --------------------------------------------------------------------------- -// PR mode (--create-pr) -// --------------------------------------------------------------------------- - -export function patchBumpVersion(): string { - const pkgPath = resolve("package.json"); - const pkg = JSON.parse(readFileSync(pkgPath, "utf-8")) as { - version: string; - description?: string; - [key: string]: unknown; - }; - const parts = pkg.version.split(".").map(Number); - if (parts.length !== 3 || parts.some(isNaN)) { - throw new Error(`Cannot patch-bump non-standard version: ${pkg.version}`); - } - parts[2] += 1; - const newVersion = parts.join("."); - pkg.version = newVersion; - - // Sync description with README subtitle - syncDescriptionFromReadme(pkg); - - writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + "\n", "utf-8"); - return newVersion; -} - -/** Keep package.json description in sync with the README subtitle. */ -function syncDescriptionFromReadme(pkg: { description?: string; [key: string]: unknown }): void { - const readmePath = resolve("README.md"); - try { - const readme = readFileSync(readmePath, "utf-8"); - // The description is the first non-empty, non-heading, non-badge, non-video line - const lines = readme.split("\n"); - for (const line of lines) { - const trimmed = line.trim(); - if ( - !trimmed || - trimmed.startsWith("#") || - trimmed.startsWith("[![") || - trimmed.startsWith("![") || - trimmed.startsWith("[") || - trimmed.startsWith("http") - ) { - continue; - } - // Found the subtitle — strip markdown formatting - const clean = trimmed.replace(/[*_`]/g, "").replace(/\s+/g, " ").trim(); - if (clean && clean !== pkg.description) { - pkg.description = clean; - } - break; - } - } catch (err: unknown) { - if ((err as NodeJS.ErrnoException).code !== "ENOENT") { - console.warn("Could not sync description from README:", err); - } - } -} - -export function addChangelogEntry(report: DriftReport, version: string): void { - const changelogPath = resolve("CHANGELOG.md"); - const existing = readFileIfExists(changelogPath) ?? ""; - - const providerSummaries = report.entries.map((entry) => { - const fields = entry.diffs.map((d) => d.path).join(", "); - return `- ${entry.provider} (${entry.scenario}): ${fields}`; - }); - - const newEntry = [ - `## ${version}`, - "", - "### Patch Changes", - "", - "- Auto-remediate API drift:", - ...providerSummaries.map((s) => ` ${s}`), - "", - ].join("\n"); - - // Insert after the title line (any line starting with "# ") - const titleMatch = existing.match(/^# .+\n/); - if (titleMatch) { - const titleLine = titleMatch[0]; - const rest = existing.slice(titleLine.length); - writeFileSync(changelogPath, titleLine + "\n" + newEntry + rest, "utf-8"); - } else { - writeFileSync(changelogPath, newEntry + "\n" + existing, "utf-8"); - } -} - -export function buildPrBody( - report: DriftReport, - changedFiles?: string[], - verdictDetail?: string, -): string { - const providers: string[] = []; - const diffs: string[] = []; - - for (const entry of report.entries) { - providers.push(`- ${entry.provider}: ${entry.scenario}`); - for (const diff of entry.diffs) { - diffs.push(`- \`${diff.path}\`: ${diff.issue}`); - } - } - - const reportJson = JSON.stringify(report, null, 2); - - const sections: string[] = [ - "## Summary", - "", - "Auto-generated drift remediation.", - "", - // Human-approval backstop (WS-2): this PR was auto-FILTERED by the - // drift-success predicate but is NOT auto-merged. A human must review CI + - // this diff + the verdict below and merge. The predicate is a strong filter, - // not a provable merge gate (the re-collect is not independent of the fix — - // WS-2b), so the merge decision stays with a human. - "> **Needs human review + merge.** This drift-fix PR was opened by the", - "> automated pipeline after the drift-success predicate passed. It is NOT", - "> auto-merged — review CI, the diff, and the verdict below, then merge.", - "", - "### Drift-success predicate verdict", - "", - verdictDetail ? `RESOLVED — ${verdictDetail}` : "RESOLVED.", - "", - "### Providers affected", - ...providers, - "", - "### Diffs fixed", - ...diffs, - "", - ]; - - // Flag skill sections that may need review based on which builders changed - const skillSections = changedFiles ? affectedSkillSections(changedFiles) : []; - if (skillSections.length > 0) { - sections.push( - "### Skill documentation", - "", - `The following write-fixtures skill sections may need review after these builder changes:`, - ...skillSections.map((s) => `- ${s}`), - "", - ); - } - - sections.push( - "## Drift Report", - "", - "
", - "Full drift report JSON", - "", - "```json", - reportJson, - "```", - "", - "
", - ); - - return truncateBody(sections.join("\n")); -} - -/** - * Parse a single line from `git status --porcelain` output into a file path. - * Handles quoted paths (special characters) and rename notation (old -> new). - */ -export function parsePorcelainLine(line: string): string { - let path = line.slice(3).trim(); - // Handle renames first: "old -> new" → take the new path - const arrowIdx = path.indexOf(" -> "); - if (arrowIdx !== -1) { - path = path.slice(arrowIdx + 4); - } - // Then strip quotes (git quotes paths with special characters) - if (path.startsWith('"') && path.endsWith('"')) { - path = path.slice(1, -1); - } - return path; -} - -/** - * Return the list of changed files from `git status --porcelain`. - */ -export function getChangedFiles(): string[] { - return exec("git status --porcelain").split("\n").filter(Boolean).map(parsePorcelainLine); -} - -/** - * Optional post-fix collector result, supplied by the workflow so createPr does - * not re-shell the (2-3 min) collector. When present, the drift-success - * predicate is the authoritative PR-open gate; when absent, createPr falls back - * to the legacy "no source files changed" guard (exit 4). - */ -export interface PostFixCollectorResult { - /** Parsed post-fix drift report (from --post-fix-report). */ - report: DriftReport; - /** Exit code of the re-run collector (from --post-fix-exit). */ - exitCode: number; -} - -/** - * The GATED commit groups for a RESOLVED verdict (CR round-3 F-C / F2). Given - * the canonicalized changed-file set and the report's sanctioned-target set, - * partition into the ONLY groups createPr is permitted to stage: - * - `builderFiles` — production mock-builder source (always allowlisted). - * - `testFiles` — ONLY report-named fixture targets under src/__tests__/ - * (any other test file would have BLOCKED at the gate, so - * it must never be staged). - * `stragglers` is every canonicalized changed file that is NOT in one of those - * gated groups. On a RESOLVED verdict it MUST be empty (the predicate allowlist - * already rejected any unclassified file); createPr never `git add`s it. The - * version bump + CHANGELOG are added separately by exact path and are not part - * of this changed-file partition. - */ -export function gatedCommitFiles( - changedFiles: string[], - sanctioned: ReadonlySet, -): { - builderFiles: string[]; - testFiles: string[]; - stragglers: string[]; -} { - const builderFiles = changedFiles.filter(isProductionFile); - const testFiles = changedFiles.filter((f) => f.startsWith("src/__tests__/") && sanctioned.has(f)); - const gated = new Set([...builderFiles, ...testFiles]); - const stragglers = changedFiles.filter((f) => !gated.has(f)); - return { builderFiles, testFiles, stragglers }; -} - -export function createPr(report: DriftReport, postFix?: PostFixCollectorResult): void { - const stamp = todayStamp(); - - // Detect uncommitted changes (staged + unstaged) BEFORE any git write ops so - // the drift-success predicate can gate PR-open ahead of branch/commit/push. - // - // Use the predicate's OWN `gitChangedFiles()` (which runs - // `git -c core.quotePath=false status --porcelain`) and canonicalize every - // path with the predicate's `canonicalizePath` (CR round-3 F-C). This keeps - // classification/staging BYTE-FOR-BYTE identical to what the predicate scored: - // both callers now read the same git invocation and the same canonical spelling, - // so a non-ASCII/C-quoted path or a `./`-prefixed spelling cannot be classified - // one way by the verdict and staged another way here. - const changedFiles = gitChangedFiles().map(canonicalizePath); - - // PR-OPEN GATE (WS-2). When the workflow supplies the post-fix collector - // result, the drift-success predicate is the authoritative decision: it - // rejects fixture-relaxation cheats (a diff that changed ONLY comparison-leg - // files, or silenced the detector) and drifts that were not actually - // resolved. This runs BEFORE any git add/commit — a blocked verdict opens no - // PR (and therefore never reaches auto-merge). - // FIX #5 — the drift-success predicate is MANDATORY; there is NO legacy - // no-post-fix fallback. The old fallback (accept a PR when - // `builderFiles.length || testFiles.length`) re-opened the original - // fixture-only cheat: a run that changed only comparison-leg test files - // satisfied `testFiles.length > 0` and sailed through to a PR. There is no - // safe "no post-fix result" path — without the authoritative post-fix - // collector signal we cannot tell a real fix from a relaxation, so we - // fail-closed to human review (COLLECTOR_INFRA) rather than open a PR. The - // real workflow (fix-drift.yml "Create PR" step) ALWAYS supplies --report - // (the PINNED pre-fix report), --post-fix-report, and --post-fix-exit, so this - // only fires on a misconfigured invocation — which must NOT auto-merge. - if (!postFix) { - console.error( - "ERROR: PR-open gate requires the post-fix collector result " + - "(--post-fix-report + --post-fix-exit). Refusing to open a PR without the " + - "authoritative drift-success signal — routing to human review.", - ); - console.log(`reason=${PredicateReason.COLLECTOR_INFRA}`); - process.exit(REASON_EXIT_CODE[PredicateReason.COLLECTOR_INFRA]); - } - let verdict; - try { - verdict = evaluateDriftResolved({ - changedFiles, - report, - postFixCollectorExit: postFix.exitCode, - postFixCriticalCount: countCriticalDiffs(postFix.report), - }); - } catch (err: unknown) { - // A malformed report or a repo-escaping changed-file path throws from the - // predicate — fail-closed to a NAMED config-error rather than an uncaught - // stacktrace, so the workflow's Slack alert names the cause (mirrors runCli). - const msg = err instanceof Error ? err.message : String(err); - console.error(`ERROR: unable to score the drift reports: ${msg}`); - console.log(`reason=${PredicateReason.CONFIG_ERROR}`); - process.exit(REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]); - } - if (!verdict.resolved) { - console.error(`ERROR: Drift NOT resolved [${verdict.reason}]: ${verdict.detail}`); - if (verdict.offendingFiles.length > 0) { - console.error(`Offending files: ${verdict.offendingFiles.join(", ")}`); - } - console.error("Aborting PR creation — this fix will be routed to human review."); - console.log(`reason=${verdict.reason}`); - process.exit(REASON_EXIT_CODE[verdict.reason]); - } - console.log(`Drift-success predicate: RESOLVED — ${verdict.detail}`); - - // Determine branch name. A git failure here fails CLOSED with a NAMED reason - // (git-push-failed) rather than a blank-reason exit 3 (see the catch at the - // end of the git-op sequence below). - let currentBranch: string; - try { - currentBranch = exec("git rev-parse --abbrev-ref HEAD"); - } catch (err: unknown) { - console.error( - `ERROR: cannot determine current branch for PR creation: ${(err as Error).message}`, - ); - console.log(`reason=${PredicateReason.GIT_PUSH_FAILED}`); - process.exit(REASON_EXIT_CODE[PredicateReason.GIT_PUSH_FAILED]); - } - - const branchName = - currentBranch === "master" || currentBranch === "main" || currentBranch === "HEAD" - ? `fix/drift-${stamp}` - : currentBranch; - - // A git checkout/stage/commit/push failure anywhere below fails CLOSED (no PR - // — the push never completes, so no partial/unversioned PR ships) but the raw - // throw would reach the top-level catch with a BLANK `reason=`. `gitOp` names - // the cause (git-push-failed) so the operator alert is not blank. The - // version-bump block keeps its OWN distinct VERSION_BUMP_FAILED reason. - const gitOp = (label: string, fn: () => void): void => { - try { - fn(); - } catch (err) { - console.error( - `ERROR: git operation failed (${label}) while opening the drift-fix PR — no PR opened:`, - err instanceof Error ? err.message : err, - ); - console.log(`reason=${PredicateReason.GIT_PUSH_FAILED}`); - process.exit(REASON_EXIT_CODE[PredicateReason.GIT_PUSH_FAILED]); - } - }; - - if (branchName !== currentBranch) { - gitOp("checkout -b", () => execFileSafe("git", ["checkout", "-b", branchName])); - console.log(`Created branch ${branchName}`); - } - - // Stage ONLY the GATED set (CR round-3 F-C / F2). The predicate already - // verified that EVERY changed file is on the sanctioned allowlist — production - // mock-builder source OR a fixture the report explicitly named. We stage - // exactly that allowlisted set here (grouped by commit purpose) and NEVER a - // catch-all `git add` of "whatever is still dirty": a straggler-add re-widened - // the PR past the verdict (any unclassified file the predicate did not judge — - // an unnamed fixture, a config file — would have BLOCKED at the gate, so - // sweeping it in AFTER the pass silently defeats the allowlist). - const sanctioned = sanctionedTargets(report); - const { builderFiles, testFiles, stragglers } = gatedCommitFiles(changedFiles, sanctioned); - - // FIX #F5 (round-4) — on a RESOLVED verdict `stragglers` MUST be empty: the - // predicate allowlist already rejected any file that is neither production - // source nor a report-named fixture target, so every changed file must fall - // into one of the gated groups above. If a straggler survives here, the - // verdict and the staging partition have DIVERGED (e.g. a future predicate - // change admitted a file gatedCommitFiles does not classify) — silently - // dropping it would ship an incomplete fix behind a green verdict. Fail closed - // to human review rather than open a PR whose diff differs from what was scored. - if (stragglers.length > 0) { - console.error( - `ERROR: RESOLVED verdict but ${stragglers.length} changed file(s) are not in any gated ` + - `commit group (${stragglers.join(", ")}) — the verdict and the staging partition have ` + - "diverged. Refusing to open a PR that would silently drop these from the diff.", - ); - console.log(`reason=${PredicateReason.UNSANCTIONED_CHANGE}`); - process.exit(REASON_EXIT_CODE[PredicateReason.UNSANCTIONED_CHANGE]); - } - - if (builderFiles.length > 0) { - gitOp("add builders", () => execFileSafe("git", ["add", ...builderFiles])); - gitOp("commit builders", () => - execFileSafe("git", ["commit", "-m", "fix: auto-remediate API drift in builder functions"]), - ); - } - - if (testFiles.length > 0) { - gitOp("add tests", () => execFileSafe("git", ["add", ...testFiles])); - gitOp("commit tests", () => - execFileSafe("git", ["commit", "-m", "test: update SDK shapes for drift remediation"]), - ); - } - - // The version bump + CHANGELOG are an EXPLICIT, gated part of the fix set — a - // release always accompanies an auto-remediation — not an unclassified - // straggler. They are workflow-authored (never LLM-authored) and staged by an - // exact path list, so they do not re-open the allowlist. - // - // WS-8 — this step is MANDATORY, so a failure here is a HARD, fail-closed - // error: opening an UNVERSIONED PR would ship a "fix" that a human might merge - // but which never publishes a release, silently delivering no value. We exit - // with a distinct VERSION_BUMP_FAILED reason (routed to the workflow's - // human-review alert) rather than warn-and-continue. No push has happened yet, - // and the builder/test commits above are local-only until the push below — - // which we never reach — so no partial/unversioned PR is ever opened. - try { - const newVersion = patchBumpVersion(); - console.log(`Bumped version to ${newVersion}`); - - addChangelogEntry(report, newVersion); - console.log("Added CHANGELOG.md entry"); - - // Commit version bump + changelog by EXACT path (not a catch-all). - execFileSafe("git", ["add", "package.json", "CHANGELOG.md"]); - execFileSafe("git", ["commit", "-m", `chore: bump version to ${newVersion}`, "--allow-empty"]); - } catch (err) { - console.error( - "ERROR: version bump / CHANGELOG step failed — refusing to open an UNVERSIONED " + - "drift-fix PR that would merge a fix which never publishes a release:", - err instanceof Error ? err.message : err, - ); - console.log(`reason=${PredicateReason.VERSION_BUMP_FAILED}`); - process.exit(REASON_EXIT_CODE[PredicateReason.VERSION_BUMP_FAILED]); - } - - gitOp("push", () => execFileSafe("git", ["push", "-u", "origin", branchName])); - console.log(`Pushed branch ${branchName}`); - - const prBody = buildPrBody(report, changedFiles, verdict.detail); - const prTitle = `fix: auto-remediate API drift (${stamp})`; - - const prBodyFile = `/tmp/aimock-drift-${process.pid}-pr-body.md`; - writeFileSync(prBodyFile, prBody, "utf-8"); - try { - execFileSafe("gh", [ - "pr", - "create", - "--title", - prTitle, - "--assignee", - "jpr5", - "--body-file", - prBodyFile, - ]); - } finally { - try { - unlinkSync(prBodyFile); - } catch (cleanupErr) { - console.warn( - `Could not clean up temp file:`, - cleanupErr instanceof Error ? cleanupErr.message : cleanupErr, - ); - } - } - - console.log("PR created successfully."); -} - -// --------------------------------------------------------------------------- -// Issue mode (--create-issue) -// --------------------------------------------------------------------------- - -function createIssue(report: DriftReport | null): void { - const stamp = todayStamp(); - const reportJson = report - ? JSON.stringify(report, null, 2) - : "(drift report was not generated — collector may have crashed)"; - const claudeOutput = - readFileIfExists(resolve("claude-code-output.log")) ?? "(no output captured)"; - - const issueBody = truncateBody( - [ - "## Drift detected but auto-fix failed", - "", - "The automated drift remediation pipeline detected API drift but was unable", - "to fix it automatically. Manual intervention is required.", - "", - "### Drift Report", - "", - "```json", - reportJson, - "```", - "", - "### Claude Code Output", - "", - "
", - "Full output", - "", - "```", - claudeOutput, - "```", - "", - "
", - ].join("\n"), - ); - - const issueTitle = `Drift detected — auto-fix failed (${stamp})`; - - const issueBodyFile = `/tmp/aimock-drift-${process.pid}-issue-body.md`; - writeFileSync(issueBodyFile, issueBody, "utf-8"); - try { - execFileSafe("gh", [ - "issue", - "create", - "--title", - issueTitle, - "--body-file", - issueBodyFile, - "--label", - "drift", - ]); - } finally { - try { - unlinkSync(issueBodyFile); - } catch (cleanupErr) { - console.warn( - `Could not clean up temp file:`, - cleanupErr instanceof Error ? cleanupErr.message : cleanupErr, - ); - } - } - - console.log("Issue created successfully."); -} - -// --------------------------------------------------------------------------- -// Main -// --------------------------------------------------------------------------- - -export function parseMode(args: string[]): "pr" | "issue" | "default" { - if (args.includes("--create-pr")) return "pr"; - if (args.includes("--create-issue")) return "issue"; - return "default"; -} - -/** - * FIX #5 — true only when BOTH post-fix collector flags are present with values. - * PR mode requires both (the drift-success predicate is the authoritative - * PR-open gate); there is no legacy no-post-fix fallback that could re-open the - * fixture-only cheat. - */ -export function hasPostFixArgs(args: string[]): boolean { - const reportIdx = args.indexOf("--post-fix-report"); - const exitIdx = args.indexOf("--post-fix-exit"); - const hasReport = reportIdx !== -1 && args[reportIdx + 1] !== undefined; - const hasExit = exitIdx !== -1 && args[exitIdx + 1] !== undefined; - return hasReport && hasExit; -} - -/** - * FIX #F7 (round-4) — parse the `--post-fix-exit` value, failing CLOSED on an - * empty/whitespace or non-integer value. `Number("")` and `Number(" ")` are - * both 0, which Number.isInteger accepts, so a missing recollect output - * (`--post-fix-exit ""`, e.g. a skipped step) would masquerade as a clean - * collector exit 0 and open a PR on an unverified fix. This mirrors the - * predicate CLI's guard (drift-success-predicate.ts:parseCliArgs). Throws on any - * empty/whitespace/non-integer input; the caller must NOT treat that as clean. - */ -export function parsePostFixExit(raw: string): number { - if (raw.trim() === "") { - throw new Error( - "--post-fix-exit is empty/whitespace — a missing collector exit code must fail closed, " + - "not be treated as clean exit 0", - ); - } - const parsed = Number(raw); - if (!Number.isInteger(parsed)) { - throw new Error(`--post-fix-exit must be an integer, got "${raw}"`); - } - return parsed; -} - -async function main(): Promise { - const args = process.argv.slice(2); - const mode = parseMode(args); - - const reportIndex = args.indexOf("--report"); - const reportPath = resolve( - reportIndex !== -1 && args[reportIndex + 1] ? args[reportIndex + 1] : "drift-report.json", - ); - - // Issue mode handles missing reports gracefully (the safety net shouldn't crash) - if (mode === "issue") { - let report: DriftReport | null = null; - try { - report = readDriftReport(reportPath); - } catch (err: unknown) { - const msg = err instanceof Error ? err.message : String(err); - console.warn(`Could not read drift report (${msg}), creating issue with available info`); - } - createIssue(report); - return; - } - - const report = readDriftReport(reportPath); - - if (report.entries.length === 0) { - console.log("No drift entries found. Nothing to do."); - process.exit(0); - } - - console.log(`Loaded drift report: ${report.entries.length} entries from ${report.timestamp}`); - - if (mode === "pr") { - const postFixReportIdx = args.indexOf("--post-fix-report"); - const postFixExitIdx = args.indexOf("--post-fix-exit"); - - // FIX #5 — the post-fix collector result is REQUIRED in PR mode. The old - // path allowed --create-pr with NO post-fix args, which fell back to the - // gameable legacy guard (a test-file-only change satisfied it and opened a - // PR). Both flags must be present; a missing/partial pair throws rather than - // silently skipping the authoritative drift-success gate. - if (!hasPostFixArgs(args)) { - throw new Error( - "--create-pr requires BOTH --post-fix-report and --post-fix-exit (the drift-success " + - "predicate is the authoritative PR-open gate; there is no legacy no-post-fix path)", - ); - } - // FIX #F7 (round-4) — fail CLOSED on an empty/whitespace/non-integer - // --post-fix-exit (see parsePostFixExit): a missing recollect output must - // not masquerade as a clean collector exit 0 and open a PR on an unverified - // fix. Mirrors the predicate CLI's guard. - // - // These parse/read failures fail closed (no PR) — but the raw throw would - // reach the top-level catch with a BLANK `reason=`, so the operator alert is - // uninformative. Emit a NAMED reason (post-fix-parse-error) before - // rethrowing so the workflow's `grep '^reason='` names the cause. - let postFixExit: number; - let postFixReport: DriftReport; - try { - postFixExit = parsePostFixExit(args[postFixExitIdx + 1]); - postFixReport = readPostFixReport(resolve(args[postFixReportIdx + 1])); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - console.error(`ERROR: could not parse/read the post-fix collector result: ${msg}`); - console.log(`reason=${PredicateReason.POST_FIX_PARSE_ERROR}`); - process.exit(REASON_EXIT_CODE[PredicateReason.POST_FIX_PARSE_ERROR]); - } - const postFix: PostFixCollectorResult = { report: postFixReport, exitCode: postFixExit }; - - createPr(report, postFix); - } else { - const prompt = buildPrompt(report); - console.log("Invoking Claude Code CLI..."); - const exitCode = await invokeClaudeCode(prompt); - console.log(`Claude Code exited with code ${exitCode}`); - process.exit(exitCode); - } -} - -const isMain = process.argv[1] === fileURLToPath(import.meta.url); -if (isMain) { - main().catch((err: unknown) => { - console.error("Fatal error:", err); - process.exit(3); - }); -} diff --git a/src/__tests__/drift-collector.test.ts b/src/__tests__/drift-collector.test.ts index 1ac243ab..e3aab9c8 100644 --- a/src/__tests__/drift-collector.test.ts +++ b/src/__tests__/drift-collector.test.ts @@ -929,6 +929,44 @@ describe("collectDriftEntries", () => { expect(exitCodeOf(result)).toBe(0); }); + it("F1: a benign infra leg is NOT batch-poisoned into quarantine by an unparseable sibling (per-leg classification)", () => { + // The recurring session failure: one leg emits genuinely-unparseable output + // in the SAME run as sibling legs that failed on benign infra (network + // flake). Pre-fix, the all-or-nothing classifyUnparseableAsInfra gate saw a + // MIXED batch (not EVERY message is infra) and quarantined EVERY unparseable + // failure — dragging the benign infra leg into the shared base report's + // quarantine (exit 5) and poisoning it for every downstream PR. Per-leg + // classification swallows the infra leg on its own and quarantines ONLY the + // genuinely-unparseable leg. + const result = makeResult([ + // Leg A: benign infra (network flake) — swallowed on its own merits. + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + title: "non-streaming text matches real API", + failureMessages: [REAL_INFRA_BODY], + }), + // Leg B: genuinely unparseable (no infra token) — legitimately quarantined. + makeAssertion({ + status: "failed", + ancestorTitles: ["Broken Suite"], + title: "unrecognized", + failureMessages: [ + "AssertionError: expected 1 to be 2\n at foo (/repo/src/__tests__/drift/b.drift.ts:1:1)", + ], + }), + ]); + const { entries, quarantine } = collectDriftEntries(result); + expect(entries).toEqual([]); + // GREEN: exactly ONE quarantine entry — leg B only; leg A (infra) swallowed. + // RED pre-fix: quarantine.length === 2 (leg A poisoned in alongside leg B). + expect(quarantine).toHaveLength(1); + expect(quarantine[0].testName).toContain("unrecognized"); + // The genuinely-unparseable leg still legitimately reds the run (exit 5) — + // real drift/quarantine detection is NOT weakened. + expect(exitCodeOf(result)).toBe(5); + }); + it("canary (unknown-model) failure → critical entry + exit 2", () => { const result = makeResult([ makeAssertion({ diff --git a/src/__tests__/drift-scripts.test.ts b/src/__tests__/drift-scripts.test.ts index 6ce34527..40dbfda7 100644 --- a/src/__tests__/drift-scripts.test.ts +++ b/src/__tests__/drift-scripts.test.ts @@ -4,18 +4,18 @@ import { join } from "node:path"; import { tmpdir } from "node:os"; // --------------------------------------------------------------------------- -// fix-drift.ts exports under test +// drift-sync.ts exports under test (C3: retargeted from the deleted +// scripts/fix-drift.ts — the LLM freewriter path, including buildPrompt and +// the predicate-gated CLI's parseMode, has been removed entirely). // --------------------------------------------------------------------------- import { readDriftReport, - buildPrompt, buildPrBody, patchBumpVersion, addChangelogEntry, parsePorcelainLine, - parseMode, todayStamp, -} from "../../scripts/fix-drift.js"; +} from "../../scripts/drift-sync.js"; import { summarizeDriftReport } from "../../scripts/drift-slack-summary.js"; @@ -116,72 +116,6 @@ describe("readDriftReport", () => { }); }); -// --------------------------------------------------------------------------- -// parseMode -// --------------------------------------------------------------------------- - -describe("parseMode", () => { - it("returns 'pr' for --create-pr", () => { - expect(parseMode(["--create-pr"])).toBe("pr"); - }); - - it("returns 'issue' for --create-issue", () => { - expect(parseMode(["--create-issue"])).toBe("issue"); - }); - - it("returns 'default' when no flag", () => { - expect(parseMode([])).toBe("default"); - expect(parseMode(["--report", "foo.json"])).toBe("default"); - }); - - it("prefers --create-pr over --create-issue when both present", () => { - expect(parseMode(["--create-pr", "--create-issue"])).toBe("pr"); - }); -}); - -// --------------------------------------------------------------------------- -// buildPrompt -// --------------------------------------------------------------------------- - -describe("buildPrompt", () => { - it("includes all drift entry details", () => { - const report = makeReport(); - const prompt = buildPrompt(report); - expect(prompt).toContain("DRIFT 1: OpenAI Chat — non-streaming text"); - expect(prompt).toContain("File: src/helpers.ts"); - expect(prompt).toContain("Functions: buildTextCompletion"); - expect(prompt).toContain("[critical] LLMOCK DRIFT"); - expect(prompt).toContain("Path: choices[0].message.refusal"); - }); - - it("includes workflow instructions", () => { - const prompt = buildPrompt(makeReport()); - expect(prompt).toContain("RED:"); - expect(prompt).toContain("GREEN:"); - expect(prompt).toContain("pnpm test"); - expect(prompt).toContain("pnpm test:drift"); - }); - - it("numbers multiple drift entries", () => { - const report = makeReport({ - entries: [ - { ...makeReport().entries[0], provider: "OpenAI Chat", scenario: "streaming" }, - { - ...makeReport().entries[0], - provider: "Anthropic", - scenario: "non-streaming text", - builderFile: "src/messages.ts", - builderFunctions: ["buildClaudeTextResponse"], - typesFile: null, - }, - ], - }); - const prompt = buildPrompt(report); - expect(prompt).toContain("DRIFT 1:"); - expect(prompt).toContain("DRIFT 2:"); - }); -}); - // --------------------------------------------------------------------------- // buildPrBody // --------------------------------------------------------------------------- diff --git a/src/__tests__/drift-success-predicate.test.ts b/src/__tests__/drift-success-predicate.test.ts deleted file mode 100644 index 3ad04a74..00000000 --- a/src/__tests__/drift-success-predicate.test.ts +++ /dev/null @@ -1,1400 +0,0 @@ -/** - * Tests for the WS-2 drift-success predicate. - * - * These exercise the REAL exported pure function `evaluateDriftResolved` and - * the CLI helpers from scripts/drift-success-predicate.ts. The predicate is a - * pure function over a small `DriftReport` fixture + a synthetic changed-file - * array — no live API, no LLM, no aimock needed. - * - * The headline case is the fixture-relaxation cheat: a run that changes ONLY - * `src/__tests__/drift/sdk-shapes.ts` (relaxing the SDK leg) must be REJECTED - * (COMPARISON_LEG_ONLY), whereas the OLD fix-drift.ts guard (`builderFiles>0 || - * testFiles>0`) would have ACCEPTED it — demonstrated by contrast below. - */ - -import { execFileSync } from "node:child_process"; -import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -import { describe, it, expect, afterEach, vi } from "vitest"; - -import type { DriftEntry, DriftReport, ParsedDiff } from "../../scripts/drift-types.js"; -import { - evaluateDriftResolved, - PredicateReason, - REASON_EXIT_CODE, - isProductionFile, - isComparisonLeg, - isSuppressionSurface, - isGameableLeg, - canonicalizePath, - isAllowlisted, - sanctionedTargets, - countCriticalDiffs, - parseCliArgs, - parsePorcelainLine, - crossCheckChangedFiles, - PredicateConfigError, - readReport, - runCli, -} from "../../scripts/drift-success-predicate.js"; - -// --------------------------------------------------------------------------- -// Fixture builders -// --------------------------------------------------------------------------- - -function diff(overrides: Partial = {}): ParsedDiff { - return { - path: "choices[0].message.content", - severity: "critical", - issue: "field present in SDK+real but missing from mock", - expected: "string", - real: "string", - mock: "", - ...overrides, - }; -} - -function entry(overrides: Partial = {}): DriftEntry { - return { - provider: "OpenAI", - scenario: "chat completion", - builderFile: "src/helpers.ts", - builderFunctions: ["buildChatCompletion"], - typesFile: "src/types.ts", - sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", - diffs: [diff()], - ...overrides, - }; -} - -function report(entries: DriftEntry[] = [entry()]): DriftReport { - return { timestamp: "2026-07-16T00:00:00.000Z", entries }; -} - -/** The OLD, gameable guard from fix-drift.ts:638 — for contrast assertions. */ -function oldGuardWouldAccept(changedFiles: string[]): boolean { - const builderFiles = changedFiles.filter( - (f) => f.startsWith("src/") && !f.startsWith("src/__tests__/"), - ); - const testFiles = changedFiles.filter((f) => f.startsWith("src/__tests__/")); - // OLD guard aborts ONLY when BOTH are empty; otherwise it proceeds. - return !(builderFiles.length === 0 && testFiles.length === 0); -} - -// --------------------------------------------------------------------------- -// RED cases — resolved:false -// --------------------------------------------------------------------------- - -describe("evaluateDriftResolved — RED (cheat/failure) cases", () => { - it("HEADLINE: fixture-relaxation-only (sdk-shapes.ts) → COMPARISON_LEG_ONLY, and the OLD guard would have ACCEPTED it", () => { - const changedFiles = ["src/__tests__/drift/sdk-shapes.ts"]; - const verdict = evaluateDriftResolved({ - changedFiles, - report: report(), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.COMPARISON_LEG_ONLY); - expect(verdict.offendingFiles).toContain("src/__tests__/drift/sdk-shapes.ts"); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(11); - - // Contrast: the OLD guard would have proceeded (testFiles non-empty). - expect(oldGuardWouldAccept(changedFiles)).toBe(true); - }); - - it("schema/allowlist edit + real builder change → SUPPRESSION_SUSPECTED (blocks even with a prod change)", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/__tests__/drift/schema.ts", "src/helpers.ts"], - report: report(), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.SUPPRESSION_SUSPECTED); - expect(verdict.offendingFiles).toContain("src/__tests__/drift/schema.ts"); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(12); - }); - - it("*.drift.ts assertion loosened only → SUPPRESSION_SUSPECTED", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/__tests__/drift/openai-chat.drift.ts"], - report: report(), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.SUPPRESSION_SUSPECTED); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(12); - }); - - it("no changes at all → NO_PRODUCTION_CHANGE", () => { - const verdict = evaluateDriftResolved({ - changedFiles: [], - report: report(), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.NO_PRODUCTION_CHANGE); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(10); - }); - - it("production change but collector still dirty (exit 2) → STILL_DIRTY", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts"], - report: report(), - postFixCollectorExit: 2, - postFixCriticalCount: 1, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.STILL_DIRTY); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(13); - }); - - it("production change + collector exit 0 but criticalCount>0 → STILL_DIRTY (belt-and-suspenders)", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts"], - report: report(), - postFixCollectorExit: 0, - postFixCriticalCount: 3, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.STILL_DIRTY); - }); - - it("post-fix quarantine (exit 5) → QUARANTINE_AFTER_FIX", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts"], - report: report(), - postFixCollectorExit: 5, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.QUARANTINE_AFTER_FIX); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(14); - }); - - it("post-fix collector infra (exit 1) → COLLECTOR_INFRA", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts"], - report: report(), - postFixCollectorExit: 1, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.COLLECTOR_INFRA); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(15); - }); - - it("production change off-target → PRODUCTION_CHANGE_OFF_TARGET", () => { - // report names src/helpers.ts; the change is an unrelated production file. - const verdict = evaluateDriftResolved({ - changedFiles: ["src/gemini.ts"], - report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.PRODUCTION_CHANGE_OFF_TARGET); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(16); - }); - - // ------------------------------------------------------------------------- - // WS-2b HYBRID CHEAT — a gameable-leg edit ACCOMPANIED by a trivial, on-target - // production edit. Pre-fix the predicate ignored the leg once ANY production - // file changed → RESOLVED (the exact WS-2b auto-merge cheat). Post-fix a leg - // edit ALWAYS blocks (SUPPRESSION_SUSPECTED), regardless of production files. - // ------------------------------------------------------------------------- - it("HEADLINE WS-2b: sdk-shapes.ts relaxation + on-target production edit → SUPPRESSION_SUSPECTED (block), NOT resolved", () => { - const changedFiles = ["src/__tests__/drift/sdk-shapes.ts", "src/helpers.ts"]; - const verdict = evaluateDriftResolved({ - changedFiles, - report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.SUPPRESSION_SUSPECTED); - expect(verdict.offendingFiles).toContain("src/__tests__/drift/sdk-shapes.ts"); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(12); - // Contrast: the OLD guard would have proceeded (both builder + test present). - expect(oldGuardWouldAccept(changedFiles)).toBe(true); - }); - - it("harness leg (providers.ts) relaxation + on-target production edit → SUPPRESSION_SUSPECTED (block)", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/__tests__/drift/providers.ts", "src/helpers.ts"], - report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.SUPPRESSION_SUSPECTED); - expect(verdict.offendingFiles).toContain("src/__tests__/drift/providers.ts"); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(12); - }); - - it("harness-only leg edit (ws-providers.ts), no production change → COMPARISON_LEG_ONLY (block, pure relaxation)", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/__tests__/drift/ws-providers.ts"], - report: report(), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - // Leg edit with NO production change → COMPARISON_LEG_ONLY (a pure - // relaxation; no mock fix even attempted). Still a hard block (exit 11). - expect(verdict.reason).toBe(PredicateReason.COMPARISON_LEG_ONLY); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(11); - }); - - // ------------------------------------------------------------------------- - // FIX #2 — dual-classification precedence: voice-models.ts is BOTH a harness - // leg AND a legit fixture target. Block-classification MUST win (fail-closed). - // ------------------------------------------------------------------------- - it("voice-models.ts (dual-classified harness+target) + on-target production edit → SUPPRESSION_SUSPECTED (block wins)", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/__tests__/drift/voice-models.ts", "src/ws-realtime.ts"], - report: report([entry({ builderFile: "src/ws-realtime.ts", typesFile: null })]), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.SUPPRESSION_SUSPECTED); - expect(verdict.offendingFiles).toContain("src/__tests__/drift/voice-models.ts"); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(12); - }); - - // ------------------------------------------------------------------------- - // FIX #3 — empty sanctioned-target set must fail closed (needs-human), not - // silently accept any production change by disabling the off-target guard. - // ------------------------------------------------------------------------- - it("empty sanctionedTargets (report entries have no usable target) → PRODUCTION_CHANGE_OFF_TARGET (fail-closed)", () => { - // Fabricate a report whose entries yield ZERO sanctioned targets: builderFile - // "" and typesFile null. (evaluateDriftResolved does not re-validate the - // report shape — it only reads builderFile/typesFile via sanctionedTargets.) - const emptyTargetReport: DriftReport = { - timestamp: "2026-07-16T00:00:00.000Z", - entries: [ - { - provider: "OpenAI", - scenario: "chat completion", - builderFile: "", - builderFunctions: ["buildChatCompletion"], - typesFile: null, - sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", - diffs: [diff()], - }, - ], - }; - expect(sanctionedTargets(emptyTargetReport).size).toBe(0); - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts"], - report: emptyTargetReport, - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.PRODUCTION_CHANGE_OFF_TARGET); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(16); - }); - - // ------------------------------------------------------------------------- - // FIX #6 — exit 5/1 WITH parseable criticalCount>0 gets its OWN reason - // (quarantine/infra), NOT STILL_DIRTY. The collector-state classification - // wins over the belt-and-suspenders criticalCount check. - // ------------------------------------------------------------------------- - it("post-fix quarantine (exit 5) with criticalCount>0 → QUARANTINE_AFTER_FIX (not STILL_DIRTY)", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts"], - report: report(), - postFixCollectorExit: 5, - postFixCriticalCount: 4, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.QUARANTINE_AFTER_FIX); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(14); - }); - - it("post-fix infra (exit 1) with criticalCount>0 → COLLECTOR_INFRA (not STILL_DIRTY)", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts"], - report: report(), - postFixCollectorExit: 1, - postFixCriticalCount: 4, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.COLLECTOR_INFRA); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(15); - }); -}); - -// --------------------------------------------------------------------------- -// GREEN cases — resolved:true -// --------------------------------------------------------------------------- - -describe("evaluateDriftResolved — GREEN (real fix) cases", () => { - it("real src/helpers.ts fix + clean collector → RESOLVED", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts", "src/types.ts"], - report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(true); - expect(verdict.reason).toBe(PredicateReason.RESOLVED); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(0); - }); - - it("legit canary: model-registry.ts + ws-realtime.ts (report sanctions BOTH) → RESOLVED", () => { - // The known-models canary routes its fix to the production ws-realtime.ts - // (builderFile) AND the model list fixture (typesFile). Under the allowlist a - // fixture is accepted ONLY when the report names it as a target — so the - // report here sanctions model-registry.ts via typesFile. - const canary = report([ - entry({ - provider: "OpenAI Realtime", - scenario: "known-models canary", - builderFile: "src/ws-realtime.ts", - builderFunctions: ["buildRealtimeSession"], - typesFile: "src/__tests__/drift/model-registry.ts", - diffs: [ - diff({ path: "knownModels", issue: "Unknown realtime model detected", mock: "" }), - ], - }), - ]); - const verdict = evaluateDriftResolved({ - changedFiles: ["src/__tests__/drift/model-registry.ts", "src/ws-realtime.ts"], - report: canary, - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(true); - expect(verdict.reason).toBe(PredicateReason.RESOLVED); - }); - - it("AG-UI: report names src/agui-types.ts; change to that file → RESOLVED", () => { - const agui = report([ - entry({ - provider: "AG-UI", - scenario: "missing event types", - builderFile: "src/agui-types.ts", - builderFunctions: ["AGUIEventType"], - typesFile: "src/agui-types.ts", - sdkShapesFile: "src/__tests__/drift/agui-schema.drift.ts", - diffs: [diff({ path: "AGUIEventType", issue: "missing event type" })], - }), - ]); - const verdict = evaluateDriftResolved({ - changedFiles: ["src/agui-types.ts"], - report: agui, - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(true); - expect(verdict.reason).toBe(PredicateReason.RESOLVED); - }); - - it("production change + accompanying report-NAMED canary fixture → RESOLVED", () => { - // model-family.ts is accepted as an accompanying change ONLY because the - // report names it (typesFile) — under the allowlist a fixture is not a free - // pass by static membership; it must be sanctioned by the report for this run. - const verdict = evaluateDriftResolved({ - changedFiles: ["src/ws-realtime.ts", "src/__tests__/drift/model-family.ts"], - report: report([ - entry({ - builderFile: "src/ws-realtime.ts", - typesFile: "src/__tests__/drift/model-family.ts", - }), - ]), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(true); - expect(verdict.reason).toBe(PredicateReason.RESOLVED); - }); -}); - -// --------------------------------------------------------------------------- -// ALLOWLIST INVERSION (round-2 CR F1/F2/F3) — a fix is RESOLVED only when EVERY -// changed file is on the allowlist. Anything not recognized as production source -// or a report-sanctioned fixture target BLOCKS. This closes path-spelling -// sneak-ins, stale-denylist gaps, and in-diff vectors (package.json / lockfiles -// / sub-fixtures / unknown paths). -// --------------------------------------------------------------------------- - -describe("allowlist inversion — non-allowlisted changed files ALWAYS block", () => { - // A report that sanctions src/helpers.ts as the fix target, so the production - // edit itself is legitimately allowlisted; the SECOND file is the attack. - const sanctioned = () => - report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]); - - const cleanSignal = { postFixCollectorExit: 0, postFixCriticalCount: 0 }; - - it("package.json changed alongside a real production fix → UNSANCTIONED_CHANGE (block)", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts", "package.json"], - report: sanctioned(), - ...cleanSignal, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); - expect(verdict.offendingFiles).toContain("package.json"); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(17); - }); - - it("pnpm-lock.yaml changed alongside a real production fix → UNSANCTIONED_CHANGE (block)", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts", "pnpm-lock.yaml"], - report: sanctioned(), - ...cleanSignal, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); - expect(verdict.offendingFiles).toContain("pnpm-lock.yaml"); - }); - - it("tsconfig.json changed alongside a real production fix → UNSANCTIONED_CHANGE (block)", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts", "tsconfig.json"], - report: sanctioned(), - ...cleanSignal, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); - }); - - it("an unknown/unrecognized path alongside a real production fix → UNSANCTIONED_CHANGE (block)", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts", "scripts/fix-drift.ts"], - report: sanctioned(), - ...cleanSignal, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); - expect(verdict.offendingFiles).toContain("scripts/fix-drift.ts"); - }); - - it("a drift-dir *.test.ts (NOT a *.drift.ts) alongside a real production fix → UNSANCTIONED_CHANGE (closes the drift-dir .test.ts gap)", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts", "src/__tests__/drift/model-registry.test.ts"], - report: sanctioned(), - ...cleanSignal, - }); - expect(verdict.resolved).toBe(false); - // A drift-dir unit test is not a gameable comparison leg (isGameableLeg is - // false for it) and is not allowlisted → UNSANCTIONED_CHANGE. - expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); - expect(verdict.offendingFiles).toContain("src/__tests__/drift/model-registry.test.ts"); - }); - - it("a non-drift __tests__ file alongside a real production fix → UNSANCTIONED_CHANGE (block)", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts", "src/__tests__/server.test.ts"], - report: sanctioned(), - ...cleanSignal, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); - }); - - it("model-registry.ts fixture NOT named by the report + prod fix → UNSANCTIONED_CHANGE (fixtures allowlisted ONLY when report-named)", () => { - // model-registry.ts is a legit fixture target but the report does NOT name it - // (builderFile/typesFile are src/helpers.ts / src/types.ts). It is therefore - // NOT on the allowlist for THIS run and must block. - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts", "src/__tests__/drift/model-registry.ts"], - report: sanctioned(), - ...cleanSignal, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); - expect(verdict.offendingFiles).toContain("src/__tests__/drift/model-registry.ts"); - }); - - it("FIX #F2 (round-4): a report-NAMED fixture target ALONE (no production change) → NO_PRODUCTION_CHANGE (routed to human, NOT auto-resolved)", () => { - // A canary that names ONLY the fixture as its target and whose diff touches - // ONLY that fixture (model-registry.ts) — with NO production/builder change. - // The module invariant (Signal 2) requires >=1 production mock-builder change - // for RESOLVED: a fixture-target-only change is not independently verifiable - // (the re-collect reads the same fixture), so it must route to human review, - // never auto-resolve. This is the canary-only bypass F2 closes. - const canary = report([ - entry({ - provider: "OpenAI Realtime", - scenario: "known-models canary", - builderFile: "src/__tests__/drift/model-registry.ts", - builderFunctions: ["knownModels"], - typesFile: null, - diffs: [diff({ path: "knownModels", issue: "new model shipped" })], - }), - ]); - const verdict = evaluateDriftResolved({ - changedFiles: ["src/__tests__/drift/model-registry.ts"], - report: canary, - ...cleanSignal, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.NO_PRODUCTION_CHANGE); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(10); - }); - - it("FIX #F2 (round-4): a report-NAMED fixture target ACCOMPANIED BY a production change → RESOLVED (the legit canary shape)", () => { - // The legit canary shape: the report names BOTH a production builder and the - // fixture, and BOTH change. The production change satisfies the >=1 - // production-change invariant, so this auto-resolves (unlike the - // fixture-only case above). - const canary = report([ - entry({ - provider: "OpenAI Realtime", - scenario: "known-models canary", - builderFile: "src/ws-realtime.ts", - builderFunctions: ["buildRealtimeSession"], - typesFile: "src/__tests__/drift/model-registry.ts", - diffs: [diff({ path: "knownModels", issue: "new model shipped" })], - }), - ]); - const verdict = evaluateDriftResolved({ - changedFiles: ["src/ws-realtime.ts", "src/__tests__/drift/model-registry.ts"], - report: canary, - ...cleanSignal, - }); - expect(verdict.resolved).toBe(true); - expect(verdict.reason).toBe(PredicateReason.RESOLVED); - }); - - it("production-source-only fix (no fixtures at all) + clean collector → RESOLVED", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts"], - report: sanctioned(), - ...cleanSignal, - }); - expect(verdict.resolved).toBe(true); - expect(verdict.reason).toBe(PredicateReason.RESOLVED); - }); -}); - -// --------------------------------------------------------------------------- -// PATH CANONICALIZATION (round-2 CR slot-1/slot-2 F1) — a leg edit presented -// under an equivalent-but-non-identical spelling must still be recognized and -// blocked; classification runs on the canonical form. -// --------------------------------------------------------------------------- - -describe("path canonicalization defeats spelling-variant leg sneak-ins", () => { - it("./src/... leading-dot spelling of the SDK leg still blocks", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["./src/__tests__/drift/sdk-shapes.ts"], - report: report(), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.COMPARISON_LEG_ONLY); - }); - - it("doubled-slash spelling of the SDK leg + prod edit still blocks (SUPPRESSION_SUSPECTED)", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src//__tests__/drift/sdk-shapes.ts", "src/helpers.ts"], - report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.SUPPRESSION_SUSPECTED); - }); - - it("trailing-dot-segment spelling of a production target canonicalizes and RESOLVES", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["./src/helpers.ts"], - report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(true); - expect(verdict.reason).toBe(PredicateReason.RESOLVED); - }); - - it("canonicalizePath normalizes ./ , // and . segments", () => { - expect(canonicalizePath("./src/helpers.ts")).toBe("src/helpers.ts"); - expect(canonicalizePath("src//__tests__/drift/sdk-shapes.ts")).toBe( - "src/__tests__/drift/sdk-shapes.ts", - ); - expect(canonicalizePath("src/./helpers.ts")).toBe("src/helpers.ts"); - expect(canonicalizePath("src/helpers.ts")).toBe("src/helpers.ts"); - }); - - it("canonicalizePath rejects a path escaping the repo root (fail-closed)", () => { - expect(() => canonicalizePath("../ag-ui/events.ts")).toThrow(PredicateConfigError); - expect(() => canonicalizePath("src/../../etc/passwd")).toThrow(PredicateConfigError); - }); -}); - -// --------------------------------------------------------------------------- -// F6 — empty / non-integer --post-fix-exit must FAIL CLOSED (Number("")===0 -// must NOT be treated as a clean exit 0). -// --------------------------------------------------------------------------- - -describe("F6 — --post-fix-exit fails closed on empty/whitespace", () => { - it("throws on an empty --post-fix-exit (Number('')===0 must not slip through)", () => { - expect(() => - parseCliArgs(["--report", "a.json", "--post-fix-report", "b.json", "--post-fix-exit", ""]), - ).toThrow(PredicateConfigError); - }); - - it("throws on a whitespace-only --post-fix-exit", () => { - expect(() => - parseCliArgs(["--report", "a.json", "--post-fix-report", "b.json", "--post-fix-exit", " "]), - ).toThrow(PredicateConfigError); - }); -}); - -// --------------------------------------------------------------------------- -// F8 — a malformed post-fix report that crashes countCriticalDiffs must be -// caught and mapped to a NAMED config-error reason, not a bare stacktrace. -// --------------------------------------------------------------------------- - -describe("F8 — malformed post-fix report → named config-error (not a bare crash)", () => { - let dir: string | null = null; - afterEach(() => { - if (dir) rmSync(dir, { recursive: true, force: true }); - dir = null; - }); - - it("runCli exits 2 (CONFIG_ERROR) when the post-fix report has entries with no diffs array", () => { - dir = mkdtempSync(join(tmpdir(), "ws2-f8-")); - const preP = join(dir, "drift-report.json"); - const postP = join(dir, "drift-report.post-fix.json"); - writeFileSync(preP, JSON.stringify(report()), "utf-8"); - // Structurally passes readReport (timestamp + entries array) but each entry - // is missing `diffs`, so countCriticalDiffs would throw. - writeFileSync( - postP, - JSON.stringify({ timestamp: "t", entries: [{ provider: "OpenAI" }] }), - "utf-8", - ); - const code = runCli(["--report", preP, "--post-fix-report", postP, "--post-fix-exit", "0"]); - expect(code).toBe(REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]); - }); -}); - -// --------------------------------------------------------------------------- -// File-classification unit coverage -// --------------------------------------------------------------------------- - -describe("file classification", () => { - it("isProductionFile matches src/** except src/__tests__/**", () => { - expect(isProductionFile("src/helpers.ts")).toBe(true); - expect(isProductionFile("src/agui-types.ts")).toBe(true); - expect(isProductionFile("src/__tests__/drift/sdk-shapes.ts")).toBe(false); - expect(isProductionFile("scripts/fix-drift.ts")).toBe(false); - }); - - it("isComparisonLeg flags SDK/schema/harness/*.drift.ts (incl dual-classified voice-models) but NOT pure legit targets", () => { - expect(isComparisonLeg("src/__tests__/drift/sdk-shapes.ts")).toBe(true); - expect(isComparisonLeg("src/__tests__/drift/schema.ts")).toBe(true); - expect(isComparisonLeg("src/__tests__/drift/providers.ts")).toBe(true); - expect(isComparisonLeg("src/__tests__/drift/ws-providers.ts")).toBe(true); - expect(isComparisonLeg("src/__tests__/drift/helpers.ts")).toBe(true); - expect(isComparisonLeg("src/__tests__/drift/openai-chat.drift.ts")).toBe(true); - // Dual-classified voice-models.ts blocks (fix #2: harness membership wins). - expect(isComparisonLeg("src/__tests__/drift/voice-models.ts")).toBe(true); - // PURE legit fixture targets (not also a harness leg) are NOT comparison legs. - expect(isComparisonLeg("src/__tests__/drift/model-registry.ts")).toBe(false); - expect(isComparisonLeg("src/__tests__/drift/model-family.ts")).toBe(false); - // Production files are not comparison legs. - expect(isComparisonLeg("src/helpers.ts")).toBe(false); - }); - - it("isSuppressionSurface flags the NARROW always-suppress set (schema + *.drift.ts) only", () => { - // Suppression surface = the actively-silencing subset: schema/allowlist and - // *.drift.ts assertions. These always map to SUPPRESSION_SUSPECTED. The - // broader legs (sdk-shapes / harness) are gameable but are NOT suppression - // surfaces (they map to COMPARISON_LEG_ONLY when standalone). - expect(isSuppressionSurface("src/__tests__/drift/schema.ts")).toBe(true); - expect(isSuppressionSurface("src/__tests__/drift/openai-chat.drift.ts")).toBe(true); - expect(isSuppressionSurface("src/__tests__/drift/sdk-shapes.ts")).toBe(false); - expect(isSuppressionSurface("src/__tests__/drift/providers.ts")).toBe(false); - expect(isSuppressionSurface("src/__tests__/drift/voice-models.ts")).toBe(false); - expect(isSuppressionSurface("src/helpers.ts")).toBe(false); - }); - - it("isGameableLeg flags every leg (incl dual-classified voice-models) but NOT model-registry/model-family/production", () => { - expect(isGameableLeg("src/__tests__/drift/sdk-shapes.ts")).toBe(true); - expect(isGameableLeg("src/__tests__/drift/schema.ts")).toBe(true); - expect(isGameableLeg("src/__tests__/drift/providers.ts")).toBe(true); - expect(isGameableLeg("src/__tests__/drift/ws-providers.ts")).toBe(true); - expect(isGameableLeg("src/__tests__/drift/helpers.ts")).toBe(true); - expect(isGameableLeg("src/__tests__/drift/openai-chat.drift.ts")).toBe(true); - // Dual-classified: harness membership wins over legit-target (fix #2). - expect(isGameableLeg("src/__tests__/drift/voice-models.ts")).toBe(true); - // Pure legit fixture targets are NOT gameable legs. - expect(isGameableLeg("src/__tests__/drift/model-registry.ts")).toBe(false); - expect(isGameableLeg("src/__tests__/drift/model-family.ts")).toBe(false); - expect(isGameableLeg("src/helpers.ts")).toBe(false); - }); - - it("isAllowlisted: production source is allowed; config/manifests/tests are not", () => { - const targets = new Set(["src/helpers.ts", "src/__tests__/drift/model-registry.ts"]); - // Production source (non-test) is always allowlisted. - expect(isAllowlisted("src/helpers.ts", targets)).toBe(true); - expect(isAllowlisted("src/agui-types.ts", targets)).toBe(true); - // A report-named fixture target is allowlisted. - expect(isAllowlisted("src/__tests__/drift/model-registry.ts", targets)).toBe(true); - // A fixture NOT named by the report is not allowlisted. - expect(isAllowlisted("src/__tests__/drift/model-family.ts", targets)).toBe(false); - // Config/manifests/lockfiles/unknown paths are not allowlisted. - expect(isAllowlisted("package.json", targets)).toBe(false); - expect(isAllowlisted("pnpm-lock.yaml", targets)).toBe(false); - expect(isAllowlisted("tsconfig.json", targets)).toBe(false); - expect(isAllowlisted("scripts/fix-drift.ts", targets)).toBe(false); - // A non-production src config-ish manifest is not allowlisted. - expect(isAllowlisted("src/__tests__/server.test.ts", targets)).toBe(false); - }); - - it("sanctionedTargets unions builderFile + non-null typesFile", () => { - const t = sanctionedTargets( - report([ - entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" }), - entry({ builderFile: "src/gemini.ts", typesFile: null }), - ]), - ); - expect(t.has("src/helpers.ts")).toBe(true); - expect(t.has("src/types.ts")).toBe(true); - expect(t.has("src/gemini.ts")).toBe(true); - expect(t.has("src/__tests__/drift/sdk-shapes.ts")).toBe(false); - }); - - it("countCriticalDiffs counts only critical severities", () => { - const r = report([ - entry({ - diffs: [ - diff({ severity: "critical" }), - diff({ severity: "warning" }), - diff({ severity: "critical" }), - ], - }), - ]); - expect(countCriticalDiffs(r)).toBe(2); - }); -}); - -// --------------------------------------------------------------------------- -// CLI arg parsing + config errors (exit 2) -// --------------------------------------------------------------------------- - -describe("parseCliArgs", () => { - it("parses a full valid arg set", () => { - const args = parseCliArgs([ - "--report", - "a.json", - "--post-fix-report", - "b.json", - "--post-fix-exit", - "0", - "--changed-file", - "src/helpers.ts", - "--changed-file", - "src/types.ts", - ]); - expect(args.reportPath).toBe("a.json"); - expect(args.postFixReportPath).toBe("b.json"); - expect(args.postFixExit).toBe(0); - expect(args.changedFiles).toEqual(["src/helpers.ts", "src/types.ts"]); - }); - - it("null changedFiles when no --changed-file flag (derive from git later)", () => { - const args = parseCliArgs([ - "--report", - "a.json", - "--post-fix-report", - "b.json", - "--post-fix-exit", - "2", - ]); - expect(args.changedFiles).toBeNull(); - }); - - it("throws on missing --report", () => { - expect(() => parseCliArgs(["--post-fix-report", "b.json", "--post-fix-exit", "0"])).toThrow( - PredicateConfigError, - ); - }); - - it("throws on non-integer --post-fix-exit", () => { - expect(() => - parseCliArgs(["--report", "a.json", "--post-fix-report", "b.json", "--post-fix-exit", "abc"]), - ).toThrow(PredicateConfigError); - }); - - it("throws on unknown argument", () => { - expect(() => parseCliArgs(["--nope"])).toThrow(PredicateConfigError); - }); -}); - -describe("readReport", () => { - let dir: string | null = null; - afterEach(() => { - if (dir) rmSync(dir, { recursive: true, force: true }); - dir = null; - }); - - it("throws PredicateConfigError on a missing file", () => { - expect(() => readReport("/no/such/drift-report.json")).toThrow(PredicateConfigError); - }); - - // FIX #6 — align the strict/loose validators. readReport must fail-closed on a - // structurally-untrustworthy report the same way fix-drift.ts:readDriftReport - // does: a missing/non-string `timestamp` is a corrupt/truncated collector run - // and must be REJECTED, never silently trusted as a clean signal. - it("rejects a report missing the timestamp field (fail-closed, aligns with readDriftReport)", () => { - dir = mkdtempSync(join(tmpdir(), "ws2-readreport-")); - const p = join(dir, "no-ts.json"); - writeFileSync(p, JSON.stringify({ entries: [] }), "utf-8"); - expect(() => readReport(p)).toThrow(PredicateConfigError); - }); - - it("rejects a non-object / non-{entries:[]} structure", () => { - dir = mkdtempSync(join(tmpdir(), "ws2-readreport-")); - const p = join(dir, "bad.json"); - writeFileSync(p, JSON.stringify({ timestamp: "t", entries: "nope" }), "utf-8"); - expect(() => readReport(p)).toThrow(PredicateConfigError); - }); - - // A legitimately-clean post-fix report IS { entries: [] } — the collector - // emits exactly that when no drift remains. It must be ACCEPTED (the trust - // anchor for "clean" is the collector EXIT CODE, corroborated by fix #1's - // always-block-on-leg-edit rule, not the entries array being non-empty). - it("accepts a well-formed EMPTY report (the genuine clean-collector signal)", () => { - dir = mkdtempSync(join(tmpdir(), "ws2-readreport-")); - const p = join(dir, "clean.json"); - writeFileSync(p, JSON.stringify({ timestamp: "t", entries: [] }), "utf-8"); - expect(() => readReport(p)).not.toThrow(); - }); - - it("accepts a well-formed non-empty report", () => { - dir = mkdtempSync(join(tmpdir(), "ws2-readreport-")); - const p = join(dir, "ok.json"); - writeFileSync(p, JSON.stringify(report()), "utf-8"); - expect(() => readReport(p)).not.toThrow(); - }); -}); - -describe("parsePorcelainLine", () => { - it("strips the 2-char status + space prefix", () => { - expect(parsePorcelainLine(" M src/helpers.ts")).toBe("src/helpers.ts"); - expect(parsePorcelainLine("?? src/new.ts")).toBe("src/new.ts"); - expect(parsePorcelainLine("A src/added.ts")).toBe("src/added.ts"); - }); - - it("takes the NEW path from rename notation (old -> new)", () => { - expect(parsePorcelainLine("R src/old.ts -> src/new.ts")).toBe("src/new.ts"); - }); - - it("unquotes paths with special characters", () => { - expect(parsePorcelainLine('?? "src/weird name.ts"')).toBe("src/weird name.ts"); - expect(parsePorcelainLine('R "src/old x.ts" -> "src/new x.ts"')).toBe("src/new x.ts"); - }); -}); - -// --------------------------------------------------------------------------- -// runCli end-to-end (in-process): exit codes over real temp report files -// --------------------------------------------------------------------------- - -describe("runCli", () => { - let dir: string | null = null; - - afterEach(() => { - if (dir) rmSync(dir, { recursive: true, force: true }); - dir = null; - }); - - function writeReports(pre: DriftReport, post: DriftReport): { pre: string; post: string } { - dir = mkdtempSync(join(tmpdir(), "ws2-predicate-")); - const preP = join(dir, "drift-report.json"); - const postP = join(dir, "drift-report.post-fix.json"); - writeFileSync(preP, JSON.stringify(pre), "utf-8"); - writeFileSync(postP, JSON.stringify(post), "utf-8"); - return { pre: preP, post: postP }; - } - - // NOTE (fix #4): runCli now ALWAYS cross-checks any --changed-file list - // against the real git working tree, so a synthetic list that does not match - // the checkout is rejected (exit 2) BEFORE the predicate runs. The exit-code- - // per-reason mapping for the cheat / real-fix cases is covered directly by the - // evaluateDriftResolved + REASON_EXIT_CODE tests above; here we lock the - // cross-check itself (the fix #4 blinding guard) at the runCli boundary. - it("exits 2 (CONFIG_ERROR) when --changed-file disagrees with git (fix #4 blinding guard)", () => { - const paths = writeReports(report(), report([])); - const code = runCli([ - "--report", - paths.pre, - "--post-fix-report", - paths.post, - "--post-fix-exit", - "0", - "--changed-file", - "src/__tests__/drift/sdk-shapes.ts", - ]); - expect(code).toBe(2); - }); - - it("exits 2 (CONFIG_ERROR) on a missing post-fix report", () => { - const paths = writeReports(report(), report()); - const code = runCli([ - "--report", - paths.pre, - "--post-fix-report", - join(dir!, "does-not-exist.json"), - "--post-fix-exit", - "0", - "--changed-file", - "src/helpers.ts", - ]); - expect(code).toBe(2); - }); - - it("exits 2 (CONFIG_ERROR) on malformed args", () => { - const code = runCli(["--bogus"]); - expect(code).toBe(2); - }); -}); - -// --------------------------------------------------------------------------- -// FIX #4 — authoritative changed-files: crossCheckChangedFiles -// --------------------------------------------------------------------------- - -describe("crossCheckChangedFiles", () => { - it("returns the git set when no explicit list is supplied", () => { - const git = ["src/helpers.ts", "src/types.ts"]; - expect(crossCheckChangedFiles(null, git)).toEqual(git); - }); - - it("accepts an explicit list that matches the git set (order-independent)", () => { - const git = ["src/helpers.ts", "src/types.ts"]; - expect(crossCheckChangedFiles(["src/types.ts", "src/helpers.ts"], git)).toEqual( - expect.arrayContaining(git), - ); - }); - - it("throws when the explicit list OMITS a file git reports (leg-blinding attack)", () => { - const git = ["src/helpers.ts", "src/__tests__/drift/sdk-shapes.ts"]; - // Attacker passes only the benign production file, hiding the relaxed leg. - expect(() => crossCheckChangedFiles(["src/helpers.ts"], git)).toThrow(PredicateConfigError); - }); - - it("throws when the explicit list ADDS a file git does not report", () => { - const git = ["src/helpers.ts"]; - expect(() => crossCheckChangedFiles(["src/helpers.ts", "src/phantom.ts"], git)).toThrow( - PredicateConfigError, - ); - }); -}); - -// --------------------------------------------------------------------------- -// REVERT GUARD + RED-COUNT LOCK -// --------------------------------------------------------------------------- - -describe("revert guard — old always-accept predicate would FAIL these locks", () => { - // The pre-hardening predicate ignored gameable legs once ANY production file - // changed (and had a legacy always-accept fallback). This models that old - // behavior; asserting it DISAGREES with the real predicate on the WS-2b cheat - // proves the hardening is load-bearing. If someone reverts evaluateDriftResolved - // to the old logic, the HEADLINE WS-2b test above flips and the suite breaks. - function oldPredicateWouldAccept(changedFiles: string[]): boolean { - const productionFiles = changedFiles.filter(isProductionFile); - // Old logic: leg files only checked when productionFiles === 0. - return productionFiles.length > 0; - } - - it("WS-2b cheat: old predicate ACCEPTS, real predicate BLOCKS (SUPPRESSION_SUSPECTED)", () => { - const cheat = ["src/__tests__/drift/sdk-shapes.ts", "src/helpers.ts"]; - // Old behavior would have accepted (a production file is present). - expect(oldPredicateWouldAccept(cheat)).toBe(true); - // Real predicate blocks. - const verdict = evaluateDriftResolved({ - changedFiles: cheat, - report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.SUPPRESSION_SUSPECTED); - }); - - // ALLOWLIST-INVERSION revert lock: the OLD denylist only blocked KNOWN gameable - // legs, so an in-diff vector NOT on the denylist (package.json, a lockfile, an - // unknown path) paired with an on-target production edit sailed through to - // RESOLVED. The allowlist inverts that: anything not explicitly allowed blocks. - // Reverting evaluateDriftResolved to a denylist model flips these to RESOLVED - // and breaks the suite. - function denylistWouldAccept(changedFiles: string[]): boolean { - // Old denylist model: block ONLY if a changed file is a known gameable leg; - // otherwise (production + package.json/lockfile/unknown) accept. - return !changedFiles.some(isGameableLeg) && changedFiles.some(isProductionFile); - } - - it("in-diff vector (package.json + prod): old DENYLIST accepts, allowlist BLOCKS (UNSANCTIONED_CHANGE)", () => { - const vector = ["src/helpers.ts", "package.json"]; - // The old denylist would have accepted (no known leg in the set). - expect(denylistWouldAccept(vector)).toBe(true); - const verdict = evaluateDriftResolved({ - changedFiles: vector, - report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); - }); -}); - -describe("exit-code distinctness lock", () => { - // Structural lock on the reason→exit-code contract: every block reason maps to - // a distinct NON-ZERO exit code and RESOLVED alone is 0. This does NOT count - // RED test cases (it asserts the code table, not the number of scenarios) — it - // guarantees the workflow can route each cause to its own Slack DETAIL without - // two reasons colliding on one exit code. - const BLOCK_REASONS = [ - PredicateReason.NO_PRODUCTION_CHANGE, - PredicateReason.COMPARISON_LEG_ONLY, - PredicateReason.SUPPRESSION_SUSPECTED, - PredicateReason.UNSANCTIONED_CHANGE, - PredicateReason.STILL_DIRTY, - PredicateReason.QUARANTINE_AFTER_FIX, - PredicateReason.COLLECTOR_INFRA, - PredicateReason.PRODUCTION_CHANGE_OFF_TARGET, - PredicateReason.CONFIG_ERROR, - ]; - - it("every block reason has a distinct non-zero exit code and RESOLVED is 0", () => { - expect(REASON_EXIT_CODE[PredicateReason.RESOLVED]).toBe(0); - const codes = BLOCK_REASONS.map((r) => REASON_EXIT_CODE[r]); - for (const c of codes) expect(c).not.toBe(0); - expect(new Set(codes).size).toBe(codes.length); // all distinct - }); -}); - -// --------------------------------------------------------------------------- -// CR round-3 slot-3 LOW gaps — unrecognized collector exit code + `..`/absolute -// canonicalization variants driven end-to-end through the predicate/runCli. -// --------------------------------------------------------------------------- - -describe("unrecognized collector exit code fails closed → COLLECTOR_INFRA", () => { - // The `postFixCollectorExit !== 0` catch-all (COLLECTOR_INFRA / exit 15) had - // no locking test. Tests only fed 0/1/2/5. A future refactor could let an - // unknown exit fall through to a clean accept — this pins it closed. - for (const exit of [3, 7, 99]) { - it(`exit ${exit} (unrecognized) with a production change → COLLECTOR_INFRA / exit 15`, () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts"], - report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), - postFixCollectorExit: exit, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.COLLECTOR_INFRA); - expect(REASON_EXIT_CODE[verdict.reason]).toBe(15); - }); - } -}); - -describe("`..`-segment and absolute path variants block/fail-closed THROUGH the predicate", () => { - it("a `..`-containing spelling of the SDK leg canonicalizes and BLOCKS (COMPARISON_LEG_ONLY)", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/__tests__/drift/foo/../sdk-shapes.ts"], - report: report(), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.COMPARISON_LEG_ONLY); - }); - - it("an absolute changed-file path throws PredicateConfigError from evaluateDriftResolved", () => { - expect(() => - evaluateDriftResolved({ - changedFiles: ["/etc/passwd"], - report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }), - ).toThrow(PredicateConfigError); - }); - - it("a `..`-escaping changed-file path throws PredicateConfigError from evaluateDriftResolved", () => { - expect(() => - evaluateDriftResolved({ - changedFiles: ["src/../../etc/passwd"], - report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), - postFixCollectorExit: 0, - postFixCriticalCount: 0, - }), - ).toThrow(PredicateConfigError); - }); -}); - -describe("runCli maps a repo-escaping/absolute changed-file to CONFIG_ERROR (exit 2)", () => { - let dir: string | null = null; - afterEach(() => { - if (dir) rmSync(dir, { recursive: true, force: true }); - dir = null; - }); - - // Drive an absolute/`..`-escaping path all the way through runCli. A supplied - // --changed-file cross-checks against git first, so an absolute path that git - // never reports would be rejected by the cross-check (still exit 2). To pin the - // fix #8 canonicalize-throw path specifically, we assert the CONFIG_ERROR exit. - it("runCli exits 2 (CONFIG_ERROR) when a --changed-file is an absolute path", () => { - dir = mkdtempSync(join(tmpdir(), "ws2-canon-")); - const preP = join(dir, "drift-report.json"); - const postP = join(dir, "drift-report.post-fix.json"); - writeFileSync(preP, JSON.stringify(report()), "utf-8"); - writeFileSync(postP, JSON.stringify(report([])), "utf-8"); - const code = runCli([ - "--report", - preP, - "--post-fix-report", - postP, - "--post-fix-exit", - "0", - "--changed-file", - "/etc/passwd", - ]); - expect(code).toBe(REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]); - }); -}); - -// --------------------------------------------------------------------------- -// FIX #F3 (round-4) — a collector-output artifact left in the repo working tree -// (drift-report.post-fix.json / drift-report.json / claude-code-output.log) -// appears as an untracked file in `git status --porcelain` and is scored by the -// predicate as UNSANCTIONED_CHANGE, breaking the happy path. The workflow fix -// writes those artifacts to $RUNNER_TEMP (outside the checkout) so they never -// enter the changed-file set. These pure-function locks prove the predicate's -// behaviour on BOTH sides of that move. -// --------------------------------------------------------------------------- -describe("FIX #F3 — collector-output artifacts must not be in the changed-file set", () => { - const sanctioned = () => - report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]); - const cleanSignal = { postFixCollectorExit: 0, postFixCriticalCount: 0 }; - - it("RED (the bug): a post-fix report artifact in the changed set → UNSANCTIONED_CHANGE (happy path broken)", () => { - // This is exactly what happened when the re-collect wrote into the repo cwd: - // the untracked drift-report.post-fix.json joined the changed set and, being - // neither production source nor a report-named target, fail-closed the run. - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts", "drift-report.post-fix.json"], - report: sanctioned(), - ...cleanSignal, - }); - expect(verdict.resolved).toBe(false); - expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); - expect(verdict.offendingFiles).toContain("drift-report.post-fix.json"); - }); - - it("GREEN (the fix): the SAME production fix WITHOUT the artifact (now in $RUNNER_TEMP) → RESOLVED", () => { - const verdict = evaluateDriftResolved({ - changedFiles: ["src/helpers.ts"], - report: sanctioned(), - ...cleanSignal, - }); - expect(verdict.resolved).toBe(true); - expect(verdict.reason).toBe(PredicateReason.RESOLVED); - }); -}); - -// --------------------------------------------------------------------------- -// TEST-TIGHTNESS (round-4 slot-3) — lock the runCli behaviour the workflow -// depends on: (F2) the machine-readable `reason=` stdout line the -// "Assert" step greps for Slack routing, and (F1) the CONFIG_ERROR path so -// deleting the parse-error catch fails a test. These drive the REAL runCli in an -// isolated temp git repo so gitChangedFiles() returns a controlled set. -// --------------------------------------------------------------------------- -describe("runCli machine-readable reason= line + CONFIG_ERROR lock (slot-3 F1/F2)", () => { - let repo: string | null = null; - let logSpy: ReturnType | null = null; - let errSpy: ReturnType | null = null; - const origCwd = process.cwd(); - - afterEach(() => { - process.chdir(origCwd); - logSpy?.mockRestore(); - errSpy?.mockRestore(); - logSpy = null; - errSpy = null; - if (repo) rmSync(repo, { recursive: true, force: true }); - repo = null; - }); - - function initRepo(): string { - const dir = mkdtempSync(join(tmpdir(), "ws2-runcli-")); - execFileSync("git", ["init", "-q"], { cwd: dir }); - execFileSync("git", ["config", "user.email", "t@t"], { cwd: dir }); - execFileSync("git", ["config", "user.name", "t"], { cwd: dir }); - return dir; - } - - function captureConsole(): void { - logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - } - - function stdoutLines(): string[] { - return (logSpy?.mock.calls ?? []).map((c) => String(c[0])); - } - - it("prints `reason=resolved` on a genuine RESOLVED verdict (the line the workflow greps)", () => { - repo = initRepo(); - // A real production change in the working tree → gitChangedFiles() reports it. - // `git add -N` (intent-to-add) makes porcelain list the INDIVIDUAL file - // (`A src/helpers.ts`) rather than collapsing an all-untracked dir to `?? src/`. - mkdirSync(join(repo, "src"), { recursive: true }); - writeFileSync(join(repo, "src", "helpers.ts"), "export const x = 1;\n", "utf-8"); - execFileSync("git", ["add", "-N", "src/helpers.ts"], { cwd: repo }); - // Report files live OUTSIDE the repo (mirrors the workflow's $RUNNER_TEMP, - // FIX #F3) so they are NOT untracked entries in `git status --porcelain` and - // do not pollute the changed-file set the predicate scores. - const outDir = mkdtempSync(join(tmpdir(), "ws2-runcli-out-")); - const rep = report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]); - const preP = join(outDir, "pre.json"); - const postP = join(outDir, "post.json"); - writeFileSync(preP, JSON.stringify(rep), "utf-8"); - writeFileSync(postP, JSON.stringify(report([])), "utf-8"); - process.chdir(repo); - captureConsole(); - try { - const code = runCli(["--report", preP, "--post-fix-report", postP, "--post-fix-exit", "0"]); - expect(code).toBe(0); - expect(stdoutLines()).toContain(`reason=${PredicateReason.RESOLVED}`); - } finally { - rmSync(outDir, { recursive: true, force: true }); - } - }); - - it("prints `reason=unsanctioned-change` (non-zero) on a blocked verdict — the Slack routing key", () => { - repo = initRepo(); - mkdirSync(join(repo, "src"), { recursive: true }); - writeFileSync(join(repo, "src", "helpers.ts"), "export const x = 1;\n", "utf-8"); - // package.json IN the repo is the unsanctioned change; reports live OUTSIDE. - writeFileSync(join(repo, "package.json"), '{"name":"x"}\n', "utf-8"); - execFileSync("git", ["add", "-N", "src/helpers.ts", "package.json"], { cwd: repo }); - const outDir = mkdtempSync(join(tmpdir(), "ws2-runcli-out-")); - const rep = report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]); - const preP = join(outDir, "pre.json"); - const postP = join(outDir, "post.json"); - writeFileSync(preP, JSON.stringify(rep), "utf-8"); - writeFileSync(postP, JSON.stringify(report([])), "utf-8"); - process.chdir(repo); - captureConsole(); - try { - const code = runCli(["--report", preP, "--post-fix-report", postP, "--post-fix-exit", "0"]); - expect(code).toBe(REASON_EXIT_CODE[PredicateReason.UNSANCTIONED_CHANGE]); - expect(stdoutLines()).toContain(`reason=${PredicateReason.UNSANCTIONED_CHANGE}`); - } finally { - rmSync(outDir, { recursive: true, force: true }); - } - }); - - it("prints `reason=config-error` and exits 2 when the report is unreadable (CONFIG_ERROR lock)", () => { - repo = initRepo(); - process.chdir(repo); - captureConsole(); - const code = runCli([ - "--report", - join(repo, "does-not-exist.json"), - "--post-fix-report", - join(repo, "also-missing.json"), - "--post-fix-exit", - "0", - ]); - expect(code).toBe(REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]); - expect(stdoutLines()).toContain(`reason=${PredicateReason.CONFIG_ERROR}`); - }); -}); - -// --------------------------------------------------------------------------- -// CR round-3 F3 — readReport ENTRY-LEVEL validation aligned with -// fix-drift.ts:readDriftReport. A structurally-valid report whose entries are -// malformed at the fields the predicate reads must fail-closed with a DISTINCT, -// NAMED PredicateConfigError (not a bare TypeError caught as an unnamed error). -// --------------------------------------------------------------------------- - -describe("readReport entry-level validation (F3)", () => { - let dir: string | null = null; - afterEach(() => { - if (dir) rmSync(dir, { recursive: true, force: true }); - dir = null; - }); - - function writeAndRead(obj: unknown): void { - dir = mkdtempSync(join(tmpdir(), "ws2-f3-")); - const p = join(dir, "r.json"); - writeFileSync(p, JSON.stringify(obj), "utf-8"); - readReport(p); - } - - it("throws a named PredicateConfigError when an entry is missing its diffs array", () => { - expect(() => - writeAndRead({ - timestamp: "t", - entries: [{ provider: "OpenAI", builderFile: "src/helpers.ts", typesFile: null }], - }), - ).toThrow(PredicateConfigError); - }); - - it('throws a named PredicateConfigError when "diffs" is present via the message', () => { - expect(() => - writeAndRead({ - timestamp: "t", - entries: [{ provider: "OpenAI", builderFile: "src/helpers.ts", typesFile: null }], - }), - ).toThrow(/diffs/); - }); - - it("throws when an entry has a non-string builderFile (cannot derive sanctioned set)", () => { - expect(() => - writeAndRead({ - timestamp: "t", - entries: [{ provider: "OpenAI", builderFile: 123, typesFile: null, diffs: [] }], - }), - ).toThrow(PredicateConfigError); - }); - - it("throws when an entry has an empty builderFile", () => { - expect(() => - writeAndRead({ - timestamp: "t", - entries: [{ provider: "OpenAI", builderFile: "", typesFile: null, diffs: [] }], - }), - ).toThrow(/builderFile/); - }); - - it("throws when an entry has a numeric typesFile (must be string or null)", () => { - expect(() => - writeAndRead({ - timestamp: "t", - entries: [{ provider: "OpenAI", builderFile: "src/helpers.ts", typesFile: 42, diffs: [] }], - }), - ).toThrow(/typesFile/); - }); - - it("throws when an entry is not an object", () => { - expect(() => writeAndRead({ timestamp: "t", entries: [null] })).toThrow(PredicateConfigError); - }); - - it("still ACCEPTS a well-formed report (empty entries + valid entries both OK)", () => { - expect(() => writeAndRead({ timestamp: "t", entries: [] })).not.toThrow(); - expect(() => - writeAndRead({ - timestamp: "t", - entries: [ - { provider: "OpenAI", builderFile: "src/helpers.ts", typesFile: null, diffs: [] }, - ], - }), - ).not.toThrow(); - }); -}); diff --git a/src/__tests__/drift-sync-check.test.ts b/src/__tests__/drift-sync-check.test.ts new file mode 100644 index 00000000..ab0b7844 --- /dev/null +++ b/src/__tests__/drift-sync-check.test.ts @@ -0,0 +1,279 @@ +/** + * drift-sync-check.ts — the trivial, deterministic data-only gate that + * REPLACES the 916-line LLM anti-cheat predicate (drift-success-predicate.ts). + * + * Three gates, tested independently and composed: + * 1. changed-file allowlist (data surfaces only) + * 2. checksum-pin re-assert (P0's logic-pin.test.ts must still be green) + * 3. clean re-collect (post-sync report has zero residual critical diffs) + * + * No LLM, no model call — every assertion here is a plain data check. + */ +import { describe, it, expect, vi } from "vitest"; + +import { + isAllowedSyncFile, + checkChangedFileAllowlist, + countCriticalDiffs, + evaluateSyncCheck, + runPinCheck, + recollect, + SyncCheckReason, + SyncCheckConfigError, + REASON_EXIT_CODE, + runCli, + type SyncCheckDeps, + type CommandResult, +} from "../../scripts/drift-sync-check.js"; +import type { DriftReport } from "../../scripts/drift-types.js"; + +function report(criticalCounts: number[]): DriftReport { + return { + timestamp: "2026-07-22T00:00:00.000Z", + entries: criticalCounts.map((n, i) => ({ + provider: `provider-${i}`, + scenario: "scenario", + builderFile: `src/provider-${i}.ts`, + builderFunctions: [], + typesFile: null, + sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", + diffs: Array.from({ length: n }, (_, j) => ({ + path: `field-${j}`, + severity: "critical" as const, + issue: "missing", + expected: "x", + real: "y", + mock: "z", + })), + })), + }; +} + +// --------------------------------------------------------------------------- +// Gate 1 — changed-file allowlist +// --------------------------------------------------------------------------- + +describe("isAllowedSyncFile / checkChangedFileAllowlist", () => { + it("allows the model-registry data file", () => { + expect(isAllowedSyncFile("src/__tests__/drift/model-registry.ts")).toBe(true); + }); + + it("allows a drift-proposals note file at any depth", () => { + expect(isAllowedSyncFile("drift-proposals/anthropic-new-family.md")).toBe(true); + expect(isAllowedSyncFile("drift-proposals/nested/dir/note.md")).toBe(true); + }); + + it("rejects the SDK-shape fixture (the primary cheat surface)", () => { + expect(isAllowedSyncFile("src/__tests__/drift/sdk-shapes.ts")).toBe(false); + }); + + it("rejects a *.drift.ts assertion file", () => { + expect(isAllowedSyncFile("src/__tests__/drift/anthropic.drift.ts")).toBe(false); + }); + + it("rejects detector/predicate/collector source", () => { + expect(isAllowedSyncFile("scripts/drift-success-predicate.ts")).toBe(false); + expect(isAllowedSyncFile("scripts/drift-report-collector.ts")).toBe(false); + expect(isAllowedSyncFile("scripts/drift-sync.ts")).toBe(false); + }); + + it("rejects a mock-builder production file", () => { + expect(isAllowedSyncFile("src/messages.ts")).toBe(false); + }); + + it("rejects the CI workflow", () => { + expect(isAllowedSyncFile(".github/workflows/fix-drift.yml")).toBe(false); + }); + + it("checkChangedFileAllowlist returns only the offenders", () => { + const offenders = checkChangedFileAllowlist([ + "src/__tests__/drift/model-registry.ts", + "drift-proposals/note.md", + "scripts/drift-success-predicate.ts", + "src/__tests__/drift/sdk-shapes.ts", + ]); + expect(offenders).toEqual([ + "scripts/drift-success-predicate.ts", + "src/__tests__/drift/sdk-shapes.ts", + ]); + }); + + it("checkChangedFileAllowlist returns [] when every file is allowed", () => { + expect( + checkChangedFileAllowlist(["src/__tests__/drift/model-registry.ts", "drift-proposals/x.md"]), + ).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// Gate 2 — checksum-pin re-assert +// --------------------------------------------------------------------------- + +describe("runPinCheck", () => { + it("reports ok when the injected runner exits 0", () => { + const runner = vi.fn((): CommandResult => ({ status: 0, output: "5 passed" })); + const result = runPinCheck(runner); + expect(result.ok).toBe(true); + expect(runner).toHaveBeenCalledWith("pnpm", [ + "exec", + "vitest", + "run", + "src/__tests__/drift/logic-pin.test.ts", + ]); + }); + + it("reports NOT ok when the injected runner exits non-zero (a pinned rule moved)", () => { + const runner = vi.fn( + (): CommandResult => ({ + status: 1, + output: "FAIL logic-pin.test.ts > freezes NON_MODEL_TOKENS", + }), + ); + const result = runPinCheck(runner); + expect(result.ok).toBe(false); + expect(result.output).toContain("NON_MODEL_TOKENS"); + }); +}); + +// --------------------------------------------------------------------------- +// Gate 3 — clean re-collect +// --------------------------------------------------------------------------- + +describe("countCriticalDiffs", () => { + it("sums critical diffs across every entry", () => { + expect(countCriticalDiffs(report([0, 2, 1]))).toBe(3); + }); + + it("is zero for an all-clean report", () => { + expect(countCriticalDiffs(report([0, 0]))).toBe(0); + }); +}); + +describe("recollect", () => { + it("fails closed (SyncCheckConfigError) when the collector produced no report file", () => { + const runner = vi.fn((): CommandResult => ({ status: 0, output: "" })); + expect(() => recollect(runner, "/nonexistent/path/does-not-exist.json")).toThrow( + SyncCheckConfigError, + ); + }); +}); + +// --------------------------------------------------------------------------- +// Composition — evaluateSyncCheck +// --------------------------------------------------------------------------- + +function deps(overrides: Partial): SyncCheckDeps { + return { + getChangedFiles: () => ["src/__tests__/drift/model-registry.ts"], + runPinCheck: () => ({ ok: true, output: "5 passed" }), + recollect: () => report([0]), + ...overrides, + }; +} + +describe("evaluateSyncCheck — RED/GREEN value-test surface", () => { + it("GREEN: a data-only excludeFamilies-style change on the allowlist, pins intact, clean re-collect -> PASSES", () => { + const verdict = evaluateSyncCheck(deps({})); + expect(verdict.ok).toBe(true); + expect(verdict.reason).toBe(SyncCheckReason.OK); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(0); + }); + + it("RED: a change touching a forbidden/source file (the detector) -> FAILS, never reaches pin/recollect", () => { + const runPinCheck = vi.fn(() => ({ ok: true, output: "" })); + const recollectFn = vi.fn(() => report([0])); + const verdict = evaluateSyncCheck( + deps({ + getChangedFiles: () => [ + "src/__tests__/drift/model-registry.ts", + "src/__tests__/drift/models.drift.ts", // the detector source (C4) + ], + runPinCheck, + recollect: recollectFn, + }), + ); + expect(verdict.ok).toBe(false); + expect(verdict.reason).toBe(SyncCheckReason.OFF_ALLOWLIST_CHANGE); + expect(verdict.offendingFiles).toEqual(["src/__tests__/drift/models.drift.ts"]); + expect(REASON_EXIT_CODE[verdict.reason]).not.toBe(0); + // fail-closed and CHEAP: never pays for the pin check or a live re-collect + // once the allowlist has already refused. + expect(runPinCheck).not.toHaveBeenCalled(); + expect(recollectFn).not.toHaveBeenCalled(); + }); + + it("RED: a change mutating a pinned rule -> FAILS even though the touched file (model-registry.ts) is on the allowlist", () => { + const recollectFn = vi.fn(() => report([0])); + const verdict = evaluateSyncCheck( + deps({ + getChangedFiles: () => ["src/__tests__/drift/model-registry.ts"], + runPinCheck: () => ({ + ok: false, + output: "FAIL logic-pin.test.ts > freezes NON_MODEL_TOKENS (rule widened)", + }), + recollect: recollectFn, + }), + ); + expect(verdict.ok).toBe(false); + expect(verdict.reason).toBe(SyncCheckReason.PIN_CHECK_FAILED); + expect(verdict.detail).toContain("NON_MODEL_TOKENS"); + expect(REASON_EXIT_CODE[verdict.reason]).not.toBe(0); + // never trusts a live re-collect once a pinned rule has moved + expect(recollectFn).not.toHaveBeenCalled(); + }); + + it("RED: a clean re-collect that still reports residual critical drift -> FAILS", () => { + const verdict = evaluateSyncCheck( + deps({ + recollect: () => report([1]), + }), + ); + expect(verdict.ok).toBe(false); + expect(verdict.reason).toBe(SyncCheckReason.RESIDUAL_CRITICAL_DRIFT); + expect(verdict.detail).toContain("1 critical"); + }); +}); + +// --------------------------------------------------------------------------- +// CLI wrapper +// --------------------------------------------------------------------------- + +describe("runCli", () => { + it("returns exit 0 and prints reason=ok on a passing verdict", () => { + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + const code = runCli(deps({})); + expect(code).toBe(0); + expect(logSpy).toHaveBeenCalledWith(`reason=${SyncCheckReason.OK}`); + logSpy.mockRestore(); + }); + + it("returns a non-zero exit and prints the offending reason on a failing verdict", () => { + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + const errSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const code = runCli( + deps({ + getChangedFiles: () => ["scripts/drift-success-predicate.ts"], + }), + ); + expect(code).toBe(REASON_EXIT_CODE[SyncCheckReason.OFF_ALLOWLIST_CHANGE]); + expect(logSpy).toHaveBeenCalledWith(`reason=${SyncCheckReason.OFF_ALLOWLIST_CHANGE}`); + logSpy.mockRestore(); + errSpy.mockRestore(); + }); + + it("fails closed to CONFIG_ERROR when a dep throws (e.g. recollect couldn't produce a report)", () => { + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + const errSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const code = runCli( + deps({ + recollect: () => { + throw new SyncCheckConfigError("no report produced"); + }, + }), + ); + expect(code).toBe(REASON_EXIT_CODE[SyncCheckReason.CONFIG_ERROR]); + expect(logSpy).toHaveBeenCalledWith(`reason=${SyncCheckReason.CONFIG_ERROR}`); + logSpy.mockRestore(); + errSpy.mockRestore(); + }); +}); diff --git a/src/__tests__/drift-sync-core.test.ts b/src/__tests__/drift-sync-core.test.ts new file mode 100644 index 00000000..4dbfdf9a --- /dev/null +++ b/src/__tests__/drift-sync-core.test.ts @@ -0,0 +1,704 @@ +/** + * C2: deterministic drift-sync CORE — the DATA-only, ZERO-LLM replacement for + * the freewriter's DECISION role on the model-churn (add/deprecate) leg. + * + * Exercises `runDriftSyncCore` and its building blocks (the mirrored + * classification predicates, the AST-located mechanical registry edits, and + * the needs-human dedup note-file mechanism) purely over injected deps — no + * real fs/git/network I/O, so every scenario below is deterministic and fast. + * + * RED (observed before this module existed): `scripts/drift-sync.ts` exported + * only the C1 git/branch/commit/PR plumbing (todayStamp, exec, getChangedFiles, + * buildPrBody, gatedCommitFiles, ...) — none of `runDriftSyncCore`, + * `detectDeprecatedFamiliesForSync`, `removeFamilyLiteralInSource`, etc. + * existed, so a live churn scenario (a new classified model, or a retired + * family) had NO mechanical sync path at all: the only remediation route was + * the LLM freewriter. Verbatim capture of that RED state (this test file + * against the pre-C2 module) is in the slot's final report. + */ +import { describe, it, expect, vi } from "vitest"; +import { execFileSync } from "node:child_process"; +import { readFileSync, writeFileSync, existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { includeFamilies } from "./drift/model-registry.js"; +import { + detectDeprecatedFamiliesForSync, + unclassifiedFamiliesForSync, + removeFamilyLiteralInSource, + addFamilyLiteralInSource, + proposalNoteRelPath, + parseProposalDecision, + renderProposalNote, + runDriftSyncCore, + computeChangesetKey, + revertSyncFiles, + SyncCoreReason, + MODEL_REGISTRY_REL_PATH, + DRIFT_PROPOSALS_DIR, + type SyncCoreDeps, + type ProviderChurnInput, + type SyncCheckResultLike, +} from "../../scripts/drift-sync.js"; + +// --------------------------------------------------------------------------- +// Test fixture: a minimal, synthetic "model-registry.ts"-shaped source text — +// same array-literal-inside-a-2-arg-call-expression shape the real file uses, +// seeded with the REAL openai includeFamilies set (so a removal target like +// "gpt-4o" is guaranteed present, and an addition target like "gpt-live" is +// guaranteed absent — mirrors C4's own test fixtures in models.drift.ts). +// --------------------------------------------------------------------------- + +function fixtureRegistrySource(): string { + const openaiFamilies = [...includeFamilies.openai]; + const lines = [ + "export const includeFamilies = {", + ' openai: set("openai", [', + ...openaiFamilies.map((f) => ` "${f}",`), + " ]),", + ' anthropic: set("anthropic", [', + ' "claude-3-5-sonnet",', + " ]),", + ' gemini: set("gemini", [', + ' "gemini-2.5-flash",', + " ]),", + "};", + ]; + return lines.join("\n"); +} + +/** In-memory fake fs + gate for runDriftSyncCore — no real disk/git touched. */ +function makeFakeDeps(overrides: Partial = {}): { + deps: SyncCoreDeps; + registry: { text: string }; + notes: Map; + writeRegistrySource: ReturnType; + writeProposalNote: ReturnType; + runSyncCheck: ReturnType; + revertFiles: ReturnType; +} { + const registry = { text: fixtureRegistrySource() }; + const notes = new Map(); + + const writeRegistrySource = vi.fn((text: string) => { + registry.text = text; + }); + const writeProposalNote = vi.fn((path: string, text: string) => { + notes.set(path, text); + }); + const runSyncCheck = vi.fn( + (): SyncCheckResultLike => ({ ok: true, reason: "ok", detail: "gate passed" }), + ); + const revertFiles = vi.fn(); + + const deps: SyncCoreDeps = { + readRegistrySource: () => registry.text, + writeRegistrySource, + readProposalNote: (path: string) => notes.get(path) ?? null, + writeProposalNote, + runSyncCheck, + revertFiles, + now: () => new Date("2026-07-22T00:00:00Z"), + ...overrides, + }; + + return { + deps, + registry, + notes, + writeRegistrySource, + writeProposalNote, + runSyncCheck, + revertFiles, + }; +} + +// --------------------------------------------------------------------------- +// Mirrored classification predicates — parity with C4's models.drift.ts +// (same fixtures/expectations as models.drift.ts's own C4 suite). +// --------------------------------------------------------------------------- + +describe("detectDeprecatedFamiliesForSync (mirrors C4's detectDeprecatedFamilies)", () => { + it("FAIL-CLOSED: an empty live listing never proposes removal", () => { + expect(detectDeprecatedFamiliesForSync([], "openai").status).toBe("skipped"); + }); + + it("FAIL-CLOSED: a short/truncated live listing never proposes removal", () => { + expect(detectDeprecatedFamiliesForSync(["gpt-4o"], "openai").status).toBe("skipped"); + }); + + it("a healthy listing omitting a classified family flags EXACTLY that family", () => { + const allButGpt4o = [...includeFamilies.openai].filter((f) => f !== "gpt-4o"); + const liveIds = [...allButGpt4o, ...allButGpt4o.map((f) => `${f}-2025-01-01`)]; + const result = detectDeprecatedFamiliesForSync(liveIds, "openai", { + isReferenced: () => false, + }); + expect(result).toEqual({ + status: "checked", + candidates: [{ provider: "openai", family: "gpt-4o", stillReferenced: false }], + }); + }); + + it("a still-referenced deprecated family is flagged with stillReferenced: true", () => { + const allButGpt4o = [...includeFamilies.openai].filter((f) => f !== "gpt-4o"); + const liveIds = [...allButGpt4o, ...allButGpt4o.map((f) => `${f}-2025-01-01`)]; + const result = detectDeprecatedFamiliesForSync(liveIds, "openai", { isReferenced: () => true }); + expect(result).toEqual({ + status: "checked", + candidates: [{ provider: "openai", family: "gpt-4o", stillReferenced: true }], + }); + }); +}); + +describe("unclassifiedFamiliesForSync (mirrors C4's unclassifiedFamilies)", () => { + it("a genuinely new family is flagged", () => { + expect(unclassifiedFamiliesForSync(["gpt-live"], "openai")).toEqual(["gpt-live"]); + }); + + it("a dated snapshot of a known family produces zero drift", () => { + expect(unclassifiedFamiliesForSync(["gpt-4o-2024-08-06"], "openai")).toEqual([]); + }); + + it("a -preview family auto-classifies with zero registry edits", () => { + expect(unclassifiedFamiliesForSync(["gpt-9-search-preview"], "openai")).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// Mechanical registry edits — AST-located, single-line surgery. +// --------------------------------------------------------------------------- + +describe("removeFamilyLiteralInSource / addFamilyLiteralInSource", () => { + it("removes an existing family, comment-marking the line, touching nothing else", () => { + const src = fixtureRegistrySource(); + const result = removeFamilyLiteralInSource( + src, + "includeFamilies", + "openai", + "gpt-4o", + "TEST-REMOVE", + ); + expect(result.changed).toBe(true); + expect(result.text).not.toContain('"gpt-4o"'); + expect(result.text).toContain("// TEST-REMOVE"); + // Every other seeded family is untouched. + for (const f of includeFamilies.openai) { + if (f === "gpt-4o") continue; + expect(result.text).toContain(`"${f}"`); + } + }); + + it("no-ops when the family is not present (never mangles the file)", () => { + const src = fixtureRegistrySource(); + const result = removeFamilyLiteralInSource( + src, + "includeFamilies", + "openai", + "zzz-not-real", + "x", + ); + expect(result.changed).toBe(false); + expect(result.text).toBe(src); + }); + + it("adds a new family literal, comment-marked", () => { + const src = fixtureRegistrySource(); + const result = addFamilyLiteralInSource( + src, + "includeFamilies", + "openai", + "gpt-live", + "TEST-ADD", + ); + expect(result.changed).toBe(true); + expect(result.text).toContain('"gpt-live", // TEST-ADD'); + }); + + it("no-ops when the family is already present (never duplicates)", () => { + const src = fixtureRegistrySource(); + const result = addFamilyLiteralInSource(src, "includeFamilies", "openai", "gpt-4o", "x"); + expect(result.changed).toBe(false); + }); + + it("the edited text is still syntactically valid TypeScript (parser round-trip)", () => { + const src = fixtureRegistrySource(); + const removed = removeFamilyLiteralInSource(src, "includeFamilies", "openai", "gpt-4o", "r"); + const added = addFamilyLiteralInSource( + removed.text, + "includeFamilies", + "openai", + "gpt-live", + "a", + ); + // A further edit against the already-edited text must still locate the + // array correctly — proves the AST-based locator survives a prior edit. + const secondAdd = addFamilyLiteralInSource( + added.text, + "includeFamilies", + "openai", + "gpt-live-2", + "b", + ); + expect(secondAdd.changed).toBe(true); + expect(secondAdd.text).toContain('"gpt-live-2"'); + }); +}); + +// --------------------------------------------------------------------------- +// Proposal note files — dedup + decision parsing. +// --------------------------------------------------------------------------- + +describe("proposal notes", () => { + it("proposalNoteRelPath is family-keyed and stable (dedup key)", () => { + expect(proposalNoteRelPath("openai", "gpt-live", "new-family")).toBe( + `${DRIFT_PROPOSALS_DIR}/openai-gpt-live-new-family.md`, + ); + expect(proposalNoteRelPath("openai", "gpt-live", "new-family")).toBe( + proposalNoteRelPath("openai", "gpt-live", "new-family"), + ); + }); + + it("parseProposalDecision defaults to pending (fail-closed, never infers approval)", () => { + expect(parseProposalDecision("Status: NEEDS HUMAN REVIEW\n")).toBe("pending"); + expect(parseProposalDecision("Decision: pending")).toBe("pending"); + expect(parseProposalDecision("garbage with no Decision line")).toBe("pending"); + }); + + it("parseProposalDecision recognizes an explicit human-authored approval", () => { + expect(parseProposalDecision("Decision: include")).toBe("include"); + }); + + it("renderProposalNote never generates a Decision line for a deprecation note", () => { + const note = renderProposalNote( + "openai", + "gpt-4o", + "still-referenced-deprecation", + "detail", + "2026-07-22", + ); + expect(note).not.toContain("## Decision"); + }); +}); + +// --------------------------------------------------------------------------- +// runDriftSyncCore — the full orchestration, over injected deps. +// --------------------------------------------------------------------------- + +describe("runDriftSyncCore", () => { + it("RED->GREEN (empty/short listing): fail-closed no-op — no edit, no gate run", () => { + const { deps, runSyncCheck, writeRegistrySource } = makeFakeDeps(); + const inputs: ProviderChurnInput[] = [{ provider: "openai", liveModelIds: [] }]; + const outcome = runDriftSyncCore(inputs, deps); + + expect(outcome.ok).toBe(true); + expect(outcome.reason).toBe(SyncCoreReason.OK_NO_CHURN); + expect(outcome.outcomes).toEqual([]); + expect(outcome.skipped).toHaveLength(1); + expect(outcome.skipped[0].reason).toMatch(/too short to trust/); + expect(writeRegistrySource).not.toHaveBeenCalled(); + expect(runSyncCheck).not.toHaveBeenCalled(); + }); + + it("RED->GREEN (deprecation, zero-reference): mechanical removal + gate passes", () => { + const { deps, registry, runSyncCheck } = makeFakeDeps({ isReferenced: () => false }); + const allButGpt4o = [...includeFamilies.openai].filter((f) => f !== "gpt-4o"); + const liveIds = [...allButGpt4o, ...allButGpt4o.map((f) => `${f}-2025-01-01`)]; + const inputs: ProviderChurnInput[] = [{ provider: "openai", liveModelIds: liveIds }]; + + const outcome = runDriftSyncCore(inputs, deps); + + expect(outcome.outcomes).toContainEqual( + expect.objectContaining({ provider: "openai", family: "gpt-4o", action: "removed" }), + ); + expect(outcome.ok).toBe(true); + expect(outcome.reason).toBe(SyncCoreReason.OK_APPLIED); + expect(runSyncCheck).toHaveBeenCalledTimes(1); + // The active Set element ("gpt-4o", with trailing comma) is gone — the + // family name may still appear in the human-readable removal comment. + expect(registry.text).not.toContain('"gpt-4o",'); + expect(registry.text).toContain("drift-sync"); + }); + + it("RED->GREEN (deprecation, STILL-REFERENCED): routed to human, no auto-edit", () => { + const { deps, registry, writeProposalNote, runSyncCheck } = makeFakeDeps({ + isReferenced: () => true, + }); + const allButGpt4o = [...includeFamilies.openai].filter((f) => f !== "gpt-4o"); + const liveIds = [...allButGpt4o, ...allButGpt4o.map((f) => `${f}-2025-01-01`)]; + const inputs: ProviderChurnInput[] = [{ provider: "openai", liveModelIds: liveIds }]; + + const outcome = runDriftSyncCore(inputs, deps); + + expect(outcome.outcomes).toContainEqual( + expect.objectContaining({ + provider: "openai", + family: "gpt-4o", + action: "needs-human-still-referenced", + }), + ); + expect(outcome.ok).toBe(false); + expect(outcome.reason).toBe(SyncCoreReason.NEEDS_HUMAN); + // Registry data itself was NEVER mechanically touched for a still-referenced family. + expect(registry.text).toContain('"gpt-4o"'); + expect(writeProposalNote).toHaveBeenCalledWith( + `${DRIFT_PROPOSALS_DIR}/openai-gpt-4o-deprecated-referenced.md`, + expect.stringContaining("still references it"), + ); + // D-M1: a note-only run has NO registry edit to re-verify, so it is NEVER + // gated behind the (recollect-bearing) drift-sync-check — otherwise gate-3 + // would re-detect the un-actioned family it just routed to a human and + // revert the note. The gate is not consulted at all here. + expect(runSyncCheck).not.toHaveBeenCalled(); + }); + + it("RED->GREEN (genuinely new family, no prior decision): RED alert + single deduped note, no auto-classify", () => { + const { deps, registry, writeProposalNote, notes } = makeFakeDeps(); + const inputs: ProviderChurnInput[] = [{ provider: "openai", liveModelIds: ["gpt-live"] }]; + // NOTE: a single unclassified live id also fails the deprecation floor + // check (too short) — that is an independent, correctly-skipped signal; + // this test only asserts the ADDITION half's behavior. + + const outcome = runDriftSyncCore(inputs, deps); + + expect(outcome.outcomes).toContainEqual( + expect.objectContaining({ + provider: "openai", + family: "gpt-live", + action: "needs-human-new-family", + }), + ); + expect(outcome.ok).toBe(false); + expect(outcome.reason).toBe(SyncCoreReason.NEEDS_HUMAN); + // NEVER auto-classified: no registry edit occurred. + expect(registry.text).not.toContain("gpt-live"); + expect(writeProposalNote).toHaveBeenCalledTimes(1); + const [path, noteText] = writeProposalNote.mock.calls[0] as [string, string]; + expect(path).toBe(`${DRIFT_PROPOSALS_DIR}/openai-gpt-live-new-family.md`); + expect(noteText).toContain("Decision: pending"); + expect(notes.size).toBe(1); + + // Re-fire: same alert, same run again — must NOT spam a second note/PR. + writeProposalNote.mockClear(); + const outcome2 = runDriftSyncCore(inputs, deps); + expect(writeProposalNote).not.toHaveBeenCalled(); + expect(outcome2.reason).toBe(SyncCoreReason.NEEDS_HUMAN); + expect(outcome2.ok).toBe(false); + }); + + it("ADDITION (human-approved via note Decision: include): mechanical registry edit + gate passes", () => { + const { deps, registry, notes, runSyncCheck } = makeFakeDeps(); + // Simulate a human having already reviewed the RED alert and flipped the + // note's Decision line to `include` (the only path that can ever add a + // genuinely-new family — never automatic, never LLM-authored). + const notePath = proposalNoteRelPath("openai", "gpt-live", "new-family"); + notes.set( + notePath, + renderProposalNote("openai", "gpt-live", "new-family", "detail", "2026-07-20").replace( + "Decision: pending", + "Decision: include", + ), + ); + + const inputs: ProviderChurnInput[] = [{ provider: "openai", liveModelIds: ["gpt-live"] }]; + const outcome = runDriftSyncCore(inputs, deps); + + expect(outcome.outcomes).toContainEqual( + expect.objectContaining({ provider: "openai", family: "gpt-live", action: "added" }), + ); + expect(outcome.ok).toBe(true); + expect(outcome.reason).toBe(SyncCoreReason.OK_APPLIED); + expect(registry.text).toContain('"gpt-live"'); + expect(registry.text).toContain(`approved via ${notePath}`); + expect(runSyncCheck).toHaveBeenCalledTimes(1); + }); + + it("a FAILING drift-sync-check gate reverts every touched file and reports GATE_FAILED", () => { + const { deps, revertFiles } = makeFakeDeps({ + isReferenced: () => false, + runSyncCheck: vi.fn( + (): SyncCheckResultLike => ({ + ok: false, + reason: "pin-check-failed", + detail: "a pinned rule moved", + }), + ), + }); + const allButGpt4o = [...includeFamilies.openai].filter((f) => f !== "gpt-4o"); + const liveIds = [...allButGpt4o, ...allButGpt4o.map((f) => `${f}-2025-01-01`)]; + const inputs: ProviderChurnInput[] = [{ provider: "openai", liveModelIds: liveIds }]; + + const outcome = runDriftSyncCore(inputs, deps); + + expect(outcome.ok).toBe(false); + expect(outcome.reason).toBe(SyncCoreReason.GATE_FAILED); + expect(outcome.detail).toContain("pin-check-failed"); + expect(revertFiles).toHaveBeenCalledWith([MODEL_REGISTRY_REL_PATH]); + }); + + it("a provider whose live listing was skipped (no key / infra error) is recorded, not treated as churn", () => { + const { deps } = makeFakeDeps(); + const inputs: ProviderChurnInput[] = [ + { provider: "anthropic", liveModelIds: null, skipReason: "ANTHROPIC_API_KEY not set" }, + ]; + const outcome = runDriftSyncCore(inputs, deps); + expect(outcome.ok).toBe(true); + expect(outcome.reason).toBe(SyncCoreReason.OK_NO_CHURN); + expect(outcome.skipped).toEqual([ + { provider: "anthropic", reason: "ANTHROPIC_API_KEY not set" }, + ]); + }); +}); + +// --------------------------------------------------------------------------- +// D-M1: the recollect gate must NOT destroy the route-to-human invariant. +// +// De-masks the default `makeFakeDeps` gate (which returned {ok:true} +// unconditionally and hid this bug): these gates model the REAL drift-sync-check +// — gate-1 (allowlist) + gate-2 (pin) pass for a data-only change, but gate-3 +// (the live re-collect) STILL reports the un-actioned family this run routed to +// a human as residual critical drift, so a `skipRecollect:false` call fails. +// --------------------------------------------------------------------------- + +/** A faithful drift-sync-check: passes with recollect skipped, FAILS with it on. */ +function faithfulRecollectGate(): ReturnType { + return vi.fn( + (opts?: { skipRecollect?: boolean }): SyncCheckResultLike => + opts?.skipRecollect + ? { ok: true, reason: "ok", detail: "allowlist + pin ok; live re-collect skipped" } + : { + ok: false, + reason: "residual-critical-drift", + detail: "1 critical diff — un-actioned family the collector still sees", + }, + ); +} + +describe("D-M1: recollect gate vs route-to-human invariant", () => { + it("RED->GREEN (note-only new family): faithful gate does NOT revert the note; NEEDS_HUMAN", () => { + const runSyncCheck = faithfulRecollectGate(); + const { deps, revertFiles } = makeFakeDeps({ runSyncCheck }); + const inputs: ProviderChurnInput[] = [{ provider: "openai", liveModelIds: ["gpt-live"] }]; + + const outcome = runDriftSyncCore(inputs, deps); + + // GREEN: the genuinely-new family persists its note and reaches the + // human-approval protocol. RED (pre-fix): the core ran the recollect gate, + // which failed, so the note was reverted and the run reported GATE_FAILED. + expect(outcome.reason).toBe(SyncCoreReason.NEEDS_HUMAN); + expect(outcome.ok).toBe(false); + expect(revertFiles).not.toHaveBeenCalled(); + // A note-only run has no registry edit to re-verify → the recollect-bearing + // gate is never even consulted. + expect(runSyncCheck).not.toHaveBeenCalled(); + }); + + it("RED->GREEN (mixed: valid removal + new-family note): removal kept, gate-3 skipped, NEEDS_HUMAN", () => { + const runSyncCheck = faithfulRecollectGate(); + const { deps, registry, revertFiles } = makeFakeDeps({ + isReferenced: () => false, + runSyncCheck, + }); + const allButGpt4o = [...includeFamilies.openai].filter((f) => f !== "gpt-4o"); + const liveIds = [...allButGpt4o, ...allButGpt4o.map((f) => `${f}-2025-01-01`), "gpt-live"]; + const inputs: ProviderChurnInput[] = [{ provider: "openai", liveModelIds: liveIds }]; + + const outcome = runDriftSyncCore(inputs, deps); + + // GREEN: the valid zero-reference removal is applied AND kept; the new + // family is deferred to a human. RED (pre-fix): the recollect gate saw the + // deferred new family as residual drift and reverted the valid removal too. + expect(outcome.reason).toBe(SyncCoreReason.NEEDS_HUMAN); + expect(outcome.ok).toBe(false); + expect(outcome.outcomes).toContainEqual( + expect.objectContaining({ family: "gpt-4o", action: "removed" }), + ); + expect(outcome.outcomes).toContainEqual( + expect.objectContaining({ family: "gpt-live", action: "needs-human-new-family" }), + ); + expect(revertFiles).not.toHaveBeenCalled(); + // The gate ran (a registry edit WAS applied) but with the live re-collect + // skipped, because a family was simultaneously deferred to a human. + expect(runSyncCheck).toHaveBeenCalledTimes(1); + expect(runSyncCheck).toHaveBeenCalledWith({ skipRecollect: true }); + // The registry edit was persisted (writeRegistrySource ran with the removal). + expect(registry.text).not.toContain('"gpt-4o",'); + }); +}); + +// --------------------------------------------------------------------------- +// G#1: a registry structural mismatch (AST locator miss) must route-to-human, +// never collapse into a silent benign no-op. +// --------------------------------------------------------------------------- + +describe("G#1: locator miss routes to human", () => { + it("RED->GREEN (deprecation locator miss): writes a note + NEEDS_HUMAN, never a silent no-op", () => { + // A registry source the AST locator cannot parse into includeFamilies — + // models the real file's structure changing out from under the editor. + const brokenSource = "export const somethingElse = { openai: [] };\n"; + const brokenRegistry = { text: brokenSource }; + const { deps, notes } = makeFakeDeps({ + isReferenced: () => false, + readRegistrySource: () => brokenRegistry.text, + writeRegistrySource: (t: string) => { + brokenRegistry.text = t; + }, + }); + const allButGpt4o = [...includeFamilies.openai].filter((f) => f !== "gpt-4o"); + const liveIds = [...allButGpt4o, ...allButGpt4o.map((f) => `${f}-2025-01-01`)]; + const inputs: ProviderChurnInput[] = [{ provider: "openai", liveModelIds: liveIds }]; + + const outcome = runDriftSyncCore(inputs, deps); + + // GREEN: routed to a human. RED (pre-fix): a silent `no-op` action, no note, + // ok-no-churn, exit 0 — a real deprecation vanishing silently. + expect(outcome.reason).toBe(SyncCoreReason.NEEDS_HUMAN); + expect(outcome.ok).toBe(false); + expect(outcome.outcomes).toContainEqual( + expect.objectContaining({ family: "gpt-4o", action: "needs-human-structural-mismatch" }), + ); + // A note actually alerts a human. + expect(notes.has(`${DRIFT_PROPOSALS_DIR}/openai-gpt-4o-structural-mismatch.md`)).toBe(true); + // The unparseable registry was never mutated. + expect(brokenRegistry.text).toBe(brokenSource); + }); +}); + +// --------------------------------------------------------------------------- +// D-M2: revertSyncFiles must handle untracked note files without throwing. +// Exercises the REAL revert surface (real git, real untracked file). +// --------------------------------------------------------------------------- + +describe("revertSyncFiles (D-M2: revert must not throw on untracked notes)", () => { + it("RED->GREEN: reverts a tracked edit AND deletes an untracked note without throwing", () => { + const repo = mkdtempSync(join(tmpdir(), "drift-sync-revert-")); + const git = (...args: string[]) => execFileSync("git", args, { cwd: repo, stdio: "ignore" }); + git("init"); + git("config", "user.email", "test@example.com"); + git("config", "user.name", "Test"); + + // A TRACKED file with committed content, then locally modified by the sync. + const trackedRel = "src/__tests__/drift/model-registry.ts"; + mkdirSync(join(repo, "src/__tests__/drift"), { recursive: true }); + writeFileSync(join(repo, trackedRel), "ORIGINAL\n"); + git("add", trackedRel); + git("commit", "-m", "seed"); + writeFileSync(join(repo, trackedRel), "MODIFIED BY SYNC\n"); + + // An UNTRACKED note git has never seen (the D-M2 trigger). + const noteRel = "drift-proposals/openai-gpt-live-new-family.md"; + mkdirSync(join(repo, "drift-proposals"), { recursive: true }); + writeFileSync(join(repo, noteRel), "note body\n"); + + const prevCwd = process.cwd(); + process.chdir(repo); + try { + // GREEN: partitions tracked vs untracked and never throws. RED (pre-fix): + // `git checkout -- ` errors on the untracked note, + // reverts NOTHING, and throws uncaught. + expect(() => revertSyncFiles([trackedRel, noteRel])).not.toThrow(); + } finally { + process.chdir(prevCwd); + } + + // Tracked file restored to its committed content; untracked note removed. + expect(readFileSync(join(repo, trackedRel), "utf-8")).toBe("ORIGINAL\n"); + expect(existsSync(join(repo, noteRel))).toBe(false); + + rmSync(repo, { recursive: true, force: true }); + }); +}); + +// --------------------------------------------------------------------------- +// Zero-LLM guarantee — static sanity check on the module's own source. +// --------------------------------------------------------------------------- + +describe("the sync core never invokes an LLM", () => { + it("scripts/drift-sync.ts contains no Claude Code invocation / free-form generation call", () => { + const src = readFileSync(new URL("../../scripts/drift-sync.ts", import.meta.url), "utf-8"); + // Check for actual invocation syntax, not the module's own explanatory + // prose (which legitimately names these as what this file does NOT do). + expect(src).not.toMatch(/invokeClaudeCode\(/); + expect(src).not.toMatch(/from ["']@anthropic-ai\/claude-code["']/); + expect(src).not.toMatch(/\bbuildPrompt\(/); + expect(src).not.toMatch(/\bspawn\(/); + }); +}); + +// --------------------------------------------------------------------------- +// G#3: computeChangesetKey — the STABLE, date-independent dedup key the CI +// workflow uses to keep BOTH PR-open paths idempotent across daily re-fires. +// +// The bug it exists to fix: the workflow's needs-human persist step deduped +// SOLELY on the committed `drift-proposals/*` note paths. In the D-M1 "mixed +// run" (a mechanical registry removal committed the SAME run a *different* +// family is deferred to a human whose note ALREADY sits on main), the diff +// carries ONLY the registry edit and NO note file — so a note-path key was +// EMPTY, the dedup was bypassed, and a brand-new near-identical PR opened on +// every daily cron run (unbounded PR-spam). The changeset key is non-empty in +// that shape (it carries the removal AND the deferred family) and identical on +// every re-fire, so the workflow can find the already-open PR and skip. +// --------------------------------------------------------------------------- + +describe("computeChangesetKey (G#3: stable, date-independent PR-dedup key)", () => { + // The D-M1 mixed run: gpt-4o removed (registry edit) + gpt-live deferred + // (needs-human) — the exact shape whose committed diff has a registry edit + // but no NEW note file, where a note-path-only dedup key is empty. + function mixedRunInputs(): ProviderChurnInput[] { + const allButGpt4o = [...includeFamilies.openai].filter((f) => f !== "gpt-4o"); + const liveIds = [...allButGpt4o, ...allButGpt4o.map((f) => `${f}-2025-01-01`), "gpt-live"]; + return [{ provider: "openai", liveModelIds: liveIds }]; + } + + it("is NON-EMPTY for a mixed run (registry edit + deferred family) — the shape a note-path-only key misses", () => { + const { deps } = makeFakeDeps({ isReferenced: () => false }); + const outcome = runDriftSyncCore(mixedRunInputs(), deps); + expect(outcome.reason).toBe(SyncCoreReason.NEEDS_HUMAN); + // The mixed-run committed diff carries NO new note file, yet the key is set + // — this is precisely what makes the workflow dedup fire on this shape. + expect(computeChangesetKey(outcome)).not.toBe(""); + // Carries BOTH the applied removal and the deferred family in its identity. + expect(outcome.outcomes).toContainEqual( + expect.objectContaining({ family: "gpt-4o", action: "removed" }), + ); + expect(outcome.outcomes).toContainEqual( + expect.objectContaining({ family: "gpt-live", action: "needs-human-new-family" }), + ); + }); + + it("is IDENTICAL across re-fires of the same drift on DIFFERENT dates (date-independent — so daily re-fires dedup)", () => { + const day1 = makeFakeDeps({ isReferenced: () => false, now: () => new Date("2026-07-22") }); + const day2 = makeFakeDeps({ isReferenced: () => false, now: () => new Date("2026-08-15") }); + const key1 = computeChangesetKey(runDriftSyncCore(mixedRunInputs(), day1.deps)); + const key2 = computeChangesetKey(runDriftSyncCore(mixedRunInputs(), day2.deps)); + expect(key1).toBe(key2); + }); + + it("is EMPTY for a no-churn run (nothing applied or deferred — no PR to dedup)", () => { + const { deps } = makeFakeDeps(); + const outcome = runDriftSyncCore([{ provider: "openai", liveModelIds: [] }], deps); + expect(outcome.reason).toBe(SyncCoreReason.OK_NO_CHURN); + expect(computeChangesetKey(outcome)).toBe(""); + }); + + it("DIFFERS for a different changeset (a pure deferral vs a mixed run) — distinct drifts get distinct PRs", () => { + const pure = makeFakeDeps(); + const pureOutcome = runDriftSyncCore( + [{ provider: "openai", liveModelIds: ["gpt-live"] }], + pure.deps, + ); + const mixed = makeFakeDeps({ isReferenced: () => false }); + const mixedOutcome = runDriftSyncCore(mixedRunInputs(), mixed.deps); + const pureKey = computeChangesetKey(pureOutcome); + const mixedKey = computeChangesetKey(mixedOutcome); + expect(pureKey).not.toBe(""); + expect(mixedKey).not.toBe(""); + expect(pureKey).not.toBe(mixedKey); + }); + + it("is a fixed-length 16-hex-char token (no substring collision between distinct keys in the PR-body marker match)", () => { + const { deps } = makeFakeDeps({ isReferenced: () => false }); + const key = computeChangesetKey(runDriftSyncCore(mixedRunInputs(), deps)); + expect(key).toMatch(/^[0-9a-f]{16}$/); + }); +}); diff --git a/src/__tests__/drift-sync.test.ts b/src/__tests__/drift-sync.test.ts new file mode 100644 index 00000000..8f809d65 --- /dev/null +++ b/src/__tests__/drift-sync.test.ts @@ -0,0 +1,90 @@ +/** + * Characterization tests for the C1 plumbing MOVE (fix-drift.ts → drift-sync.ts). + * + * The reusable git / branch / commit / PR plumbing was extracted VERBATIM into + * scripts/drift-sync.ts so the deterministic model-sync path can reuse it. + * + * C3 (delete-freewriter-predicate-rewire): `scripts/fix-drift.ts` — the LLM + * freewriter invocation + its predicate-gated CLI, which used to re-export + * this plumbing as a pass-through so its own tests kept working unchanged — + * has been DELETED entirely. The re-export-identity half of this suite (which + * asserted `fix-drift.js`'s bindings were identical to `drift-sync.js`'s own) + * no longer has a module to compare against and is dropped. What remains locks + * that the moved pure functions behave correctly at drift-sync.ts's own module + * boundary (now the only boundary). + */ +import { describe, it, expect } from "vitest"; + +import type { DriftReport } from "../../scripts/drift-types.js"; +import * as sync from "../../scripts/drift-sync.js"; + +// --------------------------------------------------------------------------- +// Behavior at the drift-sync.ts module boundary (pure functions only). +// --------------------------------------------------------------------------- + +describe("drift-sync pure plumbing behaves the same at the new boundary", () => { + it("todayStamp returns a YYYY-MM-DD stamp", () => { + expect(sync.todayStamp()).toMatch(/^\d{4}-\d{2}-\d{2}$/); + }); + + it("truncateBody passes through under the limit and marks over-limit bodies", () => { + expect(sync.truncateBody("hello", 60000)).toBe("hello"); + const out = sync.truncateBody("a".repeat(70000)); + expect(out.length).toBeLessThanOrEqual(sync.GH_BODY_SAFE_MAX); + expect(out).toContain("Body truncated"); + }); + + it("parsePorcelainLine handles quoted + rename notation", () => { + expect(sync.parsePorcelainLine(" M src/foo.ts")).toBe("src/foo.ts"); + expect(sync.parsePorcelainLine("R old.ts -> new.ts")).toBe("new.ts"); + expect(sync.parsePorcelainLine(' M "src/special chars.ts"')).toBe("src/special chars.ts"); + }); + + it("affectedSkillSections maps + dedupes + sorts builder files", () => { + expect(sync.affectedSkillSections(["src/bedrock.ts", "src/bedrock-converse.ts"])).toEqual([ + "Bedrock", + ]); + expect(sync.affectedSkillSections(["package.json"])).toEqual([]); + }); + + it("gatedCommitFiles partitions production / report-named / straggler", () => { + const sanctioned = new Set(["src/helpers.ts", "src/__tests__/drift/model-registry.ts"]); + const g = sync.gatedCommitFiles( + ["src/helpers.ts", "src/__tests__/drift/model-registry.ts", "weird-root-file.txt"], + sanctioned, + ); + expect(g.builderFiles).toEqual(["src/helpers.ts"]); + expect(g.testFiles).toEqual(["src/__tests__/drift/model-registry.ts"]); + expect(g.stragglers).toEqual(["weird-root-file.txt"]); + }); + + it("buildPrBody renders the summary + providers + diffs from a report", () => { + const report: DriftReport = { + timestamp: "2026-07-22T00:00:00.000Z", + entries: [ + { + provider: "openai", + scenario: "streaming", + builderFile: "src/helpers.ts", + builderFunctions: ["buildChatCompletion"], + typesFile: null, + sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", + diffs: [ + { + path: "response.id", + severity: "warning", + issue: "field missing", + expected: "string", + real: '"x"', + mock: "undefined", + }, + ], + }, + ], + }; + const body = sync.buildPrBody(report); + expect(body).toContain("## Summary"); + expect(body).toContain("- openai: streaming"); + expect(body).toContain("- `response.id`: field missing"); + }); +}); diff --git a/src/__tests__/drift/anthropic.drift.ts b/src/__tests__/drift/anthropic.drift.ts index 052acca5..aa089cbf 100644 --- a/src/__tests__/drift/anthropic.drift.ts +++ b/src/__tests__/drift/anthropic.drift.ts @@ -5,7 +5,7 @@ */ import http from "node:http"; -import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from "vitest"; import type { ServerInstance } from "../../server.js"; import { extractShape, triangulate, compareSSESequences, formatDriftReport } from "./schema.js"; import { @@ -16,7 +16,15 @@ import { anthropicThinkingMessageShape, anthropicThinkingStreamEventShapes, } from "./sdk-shapes.js"; -import { anthropicNonStreaming, anthropicStreaming } from "./providers.js"; +import { + isInfraSkip, + isModelNotFound, + listAnthropicModels, + resolveLiveModel, + __resetResolveLiveModelCache, + type LiveModelEntry, + type ResolvedModel, +} from "./providers.js"; import { httpPost, parseTypedSSE, startDriftServer, stopDriftServer } from "./helpers.js"; // --------------------------------------------------------------------------- @@ -34,27 +42,112 @@ afterAll(async () => { await stopDriftServer(instance); }); +// --------------------------------------------------------------------------- +// Live model resolution (self-healing — replaces the dated hardcoded pin) +// --------------------------------------------------------------------------- +// +// `claude-haiku-4-5-20251001` is a dated snapshot pin: Anthropic retires +// dated snapshots on a schedule, and a retired pin used to 404/400 the real +// API leg, batch-quarantining the whole drift baseline (exit 5) rather than +// honestly skipping. Instead of hardcoding a model id, discover a live one +// from Anthropic's own `/v1/models` listing via R0's shared +// `resolveLiveModel` helper (generalized from the cohere #325 pattern), and +// honest-skip when the listing itself hits an infra condition. + +const PREFERRED_ANTHROPIC_MODELS = [ + "claude-haiku-4-5-20251001", + "claude-3-5-haiku-20241022", + "claude-3-haiku-20240307", +]; + +/** Map Anthropic's `/v1/models` listing shape onto `resolveLiveModel`'s. */ +async function fetchAnthropicModelListing(): Promise<{ + status: number; + models: LiveModelEntry[]; +}> { + const ids = await listAnthropicModels(ANTHROPIC_API_KEY!); + // A successful listing call always means status 200 here — `listAnthropicModels` + // throws an `InfraError` (caught by `resolveLiveModel`) on any non-2xx status. + // Anthropic's listing only ever exposes currently-available model ids (a + // retired snapshot simply disappears from it), so there is no `deprecated` + // flag to map. + return { status: 200, models: ids.map((id) => ({ id })) }; +} + +/** Memoized (per `resolveLiveModel`) so the whole leg makes one listing call. */ +function getAnthropicModel(): Promise { + return resolveLiveModel("anthropic", fetchAnthropicModelListing, PREFERRED_ANTHROPIC_MODELS); +} + +// --------------------------------------------------------------------------- +// Real API helper (local, model-parameterized — mirrors the cohere.drift.ts +// template so the retrofit stays scoped to this file per R0's ownership of +// providers.ts's static-model anthropicNonStreaming/anthropicStreaming). +// --------------------------------------------------------------------------- + +async function anthropicMessagesLive( + model: string, + messages: { role: string; content: string }[], + opts: { tools?: object[]; stream?: boolean; maxTokens?: number } = {}, +): Promise<{ status: number; body: string }> { + const body: Record = { + model, + messages, + max_tokens: opts.maxTokens ?? 10, + stream: opts.stream ?? false, + }; + if (opts.tools) body.tools = opts.tools; + + const res = await fetch("https://api.anthropic.com/v1/messages", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-api-key": ANTHROPIC_API_KEY ?? "", + "anthropic-version": "2023-06-01", + }, + body: JSON.stringify(body), + }); + return { status: res.status, body: await res.text() }; +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- describe.skipIf(!ANTHROPIC_API_KEY)("Anthropic Claude Messages drift", () => { - const config = { apiKey: ANTHROPIC_API_KEY! }; + it("non-streaming text shape matches", async (ctx) => { + const resolved = await getAnthropicModel(); + if ("infra" in resolved) { + // Provider-side auth/credit/rate-limit/5xx — honest skip, not drift. + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error("Anthropic /v1/models exposed no usable non-deprecated model"); + } + const model = resolved.model; - it("non-streaming text shape matches", async () => { const sdkShape = anthropicMessageShape(); + const messages = [{ role: "user", content: "Say hello" }]; const [realRes, mockRes] = await Promise.all([ - anthropicNonStreaming(config, [{ role: "user", content: "Say hello" }]), + anthropicMessagesLive(model, messages), httpPost(`${instance.url}/v1/messages`, { - model: "claude-haiku-4-5-20251001", + model, max_tokens: 10, - messages: [{ role: "user", content: "Say hello" }], + messages, stream: false, }), ]); - const realShape = extractShape(realRes.body); + if (isInfraSkip(realRes.status) || isModelNotFound(realRes.status, realRes.body)) { + // Real probe hit a transient provider-side condition, or the resolved + // model went stale between listing and probe — honest skip, not drift. + ctx.skip(); + return; + } + + const realShape = extractShape(JSON.parse(realRes.body)); const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); @@ -66,20 +159,41 @@ describe.skipIf(!ANTHROPIC_API_KEY)("Anthropic Claude Messages drift", () => { ).toEqual([]); }); - it("streaming text event sequence and shapes match", async () => { + it("streaming text event sequence and shapes match", async (ctx) => { + const resolved = await getAnthropicModel(); + if ("infra" in resolved) { + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error("Anthropic /v1/models exposed no usable non-deprecated model"); + } + const model = resolved.model; + const sdkEvents = anthropicStreamEventShapes(); + const messages = [{ role: "user", content: "Say hello" }]; - const [realStream, mockStreamRes] = await Promise.all([ - anthropicStreaming(config, [{ role: "user", content: "Say hello" }]), + const [realRaw, mockStreamRes] = await Promise.all([ + anthropicMessagesLive(model, messages, { stream: true }), httpPost(`${instance.url}/v1/messages`, { - model: "claude-haiku-4-5-20251001", + model, max_tokens: 10, - messages: [{ role: "user", content: "Say hello" }], + messages, stream: true, }), ]); - expect(realStream.rawEvents.length, "Real API returned no SSE events").toBeGreaterThan(0); + if (isInfraSkip(realRaw.status) || isModelNotFound(realRaw.status, realRaw.body)) { + ctx.skip(); + return; + } + + const realParsed = parseTypedSSE(realRaw.body); + expect(realParsed.length, "Real API returned no SSE events").toBeGreaterThan(0); + const realEvents = realParsed.map((e) => ({ + type: e.type, + dataShape: extractShape(e.data), + })); const mockEvents = parseTypedSSE(mockStreamRes.body); expect(mockEvents.length, "Mock returned no SSE events").toBeGreaterThan(0); @@ -89,7 +203,7 @@ describe.skipIf(!ANTHROPIC_API_KEY)("Anthropic Claude Messages drift", () => { dataShape: extractShape(e.data), })); - const diffs = compareSSESequences(sdkEvents, realStream.events, mockSSEShapes); + const diffs = compareSSESequences(sdkEvents, realEvents, mockSSEShapes); const report = formatDriftReport( "Anthropic Claude (streaming text events)", diffs, @@ -102,7 +216,17 @@ describe.skipIf(!ANTHROPIC_API_KEY)("Anthropic Claude Messages drift", () => { ).toEqual([]); }); - it("non-streaming tool call shape matches", async () => { + it("non-streaming tool call shape matches", async (ctx) => { + const resolved = await getAnthropicModel(); + if ("infra" in resolved) { + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error("Anthropic /v1/models exposed no usable non-deprecated model"); + } + const model = resolved.model; + const sdkShape = anthropicMessageToolCallShape(); const tools = [ @@ -116,19 +240,25 @@ describe.skipIf(!ANTHROPIC_API_KEY)("Anthropic Claude Messages drift", () => { }, }, ]; + const messages = [{ role: "user", content: "Weather in Paris" }]; const [realRes, mockRes] = await Promise.all([ - anthropicNonStreaming(config, [{ role: "user", content: "Weather in Paris" }], tools), + anthropicMessagesLive(model, messages, { tools, maxTokens: 50 }), httpPost(`${instance.url}/v1/messages`, { - model: "claude-haiku-4-5-20251001", + model, max_tokens: 50, - messages: [{ role: "user", content: "Weather in Paris" }], + messages, stream: false, tools, }), ]); - const realShape = extractShape(realRes.body); + if (isInfraSkip(realRes.status) || isModelNotFound(realRes.status, realRes.body)) { + ctx.skip(); + return; + } + + const realShape = extractShape(JSON.parse(realRes.body)); const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); @@ -144,7 +274,17 @@ describe.skipIf(!ANTHROPIC_API_KEY)("Anthropic Claude Messages drift", () => { ).toEqual([]); }); - it("streaming tool call event sequence matches", async () => { + it("streaming tool call event sequence matches", async (ctx) => { + const resolved = await getAnthropicModel(); + if ("infra" in resolved) { + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error("Anthropic /v1/models exposed no usable non-deprecated model"); + } + const model = resolved.model; + const sdkEvents = [ ...anthropicStreamEventShapes().filter( (e) => @@ -164,19 +304,30 @@ describe.skipIf(!ANTHROPIC_API_KEY)("Anthropic Claude Messages drift", () => { }, }, ]; + const messages = [{ role: "user", content: "Weather in Paris" }]; - const [realStream, mockStreamRes] = await Promise.all([ - anthropicStreaming(config, [{ role: "user", content: "Weather in Paris" }], tools), + const [realRaw, mockStreamRes] = await Promise.all([ + anthropicMessagesLive(model, messages, { tools, stream: true, maxTokens: 50 }), httpPost(`${instance.url}/v1/messages`, { - model: "claude-haiku-4-5-20251001", + model, max_tokens: 50, - messages: [{ role: "user", content: "Weather in Paris" }], + messages, stream: true, tools, }), ]); - expect(realStream.rawEvents.length, "Real API returned no SSE events").toBeGreaterThan(0); + if (isInfraSkip(realRaw.status) || isModelNotFound(realRaw.status, realRaw.body)) { + ctx.skip(); + return; + } + + const realParsed = parseTypedSSE(realRaw.body); + expect(realParsed.length, "Real API returned no SSE events").toBeGreaterThan(0); + const realEvents = realParsed.map((e) => ({ + type: e.type, + dataShape: extractShape(e.data), + })); const mockEvents = parseTypedSSE(mockStreamRes.body); expect(mockEvents.length, "Mock returned no SSE events").toBeGreaterThan(0); @@ -186,7 +337,7 @@ describe.skipIf(!ANTHROPIC_API_KEY)("Anthropic Claude Messages drift", () => { dataShape: extractShape(e.data), })); - const diffs = compareSSESequences(sdkEvents, realStream.events, mockSSEShapes); + const diffs = compareSSESequences(sdkEvents, realEvents, mockSSEShapes); const report = formatDriftReport( "Anthropic Claude (streaming tool call events)", diffs, @@ -200,6 +351,180 @@ describe.skipIf(!ANTHROPIC_API_KEY)("Anthropic Claude Messages drift", () => { }); }); +// --------------------------------------------------------------------------- +// Regression guard for the drift-live-pr exit-5 quarantine (cross-block memo +// pollution). +// --------------------------------------------------------------------------- +// +// When ANTHROPIC_API_KEY is armed in CI, the live block above runs FIRST and +// populates `resolveLiveModel`'s per-key memo under "anthropic" with the real +// resolved id — and, exactly like the real live block, has no afterEach cache +// reset. This block SIMULATES that seeding with no credentials (a stubbed +// listing exposing the still-live `claude-haiku-4-5-20251001`) so the fixture +// self-healing block below is proven to reset the shared memo BEFORE it runs. +// +// Without that reset, the fixture block's first test read the leaked live id +// and asserted a raw `.toBe()` on model ids — an unparseable AssertionError +// (`expected 'claude-haiku-4-5-20251001' to be 'claude-haiku-5-1-20260201'`) +// that the drift collector could not parse as a structured report and +// quarantined as a fatal exit 5 (the drift-live-pr failure). Declared +// immediately before the fixture block so declaration order (vitest runs +// in-file tests in order; no shuffle configured) reproduces the CI sequence. +describe("Anthropic live-leg memo seeding (simulates the armed-key live block running first)", () => { + it("populates the shared resolveLiveModel memo under 'anthropic' (no reset — mirrors the live block)", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = (async (url: string | URL) => { + if (String(url).includes("/v1/models")) { + return new Response(JSON.stringify({ data: [{ id: "claude-haiku-4-5-20251001" }] }), { + status: 200, + }); + } + throw new Error(`Unexpected fetch in seeding test: ${String(url)}`); + }) as typeof fetch; + try { + // The live leg resolves and MEMOIZES the still-live real id under + // "anthropic". Intentionally NOT cleared here — the fixture block below + // must be the thing that resets it. + const resolved = await getAnthropicModel(); + expect(resolved).toEqual({ model: "claude-haiku-4-5-20251001" }); + } finally { + globalThis.fetch = origFetch; + } + }); +}); + +// --------------------------------------------------------------------------- +// Self-healing proof (fixture-driven — no live credentials required) +// --------------------------------------------------------------------------- +// +// Drives the REAL `getAnthropicModel`/`fetchAnthropicModelListing` resolution +// path and the REAL `isInfraSkip`/`isModelNotFound`/`triangulate` the live +// leg above uses (no reimplementation) against a simulated "Anthropic +// retired the pinned snapshot" condition. Runs unconditionally — no +// ANTHROPIC_API_KEY required — so it always exercises the retrofit even when +// live credentials aren't armed locally or in CI. +// +// RED (pre-retrofit behavior, not committed): the old hardcoded pin +// (`claude-haiku-4-5-20251001`) was sent directly to `/v1/messages` with no +// discovery/skip layer. Against this same fixture (pin retired, listing +// only exposes a newer id) that call 404s, and the old code had no honest- +// skip path — the leg would throw/critical-diff and quarantine. +// GREEN (this retrofit): `getAnthropicModel()` discovers the live id from +// the listing instead of using the stale pin, the probe against the +// discovered id succeeds, and shape-grading reports zero critical drift. +describe("Anthropic live-leg self-healing (fixture-driven, no credentials required)", () => { + const origFetch = globalThis.fetch; + // Reset the shared per-key `resolveLiveModel` memo BEFORE each case, not only + // after. The live block (and the seeding guard) above run first and leave the + // "anthropic" key memoized with a real/still-live id; without this the first + // fixture case would resolve to that leaked id instead of its own stub and + // fail a raw `.toBe()` on model ids (the drift-live-pr exit-5 quarantine). + beforeEach(() => { + __resetResolveLiveModelCache(); + }); + afterEach(() => { + globalThis.fetch = origFetch; + __resetResolveLiveModelCache(); + }); + + const RETIRED_PIN = "claude-haiku-4-5-20251001"; + const LIVE_DISCOVERED_MODEL = "claude-haiku-5-1-20260201"; + + function stubRetiredPinFixture(): void { + globalThis.fetch = (async (url: string | URL, init?: RequestInit) => { + const href = String(url); + if (href.includes("/v1/models")) { + // The pinned snapshot no longer appears in the listing — retired. + return new Response(JSON.stringify({ data: [{ id: LIVE_DISCOVERED_MODEL }] }), { + status: 200, + }); + } + if (href.includes("/v1/messages")) { + const body = JSON.parse(String(init?.body ?? "{}")) as { model?: string }; + if (body.model === RETIRED_PIN) { + // Documenting the RED this retrofit removes from the driven path: + // the retired pin genuinely 404s as "model not found". + return new Response( + JSON.stringify({ + type: "error", + error: { type: "not_found_error", message: `model: ${RETIRED_PIN}` }, + }), + { status: 404 }, + ); + } + if (body.model === LIVE_DISCOVERED_MODEL) { + return new Response( + JSON.stringify({ + id: "msg_fixture", + type: "message", + role: "assistant", + content: [{ type: "text", text: "Hello!" }], + model: LIVE_DISCOVERED_MODEL, + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 8, output_tokens: 3 }, + }), + { status: 200 }, + ); + } + } + throw new Error(`Unexpected fetch in fixture test: ${href}`); + }) as typeof fetch; + } + + it("REGRESSION: discovers a live model and shape-grades it when the pinned snapshot is retired", async () => { + stubRetiredPinFixture(); + + // RED evidence (documented, not re-executed here): sending the retired + // pin straight through — what the pre-retrofit code did — 404s against + // this exact fixture: + const staleProbe = await anthropicMessagesLive(RETIRED_PIN, [ + { role: "user", content: "Say hello" }, + ]); + expect(staleProbe.status).toBe(404); + expect(isModelNotFound(staleProbe.status, staleProbe.body)).toBe(true); + + // GREEN: the retrofitted resolution step never returns the retired pin — + // it discovers the listing's live id instead. + const resolved = await getAnthropicModel(); + if ("infra" in resolved || "unavailable" in resolved) { + throw new Error(`Expected a resolved model, got ${JSON.stringify(resolved)}`); + } + expect(resolved.model).toBe(LIVE_DISCOVERED_MODEL); + expect(resolved.model).not.toBe(RETIRED_PIN); + + const realRes = await anthropicMessagesLive(resolved.model, [ + { role: "user", content: "Say hello" }, + ]); + expect(isInfraSkip(realRes.status)).toBe(false); + expect(isModelNotFound(realRes.status, realRes.body)).toBe(false); + expect(realRes.status).toBe(200); + + // Shape-graded on the discovered-model envelope — a real drift would + // still be caught here; this fixture's envelope matches the SDK shape. + const sdkShape = anthropicMessageShape(); + const realShape = extractShape(JSON.parse(realRes.body)); + const diffs = triangulate(sdkShape, realShape, realShape); + expect(diffs.filter((d) => d.severity === "critical")).toEqual([]); + }); + + it("honest-skips (not a drift finding) when the model listing itself hits an infra condition", async () => { + // 403 avoids RETRYABLE_STATUSES (429/5xx) so the fixture stays fast. + globalThis.fetch = (async () => new Response("Forbidden", { status: 403 })) as typeof fetch; + + const resolved = await getAnthropicModel(); + expect(resolved).toEqual({ infra: 403 }); + }); + + it("classifies model-not-found probe responses (404, and 400 with a model-not-found body)", () => { + expect(isModelNotFound(404)).toBe(true); + expect( + isModelNotFound(400, JSON.stringify({ error: { message: "model_not_found: foo" } })), + ).toBe(true); + expect(isModelNotFound(400, JSON.stringify({ error: { message: "bad request" } }))).toBe(false); + }); +}); + // --------------------------------------------------------------------------- // Extended thinking // --------------------------------------------------------------------------- diff --git a/src/__tests__/drift/bedrock-stream.drift.ts b/src/__tests__/drift/bedrock-stream.drift.ts index b0fc0705..a0d2ffb5 100644 --- a/src/__tests__/drift/bedrock-stream.drift.ts +++ b/src/__tests__/drift/bedrock-stream.drift.ts @@ -27,6 +27,22 @@ const HAS_CREDENTIALS = !!process.env.AWS_SECRET_ACCESS_KEY && !!process.env.AWS_REGION; +// --------------------------------------------------------------------------- +// Model pin +// --------------------------------------------------------------------------- +// +// This leg never makes a live Bedrock call — nothing in this repo implements +// AWS SigV4 request signing (no `@aws-sdk` dependency, no HMAC-based signer +// anywhere), so every HAS_CREDENTIALS-gated test below only exercises the +// LOCAL aimock mock server against a static SDK-shape stub. The +// infra-unavailable / model-not-found "honest skip" classification used by +// the other retrofit legs (which DO drive a real provider endpoint) does not +// apply here: a 4xx/5xx from the mock server is never a live-provider +// condition, it is a mock regression, and must hard-fail like any other drift +// finding. This constant just collapses the dated snapshot id into ONE named +// value (was duplicated across every URL literal below). +const BEDROCK_MODEL_ID = "anthropic.claude-3-haiku-20240307-v1:0"; + // --------------------------------------------------------------------------- // Server lifecycle // --------------------------------------------------------------------------- @@ -184,37 +200,29 @@ describe.skipIf(!HAS_CREDENTIALS)("Bedrock drift", () => { // Bedrock streaming uses binary event-stream framing, so we test the // mock's JSON response shape for the non-streaming invoke endpoint. - const mockRes = await httpPost( - `${instance.url}/model/anthropic.claude-3-haiku-20240307-v1:0/invoke`, - { - anthropic_version: "bedrock-2023-05-31", - max_tokens: 10, - messages: [{ role: "user", content: "Say hello" }], - }, - ); + const mockRes = await httpPost(`${instance.url}/model/${BEDROCK_MODEL_ID}/invoke`, { + anthropic_version: "bedrock-2023-05-31", + max_tokens: 10, + messages: [{ role: "user", content: "Say hello" }], + }); expect(mockRes.status).toBe(200); - // When real AWS credentials are available, send the same request to - // the real Bedrock API and compare shapes. For now, validate mock - // against the SDK shape as both real and expected. - if (mockRes.status === 200) { - const mockShape = extractShape(JSON.parse(mockRes.body)); - const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Bedrock Invoke", diffs, "bedrock-invoke"); + const mockShape = extractShape(JSON.parse(mockRes.body)); + const diffs = triangulate(sdkShape, sdkShape, mockShape); + const report = formatDriftReport("Bedrock Invoke", diffs, "bedrock-invoke"); - expect( - diffs.filter((d) => d.severity === "critical"), - report, - ).toEqual([]); - } + expect( + diffs.filter((d) => d.severity === "critical"), + report, + ).toEqual([]); }); it("invoke-with-response-stream mock shape matches SDK expectations", async () => { const sdkEvents = bedrockInvokeStreamEventShapes(); const mockRes = await httpPostBinary( - `${instance.url}/model/anthropic.claude-3-haiku-20240307-v1:0/invoke-with-response-stream`, + `${instance.url}/model/${BEDROCK_MODEL_ID}/invoke-with-response-stream`, { anthropic_version: "bedrock-2023-05-31", max_tokens: 10, @@ -277,36 +285,31 @@ describe.skipIf(!HAS_CREDENTIALS)("Bedrock drift", () => { it("converse mock shape matches SDK expectations", async () => { const sdkShape = bedrockConverseResponseShape(); - const mockRes = await httpPost( - `${instance.url}/model/anthropic.claude-3-haiku-20240307-v1:0/converse`, - { - messages: [ - { - role: "user", - content: [{ text: "Say hello" }], - }, - ], - inferenceConfig: { maxTokens: 10 }, - }, - ); + const mockRes = await httpPost(`${instance.url}/model/${BEDROCK_MODEL_ID}/converse`, { + messages: [ + { + role: "user", + content: [{ text: "Say hello" }], + }, + ], + inferenceConfig: { maxTokens: 10 }, + }); expect(mockRes.status).toBe(200); - if (mockRes.status === 200) { - const mockShape = extractShape(JSON.parse(mockRes.body)); - const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Bedrock Converse", diffs, "bedrock-converse"); + const mockShape = extractShape(JSON.parse(mockRes.body)); + const diffs = triangulate(sdkShape, sdkShape, mockShape); + const report = formatDriftReport("Bedrock Converse", diffs, "bedrock-converse"); - expect( - diffs.filter((d) => d.severity === "critical"), - report, - ).toEqual([]); - } + expect( + diffs.filter((d) => d.severity === "critical"), + report, + ).toEqual([]); }); it("converse-stream payloads are flat (not double-wrapped with event type name)", async () => { const mockRes = await httpPostBinary( - `${instance.url}/model/anthropic.claude-3-haiku-20240307-v1:0/converse-stream`, + `${instance.url}/model/${BEDROCK_MODEL_ID}/converse-stream`, { messages: [ { @@ -416,7 +419,7 @@ describe.skipIf(!HAS_CREDENTIALS)("Bedrock drift", () => { it("converse-stream tool-call event shapes match SDK expectations", async () => { const mockRes = await httpPostBinary( - `${instance.url}/model/anthropic.claude-3-haiku-20240307-v1:0/converse-stream`, + `${instance.url}/model/${BEDROCK_MODEL_ID}/converse-stream`, { messages: [ { @@ -519,7 +522,7 @@ describe.skipIf(!HAS_CREDENTIALS)("Bedrock drift", () => { try { const mockRes = await httpPostBinary( - `${reasoningInstance.url}/model/anthropic.claude-3-haiku-20240307-v1:0/converse-stream`, + `${reasoningInstance.url}/model/${BEDROCK_MODEL_ID}/converse-stream`, { messages: [ { diff --git a/src/__tests__/drift/deprecation-detector.ts b/src/__tests__/drift/deprecation-detector.ts new file mode 100644 index 00000000..4a4976e7 --- /dev/null +++ b/src/__tests__/drift/deprecation-detector.ts @@ -0,0 +1,100 @@ +/** + * Zero-reference cross-check for the C4 deprecation detector (`models.drift.ts`). + * + * A model family that a healthy live `/models` listing no longer contains is + * only SAFE to auto-propose for removal from the frozen registry + * (`model-registry.ts`) if aimock's own source no longer references it — a + * still-referenced family must route to a human instead (§4.4), never be + * silently classified away. This module owns the mechanical, zero-LLM scan + * that answers that one question: does the family string still appear, + * as a real token (not a substring artifact of a longer sibling id), anywhere + * in aimock's own source tree? + * + * Deliberately co-located here (NOT in `helpers.ts` — Correction S1): this is + * drift-detector-specific I/O, not a shared cross-provider live-discovery + * utility like `providers.ts`'s `resolveLiveModel`/`isInfraSkip`. + * + * Scanned root: `src/` EXCLUDING `src/__tests__/drift/` itself. The drift + * directory is the CLASSIFICATION layer (the registry seeds, this detector, + * its tests) — every family literal trivially appears there by definition, so + * including it would make every family look "still referenced" and defeat the + * entire check. Everywhere else under `src/` (server.ts's `DEFAULT_MODELS`, + * provider builder files, non-drift test fixtures/conformance suites) is a + * legitimate signal of real usage. + */ +import { readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +/** `src/` — two levels up from this file (`src/__tests__/drift/`). */ +const SRC_ROOT = fileURLToPath(new URL("../../", import.meta.url)); + +/** Directory (relative to `SRC_ROOT`) excluded from the ref-scan — see module doc. */ +const EXCLUDED_REL_DIR = join("__tests__", "drift"); + +function isExcludedDir(name: string, relPath: string): boolean { + if (name === "node_modules" || name === "dist") return true; + return relPath === EXCLUDED_REL_DIR || relPath.startsWith(EXCLUDED_REL_DIR + "/"); +} + +/** Recursively collect every `.ts`/`.tsx` file under `root`, skipping excluded dirs. */ +function collectSourceFiles(root: string): string[] { + const files: string[] = []; + const stack: string[] = [root]; + while (stack.length > 0) { + const dir = stack.pop()!; + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name); + if (entry.isDirectory()) { + const rel = full.slice(root.length).replace(/^[/\\]+/, ""); + if (isExcludedDir(entry.name, rel)) continue; + stack.push(full); + } else if (entry.isFile() && /\.tsx?$/.test(entry.name)) { + files.push(full); + } + } + } + return files; +} + +/** Escape a string for safe interpolation into a `RegExp` literal. */ +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** Memoized concatenation of every scanned file's source text (computed once, lazily). */ +let cachedSourceText: string | null = null; + +function allSourceText(): string { + if (cachedSourceText === null) { + cachedSourceText = collectSourceFiles(SRC_ROOT) + .map((f) => readFileSync(f, "utf8")) + .join("\n"); + } + return cachedSourceText; +} + +/** Test-only: drop the memoized source cache (the source tree does not change mid-run). */ +export function __resetSourceScanCache(): void { + cachedSourceText = null; +} + +/** + * True when `family` still appears as a real token (bounded by anything other + * than a word character, `.`, or `-`) anywhere in aimock's own source outside + * `src/__tests__/drift/`. Boundary-aware so a shorter family that is a strict + * prefix of another live id's family (e.g. `gpt-4` vs. `gpt-4o` / `gpt-4-turbo`) + * is never mistaken for a hit — a naive substring scan would otherwise call + * `gpt-4` "referenced" merely because `gpt-4o` appears in source, silently + * blocking a legitimate removal proposal forever. + * + * Takes an unused second parameter slot deliberately absent: the scan is not + * provider-scoped (a family string is checked against the whole non-drift + * source tree), but its call signature is still assignable to the detector's + * injectable `(family: string, provider: Provider) => boolean` shape — a + * function with fewer parameters is always assignable there. + */ +export function isFamilyStillReferenced(family: string): boolean { + const pattern = new RegExp(`(? { + it("classifies auth/credit/rate-limit/5xx as an honest skip", () => { + for (const s of [401, 402, 403, 429, 500, 502, 503, 529]) { + expect(isInfraSkip(s)).toBe(true); + } + }); + + it("does NOT skip a success or a genuine envelope/probe error", () => { + for (const s of [200, 201, 400, 404, 422]) { + expect(isInfraSkip(s)).toBe(false); + } + }); +}); + +describe("isModelNotFound", () => { + it("treats a bare 404 as model-not-found (retired model id)", () => { + expect(isModelNotFound(404)).toBe(true); + }); + + it("treats a 400 whose body names a model-not-found condition as such", () => { + expect(isModelNotFound(400, '{"error":{"code":"model_not_found"}}')).toBe(true); + expect(isModelNotFound(400, "The model `gpt-foo` does not exist")).toBe(true); + expect(isModelNotFound(400, "unknown model: claude-retired")).toBe(true); + }); + + it("does NOT treat an unrelated 400 or a success as model-not-found", () => { + expect(isModelNotFound(400, "invalid_request_error: missing messages")).toBe(false); + expect(isModelNotFound(400)).toBe(false); + expect(isModelNotFound(200)).toBe(false); + }); +}); + +describe("selectLiveModel", () => { + it("never selects a deprecated model", () => { + const models: LiveModelEntry[] = [ + { id: "old-model", deprecated: true }, + { id: "fresh-model", deprecated: false }, + ]; + const chosen = selectLiveModel(models); + expect(chosen).not.toBe("old-model"); + expect(chosen).toBe("fresh-model"); + }); + + it("prefers an id from the preferred list when present and live", () => { + const models: LiveModelEntry[] = [{ id: "experimental" }, { id: "stable-default" }]; + expect(selectLiveModel(models, ["missing", "stable-default"])).toBe("stable-default"); + }); + + it("never prefers a deprecated id even if it is in the preferred list", () => { + const models: LiveModelEntry[] = [ + { id: "stable-default", deprecated: true }, + { id: "next-best" }, + ]; + expect(selectLiveModel(models, ["stable-default"])).toBe("next-best"); + }); + + it("falls back to the first live model when no preferred id is present", () => { + const models: LiveModelEntry[] = [ + { id: "retired", deprecated: true }, + { id: "first-live" }, + { id: "second-live" }, + ]; + expect(selectLiveModel(models, ["not-here"])).toBe("first-live"); + }); + + it("returns null when no usable live model exists", () => { + expect(selectLiveModel([{ id: "retired", deprecated: true }])).toBeNull(); + expect(selectLiveModel([])).toBeNull(); + expect(selectLiveModel([{ id: "" }])).toBeNull(); + }); +}); + +describe("resolveLiveModel", () => { + beforeEach(() => __resetResolveLiveModelCache()); + + it("resolves a live, non-deprecated id from a stubbed listing", async () => { + const resolved = await resolveLiveModel("p1", async () => ({ + status: 200, + models: [{ id: "retired", deprecated: true }, { id: "live-one" }], + })); + expect(resolved).toEqual({ model: "live-one" }); + }); + + it("honest-skips (infra) when the listing hits an auth/credit/rate/5xx status", async () => { + const resolved = await resolveLiveModel("p2", async () => ({ status: 401, models: [] })); + expect(resolved).toEqual({ infra: 401 }); + }); + + it("reports unavailable (fail-loud) on a non-infra listing error", async () => { + const resolved = await resolveLiveModel("p3", async () => ({ status: 404, models: [] })); + expect(resolved).toEqual({ unavailable: true }); + }); + + it("reports unavailable when the listing exposes no usable live model", async () => { + const resolved = await resolveLiveModel("p4", async () => ({ + status: 200, + models: [{ id: "retired", deprecated: true }], + })); + expect(resolved).toEqual({ unavailable: true }); + }); + + it("maps a thrown InfraError to an honest infra skip", async () => { + const resolved = await resolveLiveModel("p5", async () => { + throw new InfraError("INFRA_ERROR: listing down", 503); + }); + expect(resolved).toEqual({ infra: 503 }); + }); + + it("memoizes per key: the listing is fetched exactly once", async () => { + let calls = 0; + const fetchListing = async () => { + calls++; + return { status: 200, models: [{ id: "live-one" }] as LiveModelEntry[] }; + }; + const a = await resolveLiveModel("p6", fetchListing); + const b = await resolveLiveModel("p6", fetchListing); + expect(a).toEqual({ model: "live-one" }); + expect(b).toEqual({ model: "live-one" }); + expect(calls).toBe(1); + }); + + it("resets the memo cache so a fresh listing is fetched again", async () => { + let calls = 0; + const fetchListing = async () => { + calls++; + return { status: 200, models: [{ id: "live-one" }] as LiveModelEntry[] }; + }; + await resolveLiveModel("p7", fetchListing); + __resetResolveLiveModelCache(); + await resolveLiveModel("p7", fetchListing); + expect(calls).toBe(2); + }); +}); diff --git a/src/__tests__/drift/logic-pin.test.ts b/src/__tests__/drift/logic-pin.test.ts new file mode 100644 index 00000000..37d986c3 --- /dev/null +++ b/src/__tests__/drift/logic-pin.test.ts @@ -0,0 +1,171 @@ +/** + * CHECKSUM FREEZE for the model-drift CLASSIFICATION logic (Phase-0, spec §6). + * + * These surfaces decide which live `/models` families count as drift. They are + * the exact places a well-meaning bot — or an LLM told to "make the drift job + * pass" — could SILENCE the canary with a one-line edit: broaden a normalizer + * regex so unknown families collapse onto known ones, dump a new id into + * `NON_MODEL_TOKENS`, add an exclude-by-rule pattern, or short-circuit + * `isClassifiedFamily`. None of that must ever land silently. + * + * This test pins the SOURCE of each classification surface by SHA-256. Any edit + * to a frozen surface flips its checksum and fails `pnpm test`, forcing the + * change to be a deliberate, reviewed act: whoever legitimately changes a rule + * must ALSO re-pin the checksum here (and explain why in the diff), which a + * reviewer sees. A silent widening cannot slip through green CI. + * + * DO NOT "fix" a red pin by blindly pasting the new checksum. A red pin means a + * frozen classification rule moved — confirm the move is intended and reviewed + * BEFORE updating the pin. Auto-updating the pin to chase green defeats the + * entire purpose of this file. + * + * Frozen surfaces: + * model-family.ts — DATED_SNAPSHOT_SUFFIX, BUILD_TAG_SUFFIX, + * ANTHROPIC_DATE_SUFFIX, normalizeModelFamily + * model-registry.ts — PREVIEW_FAMILY, GEMMA_FAMILY, NON_MODEL_TOKENS, + * familySet, isClassifiedFamily + */ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { fileURLToPath } from "node:url"; + +import { + PREVIEW_FAMILY, + GEMMA_FAMILY, + NON_MODEL_TOKENS, + isClassifiedFamily, +} from "./model-registry.js"; +import { normalizeModelFamily } from "./model-family.js"; + +const famSrc = readFileSync(fileURLToPath(new URL("./model-family.ts", import.meta.url)), "utf8"); +const regSrc = readFileSync(fileURLToPath(new URL("./model-registry.ts", import.meta.url)), "utf8"); + +/** Extract the exact source span of a frozen surface, failing loudly if absent. */ +function extract(src: string, re: RegExp, name: string): string { + const m = src.match(re); + if (!m) { + throw new Error( + `logic-pin: could not locate frozen surface "${name}". It was renamed, ` + + `moved, or reshaped — this is itself a classification-logic change that ` + + `must be reviewed and re-pinned deliberately.`, + ); + } + return m[0]; +} + +function sha256(s: string): string { + return createHash("sha256").update(s).digest("hex"); +} + +/** + * Frozen surface → { extract, pin }. The extraction regexes anchor each + * function body on its column-0 closing brace (`\n}`), so an interior indented + * `}` never terminates the span early. + */ +const FROZEN: Record = { + DATED_SNAPSHOT_SUFFIX: { + source: extract(famSrc, /const DATED_SNAPSHOT_SUFFIX = .+;/, "DATED_SNAPSHOT_SUFFIX"), + pin: "99fcd34c515dfce4954b7c6a2bcf10a35b4f27e76107f96eb6235549225854b4", + }, + BUILD_TAG_SUFFIX: { + source: extract(famSrc, /const BUILD_TAG_SUFFIX = .+;/, "BUILD_TAG_SUFFIX"), + pin: "fe75d743b89f8eae942cd98ac8af56a25f3c86c8d64772f1aec139d1dd4fbddc", + }, + ANTHROPIC_DATE_SUFFIX: { + source: extract(famSrc, /const ANTHROPIC_DATE_SUFFIX = .+;/, "ANTHROPIC_DATE_SUFFIX"), + pin: "c79f8927776a618bb232d8b3d506296b84f9017db9f4ad56b0b20a84b9ce3a28", + }, + normalizeModelFamily: { + source: extract( + famSrc, + /export function normalizeModelFamily\([\s\S]*?\n}/, + "normalizeModelFamily", + ), + pin: "7c1236d8d644e6ec52879aae910c4b1491e51346f4dde0bb211f484013a33f50", + }, + PREVIEW_FAMILY: { + source: extract(regSrc, /export const PREVIEW_FAMILY = .+;/, "PREVIEW_FAMILY"), + pin: "da2e8a8a66ea8ac150de336429a74cc46fdaf0e3fb065614b810803a0187a3c8", + }, + GEMMA_FAMILY: { + source: extract(regSrc, /export const GEMMA_FAMILY = .+;/, "GEMMA_FAMILY"), + pin: "910e395e685e385bc924ea6900118bd8a3b93b5026513f92ea387409475555e8", + }, + NON_MODEL_TOKENS: { + source: extract( + regSrc, + /export const NON_MODEL_TOKENS: Set = new Set\(\[[\s\S]*?\]\);/, + "NON_MODEL_TOKENS", + ), + pin: "7531fa32d032016a25b7a95ce0da919bb5add8817c8f3c69c9af8fe732b0d332", + }, + familySet: { + source: extract(regSrc, /function familySet\([\s\S]*?\n}/, "familySet"), + pin: "d4ee5473b09e94a91f58b4acaff698aba00077fb2edf0b635cd8a41b6de2d58f", + }, + isClassifiedFamily: { + source: extract( + regSrc, + /export function isClassifiedFamily\([\s\S]*?\n}/, + "isClassifiedFamily", + ), + pin: "60d59a5c43f3c3d7788315a19bc0a576bfab0efce3a8e93b82e862cfb8a3d263", + }, +}; + +describe("classification-logic checksum freeze (Phase-0 anti-silence guard)", () => { + for (const [name, { source, pin }] of Object.entries(FROZEN)) { + it(`freezes ${name}`, () => { + expect( + sha256(source), + `Frozen classification surface "${name}" changed. If this edit is a ` + + `deliberate, reviewed rule change, update its pin here; if not, it is ` + + `a silent canary-silencing edit and must be reverted.`, + ).toBe(pin); + }); + } + + // Runtime-value pins — a second, human-readable lock on the exact rule shapes, + // independent of source formatting. Widening a regex or adding a routing token + // trips these as well as the source checksum above. + it("pins the PREVIEW_FAMILY exclude-by-rule pattern", () => { + expect(PREVIEW_FAMILY.source).toBe("-preview(-\\d+)?$"); + }); + + it("pins the GEMMA_FAMILY exclude-by-rule pattern", () => { + expect(GEMMA_FAMILY.source).toBe("^gemma(-|$)"); + }); + + it("pins NON_MODEL_TOKENS membership exactly", () => { + expect([...NON_MODEL_TOKENS].sort()).toEqual(["gemini-interactions"]); + }); + + // Behavioral golden anchors — prove the frozen source still MEANS what it + // should, and catch a regex widen/narrow through observed classification, not + // just bytes. These are the exact silencing surfaces §1 calls out. + it("keeps the normalizer stripping only the frozen suffix shapes", () => { + // dated snapshot + build tag stripped; single-digit tail preserved (canary) + expect(normalizeModelFamily("gpt-4o-2025-08-28", "openai")).toBe("gpt-4o"); + expect(normalizeModelFamily("tts-1-1106", "openai")).toBe("tts-1"); + expect(normalizeModelFamily("gpt-live-1", "openai")).toBe("gpt-live-1"); + // anthropic-only contiguous 8-digit snapshot + expect(normalizeModelFamily("claude-3-5-sonnet-20241022", "anthropic")).toBe( + "claude-3-5-sonnet", + ); + expect(normalizeModelFamily("gpt-weird-12345678", "openai")).toBe("gpt-weird-12345678"); + }); + + it("keeps isClassifiedFamily's known-vs-unknown boundary intact", () => { + // classified: seeded include, seeded exclude, preview-rule, gemma-rule + expect(isClassifiedFamily("gpt-4o", "openai")).toBe(true); + expect(isClassifiedFamily("tts-1", "openai")).toBe(true); + expect(isClassifiedFamily("gemini-3-pro-preview", "gemini")).toBe(true); + expect(isClassifiedFamily("gemma-4-31b-it", "gemini")).toBe(true); + // unknown families must stay flagged (the canary must not be silenced) + expect(isClassifiedFamily("gpt-live", "openai")).toBe(false); + expect(isClassifiedFamily("gemini-ultra", "gemini")).toBe(false); + // interior -preview- is NOT swept by the rule (stays explicit-only) + expect(PREVIEW_FAMILY.test("gemini-2.5-flash-preview-tts")).toBe(false); + }); +}); diff --git a/src/__tests__/drift/models.drift.ts b/src/__tests__/drift/models.drift.ts index d752b6f7..d1eedd01 100644 --- a/src/__tests__/drift/models.drift.ts +++ b/src/__tests__/drift/models.drift.ts @@ -27,10 +27,17 @@ */ import { describe, it, expect } from "vitest"; -import { listOpenAIModels, listAnthropicModels, listGeminiModels } from "./providers.js"; +import { + listOpenAIModels, + listAnthropicModels, + listGeminiModels, + InfraError, + isInfraSkip, +} from "./providers.js"; import { normalizeModelFamily } from "./model-family.js"; -import { NON_MODEL_TOKENS, isClassifiedFamily } from "./model-registry.js"; +import { NON_MODEL_TOKENS, isClassifiedFamily, includeFamilies } from "./model-registry.js"; import { formatDriftReport } from "./schema.js"; +import { isFamilyStillReferenced } from "./deprecation-detector.js"; type Provider = "openai" | "anthropic" | "gemini"; @@ -88,6 +95,224 @@ function assertNoUnclassifiedFamilies( expect(unclassified, report).toEqual([]); } +// --------------------------------------------------------------------------- +// C4: deterministic DEPRECATION detector — `classified − live` (net-new). +// +// The mirror image of `unclassifiedFamilies`: instead of a live family with no +// classification (drift = a new family), this flags a CLASSIFIED family +// (one aimock actively mocks, i.e. `includeFamilies[provider]`) that a +// healthy live `/models` listing no longer contains — a mechanical, zero-LLM +// signal the provider retired it. +// +// FAIL-CLOSED (safety-critical — a net-new DESTRUCTIVE path): never emit a +// removal signal off a listing that is empty, short/truncated, or that the +// caller could not fetch at all (infra error). A transient truncated or +// failed `/models` response must never look like "every family disappeared" +// and cascade into proposing to nuke the registry. The floor defaults to the +// number of families aimock mocks for that provider — a healthy provider +// listing returns many more raw ids than distinct families (every dated +// snapshot/build-tag variant multiplies one family into several ids), so a +// listing shorter than the family count itself is definitionally truncated, +// not a genuinely tiny catalog. +// +// A family that clears the fail-closed floor and is genuinely missing from +// the live listing is still not auto-proposed for removal if it is STILL +// REFERENCED elsewhere in aimock's own source (DEFAULT_MODELS, builders, +// fixtures — see `isFamilyStillReferenced` in `deprecation-detector.ts`): +// that case routes to a human instead (§4.4 — no silent auto-classify). +// --------------------------------------------------------------------------- + +/** One `classified − live` candidate, tagged with whether aimock's own source + * still references it (in which case it must route to a human, not be + * auto-proposed for removal — see module doc above). */ +export interface DeprecationCandidate { + provider: Provider; + family: string; + stillReferenced: boolean; +} + +export type DeprecationCheckResult = + | { status: "skipped"; reason: string } + | { status: "checked"; candidates: DeprecationCandidate[] }; + +/** + * Pure `classified − live` diff for one provider, given an already-fetched + * live `/models` id list. Fail-closed on an empty/short listing (see module + * doc). `opts.isReferenced` defaults to the real source-tree ref-scan + * (`isFamilyStillReferenced`) but is injectable so callers/tests can supply a + * deterministic stub without touching the filesystem. + */ +export function detectDeprecatedFamilies( + liveModelIds: string[], + provider: Provider, + opts: { + isReferenced?: (family: string, provider: Provider) => boolean; + minListingSize?: number; + } = {}, +): DeprecationCheckResult { + const classified = includeFamilies[provider]; + const floor = opts.minListingSize ?? classified.size; + + if (liveModelIds.length === 0 || liveModelIds.length < floor) { + return { + status: "skipped", + reason: + `live /models listing too short to trust for ${provider} ` + + `(${liveModelIds.length} raw id(s), need >= ${floor} — the number of ` + + `families aimock mocks for this provider) — never mass-removing off a ` + + `truncated or empty listing`, + }; + } + + const liveFamilies = new Set(liveModelIds.map((id) => normalizeModelFamily(id, provider))); + const missing = [...classified].filter((family) => !liveFamilies.has(family)).sort(); + const isReferenced = opts.isReferenced ?? isFamilyStillReferenced; + + return { + status: "checked", + candidates: missing.map((family) => ({ + provider, + family, + stillReferenced: isReferenced(family, provider), + })), + }; +} + +/** + * Async wrapper around {@link detectDeprecatedFamilies} for a real live-fetch + * function: classifies a thrown {@link InfraError} via `isInfraSkip` (R0) as + * an honest SKIP — the same auth/credit/rate-limit/5xx conditions the other + * live legs treat as a transient provider-side outage, never a drift/removal + * finding. A non-infra error still propagates (never silently swallowed). + */ +export async function checkDeprecatedFamiliesLive( + fetchLiveModels: () => Promise, + provider: Provider, + opts: { + isReferenced?: (family: string, provider: Provider) => boolean; + minListingSize?: number; + } = {}, +): Promise { + try { + const liveModelIds = await fetchLiveModels(); + return detectDeprecatedFamilies(liveModelIds, provider, opts); + } catch (err) { + if (err instanceof InfraError && isInfraSkip(err.status)) { + return { + status: "skipped", + reason: + `infra error (status ${err.status}) fetching live /models for ` + + `${provider} — never mass-removing off a failed listing`, + }; + } + throw err; + } +} + +describe("C4: detectDeprecatedFamilies (classified − live, fail-closed)", () => { + it("FAIL-CLOSED: an empty live listing never proposes removal", () => { + const result = detectDeprecatedFamilies([], "openai"); + expect(result.status).toBe("skipped"); + }); + + it("FAIL-CLOSED: a short/truncated live listing never proposes removal", () => { + // Far fewer raw ids than openai's include-family count — a truncated + // listing, not a genuinely tiny provider catalog. + const result = detectDeprecatedFamilies(["gpt-4o"], "openai"); + expect(result.status).toBe("skipped"); + }); + + it("a healthy listing omitting a classified family flags EXACTLY that family", () => { + // A healthy-sized live listing (>= the fail-closed floor) covering every + // include family EXCEPT gpt-4o, whose family is genuinely missing. + const allButGpt4o = [...includeFamilies.openai].filter((f) => f !== "gpt-4o"); + const liveIds = [...allButGpt4o, ...allButGpt4o.map((f) => `${f}-2025-01-01`)]; + const result = detectDeprecatedFamilies(liveIds, "openai", { isReferenced: () => false }); + expect(result.status).toBe("checked"); + if (result.status !== "checked") return; + expect(result.candidates).toEqual([ + { provider: "openai", family: "gpt-4o", stillReferenced: false }, + ]); + }); + + it("a still-referenced deprecated family routes to human (stillReferenced: true)", () => { + const allButGpt4o = [...includeFamilies.openai].filter((f) => f !== "gpt-4o"); + const liveIds = [...allButGpt4o, ...allButGpt4o.map((f) => `${f}-2025-01-01`)]; + const result = detectDeprecatedFamilies(liveIds, "openai", { isReferenced: () => true }); + expect(result.status).toBe("checked"); + if (result.status !== "checked") return; + expect(result.candidates).toEqual([ + { provider: "openai", family: "gpt-4o", stillReferenced: true }, + ]); + }); + + it("a healthy listing missing nothing reports zero candidates", () => { + const allFamilies = [...includeFamilies.openai]; + const liveIds = [...allFamilies, ...allFamilies.map((f) => `${f}-2025-01-01`)]; + const result = detectDeprecatedFamilies(liveIds, "openai"); + expect(result.status).toBe("checked"); + if (result.status !== "checked") return; + expect(result.candidates).toEqual([]); + }); +}); + +describe("C4: checkDeprecatedFamiliesLive (infra-error short-circuit via isInfraSkip)", () => { + it("an infra error (401/402/403/429/5xx) is an honest SKIP, never a removal signal", async () => { + for (const status of [401, 402, 403, 429, 503]) { + expect(isInfraSkip(status)).toBe(true); // R0 classification consumed directly below + const result = await checkDeprecatedFamiliesLive( + () => Promise.reject(new InfraError("boom", status)), + "openai", + ); + expect(result.status).toBe("skipped"); + } + }); + + it("a genuine non-infra error propagates (never silently swallowed as a skip)", async () => { + await expect( + checkDeprecatedFamiliesLive(() => Promise.reject(new Error("boom, not infra")), "openai"), + ).rejects.toThrow("boom, not infra"); + }); + + it("an InfraError with a non-skip status (e.g. a hypothetical 3xx) still propagates", () => { + expect(isInfraSkip(200)).toBe(false); + expect(isInfraSkip(404)).toBe(false); + }); + + it("a healthy fetch flows through to the deterministic diff", async () => { + const allButGpt4o = [...includeFamilies.openai].filter((f) => f !== "gpt-4o"); + const liveIds = [...allButGpt4o, ...allButGpt4o.map((f) => `${f}-2025-01-01`)]; + const result = await checkDeprecatedFamiliesLive(() => Promise.resolve(liveIds), "openai", { + isReferenced: () => false, + }); + expect(result.status).toBe("checked"); + if (result.status !== "checked") return; + expect(result.candidates).toEqual([ + { provider: "openai", family: "gpt-4o", stillReferenced: false }, + ]); + }); +}); + +describe("C4: isFamilyStillReferenced (real source-tree ref-scan, zero-LLM)", () => { + it("finds a real reference (gpt-4 and gpt-4o are both referenced in src/server.ts)", () => { + expect(isFamilyStillReferenced("gpt-4", "openai")).toBe(true); + expect(isFamilyStillReferenced("gpt-4o", "openai")).toBe(true); + }); + + it("reports zero-reference for a synthetic family that appears nowhere in src/", () => { + expect(isFamilyStillReferenced("zzz-totally-fictional-family-not-real", "openai")).toBe(false); + }); + + it("does not false-positive from being a strict substring of a longer live id", () => { + // A naive substring scan would call "gpt-4" referenced merely because + // "gpt-4-turbo"/"gpt-4o" appear in source; the boundary-aware scan must + // not confuse the two. gpt-4 IS separately, exactly referenced in + // DEFAULT_MODELS (asserted above) — this confirms the match is a real + // boundary hit, not a substring artifact. + expect(isFamilyStillReferenced("gpt-4-turbo-nonexistent-suffix", "openai")).toBe(false); + }); +}); + // --------------------------------------------------------------------------- // Regression suite (no live keys) — exercises the REAL pipeline with injected // `/models` payloads. Runs unconditionally so the drift job proves the diff --git a/src/__tests__/drift/openai-chat.drift.ts b/src/__tests__/drift/openai-chat.drift.ts index 8c08252f..11b8df17 100644 --- a/src/__tests__/drift/openai-chat.drift.ts +++ b/src/__tests__/drift/openai-chat.drift.ts @@ -17,7 +17,13 @@ import { openaiChatCompletionReasoningShape, openaiChatCompletionReasoningChunkShape, } from "./sdk-shapes.js"; -import { openaiChatNonStreaming, openaiChatStreaming } from "./providers.js"; +import { + resolveLiveModel, + isInfraSkip, + isModelNotFound, + type ResolvedModel, + type LiveModelEntry, +} from "./providers.js"; import { httpPost, parseDataOnlySSE, startDriftServer, stopDriftServer } from "./helpers.js"; // --------------------------------------------------------------------------- @@ -35,26 +41,121 @@ afterAll(async () => { await stopDriftServer(instance); }); +// --------------------------------------------------------------------------- +// Live chat-model resolution (self-healing — generalizes cohere #325 / R0) +// --------------------------------------------------------------------------- +// +// `gpt-4o-mini` is a stable OpenAI alias (not a dated snapshot), but OpenAI +// still retires aliases on its own schedule. Discover a currently-live, +// non-deprecated chat model from OpenAI's own `/v1/models` listing rather +// than trusting the hardcoded literal to remain valid forever, and treat any +// auth/credit/rate-limit/5xx condition on the listing (or on the real chat +// call itself) as an HONEST SKIP rather than a drift finding that reds the +// PR. A real shape drift (a 2xx envelope that doesn't match) is never +// skipped — only status-classified provider-side conditions are. +// +// providers.ts's own `openaiChatNonStreaming`/`openaiChatStreaming` hardcode +// "gpt-4o-mini" and take no model parameter, so the resolved model below is +// threaded through a LOCAL, model-parameterized raw fetch (mirrors +// openai-responses.drift.ts's `fetchOpenAIResponses`) rather than a +// providers.ts edit — otherwise the discovered model never reaches the real +// API and the self-healing degrades to an eternal honest-skip once +// gpt-4o-mini retires. + +const OPENAI_CHAT_URL = "https://api.openai.com/v1/chat/completions"; +const OPENAI_MODELS_URL = "https://api.openai.com/v1/models"; + +/** Maps OpenAI's `/v1/models` listing shape onto the shared `LiveModelEntry`. */ +async function fetchOpenAIModelListing(): Promise<{ status: number; models: LiveModelEntry[] }> { + const res = await fetch(OPENAI_MODELS_URL, { + headers: { Authorization: `Bearer ${OPENAI_API_KEY}` }, + }); + if (!res.ok) return { status: res.status, models: [] }; + const json = (await res.json()) as { data?: { id: string }[] }; + // OpenAI's listing does not expose a `deprecated` flag — every listed id is + // presumed live; retirement shows up as a model-not-found on the real call. + const models: LiveModelEntry[] = (json.data ?? []).map((m) => ({ id: m.id })); + return { status: res.status, models }; +} + +let openaiChatModelPromise: Promise | null = null; + +/** Memoized so the whole live leg makes exactly one model-listing call. */ +function getOpenAIChatModel(): Promise { + if (!openaiChatModelPromise) { + openaiChatModelPromise = resolveLiveModel("openai-chat", fetchOpenAIModelListing, [ + "gpt-4o-mini", + ]); + } + return openaiChatModelPromise; +} + +/** + * Raw Chat Completions fetch parameterized by a discovered model id, + * returning the raw status/body so the caller can classify a retired-model or + * provider-side condition via {@link isModelNotFound}/{@link isInfraSkip} + * BEFORE asserting success (unlike providers.ts's variants, which throw an + * opaque InfraError on any non-2xx). + */ +async function fetchOpenAIChat( + model: string, + messages: { role: string; content: string }[], + tools: object[] | undefined, + stream: boolean, +): Promise<{ status: number; raw: string }> { + const body: Record = { model, messages, stream, max_tokens: 10 }; + if (tools) body.tools = tools; + + const res = await fetch(OPENAI_CHAT_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${OPENAI_API_KEY}`, + }, + body: JSON.stringify(body), + }); + + return { status: res.status, raw: await res.text() }; +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- describe.skipIf(!OPENAI_API_KEY)("OpenAI Chat Completions drift", () => { - const config = { apiKey: OPENAI_API_KEY! }; + it("non-streaming text shape matches", async (ctx) => { + const resolved = await getOpenAIChatModel(); + if ("infra" in resolved) { + // Listing hit an auth/credit/rate-limit/5xx condition — honest skip. + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error("OpenAI /v1/models exposed no usable chat model"); + } + const model = resolved.model; - it("non-streaming text shape matches", async () => { const sdkShape = openaiChatCompletionShape(); + const messages = [{ role: "user", content: "Say hello" }]; const [realRes, mockRes] = await Promise.all([ - openaiChatNonStreaming(config, [{ role: "user", content: "Say hello" }]), + fetchOpenAIChat(model, messages, undefined, false), httpPost(`${instance.url}/v1/chat/completions`, { - model: "gpt-4o-mini", - messages: [{ role: "user", content: "Say hello" }], + model, + messages, stream: false, }), ]); - const realShape = extractShape(realRes.body); + if (isInfraSkip(realRes.status) || isModelNotFound(realRes.status, realRes.raw)) { + // Retired model id or a transient provider-side condition — honest + // skip, never a drift finding that quarantines the shared baseline. + ctx.skip(); + return; + } + expect(realRes.status, `Real API error: ${realRes.raw.slice(0, 300)}`).toBe(200); + + const realShape = extractShape(JSON.parse(realRes.raw)); const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); @@ -66,24 +167,42 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Chat Completions drift", () => { ).toEqual([]); }); - it("streaming text shape matches", async () => { + it("streaming text shape matches", async (ctx) => { + const resolved = await getOpenAIChatModel(); + if ("infra" in resolved) { + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error("OpenAI /v1/models exposed no usable chat model"); + } + const model = resolved.model; + const sdkChunkShape = openaiChatCompletionChunkShape(); + const messages = [{ role: "user", content: "Say hello" }]; - const [realStream, mockStreamRes] = await Promise.all([ - openaiChatStreaming(config, [{ role: "user", content: "Say hello" }]), + const [realRes, mockStreamRes] = await Promise.all([ + fetchOpenAIChat(model, messages, undefined, true), httpPost(`${instance.url}/v1/chat/completions`, { - model: "gpt-4o-mini", - messages: [{ role: "user", content: "Say hello" }], + model, + messages, stream: true, }), ]); + if (isInfraSkip(realRes.status) || isModelNotFound(realRes.status, realRes.raw)) { + ctx.skip(); + return; + } + expect(realRes.status, `Real API error: ${realRes.raw.slice(0, 300)}`).toBe(200); + + const realChunks = parseDataOnlySSE(realRes.raw); const mockChunks = parseDataOnlySSE(mockStreamRes.body); - expect(realStream.rawEvents.length, "Real API returned no SSE events").toBeGreaterThan(0); + expect(realChunks.length, "Real API returned no SSE events").toBeGreaterThan(0); expect(mockChunks.length, "Mock returned no SSE chunks").toBeGreaterThan(0); - const realChunkShape = extractShape(realStream.rawEvents[0].data); + const realChunkShape = extractShape(realChunks[0]); const mockChunkShape = extractShape(mockChunks[0]); const diffs = triangulate(sdkChunkShape, realChunkShape, mockChunkShape); @@ -95,7 +214,17 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Chat Completions drift", () => { ).toEqual([]); }); - it("non-streaming tool call shape matches", async () => { + it("non-streaming tool call shape matches", async (ctx) => { + const resolved = await getOpenAIChatModel(); + if ("infra" in resolved) { + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error("OpenAI /v1/models exposed no usable chat model"); + } + const model = resolved.model; + const sdkShape = openaiChatCompletionToolCallShape(); const tools = [ @@ -112,18 +241,25 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Chat Completions drift", () => { }, }, ]; + const messages = [{ role: "user", content: "Weather in Paris" }]; const [realRes, mockRes] = await Promise.all([ - openaiChatNonStreaming(config, [{ role: "user", content: "Weather in Paris" }], tools), + fetchOpenAIChat(model, messages, tools, false), httpPost(`${instance.url}/v1/chat/completions`, { - model: "gpt-4o-mini", - messages: [{ role: "user", content: "Weather in Paris" }], + model, + messages, stream: false, tools, }), ]); - const realShape = extractShape(realRes.body); + if (isInfraSkip(realRes.status) || isModelNotFound(realRes.status, realRes.raw)) { + ctx.skip(); + return; + } + expect(realRes.status, `Real API error: ${realRes.raw.slice(0, 300)}`).toBe(200); + + const realShape = extractShape(JSON.parse(realRes.raw)); const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); @@ -135,7 +271,17 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Chat Completions drift", () => { ).toEqual([]); }); - it("streaming tool call shape matches", async () => { + it("streaming tool call shape matches", async (ctx) => { + const resolved = await getOpenAIChatModel(); + if ("infra" in resolved) { + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error("OpenAI /v1/models exposed no usable chat model"); + } + const model = resolved.model; + const sdkChunkShape = openaiChatCompletionChunkShape(); const tools = [ @@ -152,23 +298,31 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Chat Completions drift", () => { }, }, ]; + const messages = [{ role: "user", content: "Weather in Paris" }]; - const [realStream, mockStreamRes] = await Promise.all([ - openaiChatStreaming(config, [{ role: "user", content: "Weather in Paris" }], tools), + const [realRes, mockStreamRes] = await Promise.all([ + fetchOpenAIChat(model, messages, tools, true), httpPost(`${instance.url}/v1/chat/completions`, { - model: "gpt-4o-mini", - messages: [{ role: "user", content: "Weather in Paris" }], + model, + messages, stream: true, tools, }), ]); + if (isInfraSkip(realRes.status) || isModelNotFound(realRes.status, realRes.raw)) { + ctx.skip(); + return; + } + expect(realRes.status, `Real API error: ${realRes.raw.slice(0, 300)}`).toBe(200); + + const realChunks = parseDataOnlySSE(realRes.raw); const mockChunks = parseDataOnlySSE(mockStreamRes.body); - expect(realStream.rawEvents.length, "Real API returned no SSE events").toBeGreaterThan(0); + expect(realChunks.length, "Real API returned no SSE events").toBeGreaterThan(0); expect(mockChunks.length, "Mock returned no SSE chunks").toBeGreaterThan(0); - const realChunkShape = extractShape(realStream.rawEvents[0].data); + const realChunkShape = extractShape(realChunks[0]); const mockChunkShape = extractShape(mockChunks[0]); const diffs = triangulate(sdkChunkShape, realChunkShape, mockChunkShape); diff --git a/src/__tests__/drift/openai-responses.drift.ts b/src/__tests__/drift/openai-responses.drift.ts index a1060266..6d5231cf 100644 --- a/src/__tests__/drift/openai-responses.drift.ts +++ b/src/__tests__/drift/openai-responses.drift.ts @@ -14,7 +14,13 @@ import { openaiResponsesToolCallEventShapes, openaiResponsesReasoningEventShapes, } from "./sdk-shapes.js"; -import { openaiResponsesNonStreaming, openaiResponsesStreaming } from "./providers.js"; +import { + resolveLiveModel, + isInfraSkip, + isModelNotFound, + type LiveModelEntry, + type ResolvedModel, +} from "./providers.js"; import { httpPost, httpPostRaw, @@ -38,26 +44,113 @@ afterAll(async () => { await stopDriftServer(instance); }); +// --------------------------------------------------------------------------- +// Live model discovery (self-healing) +// --------------------------------------------------------------------------- +// +// Replaces the hardcoded "gpt-4o-mini" pin with a live-listing lookup so a +// retired/renamed alias resolves to a currently-valid model (or honest-skips) +// instead of quarantining this leg's whole batch. Generalizes the cohere +// (#325) discovery + fal (#332) infra-skip patterns via providers.ts's shared +// resolveLiveModel/isInfraSkip/isModelNotFound. +// +// providers.ts's own `openaiResponsesNonStreaming`/`openaiResponsesStreaming` +// hardcode "gpt-4o-mini" and take no model parameter, so the live probe below +// is a local, model-parameterized fetch (mirrors cohere.drift.ts's pattern of +// a leg-local raw-fetch helper) rather than a providers.ts edit. + +const OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"; +const OPENAI_MODELS_URL = "https://api.openai.com/v1/models"; + +/** Maps OpenAI's `/v1/models` listing shape onto {@link LiveModelEntry}. */ +export async function fetchOpenAIModelsListing(): Promise<{ + status: number; + models: LiveModelEntry[]; +}> { + const res = await fetch(OPENAI_MODELS_URL, { + headers: { Authorization: `Bearer ${OPENAI_API_KEY}` }, + }); + const raw = await res.text(); + if (res.status >= 400) return { status: res.status, models: [] }; + let json: { data?: { id: string }[] }; + try { + json = JSON.parse(raw) as { data?: { id: string }[] }; + } catch { + return { status: res.status, models: [] }; + } + return { status: res.status, models: (json.data ?? []).map((m) => ({ id: m.id })) }; +} + +/** Memoized (per providers.ts `resolveLiveModel`) so this file makes one listing call. */ +export function getOpenAIResponsesModel(): Promise { + return resolveLiveModel("openai-responses", fetchOpenAIModelsListing, ["gpt-4o-mini", "gpt-4o"]); +} + +/** + * Raw Responses API fetch parameterized by a discovered model id, returning + * the raw status/body so the caller can classify a retired-model or + * provider-side condition via {@link isModelNotFound}/{@link isInfraSkip} + * BEFORE asserting success (unlike providers.ts's variants, which throw an + * opaque InfraError on any non-2xx). + */ +export async function fetchOpenAIResponses( + model: string, + input: object[], + tools: object[] | undefined, + stream: boolean, +): Promise<{ status: number; raw: string }> { + const body: Record = { model, input, stream, max_output_tokens: 50 }; + if (tools) body.tools = tools; + + const res = await fetch(OPENAI_RESPONSES_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${OPENAI_API_KEY}`, + }, + body: JSON.stringify(body), + }); + + return { status: res.status, raw: await res.text() }; +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses API drift", () => { - const config = { apiKey: OPENAI_API_KEY! }; + it("non-streaming text shape matches", async (ctx) => { + const resolved = await getOpenAIResponsesModel(); + if ("infra" in resolved) { + // Provider-side auth/credit/rate-limit/5xx — honest skip, not drift. + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error("OpenAI /v1/models exposed no usable model for the Responses API"); + } + const model = resolved.model; - it("non-streaming text shape matches", async () => { const sdkShape = openaiResponsesNonStreamingShape(); + const input = [{ role: "user", content: "Say hello" }]; const [realRes, mockRes] = await Promise.all([ - openaiResponsesNonStreaming(config, [{ role: "user", content: "Say hello" }]), + fetchOpenAIResponses(model, input, undefined, false), httpPost(`${instance.url}/v1/responses`, { - model: "gpt-4o-mini", - input: [{ role: "user", content: "Say hello" }], + model, + input, stream: false, }), ]); - const realShape = extractShape(realRes.body); + if (isInfraSkip(realRes.status) || isModelNotFound(realRes.status, realRes.raw)) { + // Retired/renamed model or a transient provider condition — honest skip. + ctx.skip(); + return; + } + expect(realRes.status, `Real API error: ${realRes.raw.slice(0, 300)}`).toBe(200); + + const realShape = extractShape(JSON.parse(realRes.raw)); const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); @@ -73,19 +166,41 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses API drift", () => { ).toEqual([]); }); - it("streaming text event sequence and shapes match", async () => { + it("streaming text event sequence and shapes match", async (ctx) => { + const resolved = await getOpenAIResponsesModel(); + if ("infra" in resolved) { + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error("OpenAI /v1/models exposed no usable model for the Responses API"); + } + const model = resolved.model; + const sdkEvents = openaiResponsesTextEventShapes(); + const input = [{ role: "user", content: "Say hello" }]; - const [realStream, mockStreamRes] = await Promise.all([ - openaiResponsesStreaming(config, [{ role: "user", content: "Say hello" }]), + const [realRes, mockStreamRes] = await Promise.all([ + fetchOpenAIResponses(model, input, undefined, true), httpPost(`${instance.url}/v1/responses`, { - model: "gpt-4o-mini", - input: [{ role: "user", content: "Say hello" }], + model, + input, stream: true, }), ]); - expect(realStream.rawEvents.length, "Real API returned no SSE events").toBeGreaterThan(0); + if (isInfraSkip(realRes.status) || isModelNotFound(realRes.status, realRes.raw)) { + ctx.skip(); + return; + } + expect(realRes.status, `Real API error: ${realRes.raw.slice(0, 300)}`).toBe(200); + + const realEvents = parseTypedSSE(realRes.raw); + expect(realEvents.length, "Real API returned no SSE events").toBeGreaterThan(0); + const realSSEShapes = realEvents.map((e) => ({ + type: e.type, + dataShape: extractShape(e.data), + })); const mockEvents = parseTypedSSE(mockStreamRes.body); expect(mockEvents.length, "Mock returned no SSE events").toBeGreaterThan(0); @@ -95,7 +210,7 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses API drift", () => { dataShape: extractShape(e.data), })); - const diffs = compareSSESequences(sdkEvents, realStream.events, mockSSEShapes); + const diffs = compareSSESequences(sdkEvents, realSSEShapes, mockSSEShapes); const report = formatDriftReport( "OpenAI Responses (streaming text events)", diffs, @@ -108,7 +223,17 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses API drift", () => { ).toEqual([]); }); - it("non-streaming tool call shape matches", async () => { + it("non-streaming tool call shape matches", async (ctx) => { + const resolved = await getOpenAIResponsesModel(); + if ("infra" in resolved) { + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error("OpenAI /v1/models exposed no usable model for the Responses API"); + } + const model = resolved.model; + const sdkShape = openaiResponsesNonStreamingShape(); const tools = [ @@ -123,18 +248,25 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses API drift", () => { }, }, ]; + const input = [{ role: "user", content: "Weather in Paris" }]; const [realRes, mockRes] = await Promise.all([ - openaiResponsesNonStreaming(config, [{ role: "user", content: "Weather in Paris" }], tools), + fetchOpenAIResponses(model, input, tools, false), httpPost(`${instance.url}/v1/responses`, { - model: "gpt-4o-mini", - input: [{ role: "user", content: "Weather in Paris" }], + model, + input, stream: false, tools, }), ]); - const realShape = extractShape(realRes.body); + if (isInfraSkip(realRes.status) || isModelNotFound(realRes.status, realRes.raw)) { + ctx.skip(); + return; + } + expect(realRes.status, `Real API error: ${realRes.raw.slice(0, 300)}`).toBe(200); + + const realShape = extractShape(JSON.parse(realRes.raw)); const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); @@ -150,7 +282,17 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses API drift", () => { ).toEqual([]); }); - it("streaming tool call event sequence matches", async () => { + it("streaming tool call event sequence matches", async (ctx) => { + const resolved = await getOpenAIResponsesModel(); + if ("infra" in resolved) { + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error("OpenAI /v1/models exposed no usable model for the Responses API"); + } + const model = resolved.model; + const sdkEvents = [ ...openaiResponsesTextEventShapes().filter( (e) => e.type === "response.created" || e.type === "response.completed", @@ -170,18 +312,30 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses API drift", () => { }, }, ]; + const input = [{ role: "user", content: "Weather in Paris" }]; - const [realStream, mockStreamRes] = await Promise.all([ - openaiResponsesStreaming(config, [{ role: "user", content: "Weather in Paris" }], tools), + const [realRes, mockStreamRes] = await Promise.all([ + fetchOpenAIResponses(model, input, tools, true), httpPost(`${instance.url}/v1/responses`, { - model: "gpt-4o-mini", - input: [{ role: "user", content: "Weather in Paris" }], + model, + input, stream: true, tools, }), ]); - expect(realStream.rawEvents.length, "Real API returned no SSE events").toBeGreaterThan(0); + if (isInfraSkip(realRes.status) || isModelNotFound(realRes.status, realRes.raw)) { + ctx.skip(); + return; + } + expect(realRes.status, `Real API error: ${realRes.raw.slice(0, 300)}`).toBe(200); + + const realEvents = parseTypedSSE(realRes.raw); + expect(realEvents.length, "Real API returned no SSE events").toBeGreaterThan(0); + const realSSEShapes = realEvents.map((e) => ({ + type: e.type, + dataShape: extractShape(e.data), + })); const mockEvents = parseTypedSSE(mockStreamRes.body); expect(mockEvents.length, "Mock returned no SSE events").toBeGreaterThan(0); @@ -191,7 +345,7 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses API drift", () => { dataShape: extractShape(e.data), })); - const diffs = compareSSESequences(sdkEvents, realStream.events, mockSSEShapes); + const diffs = compareSSESequences(sdkEvents, realSSEShapes, mockSSEShapes); const report = formatDriftReport( "OpenAI Responses (streaming tool call events)", diffs, diff --git a/src/__tests__/drift/providers.ts b/src/__tests__/drift/providers.ts index 6f4e5307..68100f53 100644 --- a/src/__tests__/drift/providers.ts +++ b/src/__tests__/drift/providers.ts @@ -48,6 +48,135 @@ export class InfraError extends Error { } } +// --------------------------------------------------------------------------- +// Shared self-healing helpers (live-model discovery + infra-skip) +// --------------------------------------------------------------------------- +// +// Generalizes the two already-merged self-healing patterns so the live-leg +// retrofits reuse ONE util instead of each inlining a divergent copy: +// - Cohere dynamic model discovery (#325 / cohere-model.ts): resolve a live, +// non-deprecated model id from the provider's own `/models` listing rather +// than hardcoding one the provider later retires (the `command-r-plus` 404 +// that quarantined the leg as exit 5). +// - fal infra-skip (#332): classify a provider-side auth/credit/rate-limit/5xx +// condition as an HONEST SKIP so a transient outage never quarantines the +// shared drift baseline. + +/** + * Environmental HTTP statuses that indicate a provider-side condition + * (auth / out-of-credit / rate-limit / upstream outage) rather than a real + * API-envelope drift. Mirrors the {@link InfraError} classification: + * 401 | 403 → stale-key 402 → out-of-credit + * 429 → rate-limited 5xx → infra-transient + * + * A live leg converts these to an HONEST SKIP so a transient provider-side + * condition never quarantines the drift collector and reds the PR. A real + * shape drift (a 2xx response with an unexpected envelope) is NEVER an infra + * status, so genuine drift still fails. Status-only by design — a 404 on a + * listing endpoint is a broken listing (unavailable), not a skip; the + * retired-MODEL case is {@link isModelNotFound}. + */ +export function isInfraSkip(status: number): boolean { + return status === 401 || status === 402 || status === 403 || status === 429 || status >= 500; +} + +/** + * True when a PROBE response indicates the requested MODEL does not exist (the + * pinned model id was retired) rather than a live-envelope drift. Providers + * signal this as a 404, or a 400 whose error body names a model-not-found + * condition (OpenAI `model_not_found`, "does not exist", "unknown model", + * etc.). The retrofit legs convert this to an honest skip — a stale pin is an + * operational nudge to re-discover, not a drift finding that should red the PR. + * + * Distinct from {@link isInfraSkip} because it is PROBE-scoped and body-aware: + * the model-listing/discovery path treats a 404 as `unavailable` (fail-loud), + * whereas a driven-probe 404 means the model id went stale. + */ +export function isModelNotFound(status: number, body?: string): boolean { + if (status === 404) return true; + if (status === 400 && body) { + return /model[_\s-]?not[_\s-]?found|does\s+not\s+exist|not\s+a\s+valid\s+model|unknown\s+model/i.test( + body, + ); + } + return false; +} + +/** Minimal shape of a `/models` listing entry we depend on for selection. */ +export interface LiveModelEntry { + /** The model identifier (OpenAI `id`, Cohere/Gemini `name`, …). */ + id: string; + /** True when the provider marks this model deprecated/retired. */ + deprecated?: boolean; +} + +/** + * Select a currently-valid, NON-deprecated model id from a `/models` listing, + * generalizing `selectCohereChatModel`. Prefers an id from `preferred` (in + * order) when it is present AND live, else falls back to the first live model. + * Returns null when the listing exposes no usable (non-deprecated, non-empty) + * model. + */ +export function selectLiveModel(models: LiveModelEntry[], preferred: string[] = []): string | null { + const liveIds = models + .filter((m) => typeof m.id === "string" && m.id.length > 0 && m.deprecated !== true) + .map((m) => m.id); + return preferred.find((p) => liveIds.includes(p)) ?? liveIds[0] ?? null; +} + +/** + * Outcome of resolving a live model for a leg: + * - `{ model }` → a valid, non-deprecated id to drive the leg + * - `{ infra }` → the listing hit an auth/credit/rate-limit/5xx + * condition; the caller SKIPS honestly (not drift) + * - `{ unavailable }` → the listing succeeded (or hit a non-infra error) but + * exposed no usable model — a genuinely broken state to + * fail loud on + */ +export type ResolvedModel = { model: string } | { infra: number } | { unavailable: true }; + +/** Per-key memo so the whole leg makes exactly one listing call. */ +const resolvedModelCache = new Map>(); + +/** + * Resolve a live model from a provider's own listing endpoint, MEMOIZED per + * `key` so repeated calls within one leg make a single listing call. + * Generalizes cohere's `resolveCohereChatModel` + `getCohereChatModel`. + * + * `fetchListing` is provider-specific: it performs the raw listing call and + * normalizes it to `{ status, models }` (mapping the provider's field names + * onto {@link LiveModelEntry}). An {@link InfraError} thrown from it (e.g. a + * network failure surfaced through {@link withInfraErrorTag}) is mapped to an + * honest `{ infra }` skip rather than propagating. + */ +export function resolveLiveModel( + key: string, + fetchListing: () => Promise<{ status: number; models: LiveModelEntry[] }>, + preferred: string[] = [], +): Promise { + const cached = resolvedModelCache.get(key); + if (cached) return cached; + const promise = (async (): Promise => { + try { + const { status, models } = await fetchListing(); + if (isInfraSkip(status)) return { infra: status }; + if (status >= 400) return { unavailable: true }; + const model = selectLiveModel(models, preferred); + return model ? { model } : { unavailable: true }; + } catch (err) { + if (err instanceof InfraError) return { infra: err.status }; + throw err; + } + })(); + resolvedModelCache.set(key, promise); + return promise; +} + +/** Test-only: clear the {@link resolveLiveModel} memo cache between cases. */ +export function __resetResolveLiveModelCache(): void { + resolvedModelCache.clear(); +} + // --------------------------------------------------------------------------- // Retry helper // --------------------------------------------------------------------------- diff --git a/src/__tests__/drift/ws-gemini-live.drift.ts b/src/__tests__/drift/ws-gemini-live.drift.ts index 0ec45f5b..03e2e12a 100644 --- a/src/__tests__/drift/ws-gemini-live.drift.ts +++ b/src/__tests__/drift/ws-gemini-live.drift.ts @@ -3,14 +3,26 @@ * * Three-way comparison: SDK types × real API (WS) × aimock output (WS). * - * Currently, the Gemini Live API only supports native-audio models - * (those with "native-audio" in the name) which cannot return TEXT responses. - * The canary test below checks the model listing API for any text-capable - * model that supports bidiGenerateContent. When Google adds one, the - * canary fails and the full drift tests can be enabled with that model. + * Currently, the Gemini Live API only supports native-audio models (those + * with "native-audio" in the name) which cannot return TEXT responses. Rather + * than a hardcoded model pin + a human-maintained `.skip`/un-skip cycle, model + * selection is SELF-HEALING: `fetchLiveCapableTextModels` + providers.ts's + * shared `resolveLiveModel` (the R0 discovery helper, generalizing the cohere + * #325 pattern) query the live model listing on every run and resolve to + * exactly one of: + * - `{ model }` — a text-capable bidiGenerateContent model exists NOW; + * the drift tests below run for real against it. + * - `{ unavailable }` — today's real-world state (no such model yet) — an + * HONEST SKIP, not a failure. + * - `{ infra }` — the listing hit an auth/rate-limit/5xx condition — + * also an honest skip (never a hard-fail). + * A WS handshake-level infra status (401/403/429/5xx) is likewise an honest + * skip via `WSHandshakeError` + `isInfraSkip` (ws-providers.ts, shared with + * every other WS live leg). Only a genuine 2xx envelope-shape mismatch reports + * drift — grading is on SHAPE (`compareSSESequences`), never on status codes. */ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { describe, it, expect, beforeAll, afterAll, afterEach, vi } from "vitest"; import type { ServerInstance } from "../../server.js"; import { extractShape, compareSSESequences, formatDriftReport } from "./schema.js"; import { @@ -18,7 +30,13 @@ import { geminiLiveTextEventShapes, geminiLiveToolCallEventShapes, } from "./sdk-shapes.js"; -import { geminiLiveWS } from "./ws-providers.js"; +import { geminiLiveWS, WSHandshakeError } from "./ws-providers.js"; +import { + resolveLiveModel, + isInfraSkip, + __resetResolveLiveModelCache, + type LiveModelEntry, +} from "./providers.js"; import { startDriftServer, stopDriftServer, @@ -44,62 +62,114 @@ afterAll(async () => { }); // --------------------------------------------------------------------------- -// Canary: detect when a text-capable model supports bidiGenerateContent +// Self-healing model discovery: find a text-capable bidiGenerateContent model // --------------------------------------------------------------------------- /** - * Query the Gemini model listing API for any model that supports - * bidiGenerateContent but is NOT a native-audio-only model. + * Query the Gemini model listing API and normalize it to the `resolveLiveModel` + * contract (R0's shared discovery helper), pre-filtered to models that support + * `bidiGenerateContent` AND are NOT native-audio-only (the only kind that can + * return TEXT responses over the Live WS protocol). A non-2xx status is + * surfaced so `resolveLiveModel`/`isInfraSkip` can classify an auth/rate-limit/ + * 5xx listing failure as an honest skip rather than this function propagating + * a bare fetch error. */ -async function findTextCapableLiveModel(apiKey: string): Promise { +async function fetchLiveCapableTextModels( + apiKey: string, +): Promise<{ status: number; models: LiveModelEntry[] }> { const url = `https://generativelanguage.googleapis.com/v1beta/models?key=${apiKey}`; const res = await fetch(url); - if (!res.ok) return null; + if (!res.ok) return { status: res.status, models: [] }; const data = (await res.json()) as { - models: { name: string; supportedGenerationMethods: string[] }[]; + models?: { name: string; supportedGenerationMethods?: string[] }[]; }; - const liveModels = data.models.filter( - (m) => - m.supportedGenerationMethods?.includes("bidiGenerateContent") && - !m.name.includes("native-audio"), - ); - return liveModels.length > 0 ? liveModels[0].name : null; + const models: LiveModelEntry[] = (data.models ?? []) + .filter( + (m) => + m.supportedGenerationMethods?.includes("bidiGenerateContent") && + !m.name.includes("native-audio"), + ) + .map((m) => ({ id: m.name.replace(/^models\//, "") })); + return { status: res.status, models }; +} + +/** + * Resolve the live text-capable Live model, memoized per the leg's cache key + * so all tests in this file make exactly one listing call. + */ +function getLiveCapableTextModel(apiKey: string) { + return resolveLiveModel("gemini-live-text", () => fetchLiveCapableTextModels(apiKey)); } describe.skipIf(!GOOGLE_API_KEY)("Gemini Live WS drift", () => { const config = { apiKey: GOOGLE_API_KEY! }; it("canary: text-capable bidiGenerateContent model availability", async () => { - const model = await findTextCapableLiveModel(config.apiKey); - if (model) { - // A text-capable Live model now exists! Time to enable the full drift tests. - // Update ws-providers.ts geminiLiveWS() to use this model, then un-skip below. + const resolved = await getLiveCapableTextModel(config.apiKey); + if ("infra" in resolved) { console.warn( - `[CANARY] Text-capable Gemini Live model found: ${model}. ` + - `Enable the skipped drift tests with this model.`, + `[CANARY] Gemini model listing hit infra status ${resolved.infra} — cannot determine ` + + `text-capable Live model availability this run.`, + ); + } else if ("model" in resolved) { + // A text-capable Live model now exists! The tests below discover and + // drive it automatically — no manual un-skip/file-edit needed. + console.warn( + `[CANARY] Text-capable Gemini Live model found: ${resolved.model}. ` + + `The drift tests below will now run for real against it.`, ); } - // This test always passes — it's a canary, not an assertion. - // When a model appears, the console warning signals it's time to act. + // This test always passes — it's a canary, not an assertion. When a model + // appears (or the listing goes infra-unavailable), the console warning + // signals what happened; the tests below self-heal either way. expect(true).toBe(true); }); - // These tests are skipped until a text-capable model supports bidiGenerateContent. - // When the canary above detects one, update the model in ws-providers.ts and remove .skip. + // These tests self-skip at runtime (via `ctx.skip()`) until a text-capable + // model supports bidiGenerateContent, discovered dynamically above — no + // hardcoded `.skip` and no manual file edit needed when Google adds one. + + it("WS text event sequence and shapes match", async (ctx) => { + const resolved = await getLiveCapableTextModel(config.apiKey); + if ("infra" in resolved) { + console.warn(`[gemini-live drift] listing infra status ${resolved.infra} — skipping`); + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + console.warn( + "[gemini-live drift] no text-capable bidiGenerateContent model available yet — skipping", + ); + ctx.skip(); + return; + } + const model = `models/${resolved.model}`; - it.skip("WS text event sequence and shapes match", async () => { const sdkEvents = [geminiLiveSetupCompleteShape(), ...geminiLiveTextEventShapes()]; - // Real API - const realResult = await geminiLiveWS(config, "Say hello"); + // Real API — a WS handshake-level infra status (auth/rate-limit/5xx) is an + // HONEST SKIP, never a hard-fail that would quarantine the shared drift + // baseline (mirrors ws-responses.drift.ts's handling of the same error type). + let realResult; + try { + realResult = await geminiLiveWS(config, "Say hello", undefined, model); + } catch (err) { + if (err instanceof WSHandshakeError && isInfraSkip(err.status)) { + console.warn(`[gemini-live drift] WS handshake infra status ${err.status} — skipping`); + ctx.skip(); + return; + } + throw err; + } // Mock — replicate Gemini Live protocol const mockWs = await connectWebSocket(instance.url, GEMINI_WS_PATH); - // Send setup + // Send setup — same discovered model as the real API call above, so both + // sides of the 3-way comparison are driven identically. mockWs.send( JSON.stringify({ - setup: { model: "models/gemini-2.5-flash" }, + setup: { model }, }), ); @@ -149,7 +219,22 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Live WS drift", () => { ).toEqual([]); }); - it.skip("WS tool call event sequence matches", async () => { + it("WS tool call event sequence matches", async (ctx) => { + const resolved = await getLiveCapableTextModel(config.apiKey); + if ("infra" in resolved) { + console.warn(`[gemini-live drift] listing infra status ${resolved.infra} — skipping`); + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + console.warn( + "[gemini-live drift] no text-capable bidiGenerateContent model available yet — skipping", + ); + ctx.skip(); + return; + } + const model = `models/${resolved.model}`; + const sdkEvents = [geminiLiveSetupCompleteShape(), ...geminiLiveToolCallEventShapes()]; const tools = [ @@ -168,16 +253,27 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Live WS drift", () => { }, ]; - // Real API - const realResult = await geminiLiveWS(config, "Weather in Paris", tools); + // Real API — WS handshake-level infra status → honest skip (see the + // first test above for the full rationale). + let realResult; + try { + realResult = await geminiLiveWS(config, "Weather in Paris", tools, model); + } catch (err) { + if (err instanceof WSHandshakeError && isInfraSkip(err.status)) { + console.warn(`[gemini-live drift] WS handshake infra status ${err.status} — skipping`); + ctx.skip(); + return; + } + throw err; + } // Mock — replicate Gemini Live protocol with tools const mockWs = await connectWebSocket(instance.url, GEMINI_WS_PATH); - // Send setup with tools + // Send setup with tools — same discovered model as the real API call above. mockWs.send( JSON.stringify({ - setup: { model: "models/gemini-2.5-flash", tools }, + setup: { model, tools }, }), ); @@ -226,3 +322,98 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Live WS drift", () => { ).toEqual([]); }); }); + +// --------------------------------------------------------------------------- +// Unit coverage for the self-healing resolution/skip logic (runs unconditionally +// — does NOT require GOOGLE_API_KEY, since the live describe block above is +// entirely gated out without one). Exercises the REAL functions the live leg +// calls (fetchLiveCapableTextModels + providers.ts's resolveLiveModel + +// ws-providers.ts's WSHandshakeError/isInfraSkip), stubbing only the network +// boundary (global fetch), so this is the fixture-driven red-green proof for +// an environment with no armed GOOGLE_API_KEY. +// --------------------------------------------------------------------------- + +describe("Gemini Live model resolution (unit)", () => { + const UNIT_CACHE_KEY = "gemini-live-text-unittest"; + + afterEach(() => { + __resetResolveLiveModelCache(); + vi.unstubAllGlobals(); + }); + + it("classifies a 401 listing response as an honest infra skip", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("", { status: 401 })), + ); + const resolved = await resolveLiveModel(UNIT_CACHE_KEY, () => + fetchLiveCapableTextModels("fake-key"), + ); + expect(resolved).toEqual({ infra: 401 }); + }); + + it("classifies a 200 listing with no bidi text-capable model as unavailable (today's real state)", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + JSON.stringify({ + models: [ + { + name: "models/gemini-2.5-flash-native-audio-preview", + supportedGenerationMethods: ["bidiGenerateContent"], + }, + { + name: "models/gemini-2.5-flash", + supportedGenerationMethods: ["generateContent"], + }, + ], + }), + { status: 200 }, + ), + ), + ); + const resolved = await resolveLiveModel(UNIT_CACHE_KEY, () => + fetchLiveCapableTextModels("fake-key"), + ); + expect(resolved).toEqual({ unavailable: true }); + }); + + it("discovers a text-capable bidiGenerateContent model when the provider adds one", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + JSON.stringify({ + models: [ + { + name: "models/gemini-2.5-flash-native-audio-preview", + supportedGenerationMethods: ["bidiGenerateContent"], + }, + { + name: "models/gemini-live-text-preview", + supportedGenerationMethods: ["bidiGenerateContent"], + }, + ], + }), + { status: 200 }, + ), + ), + ); + const resolved = await resolveLiveModel(UNIT_CACHE_KEY, () => + fetchLiveCapableTextModels("fake-key"), + ); + expect(resolved).toEqual({ model: "gemini-live-text-preview" }); + }); + + it("classifies WS handshake auth/rate-limit/5xx statuses as an honest skip via isInfraSkip", () => { + expect(isInfraSkip(new WSHandshakeError("unauthorized", 401).status)).toBe(true); + expect(isInfraSkip(new WSHandshakeError("rate limited", 429).status)).toBe(true); + expect(isInfraSkip(new WSHandshakeError("upstream outage", 503).status)).toBe(true); + // A non-infra handshake status (e.g. a malformed-request 400) is NOT an + // honest skip — it's a real problem the leg should still surface. + expect(isInfraSkip(new WSHandshakeError("bad request", 400).status)).toBe(false); + }); +}); diff --git a/src/__tests__/drift/ws-providers.ts b/src/__tests__/drift/ws-providers.ts index 351dc8fe..f399e7b3 100644 --- a/src/__tests__/drift/ws-providers.ts +++ b/src/__tests__/drift/ws-providers.ts @@ -43,6 +43,103 @@ export function classifyGeminiMessage(msg: Record): string { return "unknown"; } +// --------------------------------------------------------------------------- +// Self-healing WS classification (shared by the WS live-leg retrofits) +// --------------------------------------------------------------------------- +// +// Generalizes the HTTP self-healing patterns (providers.ts `resolveLiveModel` / +// `isInfraSkip` / `isModelNotFound`) to the WS surface, which has TWO distinct +// failure channels a plain HTTP probe doesn't: +// 1. Handshake-level: the upgrade request itself gets a non-101 HTTP +// response (auth/rate-limit/5xx) — surfaced as {@link WSHandshakeError} +// with the parsed status so callers can classify it with +// `isInfraSkip(status)` exactly like an HTTP leg. +// 2. Message-level: the socket upgrades fine but a request-level problem +// (e.g. a retired/invalid model id) is reported IN-BAND as a +// `{"type":"error",...}` protocol frame, since the Responses WS protocol +// validates the model per-message rather than via a connect-time query +// param (unlike Realtime's `?model=`). {@link extractWSErrorBody} surfaces +// that frame's body so callers can classify it with +// `isModelNotFound(400, body)`. + +/** + * Raised by {@link connectTLSWebSocket} when the WS upgrade handshake itself + * fails (a non-101 HTTP response). Carries the parsed numeric status (0 when + * unparseable) so callers can classify an auth/rate-limit/5xx condition as an + * honest skip via `isInfraSkip(status)` — the same classification an HTTP leg + * gets from a non-2xx response, just surfaced through a different channel. + */ +export class WSHandshakeError extends Error { + readonly status: number; + + constructor(message: string, status: number) { + super(message); + this.name = "WSHandshakeError"; + this.status = status; + } +} + +/** + * Extract the numeric HTTP status code from a WS handshake's status line + * (e.g. `"HTTP/1.1 401 Unauthorized"` -> `401`). Returns `null` when no + * 3-digit code is present (a malformed/unexpected response). + */ +export function parseHandshakeStatus(statusLine: string): number | null { + const match = statusLine.match(/^HTTP\/\d\.\d\s+(\d{3})/); + return match ? Number(match[1]) : null; +} + +/** + * True when a raw WS protocol message is a request-level error frame + * (`{"type":"error",...}`) rather than a successful completion event. + */ +export function isResponsesWSError(msg: unknown): boolean { + return !!msg && typeof msg === "object" && (msg as Record).type === "error"; +} + +/** + * Terminal predicate for the OpenAI Responses WS protocol: a successful + * completion (`response.completed` / `response.done`, both observed in the + * wild) OR a request-level error frame. Including the error frame as terminal + * means a retired/invalid model id surfaces here promptly instead of the + * `waitUntil` call timing out after 30s waiting for a completion event that + * will never arrive. + */ +export function isResponsesWSTerminal(msg: unknown): boolean { + const m = msg as Record | null; + return m?.type === "response.completed" || m?.type === "response.done" || isResponsesWSError(msg); +} + +/** + * Extract the JSON body of a terminal error frame for {@link isModelNotFound} + * classification, or `null` when the messages ended in a normal completion. + */ +export function extractWSErrorBody(messages: unknown[]): string | null { + const last = messages[messages.length - 1]; + return isResponsesWSError(last) ? JSON.stringify(last) : null; +} + +/** + * Build the `response.create` WS message body. The model is a PARAMETER, never + * hardcoded here, so callers thread a live-discovered id (via + * `resolveLiveModel` in providers.ts) instead of baking a value into this file + * that the provider can later retire out from under the drift suite. + */ +export function buildResponsesCreateMessage( + model: string, + input: object[], + tools?: object[], +): Record { + const msg: Record = { + type: "response.create", + model, + input, + max_output_tokens: 50, + }; + if (tools) msg.tools = tools; + return msg; +} + // --------------------------------------------------------------------------- // Masked frame helpers // --------------------------------------------------------------------------- @@ -150,7 +247,16 @@ export function connectTLSWebSocket( if (headerEnd === -1) return; const headerStr = buffer.subarray(0, headerEnd).toString(); if (!headerStr.includes("101")) { - reject(new Error(`WebSocket upgrade failed: ${headerStr.split("\r\n")[0]}`)); + const statusLine = headerStr.split("\r\n")[0]; + // Structured, not a bare Error: callers classify an auth/rate-limit/ + // 5xx handshake failure as an honest skip via `isInfraSkip(status)` + // (see WSHandshakeError above) instead of the whole leg hard-failing. + reject( + new WSHandshakeError( + `WebSocket upgrade failed: ${statusLine}`, + parseHandshakeStatus(statusLine) ?? 0, + ), + ); return; } handshakeDone = true; @@ -313,27 +419,23 @@ export async function openaiResponsesWS( config: ProviderConfig, input: object[], tools?: object[], + model = "gpt-4o-mini", ): Promise { const ws = await connectTLSWebSocket("api.openai.com", "/v1/responses", { Authorization: `Bearer ${config.apiKey}`, }); // Real Responses WS API uses flat format: model/input/tools at the top level - // of the response.create message (not nested inside a "response" object) - const msg: Record = { - type: "response.create", - model: "gpt-4o-mini", - input, - max_output_tokens: 50, - }; - if (tools) msg.tools = tools; + // of the response.create message (not nested inside a "response" object). + // `model` is a caller-supplied parameter (default retained for back-compat) + // so the drift leg can thread a live-discovered id via `resolveLiveModel` + // instead of this file hardcoding one the provider can later retire. + ws.send(JSON.stringify(buildResponsesCreateMessage(model, input, tools))); - ws.send(JSON.stringify(msg)); - - // Terminal event: "response.completed" or "response.done" (both observed in the wild) - const rawMessages = await ws.waitUntil( - (msg: any) => msg?.type === "response.completed" || msg?.type === "response.done", - ); + // Terminal: a completion event ("response.completed"/"response.done", both + // observed in the wild) OR a request-level error frame — see + // isResponsesWSTerminal for why the error frame must be terminal too. + const rawMessages = await ws.waitUntil(isResponsesWSTerminal); ws.close(); @@ -430,14 +532,19 @@ export async function geminiLiveWS( config: ProviderConfig, text: string, tools?: object[], + model = "models/gemini-2.5-flash", ): Promise { const path = `/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key=${config.apiKey}`; const ws = await connectTLSWebSocket("generativelanguage.googleapis.com", path); - // Step 1: Send setup + // Step 1: Send setup. `model` is a caller-supplied parameter (default + // retained for back-compat) so the drift leg can thread a live-discovered + // id via `resolveLiveModel` instead of this file hardcoding one that the + // provider can later retire out from under the drift suite (mirrors the + // `buildResponsesCreateMessage` model-as-parameter pattern above). const setup: Record = { - model: "models/gemini-2.5-flash", + model, generationConfig: { responseModalities: ["TEXT"] }, }; if (tools) setup.tools = tools; diff --git a/src/__tests__/drift/ws-responses-canary-skip.test.ts b/src/__tests__/drift/ws-responses-canary-skip.test.ts new file mode 100644 index 00000000..57e9fd3b --- /dev/null +++ b/src/__tests__/drift/ws-responses-canary-skip.test.ts @@ -0,0 +1,146 @@ +/** + * Regression + guard tests for the Responses WS live-leg self-healing retrofit + * (R5, aligning ws-responses.drift.ts to the ws-realtime discovery/GA pattern + * from #324). + * + * The old leg hardcoded `model: "gpt-4o-mini"` in both the real WS request AND + * hardcoded the SAME literal in the mock request, and its terminal predicate + * only recognized `response.completed`/`response.done` — a retired endpoint + * (handshake auth/rate/5xx) surfaced as a bare, unclassified `Error`, and a + * retired/invalid model id surfaced as an in-band `{"type":"error"}` frame the + * old predicate never terminated on (so the leg would hang until the 30s + * `waitUntil` timeout and then hard-fail/quarantine the batch instead of + * skipping honestly). + * + * These tests drive the REAL exported ws-providers.ts surface (no local + * reimplementation of the classification logic) so they exercise the actual + * failure surface the live leg runs against, without requiring a live + * OPENAI_API_KEY or a real TLS connection. + * + * RED (pre-fix): these exports (`WSHandshakeError`, `parseHandshakeStatus`, + * `isResponsesWSTerminal`, `extractWSErrorBody`, `buildResponsesCreateMessage`, + * and `openaiResponsesWS`'s `model` parameter) did not exist — this file + * fails to even type-check/import against the pre-retrofit ws-providers.ts. + * GREEN (post-fix): the exports exist and classify each condition correctly. + */ +import { describe, it, expect } from "vitest"; +import { + WSHandshakeError, + parseHandshakeStatus, + isResponsesWSTerminal, + extractWSErrorBody, + buildResponsesCreateMessage, +} from "./ws-providers.js"; +import { isInfraSkip, isModelNotFound } from "./providers.js"; + +describe("parseHandshakeStatus", () => { + it("extracts the numeric status from a standard HTTP status line", () => { + expect(parseHandshakeStatus("HTTP/1.1 401 Unauthorized")).toBe(401); + expect(parseHandshakeStatus("HTTP/1.1 429 Too Many Requests")).toBe(429); + expect(parseHandshakeStatus("HTTP/1.1 503 Service Unavailable")).toBe(503); + expect(parseHandshakeStatus("HTTP/1.1 101 Switching Protocols")).toBe(101); + }); + + it("returns null for a malformed/unexpected status line", () => { + expect(parseHandshakeStatus("garbage")).toBeNull(); + expect(parseHandshakeStatus("")).toBeNull(); + }); +}); + +describe("REGRESSION: WS handshake failure classification (#R5)", () => { + it("RED (old shape): a bare Error carries no status — cannot be classified as an honest skip", () => { + // This is exactly what connectTLSWebSocket threw BEFORE the retrofit: an + // unstructured Error with the status only embedded in prose. A caller has + // no reliable field to feed isInfraSkip, so a 401/429/503 handshake + // rejection could only be treated as a hard failure — the leg would + // quarantine on any transient provider-side condition. + const oldStyleError = new Error("WebSocket upgrade failed: HTTP/1.1 401 Unauthorized"); + expect(oldStyleError).not.toBeInstanceOf(WSHandshakeError); + expect((oldStyleError as { status?: number }).status).toBeUndefined(); + }); + + it("GREEN (new shape): WSHandshakeError carries a parsed status isInfraSkip can classify", () => { + const err = new WSHandshakeError("WebSocket upgrade failed: HTTP/1.1 401 Unauthorized", 401); + expect(err).toBeInstanceOf(WSHandshakeError); + expect(err.status).toBe(401); + expect(isInfraSkip(err.status)).toBe(true); + }); + + it("classifies the full infra-status set (401/402/403/429/5xx) as skippable", () => { + for (const status of [401, 402, 403, 429, 500, 502, 503]) { + const err = new WSHandshakeError(`WebSocket upgrade failed: HTTP/1.1 ${status}`, status); + expect(isInfraSkip(err.status), `status ${status}`).toBe(true); + } + }); + + it("GUARD: a genuine 4xx that is NOT an infra status is not swallowed as a skip", () => { + // e.g. 400 Bad Request from a malformed handshake — a real bug, not a + // provider-side outage. Must stay unclassified so it fails loud. + const err = new WSHandshakeError("WebSocket upgrade failed: HTTP/1.1 400 Bad Request", 400); + expect(isInfraSkip(err.status)).toBe(false); + }); +}); + +describe("REGRESSION: in-band model-not-found detection (#R5)", () => { + const modelNotFoundFrame = { + type: "error", + error: { type: "invalid_request_error", code: "model_not_found", message: "model retired" }, + }; + const completionFrame = { type: "response.completed", response: { id: "resp_1" } }; + + it("RED (old predicate): the old terminal predicate never recognizes an error frame", () => { + // The exact predicate the pre-retrofit code used inline — copied here ONLY + // to document the bug being fixed, not as the code under test. + const oldTerminalPredicate = (msg: unknown) => { + const m = msg as { type?: string } | null; + return m?.type === "response.completed" || m?.type === "response.done"; + }; + // An error frame never satisfies the old predicate — waitUntil would spin + // until the 30s timeout, then hard-fail with an unclassified timeout error + // instead of an honest, fast skip. + expect(oldTerminalPredicate(modelNotFoundFrame)).toBe(false); + }); + + it("GREEN (new predicate): isResponsesWSTerminal recognizes the error frame as terminal", () => { + expect(isResponsesWSTerminal(modelNotFoundFrame)).toBe(true); + expect(isResponsesWSTerminal(completionFrame)).toBe(true); + expect(isResponsesWSTerminal({ type: "response.output_text.delta" })).toBe(false); + }); + + it("extractWSErrorBody surfaces the error frame body for isModelNotFound classification", () => { + const body = extractWSErrorBody([completionFrame, modelNotFoundFrame]); + expect(body).not.toBeNull(); + expect(isModelNotFound(400, body!)).toBe(true); + }); + + it("extractWSErrorBody returns null on a normal completion — no false-positive skip", () => { + expect(extractWSErrorBody([completionFrame])).toBeNull(); + }); + + it("GUARD: a genuine invalid_request_error unrelated to the model is NOT classified as model-not-found", () => { + const unrelatedError = { + type: "error", + error: { type: "invalid_request_error", code: "invalid_json", message: "malformed input" }, + }; + const body = extractWSErrorBody([unrelatedError]); + expect(body).not.toBeNull(); + expect(isModelNotFound(400, body!)).toBe(false); + }); +}); + +describe("REGRESSION: model is threaded, never hardcoded (#R5)", () => { + it("buildResponsesCreateMessage reflects the caller-supplied (live-discovered) model", () => { + const discovered = buildResponsesCreateMessage("gpt-4o-mini-2024-11-20", [ + { role: "user", content: "hi" }, + ]); + expect(discovered.model).toBe("gpt-4o-mini-2024-11-20"); + expect(discovered.type).toBe("response.create"); + }); + + it("includes tools only when provided", () => { + const withTools = buildResponsesCreateMessage("gpt-4o-mini", [], [{ type: "function" }]); + expect(withTools.tools).toEqual([{ type: "function" }]); + const withoutTools = buildResponsesCreateMessage("gpt-4o-mini", []); + expect(withoutTools.tools).toBeUndefined(); + }); +}); diff --git a/src/__tests__/drift/ws-responses.drift.ts b/src/__tests__/drift/ws-responses.drift.ts index 4dc99232..c2ebe16e 100644 --- a/src/__tests__/drift/ws-responses.drift.ts +++ b/src/__tests__/drift/ws-responses.drift.ts @@ -12,7 +12,8 @@ import { openaiResponsesTextEventShapes, openaiResponsesToolCallEventShapes, } from "./sdk-shapes.js"; -import { openaiResponsesWS } from "./ws-providers.js"; +import { openaiResponsesWS, WSHandshakeError, extractWSErrorBody } from "./ws-providers.js"; +import { resolveLiveModel, isInfraSkip, isModelNotFound, listOpenAIModels } from "./providers.js"; import { startDriftServer, stopDriftServer, collectMockWSMessages } from "./helpers.js"; import { connectWebSocket } from "../ws-test-client.js"; @@ -31,6 +32,28 @@ afterAll(async () => { await stopDriftServer(instance); }); +// --------------------------------------------------------------------------- +// Live model discovery +// --------------------------------------------------------------------------- + +/** + * Resolve a live, non-deprecated chat model for the Responses WS probe via + * `GET /v1/models` (the same discovery `resolveLiveModel` generalizes from the + * Cohere #325 pattern), preferring `gpt-4o-mini` but falling back to whatever + * the account's listing exposes. Memoized per-key so both tests below make a + * single listing call. + */ +function getResponsesWSModel() { + return resolveLiveModel( + "openai-responses-ws", + async () => { + const ids = await listOpenAIModels(OPENAI_API_KEY!); + return { status: 200, models: ids.map((id) => ({ id })) }; + }, + ["gpt-4o-mini"], + ); +} + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -38,18 +61,52 @@ afterAll(async () => { describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses WS drift", () => { const config = { apiKey: OPENAI_API_KEY! }; - it("WS text event sequence and shapes match", async () => { + it("WS text event sequence and shapes match", async (ctx) => { + const resolved = await getResponsesWSModel(); + if ("infra" in resolved) { + // Provider-side auth/credit/rate-limit/5xx on the listing — honest skip. + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error("OpenAI /v1/models exposed no usable chat model for the Responses WS probe"); + } + const model = resolved.model; + const sdkEvents = openaiResponsesTextEventShapes(); - // Real API via WS - const realResult = await openaiResponsesWS(config, [{ role: "user", content: "Say hello" }]); + // Real API via WS — a retired endpoint (handshake auth/rate/5xx) or a + // stale/retired model id (in-band error frame) is an HONEST SKIP, never a + // hard-fail that would quarantine the shared drift baseline. + let realResult; + try { + realResult = await openaiResponsesWS( + config, + [{ role: "user", content: "Say hello" }], + undefined, + model, + ); + } catch (err) { + if (err instanceof WSHandshakeError && isInfraSkip(err.status)) { + console.warn(`[ws-responses drift] WS handshake infra status ${err.status} — skipping`); + ctx.skip(); + return; + } + throw err; + } + const errBody = extractWSErrorBody(realResult.rawMessages); + if (errBody && isModelNotFound(400, errBody)) { + console.warn(`[ws-responses drift] model-not-found: ${errBody} — skipping`); + ctx.skip(); + return; + } // Mock via WS — uses flat format matching real API const mockWs = await connectWebSocket(instance.url, "/v1/responses"); mockWs.send( JSON.stringify({ type: "response.create", - model: "gpt-4o-mini", + model, input: [{ role: "user", content: "Say hello" }], }), ); @@ -62,6 +119,8 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses WS drift", () => { expect(realResult.rawMessages.length, "Real API returned no WS messages").toBeGreaterThan(0); expect(mockResult.events.length, "Mock returned no WS messages").toBeGreaterThan(0); + // Grade envelope SHAPE, never status/connection codes — the honest-skip + // branches above already handled the non-shape failure modes. const diffs = compareSSESequences(sdkEvents, realResult.events, mockResult.events); const report = formatDriftReport( "OpenAI Responses WS (text events)", @@ -75,7 +134,17 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses WS drift", () => { ).toEqual([]); }); - it("WS tool call event sequence matches", async () => { + it("WS tool call event sequence matches", async (ctx) => { + const resolved = await getResponsesWSModel(); + if ("infra" in resolved) { + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error("OpenAI /v1/models exposed no usable chat model for the Responses WS probe"); + } + const model = resolved.model; + const sdkEvents = [ ...openaiResponsesTextEventShapes().filter( (e) => e.type === "response.created" || e.type === "response.completed", @@ -97,18 +166,35 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses WS drift", () => { ]; // Real API via WS - const realResult = await openaiResponsesWS( - config, - [{ role: "user", content: "Weather in Paris" }], - tools, - ); + let realResult; + try { + realResult = await openaiResponsesWS( + config, + [{ role: "user", content: "Weather in Paris" }], + tools, + model, + ); + } catch (err) { + if (err instanceof WSHandshakeError && isInfraSkip(err.status)) { + console.warn(`[ws-responses drift] WS handshake infra status ${err.status} — skipping`); + ctx.skip(); + return; + } + throw err; + } + const errBody = extractWSErrorBody(realResult.rawMessages); + if (errBody && isModelNotFound(400, errBody)) { + console.warn(`[ws-responses drift] model-not-found: ${errBody} — skipping`); + ctx.skip(); + return; + } // Mock via WS — uses flat format matching real API const mockWs = await connectWebSocket(instance.url, "/v1/responses"); mockWs.send( JSON.stringify({ type: "response.create", - model: "gpt-4o-mini", + model, input: [{ role: "user", content: "Weather in Paris" }], tools, }), diff --git a/src/__tests__/fix-drift-invoke.test.ts b/src/__tests__/fix-drift-invoke.test.ts deleted file mode 100644 index e000c2ce..00000000 --- a/src/__tests__/fix-drift-invoke.test.ts +++ /dev/null @@ -1,77 +0,0 @@ -/** - * WS-4 residual lock (slot2-F1) — a SYNCHRONOUS throw during the Promise - * executor setup of `invokeClaudeCode` must NOT strand the already-armed 30-min - * `killTimer`. - * - * The executor arms `killTimer` (a 30-minute setTimeout that would later - * group-kill `child.pid`) BEFORE it wires the stdout/stderr handlers. If - * `child.stdout` is null (a spawn edge case), `child.stdout.on(...)` throws a - * TypeError synchronously inside the executor. Without the fix, that throw - * rejects the Promise but leaves `killTimer` LIVE — a leaked 30-min timer that - * could later SIGKILL a reused PID. The fix clears the timer before rejecting. - * - * We mock `spawn` to return a fake child with NULL streams, drive - * `invokeClaudeCode`, and assert (a) the Promise REJECTS (not hangs) and (b) no - * timer is left pending (fake timers report zero pending after the reject). - */ -import { EventEmitter } from "node:events"; - -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; - -vi.mock("node:child_process", async () => { - const actual = await vi.importActual("node:child_process"); - return { ...actual, spawn: vi.fn() }; -}); - -import { spawn } from "node:child_process"; - -import { invokeClaudeCode } from "../../scripts/fix-drift.js"; - -const mockedSpawn = vi.mocked(spawn); - -/** A fake detached child whose stdout/stderr are NULL (the edge case). */ -function makeNullStreamChild(pid = 4242): EventEmitter & { - pid: number; - stdout: null; - stderr: null; -} { - const child = new EventEmitter() as EventEmitter & { - pid: number; - stdout: null; - stderr: null; - }; - child.pid = pid; - child.stdout = null; - child.stderr = null; - return child; -} - -describe("invokeClaudeCode — executor-setup throw must not strand killTimer (WS-4/slot2-F1)", () => { - beforeEach(() => { - vi.clearAllMocks(); - vi.useFakeTimers(); - vi.spyOn(console, "error").mockImplementation(() => {}); - }); - - afterEach(() => { - vi.clearAllTimers(); - vi.useRealTimers(); - vi.restoreAllMocks(); - }); - - it("REJECTS when the child has null streams (no stdout/stderr to attach handlers to)", async () => { - mockedSpawn.mockReturnValue(makeNullStreamChild() as never); - await expect(invokeClaudeCode("prompt")).rejects.toThrow(/no stdout\/stderr pipe/i); - }); - - it("clears the 30-min killTimer on the setup-throw path — NO timer is left pending", async () => { - mockedSpawn.mockReturnValue(makeNullStreamChild() as never); - - // The executor arms killTimer, then throws attaching the null-stream - // handler. The fix must clearTimeout(killTimer) before rejecting, so after - // the reject settles there must be ZERO pending fake timers. Without the - // fix, the 30-min killTimer would still be pending here. - await expect(invokeClaudeCode("prompt")).rejects.toThrow(); - expect(vi.getTimerCount()).toBe(0); - }); -}); diff --git a/src/__tests__/fix-drift-kill.test.ts b/src/__tests__/fix-drift-kill.test.ts deleted file mode 100644 index 0f0e62d8..00000000 --- a/src/__tests__/fix-drift-kill.test.ts +++ /dev/null @@ -1,235 +0,0 @@ -/** - * WS-4 regression locks — real process-group subprocess control. - * - * The original `invokeClaudeCode` had two live defects: - * 1. `spawn("npx", …)` had NO `detached: true`, so signalling the child pid - * reached only the `npx` wrapper, never the `@anthropic-ai/claude-code` - * grandchild — a wedged fixer survived and burned the 30-min job budget. - * 2. The SIGKILL escalation was gated on `if (!child.killed)`, but Node sets - * `child.killed = true` the instant SIGTERM is DELIVERED (not when the - * process exits), so `!child.killed` was ~always false and SIGKILL NEVER - * fired against a process that ignored SIGTERM. - * - * These tests exercise a REAL controlled subprocess that traps SIGTERM and - * sleeps, proving the OLD logic leaves it alive and the NEW logic - * (`killProcessGroup` + `scheduleEscalatingKill`, gated on a real has-exited - * flag) kills it and its whole group within the grace window. - */ -import { spawn } from "node:child_process"; - -import { describe, it, expect, vi } from "vitest"; - -/** Real subprocess spin-up + grace windows need more than the default budget. */ -const SUBPROC_TIMEOUT = 15000; - -import { killProcessGroup, scheduleEscalatingKill } from "../../scripts/fix-drift.js"; - -/** A child that TRAPS SIGTERM and keeps sleeping — models a wedged fixer. */ -const WEDGED_CHILD = ` -process.on("SIGTERM", () => { /* ignore — wedged, refuse to die on SIGTERM */ }); -setTimeout(() => process.exit(0), 60000); -`; - -/** A child that exits cleanly on SIGTERM — models a well-behaved fixer. */ -const OBEDIENT_CHILD = ` -process.on("SIGTERM", () => process.exit(0)); -setTimeout(() => process.exit(0), 60000); -`; - -function isAlive(pid: number): boolean { - try { - process.kill(pid, 0); - return true; - } catch { - return false; - } -} - -function sleep(ms: number): Promise { - return new Promise((r) => setTimeout(r, ms)); -} - -describe("killProcessGroup", () => { - it( - "delivers a signal to the whole GROUP of a detached child (kills a SIGTERM-trapping grandchild-style process)", - { timeout: SUBPROC_TIMEOUT }, - async () => { - const child = spawn("node", ["-e", WEDGED_CHILD], { stdio: "ignore", detached: true }); - const pid = child.pid!; - await sleep(300); - expect(isAlive(pid)).toBe(true); - - // SIGTERM to the group is IGNORED by the wedged child (it traps it). - expect(killProcessGroup(pid, "SIGTERM")).toBe(true); - await sleep(200); - expect(isAlive(pid)).toBe(true); // still alive — it trapped SIGTERM - - // SIGKILL to the GROUP cannot be trapped — it dies. - expect(killProcessGroup(pid, "SIGKILL")).toBe(true); - await sleep(400); - expect(isAlive(pid)).toBe(false); - }, - ); - - it("tolerates ESRCH (group already gone) and returns false, never throwing", () => { - // A pid that is essentially certain not to exist as a group leader. - const missing = 2 ** 30; - expect(() => killProcessGroup(missing, "SIGTERM")).not.toThrow(); - expect(killProcessGroup(missing, "SIGTERM")).toBe(false); - }); - - it("does NOT silently treat EPERM as success — logs a visible warning and attempts a single-PID fallback (slot2-F5)", () => { - // EPERM means the group EXISTS but is unkillable by us (re-credentialed / - // re-parented child) — it may still be ALIVE burning the budget. It must - // NOT be swallowed as a benign "nothing to kill" like ESRCH. Assert we (a) - // log a distinct WARNING and (b) attempt the single-PID fallback. - const errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - let call = 0; - const killSpy = vi.spyOn(process, "kill").mockImplementation(((pidArg: number) => { - call += 1; - if (call === 1) { - // First call: group signal (negative pid) → EPERM. - expect(pidArg).toBeLessThan(0); - const e = new Error("EPERM") as NodeJS.ErrnoException; - e.code = "EPERM"; - throw e; - } - // Second call: the single-PID fallback (positive pid) succeeds. - expect(pidArg).toBeGreaterThan(0); - return true; - }) as never); - try { - expect(killProcessGroup(12345, "SIGKILL")).toBe(true); // fallback delivered - expect(call).toBe(2); // group attempt + single-PID fallback - const warned = errSpy.mock.calls.some((c) => String(c[0]).includes("EPERM")); - expect(warned).toBe(true); // distinct, visible EPERM warning — not silent - } finally { - killSpy.mockRestore(); - errSpy.mockRestore(); - } - }); - - it("re-throws unexpected errors (e.g. EINVAL from a bad signal)", () => { - const child = spawn("node", ["-e", OBEDIENT_CHILD], { stdio: "ignore", detached: true }); - const pid = child.pid!; - try { - // An invalid signal name yields a non-ESRCH/EPERM error, which must - // propagate rather than be swallowed as "nothing to kill". - expect(() => killProcessGroup(pid, "SIGNOTAREALSIGNAL" as NodeJS.Signals)).toThrow(); - } finally { - try { - process.kill(-pid, "SIGKILL"); - } catch { - /* best effort */ - } - } - }); -}); - -describe("scheduleEscalatingKill — SIGKILL escalation gated on a REAL exit flag", () => { - it( - "SIGKILLs a wedged (SIGTERM-trapping) subprocess group within the grace window (GREEN)", - { timeout: SUBPROC_TIMEOUT }, - async () => { - const child = spawn("node", ["-e", WEDGED_CHILD], { stdio: "ignore", detached: true }); - const pid = child.pid!; - let exited = false; - child.on("close", () => { - exited = true; - }); - await sleep(300); - expect(isAlive(pid)).toBe(true); - - // Short grace so the test is fast. hasExited() is backed by the real - // `close` event, NOT child.killed. - const timer = scheduleEscalatingKill(pid, () => exited, 150); - // Before the grace elapses, SIGTERM has been delivered but the wedged child - // is still alive (it trapped SIGTERM). - await sleep(80); - expect(isAlive(pid)).toBe(true); - // After the grace, the escalation SIGKILLs the group. - await sleep(400); - expect(isAlive(pid)).toBe(false); - clearTimeout(timer); - }, - ); - - it( - "does NOT SIGKILL when hasExited() is true — the gate genuinely SKIPS escalation (a SIGTERM-trapping child that WOULD have died to a stray SIGKILL stays alive)", - { timeout: SUBPROC_TIMEOUT }, - async () => { - // The previous version of this test used an OBEDIENT child and asserted - // it was dead — but an obedient child dies from the unconditional SIGTERM - // that scheduleEscalatingKill sends FIRST, regardless of whether the - // SIGKILL escalation is skipped, so it passed for the WRONG reason (it - // could not distinguish "escalation skipped" from "escalation fired"). - // - // Use a WEDGED child that TRAPS SIGTERM instead: the first SIGTERM leaves - // it alive, so the ONLY thing that could kill it within the window is the - // SIGKILL escalation. With hasExited() forced true, that escalation MUST - // be skipped — so the child must remain ALIVE after the grace elapses. If - // the has-exited gate regressed to always-escalate (the WS-4 defect), the - // group SIGKILL would fire and the child would be DEAD — turning this RED. - const child = spawn("node", ["-e", WEDGED_CHILD], { stdio: "ignore", detached: true }); - const pid = child.pid!; - await sleep(300); - expect(isAlive(pid)).toBe(true); - - const timer = scheduleEscalatingKill(pid, () => true, 100); - // Wait well past the grace window: the escalation callback has run (and, - // gated on hasExited()===true, SKIPPED the SIGKILL). - await sleep(400); - // Wedged child trapped the SIGTERM and no SIGKILL was sent → STILL ALIVE. - expect(isAlive(pid)).toBe(true); - - // The grace timer must have already fired-and-skipped (not still pending): - // clearing it now is a no-op, and no LATE SIGKILL can arrive after this. - clearTimeout(timer); - await sleep(200); - expect(isAlive(pid)).toBe(true); // no late kill - - // Clean up the still-alive wedged group. - try { - process.kill(-pid, "SIGKILL"); - } catch { - /* best effort */ - } - await sleep(300); - expect(isAlive(pid)).toBe(false); - }, - ); - - it( - "the returned grace timer, once cleared before it fires, delivers NO late SIGKILL to a still-running group", - { timeout: SUBPROC_TIMEOUT }, - async () => { - // Locks the caller-side lifecycle used by invokeClaudeCode's `close` - // handler: on a clean early exit the caller clears the returned timer, and - // a pending SIGKILL escalation must NOT fire late against a (possibly - // reused) PID. Here the child is still running and hasExited() would be - // false, so ONLY a fired escalation could kill it — we clear the timer - // BEFORE the grace elapses and assert the child survives. - const child = spawn("node", ["-e", WEDGED_CHILD], { stdio: "ignore", detached: true }); - const pid = child.pid!; - await sleep(300); - expect(isAlive(pid)).toBe(true); - - // Long grace so we can cancel before it fires. hasExited stays false. - const timer = scheduleEscalatingKill(pid, () => false, 5000); - // The initial SIGTERM is trapped; child alive. Cancel before the grace. - await sleep(100); - clearTimeout(timer); - // Well past what the grace would have been — no SIGKILL arrives. - await sleep(300); - expect(isAlive(pid)).toBe(true); - - try { - process.kill(-pid, "SIGKILL"); - } catch { - /* best effort */ - } - await sleep(300); - expect(isAlive(pid)).toBe(false); - }, - ); -}); diff --git a/src/__tests__/fix-drift-straggler.test.ts b/src/__tests__/fix-drift-straggler.test.ts deleted file mode 100644 index 4bd78a49..00000000 --- a/src/__tests__/fix-drift-straggler.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -/** - * FIX #F5 (round-4) — createPr's STRAGGLER fail-closed guard, in isolation. - * - * On a RESOLVED verdict every changed file MUST fall into a gated commit group - * (production builder, report-named fixture target, or skills/). A "straggler" - * (a changed file in none of those groups) means the predicate verdict and the - * staging partition have DIVERGED — staging would silently drop it and ship an - * incomplete fix behind a green verdict. createPr must exit UNSANCTIONED_CHANGE - * and stage NOTHING. - * - * By construction with the CURRENT predicate + gatedCommitFiles there is no file - * that both PASSES the predicate and becomes a straggler (the allowlist blocks - * everything gatedCommitFiles would not classify). This guard is therefore - * defense-in-depth against a FUTURE predicate change. To lock it load-bearingly - * we mock `evaluateDriftResolved` to force a RESOLVED verdict while git reports a - * straggler in the working tree, then assert createPr fail-closes before any - * `git add`. This is a pure-function/mock test — it never runs the autofix - * subprocess. - */ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; - -import type { DriftReport } from "../../scripts/drift-types.js"; - -// Mock git so gitChangedFiles()/exec() are controllable, and force the predicate -// to RESOLVED so the straggler guard (which runs AFTER the verdict) is reached. -vi.mock("node:fs", async () => { - const actual = await vi.importActual("node:fs"); - return { ...actual, readFileSync: vi.fn(actual.readFileSync), writeFileSync: vi.fn() }; -}); - -vi.mock("node:child_process", async () => { - const actual = await vi.importActual("node:child_process"); - return { ...actual, execFileSync: vi.fn(), execSync: vi.fn() }; -}); - -vi.mock("../../scripts/drift-success-predicate.js", async () => { - const actual = await vi.importActual( - "../../scripts/drift-success-predicate.js", - ); - return { - ...actual, - // Force RESOLVED regardless of inputs so the guard downstream is exercised. - evaluateDriftResolved: vi.fn(() => ({ - resolved: true, - reason: actual.PredicateReason.RESOLVED, - detail: "forced resolved for the straggler-guard test", - offendingFiles: [], - })), - // gitChangedFiles is used by createPr — return a production file PLUS a - // straggler (a root file gatedCommitFiles cannot classify). - gitChangedFiles: vi.fn(() => ["src/helpers.ts", "weird-root-file.txt"]), - }; -}); - -import { execSync, execFileSync } from "node:child_process"; - -import { createPr } from "../../scripts/fix-drift.js"; - -const mockedExecSync = vi.mocked(execSync); -const mockedExecFileSync = vi.mocked(execFileSync); - -describe("createPr straggler guard fail-closed (fix #F5, isolated)", () => { - let logSpy: ReturnType; - let errSpy: ReturnType; - let exitSpy: ReturnType; - - beforeEach(() => { - vi.clearAllMocks(); - logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { - throw new Error(`__exit__${code}`); - }) as never); - // Default git calls return empty (branch lookups etc.). - mockedExecSync.mockReturnValue("fix/drift-2026-07-16\n" as unknown as string); - }); - - afterEach(() => { - logSpy.mockRestore(); - errSpy.mockRestore(); - exitSpy.mockRestore(); - }); - - const rep: DriftReport = { - timestamp: "2026-07-16T00:00:00.000Z", - entries: [ - { - provider: "OpenAI", - scenario: "chat completion", - builderFile: "src/helpers.ts", - builderFunctions: ["buildChatCompletion"], - typesFile: null, - sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", - diffs: [ - { - path: "x", - severity: "critical", - issue: "missing", - expected: "string", - real: "string", - mock: "", - }, - ], - }, - ], - }; - - it("exits UNSANCTIONED_CHANGE (17) and NEVER stages the straggler even on a RESOLVED verdict", () => { - expect(() => createPr(rep, { report: { timestamp: "t", entries: [] }, exitCode: 0 })).toThrow( - /__exit__17/, - ); - - const staged = mockedExecFileSync.mock.calls.some( - (c) => - c[0] === "git" && Array.isArray(c[1]) && (c[1] as string[]).includes("weird-root-file.txt"), - ); - expect(staged).toBe(false); - - const lines = logSpy.mock.calls.map((c) => String(c[0])); - expect(lines).toContain("reason=unsanctioned-change"); - }); -}); diff --git a/src/__tests__/fix-drift-workflow.test.ts b/src/__tests__/fix-drift-workflow.test.ts index e48a5263..2ede654e 100644 --- a/src/__tests__/fix-drift-workflow.test.ts +++ b/src/__tests__/fix-drift-workflow.test.ts @@ -1,20 +1,28 @@ /** * Static (text-level) assertions on .github/workflows/fix-drift.yml. * - * These pin the LOAD-BEARING wiring that the drift-success predicate/guard now - * REQUIRE (CR round-3): + * C3 (delete-freewriter-predicate-rewire): this workflow used to invoke an + * autonomous coding-agent subprocess to freewrite a fix for whatever drift the + * collector found, then gate the resulting diff behind a 916-line anti-cheat + * verdict function (`scripts/drift-success-predicate.ts`) before opening a PR. + * Both have been DELETED entirely. This suite (retargeted from the deleted + * predicate-era assertions) pins the NEW, load-bearing wiring instead: * - * F1 — the workflow must (a) re-collect drift AUTHORITATIVELY after the autofix - * to a distinct post-fix path, capturing its exit code, and (b) pass BOTH - * --post-fix-report and --post-fix-exit into the `--create-pr` invocation. - * Without these, the mandatory-post-fix guard fails closed and NO PR is - * ever opened (the gate would be inert). - * - * F-A — the PRE-fix report the allowlist's sanctioned-target set is derived - * from must be PINNED outside the LLM-writable repo checkout BEFORE the - * autofix runs, and both the Assert and Create-PR steps must read - * --report from that pinned copy — NEVER the in-repo drift-report.json - * the autofix LLM could overwrite. + * - the workflow triggers on workflow_dispatch, a SCHEDULED cron (the + * deprecation detector fires independently of drift-test failure — a + * vanished model family does not, by itself, red the Drift Tests + * workflow), and workflow_run(Drift Tests, failure). + * - the "Auto-fix drift" step is replaced by `scripts/drift-sync.ts` (the + * deterministic, zero-LLM model-family sync core). + * - the "Assert drift truly resolved" step is replaced by + * `scripts/drift-sync-check.ts` (the trivial allowlist + pin + re-collect + * gate). + * - the PR-open path is gated on `reason == 'ok-applied'`, never on a + * verdict function. + * - the NO-AUTO-MERGE human-approval backstop is preserved verbatim (Phase + * 0/1 — auto-merge is an explicit, opt-in Phase-4 exception, out of scope + * here). + * - there is NO remaining reference to the deleted predicate/LLM machinery. * * No YAML dependency is added; the repo ships none. These are deliberately * text-shape assertions on the committed workflow — an actionlint run in CI @@ -31,112 +39,120 @@ const wf = readFileSync(WORKFLOW_PATH, "utf-8"); /** Collapse runs of whitespace so multi-line YAML `run:` blocks match linearly. */ const wfFlat = wf.replace(/\s+/g, " "); -describe("fix-drift.yml — F1: post-fix re-collect + args wired into --create-pr", () => { - it("has an authoritative post-fix re-collect step writing a DISTINCT report path OUTSIDE the repo (FIX #F3)", () => { - expect(wf).toContain("Re-collect drift (authoritative)"); - // FIX #F3 — the re-collect writes to $RUNNER_TEMP (via the POST_FIX_REPORT - // env), NOT the repo cwd, so it is never scored by the predicate's git scan. - expect(wf).toContain('npx tsx scripts/drift-report-collector.ts --out "$POST_FIX_REPORT"'); - expect(wf).toContain("POST_FIX_REPORT: ${{ runner.temp }}/drift-report.post-fix.json"); +describe("fix-drift.yml — the LLM freewriter + anti-cheat predicate are GONE", () => { + it("never references the deleted invokeClaudeCode / Claude Code CLI spawn", () => { + expect(wf).not.toMatch(/invokeClaudeCode/i); + expect(wf).not.toMatch(/@anthropic-ai\/claude-code/); + expect(wf).not.toContain("Claude Code"); }); - it("captures the post-fix collector exit code as a step output", () => { - expect(wfFlat).toContain('POST_FIX_EXIT=$? set -e echo "post_fix_exit=$POST_FIX_EXIT"'); + it("never references the deleted drift-success-predicate.ts", () => { + expect(wf).not.toContain("drift-success-predicate"); }); - it("passes BOTH --post-fix-report and --post-fix-exit into `fix-drift.ts --create-pr`", () => { - expect(wfFlat).toContain("npx tsx scripts/fix-drift.ts --create-pr"); - expect(wfFlat).toMatch( - /fix-drift\.ts --create-pr[^]*?--post-fix-report "\$\{POST_FIX_REPORT\}"[^]*?--post-fix-exit "\$\{POST_FIX_EXIT\}"/, - ); + it("never invokes the deleted scripts/fix-drift.ts", () => { + expect(wf).not.toMatch(/scripts\/fix-drift\.ts/); }); - it("the Assert step runs the predicate with post-fix args (the happy-path gate)", () => { - expect(wf).toContain("Assert drift truly resolved"); - expect(wfFlat).toMatch( - /drift-success-predicate\.ts[^]*?--post-fix-report "\$\{POST_FIX_REPORT\}"[^]*?--post-fix-exit "\$\{POST_FIX_EXIT\}"/, - ); + it("has no step named 'Auto-fix drift' or 'Assert drift truly resolved' (the old predicate-era step names)", () => { + expect(wf).not.toContain("name: Auto-fix drift"); + expect(wf).not.toContain("name: Assert drift truly resolved"); }); -}); -describe("fix-drift.yml — F-A: PRE-fix report pinned outside the LLM-writable checkout", () => { - it("has a pin step that copies the pre-fix report into runner.temp before autofix", () => { - expect(wf).toContain("Pin pre-fix drift report (integrity)"); - // FIX #F3 — the pre-fix report is itself collected into $RUNNER_TEMP - // (PRE_FIX_REPORT), so the pin copies from that out-of-repo path, never the - // repo cwd. Both source and destination are outside the LLM-writable checkout. - expect(wf).toContain('cp "$PRE_FIX_REPORT" "$PINNED_REPORT"'); - expect(wf).toContain("PINNED_REPORT: ${{ runner.temp }}/drift-report.pinned.json"); + it("carries no now-unused agent/predicate-only secrets or env beyond the legitimate provider keys", () => { + // ANTHROPIC_API_KEY legitimately remains — drift-sync.ts uses it to list + // live Anthropic models, and the collector's Anthropic drift leg uses it + // too. Neither is the deleted agent invocation. + expect(wf).toContain("ANTHROPIC_API_KEY"); + expect(wf).not.toMatch(/claude-code-output/); }); +}); - it("the pin step runs BEFORE the Auto-fix step (so the LLM cannot pre-tamper the pin)", () => { - const pinIdx = wf.indexOf("Pin pre-fix drift report"); - const autofixIdx = wf.indexOf("name: Auto-fix drift"); - expect(pinIdx).toBeGreaterThan(-1); - expect(autofixIdx).toBeGreaterThan(-1); - expect(pinIdx).toBeLessThan(autofixIdx); +describe("fix-drift.yml — triggers on workflow_dispatch, a SCHEDULED cron, and drift-test failure", () => { + it("triggers on workflow_dispatch", () => { + expect(wf).toMatch(/on:\s*\n\s*workflow_dispatch:/); }); - it("the Assert step reads --report from the PINNED copy, not the in-repo file", () => { - // The YAML line-continuation `\` survives whitespace-flattening, so match - // tolerantly across it. - expect(wfFlat).toMatch(/drift-success-predicate\.ts \\? *--report "\$\{PINNED_REPORT\}"/); + it("has a schedule/cron trigger, independent of the drift-failure gate", () => { + expect(wf).toMatch(/schedule:\s*\n\s*-\s*cron:/); }); - it("the Create PR step reads --report from the PINNED copy, not the in-repo file", () => { - expect(wfFlat).toMatch( - /scripts\/fix-drift\.ts --create-pr \\? *--report "\$\{PINNED_REPORT\}"/, - ); + it("still triggers on workflow_run of the Drift Tests workflow completing", () => { + expect(wf).toContain('workflows: ["Drift Tests"]'); + expect(wf).toContain("types: [completed]"); }); - it("neither the Assert nor Create-PR predicate invocation reads --report drift-report.json (the LLM-writable file)", () => { - // The in-repo drift-report.json is still uploaded as an artifact + copied by - // the pin step, but must NEVER be the --report source for the gate. - expect(wfFlat).not.toMatch(/drift-success-predicate\.ts \\? *--report drift-report\.json/); - expect(wfFlat).not.toMatch(/fix-drift\.ts --create-pr \\? *--report drift-report\.json/); + it("the job runs on workflow_dispatch, schedule, OR a failed Drift Tests run", () => { + const idx = wf.indexOf("if: >-"); + expect(idx).toBeGreaterThan(-1); + const block = wf.slice(idx, wf.indexOf("runs-on:", idx)); + expect(block).toContain("github.event_name == 'workflow_dispatch'"); + expect(block).toContain("github.event_name == 'schedule'"); + expect(block).toContain("github.event.workflow_run.conclusion == 'failure'"); }); }); -// --------------------------------------------------------------------------- -// report-path — the Auto-fix step must pass --report from the PINNED copy. -// -// FIX #F3 moved the collector's report to $RUNNER_TEMP (outside the repo -// checkout), so the repo-root drift-report.json that fix-drift.ts defaults to no -// longer exists. The Assert and Create-PR steps were updated to read --report -// from the pinned copy, but the Auto-fix step still ran `fix-drift.ts` with NO -// --report, so it fell back to the missing repo-root path and died with "Drift -// report not found" (readDriftReport) — the recurring drift-job failure. These -// lock that the Auto-fix step reads --report from the pinned copy, matching the -// downstream integrity gate. -// --------------------------------------------------------------------------- -describe("fix-drift.yml — report-path: Auto-fix step reads --report from the pinned copy", () => { - it("passes --report from the PINNED copy into the Auto-fix `fix-drift.ts` invocation", () => { - expect(wfFlat).toMatch(/scripts\/fix-drift\.ts --report "\$\{PINNED_REPORT\}"/); +describe("fix-drift.yml — deterministic sync + sync-check replace the fixer + predicate", () => { + it("runs scripts/drift-sync.ts as the remediation step", () => { + expect(wfFlat).toContain("npx tsx scripts/drift-sync.ts"); }); - it("does NOT run the Auto-fix step against the default (missing) repo-root drift-report.json", () => { - // The bare `npx tsx scripts/fix-drift.ts` (no --report) is the regression: - // it defaults to the repo-root drift-report.json which FIX #F3 no longer - // writes. Ensure the autofix invocation always carries a --report flag. - expect(wfFlat).not.toMatch(/npx tsx scripts\/fix-drift\.ts(?! --)/); + it("captures drift-sync's reason= output as a step output", () => { + expect(wfFlat).toContain("id: sync"); + expect(wfFlat).toMatch(/grep '\^reason=' "\$\{SYNC_LOG\}"/); + expect(wfFlat).toContain('echo "reason=${REASON}" >> "$GITHUB_OUTPUT"'); }); - it("exposes PINNED_REPORT as an env in the Auto-fix step", () => { - const idx = wf.indexOf("name: Auto-fix drift"); + it("runs scripts/drift-sync-check.ts as a defense-in-depth re-assertion, gated on reason == 'ok-applied'", () => { + expect(wf).toContain("name: Assert drift-sync-check (defense-in-depth)"); + expect(wf).toContain("if: steps.sync.outputs.reason == 'ok-applied'"); + expect(wfFlat).toContain("npx tsx scripts/drift-sync-check.ts"); + }); + + it("the PR-open step is gated on reason == 'ok-applied', not on a verdict function", () => { + const idx = wf.indexOf("name: Push branch + create PR"); expect(idx).toBeGreaterThan(-1); const nextStep = wf.indexOf("\n - name:", idx + 1); const stepBlock = wf.slice(idx, nextStep === -1 ? undefined : nextStep); - expect(stepBlock).toContain("PINNED_REPORT: ${{ runner.temp }}/drift-report.pinned.json"); + expect(stepBlock).toContain("if: steps.sync.outputs.reason == 'ok-applied' && success()"); + expect(stepBlock).toContain("gh pr create"); + }); +}); + +describe("fix-drift.yml — needs-human vs gate-failure are DISTINCT alerts", () => { + it("alerts distinctly when the sync routes to a human decision (new family / still-referenced deprecation)", () => { + expect(wf).toContain("name: Alert on needs-human decision"); + expect(wf).toContain("steps.sync.outputs.reason == 'needs-human'"); + }); + + it("alerts distinctly (and separately) when drift-sync-check refuses the gate — a tooling fault, not a product decision", () => { + expect(wf).toContain("name: Alert on drift-sync-check gate failure"); + expect(wf).toContain("steps.sync.outputs.reason == 'gate-failed'"); + }); + + it("both needs-human and gate-failure alerts fail the job (non-green), so a human sees it in CI status too", () => { + const needsHumanIdx = wf.indexOf("name: Alert on needs-human decision"); + const needsHumanBlock = wf.slice( + needsHumanIdx, + wf.indexOf("\n - name:", needsHumanIdx + 1), + ); + expect(needsHumanBlock).toMatch(/\n\s*exit 1\b/); + + const gateFailedIdx = wf.indexOf("name: Alert on drift-sync-check gate failure"); + const gateFailedBlock = wf.slice( + gateFailedIdx, + wf.indexOf("\n - name:", gateFailedIdx + 1), + ); + expect(gateFailedBlock).toMatch(/\n\s*exit 1\b/); + }); + + it("re-fires never spam a duplicate PR for an already-proposed family (documented: dedup note file + open-PR marker check)", () => { + expect(wf).toContain("no PR spam"); }); }); // --------------------------------------------------------------------------- -// FIX (round-4, user-approved) — HUMAN-APPROVAL BACKSTOP. The drift path opens a -// PR but must NEVER auto-merge: the predicate is a strong AUTO-FILTER, not a -// provable merge gate (the re-collect is not independent of the fix — WS-2b), so -// a human reviews CI + the diff + the verdict and merges. These lock that the -// unattended in-workflow merge is GONE and the Slack copy no longer claims -// "merged to main". +// Human-approval backstop — preserved verbatim from the pre-C3 workflow. // --------------------------------------------------------------------------- describe("fix-drift.yml — human-approval backstop: no unattended auto-merge", () => { it("has NO auto-merge step and never runs `gh pr merge`", () => { @@ -144,142 +160,233 @@ describe("fix-drift.yml — human-approval backstop: no unattended auto-merge", expect(wf).not.toMatch(/gh pr merge/); }); - it("documents WHY the drift path is human-gated (predicate is a filter, not a merge gate)", () => { + it("documents WHY the drift-sync path is human-gated (drift-sync-check is a filter, not a merge gate)", () => { expect(wf).toContain("NO AUTO-MERGE"); - // The rationale wraps across comment lines (a `#` marker survives flattening), - // so match tolerantly across the wrap. expect(wfFlat).toMatch(/AUTO-FILTER, NOT a provable merge (# )?gate/i); }); it("the success Slack message says the PR needs human review + merge, NOT merged to main", () => { - expect(wf).not.toContain("Drift auto-fix merged to main"); - expect(wf).toContain("Drift-fix PR opened — needs human review + merge"); - }); - - it("the fix-failure Slack step no longer references the removed merge step outputs", () => { - expect(wf).not.toContain("steps.merge.outputs"); - expect(wf).not.toContain("MERGE_REASON"); + expect(wf).not.toContain("merged to main"); + expect(wf).toContain("Drift-sync PR opened — needs human review + merge"); }); }); // --------------------------------------------------------------------------- -// WS-6 — end-of-job CATCH-ALL alert for the EARLY-infra window. The four -// specific alerts (collector-crash / autofix-step-fail / quarantine / -// fix-failure) are all gated on step outputs that only exist AFTER the -// collector ran. An EARLY failure (checkout / mint-app-token / pnpm install / -// clone ag-ui / git config) leaves those outputs empty, so without a catch-all -// the job dies red with ZERO Slack signal. These lock the catch-all's presence, -// its UNCONDITIONAL `if: failure()` gating, the anti-double-alert guard, and the -// infra-vs-drift-fix distinction. +// Early-infra catch-all — preserved (adapted to the new step id `sync`). // --------------------------------------------------------------------------- -describe("fix-drift.yml — WS-6: early-infra catch-all failure alert", () => { +describe("fix-drift.yml — early-infra catch-all failure alert", () => { it("has an end-of-job catch-all alert step", () => { expect(wf).toContain("Alert on early-infra failure (catch-all)"); }); - it("the catch-all is gated on failure() and is UNCONDITIONAL on the earlier step OUTCOMES", () => { - // Isolate the catch-all step's `if:` expression. + it("the catch-all is gated on failure() and is UNCONDITIONAL on the sync step's REASON output being unset", () => { const idx = wf.indexOf("Alert on early-infra failure (catch-all)"); expect(idx).toBeGreaterThan(-1); - const stepBlock = wf.slice(idx, idx + 600); - const ifMatch = stepBlock.match(/if:\s*>-([\s\S]*?)\n\s{8}env:/); - expect(ifMatch).not.toBeNull(); - const ifExpr = (ifMatch?.[1] ?? "").replace(/\s+/g, " ").trim(); - - // MUST fire on any job failure. - expect(ifExpr).toContain("failure()"); - // MUST NOT gate on the autofix step OUTCOME (that would re-open the - // early-infra silence: autofix.outcome is empty pre-detect). - expect(ifExpr).not.toContain("steps.autofix.outcome"); - // Anti-double-alert guard: only fires when NONE of the specific alerts did - // (collector_crashed unset, quarantine unset, and check never ran so its - // skip output is empty). - expect(ifExpr).toContain("steps.detect.outputs.collector_crashed != 'true'"); - expect(ifExpr).toContain("steps.check.outputs.quarantine != 'true'"); - expect(ifExpr).toContain("steps.check.outputs.skip == ''"); - }); - - it("distinguishes an INFRA/SETUP failure from a drift-fix failure in its message", () => { + const stepBlock = wf.slice(idx, idx + 400); + expect(stepBlock).toContain("if: failure() && steps.sync.outputs.reason == ''"); + }); + + it("distinguishes an INFRA/SETUP failure from a sync-gate failure in its message", () => { const idx = wf.indexOf("Alert on early-infra failure (catch-all)"); const stepBlock = wf.slice(idx, idx + 1400); expect(stepBlock).toMatch(/INFRA\/SETUP failure/); - // Missing-webhook must be a VISIBLE ::error:: + step failure, never silent. expect(stepBlock).toContain("SLACK_WEBHOOK is not set"); expect(stepBlock).toMatch(/::error::/); }); +}); - it("the autofix-step-failure alert requires `check` to have RUN (skip == 'false'), so it never misfires on the early-infra window", () => { - // The autofix-failure alert must NOT fire before the collector ran — that - // window belongs to the catch-all. Gating on skip == 'false' (never empty - // once `check` executed) ensures the two are mutually exclusive. - const idx = wf.indexOf("Alert on autofix step failure"); +// --------------------------------------------------------------------------- +// F#1 (mandatory): the "Push branch + create PR" step can itself fail (branch +// push rejected, `gh pr create` error, or the head-SHA PR-match polling loop +// exhausting its attempts) AFTER the defense-in-depth Assert step already +// SUCCEEDED. In that window: reason stays 'ok-applied', steps.assert.outcome +// stays 'success', and reason is non-empty — so none of needs-human, +// gate-failure (assert.outcome-only check), or the early-infra catch-all +// (reason=='') fire. The job goes red with ZERO Slack signal on an unattended +// daily cron. The gate-failure alert must widen to also catch this window. +// --------------------------------------------------------------------------- +describe("fix-drift.yml — gate-failure alert also covers a later step failing after an ok-applied sync + successful assert", () => { + it("the gate-failure alert fires on ok-applied + failure(), not only on steps.assert.outcome == 'failure' (so a Push/PR-create failure is caught too)", () => { + const idx = wf.indexOf("name: Alert on drift-sync-check gate failure"); expect(idx).toBeGreaterThan(-1); - const stepBlock = wf.slice(idx, idx + 400); - expect(stepBlock).toContain("steps.check.outputs.skip == 'false'"); + const nextStep = wf.indexOf("\n - name:", idx + 1); + const stepBlock = wf.slice(idx, nextStep === -1 ? undefined : nextStep); + // Must be gated on general failure() in the ok-applied branch, not + // narrowly on steps.assert.outcome == 'failure' — otherwise a failure in + // a step AFTER assert (Push branch + create PR) is invisible to this + // condition. + expect(stepBlock).toMatch(/steps\.sync\.outputs\.reason == 'ok-applied' && failure\(\)/); + }); + + it("the gate-failure alert message names which step actually failed", () => { + const idx = wf.indexOf("name: Alert on drift-sync-check gate failure"); + const nextStep = wf.indexOf("\n - name:", idx + 1); + const stepBlock = wf.slice(idx, nextStep === -1 ? undefined : nextStep); + // The step must reference the outcomes of both the assert step and the PR + // step so its message can distinguish "assert refused" from "push/PR + // creation failed" rather than a single generic message. + expect(stepBlock).toContain("steps.assert.outcome"); + expect(stepBlock).toContain("steps.pr.outcome"); }); }); // --------------------------------------------------------------------------- -// WS-8 — the version-bump fail-closed reason must be NAMED in the failure -// alert, and the Create-PR step must surface the script's `reason=` on a -// non-zero exit so a fail-closed exit is not reported blank. +// G#2 (mandatory): a needs-human run WRITES a `drift-proposals/` note (the +// Bucket-B human touchpoint) into CI's working tree, but the workflow only ever +// pushed a branch + opened a PR on reason == 'ok-applied'. On a needs-human run +// the registry is unchanged, so NOTHING was pushed — the note was discarded +// with the runner. The self-service human-decision path (human sets +// `Decision: include`, the NEXT run reads the approved note and applies it) was +// therefore unreachable: the note never landed in the repo. The workflow must +// persist the note on needs-human by pushing a branch + opening a (distinct, +// never auto-merged) PR. // --------------------------------------------------------------------------- -describe("fix-drift.yml — WS-8: version-bump-failed reason wiring", () => { - it("the fix-failure Slack alert names the version-bump-failed reason", () => { - expect(wf).toContain("version-bump-failed)"); - expect(wf).toMatch(/version-bump-failed\)\s+DETAIL=.*UNVERSIONED PR/); +/** Split the workflow into per-step blocks (text after each `- name:` header). */ +function stepBlocks(): string[] { + return wf.split(/\n {6}- name: /).slice(1); +} + +describe("fix-drift.yml — needs-human notes are PERSISTED (pushed + PR'd), not discarded", () => { + it("has a step gated on reason == 'needs-human' that pushes a branch AND opens a PR (so the note reaches the repo)", () => { + // Concept-level: SOME step must both be conditioned on the needs-human + // outcome and perform a git push + `gh pr create`. Pre-fix, the only + // `gh pr create` lives in the ok-applied "Push branch + create PR" step, + // so this finds nothing and FAILS (RED). + const persistSteps = stepBlocks().filter( + (b) => + b.includes("steps.sync.outputs.reason == 'needs-human'") && + /git push\b/.test(b) && + b.includes("gh pr create"), + ); + expect(persistSteps.length).toBeGreaterThan(0); }); - it("the Create-PR step captures the script exit code + reason and surfaces it as a step output on failure", () => { - expect(wfFlat).toContain("PR_EXIT=${PIPESTATUS[0]}"); - expect(wfFlat).toContain("reason=${PR_REASON}"); + it("the needs-human persist step uses a DISTINCT branch (not colliding with the ok-applied fix/drift-* branch)", () => { + const persist = stepBlocks().find( + (b) => b.includes("steps.sync.outputs.reason == 'needs-human'") && b.includes("gh pr create"), + ); + expect(persist).toBeDefined(); + // A dedicated needs-human branch prefix keeps the two PR classes separate. + expect(persist!).toMatch(/drift-needs-human/); + }); + + it("the needs-human persist step de-dups: it skips opening a second PR when one is already open for the same note", () => { + const persist = stepBlocks().find( + (b) => b.includes("steps.sync.outputs.reason == 'needs-human'") && b.includes("gh pr create"), + ); + expect(persist).toBeDefined(); + // Must consult already-open PRs before creating a new one. + expect(persist!).toContain("gh pr list"); + }); + + it("the needs-human persist step's PR body tells a human to set Decision: include and merge (closing the two-run loop), never auto-merged", () => { + const persist = stepBlocks().find( + (b) => b.includes("steps.sync.outputs.reason == 'needs-human'") && b.includes("gh pr create"), + ); + expect(persist).toBeDefined(); + expect(persist!).toContain("Decision: include"); + // No `gh pr merge` anywhere (asserted globally too) — human merges. + expect(persist!).not.toMatch(/gh pr merge/); + }); + + it("the needs-human persist step's OWN failure is alerted (gate-failure alert references its outcome)", () => { + const gateAlert = stepBlocks().find((b) => + b.startsWith("Alert on drift-sync-check gate failure"), + ); + expect(gateAlert).toBeDefined(); + expect(gateAlert!).toContain("steps.needs_human_pr.outcome"); }); }); // --------------------------------------------------------------------------- -// slot2-F3 — QUARANTINE must FAIL THE JOB (non-green), not just Slack-ping. -// A human watching CI status (not Slack) must see quarantine as a failure, like -// the collector-crash / autofix-failure alerts. Because quarantine sets -// check.outputs.skip == 'true', the fix-failure alert (needs skip != 'true') -// and the catch-all (needs skip == '') are both disjoint from it, so making the -// quarantine step exit 1 does NOT double-alert. +// F#2 / G#2 (should-fix): DRIFT.md (and the workflow's own PR-body fallback +// text) claim a `drift-sync-check-log` artifact exists, but historically the +// workflow only uploaded `drift-sync-log`. Assert the claim matches reality. // --------------------------------------------------------------------------- -describe("fix-drift.yml — slot2-F3: quarantine fails the job (non-green)", () => { - it("the quarantine alert step exits non-zero on the happy (webhook-sent) path too", () => { - const idx = wf.indexOf("Alert on drift quarantine"); - expect(idx).toBeGreaterThan(-1); - // The step body runs until the next `- name:` step. - const nextStep = wf.indexOf("\n - name:", idx + 1); - const stepBlock = wf.slice(idx, nextStep === -1 ? undefined : nextStep); - // The curl (happy path) must be FOLLOWED by an `exit 1` — the step is not - // allowed to end green after sending the Slack ping. - const curlIdx = stepBlock.lastIndexOf("curl -fsS"); - expect(curlIdx).toBeGreaterThan(-1); - expect(stepBlock.slice(curlIdx)).toMatch(/\n\s*exit 1\b/); +// --------------------------------------------------------------------------- +// G#3 (mandatory): the needs-human persist step's de-dup was keyed SOLELY on +// the committed `drift-proposals/*` note paths. In the D-M1 "mixed run" (a +// mechanical registry edit committed the SAME run a *different* family is +// deferred to a human, whose note already sits on main), the committed diff is +// ONLY the registry edit and NO new note file — so the note-path list is +// EMPTY, the per-note dedup for-loop runs zero times, and the step falls +// straight through to an unconditional `git push` + `gh pr create`. Because the +// edit is never auto-merged and the unrelated deprecation is re-detected every +// daily cron run, this opens a brand-new near-identical PR every single day +// (unbounded PR-spam). Both PR-open paths must instead de-dup on a STABLE, +// date-independent changeset key that exists for EVERY committed changeset. +// --------------------------------------------------------------------------- +describe("fix-drift.yml — G#3: PR-open paths de-dup on a STABLE changeset key (idempotent in EVERY run shape, incl. the mixed run with NO new note file)", () => { + it("the sync step emits a stable changeset_key step output (grepped from drift-sync.ts's changeset-key= line)", () => { + // RED (pre-fix): drift-sync.ts printed no changeset-key line and the sync + // step captured no such output — nothing existed to dedup a note-less + // mixed run on. + expect(wfFlat).toMatch(/grep '\^changeset-key=' "\$\{SYNC_LOG\}"/); + expect(wf).toContain('echo "changeset_key=${CHANGESET_KEY}"'); + // Written to the step's outputs (block-redirected to $GITHUB_OUTPUT). + expect(wfFlat).toContain('echo "changeset_key=${CHANGESET_KEY}" } >> "$GITHUB_OUTPUT"'); + }); + + it("the needs-human persist step's PRIMARY de-dup is keyed on the changeset key and runs BEFORE the note-file scan (so it fires even when the committed diff carries NO drift-proposals/* note)", () => { + const persist = stepBlocks().find( + (b) => b.includes("steps.sync.outputs.reason == 'needs-human'") && b.includes("gh pr create"), + ); + expect(persist).toBeDefined(); + // Keyed on the changeset key wired from the sync step. + expect(persist!).toContain("CHANGESET_KEY: ${{ steps.sync.outputs.changeset_key }}"); + expect(persist!).toContain("drift-changeset: ${CHANGESET_KEY}"); + // CRITICAL: the changeset-key dedup guard must appear BEFORE the + // `mapfile ... NOTES` scan. Pre-fix, the ONLY dedup lived inside the + // per-note for-loop, reachable only when a note file was in the diff — + // exactly what the empty-NOTES mixed run bypasses. + const guardIdx = persist!.indexOf("drift-changeset: ${CHANGESET_KEY}"); + const mapfileIdx = persist!.indexOf("mapfile -t COMMITTED"); + expect(guardIdx).toBeGreaterThan(-1); + expect(mapfileIdx).toBeGreaterThan(-1); + expect(guardIdx).toBeLessThan(mapfileIdx); }); - it("does not overlap the fix-failure alert or the catch-all (both disjoint from quarantine's skip=='true')", () => { - // fix-failure requires skip != 'true'; catch-all requires skip == ''; - // quarantine sets skip == 'true' — so neither fires alongside it. - expect(wf).toContain("steps.check.outputs.skip != 'true'"); // fix-failure guard - const catchAllIdx = wf.indexOf("Alert on early-infra failure (catch-all)"); - expect(wf.slice(catchAllIdx, catchAllIdx + 600)).toContain("steps.check.outputs.skip == ''"); + it("the ok-applied Push+PR step ALSO de-dups on the changeset key (a never-auto-merged applied edit deserves exactly ONE open PR, re-findable across daily re-fires)", () => { + const okApplied = stepBlocks().find( + (b) => + b.includes("steps.sync.outputs.reason == 'ok-applied' && success()") && + b.includes("gh pr create"), + ); + expect(okApplied).toBeDefined(); + expect(okApplied!).toContain("CHANGESET_KEY: ${{ steps.sync.outputs.changeset_key }}"); + expect(okApplied!).toContain("drift-changeset: ${CHANGESET_KEY}"); + expect(okApplied!).toContain("gh pr list"); + // The dedup skip must precede the push (skip a duplicate BEFORE pushing). + const guardIdx = okApplied!.indexOf("not opening a duplicate"); + const pushIdx = okApplied!.indexOf("git push -u origin"); + expect(guardIdx).toBeGreaterThan(-1); + expect(pushIdx).toBeGreaterThan(-1); + expect(guardIdx).toBeLessThan(pushIdx); + }); + + it("the needs-human persist step RETAINS the per-note body marker as a secondary guard (note-path de-dup not regressed)", () => { + const persist = stepBlocks().find( + (b) => b.includes("steps.sync.outputs.reason == 'needs-human'") && b.includes("gh pr create"), + ); + expect(persist).toBeDefined(); + expect(persist!).toContain("drift-proposal-note: ${note}"); + }); + + it("BOTH PR bodies embed the stable drift-changeset marker the dedup guards match on", () => { + const markerCount = (wf.match(//g) || []).length; + expect(markerCount).toBeGreaterThanOrEqual(2); }); }); -// --------------------------------------------------------------------------- -// slot2-F7/F12 — the fail-closed parse/git paths must be NAMED in the failure -// alert (post-fix-parse-error / git-push-failed), not blank. The code emits the -// reason; the workflow's case block must translate it to a human DETAIL. -// --------------------------------------------------------------------------- -describe("fix-drift.yml — slot2-F7/F12: fail-closed reasons are named in the alert", () => { - it("the fix-failure alert names the post-fix-parse-error reason", () => { - expect(wf).toContain("post-fix-parse-error)"); - expect(wfFlat).toMatch(/post-fix-parse-error\) DETAIL=.*Failed closed/); +describe("fix-drift.yml — drift-sync-check-log artifact matches DRIFT.md's claim", () => { + it("DRIFT.md claims a drift-sync-check-log artifact exists", () => { + const driftMd = readFileSync(resolve(__dirname, "../../DRIFT.md"), "utf-8"); + expect(driftMd).toContain("drift-sync-check-log"); }); - it("the fix-failure alert names the git-push-failed reason", () => { - expect(wf).toContain("git-push-failed)"); - expect(wfFlat).toMatch(/git-push-failed\) DETAIL=.*no PR opened/); + it("the workflow actually uploads a drift-sync-check-log artifact (matching the drift-sync-log sibling's retention)", () => { + expect(wf).toContain("name: drift-sync-check-log"); + expect(wfFlat).toContain("path: ${{ runner.temp }}/drift-sync-check.log"); + expect(wfFlat).toContain("retention-days: 30"); }); }); diff --git a/src/__tests__/fix-drift.test.ts b/src/__tests__/fix-drift.test.ts index 24bee46c..da6d8702 100644 --- a/src/__tests__/fix-drift.test.ts +++ b/src/__tests__/fix-drift.test.ts @@ -1,4 +1,27 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +/** + * C3 (delete-freewriter-predicate-rewire) — retargeted from `scripts/fix-drift.js` + * to `scripts/drift-sync.js`. + * + * `scripts/fix-drift.ts` (the LLM freewriter invocation + its predicate-gated + * `createPr`/`createIssue`) and `scripts/drift-success-predicate.ts` (the + * 916-line anti-cheat predicate) have been DELETED entirely — there is no + * arbitrary/free-form code generation left in the drift-remediation pipeline to + * police. The reusable git/branch/commit/PR plumbing C1 originally moved into + * `drift-sync.ts` (and `fix-drift.ts` re-exported) now lives ONLY in + * `drift-sync.ts`, so this suite imports from there directly. + * + * Dropped in this retarget (no surviving equivalent): + * - `buildPrompt`, `invokeClaudeCode`, `killProcessGroup`, `scheduleEscalatingKill` + * — the deleted LLM freewriter path. + * - `parseMode`, `hasPostFixArgs`, `parsePostFixExit`, `createPr`, `createIssue`, + * `PostFixCollectorResult` — the deleted predicate-gated CLI (`fix-drift.ts`'s + * own `main()`/`--create-pr`/`--create-issue` modes). The deterministic + * model-sync path (`runDriftSyncCli` in drift-sync.ts) commits its own + * mechanical edits directly; there is no freeform diff left to gate behind a + * verdict, so `sanctionedTargets`/`evaluateDriftResolved`-shaped tests do not + * carry over. + */ +import { describe, it, expect, vi, beforeEach } from "vitest"; import { resolve } from "node:path"; import type { @@ -31,26 +54,20 @@ vi.mock("node:child_process", async () => { import { todayStamp, readDriftReport, - buildPrompt, patchBumpVersion, addChangelogEntry, buildPrBody, parsePorcelainLine, readFileIfExists, execFileSafe, - parseMode, - hasPostFixArgs, - parsePostFixExit, getChangedFiles, gatedCommitFiles, - createPr, affectedSkillSections, BUILDER_TO_SKILL_SECTION, truncateBody, GH_BODY_MAX, GH_BODY_SAFE_MAX, -} from "../../scripts/fix-drift.js"; -import { sanctionedTargets } from "../../scripts/drift-success-predicate.js"; +} from "../../scripts/drift-sync.js"; import { readFileSync, writeFileSync, existsSync } from "node:fs"; import { execFileSync, execSync } from "node:child_process"; @@ -343,75 +360,6 @@ describe("readDriftReport", () => { }); }); -// --------------------------------------------------------------------------- -// buildPrompt -// --------------------------------------------------------------------------- - -describe("buildPrompt", () => { - it("includes workflow instructions", () => { - const prompt = buildPrompt(makeReport()); - expect(prompt).toContain("## Workflow"); - expect(prompt).toContain("RED:"); - expect(prompt).toContain("GREEN:"); - expect(prompt).toContain("REFACTOR:"); - }); - - it("renders a single drift entry", () => { - const report = makeReport(); - const prompt = buildPrompt(report); - - expect(prompt).toContain("DRIFT 1: openai"); - expect(prompt).toContain("non-streaming text"); - expect(prompt).toContain("File: src/builders/openai.ts"); - expect(prompt).toContain("Functions: buildTextResponse"); - expect(prompt).toContain("Types file: src/types.ts"); - expect(prompt).toContain("[warning] missing field"); - }); - - it("renders multiple drift entries with sequential numbering", () => { - const report = makeReport({ - entries: [ - makeEntry({ provider: "openai", scenario: "streaming" }), - makeEntry({ provider: "anthropic", scenario: "non-streaming" }), - ], - }); - const prompt = buildPrompt(report); - - expect(prompt).toContain("DRIFT 1: openai"); - expect(prompt).toContain("DRIFT 2: anthropic"); - }); - - it('renders "N/A" when typesFile is null', () => { - const report = makeReport({ - entries: [makeEntry({ typesFile: null })], - }); - const prompt = buildPrompt(report); - expect(prompt).toContain("Types file: N/A"); - }); - - it("includes after-fixes section", () => { - const prompt = buildPrompt(makeReport()); - expect(prompt).toContain("## After all fixes"); - expect(prompt).toContain("pnpm test"); - expect(prompt).toContain("pnpm test:drift"); - }); - - it("renders diff details (path, real, mock)", () => { - const diff = makeDiff({ - path: "body.model", - real: '"gpt-4o"', - mock: '"gpt-4"', - }); - const report = makeReport({ entries: [makeEntry({ diffs: [diff] })] }); - const prompt = buildPrompt(report); - - expect(prompt).toContain("Path: body.model"); - expect(prompt).toContain("SDK type: string"); - expect(prompt).toContain('Real API: "gpt-4o"'); - expect(prompt).toContain('Mock: "gpt-4"'); - }); -}); - // --------------------------------------------------------------------------- // patchBumpVersion // --------------------------------------------------------------------------- @@ -757,90 +705,6 @@ describe("execFileSafe", () => { }); }); -// --------------------------------------------------------------------------- -// parseMode -// --------------------------------------------------------------------------- - -describe("parseMode", () => { - it("returns 'pr' for --create-pr flag", () => { - expect(parseMode(["--create-pr"])).toBe("pr"); - }); - - it("returns 'issue' for --create-issue flag", () => { - expect(parseMode(["--create-issue"])).toBe("issue"); - }); - - it("returns 'default' with no flags", () => { - expect(parseMode([])).toBe("default"); - }); - - it("returns 'default' with unrelated flags", () => { - expect(parseMode(["--report", "drift-report.json"])).toBe("default"); - }); - - it("returns 'pr' even with other flags present", () => { - expect(parseMode(["--report", "drift-report.json", "--create-pr"])).toBe("pr"); - }); -}); - -// --------------------------------------------------------------------------- -// FIX #5 — hasPostFixArgs: PR mode REQUIRES both post-fix flags. The old legacy -// no-post-fix fallback re-opened the fixture-only cheat (a test-file-only change -// satisfied it and opened a PR), so main() throws unless BOTH are present. A -// live subprocess red-green confirmed the OLD code proceeded to `gh pr create` -// with no post-fix args while the NEW code fails-closed BEFORE any git op. -// --------------------------------------------------------------------------- -describe("hasPostFixArgs (fix #5 legacy-fallback closure)", () => { - it("false when NO post-fix flags (the legacy cheat path — must be rejected)", () => { - expect(hasPostFixArgs(["--create-pr", "--report", "drift-report.json"])).toBe(false); - }); - - it("false when only --post-fix-report is present", () => { - expect(hasPostFixArgs(["--create-pr", "--post-fix-report", "post.json"])).toBe(false); - }); - - it("false when only --post-fix-exit is present", () => { - expect(hasPostFixArgs(["--create-pr", "--post-fix-exit", "0"])).toBe(false); - }); - - it("false when a flag is present but its value is missing (trailing flag)", () => { - expect(hasPostFixArgs(["--post-fix-report", "post.json", "--post-fix-exit"])).toBe(false); - }); - - it("true only when BOTH flags carry values", () => { - expect( - hasPostFixArgs(["--create-pr", "--post-fix-report", "post.json", "--post-fix-exit", "0"]), - ).toBe(true); - }); -}); - -// --------------------------------------------------------------------------- -// FIX #F7 (round-4) — parsePostFixExit: the PR path must fail CLOSED on an -// empty/whitespace --post-fix-exit rather than accept Number("")===0 as a clean -// collector exit 0. Mirrors the predicate CLI's guard so a missing recollect -// output never masquerades as clean and opens a PR on an unverified fix. -// --------------------------------------------------------------------------- -describe("parsePostFixExit (fix #F7 empty-exit fail-closed)", () => { - it("throws on an empty string (Number('')===0 must NOT slip through as clean)", () => { - expect(() => parsePostFixExit("")).toThrow(/empty\/whitespace/); - }); - - it("throws on a whitespace-only value", () => { - expect(() => parsePostFixExit(" ")).toThrow(/empty\/whitespace/); - }); - - it("throws on a non-integer value", () => { - expect(() => parsePostFixExit("abc")).toThrow(/integer/); - expect(() => parsePostFixExit("1.5")).toThrow(/integer/); - }); - - it("returns the integer for a valid value", () => { - expect(parsePostFixExit("0")).toBe(0); - expect(parsePostFixExit("2")).toBe(2); - expect(parsePostFixExit("-1")).toBe(-1); - }); -}); - // --------------------------------------------------------------------------- // getChangedFiles // --------------------------------------------------------------------------- @@ -874,17 +738,10 @@ describe("getChangedFiles", () => { }); // --------------------------------------------------------------------------- -// affectedSkillSections +// gatedCommitFiles — no straggler catch-all // --------------------------------------------------------------------------- -// --------------------------------------------------------------------------- -// CR round-3 F-C / F2 — gatedCommitFiles: createPr stages ONLY the allowlisted -// set (production source + report-named fixture targets), NEVER a straggler -// catch-all. A file the predicate would have blocked (config/manifest/unnamed -// fixture) must land in `stragglers`, which createPr never `git add`s. -// --------------------------------------------------------------------------- - -describe("gatedCommitFiles (F-C: no straggler catch-all)", () => { +describe("gatedCommitFiles (no straggler catch-all)", () => { const sanctioned = new Set([ "src/helpers.ts", "src/__tests__/drift/model-registry.ts", @@ -897,7 +754,7 @@ describe("gatedCommitFiles (F-C: no straggler catch-all)", () => { expect(g.stragglers).toEqual([]); }); - it("stages a report-named fixture target as a testFile", () => { + it("stages a sanctioned fixture target as a testFile", () => { const g = gatedCommitFiles(["src/__tests__/drift/model-registry.ts"], sanctioned); expect(g.testFiles).toEqual(["src/__tests__/drift/model-registry.ts"]); expect(g.stragglers).toEqual([]); @@ -918,287 +775,21 @@ describe("gatedCommitFiles (F-C: no straggler catch-all)", () => { expect(g.stragglers).toEqual(["package.json", "pnpm-lock.yaml", "tsconfig.json"]); }); - it("stragglers is empty for a clean allowlisted set (the RESOLVED-verdict invariant)", () => { + it("stragglers is empty for a clean allowlisted set", () => { const g = gatedCommitFiles( ["src/helpers.ts", "src/__tests__/drift/model-registry.ts"], sanctioned, ); expect(g.stragglers).toEqual([]); }); -}); - -// --------------------------------------------------------------------------- -// FIX #F5 (round-4) — createPr MUST fail-closed if any changed file falls -// outside every gated commit group (a straggler): staging it as part of an -// allowlisted group would silently drop or mis-stage it. Because the predicate -// allowlist already blocks any non-production, non-report-named file BEFORE -// createPr stages, an unclassified working-tree file causes createPr to exit -// with UNSANCTIONED_CHANGE and stage NOTHING — the straggler is never `git add`ed. -// -// Driven against the REAL createPr with git mocked (no autofix subprocess). -// --------------------------------------------------------------------------- -describe("createPr straggler / unsanctioned fail-closed (fix #F5)", () => { - const mockedExecSync = vi.mocked(execSync); - let logSpy: ReturnType | null = null; - let errSpy: ReturnType | null = null; - let exitSpy: ReturnType | null = null; - - beforeEach(() => { - vi.clearAllMocks(); - logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { - throw new Error(`__exit__${code}`); - }) as never); - void errSpy; - void exitSpy; - }); - - afterEach(() => { - logSpy?.mockRestore(); - errSpy?.mockRestore(); - exitSpy?.mockRestore(); - }); - - function stdoutLines(): string[] { - return (logSpy?.mock.calls ?? []).map((c) => String(c[0])); - } - - const rep: DriftReport = { - timestamp: "2026-07-16T00:00:00.000Z", - entries: [ - { - provider: "OpenAI", - scenario: "chat completion", - builderFile: "src/helpers.ts", - builderFunctions: ["buildChatCompletion"], - typesFile: null, - sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", - diffs: [ - { - path: "x", - severity: "critical", - issue: "missing", - expected: "string", - real: "string", - mock: "", - }, - ], - }, - ], - }; - - it("exits UNSANCTIONED_CHANGE (17) and stages NOTHING when an unclassified file is in the tree", () => { - // A production fix (helpers.ts, allowlisted) PLUS an unclassified root file. - // createPr fail-closes (the root file is neither allowlisted nor a gated - // group) BEFORE any staging — the straggler is never git-added. - mockedExecSync.mockImplementation((cmd: unknown) => { - if (typeof cmd === "string" && cmd.includes("status --porcelain")) { - return "M src/helpers.ts\n?? weird-root-file.txt\n" as unknown as string; - } - return "" as unknown as string; - }); - - expect(() => createPr(rep, { report: { timestamp: "t", entries: [] }, exitCode: 0 })).toThrow( - /__exit__17/, - ); - - const addedStraggler = mockedExecFileSync.mock.calls.some( - (c) => - c[0] === "git" && Array.isArray(c[1]) && (c[1] as string[]).includes("weird-root-file.txt"), - ); - expect(addedStraggler).toBe(false); - expect(stdoutLines()).toContain("reason=unsanctioned-change"); - }); - it("gatedCommitFiles never leaves a production or report-named file as a straggler (the invariant createPr asserts)", () => { - // The straggler guard in createPr is defense-in-depth: prove the partition - // it relies on classifies every allowlisted file into a gated group so - // stragglers is empty on any RESOLVED-shaped set. - const s = sanctionedTargets(rep); - const g = gatedCommitFiles(["src/helpers.ts", "src/foo.ts"], s); + it("never leaves a production file as a straggler regardless of the sanctioned set", () => { + const g = gatedCommitFiles(["src/helpers.ts", "src/foo.ts"], new Set()); expect(g.stragglers).toEqual([]); expect(g.builderFiles).toEqual(["src/helpers.ts", "src/foo.ts"]); }); }); -// --------------------------------------------------------------------------- -// CR round-3 F-A — the sanctioned/allowlist set is derived SOLELY from the -// (pinned) report object passed to createPr, NOT from any on-disk file. The -// workflow pins the pre-fix report OUTSIDE the LLM-writable repo checkout and -// passes THAT copy via --report; here we prove that a DIFFERENT on-disk report -// cannot expand the allowlist, because the set is a pure function of the passed -// report — the file the LLM could overwrite has no bearing on the sanctioned set. -// --------------------------------------------------------------------------- - -describe("F-A: sanctioned set comes from the passed (pinned) report, not on-disk", () => { - it("sanctionedTargets is a pure function of the report — a forged on-disk file cannot widen it", () => { - // The PINNED report names only the production builder as a target. - const pinned = makeReport({ - entries: [makeEntry({ builderFile: "src/helpers.ts", typesFile: null })], - }); - // A FORGED report (what an autofix LLM might write to drift-report.json in the - // repo) tries to sanction the SDK-shape fixture as a target. - const forged = makeReport({ - entries: [makeEntry({ builderFile: "src/__tests__/drift/sdk-shapes.ts", typesFile: null })], - }); - - const pinnedSet = sanctionedTargets(pinned); - const forgedSet = sanctionedTargets(forged); - - // The sanctioned set is derived purely from the object passed in. - expect(pinnedSet.has("src/helpers.ts")).toBe(true); - expect(pinnedSet.has("src/__tests__/drift/sdk-shapes.ts")).toBe(false); - // The forged set (only relevant if createPr were fed the LLM-writable file) - // would sanction the fixture — which is EXACTLY why the workflow must pin the - // pre-fix report and pass THAT copy, never the in-repo drift-report.json. - expect(forgedSet.has("src/__tests__/drift/sdk-shapes.ts")).toBe(true); - // The two sets are disjoint on the fixture: pinning the report is what keeps - // the fixture OUT of the allowlist. - expect(pinnedSet.has("src/__tests__/drift/sdk-shapes.ts")).not.toBe( - forgedSet.has("src/__tests__/drift/sdk-shapes.ts"), - ); - }); -}); - -// --------------------------------------------------------------------------- -// WS-8 — version-bump failure must FAIL CLOSED, not warn-and-continue. -// -// The original createPr wrapped patchBumpVersion()+changelog+version-commit in -// a try/catch that did `console.warn("Version bump failed, skipping")` and -// CONTINUED to push + open the PR — shipping an UNVERSIONED "fix" that a human -// might merge but which never publishes a release (silent value loss). The fix -// makes a bump failure a HARD, fail-closed error: exit VERSION_BUMP_FAILED (18) -// with a named reason and NO push / NO PR. -// -// Driven against the REAL createPr with git mocked so the version-bump commit -// throws (no autofix subprocess). -// --------------------------------------------------------------------------- -describe("createPr version-bump fail-closed (WS-8)", () => { - const mockedExecSync = vi.mocked(execSync); - let logSpy: ReturnType | null = null; - let errSpy: ReturnType | null = null; - let warnSpy: ReturnType | null = null; - let exitSpy: ReturnType | null = null; - - beforeEach(() => { - vi.clearAllMocks(); - logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); - errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); - warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); - exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { - throw new Error(`__exit__${code}`); - }) as never); - void errSpy; - void exitSpy; - }); - - afterEach(() => { - logSpy?.mockRestore(); - errSpy?.mockRestore(); - warnSpy?.mockRestore(); - exitSpy?.mockRestore(); - }); - - function stdoutLines(): string[] { - return (logSpy?.mock.calls ?? []).map((c) => String(c[0])); - } - - // A clean RESOLVED-shaped report: ONE production builder changed (helpers.ts, - // sanctioned), no stragglers — so createPr proceeds PAST the straggler guard - // into the version-bump step. - const rep: DriftReport = { - timestamp: "2026-07-16T00:00:00.000Z", - entries: [ - { - provider: "OpenAI", - scenario: "chat completion", - builderFile: "src/helpers.ts", - builderFunctions: ["buildChatCompletion"], - typesFile: null, - sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", - diffs: [ - { - path: "x", - severity: "critical", - issue: "missing", - expected: "string", - real: "string", - mock: "", - }, - ], - }, - ], - }; - - function onlyHelpersChanged(cmd: unknown): string { - if (typeof cmd === "string" && cmd.includes("status --porcelain")) { - return "M src/helpers.ts\n" as unknown as string; - } - return "" as unknown as string; - } - - it("exits VERSION_BUMP_FAILED (18) with a named reason and opens NO PR when the version-bump commit throws", () => { - mockedExecSync.mockImplementation(onlyHelpersChanged as never); - // Make the version-bump git commit throw (mock git so the bump step fails). - mockedExecFileSync.mockImplementation((file: unknown, args: unknown) => { - if ( - file === "git" && - Array.isArray(args) && - (args as string[]).includes("commit") && - (args as string[]).some((a) => typeof a === "string" && a.includes("bump version")) - ) { - throw new Error("simulated git failure during version-bump commit"); - } - return Buffer.from("") as unknown as void; - }); - - expect(() => createPr(rep, { report: { timestamp: "t", entries: [] }, exitCode: 0 })).toThrow( - /__exit__18/, - ); - - // Named, fail-closed reason emitted for the workflow's failure alert. - expect(stdoutLines()).toContain("reason=version-bump-failed"); - - // FAIL CLOSED: it must NOT warn-and-continue, and must NEVER push or open a PR. - expect(warnSpy?.mock.calls ?? []).toEqual([]); - const pushed = mockedExecFileSync.mock.calls.some( - (c) => c[0] === "git" && Array.isArray(c[1]) && (c[1] as string[]).includes("push"), - ); - const openedPr = mockedExecFileSync.mock.calls.some( - (c) => c[0] === "gh" && Array.isArray(c[1]) && (c[1] as string[]).includes("pr"), - ); - expect(pushed).toBe(false); - expect(openedPr).toBe(false); - }); - - it("exits GIT_PUSH_FAILED (20) with a named reason and opens NO PR when `git push` throws", () => { - // slot2-F12 lock: a git push failure after the local commits must fail - // CLOSED (no PR) AND emit a NAMED reason — historically it reached the - // top-level catch as a blank-reason exit 3. - mockedExecSync.mockImplementation(onlyHelpersChanged as never); - mockedExecFileSync.mockImplementation((file: unknown, args: unknown) => { - if (file === "git" && Array.isArray(args) && (args as string[]).includes("push")) { - throw new Error("simulated git push failure (auth/network)"); - } - return Buffer.from("") as unknown as void; - }); - - expect(() => createPr(rep, { report: { timestamp: "t", entries: [] }, exitCode: 0 })).toThrow( - /__exit__20/, - ); - - // Named, fail-closed reason (not blank) for the workflow's failure alert. - expect(stdoutLines()).toContain("reason=git-push-failed"); - - // No PR opened — push failed before `gh pr create`. - const openedPr = mockedExecFileSync.mock.calls.some( - (c) => c[0] === "gh" && Array.isArray(c[1]) && (c[1] as string[]).includes("pr"), - ); - expect(openedPr).toBe(false); - }); -}); - describe("affectedSkillSections", () => { it("returns empty array when no builder files are present", () => { expect(affectedSkillSections(["src/__tests__/foo.test.ts", "package.json"])).toEqual([]);