diff --git a/.github/workflows/fix-drift.yml b/.github/workflows/fix-drift.yml index 022192e4..298c519b 100644 --- a/.github/workflows/fix-drift.yml +++ b/.github/workflows/fix-drift.yml @@ -23,6 +23,8 @@ jobs: permissions: contents: write pull-requests: write + checks: read + statuses: read # issues: write # removed — no longer creating issues on drift failure steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 @@ -36,6 +38,10 @@ jobs: private-key: ${{ secrets.DEVOPS_BOT_PRIVATE_KEY }} permission-contents: write permission-pull-requests: write + # Read check runs + commit statuses so the merge gate can assert + # the PR is truly green before merging. + permission-checks: read + permission-statuses: read - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -68,13 +74,18 @@ jobs: npx tsx scripts/drift-report-collector.ts EXIT_CODE=$? set -e - echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT + echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT" if [ "$EXIT_CODE" -eq 2 ]; then : # critical drift found, continue elif [ "$EXIT_CODE" -eq 5 ]; then : # quarantine: collector encountered unparseable output, continue without aborting elif [ "$EXIT_CODE" -ne 0 ]; then - echo "::error::Collector script crashed with exit code $EXIT_CODE" + # A Step-1 collector crash is a DISTINCT failure cause from an + # auto-fix step failure. Record it as an output BEFORE aborting so + # the dedicated collector-crash alert (not the "auto-fix STEP + # failed" alert) fires and names the real cause. + echo "collector_crashed=true" >> "$GITHUB_OUTPUT" + echo "::error::Drift collector (Step 1) crashed with exit code $EXIT_CODE — this is a collector failure, NOT an auto-fix step failure" exit $EXIT_CODE fi @@ -94,11 +105,19 @@ jobs: env: DETECT_EXIT_CODE: ${{ steps.detect.outputs.exit_code }} run: | + set -euo pipefail if [ "$DETECT_EXIT_CODE" = "2" ]; then - echo "skip=false" >> $GITHUB_OUTPUT + echo "skip=false" >> "$GITHUB_OUTPUT" + echo "quarantine=false" >> "$GITHUB_OUTPUT" echo "Critical drift detected" + elif [ "$DETECT_EXIT_CODE" = "5" ]; then + # Unparseable / quarantine: NOT a silent skip — a human must look. + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "quarantine=true" >> "$GITHUB_OUTPUT" + echo "::warning::Drift collector returned quarantine (exit 5) — unparseable output, routing to human alert" else - echo "skip=true" >> $GITHUB_OUTPUT + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "quarantine=false" >> "$GITHUB_OUTPUT" echo "No critical drift detected (exit code: $DETECT_EXIT_CODE) — skipping fix" fi @@ -147,77 +166,328 @@ jobs: TOKEN: ${{ steps.app-token.outputs.token }} # Step 5: Create PR on success + # + # Select the PR by HEAD-SHA match (not `gh pr list ... .[0]`) so a stale + # or unrelated open PR on the same head branch cannot be mistaken for the + # one we just pushed. - name: Create PR id: pr if: steps.autofix.outcome == 'success' && success() && steps.check.outputs.skip != 'true' env: GH_TOKEN: ${{ steps.app-token.outputs.token }} run: | + set -euo pipefail npx tsx scripts/fix-drift.ts --create-pr - PR_URL=$(gh pr list --head "$(git branch --show-current)" --json url --jq '.[0].url') - if [ -z "$PR_URL" ]; then echo "No PR URL found"; exit 1; fi - echo "url=$PR_URL" >> $GITHUB_OUTPUT + HEAD_SHA="$(git rev-parse HEAD)" + echo "Pushed head SHA: $HEAD_SHA" + # Find the open PR whose headRefOid matches the exact SHA we pushed. + PR_URL="" + PR_NUMBER="" + for attempt in $(seq 1 6); do + MATCH="$(gh pr list --state open --json url,number,headRefOid \ + --jq ".[] | select(.headRefOid == \"$HEAD_SHA\") | {url, number}" 2>/dev/null || true)" + if [ -n "$MATCH" ]; then + PR_URL="$(printf '%s' "$MATCH" | jq -r '.url')" + PR_NUMBER="$(printf '%s' "$MATCH" | jq -r '.number')" + break + fi + echo "PR for head SHA not indexed yet (attempt $attempt/6), waiting 10s..." + sleep 10 + done + if [ -z "$PR_URL" ]; then + echo "::error::No open PR found whose head matches pushed SHA $HEAD_SHA" + echo "reason=no-pr-match" >> "$GITHUB_OUTPUT" + exit 1 + fi + echo "Matched PR #$PR_NUMBER: $PR_URL" + { + echo "url=$PR_URL" + echo "number=$PR_NUMBER" + echo "head_sha=$HEAD_SHA" + } >> "$GITHUB_OUTPUT" - # Step 5.5: Auto-merge the drift fix PR + # Step 5.5: Auto-merge the drift fix PR behind a REAL green gate. + # + # A ruleset bypass actor bypasses the entire ruleset, so the green gate + # must be enforced HERE, not in branch protection. The old gate counted + # `gh pr checks | wc -l` rows (a "no checks" message is a row) and trusted + # `--watch --fail-fast`, which exits 0 on empty/pending/skipped/neutral — + # a false-green that could merge an unverified PR. We now: + # 1. poll on machine-readable JSON length for check REGISTRATION, + # 2. `--watch` (with a hard timeout) to let checks run to completion, + # 3. re-query JSON and hand it to scripts/ci-merge-gate.sh, which exits + # 0 ONLY when pass>=1, nothing pending/fail/cancel, and every + # required context is present and passing. - name: Auto-merge PR + id: merge if: success() && steps.pr.outputs.url != '' env: GH_TOKEN: ${{ steps.app-token.outputs.token }} PR_URL: ${{ steps.pr.outputs.url }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} run: | - # Wait for CI to register and pass before merging. A ruleset bypass - # actor bypasses the entire ruleset, so the green gate must live here, - # not in branch protection. `gh pr checks --watch` blocks until all - # checks complete and exits non-zero if any fail; --fail-fast bails on - # first failure. Treat "no checks" as NOT-green: require at least one. - # - # First poll until at least one check is registered (up to ~5 min). - count=0 + set -euo pipefail + + # Phase 1: poll on JSON length until at least one check is registered + # (guards against a registration race letting an empty set through). + registered=0 for i in $(seq 1 10); do - count=$(gh pr checks "$PR_URL" 2>/dev/null | wc -l) - if [ "$count" -gt 0 ]; then - echo "CI checks registered ($count rows), proceeding to watch" - break + # Distinguish a `gh` API error from a genuine zero-check response: + # capture stdout and the exit status separately. `|| echo 0` would + # collapse a transient API failure into a spurious "no checks" and + # could let the poll exhaust on an error rather than a real empty set. + if count="$(gh pr checks "$PR_URL" --json name --jq 'length' 2>/dev/null)"; then + if [ "${count:-0}" -gt 0 ]; then + echo "CI checks registered ($count checks), proceeding to watch" + registered=1 + break + fi + echo "No checks registered yet (attempt $i/10), waiting 30s..." + else + echo "::warning::gh pr checks query failed (attempt $i/10) — retrying, not treating as zero checks" fi - echo "No checks registered yet (attempt $i/10), waiting 30s..." sleep 30 done - if [ "$count" -eq 0 ]; then + if [ "$registered" -eq 0 ]; then echo "::error::No CI checks ever registered — not merging" + echo "reason=no-checks" >> "$GITHUB_OUTPUT" + exit 1 + fi + + # Phase 2: let checks run to completion under a hard timeout so we + # never block the job indefinitely. `--watch` exit status is advisory + # only — the authoritative decision is the gate in phase 3. + if timeout 1200 gh pr checks "$PR_URL" --watch --interval 30; then + echo "watch completed" + else + watch_rc=$? + if [ "$watch_rc" -eq 124 ]; then + echo "::error::CI checks timed out after 20m — not merging" + echo "reason=timeout" >> "$GITHUB_OUTPUT" + exit 1 + fi + echo "watch exited non-zero ($watch_rc) — deferring to gate assertion" + fi + + # Phase 3: authoritative green-gate assertion on machine-readable JSON. + # + # `gh pr checks --json` exits NON-ZERO whenever any check is failing or + # pending — even in --json mode — so under `set -euo pipefail` the job + # would abort HERE and never reach the gate script (reason would never + # be set to not-green). Append `|| true` so the JSON capture is + # advisory and the gate script is the sole authority on the decision. + # The JSON the gate scores MUST be tied to the exact SHA we pushed. + # If HEAD_SHA is empty the anti-TOCTOU guarantee is void — HARD-FAIL + # here rather than re-querying the live head (which would re-open the + # TOCTOU window the pin exists to close). Distinct reason so the Slack + # alert names the real cause. + if [ -z "${HEAD_SHA:-}" ]; then + echo "::error::Merge gate: HEAD_SHA is empty — cannot pin the gate snapshot to the pushed commit, refusing to merge" + echo "reason=head-sha-empty" >> "$GITHUB_OUTPUT" + exit 1 + fi + + # Score the checks for the pinned SHA. gh does not filter --json by + # SHA, so we verify the live head still equals HEAD_SHA before scoring; + # if it moved, the snapshot would be stale — bail with a distinct + # reason rather than score checks that belong to a different commit. + LIVE_SHA="$(gh pr view "$PR_URL" --json headRefOid --jq '.headRefOid' 2>/dev/null || true)" + if [ -z "$LIVE_SHA" ]; then + echo "::error::Merge gate: could not read the PR's live head SHA to confirm the gate snapshot — not merging" + echo "reason=head-requery-error" >> "$GITHUB_OUTPUT" + exit 1 + fi + if [ "$LIVE_SHA" != "$HEAD_SHA" ]; then + echo "::error::Merge gate: head moved ($HEAD_SHA -> $LIVE_SHA) before scoring — snapshot is stale, not merging" + echo "reason=head-moved" >> "$GITHUB_OUTPUT" exit 1 fi - gh pr checks "$PR_URL" --watch --fail-fast --interval 30 || { - echo "::error::CI checks failed or none present — not merging"; exit 1; - } - gh pr merge "$PR_URL" --merge - # Step 7: Slack notification on successful fix + gh pr checks "$PR_URL" --json name,state,bucket > /tmp/pr-checks.json || true + if [ ! -s /tmp/pr-checks.json ]; then + echo "::error::Merge gate: could not read checks JSON — not merging" + echo "reason=not-green" >> "$GITHUB_OUTPUT" + exit 1 + fi + + # Run the gate. Exit 2 is a distinct CONFIG/parse error (malformed + # input, empty/contradictory required set, jq failure) — it is NOT the + # same as "PR is not green" (exit 1) and must be labeled distinctly in + # Slack so a human triages a broken gate, not a red PR. + set +e + ./scripts/ci-merge-gate.sh /tmp/pr-checks.json + gate_rc=$? + set -e + if [ "$gate_rc" -eq 2 ]; then + echo "::error::Merge gate: config/parse error (exit 2) — the gate could not score the checks, not merging" + echo "reason=gate-config-error" >> "$GITHUB_OUTPUT" + exit 1 + fi + if [ "$gate_rc" -ne 0 ]; then + echo "::error::Merge gate refused: PR is not truly green — not merging" + echo "reason=not-green" >> "$GITHUB_OUTPUT" + exit 1 + fi + + # Re-verify the head SHA at merge time matches the SHA the gate + # snapshot was taken against, closing the TOCTOU window where a new + # push lands between the gate check and the merge. `--match-head-commit` + # makes `gh pr merge` refuse if the head moved. HEAD_SHA is guaranteed + # non-empty (asserted above), so there is no live-head fallback. + # + # If the merge command itself fails (head moved and --match-head-commit + # refused, a merge conflict, a transient API error, etc.) set a + # SPECIFIC reason so the Slack alert names "merge command failed" + # rather than falling back to a generic/blank detail. + set +e + gh pr merge "$PR_URL" --merge --match-head-commit "$HEAD_SHA" + merge_rc=$? + set -e + if [ "$merge_rc" -ne 0 ]; then + echo "::error::Merge gate: 'gh pr merge --match-head-commit' failed (rc=$merge_rc) — head may have moved after the gate scored it, or the merge was rejected" + echo "reason=merge-command-failed" >> "$GITHUB_OUTPUT" + exit 1 + fi + echo "merged=true" >> "$GITHUB_OUTPUT" + + # Alert: the Step-1 drift collector crashed (exit code other than + # 0/2/5). This is a DISTINCT cause from an auto-fix step failure — the + # fixer never ran because we could not even produce a drift report — so it + # gets its own message instead of the misleading "auto-fix STEP failed". + - name: Alert on collector crash + if: always() && steps.detect.outputs.collector_crashed == 'true' + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + COLLECTOR_EXIT: ${{ steps.detect.outputs.exit_code }} + run: | + set -euo pipefail + echo "::error::Drift collector (Step 1) crashed (exit ${COLLECTOR_EXIT}) — no drift report produced, auto-fix never ran" + if [ -z "${SLACK_WEBHOOK:-}" ]; then + echo "::error::SLACK_WEBHOOK not set — cannot send collector-crash alert" + else + MSG="💥❌ *Drift collector crashed* (Step 1, exit ${COLLECTOR_EXIT}) — no drift report was produced and no auto-fix was attempted. This is a collector failure, not an auto-fix failure. Manual intervention needed.\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" + curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ + -H "Content-Type: application/json" \ + -d "$PAYLOAD" + fi + exit 1 + + # Alert: the autofix step itself failed. Because that step is + # `continue-on-error: true`, on failure the job would otherwise sail on + # green with no signal. Detect outcome != 'success' (and not the skip + # case), alert distinctly, then fail the job. + - name: Alert on autofix step failure + if: >- + always() && steps.check.outputs.skip != 'true' && + steps.autofix.outcome != 'success' && + steps.detect.outputs.collector_crashed != 'true' + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + AUTOFIX_OUTCOME: ${{ steps.autofix.outcome }} + run: | + set -euo pipefail + echo "::error::Auto-fix step outcome was '${AUTOFIX_OUTCOME}' (not success) — failing the job" + if [ -z "${SLACK_WEBHOOK:-}" ]; then + echo "::error::SLACK_WEBHOOK not set — cannot send autofix-failure alert" + else + MSG="🛠️❌ *Drift auto-fix STEP failed* (outcome: ${AUTOFIX_OUTCOME}) — the fixer errored before producing a PR. Manual intervention needed.\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" + curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ + -H "Content-Type: application/json" \ + -d "$PAYLOAD" + fi + exit 1 + + # Alert: drift collector returned quarantine (exit 5) — unparseable + # output that a human must triage. Routed here instead of the silent + # no-drift skip path. + - name: Alert on drift quarantine + if: always() && steps.check.outputs.quarantine == 'true' + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + # A missing webhook must NOT swallow a quarantine — it is a real + # needs-human event. Emit a VISIBLE ::error:: and FAIL the step (same + # discipline as the autofix-failure alert), never a silent exit 0. + if [ -z "${SLACK_WEBHOOK:-}" ]; then + echo "::error::Drift quarantine (exit 5) — a human must inspect the drift report — AND SLACK_WEBHOOK is not set, so no alert could be sent. See run https://github.com/${REPO}/actions/runs/${RUN_ID}" + exit 1 + fi + MSG="⚠️ *Drift quarantine — needs human* — the drift collector produced unparseable output (exit 5). No auto-fix attempted; a human must inspect the drift report artifact.\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" + curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ + -H "Content-Type: application/json" \ + -d "$PAYLOAD" + + # Slack notification on success. Distinguishes "merged to main" (the merge + # command returned 0) from "PR open, not yet merged" so the message never + # overstates what happened. - name: Notify Slack on fix success if: success() && steps.pr.outputs.url != '' env: SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} PR_URL: ${{ steps.pr.outputs.url }} + MERGED: ${{ steps.merge.outputs.merged }} REPO: ${{ github.repository }} RUN_ID: ${{ github.run_id }} run: | - if [ -z "$SLACK_WEBHOOK" ]; then echo "SLACK_WEBHOOK not set, skipping"; exit 0; fi - MSG="✅ *Drift auto-fix PR created with auto-merge enabled*\nPR: ${PR_URL}\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" - PAYLOAD=$(jq -n --arg text "$MSG" '{text: $text}') - curl -sf -X POST "$SLACK_WEBHOOK" \ + set -euo pipefail + if [ -z "${SLACK_WEBHOOK:-}" ]; then + echo "::error::SLACK_WEBHOOK not set — cannot send fix-success alert" + exit 0 + fi + if [ "${MERGED:-}" = "true" ]; then + MSG="✅ *Drift auto-fix merged to main*\nPR: ${PR_URL}\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + else + MSG="✅ *Drift auto-fix PR open* (not yet merged — gate did not complete a merge)\nPR: ${PR_URL}\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + fi + PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" + curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ -H "Content-Type: application/json" \ -d "$PAYLOAD" - # Step 8: Slack notification on fix failure + # Slack notification on failure. A missing SLACK_WEBHOOK is a VISIBLE + # ::error:: annotation (not a silent exit 0). Distinguishes the merge-gate + # timeout / not-green reasons. - name: Notify Slack on fix failure - if: failure() && steps.check.outputs.skip != 'true' + if: failure() && steps.check.outputs.skip != 'true' && steps.autofix.outcome == 'success' env: SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} REPO: ${{ github.repository }} RUN_ID: ${{ github.run_id }} + MERGE_REASON: ${{ steps.merge.outputs.reason }} + PR_REASON: ${{ steps.pr.outputs.reason }} run: | - if [ -z "$SLACK_WEBHOOK" ]; then echo "SLACK_WEBHOOK not set, skipping"; exit 0; fi - MSG="❌ *Drift auto-fix failed* — manual intervention needed\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" - PAYLOAD=$(jq -n --arg text "$MSG" '{text: $text}') - curl -sf -X POST "$SLACK_WEBHOOK" \ + set -euo pipefail + # The merge-step reason takes precedence; fall back to the PR-step + # reason so a SHA-match exhaustion in Create PR is not reported blank. + REASON="${MERGE_REASON:-${PR_REASON:-}}" + case "$REASON" in + timeout) DETAIL=" (CI checks timed out before completing)";; + not-green) DETAIL=" (merge gate refused — PR not truly green)";; + gate-config-error) DETAIL=" (merge gate CONFIG/parse error — the gate could not score checks; not a red PR, the gate itself needs attention)";; + head-sha-empty) DETAIL=" (merge gate could not pin the snapshot — pushed HEAD SHA was empty)";; + head-requery-error) DETAIL=" (merge gate could not read the PR live head SHA to confirm the snapshot)";; + head-moved) DETAIL=" (PR head moved before the gate scored it — stale snapshot, refused)";; + merge-command-failed) DETAIL=" (the merge command failed — head moved after the gate scored it, or the merge was rejected)";; + no-checks) DETAIL=" (no CI checks ever registered)";; + no-pr-match) DETAIL=" (no open PR matched the pushed head SHA)";; + *) DETAIL="";; + esac + if [ -z "${SLACK_WEBHOOK:-}" ]; then + echo "::error::Drift auto-fix failed${DETAIL} AND SLACK_WEBHOOK is not set — no alert could be sent. See run https://github.com/${REPO}/actions/runs/${RUN_ID}" + exit 0 + fi + MSG="❌ *Drift auto-fix failed*${DETAIL} — manual intervention needed\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" + curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ -H "Content-Type: application/json" \ -d "$PAYLOAD" diff --git a/scripts/ci-merge-gate.sh b/scripts/ci-merge-gate.sh new file mode 100755 index 00000000..870d9ac3 --- /dev/null +++ b/scripts/ci-merge-gate.sh @@ -0,0 +1,327 @@ +#!/usr/bin/env bash +# +# ci-merge-gate.sh — the auto-merge green-gate decision, factored out of the +# fix-drift workflow so it can be unit-tested in isolation. +# +# Reads the machine-readable check-state JSON produced by +# `gh pr checks --json name,state,bucket` (an array of objects) from a +# file argument or stdin, and exits 0 ONLY when the PR is truly green: +# +# 1. at least one check is in the "pass" bucket (>=1), AND +# 2. EVERY present check lands in a recognized bucket and is either in the +# "pass" bucket or on the explicit, documented IGNORE_CONTEXTS allow-list +# — an unknown bucket/state, or a non-ignored skipped/neutral/pending/ +# fail/cancel check, fails the gate (we never silently accept), AND +# 3. every REQUIRED context is present AND concluded in the "pass" bucket. +# +# Policy on neutral/skipped: the "skipping" bucket does NOT count toward +# pass>=1, a required context in "skipping" is treated as NOT satisfied, and a +# NON-required skipped/neutral check FAILS the gate unless its exact name is on +# IGNORE_CONTEXTS. This closes the "newly-added gating check resolves skipped/ +# neutral and is silently ignored → false-green" hole: to accept a skipped +# check you must name it explicitly. +# +# `gh pr checks --json` buckets (from cli/cli): pass | fail | pending | +# skipping | cancel. `state` is the raw check/status conclusion. We key the +# decision off `bucket` and fall back to `state` when bucket is absent so the +# gate is robust to either shape. Any check whose bucket/state does not map to +# one of the five recognized buckets is treated as NOT-pass (fails the gate); +# the recognized buckets must sum to the total check count or we abort. +# +# Required contexts default to the set a drift-fix PR must pass on this repo +# (Static Quality + Unit Tests matrix + Drift Tests PR legs + Zizmor). Override +# via REQUIRED_CONTEXTS (newline- or comma-separated) when the expected set +# changes. An empty/whitespace-only REQUIRED_CONTEXTS is a configuration error +# (exit 2), never a silent no-op that lets a PR through with zero requirements. +# +# Non-required checks that are legitimately skipped may be allow-listed via +# IGNORE_CONTEXTS (newline- or comma-separated, same parsing as +# REQUIRED_CONTEXTS). Anything not on that list and not passing fails the gate. +# +# Usage: +# ci-merge-gate.sh [checks.json] # or pipe JSON on stdin +# REQUIRED_CONTEXTS="a,b,c" ci-merge-gate.sh checks.json +# IGNORE_CONTEXTS="notify,drift" ci-merge-gate.sh checks.json +# +# Exit codes: +# 0 true-green — safe to merge +# 1 not green — do NOT merge (reason printed to stderr) +# 2 usage / malformed-input / configuration error. This is a FAIL-CLOSED +# assertion, not a downstream side effect: a non-array/non-object input, a +# non-string .bucket/.state, an empty/contradictory required set, or ANY +# failure of the SINGLE guarded verdict computation (jq parse/runtime +# error, empty jq output, or a verdict object lacking a boolean .green) +# exits 2 here rather than reading an emptied jq result as green. The whole +# pass/fail decision is computed by ONE jq program and validated ONCE +# before the shell reads any field; the gate never merges on a verdict it +# could not trust. + +set -euo pipefail + +DEFAULT_REQUIRED_CONTEXTS='prettier +eslint +exports +commitlint +test (20) +test (22) +test (24) +agui-schema-drift +drift-live-pr +zizmor' + +# Non-required checks that legitimately do not run on a drift-fix PR and must +# NOT block the merge when they resolve skipped/neutral. Keep this list tight — +# every name here is an explicit, reviewed decision to tolerate a non-passing +# state for that context. +DEFAULT_IGNORE_CONTEXTS='notify +drift' + +REQUIRED_CONTEXTS="${REQUIRED_CONTEXTS:-$DEFAULT_REQUIRED_CONTEXTS}" +IGNORE_CONTEXTS="${IGNORE_CONTEXTS:-$DEFAULT_IGNORE_CONTEXTS}" + +# Read the check JSON from the file arg or stdin. +if [ "$#" -ge 1 ] && [ "$1" != "-" ]; then + if [ ! -f "$1" ]; then + echo "::error::ci-merge-gate: input file not found: $1" >&2 + exit 2 + fi + CHECKS_JSON="$(cat "$1")" +else + CHECKS_JSON="$(cat)" +fi + +if [ -z "${CHECKS_JSON//[[:space:]]/}" ]; then + echo "::error::ci-merge-gate: empty check JSON — treating as NOT green" >&2 + exit 1 +fi + +# Validate it parses as a JSON array whose every element is an object. An +# array of non-objects (e.g. [1,2,3] or ["a"]) would pass a bare type=="array" +# guard and then crash jq on `.bucket` indexing (undocumented exit 5); the +# script's contract is 0/1/2 only, so malformed shapes must exit 2 here. +if ! echo "$CHECKS_JSON" | jq -e 'type == "array"' >/dev/null 2>&1; then + echo "::error::ci-merge-gate: check JSON is not a JSON array" >&2 + exit 2 +fi +if ! echo "$CHECKS_JSON" | jq -e 'all(.[]; type == "object")' >/dev/null 2>&1; then + echo "::error::ci-merge-gate: check JSON array contains a non-object element — malformed input" >&2 + exit 2 +fi + +# Field-type guard: a valid JSON object can still carry a non-string `.bucket` +# or `.state` (a number/object/array) — that passes the object guard above but +# is malformed check data from `gh` and, without this guard, would throw +# "explode input must be a string" inside `ascii_downcase` (undocumented jq +# exit 5). We validate here and FAIL CLOSED with the documented config-error +# exit 2. Absent/null fields are fine (handled downstream as derive-from-state / +# unknown); only a PRESENT non-string value is rejected. +if ! echo "$CHECKS_JSON" | jq -e \ + 'all(.[]; (.bucket | type | . == "string" or . == "null") and (.state | type | . == "string" or . == "null"))' \ + >/dev/null 2>&1; then + echo "::error::ci-merge-gate: a check has a non-string .bucket or .state — malformed check data, treating as config error" >&2 + exit 2 +fi + +# Normalize a comma-/newline-separated list into a JSON array so jq can reason +# over it. Trims surrounding whitespace, drops blank entries. Uses `|| true` so +# an all-blank input (grep matches nothing → exit 1) does not abort under +# `set -e`; the caller inspects the resulting array length. +normalize_list() { + printf '%s' "$1" \ + | tr ',' '\n' \ + | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' \ + | { grep -v '^$' || true; } \ + | jq -R . \ + | jq -s . +} + +REQUIRED_JSON="$(normalize_list "$REQUIRED_CONTEXTS")" +IGNORE_JSON="$(normalize_list "$IGNORE_CONTEXTS")" + +# An empty required set is a configuration error, not a silent green: a gate +# with zero requirements would merge a PR that ran no gating checks at all. +if [ "$(echo "$REQUIRED_JSON" | jq 'length')" -eq 0 ]; then + echo "::error::ci-merge-gate: REQUIRED_CONTEXTS is empty or whitespace-only — refusing to run a gate with no required checks (config error)" >&2 + exit 2 +fi + +# Contradictory config: a name that is BOTH required and ignored is +# self-contradictory (a required context can never be "safe to skip"). Rather +# than silently resolve the ambiguity one way, fail closed as a config error. +CONFLICTING_CONTEXTS="$( + jq -rn --argjson req "$REQUIRED_JSON" --argjson ign "$IGNORE_JSON" \ + '$req - ($req - $ign) | .[]' +)" +if [ -n "$CONFLICTING_CONTEXTS" ]; then + echo "::error::ci-merge-gate: context(s) appear in BOTH REQUIRED_CONTEXTS and IGNORE_CONTEXTS — contradictory config, refusing to run:" >&2 + while IFS= read -r line; do + [ -n "$line" ] && echo "::error:: - $line" >&2 + done <<<"$CONFLICTING_CONTEXTS" + exit 2 +fi + +# Effective bucket for a check: prefer `.bucket`; else derive from `.state`. +# gh buckets: pass | fail | pending | skipping | cancel. Anything we cannot map +# to one of those five becomes the sentinel "unknown" so it is counted, never +# silently dropped. +# +# `.bucket`/`.state` are coerced to strings BEFORE `ascii_downcase`: a valid +# JSON object can carry a non-string `.bucket` (e.g. a number or object) that +# passes the array-of-objects guard above but throws "explode input must be a +# string" inside `ascii_downcase` (undocumented jq exit 5). A non-string field +# is not a recognized bucket/state, so `coerce_str` maps it to "" which then +# resolves to the "unknown" sentinel — counted, never dropped, never a crash. +# shellcheck disable=SC2016 # jq program; $-vars are jq's, must NOT be shell-expanded +JQ_BUCKET=' + def known: ["pass","fail","pending","skipping","cancel"]; + def coerce_str: if type == "string" then . else "" end; + def eff_bucket: + ( if ((.bucket | coerce_str)) != "" then (.bucket | coerce_str | ascii_downcase) + else + (.state | coerce_str | ascii_downcase) as $s + | if ($s == "success") then "pass" + elif ($s | test("^(neutral|skipped)$")) then "skipping" + elif ($s | test("^(failure|error|action_required|startup_failure|timed_out)$")) then "fail" + elif ($s | test("^(cancelled|canceled|stale)$")) then "cancel" + elif ($s == "") then "unknown" + else "pending" + end + end ) as $b + | if (known | index($b)) != null then $b else "unknown" end; +' + +# --------------------------------------------------------------------------- +# SINGLE GUARDED VERDICT (the structural fix that kills the recurring +# "bare-jq-assignment defaults to empty → green-signal" class). +# +# INVARIANT: there is EXACTLY ONE jq computation over the check data, and its +# output is validated ONCE before the shell reads any field from it. The old +# gate scored the checks through ~9 separate `VAR="$(echo "$CHECKS_JSON" | jq +# ...)"` command-substitutions. `set -e` does NOT guard a command-substitution +# RHS, so a jq crash in any of them produced an EMPTY string — and for several +# (notably UNACCEPTED_CHECKS and MISSING_REQUIRED, whose emptiness means "no +# unaccepted checks" / "no missing required") that empty read as the GREEN +# signal. Consolidating into one call + one assertion means ANY jq/parse +# failure is caught HERE as exit 2 (fail-closed), and it is structurally +# impossible for an unguarded empty jq result to be interpreted as green. +# +# The jq program computes the ENTIRE decision — the green/not-green boolean AND +# the human reason AND every scalar — inside jq, so the shell only consumes the +# already-validated object. It emits ONE JSON object: +# { green: bool, reason: string, total, pass, pending, fail, cancel, skip, +# unknown, unaccepted: [string], missing_required: [string] } +# The shell then asserts jq exited 0 AND produced non-empty valid JSON with a +# boolean `.green`; otherwise `::error::` + exit 2. There is NO bare +# `VAR=$(jq)` whose emptiness can be read as green anywhere below. +# --------------------------------------------------------------------------- + +# shellcheck disable=SC2016 # jq program; $-vars are jq's, must NOT be shell-expanded +JQ_VERDICT="$JQ_BUCKET"' + ('"$REQUIRED_JSON"') as $required + | ('"$IGNORE_JSON"') as $ignored + | . as $checks + # Per-check effective bucket, computed once. + | [ $checks[] | { name: .name, b: (. | eff_bucket) } ] as $scored + | ($scored | length) as $total + | ([ $scored[] | select(.b == "pass") ] | length) as $pass + | ([ $scored[] | select(.b == "pending") ] | length) as $pending + | ([ $scored[] | select(.b == "fail") ] | length) as $fail + | ([ $scored[] | select(.b == "cancel") ] | length) as $cancel + | ([ $scored[] | select(.b == "skipping") ] | length) as $skip + | ([ $scored[] | select(.b == "unknown") ] | length) as $unknown + | ($pass + $pending + $fail + $cancel + $skip) as $recognized_sum + # Non-passing checks that are neither required nor on the ignore allow-list. + # Capture the element into $c first: inside `$required | index(...)` the pipe + # rebinds `.` to $required (the array), so `.name` there would index the array + # (jq error) — reference the captured $c.name instead. + | [ $scored[] + | . as $c + | select($c.b != "pass") + | select( ($required | index($c.name)) == null ) + | select( ($ignored | index($c.name)) == null ) + | "\($c.name) [\($c.b)]" + ] as $unaccepted + # Required contexts that are NOT present-and-passing. Only STRING names of + # pass-bucket checks can satisfy a requirement: a pass check with a + # null/absent/non-string name counts toward pass>=1 but must NOT resolve a + # named requirement. + | ( [ $scored[] | select(.b == "pass") | .name | select(type == "string") ] ) as $passing + | ( $required | map(. as $name | select( ($passing | index($name)) == null )) ) as $missing_required + # ALL triggered reasons, in the same order the old sequence of `if` blocks + # emitted them (a single check can trip more than one, e.g. a required + # context in the cancel bucket trips BOTH "cancelled/stale" AND + # "required context missing"). We collect every applicable reason rather than + # short-circuiting on the first so the human-facing diagnostics — and the + # tests that assert on specific reasons — see exactly the same messages as + # before. An empty reasons array == green. + | ( [ (if ($recognized_sum != $total) then "bucket sum mismatch — recognized=\($recognized_sum) total=\($total) (unknown-bucket check(s) present) — NOT green" else empty end), + (if ($unknown > 0) then "\($unknown) check(s) in an unrecognized bucket/state — NOT green" else empty end), + (if ($pass < 1) then "no checks in '"'"'pass'"'"' bucket (pass=\($pass)) — NOT green" else empty end), + (if ($pending > 0) then "\($pending) check(s) still pending/queued/in_progress — NOT green" else empty end), + (if ($fail > 0) then "\($fail) check(s) failed/errored — NOT green" else empty end), + (if ($cancel > 0) then "\($cancel) check(s) cancelled/stale — NOT green" else empty end), + (if (($unaccepted | length) > 0) then "non-passing check(s) not required and not on IGNORE_CONTEXTS allow-list — NOT green" else empty end), + (if (($missing_required | length) > 0) then "required context(s) missing or not passing" else empty end) + ] ) as $reasons + | { + green: (($reasons | length) == 0), + reasons: $reasons, + total: $total, + pass: $pass, + pending: $pending, + fail: $fail, + cancel: $cancel, + skip: $skip, + unknown: $unknown, + unaccepted: $unaccepted, + missing_required: $missing_required + } +' + +# THE one guarded jq computation. Capture and IMMEDIATELY assert: jq exited 0 +# AND emitted a non-empty object with a boolean `.green`. Any failure (parse +# error, runtime crash, empty output, missing/non-boolean `.green`) is a hard +# config error — exit 2, NEVER green. This assertion is what makes it +# impossible for an emptied jq result to be read as a pass. +set +e +VERDICT="$(echo "$CHECKS_JSON" | jq -c "$JQ_VERDICT")" +verdict_rc=$? +set -e +if [ "$verdict_rc" -ne 0 ] || [ -z "${VERDICT//[[:space:]]/}" ] \ + || ! printf '%s' "$VERDICT" | jq -e 'type == "object" and (.green | type == "boolean")' >/dev/null 2>&1; then + echo "::error::ci-merge-gate: jq failed to compute a valid verdict over the input (parse/runtime error, empty output, or missing boolean .green) — cannot score checks, treating as config error" >&2 + exit 2 +fi + +# Extract scalars from the ALREADY-VALIDATED verdict object. These reads are +# safe: the object was asserted to be valid JSON with a boolean .green above, +# so a jq extraction here cannot silently produce a green-reading empty. +GREEN="$(printf '%s' "$VERDICT" | jq -r '.green')" +PASS_COUNT="$(printf '%s' "$VERDICT" | jq -r '.pass')" +SKIP_COUNT="$(printf '%s' "$VERDICT" | jq -r '.skip')" + +if [ "$GREEN" != "true" ]; then + # Emit every triggered reason, and — right after the unaccepted-checks reason + # and the missing-required reason — the per-line `- ` detail so the + # human-facing diagnostics match the previous multi-if output exactly. + while IFS= read -r reason; do + [ -z "$reason" ] && continue + echo "::error::ci-merge-gate: $reason" >&2 + case "$reason" in + "non-passing check(s) not required and not on IGNORE_CONTEXTS allow-list"*) + printf '%s' "$VERDICT" | jq -r '.unaccepted[]?' | while IFS= read -r line; do + [ -n "$line" ] && echo "::error:: - $line" >&2 + done + ;; + "required context(s) missing or not passing"*) + printf '%s' "$VERDICT" | jq -r '.missing_required[]?' | while IFS= read -r line; do + [ -n "$line" ] && echo "::error:: - $line" >&2 + done + ;; + esac + done < <(printf '%s' "$VERDICT" | jq -r '.reasons[]?') + exit 1 +fi + +echo "ci-merge-gate: GREEN — pass=$PASS_COUNT, pending=0, fail=0, cancel=0, skipping=$SKIP_COUNT (all allow-listed), all required contexts present and passing" +exit 0 diff --git a/src/__tests__/ci-merge-gate.test.ts b/src/__tests__/ci-merge-gate.test.ts new file mode 100644 index 00000000..0e9a7d01 --- /dev/null +++ b/src/__tests__/ci-merge-gate.test.ts @@ -0,0 +1,592 @@ +import { describe, it, expect } from "vitest"; +import { spawnSync } from "node:child_process"; +import { readFileSync, writeFileSync, mkdtempSync, rmSync } from "node:fs"; +import { resolve, join } from "node:path"; +import { tmpdir } from "node:os"; + +// --------------------------------------------------------------------------- +// Tests for scripts/ci-merge-gate.sh — the auto-merge green-gate decision. +// +// EVERY case here INVOKES THE REAL scripts/ci-merge-gate.sh with fixture JSON +// and asserts its exit code / stderr. Nothing is asserted against a JS replica +// of the gate — so if the gate script were deleted or reverted to the old +// row-count logic, this suite FAILS (see the "reverting the gate" guard test). +// +// Exit-code contract of the gate: 0 = true-green (merge), 1 = not green +// (do not merge), 2 = usage / malformed-input / config error. +// --------------------------------------------------------------------------- + +const GATE = resolve(__dirname, "../../scripts/ci-merge-gate.sh"); + +type Check = { name: string; state: string; bucket?: string }; + +/** Run the real gate with a check array (or raw string) on stdin. */ +function runGate( + input: Check[] | string, + env: Record = {}, +): { code: number; stdout: string; stderr: string } { + const payload = typeof input === "string" ? input : JSON.stringify(input); + const r = spawnSync("bash", [GATE], { + input: payload, + encoding: "utf8", + env: { ...process.env, ...env }, + }); + return { code: r.status ?? -1, stdout: r.stdout ?? "", stderr: r.stderr ?? "" }; +} + +/** Run the real gate passing the JSON via a FILE ARGUMENT (not stdin). */ +function runGateWithFile( + input: Check[] | string, + env: Record = {}, +): { code: number; stdout: string; stderr: string } { + const payload = typeof input === "string" ? input : JSON.stringify(input); + const dir = mkdtempSync(join(tmpdir(), "gate-file-")); + try { + const file = join(dir, "checks.json"); + writeFileSync(file, payload); + const r = spawnSync("bash", [GATE, file], { + encoding: "utf8", + env: { ...process.env, ...env }, + }); + return { code: r.status ?? -1, stdout: r.stdout ?? "", stderr: r.stderr ?? "" }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +/** Run a COPY of the gate whose source has been mutated, from a temp dir. */ +function runMutatedGate( + mutate: (src: string) => string, + input: Check[] | string, + env: Record = {}, +): { code: number; stdout: string; stderr: string } { + const payload = typeof input === "string" ? input : JSON.stringify(input); + const dir = mkdtempSync(join(tmpdir(), "gate-mut-")); + try { + const mutated = mutate(readFileSync(GATE, "utf8")); + const script = join(dir, "ci-merge-gate.sh"); + writeFileSync(script, mutated); + const r = spawnSync("bash", [script], { + input: payload, + encoding: "utf8", + env: { ...process.env, ...env }, + }); + return { code: r.status ?? -1, stdout: r.stdout ?? "", stderr: r.stderr ?? "" }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +// The canonical required-context set a drift-fix PR must pass on this repo. +// This mirrors the branch's real gating checks. The regression test below +// asserts the gate script's DEFAULT_REQUIRED_CONTEXTS matches this list, so if +// the two drift (a gating check is added/removed in one place only) CI fails +// loudly here instead of silently merging an unverified PR in prod. +const CANONICAL_REQUIRED = [ + "prettier", + "eslint", + "exports", + "commitlint", + "test (20)", + "test (22)", + "test (24)", + "agui-schema-drift", + "drift-live-pr", + "zizmor", +]; + +// All-required-green shape (mirrors PR #305's passing checks). notify/drift are +// non-required extras that legitimately skip; they are on the gate's default +// IGNORE_CONTEXTS allow-list so they must not block. +const ALL_GREEN: Check[] = [ + ...CANONICAL_REQUIRED.map((name) => ({ name, state: "SUCCESS", bucket: "pass" })), + { name: "notify", state: "SKIPPED", bucket: "skipping" }, + { name: "drift", state: "SKIPPED", bucket: "skipping" }, +]; + +describe("ci-merge-gate.sh — refuses every false-green shape (real gate invoked)", () => { + it("empty array [] — REFUSES (exit 1)", () => { + const r = runGate([]); + expect(r.code).toBe(1); + // Tight reason: an empty array specifically has zero pass-bucket checks. + expect(r.stderr).toMatch(/no checks in 'pass' bucket/); + }); + + it("historical no-rows / empty-string input — REFUSES (exit 1)", () => { + // The old gate counted a "no checks reported" stdout LINE as a row and + // sailed through. The real gate treats empty/whitespace input as NOT green. + const r = runGate(" \n "); + expect(r.code).toBe(1); + // Tight reason: whitespace-only input is the empty-JSON path specifically. + expect(r.stderr).toMatch(/empty check JSON/); + }); + + it("pending-only — REFUSES (exit 1)", () => { + const r = runGate([{ name: "test (20)", state: "IN_PROGRESS", bucket: "pending" }]); + expect(r.code).toBe(1); + // Specific pending reason — not any stray occurrence of "pending". + expect(r.stderr).toMatch(/check\(s\) still pending\/queued\/in_progress — NOT green/); + }); + + it("skipped/neutral-only — REFUSES (exit 1), skips never count as pass", () => { + const r = runGate([ + { name: "prettier", state: "SKIPPED", bucket: "skipping" }, + { name: "eslint", state: "NEUTRAL", bucket: "skipping" }, + ]); + expect(r.code).toBe(1); + expect(r.stderr).toMatch(/no checks in 'pass' bucket/); + }); + + it("one unrelated pass, a required context missing — REFUSES (exit 1)", () => { + const r = runGate([{ name: "Continuous Releases", state: "SUCCESS", bucket: "pass" }]); + expect(r.code).toBe(1); + expect(r.stderr).toMatch(/required context\(s\) missing/); + }); + + it("a required context in the fail bucket — REFUSES (exit 1)", () => { + const checks = ALL_GREEN.map((c) => + c.name === "eslint" ? { ...c, state: "FAILURE", bucket: "fail" } : c, + ); + const r = runGate(checks); + expect(r.code).toBe(1); + expect(r.stderr).toMatch(/failed\/errored/); + }); + + it("a required context in the pending bucket — REFUSES (exit 1)", () => { + const checks = ALL_GREEN.map((c) => + c.name === "test (24)" ? { ...c, state: "IN_PROGRESS", bucket: "pending" } : c, + ); + const r = runGate(checks); + expect(r.code).toBe(1); + expect(r.stderr).toMatch(/check\(s\) still pending\/queued\/in_progress — NOT green/); + // A required context that is pending is ALSO missing-required (not passing). + expect(r.stderr).toMatch(/required context\(s\) missing or not passing/); + expect(r.stderr).toMatch(/- test \(24\)$/m); + }); + + it("a check in the cancel/stale bucket — REFUSES (exit 1)", () => { + const checks = [...ALL_GREEN, { name: "extra", state: "STALE", bucket: "cancel" }]; + const r = runGate(checks); + expect(r.code).toBe(1); + expect(r.stderr).toMatch(/cancelled\/stale/); + }); + + it("action_required (derived from state, no bucket field) — REFUSES (exit 1)", () => { + const r = runGate([{ name: "only-check", state: "ACTION_REQUIRED" }], { + REQUIRED_CONTEXTS: "only-check", + }); + expect(r.code).toBe(1); + expect(r.stderr).toMatch(/failed\/errored/); + }); + + // Exercise each derived-state leg INDIVIDUALLY (bucket field absent, so + // eff_bucket derives the bucket from raw .state). Each state maps to a + // specific effective bucket with a specific refusal reason; a single lumped + // assertion could pass on the wrong mapping, so we pin each leg to its exact + // reason. The single required "only-check" is present-and-passing, so the + // ONLY reason to refuse is the extra check's derived bucket. + const DERIVED_STATE_LEGS: Array<{ state: string; reason: RegExp }> = [ + { state: "ERROR", reason: /check\(s\) failed\/errored — NOT green/ }, + { state: "STARTUP_FAILURE", reason: /check\(s\) failed\/errored — NOT green/ }, + { state: "TIMED_OUT", reason: /check\(s\) failed\/errored — NOT green/ }, + { state: "CANCELED", reason: /check\(s\) cancelled\/stale — NOT green/ }, + { state: "NEUTRAL", reason: /not required and not on IGNORE_CONTEXTS/ }, + ]; + for (const { state, reason } of DERIVED_STATE_LEGS) { + it(`derived-state leg '${state}' (no bucket field) maps to its bucket and REFUSES (exit 1)`, () => { + const r = runGate( + [ + { name: "only-check", state: "SUCCESS", bucket: "pass" }, + { name: "extra", state }, + ], + { REQUIRED_CONTEXTS: "only-check" }, + ); + expect(r.code).toBe(1); + expect(r.stderr).toMatch(reason); + }); + } + + it("a non-required skipped check NOT on the ignore-list — REFUSES (exit 1)", () => { + // Guards finding #1: a newly-added gating check that resolves skipped must + // NOT be silently ignored just because it is not (yet) in REQUIRED_CONTEXTS. + const r = runGate([ + { name: "only-check", state: "SUCCESS", bucket: "pass" }, + { name: "new-gating-check", state: "SKIPPED", bucket: "skipping" }, + ]); + // only-check is not in the default REQUIRED set → required missing too, + // but the salient assertion is the unaccepted-check refusal. + expect(r.code).toBe(1); + expect(r.stderr).toMatch(/not required and not on IGNORE_CONTEXTS/); + }); + + it("an unknown-bucket check — REFUSES (exit 1), never silently dropped", () => { + // Guards finding #2: a check whose bucket/state spelling is unrecognized + // must be treated as NOT-pass and fail the gate. + const r = runGate( + [ + { name: "only-check", state: "SUCCESS", bucket: "pass" }, + { name: "weird", state: "WHAT", bucket: "mystery" }, + ], + { REQUIRED_CONTEXTS: "only-check" }, + ); + expect(r.code).toBe(1); + // An unknown-bucket check trips BOTH the unknown-count reason AND the sum + // mismatch (it does not land in any recognized bucket). Assert both specific + // reasons rather than an either-or match. + expect(r.stderr).toMatch(/check\(s\) in an unrecognized bucket\/state — NOT green/); + expect(r.stderr).toMatch(/bucket sum mismatch — recognized=\d+ total=\d+/); + }); + + it("malformed JSON (array of non-objects [1,2,3]) — exit 2, SPECIFICALLY the object-guard branch", () => { + // Guards finding #4: this used to slip past the type=="array" guard and + // crash jq with an undocumented exit 5. Assert the SPECIFIC object-guard + // message so this cannot pass on the field-type guard's "non-string .bucket + // or .state" message instead — the two guards catch different malformations + // and a test that matched either would not lock the object-guard branch. + const r = runGate("[1,2,3]"); + expect(r.code).toBe(2); + expect(r.stderr).toMatch(/check JSON array contains a non-object element/); + // NOT the field-type guard: [1,2,3] fails the object guard first. + expect(r.stderr).not.toMatch(/non-string \.bucket or \.state/); + }); + + it("malformed JSON (not an array) — exit 2 per contract", () => { + const r = runGate('{"name":"x"}'); + expect(r.code).toBe(2); + expect(r.stderr).toMatch(/not a JSON array/); + }); + + it("empty/whitespace REQUIRED_CONTEXTS — exit 2 config error, never a no-op green", () => { + // Guards finding #7: a gate with zero requirements must NOT merge. + const r = runGate([{ name: "x", state: "SUCCESS", bucket: "pass" }], { + REQUIRED_CONTEXTS: " ", + }); + expect(r.code).toBe(2); + expect(r.stderr).toMatch(/REQUIRED_CONTEXTS is empty/); + }); + + it("a REQUIRED context resolved to skipping — REFUSES (exit 1), skip never satisfies a requirement", () => { + const checks = ALL_GREEN.map((c) => + c.name === "zizmor" ? { ...c, state: "SKIPPED", bucket: "skipping" } : c, + ); + const r = runGate(checks); + expect(r.code).toBe(1); + expect(r.stderr).toMatch(/required context\(s\) missing or not passing/); + // The salient reason is the missing-required refusal, NOT a generic skip. + expect(r.stderr).toMatch(/- zizmor$/m); + }); + + it("a REQUIRED context in the cancel bucket — REFUSES (exit 1)", () => { + const checks = ALL_GREEN.map((c) => + c.name === "commitlint" ? { ...c, state: "CANCELLED", bucket: "cancel" } : c, + ); + const r = runGate(checks); + expect(r.code).toBe(1); + expect(r.stderr).toMatch(/cancelled\/stale/); + expect(r.stderr).toMatch(/required context\(s\) missing or not passing/); + }); + + it("duplicate same-name REQUIRED where one leg FAILS — REFUSES (exit 1)", () => { + // A required context with a passing leg AND a failing leg is NOT green: the + // failing leg must sink the gate even though the name is technically present + // in a pass bucket. + const checks = [...ALL_GREEN, { name: "eslint", state: "FAILURE", bucket: "fail" }]; + const r = runGate(checks); + expect(r.code).toBe(1); + expect(r.stderr).toMatch(/failed\/errored/); + }); + + it("empty-string state AND bucket → unknown (exit 1), never silently pass", () => { + // A blank state+bucket must resolve to the "unknown" sentinel and fail — + // never be dropped or defaulted to pass. + const r = runGate([{ name: "only-check", state: "", bucket: "" }], { + REQUIRED_CONTEXTS: "only-check", + }); + expect(r.code).toBe(1); + expect(r.stderr).toMatch(/check\(s\) in an unrecognized bucket\/state — NOT green/); + expect(r.stderr).toMatch(/bucket sum mismatch — recognized=\d+ total=\d+/); + }); + + it("non-string .bucket (object) with state SUCCESS — exit 2 config error (closes a false-GREEN, not a jq crash)", () => { + // CORRECTED RATIONALE: the coerce_str helper already maps a non-string + // .bucket to "" (no ascii_downcase crash / exit 5). The REAL hole this + // field-type guard closes is a FALSE-GREEN via the state fallback: a check + // with a non-string .bucket but state="SUCCESS" would, absent the guard, + // fall through coerce_str ("") into the state branch and resolve to the + // "pass" bucket — a malformed check silently scored as passing. The guard + // rejects it as the documented config error (exit 2) instead. The + // ISOLATED-VECTOR test below proves that flip directly. + const r = runGate('[{"name":"x","state":"SUCCESS","bucket":{"weird":true}}]', { + REQUIRED_CONTEXTS: "x", + }); + expect(r.code).toBe(2); + expect(r.stderr).toMatch(/non-string \.bucket or \.state/); + }); + + it("ISOLATED VECTOR: non-string .bucket→state-fallback false-green (guard removed → GREEN; present → refused)", () => { + // Prove the guard's VALUE, not just its presence: the same malformed input + // (non-string .bucket, state SUCCESS) flips from a false-GREEN to a refusal + // when the field-type guard is present. Mutating the guard out demonstrates + // the exact false-green the guard closes. + const payload = '[{"name":"x","state":"SUCCESS","bucket":{"weird":true}}]'; + // Guard REMOVED: the non-string bucket coerces to "" → state fallback → + // "pass" bucket → the gate would MERGE this malformed check (false-green). + const withoutGuard = runMutatedGate( + (src) => + src.replace( + /# Field-type guard:[\s\S]*?exit 2\nfi\n/, + "# field-type guard removed for this test\n", + ), + payload, + { REQUIRED_CONTEXTS: "x" }, + ); + expect(withoutGuard.code).toBe(0); + expect(withoutGuard.stdout).toMatch(/GREEN/); + // Guard PRESENT (real gate): the malformed check is refused as exit 2. + const withGuard = runGate(payload, { REQUIRED_CONTEXTS: "x" }); + expect(withGuard.code).toBe(2); + expect(withGuard.stderr).toMatch(/non-string \.bucket or \.state/); + }); + + it("non-string .state (number) — exit 2 config error (malformed check data, not a jq crash)", () => { + // Same field-type guard on .state: a numeric state is malformed check data + // from gh and must fail closed as the documented config error, not be + // coerced and scored. + const r = runGate('[{"name":"x","state":123}]', { REQUIRED_CONTEXTS: "x" }); + expect(r.code).toBe(2); + expect(r.stderr).toMatch(/non-string \.bucket or \.state/); + }); + + it("contradictory config: a name in BOTH required and ignore — exit 2 config error", () => { + // Guards finding #5: a required context can never be "safe to skip". The + // gate fails closed rather than silently resolving the contradiction. + const r = runGate([{ name: "x", state: "SUCCESS", bucket: "pass" }], { + REQUIRED_CONTEXTS: "x,dup", + IGNORE_CONTEXTS: "dup", + }); + expect(r.code).toBe(2); + expect(r.stderr).toMatch(/BOTH REQUIRED_CONTEXTS and IGNORE_CONTEXTS/); + }); + + it("null .name in the pass bucket does NOT satisfy a required context — REFUSES (exit 1)", () => { + // Guards finding #5: a nameless pass check counts toward pass>=1 but must + // not be able to resolve a named required context. + const r = runGate('[{"name":null,"state":"SUCCESS","bucket":"pass"}]', { + REQUIRED_CONTEXTS: "realreq", + }); + expect(r.code).toBe(1); + expect(r.stderr).toMatch(/required context\(s\) missing or not passing/); + expect(r.stderr).toMatch(/- realreq$/m); + }); +}); + +describe("ci-merge-gate.sh — accepts genuine true-green (real gate invoked)", () => { + it("all-required-green (extras skipped + allow-listed) — ACCEPTS (exit 0)", () => { + const r = runGate(ALL_GREEN); + expect(r.code).toBe(0); + expect(r.stdout).toMatch(/GREEN/); + }); + + it("duplicate/mixed same-name required context (one pass) — ACCEPTS (exit 0)", () => { + // A required context appearing twice (e.g. re-run) where at least one leg + // passes and none fail/pend is satisfied. + const checks = [ + ...ALL_GREEN, + { name: "eslint", state: "SUCCESS", bucket: "pass" }, // duplicate pass + ]; + const r = runGate(checks); + expect(r.code).toBe(0); + }); + + it("duplicate same-name required with one leg still pending — REFUSES (exit 1)", () => { + const checks = [ + ...ALL_GREEN, + { name: "eslint", state: "IN_PROGRESS", bucket: "pending" }, // duplicate pending + ]; + const r = runGate(checks); + expect(r.code).toBe(1); + expect(r.stderr).toMatch(/check\(s\) still pending\/queued\/in_progress — NOT green/); + }); + + it("honors REQUIRED_CONTEXTS override (comma-separated)", () => { + const r = runGate([{ name: "only-check", state: "SUCCESS", bucket: "pass" }], { + REQUIRED_CONTEXTS: "only-check", + }); + expect(r.code).toBe(0); + }); + + it("honors IGNORE_CONTEXTS override for a non-required skipped check", () => { + const r = runGate( + [ + { name: "only-check", state: "SUCCESS", bucket: "pass" }, + { name: "optional-skip", state: "SKIPPED", bucket: "skipping" }, + ], + { REQUIRED_CONTEXTS: "only-check", IGNORE_CONTEXTS: "optional-skip" }, + ); + expect(r.code).toBe(0); + }); + + it("derives bucket from raw state when the bucket field is absent", () => { + const checks = ALL_GREEN.filter((c) => c.bucket === "pass").map(({ name, state }) => ({ + name, + state, + })); + const r = runGate(checks as Check[]); + expect(r.code).toBe(0); + }); +}); + +describe("ci-merge-gate.sh — file-argument input path (real gate invoked)", () => { + it("reads JSON from a FILE ARG and accepts true-green (exit 0)", () => { + const r = runGateWithFile(ALL_GREEN); + expect(r.code).toBe(0); + expect(r.stdout).toMatch(/GREEN/); + }); + + it("reads JSON from a FILE ARG and refuses a false-green (exit 1)", () => { + const r = runGateWithFile([{ name: "test (20)", state: "IN_PROGRESS", bucket: "pending" }]); + expect(r.code).toBe(1); + expect(r.stderr).toMatch(/check\(s\) still pending\/queued\/in_progress — NOT green/); + }); + + it("file-not-found → exit 2 config error", () => { + const r = spawnSync("bash", [GATE, "/nonexistent/path/does-not-exist.json"], { + encoding: "utf8", + env: { ...process.env }, + }); + expect(r.status).toBe(2); + expect(r.stderr).toMatch(/input file not found/); + }); +}); + +describe("ci-merge-gate.sh — drift + revert guards", () => { + it("gate script's DEFAULT_REQUIRED_CONTEXTS matches the canonical required set", () => { + // Finding #1(b): catch REQUIRED_CONTEXTS drifting from the repo's real + // required checks in CI, not in prod. If a gating check is added/removed in + // only one place, this fails loudly. + const src = readFileSync(GATE, "utf8"); + const m = src.match(/DEFAULT_REQUIRED_CONTEXTS='([^']*)'/); + expect(m, "DEFAULT_REQUIRED_CONTEXTS assignment not found in gate script").toBeTruthy(); + const scriptList = m![1] + .split("\n") + .map((s) => s.trim()) + .filter(Boolean); + expect(scriptList).toEqual(CANONICAL_REQUIRED); + }); + + it("gate script's DEFAULT_IGNORE_CONTEXTS is exactly the tight reviewed set (widening fails here)", () => { + // Drift-guard on the DANGEROUS direction: widening IGNORE_CONTEXTS silently + // tolerates more non-passing checks. Pin it to the exact reviewed set so any + // addition to the allow-list must be a conscious, test-updating change. + const src = readFileSync(GATE, "utf8"); + const m = src.match(/DEFAULT_IGNORE_CONTEXTS='([^']*)'/); + expect(m, "DEFAULT_IGNORE_CONTEXTS assignment not found in gate script").toBeTruthy(); + const scriptList = m![1] + .split("\n") + .map((s) => s.trim()) + .filter(Boolean); + expect(scriptList).toEqual(["notify", "drift"]); + }); + + it("CONSOLIDATED-VERDICT GUARD: a runtime jq failure in the verdict program exits 2, never masks to green", () => { + // Structural fix (single guarded verdict): the entire decision is computed + // by ONE jq program whose output is validated ONCE before the shell reads + // any field. Mutate a shared helper (coerce_str) so eff_bucket — and thus + // the whole verdict program — throws at runtime, and assert the gate fails + // CLOSED with the documented config-error exit 2 (the verdict validity + // assertion), NOT a scored 0/1 decision, not a false-green, not exit 5. + const r = runMutatedGate( + (src) => src.replace(/def coerce_str:[^\n]*\n/, "def coerce_str: (undefined_fn);\n"), + [{ name: "x", state: "SUCCESS", bucket: "pass" }], + { REQUIRED_CONTEXTS: "x" }, + ); + expect(r.code).toBe(2); + // The SPECIFIC consolidated-verdict assertion message — not a generic match. + expect(r.stderr).toMatch(/jq failed to compute a valid verdict over the input/); + }); + + it("CONSOLIDATED-VERDICT GUARD: an EMPTY verdict (jq emits nothing) exits 2, never reads as green", () => { + // This is the crux of the recurring class: `set -e` does NOT guard a + // command-substitution RHS, so a crashed jq yields an EMPTY VERDICT. The + // old scattered `UNACCEPTED_CHECKS`/`MISSING_REQUIRED` assignments read that + // empty as "no unaccepted / no missing" = GREEN. Force the single verdict + // capture to emit nothing and confirm the validity assertion rejects it as + // exit 2 — an empty verdict can NEVER be interpreted as a pass. + const r = runMutatedGate( + (src) => + src.replace( + /VERDICT="\$\(echo "\$CHECKS_JSON" \| jq -c "\$JQ_VERDICT"\)"/, + 'VERDICT="$(printf %s "")"', + ), + [{ name: "x", state: "SUCCESS", bucket: "pass" }], + { REQUIRED_CONTEXTS: "x" }, + ); + expect(r.code).toBe(2); + expect(r.stderr).toMatch(/jq failed to compute a valid verdict over the input/); + }); + + it("CONSOLIDATED-VERDICT GUARD: a verdict object with a non-boolean .green exits 2, never green", () => { + // The validity assertion requires .green to be a BOOLEAN. A verdict whose + // .green somehow becomes a non-boolean (e.g. a string) must fail closed as + // exit 2 rather than let `[ "$GREEN" != "true" ]` misfire. Mutate the + // verdict program to emit green as a string and confirm exit 2. + const r = runMutatedGate( + (src) => src.replace(/green: \(\(\$reasons \| length\) == 0\),/, 'green: "yes",'), + [{ name: "x", state: "SUCCESS", bucket: "pass" }], + { REQUIRED_CONTEXTS: "x" }, + ); + expect(r.code).toBe(2); + expect(r.stderr).toMatch(/jq failed to compute a valid verdict over the input/); + }); + + it("STRUCTURAL INVARIANT: no bare `VAR=$(jq ...)` whose emptiness could read as green", () => { + // The recurring bug class is a scattered `VAR="$(echo "$CHECKS_JSON" | jq + // ...)"` whose crash-emptied output is later interpreted as a green signal. + // Lock the invariant structurally: the ONLY jq invocation that consumes + // $CHECKS_JSON to score checks is the single guarded $JQ_VERDICT capture. + // No `$CHECKS_JSON | jq` assignment may exist outside that one line. + const src = readFileSync(GATE, "utf8"); + const checksJsonJqAssignments = src + .split("\n") + .filter((l) => !/^\s*#/.test(l)) // ignore comment lines + .filter((l) => /=\s*"?\$\(\s*echo "\$CHECKS_JSON" \| jq/.test(l)); + // Exactly one: the guarded VERDICT capture. + expect(checksJsonJqAssignments.length).toBe(1); + expect(checksJsonJqAssignments[0]).toMatch( + /VERDICT="\$\(echo "\$CHECKS_JSON" \| jq -c "\$JQ_VERDICT"\)"/, + ); + }); + + it("REVERT GUARD: the old row-count logic would NOT pass this suite", () => { + // Prove the suite is anchored to the real gate, not a replica: model the + // OLD gate (merge iff row-count>0 AND nothing in fail/cancel) and assert it + // gives the WRONG answer on false-green shapes the real gate refuses. If + // someone reverted ci-merge-gate.sh to the old logic, the false-green cases + // above would flip to exit 0 and this expectation documents why they fail. + const oldGateWouldMerge = (checks: Check[]): boolean => { + const rowCount = checks.length === 0 ? 1 : checks.length; + if (rowCount === 0) return false; + const watchFails = checks.some((c) => c.bucket === "fail" || c.bucket === "cancel"); + return !watchFails; + }; + // Empty, pending-only, skipped-only, unrelated-pass, skipped-required: old + // logic MERGES (bug) on all of these. + expect(oldGateWouldMerge([])).toBe(true); + expect(oldGateWouldMerge([{ name: "t", state: "IN_PROGRESS", bucket: "pending" }])).toBe(true); + expect(oldGateWouldMerge([{ name: "t", state: "SKIPPED", bucket: "skipping" }])).toBe(true); + // An UNRELATED pass with a required missing: old row-count logic sees >0 rows + // and no fail/cancel → MERGES (bug); the real gate refuses (required missing). + const unrelatedPass = [{ name: "Continuous Releases", state: "SUCCESS", bucket: "pass" }]; + expect(oldGateWouldMerge(unrelatedPass)).toBe(true); + // A skipped-ONLY (no pass at all) set: old logic MERGES (bug); real gate + // refuses (no pass bucket). + const skippedOnly = [{ name: "t", state: "SKIPPED", bucket: "skipping" }]; + expect(oldGateWouldMerge(skippedOnly)).toBe(true); + // The REAL gate refuses ALL of those, so a revert to old logic flips these + // exit codes from 1 → 0 and breaks CI on the broadened cases too. + expect(runGate([]).code).toBe(1); + expect(runGate([{ name: "t", state: "IN_PROGRESS", bucket: "pending" }]).code).toBe(1); + expect(runGate(unrelatedPass).code).toBe(1); + expect(runGate(skippedOnly).code).toBe(1); + }); +});