feat: run OpenAI Realtime drift probes live in CI via OPENAI_API_KEY fallback

scripts/drift-report-collector.ts

@@ -350,6 +350,48 @@ export function parseKnownModelsCanary(text: string): CanaryParseResult | null {
   return truncated ? { ids, truncated: true } : { ids };
 }
 
+// ---------------------------------------------------------------------------
+// WS handshake-failure recognizer
+// ---------------------------------------------------------------------------
+
+/**
+ * A parsed OpenAI-Realtime WS handshake failure. The socket UPGRADED (101) and
+ * the live API sent back an `error` event, but the expected session-lifecycle
+ * event never arrived, so the probe's `waitUntil(...)` timed out. That shape is
+ * a genuine, actionable protocol drift (e.g. a GA session-config field the
+ * probe/mock stopped sending) — NOT a benign network flake (a flake times out
+ * having collected ZERO messages and carries no `error` body).
+ *
+ * Recognizing it here diverts it from the opaque exit-5 quarantine into a
+ * parseable, attributed critical DriftEntry (exit 2), so the failing handshake
+ * and its error payload are visible and route to a builder for remediation.
+ * Narrowly gated (realtime probe origin + a surfaced `error` event body) so it
+ * can never reclassify another provider's failure or a bare network timeout.
+ */
+export interface WSHandshakeFailure {
+  errorType: string;
+  errorCode: string;
+  errorMessage: string;
+}
+
+export function parseWSHandshakeFailure(text: string): WSHandshakeFailure | null {
+  // Gate 1: the probe timed out waiting for a lifecycle event (handshake never
+  // completed). Gate 2: it is the OpenAI Realtime WS probe (its stack frame is
+  // always present on a real failure; the surfaced `error` body comes from
+  // ws-providers' openaiRealtimeWS). Gate 3: an `error` event body was surfaced
+  // — this is the "connect succeeded but handshake didn't complete WITH a
+  // protocol error" case. A pure network flake (zero messages, no error body)
+  // fails Gate 3 and stays in the quarantine lane for human review.
+  if (!/waitUntil timeout/.test(text)) return null;
+  if (!/ws-realtime\.drift\.ts/.test(text)) return null;
+  if (!/"type"\s*:\s*"error"/.test(text)) return null;
+
+  const errorType = text.match(/"error"\s*:\s*\{[^}]*?"type"\s*:\s*"([^"]+)"/)?.[1] ?? "unknown";
+  const errorCode = text.match(/"code"\s*:\s*"([^"]+)"/)?.[1] ?? "unknown";
+  const errorMessage = text.match(/"message"\s*:\s*"((?:[^"\\]|\\.)*)"/)?.[1] ?? "unknown";
+  return { errorType, errorCode, errorMessage };
+}
+
 // ---------------------------------------------------------------------------
 // Run drift tests and collect results
 // ---------------------------------------------------------------------------
@@ -740,6 +782,39 @@ export function collectDriftEntries(results: VitestJsonResult): CollectResult {
           });
           continue;
         }
+
+        // A WS handshake that upgraded but never completed, carrying a surfaced
+        // provider `error` event, is a parseable critical drift (exit 2) — not
+        // an opaque exit-5 quarantine. Recognized narrowly so a bare network
+        // timeout (no error body) still falls through to the quarantine lane.
+        const wsFailure = parseWSHandshakeFailure(fullMessage);
+        if (wsFailure !== null) {
+          const mapping = SURFACE_REGISTRY["openai-realtime"];
+          entries.push({
+            provider: "OpenAI Realtime",
+            scenario: "WS handshake",
+            builderFile: mapping.builderFile,
+            builderFunctions: mapping.builderFunctions,
+            typesFile: mapping.typesFile ?? null,
+            sdkShapesFile: SDK_SHAPES_FILE,
+            diffs: [
+              {
+                severity: "critical" as const,
+                issue:
+                  "OpenAI Realtime WS handshake did not complete — the live API returned an " +
+                  `error event (${wsFailure.errorType}/${wsFailure.errorCode}) and the expected ` +
+                  "session lifecycle event never arrived. The realtime session config sent by the " +
+                  `probe/mock likely drifted from the live protocol. Error: ${wsFailure.errorMessage}`,
+                path: `session.${wsFailure.errorCode}`,
+                expected: "(handshake completes: session.created/updated received)",
+                real: `error ${wsFailure.errorType}: ${wsFailure.errorMessage}`,
+                mock: "<no mock leg — live handshake probe>",
+                id: `ws-handshake:${wsFailure.errorCode}`,
+              },
+            ],
+          });
+          continue;
+        }
         unparseable++;
         continue;
       }
@@ -834,9 +909,10 @@ export function collectDriftEntries(results: VitestJsonResult): CollectResult {
         const fullMessage = assertion.failureMessages.join("\n");
         const parsed = parseDriftBlock(fullMessage);
         if (!parsed || parsed.diffs.length === 0) {
-          // Canary shapes are handled above (they became entries) — only truly
-          // unparseable messages reach here.
+          // Canary and WS-handshake shapes are handled above (they became
+          // entries) — only truly unparseable messages reach here.
           if (parseKnownModelsCanary(fullMessage) !== null) continue;
+          if (parseWSHandshakeFailure(fullMessage) !== null) continue;
           unparseableFailures.push({
             message: fullMessage,
             testName: `${assertion.ancestorTitles.join(" ")} > ${assertion.title}`,

src/__tests__/drift-collector.test.ts

@@ -418,6 +418,64 @@ describe("collectDriftEntries", () => {
     expect(exitCodeOf(result)).toBe(5);
   });
 
+  it("recognizes an OpenAI Realtime WS handshake failure as a critical DriftEntry (exit 2), NOT an opaque exit-5 quarantine", () => {
+    // REAL vitest failure-message shape captured from the drift-live-pr CI run
+    // that surfaced the GA session.type protocol change: the socket upgraded,
+    // the live API returned ONE `error` event, then the probe timed out waiting
+    // for session.updated. Before the WS-handshake recognizer this fell through
+    // to exit-5 quarantine (opaque red); now it is a parseable critical drift.
+    const wsHandshakeFailure =
+      "Error: waitUntil timeout after 30000ms. Collected 1 messages: [error] " +
+      'bodies=[{"type":"error","event_id":"event_E4b9BfUiVmC9qkIgQZSni",' +
+      '"error":{"type":"invalid_request_error","code":"missing_required_parameter",' +
+      '"message":"Missing required parameter: \'session.type\'.","param":"session.type"}}]\n' +
+      "    at openaiRealtimeWS (/repo/src/__tests__/drift/ws-providers.ts:214:23)\n" +
+      "    at /repo/src/__tests__/drift/ws-realtime.drift.ts:138:26";
+    const result = makeResult([
+      makeAssertion({
+        status: "failed",
+        ancestorTitles: ["OpenAI Realtime API drift"],
+        title: "WS text event sequence and shapes match (GA)",
+        failureMessages: [wsHandshakeFailure],
+      }),
+    ]);
+
+    // GREEN: one attributed critical entry, no quarantine, exit 2.
+    expect(quarantineOf(result)).toEqual([]);
+    const entries = entriesOf(result);
+    expect(entries).toHaveLength(1);
+    expect(entries[0].provider).toBe("OpenAI Realtime");
+    expect(entries[0].builderFile).toBe("src/ws-realtime.ts");
+    expect(entries[0].diffs).toHaveLength(1);
+    expect(entries[0].diffs[0].severity).toBe("critical");
+    // The surfaced error payload (type/code/message) is carried into the entry.
+    expect(entries[0].diffs[0].real).toContain("invalid_request_error");
+    expect(entries[0].diffs[0].issue).toContain("missing_required_parameter");
+    expect(entries[0].diffs[0].issue).toContain("session.type");
+    expect(exitCodeOf(result)).toBe(2);
+  });
+
+  it("does NOT recognize a bare WS network timeout (zero messages, no error body) as handshake drift → stays quarantined (exit 5)", () => {
+    // A genuine transient network flake times out having collected ZERO
+    // messages and carries no provider `error` body. It must NOT be reclassified
+    // as protocol drift — it stays in the quarantine lane for human review.
+    const bareTimeout =
+      "Error: waitUntil timeout after 30000ms. Collected 0 messages: []\n" +
+      "    at openaiRealtimeWS (/repo/src/__tests__/drift/ws-providers.ts:372:20)\n" +
+      "    at /repo/src/__tests__/drift/ws-realtime.drift.ts:138:26";
+    const result = makeResult([
+      makeAssertion({
+        status: "failed",
+        ancestorTitles: ["OpenAI Realtime API drift"],
+        title: "WS text event sequence and shapes match (GA)",
+        failureMessages: [bareTimeout],
+      }),
+    ]);
+    expect(entriesOf(result)).toEqual([]);
+    expect(quarantineOf(result)).toHaveLength(1);
+    expect(exitCodeOf(result)).toBe(5);
+  });
+
   it("returns valid entries and tolerates unparseable failures mixed in", () => {
     const driftText = formatDriftReport("OpenAI Chat (non-streaming text)", [SAMPLE_DIFF]);
     const result = makeResult([

src/__tests__/drift/ws-providers.ts

@@ -200,10 +200,19 @@ export function connectTLSWebSocket(
                     settled = true;
                     removeResolver();
                     const types = collected.map((m: any) => m?.type ?? "unknown").join(", ");
+                    // Surface collected message bodies (truncated) so an early
+                    // `error` event's code/message is visible in CI logs rather
+                    // than swallowed behind the bare type list.
+                    let bodies = "";
+                    try {
+                      bodies = ` bodies=${JSON.stringify(collected).slice(0, 800)}`;
+                    } catch {
+                      /* non-serializable payload; type list is enough */
+                    }
                     reject(
                       new Error(
                         `waitUntil timeout after ${timeoutMs}ms. ` +
-                          `Collected ${collected.length} messages: [${types}]`,
+                          `Collected ${collected.length} messages: [${types}]${bodies}`,
                       ),
                     );
                   }
@@ -344,28 +353,34 @@ export async function openaiRealtimeWS(
   config: ProviderConfig,
   text: string,
   tools?: object[],
-  beta = true,
 ): Promise<WSResult> {
-  // Realtime API requires a realtime-specific model (gpt-4o-mini doesn't work)
+  // GA-only probe. The Realtime Beta API is retired — a live Beta handshake
+  // ("OpenAI-Beta: realtime=v1") is now rejected with
+  // {"code":"beta_api_shape_disabled","message":"The Realtime Beta API is no
+  // longer supported. Please use /v1/realtime for the GA API."}. So this probe
+  // exercises ONLY the GA surface, which is the surface aimock mocks.
+  //
+  // Realtime API requires a realtime-specific model (gpt-4o-mini doesn't work).
   const headers: Record<string, string> = {
     Authorization: `Bearer ${config.apiKey}`,
   };
-  if (beta) {
-    headers["OpenAI-Beta"] = "realtime=v1";
-  }
   const ws = await connectTLSWebSocket(
     "api.openai.com",
-    "/v1/realtime?model=gpt-4o-mini-realtime-preview",
+    "/v1/realtime?model=gpt-realtime-mini",
     headers,
   );
 
   // Step 1: Wait for session.created
   const sessionCreated = await ws.waitUntil((msg: any) => msg?.type === "session.created");
 
-  // Step 2: Send session.update
+  // Step 2: Send session.update.
+  // GA requires session.type:"realtime" and renames the legacy `modalities`
+  // field to `output_modalities`. Confirmed live: a GA session.update without
+  // session.type is rejected with "Missing required parameter: 'session.type'".
   const session: Record<string, unknown> = {
-    model: "gpt-4o-mini-realtime-preview",
-    modalities: ["text"],
+    type: "realtime",
+    model: "gpt-realtime-mini",
+    output_modalities: ["text"],
   };
   if (tools) session.tools = tools;
   ws.send(JSON.stringify({ type: "session.update", session }));
@@ -385,11 +400,8 @@ export async function openaiRealtimeWS(
     }),
   );
 
-  // Step 5: Wait for conversation.item.created (Beta) or conversation.item.added (GA)
-  const itemCreated = await ws.waitUntil(
-    (msg: any) =>
-      msg?.type === "conversation.item.created" || msg?.type === "conversation.item.added",
-  );
+  // Step 5: Wait for conversation.item.added (GA)
+  const itemCreated = await ws.waitUntil((msg: any) => msg?.type === "conversation.item.added");
 
   // Step 6: Send response.create
   ws.send(JSON.stringify({ type: "response.create" }));