From d1429e4da6ddd6baf79843c18cf8070f95009c15 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 22 Jul 2026 16:44:23 -0700 Subject: [PATCH 1/6] feat: run OpenAI Realtime drift probes live in CI Gate the 3 realtime WS protocol probes on OPENAI_REALTIME_KEY ?? OPENAI_API_KEY so they run live in CI via the existing key instead of skipping. No new secret. --- src/__tests__/drift/ws-realtime.drift.ts | 225 ++++++++++++----------- 1 file changed, 117 insertions(+), 108 deletions(-) diff --git a/src/__tests__/drift/ws-realtime.drift.ts b/src/__tests__/drift/ws-realtime.drift.ts index 7450ea6c..56446f3e 100644 --- a/src/__tests__/drift/ws-realtime.drift.ts +++ b/src/__tests__/drift/ws-realtime.drift.ts @@ -42,12 +42,23 @@ let instance: ServerInstance; // CI actually provides to the drift job) and is gated ONLY on OPENAI_API_KEY, // guaranteeing it runs in CI and cannot silently skip. // -// The real realtime WS *session* tests connect a live socket that legitimately -// requires realtime access, so they gate on OPENAI_REALTIME_KEY and pass that -// same credential in their session config — using the chat-only OPENAI_API_KEY -// there could produce a spurious auth-failure "drift" if the two keys differ. +// The real realtime WS *session* tests connect a live socket that requires +// realtime model access. OpenAI Realtime authenticates with the STANDARD +// project key (scope is project-level; there is no separate "realtime key" +// type), so these probes resolve their credential as OPENAI_REALTIME_KEY (an +// optional explicit override, kept for the rare case where realtime access +// lives on a different project) ELSE the standard OPENAI_API_KEY that CI +// already provides. Preferring the explicit override but falling back to the +// standard key lets the protocol probes run LIVE in daily CI without adding a +// new secret. (If CI ever shows these probes failing with an auth/scope error, +// it means this project's OPENAI_API_KEY lacks Realtime access and a separate +// OPENAI_REALTIME_KEY genuinely is required.) const OPENAI_API_KEY = process.env.OPENAI_API_KEY; const OPENAI_REALTIME_KEY = process.env.OPENAI_REALTIME_KEY; +// Resolved realtime credential: explicit override preferred, standard key as +// fallback. Non-empty whenever the describe below runs (it gates on +// OPENAI_API_KEY), so the protocol probes RUN in CI instead of skipping. +const OPENAI_REALTIME_CREDENTIAL = OPENAI_REALTIME_KEY ?? OPENAI_API_KEY; beforeAll(async () => { instance = await startDriftServer(); @@ -62,10 +73,11 @@ afterAll(async () => { // --------------------------------------------------------------------------- describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime API drift", () => { - // Session config for the real WS realtime tests uses the realtime credential, - // NOT the chat-only OPENAI_API_KEY, so that a differing realtime key can't - // cause a spurious auth-failure "drift". The canary below does NOT use this. - const config = { apiKey: OPENAI_REALTIME_KEY! }; + // Session config for the real WS realtime tests: explicit OPENAI_REALTIME_KEY + // override if set, else the standard OPENAI_API_KEY (which CI provides and + // which authenticates Realtime at project scope). The canary below reads + // OPENAI_API_KEY directly and does NOT use this. + const config = { apiKey: OPENAI_REALTIME_CREDENTIAL! }; it("canary: GA realtime models available", async () => { // Fetch via GET /v1/models, which lists ALL models for ANY valid key. Use @@ -117,7 +129,7 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime API drift", () => { expect(unknown, `UNKNOWN_REALTIME_MODELS=${unknown.join(",")}`).toEqual([]); }); - it.skipIf(!process.env.OPENAI_REALTIME_KEY)( + it.skipIf(!OPENAI_REALTIME_CREDENTIAL)( "WS text event sequence and shapes match (GA)", async () => { const sdkEvents = openaiRealtimeTextEventShapes(); @@ -195,108 +207,105 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime API drift", () => { }, ); - it.skipIf(!process.env.OPENAI_REALTIME_KEY)( - "WS tool call event sequence matches (GA)", - async () => { - const sdkEvents = [ - ...openaiRealtimeTextEventShapes().filter( - (e) => - e.type === "session.created" || - e.type === "session.updated" || - e.type === "conversation.item.added" || - e.type === "response.created" || - e.type === "response.done", - ), - ...openaiRealtimeToolCallEventShapes(), - ]; - - const tools = [ - { - type: "function", - name: "get_weather", - description: "Get weather", - parameters: { - type: "object", - properties: { city: { type: "string" } }, - required: ["city"], - }, + it.skipIf(!OPENAI_REALTIME_CREDENTIAL)("WS tool call event sequence matches (GA)", async () => { + const sdkEvents = [ + ...openaiRealtimeTextEventShapes().filter( + (e) => + e.type === "session.created" || + e.type === "session.updated" || + e.type === "conversation.item.added" || + e.type === "response.created" || + e.type === "response.done", + ), + ...openaiRealtimeToolCallEventShapes(), + ]; + + const tools = [ + { + type: "function", + name: "get_weather", + description: "Get weather", + parameters: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], }, - ]; - - // Real API — GA mode - const realResult = await openaiRealtimeWS(config, "Weather in Paris", tools, false); - - // Mock — replicate the Realtime protocol sequence - const mockWs = await connectWebSocket(instance.url, "/v1/realtime"); - - // session.created - const sessionCreatedMsgs = await mockWs.waitForMessages(1); - const allMockRaw: unknown[] = [JSON.parse(sessionCreatedMsgs[0])]; - - // session.update with tools - mockWs.send( - JSON.stringify({ - type: "session.update", - session: { model: "gpt-4o-mini", modalities: ["text"], tools }, - }), - ); - const sessionUpdatedMsgs = await mockWs.waitForMessages(2); - allMockRaw.push(JSON.parse(sessionUpdatedMsgs[1])); - - // conversation.item.create - mockWs.send( - JSON.stringify({ - type: "conversation.item.create", - item: { - type: "message", - role: "user", - content: [{ type: "input_text", text: "Weather in Paris" }], - }, - }), - ); - const itemCreatedMsgs = await mockWs.waitForMessages(3); - allMockRaw.push(JSON.parse(itemCreatedMsgs[2])); - - // response.create - mockWs.send(JSON.stringify({ type: "response.create" })); - - // Collect remaining messages until response.done - const responseMsgs = await collectMockWSMessages( - mockWs, - (msg) => (msg as Record).type === "response.done", - 15000, - 3, - ); - allMockRaw.push(...responseMsgs.rawMessages); - mockWs.close(); - - // Build mock events - const mockEvents = allMockRaw.map((msg) => { - const m = msg as Record; - return { - type: m.type as string, - dataShape: extractShape(msg), - }; - }); - - expect(realResult.rawMessages.length, "Real API returned no WS messages").toBeGreaterThan(0); - expect(mockEvents.length, "Mock returned no WS messages").toBeGreaterThan(0); - - const diffs = compareSSESequences(sdkEvents, realResult.events, mockEvents); - const report = formatDriftReport( - "OpenAI Realtime WS (GA tool call events)", - diffs, - "openai-realtime", - ); + }, + ]; + + // Real API — GA mode + const realResult = await openaiRealtimeWS(config, "Weather in Paris", tools, false); + + // Mock — replicate the Realtime protocol sequence + const mockWs = await connectWebSocket(instance.url, "/v1/realtime"); + + // session.created + const sessionCreatedMsgs = await mockWs.waitForMessages(1); + const allMockRaw: unknown[] = [JSON.parse(sessionCreatedMsgs[0])]; + + // session.update with tools + mockWs.send( + JSON.stringify({ + type: "session.update", + session: { model: "gpt-4o-mini", modalities: ["text"], tools }, + }), + ); + const sessionUpdatedMsgs = await mockWs.waitForMessages(2); + allMockRaw.push(JSON.parse(sessionUpdatedMsgs[1])); + + // conversation.item.create + mockWs.send( + JSON.stringify({ + type: "conversation.item.create", + item: { + type: "message", + role: "user", + content: [{ type: "input_text", text: "Weather in Paris" }], + }, + }), + ); + const itemCreatedMsgs = await mockWs.waitForMessages(3); + allMockRaw.push(JSON.parse(itemCreatedMsgs[2])); + + // response.create + mockWs.send(JSON.stringify({ type: "response.create" })); + + // Collect remaining messages until response.done + const responseMsgs = await collectMockWSMessages( + mockWs, + (msg) => (msg as Record).type === "response.done", + 15000, + 3, + ); + allMockRaw.push(...responseMsgs.rawMessages); + mockWs.close(); + + // Build mock events + const mockEvents = allMockRaw.map((msg) => { + const m = msg as Record; + return { + type: m.type as string, + dataShape: extractShape(msg), + }; + }); + + expect(realResult.rawMessages.length, "Real API returned no WS messages").toBeGreaterThan(0); + expect(mockEvents.length, "Mock returned no WS messages").toBeGreaterThan(0); + + const diffs = compareSSESequences(sdkEvents, realResult.events, mockEvents); + const report = formatDriftReport( + "OpenAI Realtime WS (GA tool call events)", + diffs, + "openai-realtime", + ); - expect( - diffs.filter((d) => d.severity === "critical"), - report, - ).toEqual([]); - }, - ); + expect( + diffs.filter((d) => d.severity === "critical"), + report, + ).toEqual([]); + }); - it.skipIf(!process.env.OPENAI_REALTIME_KEY)( + it.skipIf(!OPENAI_REALTIME_CREDENTIAL)( "GA and Beta event sequences are consistent after normalization", async () => { // GA connection (no Beta header) From 558d486208da6e8a150b4249b8a4378483c41804 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 22 Jul 2026 16:53:51 -0700 Subject: [PATCH 2/6] fix: point live OpenAI Realtime drift probe at GA gpt-realtime-mini (retired -preview id) --- src/__tests__/drift/ws-providers.ts | 4 ++-- src/__tests__/drift/ws-realtime.drift.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/__tests__/drift/ws-providers.ts b/src/__tests__/drift/ws-providers.ts index 03e8a7f6..c2d8295c 100644 --- a/src/__tests__/drift/ws-providers.ts +++ b/src/__tests__/drift/ws-providers.ts @@ -355,7 +355,7 @@ export async function openaiRealtimeWS( } const ws = await connectTLSWebSocket( "api.openai.com", - "/v1/realtime?model=gpt-4o-mini-realtime-preview", + "/v1/realtime?model=gpt-realtime-mini", headers, ); @@ -364,7 +364,7 @@ export async function openaiRealtimeWS( // Step 2: Send session.update const session: Record = { - model: "gpt-4o-mini-realtime-preview", + model: "gpt-realtime-mini", modalities: ["text"], }; if (tools) session.tools = tools; diff --git a/src/__tests__/drift/ws-realtime.drift.ts b/src/__tests__/drift/ws-realtime.drift.ts index 56446f3e..a4ff91e1 100644 --- a/src/__tests__/drift/ws-realtime.drift.ts +++ b/src/__tests__/drift/ws-realtime.drift.ts @@ -2,7 +2,7 @@ * OpenAI Realtime API WebSocket drift tests. * * Three-way comparison: SDK types x real API (WS) x aimock output (WS). - * Updated for GA protocol — uses gpt-realtime-2 and GA event names. + * Updated for GA protocol — uses gpt-realtime-mini and GA event names. */ import { describe, it, expect, beforeAll, afterAll } from "vitest"; From 43f0d30ba9d85ddcc7c724638217081f7a331353 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 22 Jul 2026 16:59:17 -0700 Subject: [PATCH 3/6] fix: surface WS error event body in realtime probe timeout for CI triage --- src/__tests__/drift/ws-providers.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/__tests__/drift/ws-providers.ts b/src/__tests__/drift/ws-providers.ts index c2d8295c..eda5b6af 100644 --- a/src/__tests__/drift/ws-providers.ts +++ b/src/__tests__/drift/ws-providers.ts @@ -200,10 +200,19 @@ export function connectTLSWebSocket( settled = true; removeResolver(); const types = collected.map((m: any) => m?.type ?? "unknown").join(", "); + // Surface collected message bodies (truncated) so an early + // `error` event's code/message is visible in CI logs rather + // than swallowed behind the bare type list. + let bodies = ""; + try { + bodies = ` bodies=${JSON.stringify(collected).slice(0, 800)}`; + } catch { + /* non-serializable payload; type list is enough */ + } reject( new Error( `waitUntil timeout after ${timeoutMs}ms. ` + - `Collected ${collected.length} messages: [${types}]`, + `Collected ${collected.length} messages: [${types}]${bodies}`, ), ); } From df998b17c6224493b497fd58735579f14c49e2a1 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 22 Jul 2026 17:19:07 -0700 Subject: [PATCH 4/6] fix: send GA session.type + output_modalities in live OpenAI Realtime probe --- src/__tests__/drift/ws-providers.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/__tests__/drift/ws-providers.ts b/src/__tests__/drift/ws-providers.ts index eda5b6af..8927f8af 100644 --- a/src/__tests__/drift/ws-providers.ts +++ b/src/__tests__/drift/ws-providers.ts @@ -371,11 +371,14 @@ export async function openaiRealtimeWS( // Step 1: Wait for session.created const sessionCreated = await ws.waitUntil((msg: any) => msg?.type === "session.created"); - // Step 2: Send session.update - const session: Record = { - model: "gpt-realtime-mini", - modalities: ["text"], - }; + // Step 2: Send session.update. + // GA (no Beta header) requires session.type:"realtime" and renames the legacy + // `modalities` field to `output_modalities`. Beta keeps the legacy field name + // and has no session.type. Confirmed live: a GA session.update without + // session.type is rejected with "Missing required parameter: 'session.type'". + const session: Record = beta + ? { model: "gpt-realtime-mini", modalities: ["text"] } + : { type: "realtime", model: "gpt-realtime-mini", output_modalities: ["text"] }; if (tools) session.tools = tools; ws.send(JSON.stringify({ type: "session.update", session })); From 5ef07802fdbb8ed3286f8e2b3476d3a847f63873 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 22 Jul 2026 17:19:07 -0700 Subject: [PATCH 5/6] fix: surface OpenAI Realtime WS handshake failure as drift, not opaque quarantine --- scripts/drift-report-collector.ts | 80 ++++++++++++++++++++++++++- src/__tests__/drift-collector.test.ts | 58 +++++++++++++++++++ 2 files changed, 136 insertions(+), 2 deletions(-) diff --git a/scripts/drift-report-collector.ts b/scripts/drift-report-collector.ts index 4e440402..a78807c3 100644 --- a/scripts/drift-report-collector.ts +++ b/scripts/drift-report-collector.ts @@ -350,6 +350,48 @@ export function parseKnownModelsCanary(text: string): CanaryParseResult | null { return truncated ? { ids, truncated: true } : { ids }; } +// --------------------------------------------------------------------------- +// WS handshake-failure recognizer +// --------------------------------------------------------------------------- + +/** + * A parsed OpenAI-Realtime WS handshake failure. The socket UPGRADED (101) and + * the live API sent back an `error` event, but the expected session-lifecycle + * event never arrived, so the probe's `waitUntil(...)` timed out. That shape is + * a genuine, actionable protocol drift (e.g. a GA session-config field the + * probe/mock stopped sending) — NOT a benign network flake (a flake times out + * having collected ZERO messages and carries no `error` body). + * + * Recognizing it here diverts it from the opaque exit-5 quarantine into a + * parseable, attributed critical DriftEntry (exit 2), so the failing handshake + * and its error payload are visible and route to a builder for remediation. + * Narrowly gated (realtime probe origin + a surfaced `error` event body) so it + * can never reclassify another provider's failure or a bare network timeout. + */ +export interface WSHandshakeFailure { + errorType: string; + errorCode: string; + errorMessage: string; +} + +export function parseWSHandshakeFailure(text: string): WSHandshakeFailure | null { + // Gate 1: the probe timed out waiting for a lifecycle event (handshake never + // completed). Gate 2: it is the OpenAI Realtime WS probe (its stack frame is + // always present on a real failure; the surfaced `error` body comes from + // ws-providers' openaiRealtimeWS). Gate 3: an `error` event body was surfaced + // — this is the "connect succeeded but handshake didn't complete WITH a + // protocol error" case. A pure network flake (zero messages, no error body) + // fails Gate 3 and stays in the quarantine lane for human review. + if (!/waitUntil timeout/.test(text)) return null; + if (!/ws-realtime\.drift\.ts/.test(text)) return null; + if (!/"type"\s*:\s*"error"/.test(text)) return null; + + const errorType = text.match(/"error"\s*:\s*\{[^}]*?"type"\s*:\s*"([^"]+)"/)?.[1] ?? "unknown"; + const errorCode = text.match(/"code"\s*:\s*"([^"]+)"/)?.[1] ?? "unknown"; + const errorMessage = text.match(/"message"\s*:\s*"((?:[^"\\]|\\.)*)"/)?.[1] ?? "unknown"; + return { errorType, errorCode, errorMessage }; +} + // --------------------------------------------------------------------------- // Run drift tests and collect results // --------------------------------------------------------------------------- @@ -740,6 +782,39 @@ export function collectDriftEntries(results: VitestJsonResult): CollectResult { }); continue; } + + // A WS handshake that upgraded but never completed, carrying a surfaced + // provider `error` event, is a parseable critical drift (exit 2) — not + // an opaque exit-5 quarantine. Recognized narrowly so a bare network + // timeout (no error body) still falls through to the quarantine lane. + const wsFailure = parseWSHandshakeFailure(fullMessage); + if (wsFailure !== null) { + const mapping = SURFACE_REGISTRY["openai-realtime"]; + entries.push({ + provider: "OpenAI Realtime", + scenario: "WS handshake", + builderFile: mapping.builderFile, + builderFunctions: mapping.builderFunctions, + typesFile: mapping.typesFile ?? null, + sdkShapesFile: SDK_SHAPES_FILE, + diffs: [ + { + severity: "critical" as const, + issue: + "OpenAI Realtime WS handshake did not complete — the live API returned an " + + `error event (${wsFailure.errorType}/${wsFailure.errorCode}) and the expected ` + + "session lifecycle event never arrived. The realtime session config sent by the " + + `probe/mock likely drifted from the live protocol. Error: ${wsFailure.errorMessage}`, + path: `session.${wsFailure.errorCode}`, + expected: "(handshake completes: session.created/updated received)", + real: `error ${wsFailure.errorType}: ${wsFailure.errorMessage}`, + mock: "", + id: `ws-handshake:${wsFailure.errorCode}`, + }, + ], + }); + continue; + } unparseable++; continue; } @@ -834,9 +909,10 @@ export function collectDriftEntries(results: VitestJsonResult): CollectResult { const fullMessage = assertion.failureMessages.join("\n"); const parsed = parseDriftBlock(fullMessage); if (!parsed || parsed.diffs.length === 0) { - // Canary shapes are handled above (they became entries) — only truly - // unparseable messages reach here. + // Canary and WS-handshake shapes are handled above (they became + // entries) — only truly unparseable messages reach here. if (parseKnownModelsCanary(fullMessage) !== null) continue; + if (parseWSHandshakeFailure(fullMessage) !== null) continue; unparseableFailures.push({ message: fullMessage, testName: `${assertion.ancestorTitles.join(" ")} > ${assertion.title}`, diff --git a/src/__tests__/drift-collector.test.ts b/src/__tests__/drift-collector.test.ts index 5c095188..1ac243ab 100644 --- a/src/__tests__/drift-collector.test.ts +++ b/src/__tests__/drift-collector.test.ts @@ -418,6 +418,64 @@ describe("collectDriftEntries", () => { expect(exitCodeOf(result)).toBe(5); }); + it("recognizes an OpenAI Realtime WS handshake failure as a critical DriftEntry (exit 2), NOT an opaque exit-5 quarantine", () => { + // REAL vitest failure-message shape captured from the drift-live-pr CI run + // that surfaced the GA session.type protocol change: the socket upgraded, + // the live API returned ONE `error` event, then the probe timed out waiting + // for session.updated. Before the WS-handshake recognizer this fell through + // to exit-5 quarantine (opaque red); now it is a parseable critical drift. + const wsHandshakeFailure = + "Error: waitUntil timeout after 30000ms. Collected 1 messages: [error] " + + 'bodies=[{"type":"error","event_id":"event_E4b9BfUiVmC9qkIgQZSni",' + + '"error":{"type":"invalid_request_error","code":"missing_required_parameter",' + + '"message":"Missing required parameter: \'session.type\'.","param":"session.type"}}]\n' + + " at openaiRealtimeWS (/repo/src/__tests__/drift/ws-providers.ts:214:23)\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:138:26"; + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "WS text event sequence and shapes match (GA)", + failureMessages: [wsHandshakeFailure], + }), + ]); + + // GREEN: one attributed critical entry, no quarantine, exit 2. + expect(quarantineOf(result)).toEqual([]); + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + expect(entries[0].provider).toBe("OpenAI Realtime"); + expect(entries[0].builderFile).toBe("src/ws-realtime.ts"); + expect(entries[0].diffs).toHaveLength(1); + expect(entries[0].diffs[0].severity).toBe("critical"); + // The surfaced error payload (type/code/message) is carried into the entry. + expect(entries[0].diffs[0].real).toContain("invalid_request_error"); + expect(entries[0].diffs[0].issue).toContain("missing_required_parameter"); + expect(entries[0].diffs[0].issue).toContain("session.type"); + expect(exitCodeOf(result)).toBe(2); + }); + + it("does NOT recognize a bare WS network timeout (zero messages, no error body) as handshake drift → stays quarantined (exit 5)", () => { + // A genuine transient network flake times out having collected ZERO + // messages and carries no provider `error` body. It must NOT be reclassified + // as protocol drift — it stays in the quarantine lane for human review. + const bareTimeout = + "Error: waitUntil timeout after 30000ms. Collected 0 messages: []\n" + + " at openaiRealtimeWS (/repo/src/__tests__/drift/ws-providers.ts:372:20)\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:138:26"; + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "WS text event sequence and shapes match (GA)", + failureMessages: [bareTimeout], + }), + ]); + expect(entriesOf(result)).toEqual([]); + expect(quarantineOf(result)).toHaveLength(1); + expect(exitCodeOf(result)).toBe(5); + }); + it("returns valid entries and tolerates unparseable failures mixed in", () => { const driftText = formatDriftReport("OpenAI Chat (non-streaming text)", [SAMPLE_DIFF]); const result = makeResult([ From ab8db68ea2f9e1c63c0f8cc5a00f87c432daa72e Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 22 Jul 2026 18:58:59 -0700 Subject: [PATCH 6/6] fix: drop retired OpenAI Realtime Beta probe from drift leg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OpenAI retired the Realtime Beta API — a live Beta handshake now returns {"code":"beta_api_shape_disabled","message":"The Realtime Beta API is no longer supported. Please use /v1/realtime for the GA API."}. The drift leg's Beta branch therefore hit a dead endpoint, surfaced an unparseable error, and quarantined the whole leg (exit 5), turning drift-live-pr red at HEAD. Remove the Beta branch from openaiRealtimeWS (GA-only: session.type:"realtime" + output_modalities) and delete the now-dead GA<->Beta consistency probe plus its normalization constants. GA is the only live surface and the only surface aimock mocks. --- src/__tests__/drift/ws-providers.ts | 32 +++++++-------- src/__tests__/drift/ws-realtime.drift.ts | 52 ++++-------------------- 2 files changed, 24 insertions(+), 60 deletions(-) diff --git a/src/__tests__/drift/ws-providers.ts b/src/__tests__/drift/ws-providers.ts index 8927f8af..351dc8fe 100644 --- a/src/__tests__/drift/ws-providers.ts +++ b/src/__tests__/drift/ws-providers.ts @@ -353,15 +353,17 @@ export async function openaiRealtimeWS( config: ProviderConfig, text: string, tools?: object[], - beta = true, ): Promise { - // Realtime API requires a realtime-specific model (gpt-4o-mini doesn't work) + // GA-only probe. The Realtime Beta API is retired — a live Beta handshake + // ("OpenAI-Beta: realtime=v1") is now rejected with + // {"code":"beta_api_shape_disabled","message":"The Realtime Beta API is no + // longer supported. Please use /v1/realtime for the GA API."}. So this probe + // exercises ONLY the GA surface, which is the surface aimock mocks. + // + // Realtime API requires a realtime-specific model (gpt-4o-mini doesn't work). const headers: Record = { Authorization: `Bearer ${config.apiKey}`, }; - if (beta) { - headers["OpenAI-Beta"] = "realtime=v1"; - } const ws = await connectTLSWebSocket( "api.openai.com", "/v1/realtime?model=gpt-realtime-mini", @@ -372,13 +374,14 @@ export async function openaiRealtimeWS( const sessionCreated = await ws.waitUntil((msg: any) => msg?.type === "session.created"); // Step 2: Send session.update. - // GA (no Beta header) requires session.type:"realtime" and renames the legacy - // `modalities` field to `output_modalities`. Beta keeps the legacy field name - // and has no session.type. Confirmed live: a GA session.update without + // GA requires session.type:"realtime" and renames the legacy `modalities` + // field to `output_modalities`. Confirmed live: a GA session.update without // session.type is rejected with "Missing required parameter: 'session.type'". - const session: Record = beta - ? { model: "gpt-realtime-mini", modalities: ["text"] } - : { type: "realtime", model: "gpt-realtime-mini", output_modalities: ["text"] }; + const session: Record = { + type: "realtime", + model: "gpt-realtime-mini", + output_modalities: ["text"], + }; if (tools) session.tools = tools; ws.send(JSON.stringify({ type: "session.update", session })); @@ -397,11 +400,8 @@ export async function openaiRealtimeWS( }), ); - // Step 5: Wait for conversation.item.created (Beta) or conversation.item.added (GA) - const itemCreated = await ws.waitUntil( - (msg: any) => - msg?.type === "conversation.item.created" || msg?.type === "conversation.item.added", - ); + // Step 5: Wait for conversation.item.added (GA) + const itemCreated = await ws.waitUntil((msg: any) => msg?.type === "conversation.item.added"); // Step 6: Send response.create ws.send(JSON.stringify({ type: "response.create" })); diff --git a/src/__tests__/drift/ws-realtime.drift.ts b/src/__tests__/drift/ws-realtime.drift.ts index a4ff91e1..9f76770b 100644 --- a/src/__tests__/drift/ws-realtime.drift.ts +++ b/src/__tests__/drift/ws-realtime.drift.ts @@ -15,22 +15,6 @@ import { detectVoiceModelDrift } from "./voice-models.js"; import { startDriftServer, stopDriftServer, collectMockWSMessages } from "./helpers.js"; import { connectWebSocket } from "../ws-test-client.js"; -// --------------------------------------------------------------------------- -// GA <-> Beta event name mapping (local copy for normalization in tests) -// --------------------------------------------------------------------------- - -const GA_TO_BETA_EVENT: Record = { - "response.output_text.delta": "response.text.delta", - "response.output_text.done": "response.text.done", - "response.output_audio.delta": "response.audio.delta", - "response.output_audio.done": "response.audio.done", - "response.output_audio_transcript.delta": "response.audio_transcript.delta", - "response.output_audio_transcript.done": "response.audio_transcript.done", - "conversation.item.added": "conversation.item.created", -}; - -const BETA_SUPPRESSED_EVENTS = new Set(["conversation.item.done"]); - // --------------------------------------------------------------------------- // Server lifecycle // --------------------------------------------------------------------------- @@ -135,7 +119,7 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime API drift", () => { const sdkEvents = openaiRealtimeTextEventShapes(); // Real API — GA mode (no Beta header) - const realResult = await openaiRealtimeWS(config, "Say hello", undefined, false); + const realResult = await openaiRealtimeWS(config, "Say hello", undefined); // Mock — replicate the Realtime protocol sequence (GA mode) const mockWs = await connectWebSocket(instance.url, "/v1/realtime"); @@ -234,7 +218,7 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime API drift", () => { ]; // Real API — GA mode - const realResult = await openaiRealtimeWS(config, "Weather in Paris", tools, false); + const realResult = await openaiRealtimeWS(config, "Weather in Paris", tools); // Mock — replicate the Realtime protocol sequence const mockWs = await connectWebSocket(instance.url, "/v1/realtime"); @@ -305,30 +289,10 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime API drift", () => { ).toEqual([]); }); - it.skipIf(!OPENAI_REALTIME_CREDENTIAL)( - "GA and Beta event sequences are consistent after normalization", - async () => { - // GA connection (no Beta header) - const gaResult = await openaiRealtimeWS(config, "Say hello in one word.", undefined, false); - - // Beta connection - const betaResult = await openaiRealtimeWS(config, "Say hello in one word.", undefined, true); - - // Normalize GA events to Beta names for comparison - const gaToComparable = (type: string) => GA_TO_BETA_EVENT[type] ?? type; - - const gaTypes = gaResult.events - .map((e) => e.type) - .filter((t) => !BETA_SUPPRESSED_EVENTS.has(t)) - .map(gaToComparable); - const betaTypes = betaResult.events.map((e) => e.type); - - // Deduplicate consecutive repeated types so that differences in delta - // count (non-deterministic LLM output length) don't cause false failures. - function dedupeConsecutive(types: string[]): string[] { - return types.filter((t, i) => i === 0 || t !== types[i - 1]); - } - expect(dedupeConsecutive(gaTypes)).toEqual(dedupeConsecutive(betaTypes)); - }, - ); + // NOTE: A GA<->Beta event-sequence consistency probe used to live here. It was + // removed because OpenAI RETIRED the Realtime Beta API — a live Beta handshake + // now returns {"code":"beta_api_shape_disabled","message":"The Realtime Beta + // API is no longer supported. Please use /v1/realtime for the GA API."}, which + // quarantined the whole leg (exit 5). GA is the only live surface and the only + // surface aimock mocks, so the drift suite probes GA exclusively. });