fix(drift): flag new voice/audio model families beyond the "realtime" substring

src/__tests__/drift/voice-models.ts

@@ -0,0 +1,154 @@
+/**
+ * Known voice/audio model FAMILIES + drift detection for the OpenAI realtime
+ * canary. Detection compares each candidate's NORMALIZED family (trailing dated
+ * snapshot / build-tag suffixes stripped) against a known-family set, so dated
+ * snapshots of a known family don't churn as false-positive drift.
+ *
+ * Extracted into its own side-effect-free module (no `describe`/`beforeAll`) so
+ * both the live canary in ws-realtime.drift.ts AND its unit test can import the
+ * SAME detection code path without the unit test transitively registering the
+ * drift suite (which would spin up the drift server).
+ */
+
+/**
+ * The GA realtime model FAMILIES. At least one voice/audio model whose
+ * normalized family (see `normalizeVoiceModelFamily`) is one of these MUST
+ * appear in the account's model list, otherwise the family was renamed/removed
+ * (NO_GA drift). Entries are already normalized family keys — dated snapshots
+ * such as `gpt-realtime-2025-08-28` collapse onto `gpt-realtime` and so match
+ * here without being listed separately.
+ */
+export const gaRealtimeModels = [
+  "gpt-realtime",
+  "gpt-realtime-2",
+  "gpt-realtime-2.1",
+  "gpt-realtime-2.1-mini",
+  "gpt-realtime-1.5",
+  "gpt-realtime-mini",
+];
+
+/**
+ * Normalize a model id to its FAMILY KEY by stripping trailing version/snapshot
+ * suffixes that OpenAI appends to already-known families. New dated snapshots of
+ * an existing family land constantly (`tts-1-1106`, `gpt-audio-2025-08-28`,
+ * `gpt-4o-mini-tts-2025-12-15`, …); appending every one to a known-ID set never
+ * converges and turns the daily drift job permanently red on false positives.
+ * Comparing the NORMALIZED family instead means only a genuinely new family
+ * (e.g. `gpt-live`) is ever flagged.
+ *
+ * Two suffix shapes are stripped, repeatedly, from the END of the id:
+ *   - a dated snapshot `-YYYY-MM-DD`  (e.g. `-2025-08-28`)
+ *   - a build/version tag `-NNN` or `-NNNN`  (3–4 digits, e.g. `-1106`)
+ *
+ * Both are anchored to the end and applied in a loop so a trailing dated
+ * snapshot that itself follows a build tag is fully reduced. A short numeric
+ * suffix like `gpt-live-1`'s trailing `-1` is a SINGLE digit and is deliberately
+ * NOT stripped, so `gpt-live-1` normalizes to `gpt-live-1` — an unknown family —
+ * and stays flagged (the whole point of the canary).
+ */
+const DATED_SNAPSHOT_SUFFIX = /-\d{4}-\d{2}-\d{2}$/;
+const BUILD_TAG_SUFFIX = /-\d{3,4}$/;
+
+export function normalizeVoiceModelFamily(id: string): string {
+  let family = id;
+  for (;;) {
+    const stripped = family.replace(DATED_SNAPSHOT_SUFFIX, "").replace(BUILD_TAG_SUFFIX, "");
+    if (stripped === family) break;
+    family = stripped;
+  }
+  return family;
+}
+
+/**
+ * The set of voice/audio model FAMILIES we already know about, keyed by the
+ * normalized family (see `normalizeVoiceModelFamily`). A voice/audio model whose
+ * NORMALIZED family is not in this set is surfaced as new/unknown drift, so a
+ * newly-shipped family (e.g. `gpt-live`) is flagged the first time it appears —
+ * while dated snapshots of a known family (e.g. `gpt-audio-2025-08-28`) collapse
+ * onto their family and stay green.
+ *
+ * The listed ids are the family keys; the seed values are already normalized
+ * (they carry no dated/build suffix), so building the set through
+ * `normalizeVoiceModelFamily` is idempotent and keeps the two in lockstep.
+ */
+export const knownVoiceModelFamilies = new Set(
+  [
+    // GA realtime family (dated/versioned variants normalize onto these).
+    "gpt-realtime",
+    "gpt-realtime-2",
+    "gpt-realtime-2.1",
+    "gpt-realtime-2.1-mini",
+    "gpt-realtime-1.5",
+    "gpt-realtime-mini",
+    // Translate/whisper realtime variants
+    "gpt-realtime-translate",
+    "gpt-realtime-whisper",
+    // Audio models also valid in realtime sessions
+    "gpt-audio",
+    "gpt-audio-1.5",
+    "gpt-audio-mini",
+    // Transcription/translation models
+    "gpt-4o-transcribe",
+    "gpt-4o-mini-transcribe",
+    "gpt-4o-transcribe-diarize",
+    "whisper-1",
+    // Legacy preview models (may still appear)
+    "gpt-4o-realtime-preview",
+    "gpt-4o-mini-realtime-preview",
+    // TTS / speech-out models (voice family, no "realtime" substring)
+    "gpt-4o-mini-tts",
+    "tts-1",
+    "tts-1-hd",
+  ].map(normalizeVoiceModelFamily),
+);
+
+/**
+ * Match a model id that belongs to the voice/audio family the realtime canary
+ * is responsible for. This is DELIBERATELY broader than the old
+ * `id.includes("realtime")` filter: a new full-duplex voice family whose id
+ * lacks the "realtime" substring (e.g. OpenAI's `gpt-live-1` / `gpt-live-1-mini`)
+ * would previously never enter the unknown-model computation and so slip past
+ * the canary silently. Matching on the broader voice/audio vocabulary closes
+ * that blind spot generally — the point is "a new audio/voice model family the
+ * account hasn't seen before gets flagged", not a one-off hardcode of gpt-live.
+ *
+ * Chat/text/image/embedding models (gpt-4o, gpt-5, dall-e, text-embedding-*,
+ * etc.) do NOT match, so they never become false-positive "unknown voice" drift.
+ */
+export function isVoiceModelId(id: string): boolean {
+  return /(?:realtime|audio|\blive\b|-live|transcribe|whisper|voice|\btts\b|-tts)/i.test(id);
+}
+
+/**
+ * Result of running the known-voice-models drift detection over a model list.
+ */
+export interface VoiceModelDriftResult {
+  /** Every model id that matched the voice/audio family matcher. */
+  candidateModels: string[];
+  /**
+   * Voice/audio ids whose NORMALIZED family is not in knownVoiceModelFamilies —
+   * new/unknown drift. Dated snapshots of a known family (e.g.
+   * `gpt-audio-2025-08-28`) collapse onto their family and are NOT listed.
+   */
+  unknown: string[];
+  /** Whether at least one GA realtime model (by family) is present. */
+  hasGA: boolean;
+}
+
+/**
+ * The single detection code path shared by the live canary AND its unit test.
+ * Given a raw `GET /v1/models` id list, compute the voice/audio candidates, the
+ * unknown (new-family) subset, and GA presence. Keeping this pure lets the unit
+ * test drive the EXACT logic the live canary runs against a representative
+ * payload without a network call.
+ */
+export function detectVoiceModelDrift(models: string[]): VoiceModelDriftResult {
+  const candidateModels = models.filter(isVoiceModelId);
+  const unknown = candidateModels.filter(
+    (m) => !knownVoiceModelFamilies.has(normalizeVoiceModelFamily(m)),
+  );
+  const hasGA = candidateModels.some((m) =>
+    gaRealtimeModels.includes(normalizeVoiceModelFamily(m)),
+  );
+  return { candidateModels, unknown, hasGA };
+}

src/__tests__/drift/ws-realtime.drift.ts

@@ -11,6 +11,7 @@ import { extractShape, compareSSESequences, formatDriftReport } from "./schema.j
 import { openaiRealtimeTextEventShapes, openaiRealtimeToolCallEventShapes } from "./sdk-shapes.js";
 import { openaiRealtimeWS } from "./ws-providers.js";
 import { listOpenAIModels } from "./providers.js";
+import { detectVoiceModelDrift } from "./voice-models.js";
 import { startDriftServer, stopDriftServer, collectMockWSMessages } from "./helpers.js";
 import { connectWebSocket } from "../ws-test-client.js";
 
@@ -74,51 +75,20 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime API drift", () => {
     // silently skipped the one check this whole suite exists to run.
     const models = await listOpenAIModels(OPENAI_API_KEY!);
 
-    const gaModels = [
-      "gpt-realtime",
-      "gpt-realtime-2",
-      "gpt-realtime-2.1",
-      "gpt-realtime-2.1-mini",
-      "gpt-realtime-2025-08-28",
-      "gpt-realtime-1.5",
-      "gpt-realtime-mini",
-      "gpt-realtime-mini-2025-10-06",
-      "gpt-realtime-mini-2025-12-15",
-    ];
-    const knownModels = new Set([
-      ...gaModels,
-      // Translate/whisper models (also contain "realtime" in some variants)
-      "gpt-realtime-translate",
-      "gpt-realtime-whisper",
-      // Audio models also valid in realtime sessions
-      "gpt-audio-1.5",
-      "gpt-audio-mini",
-      "gpt-audio-mini-2025-10-06",
-      "gpt-audio-mini-2025-12-15",
-      // Transcription/translation models
-      "gpt-4o-transcribe",
-      "gpt-4o-mini-transcribe",
-      "whisper-1",
-      // Legacy preview models (may still appear)
-      "gpt-4o-realtime-preview",
-      "gpt-4o-mini-realtime-preview",
-      "gpt-4o-realtime-preview-2024-10-01",
-      "gpt-4o-realtime-preview-2024-12-17",
-      "gpt-4o-realtime-preview-2025-06-03",
-      "gpt-4o-mini-realtime-preview-2024-12-17",
-    ]);
-
-    const realtimeModels = models.filter((m) => m.includes("realtime"));
+    // Run the SHARED detection code path (also driven directly by the unit test
+    // in ws-realtime-canary.test.ts). The voice/audio family matcher is broader
+    // than the old `includes("realtime")` filter so a NEW voice family whose id
+    // lacks the "realtime" substring (e.g. gpt-live-1) is still flagged.
+    const { candidateModels: realtimeModels, unknown, hasGA } = detectVoiceModelDrift(models);
 
     // Compute the unknown-model list BEFORE the hasGA assertion. A run can be
     // BOTH GA-family-gone AND carry new unknown models; because the hasGA
     // assertion below throws first, the later unknown-models assertion would
     // never run and its list would be lost from the NO_GA failure message (and
     // therefore from the auto-fix prompt). So we carry the unknown list into the
     // NO_GA marker too — no information is lost in the combined case.
-    const unknown = realtimeModels.filter((m) => !knownModels.has(m));
     if (unknown.length > 0) {
-      console.warn(`[DRIFT] Unknown realtime models detected: ${unknown.join(", ")}`);
+      console.warn(`[DRIFT] Unknown voice/audio models detected: ${unknown.join(", ")}`);
     }
 
     // At least one GA model should exist. Carry the OBSERVED realtime models in
@@ -134,7 +104,6 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime API drift", () => {
     // segment so the combined (no-GA AND unknown-models-present) case does not
     // lose the unknown list when this assertion short-circuits the one below.
     // The collector splits the two markers apart; each list stays clean.
-    const hasGA = realtimeModels.some((m) => gaModels.includes(m));
     expect(
       hasGA,
       `NO_GA_REALTIME_MODELS=${realtimeModels.join(",")} | UNKNOWN_REALTIME_MODELS=${unknown.join(",")}`,

src/__tests__/ws-realtime-canary.test.ts

@@ -0,0 +1,162 @@
+/**
+ * Unit test for the ws-realtime known-voice-models drift canary.
+ *
+ * This drives the SAME detection code path the live canary runs
+ * (`detectVoiceModelDrift` from ws-realtime.drift.ts) against a representative
+ * `GET /v1/models` id list — no network call, but NOT a reimplemented fake: the
+ * exported function under test IS the one the live canary invokes.
+ *
+ * Regression guard for the voice-model-family blind spot: the canary used to
+ * filter the model list with `models.filter((m) => m.includes("realtime"))`, so
+ * a NEW full-duplex voice family whose id lacks the "realtime" substring (e.g.
+ * OpenAI's gpt-live-1 / gpt-live-1-mini) never entered the unknown-model
+ * computation and slipped past the canary silently. The broadened voice/audio
+ * matcher closes that blind spot generally.
+ */
+
+import { describe, it, expect } from "vitest";
+import {
+  detectVoiceModelDrift,
+  isVoiceModelId,
+  knownVoiceModelFamilies,
+  normalizeVoiceModelFamily,
+} from "./drift/voice-models.js";
+
+// Dated-snapshot / build-tag variants of families that are ALREADY known. These
+// are the exact ids the live Drift Tests run (28968203340) flagged as false
+// positives before family normalization: each normalizes onto a known family
+// (tts-1, tts-1-hd, gpt-audio, gpt-4o-mini-transcribe, gpt-4o-mini-tts) and so
+// MUST NOT be flagged.
+const KNOWN_FAMILY_SNAPSHOTS = [
+  "tts-1-1106",
+  "tts-1-hd-1106",
+  "gpt-audio-2025-08-28",
+  "gpt-4o-mini-transcribe-2025-12-15",
+  "gpt-4o-mini-transcribe-2025-03-20",
+  "gpt-4o-mini-tts-2025-03-20",
+  "gpt-4o-mini-tts-2025-12-15",
+];
+
+// A representative GET /v1/models id list: the known GA realtime family, some
+// known audio/transcribe/tts models (incl. dated snapshots of known families
+// that MUST normalize onto their family), a batch of non-voice
+// chat/image/embedding models that MUST NOT be flagged, and the newly-shipped
+// gpt-live-* full-duplex voice family that is a genuinely new family.
+const REPRESENTATIVE_MODELS = [
+  // --- known voice/audio family (should stay green) ---
+  "gpt-realtime",
+  "gpt-realtime-2.1",
+  "gpt-realtime-mini",
+  "gpt-audio-mini",
+  "gpt-4o-transcribe",
+  "whisper-1",
+  "gpt-4o-realtime-preview",
+  "tts-1",
+  // --- dated-snapshot / build-tag variants of KNOWN families (stay green) ---
+  ...KNOWN_FAMILY_SNAPSHOTS,
+  // --- non-voice models (must NOT be flagged as voice drift) ---
+  "gpt-4o",
+  "gpt-4o-mini",
+  "gpt-5",
+  "gpt-5-mini",
+  "o1",
+  "o3-mini",
+  "chatgpt-4o-latest",
+  "dall-e-3",
+  "text-embedding-3-large",
+  "text-embedding-3-small",
+  "omni-moderation-latest",
+  // --- NEW voice family with no "realtime" substring (the blind spot) ---
+  "gpt-live-1",
+  "gpt-live-1-mini",
+];
+
+describe("ws-realtime known-voice-models canary detection", () => {
+  it("flags a new voice family whose id lacks the 'realtime' substring (gpt-live-*)", () => {
+    const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS);
+
+    // The whole point: the new gpt-live-* family is surfaced as unknown drift.
+    expect(unknown).toContain("gpt-live-1");
+    expect(unknown).toContain("gpt-live-1-mini");
+  });
+
+  it("does not flag legitimately-known voice/audio models (no false positives)", () => {
+    const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS);
+
+    // Every known-voice model in the payload stays green.
+    for (const known of [
+      "gpt-realtime",
+      "gpt-realtime-2.1",
+      "gpt-realtime-mini",
+      "gpt-audio-mini",
+      "gpt-4o-transcribe",
+      "whisper-1",
+      "gpt-4o-realtime-preview",
+      "tts-1",
+    ]) {
+      expect(knownVoiceModelFamilies.has(normalizeVoiceModelFamily(known))).toBe(true);
+      expect(unknown).not.toContain(known);
+    }
+  });
+
+  it("does not flag dated-snapshot / build-tag variants of known families", () => {
+    // Regression guard for live Drift Tests run 28968203340: these 7 real ids
+    // were flagged as critical drift by the pre-normalization id-set matcher.
+    // Family normalization collapses each onto an already-known family, so none
+    // may appear in `unknown`.
+    const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS);
+    for (const snapshot of KNOWN_FAMILY_SNAPSHOTS) {
+      expect(unknown).not.toContain(snapshot);
+    }
+  });
+
+  it("normalizes dated snapshots and build tags onto their family", () => {
+    // Dated snapshot `-YYYY-MM-DD` and build tag `-NNN`/`-NNNN` are stripped.
+    expect(normalizeVoiceModelFamily("tts-1-1106")).toBe("tts-1");
+    expect(normalizeVoiceModelFamily("tts-1-hd-1106")).toBe("tts-1-hd");
+    expect(normalizeVoiceModelFamily("gpt-audio-2025-08-28")).toBe("gpt-audio");
+    expect(normalizeVoiceModelFamily("gpt-4o-mini-transcribe-2025-12-15")).toBe(
+      "gpt-4o-mini-transcribe",
+    );
+    expect(normalizeVoiceModelFamily("gpt-4o-mini-tts-2025-03-20")).toBe("gpt-4o-mini-tts");
+    // A single-digit trailing tag is NOT a build tag: gpt-live-1 stays a new
+    // family and must remain flaggable.
+    expect(normalizeVoiceModelFamily("gpt-live-1")).toBe("gpt-live-1");
+    expect(normalizeVoiceModelFamily("gpt-live-1-mini")).toBe("gpt-live-1-mini");
+    expect(knownVoiceModelFamilies.has("gpt-live-1")).toBe(false);
+  });
+
+  it("does not flag non-voice chat/image/embedding models as voice drift", () => {
+    const { candidateModels, unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS);
+
+    for (const nonVoice of [
+      "gpt-4o",
+      "gpt-4o-mini",
+      "gpt-5",
+      "gpt-5-mini",
+      "o1",
+      "o3-mini",
+      "chatgpt-4o-latest",
+      "dall-e-3",
+      "text-embedding-3-large",
+      "text-embedding-3-small",
+      "omni-moderation-latest",
+    ]) {
+      expect(isVoiceModelId(nonVoice)).toBe(false);
+      expect(candidateModels).not.toContain(nonVoice);
+      expect(unknown).not.toContain(nonVoice);
+    }
+  });
+
+  it("reports GA realtime presence when a GA model exists", () => {
+    const { hasGA } = detectVoiceModelDrift(REPRESENTATIVE_MODELS);
+    expect(hasGA).toBe(true);
+  });
+
+  it("gpt-live-* is the ONLY unknown in the representative payload", () => {
+    // Confirms the matcher is neither too narrow (misses gpt-live) nor too broad
+    // (drags in non-voice models). The unknown set is exactly the new family.
+    const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS);
+    expect([...unknown].sort()).toEqual(["gpt-live-1", "gpt-live-1-mini"]);
+  });
+});