From 6abd51500a5911fb39fa82e65f33934d22df3eed Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 8 Jul 2026 11:49:21 -0700 Subject: [PATCH 1/2] fix(drift): flag new voice/audio model families beyond the "realtime" substring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ws-realtime known-models canary filtered GET /v1/models with `models.filter((m) => m.includes("realtime"))`, so a NEW full-duplex voice family whose id lacks the "realtime" substring (e.g. OpenAI's gpt-live-1 / gpt-live-1-mini) never entered the unknown-model computation and slipped past the canary silently. The general models.drift.ts canary only detects deprecation of already-referenced models, not the arrival of new ones — this was the blind spot. Extract the detection into a side-effect-free module (voice-models.ts) with a broadened voice/audio family matcher (realtime|audio|live|transcribe|whisper| voice|tts) and a knownVoiceModels allowlist. Both the live canary and a new unit test drive the SAME detectVoiceModelDrift code path. The canary's critical -vs-crash markers (NO_GA_REALTIME_MODELS= / UNKNOWN_REALTIME_MODELS=, added in 1e3e638) are preserved verbatim, so the drift collector still emits exit-2 critical drift rather than crashing. Generalizes beyond gpt-live: any unseen voice/audio family is flagged the first time it appears; chat/image/embedding models never become false positives. --- src/__tests__/drift/voice-models.ts | 101 ++++++++++++++++++++ src/__tests__/drift/ws-realtime.drift.ts | 45 ++------- src/__tests__/ws-realtime-canary.test.ts | 112 +++++++++++++++++++++++ 3 files changed, 220 insertions(+), 38 deletions(-) create mode 100644 src/__tests__/drift/voice-models.ts create mode 100644 src/__tests__/ws-realtime-canary.test.ts diff --git a/src/__tests__/drift/voice-models.ts b/src/__tests__/drift/voice-models.ts new file mode 100644 index 00000000..b422d308 --- /dev/null +++ b/src/__tests__/drift/voice-models.ts @@ -0,0 +1,101 @@ +/** + * Known voice/audio model set + drift detection for the OpenAI realtime canary. + * + * Extracted into its own side-effect-free module (no `describe`/`beforeAll`) so + * both the live canary in ws-realtime.drift.ts AND its unit test can import the + * SAME detection code path without the unit test transitively registering the + * drift suite (which would spin up the drift server). + */ + +/** + * The GA realtime model family. At least one of these MUST appear in the + * account's model list, otherwise the family was renamed/removed (NO_GA drift). + */ +export const gaRealtimeModels = [ + "gpt-realtime", + "gpt-realtime-2", + "gpt-realtime-2.1", + "gpt-realtime-2.1-mini", + "gpt-realtime-2025-08-28", + "gpt-realtime-1.5", + "gpt-realtime-mini", + "gpt-realtime-mini-2025-10-06", + "gpt-realtime-mini-2025-12-15", +]; + +/** + * The full set of voice/audio model ids we already know about. Any voice/audio + * model id NOT in this set is surfaced as new/unknown drift so a newly-shipped + * family is flagged the first time it appears on the account. + */ +export const knownVoiceModels = new Set([ + ...gaRealtimeModels, + // Translate/whisper models (also contain "realtime" in some variants) + "gpt-realtime-translate", + "gpt-realtime-whisper", + // Audio models also valid in realtime sessions + "gpt-audio", + "gpt-audio-1.5", + "gpt-audio-mini", + "gpt-audio-mini-2025-10-06", + "gpt-audio-mini-2025-12-15", + // Transcription/translation models + "gpt-4o-transcribe", + "gpt-4o-mini-transcribe", + "gpt-4o-transcribe-diarize", + "whisper-1", + // Legacy preview models (may still appear) + "gpt-4o-realtime-preview", + "gpt-4o-mini-realtime-preview", + "gpt-4o-realtime-preview-2024-10-01", + "gpt-4o-realtime-preview-2024-12-17", + "gpt-4o-realtime-preview-2025-06-03", + "gpt-4o-mini-realtime-preview-2024-12-17", + // TTS / speech-out models (voice family, no "realtime" substring) + "gpt-4o-mini-tts", + "tts-1", + "tts-1-hd", +]); + +/** + * Match a model id that belongs to the voice/audio family the realtime canary + * is responsible for. This is DELIBERATELY broader than the old + * `id.includes("realtime")` filter: a new full-duplex voice family whose id + * lacks the "realtime" substring (e.g. OpenAI's `gpt-live-1` / `gpt-live-1-mini`) + * would previously never enter the unknown-model computation and so slip past + * the canary silently. Matching on the broader voice/audio vocabulary closes + * that blind spot generally — the point is "a new audio/voice model family the + * account hasn't seen before gets flagged", not a one-off hardcode of gpt-live. + * + * Chat/text/image/embedding models (gpt-4o, gpt-5, dall-e, text-embedding-*, + * etc.) do NOT match, so they never become false-positive "unknown voice" drift. + */ +export function isVoiceModelId(id: string): boolean { + return /(?:realtime|audio|\blive\b|-live|transcribe|whisper|voice|\btts\b|-tts)/i.test(id); +} + +/** + * Result of running the known-voice-models drift detection over a model list. + */ +export interface VoiceModelDriftResult { + /** Every model id that matched the voice/audio family matcher. */ + candidateModels: string[]; + /** Voice/audio ids not present in knownVoiceModels — new/unknown drift. */ + unknown: string[]; + /** Whether at least one GA realtime model is present. */ + hasGA: boolean; +} + +/** + * The single detection code path shared by the live canary AND its unit test. + * Given a raw `GET /v1/models` id list, compute the voice/audio candidates, the + * unknown (new-family) subset, and GA presence. Keeping this pure lets the unit + * test drive the EXACT logic the live canary runs against a representative + * payload without a network call. + */ +export function detectVoiceModelDrift(models: string[]): VoiceModelDriftResult { + const candidateModels = models.filter(isVoiceModelId); + const unknown = candidateModels.filter((m) => !knownVoiceModels.has(m)); + const hasGA = candidateModels.some((m) => gaRealtimeModels.includes(m)); + return { candidateModels, unknown, hasGA }; +} diff --git a/src/__tests__/drift/ws-realtime.drift.ts b/src/__tests__/drift/ws-realtime.drift.ts index 2c312f12..fdb271cb 100644 --- a/src/__tests__/drift/ws-realtime.drift.ts +++ b/src/__tests__/drift/ws-realtime.drift.ts @@ -11,6 +11,7 @@ import { extractShape, compareSSESequences, formatDriftReport } from "./schema.j import { openaiRealtimeTextEventShapes, openaiRealtimeToolCallEventShapes } from "./sdk-shapes.js"; import { openaiRealtimeWS } from "./ws-providers.js"; import { listOpenAIModels } from "./providers.js"; +import { detectVoiceModelDrift } from "./voice-models.js"; import { startDriftServer, stopDriftServer, collectMockWSMessages } from "./helpers.js"; import { connectWebSocket } from "../ws-test-client.js"; @@ -74,41 +75,11 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime API drift", () => { // silently skipped the one check this whole suite exists to run. const models = await listOpenAIModels(OPENAI_API_KEY!); - const gaModels = [ - "gpt-realtime", - "gpt-realtime-2", - "gpt-realtime-2.1", - "gpt-realtime-2.1-mini", - "gpt-realtime-2025-08-28", - "gpt-realtime-1.5", - "gpt-realtime-mini", - "gpt-realtime-mini-2025-10-06", - "gpt-realtime-mini-2025-12-15", - ]; - const knownModels = new Set([ - ...gaModels, - // Translate/whisper models (also contain "realtime" in some variants) - "gpt-realtime-translate", - "gpt-realtime-whisper", - // Audio models also valid in realtime sessions - "gpt-audio-1.5", - "gpt-audio-mini", - "gpt-audio-mini-2025-10-06", - "gpt-audio-mini-2025-12-15", - // Transcription/translation models - "gpt-4o-transcribe", - "gpt-4o-mini-transcribe", - "whisper-1", - // Legacy preview models (may still appear) - "gpt-4o-realtime-preview", - "gpt-4o-mini-realtime-preview", - "gpt-4o-realtime-preview-2024-10-01", - "gpt-4o-realtime-preview-2024-12-17", - "gpt-4o-realtime-preview-2025-06-03", - "gpt-4o-mini-realtime-preview-2024-12-17", - ]); - - const realtimeModels = models.filter((m) => m.includes("realtime")); + // Run the SHARED detection code path (also driven directly by the unit test + // in ws-realtime-canary.test.ts). The voice/audio family matcher is broader + // than the old `includes("realtime")` filter so a NEW voice family whose id + // lacks the "realtime" substring (e.g. gpt-live-1) is still flagged. + const { candidateModels: realtimeModels, unknown, hasGA } = detectVoiceModelDrift(models); // Compute the unknown-model list BEFORE the hasGA assertion. A run can be // BOTH GA-family-gone AND carry new unknown models; because the hasGA @@ -116,9 +87,8 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime API drift", () => { // never run and its list would be lost from the NO_GA failure message (and // therefore from the auto-fix prompt). So we carry the unknown list into the // NO_GA marker too — no information is lost in the combined case. - const unknown = realtimeModels.filter((m) => !knownModels.has(m)); if (unknown.length > 0) { - console.warn(`[DRIFT] Unknown realtime models detected: ${unknown.join(", ")}`); + console.warn(`[DRIFT] Unknown voice/audio models detected: ${unknown.join(", ")}`); } // At least one GA model should exist. Carry the OBSERVED realtime models in @@ -134,7 +104,6 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime API drift", () => { // segment so the combined (no-GA AND unknown-models-present) case does not // lose the unknown list when this assertion short-circuits the one below. // The collector splits the two markers apart; each list stays clean. - const hasGA = realtimeModels.some((m) => gaModels.includes(m)); expect( hasGA, `NO_GA_REALTIME_MODELS=${realtimeModels.join(",")} | UNKNOWN_REALTIME_MODELS=${unknown.join(",")}`, diff --git a/src/__tests__/ws-realtime-canary.test.ts b/src/__tests__/ws-realtime-canary.test.ts new file mode 100644 index 00000000..33d3cf01 --- /dev/null +++ b/src/__tests__/ws-realtime-canary.test.ts @@ -0,0 +1,112 @@ +/** + * Unit test for the ws-realtime known-voice-models drift canary. + * + * This drives the SAME detection code path the live canary runs + * (`detectVoiceModelDrift` from ws-realtime.drift.ts) against a representative + * `GET /v1/models` id list — no network call, but NOT a reimplemented fake: the + * exported function under test IS the one the live canary invokes. + * + * Regression guard for the voice-model-family blind spot: the canary used to + * filter the model list with `models.filter((m) => m.includes("realtime"))`, so + * a NEW full-duplex voice family whose id lacks the "realtime" substring (e.g. + * OpenAI's gpt-live-1 / gpt-live-1-mini) never entered the unknown-model + * computation and slipped past the canary silently. The broadened voice/audio + * matcher closes that blind spot generally. + */ + +import { describe, it, expect } from "vitest"; +import { detectVoiceModelDrift, isVoiceModelId, knownVoiceModels } from "./drift/voice-models.js"; + +// A representative GET /v1/models id list: the known GA realtime family, some +// known audio/transcribe/tts models, a batch of non-voice chat/image/embedding +// models that MUST NOT be flagged, and the newly-shipped gpt-live-* full-duplex +// voice family that the old "realtime"-substring filter would have missed. +const REPRESENTATIVE_MODELS = [ + // --- known voice/audio family (should stay green) --- + "gpt-realtime", + "gpt-realtime-2.1", + "gpt-realtime-mini", + "gpt-audio-mini", + "gpt-4o-transcribe", + "whisper-1", + "gpt-4o-realtime-preview", + "tts-1", + // --- non-voice models (must NOT be flagged as voice drift) --- + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "o1", + "o3-mini", + "chatgpt-4o-latest", + "dall-e-3", + "text-embedding-3-large", + "text-embedding-3-small", + "omni-moderation-latest", + // --- NEW voice family with no "realtime" substring (the blind spot) --- + "gpt-live-1", + "gpt-live-1-mini", +]; + +describe("ws-realtime known-voice-models canary detection", () => { + it("flags a new voice family whose id lacks the 'realtime' substring (gpt-live-*)", () => { + const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + + // The whole point: the new gpt-live-* family is surfaced as unknown drift. + expect(unknown).toContain("gpt-live-1"); + expect(unknown).toContain("gpt-live-1-mini"); + }); + + it("does not flag legitimately-known voice/audio models (no false positives)", () => { + const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + + // Every known-voice model in the payload stays green. + for (const known of [ + "gpt-realtime", + "gpt-realtime-2.1", + "gpt-realtime-mini", + "gpt-audio-mini", + "gpt-4o-transcribe", + "whisper-1", + "gpt-4o-realtime-preview", + "tts-1", + ]) { + expect(knownVoiceModels.has(known)).toBe(true); + expect(unknown).not.toContain(known); + } + }); + + it("does not flag non-voice chat/image/embedding models as voice drift", () => { + const { candidateModels, unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + + for (const nonVoice of [ + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "o1", + "o3-mini", + "chatgpt-4o-latest", + "dall-e-3", + "text-embedding-3-large", + "text-embedding-3-small", + "omni-moderation-latest", + ]) { + expect(isVoiceModelId(nonVoice)).toBe(false); + expect(candidateModels).not.toContain(nonVoice); + expect(unknown).not.toContain(nonVoice); + } + }); + + it("reports GA realtime presence when a GA model exists", () => { + const { hasGA } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + expect(hasGA).toBe(true); + }); + + it("gpt-live-* is the ONLY unknown in the representative payload", () => { + // Confirms the matcher is neither too narrow (misses gpt-live) nor too broad + // (drags in non-voice models). The unknown set is exactly the new family. + const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + expect([...unknown].sort()).toEqual(["gpt-live-1", "gpt-live-1-mini"]); + }); +}); From 2e57ec61d93f792ee05b23f161a9e44a7fb2e393 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 8 Jul 2026 12:09:21 -0700 Subject: [PATCH 2/2] fix(drift): normalize voice model ids to family keys to kill snapshot false positives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The broadened voice/audio matcher correctly widened coverage to catch new families like gpt-live, but it compared full model ids against a known-ID set. OpenAI ships dated snapshots of already-known families constantly, so the live Drift Tests run (28968203340) flagged 7 legitimate snapshots as critical drift (tts-1-1106, tts-1-hd-1106, gpt-audio-2025-08-28, gpt-4o-mini-transcribe-*, gpt-4o-mini-tts-*). Appending each new snapshot never converges — the daily job would go red every day and bury a real new-family signal. Fix: normalize each candidate id to a FAMILY KEY by stripping trailing dated snapshot (-YYYY-MM-DD) and build-tag (-NNN/-NNNN) suffixes, then compare the normalized family against a set of known FAMILIES. Dated snapshots of a known family collapse onto it and stay green; a genuinely new family (gpt-live, whose single-digit -1 is deliberately not stripped) still normalizes to an unknown family and is flagged. The NO_GA_REALTIME_MODELS= / UNKNOWN_REALTIME_MODELS= markers in ws-realtime.drift.ts are preserved. --- src/__tests__/drift/voice-models.ts | 135 ++++++++++++++++------- src/__tests__/ws-realtime-canary.test.ts | 60 +++++++++- 2 files changed, 149 insertions(+), 46 deletions(-) diff --git a/src/__tests__/drift/voice-models.ts b/src/__tests__/drift/voice-models.ts index b422d308..e1e28166 100644 --- a/src/__tests__/drift/voice-models.ts +++ b/src/__tests__/drift/voice-models.ts @@ -1,5 +1,8 @@ /** - * Known voice/audio model set + drift detection for the OpenAI realtime canary. + * Known voice/audio model FAMILIES + drift detection for the OpenAI realtime + * canary. Detection compares each candidate's NORMALIZED family (trailing dated + * snapshot / build-tag suffixes stripped) against a known-family set, so dated + * snapshots of a known family don't churn as false-positive drift. * * Extracted into its own side-effect-free module (no `describe`/`beforeAll`) so * both the live canary in ws-realtime.drift.ts AND its unit test can import the @@ -8,54 +11,96 @@ */ /** - * The GA realtime model family. At least one of these MUST appear in the - * account's model list, otherwise the family was renamed/removed (NO_GA drift). + * The GA realtime model FAMILIES. At least one voice/audio model whose + * normalized family (see `normalizeVoiceModelFamily`) is one of these MUST + * appear in the account's model list, otherwise the family was renamed/removed + * (NO_GA drift). Entries are already normalized family keys — dated snapshots + * such as `gpt-realtime-2025-08-28` collapse onto `gpt-realtime` and so match + * here without being listed separately. */ export const gaRealtimeModels = [ "gpt-realtime", "gpt-realtime-2", "gpt-realtime-2.1", "gpt-realtime-2.1-mini", - "gpt-realtime-2025-08-28", "gpt-realtime-1.5", "gpt-realtime-mini", - "gpt-realtime-mini-2025-10-06", - "gpt-realtime-mini-2025-12-15", ]; /** - * The full set of voice/audio model ids we already know about. Any voice/audio - * model id NOT in this set is surfaced as new/unknown drift so a newly-shipped - * family is flagged the first time it appears on the account. + * Normalize a model id to its FAMILY KEY by stripping trailing version/snapshot + * suffixes that OpenAI appends to already-known families. New dated snapshots of + * an existing family land constantly (`tts-1-1106`, `gpt-audio-2025-08-28`, + * `gpt-4o-mini-tts-2025-12-15`, …); appending every one to a known-ID set never + * converges and turns the daily drift job permanently red on false positives. + * Comparing the NORMALIZED family instead means only a genuinely new family + * (e.g. `gpt-live`) is ever flagged. + * + * Two suffix shapes are stripped, repeatedly, from the END of the id: + * - a dated snapshot `-YYYY-MM-DD` (e.g. `-2025-08-28`) + * - a build/version tag `-NNN` or `-NNNN` (3–4 digits, e.g. `-1106`) + * + * Both are anchored to the end and applied in a loop so a trailing dated + * snapshot that itself follows a build tag is fully reduced. A short numeric + * suffix like `gpt-live-1`'s trailing `-1` is a SINGLE digit and is deliberately + * NOT stripped, so `gpt-live-1` normalizes to `gpt-live-1` — an unknown family — + * and stays flagged (the whole point of the canary). + */ +const DATED_SNAPSHOT_SUFFIX = /-\d{4}-\d{2}-\d{2}$/; +const BUILD_TAG_SUFFIX = /-\d{3,4}$/; + +export function normalizeVoiceModelFamily(id: string): string { + let family = id; + for (;;) { + const stripped = family.replace(DATED_SNAPSHOT_SUFFIX, "").replace(BUILD_TAG_SUFFIX, ""); + if (stripped === family) break; + family = stripped; + } + return family; +} + +/** + * The set of voice/audio model FAMILIES we already know about, keyed by the + * normalized family (see `normalizeVoiceModelFamily`). A voice/audio model whose + * NORMALIZED family is not in this set is surfaced as new/unknown drift, so a + * newly-shipped family (e.g. `gpt-live`) is flagged the first time it appears — + * while dated snapshots of a known family (e.g. `gpt-audio-2025-08-28`) collapse + * onto their family and stay green. + * + * The listed ids are the family keys; the seed values are already normalized + * (they carry no dated/build suffix), so building the set through + * `normalizeVoiceModelFamily` is idempotent and keeps the two in lockstep. */ -export const knownVoiceModels = new Set([ - ...gaRealtimeModels, - // Translate/whisper models (also contain "realtime" in some variants) - "gpt-realtime-translate", - "gpt-realtime-whisper", - // Audio models also valid in realtime sessions - "gpt-audio", - "gpt-audio-1.5", - "gpt-audio-mini", - "gpt-audio-mini-2025-10-06", - "gpt-audio-mini-2025-12-15", - // Transcription/translation models - "gpt-4o-transcribe", - "gpt-4o-mini-transcribe", - "gpt-4o-transcribe-diarize", - "whisper-1", - // Legacy preview models (may still appear) - "gpt-4o-realtime-preview", - "gpt-4o-mini-realtime-preview", - "gpt-4o-realtime-preview-2024-10-01", - "gpt-4o-realtime-preview-2024-12-17", - "gpt-4o-realtime-preview-2025-06-03", - "gpt-4o-mini-realtime-preview-2024-12-17", - // TTS / speech-out models (voice family, no "realtime" substring) - "gpt-4o-mini-tts", - "tts-1", - "tts-1-hd", -]); +export const knownVoiceModelFamilies = new Set( + [ + // GA realtime family (dated/versioned variants normalize onto these). + "gpt-realtime", + "gpt-realtime-2", + "gpt-realtime-2.1", + "gpt-realtime-2.1-mini", + "gpt-realtime-1.5", + "gpt-realtime-mini", + // Translate/whisper realtime variants + "gpt-realtime-translate", + "gpt-realtime-whisper", + // Audio models also valid in realtime sessions + "gpt-audio", + "gpt-audio-1.5", + "gpt-audio-mini", + // Transcription/translation models + "gpt-4o-transcribe", + "gpt-4o-mini-transcribe", + "gpt-4o-transcribe-diarize", + "whisper-1", + // Legacy preview models (may still appear) + "gpt-4o-realtime-preview", + "gpt-4o-mini-realtime-preview", + // TTS / speech-out models (voice family, no "realtime" substring) + "gpt-4o-mini-tts", + "tts-1", + "tts-1-hd", + ].map(normalizeVoiceModelFamily), +); /** * Match a model id that belongs to the voice/audio family the realtime canary @@ -80,9 +125,13 @@ export function isVoiceModelId(id: string): boolean { export interface VoiceModelDriftResult { /** Every model id that matched the voice/audio family matcher. */ candidateModels: string[]; - /** Voice/audio ids not present in knownVoiceModels — new/unknown drift. */ + /** + * Voice/audio ids whose NORMALIZED family is not in knownVoiceModelFamilies — + * new/unknown drift. Dated snapshots of a known family (e.g. + * `gpt-audio-2025-08-28`) collapse onto their family and are NOT listed. + */ unknown: string[]; - /** Whether at least one GA realtime model is present. */ + /** Whether at least one GA realtime model (by family) is present. */ hasGA: boolean; } @@ -95,7 +144,11 @@ export interface VoiceModelDriftResult { */ export function detectVoiceModelDrift(models: string[]): VoiceModelDriftResult { const candidateModels = models.filter(isVoiceModelId); - const unknown = candidateModels.filter((m) => !knownVoiceModels.has(m)); - const hasGA = candidateModels.some((m) => gaRealtimeModels.includes(m)); + const unknown = candidateModels.filter( + (m) => !knownVoiceModelFamilies.has(normalizeVoiceModelFamily(m)), + ); + const hasGA = candidateModels.some((m) => + gaRealtimeModels.includes(normalizeVoiceModelFamily(m)), + ); return { candidateModels, unknown, hasGA }; } diff --git a/src/__tests__/ws-realtime-canary.test.ts b/src/__tests__/ws-realtime-canary.test.ts index 33d3cf01..7a6cacd1 100644 --- a/src/__tests__/ws-realtime-canary.test.ts +++ b/src/__tests__/ws-realtime-canary.test.ts @@ -15,12 +15,33 @@ */ import { describe, it, expect } from "vitest"; -import { detectVoiceModelDrift, isVoiceModelId, knownVoiceModels } from "./drift/voice-models.js"; +import { + detectVoiceModelDrift, + isVoiceModelId, + knownVoiceModelFamilies, + normalizeVoiceModelFamily, +} from "./drift/voice-models.js"; + +// Dated-snapshot / build-tag variants of families that are ALREADY known. These +// are the exact ids the live Drift Tests run (28968203340) flagged as false +// positives before family normalization: each normalizes onto a known family +// (tts-1, tts-1-hd, gpt-audio, gpt-4o-mini-transcribe, gpt-4o-mini-tts) and so +// MUST NOT be flagged. +const KNOWN_FAMILY_SNAPSHOTS = [ + "tts-1-1106", + "tts-1-hd-1106", + "gpt-audio-2025-08-28", + "gpt-4o-mini-transcribe-2025-12-15", + "gpt-4o-mini-transcribe-2025-03-20", + "gpt-4o-mini-tts-2025-03-20", + "gpt-4o-mini-tts-2025-12-15", +]; // A representative GET /v1/models id list: the known GA realtime family, some -// known audio/transcribe/tts models, a batch of non-voice chat/image/embedding -// models that MUST NOT be flagged, and the newly-shipped gpt-live-* full-duplex -// voice family that the old "realtime"-substring filter would have missed. +// known audio/transcribe/tts models (incl. dated snapshots of known families +// that MUST normalize onto their family), a batch of non-voice +// chat/image/embedding models that MUST NOT be flagged, and the newly-shipped +// gpt-live-* full-duplex voice family that is a genuinely new family. const REPRESENTATIVE_MODELS = [ // --- known voice/audio family (should stay green) --- "gpt-realtime", @@ -31,6 +52,8 @@ const REPRESENTATIVE_MODELS = [ "whisper-1", "gpt-4o-realtime-preview", "tts-1", + // --- dated-snapshot / build-tag variants of KNOWN families (stay green) --- + ...KNOWN_FAMILY_SNAPSHOTS, // --- non-voice models (must NOT be flagged as voice drift) --- "gpt-4o", "gpt-4o-mini", @@ -71,11 +94,38 @@ describe("ws-realtime known-voice-models canary detection", () => { "gpt-4o-realtime-preview", "tts-1", ]) { - expect(knownVoiceModels.has(known)).toBe(true); + expect(knownVoiceModelFamilies.has(normalizeVoiceModelFamily(known))).toBe(true); expect(unknown).not.toContain(known); } }); + it("does not flag dated-snapshot / build-tag variants of known families", () => { + // Regression guard for live Drift Tests run 28968203340: these 7 real ids + // were flagged as critical drift by the pre-normalization id-set matcher. + // Family normalization collapses each onto an already-known family, so none + // may appear in `unknown`. + const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + for (const snapshot of KNOWN_FAMILY_SNAPSHOTS) { + expect(unknown).not.toContain(snapshot); + } + }); + + it("normalizes dated snapshots and build tags onto their family", () => { + // Dated snapshot `-YYYY-MM-DD` and build tag `-NNN`/`-NNNN` are stripped. + expect(normalizeVoiceModelFamily("tts-1-1106")).toBe("tts-1"); + expect(normalizeVoiceModelFamily("tts-1-hd-1106")).toBe("tts-1-hd"); + expect(normalizeVoiceModelFamily("gpt-audio-2025-08-28")).toBe("gpt-audio"); + expect(normalizeVoiceModelFamily("gpt-4o-mini-transcribe-2025-12-15")).toBe( + "gpt-4o-mini-transcribe", + ); + expect(normalizeVoiceModelFamily("gpt-4o-mini-tts-2025-03-20")).toBe("gpt-4o-mini-tts"); + // A single-digit trailing tag is NOT a build tag: gpt-live-1 stays a new + // family and must remain flaggable. + expect(normalizeVoiceModelFamily("gpt-live-1")).toBe("gpt-live-1"); + expect(normalizeVoiceModelFamily("gpt-live-1-mini")).toBe("gpt-live-1-mini"); + expect(knownVoiceModelFamilies.has("gpt-live-1")).toBe(false); + }); + it("does not flag non-voice chat/image/embedding models as voice drift", () => { const { candidateModels, unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS);