diff --git a/.github/workflows/fix-drift.yml b/.github/workflows/fix-drift.yml index 5ea33041..de9456c0 100644 --- a/.github/workflows/fix-drift.yml +++ b/.github/workflows/fix-drift.yml @@ -103,6 +103,8 @@ jobs: OLLAMA_HOST: 127.0.0.1:11434 OLLAMA_MODEL: qwen2:0.5b FAL_KEY: ${{ secrets.FAL_KEY }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }} PRE_FIX_REPORT: ${{ runner.temp }}/drift-report.json run: | set +e @@ -241,6 +243,8 @@ jobs: OLLAMA_HOST: 127.0.0.1:11434 OLLAMA_MODEL: qwen2:0.5b FAL_KEY: ${{ secrets.FAL_KEY }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }} run: pnpm test:drift # Step 4c: Re-collect drift AUTHORITATIVELY against the LIVE SDK, writing @@ -270,6 +274,8 @@ jobs: OLLAMA_HOST: 127.0.0.1:11434 OLLAMA_MODEL: qwen2:0.5b FAL_KEY: ${{ secrets.FAL_KEY }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }} POST_FIX_REPORT: ${{ runner.temp }}/drift-report.post-fix.json run: | set +e diff --git a/.github/workflows/test-drift.yml b/.github/workflows/test-drift.yml index 4c97902b..0451767e 100644 --- a/.github/workflows/test-drift.yml +++ b/.github/workflows/test-drift.yml @@ -190,6 +190,8 @@ jobs: OLLAMA_HOST: 127.0.0.1:11434 OLLAMA_MODEL: qwen2:0.5b FAL_KEY: ${{ secrets.FAL_KEY }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }} run: | set +e npx tsx scripts/drift-retry.ts @@ -372,6 +374,8 @@ jobs: GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} FAL_KEY: ${{ secrets.FAL_KEY }} + COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }} steps: # HEAD checkout: the PR ref (default checkout behavior on pull_request). # persist-credentials:false — this leg never pushes; keeps the token off diff --git a/src/__tests__/drift/cohere-model.test.ts b/src/__tests__/drift/cohere-model.test.ts new file mode 100644 index 00000000..2337e668 --- /dev/null +++ b/src/__tests__/drift/cohere-model.test.ts @@ -0,0 +1,61 @@ +/** + * Unit tests for the Cohere live-leg model selection + infra classification. + * + * These exercise the exact logic that fixes the drift-live-pr quarantine: the + * live leg was hardcoded to `command-r-plus`, which Cohere retired 2026-04-04, + * so the real `/v2/chat` call returned 404 (model not found) and quarantined + * as exit 5. The selector must skip deprecated models and pick a live one. + */ + +import { describe, it, expect } from "vitest"; +import { isInfraStatus, selectCohereChatModel, type CohereModelEntry } from "./cohere-model.js"; + +describe("selectCohereChatModel", () => { + it("never selects a deprecated model (the command-r-plus 404 regression)", () => { + const models: CohereModelEntry[] = [ + { name: "command-r-plus", is_deprecated: true, endpoints: ["chat"] }, + { name: "command-a-03-2025", is_deprecated: false, endpoints: ["chat"] }, + ]; + const chosen = selectCohereChatModel(models); + expect(chosen).not.toBe("command-r-plus"); + expect(chosen).toBe("command-a-03-2025"); + }); + + it("prefers a stable default when present", () => { + const models: CohereModelEntry[] = [ + { name: "some-experimental-model", is_deprecated: false, endpoints: ["chat"] }, + { name: "command-a-03-2025", is_deprecated: false, endpoints: ["chat"] }, + ]; + expect(selectCohereChatModel(models)).toBe("command-a-03-2025"); + }); + + it("falls back to the first non-deprecated model when no preferred present", () => { + const models: CohereModelEntry[] = [ + { name: "command-r-plus", is_deprecated: true, endpoints: ["chat"] }, + { name: "north-mini-code-1-0", is_deprecated: false, endpoints: ["chat"] }, + ]; + expect(selectCohereChatModel(models)).toBe("north-mini-code-1-0"); + }); + + it("returns null when no usable chat model exists", () => { + const models: CohereModelEntry[] = [ + { name: "command-r-plus", is_deprecated: true, endpoints: ["chat"] }, + ]; + expect(selectCohereChatModel(models)).toBeNull(); + expect(selectCohereChatModel([])).toBeNull(); + }); +}); + +describe("isInfraStatus", () => { + it("classifies auth/credit/rate-limit/5xx as infra (honest-skip conditions)", () => { + for (const s of [401, 402, 403, 429, 500, 502, 503, 529]) { + expect(isInfraStatus(s)).toBe(true); + } + }); + + it("does NOT classify success or a genuine drift/probe error as infra", () => { + for (const s of [200, 400, 404, 422]) { + expect(isInfraStatus(s)).toBe(false); + } + }); +}); diff --git a/src/__tests__/drift/cohere-model.ts b/src/__tests__/drift/cohere-model.ts new file mode 100644 index 00000000..2db7d1a3 --- /dev/null +++ b/src/__tests__/drift/cohere-model.ts @@ -0,0 +1,52 @@ +/** + * Pure helpers for the Cohere live drift leg: chat-model selection and + * infra-status classification. + * + * Extracted from cohere.drift.ts so the selection/classification logic is + * unit-testable WITHOUT live credentials (the live describe block is + * credential-gated and cannot run locally). + */ + +/** Cohere REST base URL. */ +export const COHERE_BASE_URL = "https://api.cohere.com"; + +/** + * Environmental HTTP statuses that indicate a provider-side condition + * (auth / out-of-credit / rate-limit / upstream outage) rather than a real + * API-envelope drift. Mirrors the providers.ts InfraError classification: + * 401|403 → stale-key, 429 → rate-limited, 5xx → infra-transient; 402 → + * out-of-credit. + * + * The live leg converts these to an HONEST SKIP so a transient provider-side + * condition never quarantines the drift collector and reds the PR. A real + * shape drift (a 200 response with an unexpected envelope) is NEVER skipped. + */ +export function isInfraStatus(status: number): boolean { + return status === 401 || status === 402 || status === 403 || status === 429 || status >= 500; +} + +/** Minimal shape of a /v1/models listing entry we depend on. */ +export interface CohereModelEntry { + name?: string; + is_deprecated?: boolean; + endpoints?: string[]; +} + +/** + * Select a currently-valid, non-deprecated Cohere chat model from a + * `/v1/models?endpoint=chat` listing. + * + * Cohere retires model IDs on a schedule (e.g. `command-r-plus` was removed + * 2026-04-04), so a hardcoded model name is a standing 404 / quarantine risk. + * Self-selecting from the live listing makes the leg resilient to future + * deprecations. Prefers a stable, widely-available default when present, else + * falls back to the first non-deprecated chat model. Returns null when the + * listing exposes no usable chat model. + */ +export function selectCohereChatModel(models: CohereModelEntry[]): string | null { + const names = models + .filter((m) => typeof m.name === "string" && m.is_deprecated !== true) + .map((m) => m.name as string); + const preferred = ["command-a-03-2025", "command-r-08-2024", "command-r7b-12-2024"]; + return preferred.find((p) => names.includes(p)) ?? names[0] ?? null; +} diff --git a/src/__tests__/drift/cohere.drift.ts b/src/__tests__/drift/cohere.drift.ts index 3b882d99..ef6548cb 100644 --- a/src/__tests__/drift/cohere.drift.ts +++ b/src/__tests__/drift/cohere.drift.ts @@ -17,6 +17,12 @@ import { startDriftServer, stopDriftServer, } from "./helpers.js"; +import { + COHERE_BASE_URL, + isInfraStatus, + selectCohereChatModel, + type CohereModelEntry, +} from "./cohere-model.js"; // --------------------------------------------------------------------------- // Credentials check @@ -87,16 +93,17 @@ function cohereChatStreamChunkShape() { // --------------------------------------------------------------------------- async function cohereChatNonStreaming( + model: string, messages: { role: string; content: string }[], ): Promise<{ status: number; body: string }> { - const res = await fetch("https://api.cohere.com/v2/chat", { + const res = await fetch(`${COHERE_BASE_URL}/v2/chat`, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${COHERE_API_KEY}`, }, body: JSON.stringify({ - model: "command-r-plus", + model, messages, stream: false, max_tokens: 10, @@ -106,16 +113,17 @@ async function cohereChatNonStreaming( } async function cohereChatStreaming( + model: string, messages: { role: string; content: string }[], ): Promise<{ status: number; body: string }> { - const res = await fetch("https://api.cohere.com/v2/chat", { + const res = await fetch(`${COHERE_BASE_URL}/v2/chat`, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${COHERE_API_KEY}`, }, body: JSON.stringify({ - model: "command-r-plus", + model, messages, stream: true, max_tokens: 10, @@ -124,6 +132,45 @@ async function cohereChatStreaming( return { status: res.status, body: await res.text() }; } +// --------------------------------------------------------------------------- +// Live chat-model resolution +// --------------------------------------------------------------------------- + +/** + * Outcome of resolving a live Cohere chat model: + * - { model } → a valid, non-deprecated chat model to drive the leg + * - { infra } → the listing call hit an auth/credit/rate-limit/5xx + * condition; the caller SKIPS honestly (not drift) + * - { unavailable } → the listing succeeded but exposed no usable chat + * model (genuinely broken state — fail loud) + */ +type ResolvedModel = { model: string } | { infra: number } | { unavailable: true }; + +let cohereChatModelPromise: Promise | null = null; + +/** + * Discover a currently-valid chat model from Cohere's own model listing rather + * than hardcoding one. Cohere retires model IDs on a schedule (command-r-plus + * was removed 2026-04-04, which is what quarantined this leg), so the listing + * is the only drift-resilient source of a live model name. + */ +async function resolveCohereChatModel(): Promise { + const res = await fetch(`${COHERE_BASE_URL}/v1/models?endpoint=chat&page_size=1000`, { + headers: { Authorization: `Bearer ${COHERE_API_KEY}` }, + }); + if (isInfraStatus(res.status)) return { infra: res.status }; + if (!res.ok) return { unavailable: true }; + const json = (await res.json()) as { models?: CohereModelEntry[] }; + const model = selectCohereChatModel(json.models ?? []); + return model ? { model } : { unavailable: true }; +} + +/** Memoized so the whole live leg makes exactly one model-listing call. */ +function getCohereChatModel(): Promise { + if (!cohereChatModelPromise) cohereChatModelPromise = resolveCohereChatModel(); + return cohereChatModelPromise; +} + // --------------------------------------------------------------------------- // Error shape stubs // --------------------------------------------------------------------------- @@ -235,19 +282,38 @@ describe("Cohere error shapes", () => { }); describe.skipIf(!HAS_CREDENTIALS)("Cohere drift", () => { - it("non-streaming /v2/chat shape matches", async () => { + it("non-streaming /v2/chat shape matches", async (ctx) => { + const resolved = await getCohereChatModel(); + if ("infra" in resolved) { + // Provider-side auth/credit/rate-limit/5xx — honest skip, not drift. + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error( + "Cohere /v1/models?endpoint=chat exposed no usable non-deprecated chat model", + ); + } + const model = resolved.model; + const sdkShape = cohereChatResponseShape(); const messages = [{ role: "user", content: "Say hello" }]; const [realRes, mockRes] = await Promise.all([ - cohereChatNonStreaming(messages), + cohereChatNonStreaming(model, messages), httpPost(`${instance.url}/v2/chat`, { - model: "command-r-plus", + model, messages, stream: false, }), ]); + if (isInfraStatus(realRes.status)) { + // Real API hit a transient provider-side condition — honest skip. + ctx.skip(); + return; + } + expect(realRes.status).toBe(200); expect(mockRes.status).toBeLessThan(500); @@ -265,19 +331,38 @@ describe.skipIf(!HAS_CREDENTIALS)("Cohere drift", () => { } }); - it("streaming /v2/chat shape matches", async () => { + it("streaming /v2/chat shape matches", async (ctx) => { + const resolved = await getCohereChatModel(); + if ("infra" in resolved) { + // Provider-side auth/credit/rate-limit/5xx — honest skip, not drift. + ctx.skip(); + return; + } + if ("unavailable" in resolved) { + throw new Error( + "Cohere /v1/models?endpoint=chat exposed no usable non-deprecated chat model", + ); + } + const model = resolved.model; + const sdkChunkShape = cohereChatStreamChunkShape(); const messages = [{ role: "user", content: "Say hello" }]; const [realRes, mockRes] = await Promise.all([ - cohereChatStreaming(messages), + cohereChatStreaming(model, messages), httpPost(`${instance.url}/v2/chat`, { - model: "command-r-plus", + model, messages, stream: true, }), ]); + if (isInfraStatus(realRes.status)) { + // Real API hit a transient provider-side condition — honest skip. + ctx.skip(); + return; + } + expect(realRes.status).toBe(200); expect(mockRes.status).toBeLessThan(500);