Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/fix-drift.yml
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,8 @@ jobs:
OLLAMA_HOST: 127.0.0.1:11434
OLLAMA_MODEL: qwen2:0.5b
FAL_KEY: ${{ secrets.FAL_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }}
PRE_FIX_REPORT: ${{ runner.temp }}/drift-report.json
run: |
set +e
Expand Down Expand Up @@ -241,6 +243,8 @@ jobs:
OLLAMA_HOST: 127.0.0.1:11434
OLLAMA_MODEL: qwen2:0.5b
FAL_KEY: ${{ secrets.FAL_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }}
run: pnpm test:drift

# Step 4c: Re-collect drift AUTHORITATIVELY against the LIVE SDK, writing
Expand Down Expand Up @@ -270,6 +274,8 @@ jobs:
OLLAMA_HOST: 127.0.0.1:11434
OLLAMA_MODEL: qwen2:0.5b
FAL_KEY: ${{ secrets.FAL_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }}
POST_FIX_REPORT: ${{ runner.temp }}/drift-report.post-fix.json
run: |
set +e
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/test-drift.yml
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,8 @@ jobs:
OLLAMA_HOST: 127.0.0.1:11434
OLLAMA_MODEL: qwen2:0.5b
FAL_KEY: ${{ secrets.FAL_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }}
run: |
set +e
npx tsx scripts/drift-retry.ts
Expand Down Expand Up @@ -372,6 +374,8 @@ jobs:
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
FAL_KEY: ${{ secrets.FAL_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
ELEVENLABS_API_KEY: ${{ secrets.ELEVENLABS_API_KEY }}
steps:
# HEAD checkout: the PR ref (default checkout behavior on pull_request).
# persist-credentials:false — this leg never pushes; keeps the token off
Expand Down
61 changes: 61 additions & 0 deletions src/__tests__/drift/cohere-model.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/**
* Unit tests for the Cohere live-leg model selection + infra classification.
*
* These exercise the exact logic that fixes the drift-live-pr quarantine: the
* live leg was hardcoded to `command-r-plus`, which Cohere retired 2026-04-04,
* so the real `/v2/chat` call returned 404 (model not found) and quarantined
* as exit 5. The selector must skip deprecated models and pick a live one.
*/

import { describe, it, expect } from "vitest";
import { isInfraStatus, selectCohereChatModel, type CohereModelEntry } from "./cohere-model.js";

describe("selectCohereChatModel", () => {
it("never selects a deprecated model (the command-r-plus 404 regression)", () => {
const models: CohereModelEntry[] = [
{ name: "command-r-plus", is_deprecated: true, endpoints: ["chat"] },
{ name: "command-a-03-2025", is_deprecated: false, endpoints: ["chat"] },
];
const chosen = selectCohereChatModel(models);
expect(chosen).not.toBe("command-r-plus");
expect(chosen).toBe("command-a-03-2025");
});

it("prefers a stable default when present", () => {
const models: CohereModelEntry[] = [
{ name: "some-experimental-model", is_deprecated: false, endpoints: ["chat"] },
{ name: "command-a-03-2025", is_deprecated: false, endpoints: ["chat"] },
];
expect(selectCohereChatModel(models)).toBe("command-a-03-2025");
});

it("falls back to the first non-deprecated model when no preferred present", () => {
const models: CohereModelEntry[] = [
{ name: "command-r-plus", is_deprecated: true, endpoints: ["chat"] },
{ name: "north-mini-code-1-0", is_deprecated: false, endpoints: ["chat"] },
];
expect(selectCohereChatModel(models)).toBe("north-mini-code-1-0");
});

it("returns null when no usable chat model exists", () => {
const models: CohereModelEntry[] = [
{ name: "command-r-plus", is_deprecated: true, endpoints: ["chat"] },
];
expect(selectCohereChatModel(models)).toBeNull();
expect(selectCohereChatModel([])).toBeNull();
});
});

describe("isInfraStatus", () => {
it("classifies auth/credit/rate-limit/5xx as infra (honest-skip conditions)", () => {
for (const s of [401, 402, 403, 429, 500, 502, 503, 529]) {
expect(isInfraStatus(s)).toBe(true);
}
});

it("does NOT classify success or a genuine drift/probe error as infra", () => {
for (const s of [200, 400, 404, 422]) {
expect(isInfraStatus(s)).toBe(false);
}
});
});
52 changes: 52 additions & 0 deletions src/__tests__/drift/cohere-model.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/**
* Pure helpers for the Cohere live drift leg: chat-model selection and
* infra-status classification.
*
* Extracted from cohere.drift.ts so the selection/classification logic is
* unit-testable WITHOUT live credentials (the live describe block is
* credential-gated and cannot run locally).
*/

/** Cohere REST base URL. */
export const COHERE_BASE_URL = "https://api.cohere.com";

/**
* Environmental HTTP statuses that indicate a provider-side condition
* (auth / out-of-credit / rate-limit / upstream outage) rather than a real
* API-envelope drift. Mirrors the providers.ts InfraError classification:
* 401|403 → stale-key, 429 → rate-limited, 5xx → infra-transient; 402 →
* out-of-credit.
*
* The live leg converts these to an HONEST SKIP so a transient provider-side
* condition never quarantines the drift collector and reds the PR. A real
* shape drift (a 200 response with an unexpected envelope) is NEVER skipped.
*/
export function isInfraStatus(status: number): boolean {
return status === 401 || status === 402 || status === 403 || status === 429 || status >= 500;
}

/** Minimal shape of a /v1/models listing entry we depend on. */
export interface CohereModelEntry {
name?: string;
is_deprecated?: boolean;
endpoints?: string[];
}

/**
* Select a currently-valid, non-deprecated Cohere chat model from a
* `/v1/models?endpoint=chat` listing.
*
* Cohere retires model IDs on a schedule (e.g. `command-r-plus` was removed
* 2026-04-04), so a hardcoded model name is a standing 404 / quarantine risk.
* Self-selecting from the live listing makes the leg resilient to future
* deprecations. Prefers a stable, widely-available default when present, else
* falls back to the first non-deprecated chat model. Returns null when the
* listing exposes no usable chat model.
*/
export function selectCohereChatModel(models: CohereModelEntry[]): string | null {
const names = models
.filter((m) => typeof m.name === "string" && m.is_deprecated !== true)
.map((m) => m.name as string);
const preferred = ["command-a-03-2025", "command-r-08-2024", "command-r7b-12-2024"];
return preferred.find((p) => names.includes(p)) ?? names[0] ?? null;
}
105 changes: 95 additions & 10 deletions src/__tests__/drift/cohere.drift.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ import {
startDriftServer,
stopDriftServer,
} from "./helpers.js";
import {
COHERE_BASE_URL,
isInfraStatus,
selectCohereChatModel,
type CohereModelEntry,
} from "./cohere-model.js";

// ---------------------------------------------------------------------------
// Credentials check
Expand Down Expand Up @@ -87,16 +93,17 @@ function cohereChatStreamChunkShape() {
// ---------------------------------------------------------------------------

async function cohereChatNonStreaming(
model: string,
messages: { role: string; content: string }[],
): Promise<{ status: number; body: string }> {
const res = await fetch("https://api.cohere.com/v2/chat", {
const res = await fetch(`${COHERE_BASE_URL}/v2/chat`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${COHERE_API_KEY}`,
},
body: JSON.stringify({
model: "command-r-plus",
model,
messages,
stream: false,
max_tokens: 10,
Expand All @@ -106,16 +113,17 @@ async function cohereChatNonStreaming(
}

async function cohereChatStreaming(
model: string,
messages: { role: string; content: string }[],
): Promise<{ status: number; body: string }> {
const res = await fetch("https://api.cohere.com/v2/chat", {
const res = await fetch(`${COHERE_BASE_URL}/v2/chat`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${COHERE_API_KEY}`,
},
body: JSON.stringify({
model: "command-r-plus",
model,
messages,
stream: true,
max_tokens: 10,
Expand All @@ -124,6 +132,45 @@ async function cohereChatStreaming(
return { status: res.status, body: await res.text() };
}

// ---------------------------------------------------------------------------
// Live chat-model resolution
// ---------------------------------------------------------------------------

/**
* Outcome of resolving a live Cohere chat model:
* - { model } → a valid, non-deprecated chat model to drive the leg
* - { infra } → the listing call hit an auth/credit/rate-limit/5xx
* condition; the caller SKIPS honestly (not drift)
* - { unavailable } → the listing succeeded but exposed no usable chat
* model (genuinely broken state — fail loud)
*/
type ResolvedModel = { model: string } | { infra: number } | { unavailable: true };

let cohereChatModelPromise: Promise<ResolvedModel> | null = null;

/**
* Discover a currently-valid chat model from Cohere's own model listing rather
* than hardcoding one. Cohere retires model IDs on a schedule (command-r-plus
* was removed 2026-04-04, which is what quarantined this leg), so the listing
* is the only drift-resilient source of a live model name.
*/
async function resolveCohereChatModel(): Promise<ResolvedModel> {
const res = await fetch(`${COHERE_BASE_URL}/v1/models?endpoint=chat&page_size=1000`, {
headers: { Authorization: `Bearer ${COHERE_API_KEY}` },
});
if (isInfraStatus(res.status)) return { infra: res.status };
if (!res.ok) return { unavailable: true };
const json = (await res.json()) as { models?: CohereModelEntry[] };
const model = selectCohereChatModel(json.models ?? []);
return model ? { model } : { unavailable: true };
}

/** Memoized so the whole live leg makes exactly one model-listing call. */
function getCohereChatModel(): Promise<ResolvedModel> {
if (!cohereChatModelPromise) cohereChatModelPromise = resolveCohereChatModel();
return cohereChatModelPromise;
}

// ---------------------------------------------------------------------------
// Error shape stubs
// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -235,19 +282,38 @@ describe("Cohere error shapes", () => {
});

describe.skipIf(!HAS_CREDENTIALS)("Cohere drift", () => {
it("non-streaming /v2/chat shape matches", async () => {
it("non-streaming /v2/chat shape matches", async (ctx) => {
const resolved = await getCohereChatModel();
if ("infra" in resolved) {
// Provider-side auth/credit/rate-limit/5xx — honest skip, not drift.
ctx.skip();
return;
}
if ("unavailable" in resolved) {
throw new Error(
"Cohere /v1/models?endpoint=chat exposed no usable non-deprecated chat model",
);
}
const model = resolved.model;

const sdkShape = cohereChatResponseShape();
const messages = [{ role: "user", content: "Say hello" }];

const [realRes, mockRes] = await Promise.all([
cohereChatNonStreaming(messages),
cohereChatNonStreaming(model, messages),
httpPost(`${instance.url}/v2/chat`, {
model: "command-r-plus",
model,
messages,
stream: false,
}),
]);

if (isInfraStatus(realRes.status)) {
// Real API hit a transient provider-side condition — honest skip.
ctx.skip();
return;
}

expect(realRes.status).toBe(200);
expect(mockRes.status).toBeLessThan(500);

Expand All @@ -265,19 +331,38 @@ describe.skipIf(!HAS_CREDENTIALS)("Cohere drift", () => {
}
});

it("streaming /v2/chat shape matches", async () => {
it("streaming /v2/chat shape matches", async (ctx) => {
const resolved = await getCohereChatModel();
if ("infra" in resolved) {
// Provider-side auth/credit/rate-limit/5xx — honest skip, not drift.
ctx.skip();
return;
}
if ("unavailable" in resolved) {
throw new Error(
"Cohere /v1/models?endpoint=chat exposed no usable non-deprecated chat model",
);
}
const model = resolved.model;

const sdkChunkShape = cohereChatStreamChunkShape();
const messages = [{ role: "user", content: "Say hello" }];

const [realRes, mockRes] = await Promise.all([
cohereChatStreaming(messages),
cohereChatStreaming(model, messages),
httpPost(`${instance.url}/v2/chat`, {
model: "command-r-plus",
model,
messages,
stream: true,
}),
]);

if (isInfraStatus(realRes.status)) {
// Real API hit a transient provider-side condition — honest skip.
ctx.skip();
return;
}

expect(realRes.status).toBe(200);
expect(mockRes.status).toBeLessThan(500);

Expand Down
Loading