diff --git a/.github/workflows/fix-drift.yml b/.github/workflows/fix-drift.yml index 8a03713f..9640e58b 100644 --- a/.github/workflows/fix-drift.yml +++ b/.github/workflows/fix-drift.yml @@ -77,6 +77,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} PRE_FIX_REPORT: ${{ runner.temp }}/drift-report.json run: | set +e @@ -176,6 +177,7 @@ jobs: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} PINNED_REPORT: ${{ runner.temp }}/drift-report.pinned.json run: npx tsx scripts/fix-drift.ts --report "${PINNED_REPORT}" @@ -207,6 +209,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} run: pnpm test:drift # Step 4c: Re-collect drift AUTHORITATIVELY against the LIVE SDK, writing @@ -230,6 +233,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} POST_FIX_REPORT: ${{ runner.temp }}/drift-report.post-fix.json run: | set +e diff --git a/.github/workflows/test-drift.yml b/.github/workflows/test-drift.yml index ed190275..99fd9a4e 100644 --- a/.github/workflows/test-drift.yml +++ b/.github/workflows/test-drift.yml @@ -80,6 +80,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} run: | # Written at the checkout root (cwd) so the relative provider import # below resolves against the repo. ESM relative specifiers key off the @@ -161,6 +162,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} run: | set +e npx tsx scripts/drift-retry.ts @@ -341,6 +343,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} steps: # HEAD checkout: the PR ref (default checkout behavior on pull_request). # persist-credentials:false — this leg never pushes; keeps the token off diff --git a/src/__tests__/drift/openrouter-video.drift.ts b/src/__tests__/drift/openrouter-video.drift.ts new file mode 100644 index 00000000..ee3e8c74 --- /dev/null +++ b/src/__tests__/drift/openrouter-video.drift.ts @@ -0,0 +1,282 @@ +/** + * OpenRouter video-proxy drift tests (surface: `openrouter-video`). + * + * The OpenRouter async video lifecycle proxy (`src/openrouter-video.ts`, + * `video-proxy-shared.ts`) is SHIPPED on master but had ZERO drift coverage. + * This leg closes that gap COST-SAFELY — video generation costs real money per + * job, so NOTHING here submits a paid generation: + * + * 1. LIVE canary (FREE) — authenticate with `OPENROUTER_API_KEY` and hit the + * public video-models LISTING endpoint (`GET /api/v1/videos/models`), + * asserting the video-model FAMILIES aimock's proxy mirrors are still + * present. Metadata-only; no generation. Gated on the key (skips in CI + * until `OPENROUTER_API_KEY` is mirrored to repo secrets). + * + * 2. Envelope shapes (STATIC, no key, no live generation) — drive the aimock + * server over HTTP and triangulate the job-lifecycle envelope shapes the + * proxy emits (submit response, completed status poll, models listing) + * against hand-authored conformant exemplars via the documented static + * `triangulate(sdkShape, sdkShape, mockShape)` form. This exercises the + * REAL handler + triangulate + collector routing path, not a unit fake. + * + * The models listing shape asserts the shape the mock is CONTRACTED to emit + * (mock-vs-exemplar, mirroring video.drift.ts) — the live canary above is what + * catches provider-side model drift. + */ + +import http from "node:http"; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { createServer, type ServerInstance } from "../../server.js"; +import type { Fixture } from "../../types.js"; +import { extractShape, triangulate, formatDriftReport } from "./schema.js"; +import { httpPost } from "./helpers.js"; +import { listOpenRouterVideoModels } from "./providers.js"; + +const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY; + +// --------------------------------------------------------------------------- +// HTTP GET helper (drift helpers.ts only exports httpPost — mirror video.drift.ts) +// --------------------------------------------------------------------------- + +async function httpGet( + url: string, +): Promise<{ status: number; headers: http.IncomingHttpHeaders; body: string }> { + return new Promise((resolve, reject) => { + const req = http.request(url, { method: "GET" }, (res) => { + const chunks: Buffer[] = []; + res.on("data", (c) => chunks.push(c)); + res.on("end", () => + resolve({ + status: res.statusCode!, + headers: res.headers, + body: Buffer.concat(chunks).toString(), + }), + ); + }); + req.on("error", reject); + req.end(); + }); +} + +// --------------------------------------------------------------------------- +// Fixtures — a completed OpenRouter video job. `endpoint: "video"` + +// `model` scopes the match to the submit's synthetic request. Default +// progression (0/0) seeds the job terminal at submit, so the first status +// poll reports "completed" without a paid generation ever happening. +// --------------------------------------------------------------------------- + +const OPENROUTER_VIDEO_MODEL = "bytedance/seedance-2.0"; + +const VIDEO_COMPLETED_FIXTURE: Fixture = { + match: { + userMessage: "a serene beach at sunset", + endpoint: "video", + model: OPENROUTER_VIDEO_MODEL, + }, + response: { + // "AAAA" is valid base64 (decodes to 3 bytes) — the content endpoint is + // not exercised here; the bytes just have to be a well-formed payload. + video: { id: "vid_or_completed", status: "completed", b64: "AAAA", cost: 0.12 }, + }, +}; + +// --------------------------------------------------------------------------- +// Expected envelope shapes (hand-authored conformant exemplars — mirrors the +// sdk-shapes.ts "minimal conformant instance" philosophy). These describe the +// shape openrouter-video.ts is CONTRACTED to emit on each lifecycle step. +// --------------------------------------------------------------------------- + +/** POST /api/v1/videos submit response: `{ id, polling_url, status }`. */ +function submitEnvelopeShape() { + return extractShape({ + id: "vid_or_completed", + polling_url: "http://localhost/api/v1/videos/vid_or_completed", + status: "pending", + }); +} + +/** + * GET /api/v1/videos/{jobId} completed status: + * `{ id, status, unsigned_urls: string[], usage: { cost } }`. + */ +function completedStatusShape() { + return extractShape({ + id: "vid_or_completed", + status: "completed", + unsigned_urls: ["http://localhost/api/v1/videos/vid_or_completed/content?index=0"], + usage: { cost: 0.12 }, + }); +} + +/** + * GET /api/v1/videos/models per-model entry shape (the mock's synthesized + * `modelEntry` — see handleOpenRouterVideoModels). Wrapped in `{ data: [...] }`. + */ +function modelsListingShape() { + return extractShape({ + data: [ + { + id: OPENROUTER_VIDEO_MODEL, + name: OPENROUTER_VIDEO_MODEL, + supported_durations: [4, 8], + supported_resolutions: ["720p", "1080p"], + supported_aspect_ratios: ["16:9", "9:16", "1:1"], + supported_frame_images: [], + supported_sizes: [], + generate_audio: false, + seed: true, + pricing_skus: [], + }, + ], + }); +} + +// --------------------------------------------------------------------------- +// Server lifecycle +// --------------------------------------------------------------------------- + +let instance: ServerInstance; + +beforeAll(async () => { + instance = await createServer([VIDEO_COMPLETED_FIXTURE], { port: 0 }); +}); + +afterAll(async () => { + await new Promise((r) => instance.server.close(() => r())); +}); + +// --------------------------------------------------------------------------- +// Envelope-shape drift (STATIC — no key, runs in CI) +// --------------------------------------------------------------------------- + +describe("OpenRouter video-proxy envelope shapes", () => { + it("submit returns { id, polling_url, status }", async () => { + const res = await httpPost(`${instance.url}/api/v1/videos`, { + model: OPENROUTER_VIDEO_MODEL, + prompt: "a serene beach at sunset", + }); + + expect(res.status, res.body).toBe(200); + const body = JSON.parse(res.body); + expect(body.id).toBeDefined(); + expect(body.status).toBe("pending"); + expect(typeof body.polling_url).toBe("string"); + + const sdkShape = submitEnvelopeShape(); + const mockShape = extractShape(body); + const diffs = triangulate(sdkShape, sdkShape, mockShape); + const report = formatDriftReport("OpenRouter video submit", diffs, "openrouter-video"); + + expect( + diffs.filter((d) => d.severity === "critical"), + report, + ).toEqual([]); + }); + + it("completed status poll returns { id, status, unsigned_urls, usage }", async () => { + // Submit, then poll the returned polling_url. Default 0/0 progression seeds + // the job terminal at submit, so the first poll is already "completed". + const submitRes = await httpPost(`${instance.url}/api/v1/videos`, { + model: OPENROUTER_VIDEO_MODEL, + prompt: "a serene beach at sunset", + }); + expect(submitRes.status, submitRes.body).toBe(200); + const { polling_url } = JSON.parse(submitRes.body); + + const poll = await httpGet(polling_url); + expect(poll.status, poll.body).toBe(200); + const body = JSON.parse(poll.body); + expect(body.status).toBe("completed"); + expect(Array.isArray(body.unsigned_urls)).toBe(true); + expect(body.unsigned_urls.length).toBeGreaterThan(0); + + const sdkShape = completedStatusShape(); + const mockShape = extractShape(body); + const diffs = triangulate(sdkShape, sdkShape, mockShape); + const report = formatDriftReport( + "OpenRouter video completed status", + diffs, + "openrouter-video", + ); + + expect( + diffs.filter((d) => d.severity === "critical"), + report, + ).toEqual([]); + }); + + it("models listing returns { data: [ { id, name, supported_* , ... } ] }", async () => { + const res = await httpGet(`${instance.url}/api/v1/videos/models`); + + expect(res.status, res.body).toBe(200); + const body = JSON.parse(res.body); + expect(Array.isArray(body.data)).toBe(true); + expect(body.data.length).toBeGreaterThan(0); + + const sdkShape = modelsListingShape(); + const mockShape = extractShape(body); + const diffs = triangulate(sdkShape, sdkShape, mockShape); + const report = formatDriftReport("OpenRouter video models listing", diffs, "openrouter-video"); + + expect( + diffs.filter((d) => d.severity === "critical"), + report, + ).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// LIVE models-listing canary (FREE — metadata only, NO generation). +// Gated on OPENROUTER_API_KEY: skips in CI until the secret is mirrored. +// --------------------------------------------------------------------------- + +/** + * Reduce an OpenRouter video-model slug to its FAMILY key: drop the provider + * prefix (`bytedance/seedance-2.0` → `seedance-2.0`) then take the leading + * alpha token before the version (`seedance-2.0` → `seedance`, `sora-2` → + * `sora`, `veo-3.1` → `veo`). Comparing normalized families (not raw dated + * slugs) is what keeps the canary from false-positiving on every new version. + */ +export function openRouterVideoFamily(id: string): string { + const afterProvider = id.includes("/") ? id.slice(id.indexOf("/") + 1) : id; + const leading = afterProvider.match(/^[a-zA-Z]+/); + return (leading ? leading[0] : afterProvider).toLowerCase(); +} + +// The families aimock's OpenRouter video proxy hardcodes as its default set +// (DEFAULT_OPENROUTER_VIDEO_MODELS = ["bytedance/seedance-2.0", "openai/sora-2"] +// in src/openrouter-video.ts) — the ground truth of "what the proxy mirrors". +// A missing family means aimock is mirroring a model the provider retired (or +// the endpoint contract moved), which is exactly the drift signal. +const REQUIRED_VIDEO_FAMILIES = ["seedance", "sora"] as const; + +describe.skipIf(!OPENROUTER_API_KEY)("OpenRouter video model-family availability (live)", () => { + it("live /api/v1/videos/models contains the families aimock mirrors", async () => { + const ids = await listOpenRouterVideoModels(OPENROUTER_API_KEY!); + expect(ids.length, "OpenRouter returned an empty video-model listing").toBeGreaterThan(0); + + const families = new Set(ids.map(openRouterVideoFamily)); + const missing = REQUIRED_VIDEO_FAMILIES.filter((f) => !families.has(f)); + + const report = + missing.length > 0 + ? formatDriftReport( + "OpenRouter video (live /api/v1/videos/models family canary)", + missing.map((family) => ({ + path: `videos/models/${family}`, + severity: "critical" as const, + issue: + `aimock's OpenRouter video proxy mirrors the "${family}" family, but the live ` + + `/api/v1/videos/models listing no longer contains it — update ` + + `DEFAULT_OPENROUTER_VIDEO_MODELS in src/openrouter-video.ts`, + expected: `(family "${family}" present in live listing)`, + real: [...families].sort().join(", "), + mock: family, + })), + "openrouter-video", + ) + : "No drift detected: OpenRouter video family canary"; + + expect(missing, report).toEqual([]); + }); +}); diff --git a/src/__tests__/drift/providers.ts b/src/__tests__/drift/providers.ts index c2b2c309..434749e6 100644 --- a/src/__tests__/drift/providers.ts +++ b/src/__tests__/drift/providers.ts @@ -132,7 +132,9 @@ function parseDataOnlySSE(text: string): { data: unknown }[] { try { return { data: JSON.parse(json) }; } catch (err) { - throw new Error(`Malformed SSE JSON in frame: ${json.slice(0, 100)}`, { cause: err }); + throw new Error( + `Malformed SSE JSON in frame: ${json.slice(0, 100)} — ${err instanceof Error ? err.message : String(err)}`, + ); } }); } @@ -162,7 +164,9 @@ function parseTypedSSE(text: string): { type: string; data: unknown }[] { data: JSON.parse(json), }; } catch (err) { - throw new Error(`Malformed SSE JSON in frame: ${json.slice(0, 100)}`, { cause: err }); + throw new Error( + `Malformed SSE JSON in frame: ${json.slice(0, 100)} — ${err instanceof Error ? err.message : String(err)}`, + ); } }); } @@ -697,3 +701,29 @@ export async function listGeminiModels(apiKey: string): Promise { return json.models.map((m) => m.name.replace(/^models\//, "")); }); } + +/** + * List OpenRouter's video-capable models via the dedicated (FREE, no + * generation) listing endpoint `GET /api/v1/videos/models`. Video models do + * NOT appear in the plain `/api/v1/models` listing, hence the dedicated route + * (see src/openrouter-video.ts handleOpenRouterVideoModels). Returns the raw + * provider-prefixed slugs (e.g. "openai/sora-2", "bytedance/seedance-2.0", + * "google/veo-3.1"). This is the cost-safe daily live-reality canary for the + * OpenRouter video proxy surface — it authenticates and reads metadata only, + * never submitting a paid generation job. + */ +export async function listOpenRouterVideoModels(apiKey: string): Promise { + return withInfraErrorTag("OpenRouter Video Models", async () => { + const url = "https://openrouter.ai/api/v1/videos/models"; + const res = await fetchWithRetry(url, { + method: "GET", + headers: { Authorization: `Bearer ${apiKey}` }, + }); + + const raw = await res.text(); + const json = parseJsonResponse(raw, res.status, "OpenRouter video model list", url) as { + data: { id: string }[]; + }; + return json.data.map((m) => m.id); + }); +} diff --git a/src/__tests__/drift/surface-registry.ts b/src/__tests__/drift/surface-registry.ts index 32d7d89f..bf2fa626 100644 --- a/src/__tests__/drift/surface-registry.ts +++ b/src/__tests__/drift/surface-registry.ts @@ -255,6 +255,17 @@ export const SURFACE_REGISTRY: Record = { ], typesFile: null, }, + "openrouter-video": { + provider: "OpenRouter Video", + builderFile: "src/openrouter-video.ts", + builderFunctions: [ + "handleOpenRouterVideoCreate", + "handleOpenRouterVideoStatus", + "handleOpenRouterVideoModels", + "handleOpenRouterVideoContent", + ], + typesFile: null, + }, }; /**