/*--------------------------------------------------------------------------------------------- * Copyright (c) Microsoft Corporation. All rights reserved. *--------------------------------------------------------------------------------------------*/ import { mkdtemp, readFile, rm, writeFile } from "fs/promises"; import http from "http"; import type { ChatCompletion, ChatCompletionChunk, ChatCompletionMessageFunctionToolCall, } from "openai/resources/chat/completions"; import os from "os"; import path from "path"; import { afterEach, beforeEach, describe, expect, test } from "vitest"; import yaml from "yaml"; import { NormalizedData, ReplayingCapiProxy, ToolResultNormalizer, workingDirPlaceholder, } from "./replayingCapiProxy"; import { ShellConfig } from "./util"; describe("ReplayingCapiProxy", () => { let tempDir: string; let workDir: string; let githubActions: string | undefined; beforeEach(async () => { githubActions = process.env.GITHUB_ACTIONS; delete process.env.GITHUB_ACTIONS; tempDir = await mkdtemp(path.join(os.tmpdir(), "capi-proxy-test-")); workDir = path.join(tempDir, "work"); }); afterEach(async () => { if (githubActions === undefined) { delete process.env.GITHUB_ACTIONS; } else { process.env.GITHUB_ACTIONS = githubActions; } await rm(tempDir, { recursive: true, force: true }); }); async function createProxy( httpExchanges: Array<{ url: string; requestBody: string; responseBody: string; }>, options?: { toolResultNormalizers?: ToolResultNormalizer[] }, ) { const outputPath = path.join(tempDir, "output.yaml"); const proxy = new ReplayingCapiProxy( "http://localhost", outputPath, workDir, ); for (const normalizer of options?.toolResultNormalizers ?? []) { proxy.addToolResultNormalizer(normalizer.toolName, normalizer.normalizer); } for (const exchange of httpExchanges) { (proxy.exchanges as Array).push({ request: { url: exchange.url, method: "POST", body: exchange.requestBody, }, response: { statusCode: 200, body: exchange.responseBody }, }); } await proxy.stop(); return outputPath; } async function readYamlOutput(outputPath: string): Promise { const content = await readFile(outputPath, "utf-8"); return yaml.parse(content) as NormalizedData; } test("does not write file when no chat completion exchanges", async () => { const outputPath = path.join(tempDir, "output.yaml"); const proxy = new ReplayingCapiProxy( "http://localhost", outputPath, workDir, ); await proxy.stop(); await expect(readFile(outputPath)).rejects.toThrow(/ENOENT/); }); test("captures chat completion request and response", async () => { const requestBody = JSON.stringify({ messages: [ { role: "system", content: "You are helpful" }, { role: "user", content: "Hello" }, ], }); const responseBody = JSON.stringify({ choices: [{ message: { role: "assistant", content: "Hi there!" } }], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); expect(result.conversations).toHaveLength(1); expect(result.conversations[0].messages).toEqual([ { role: "system", content: "${system}" }, { role: "user", content: "Hello" }, { role: "assistant", content: "Hi there!" }, ]); }); test("normalizes tool call IDs to sequential values", async () => { const requestBody = JSON.stringify({ messages: [{ role: "user", content: "Do something" }], }); const responseBody = JSON.stringify({ choices: [ { message: { role: "assistant", tool_calls: [ { id: "toolu_abc123xyz", type: "function", function: { name: "view", arguments: "{}" }, }, ], }, }, ], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); expect(result.conversations[0].messages[1].tool_calls![0].id).toBe( "toolcall_0", ); }); test("normalizes shell tool names to platform-agnostic placeholders", async () => { const originalShellConfig = process.platform === "win32" ? ShellConfig.powerShell : ShellConfig.bash; const requestBody = JSON.stringify({ messages: [{ role: "user", content: "Do something" }], }); const responseBody = JSON.stringify({ choices: [ { message: { role: "assistant", tool_calls: [ { id: "t0", type: "function", function: { name: originalShellConfig.shellToolName, arguments: "{}", }, }, { id: "t1", type: "function", function: { name: originalShellConfig.readShellToolName, arguments: "{}", }, }, { id: "t2", type: "function", function: { name: originalShellConfig.writeShellToolName, arguments: "{}", }, }, { id: "t3", type: "function", function: { name: "someOtherName", arguments: "{}" }, }, ], }, }, ], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); expect( result.conversations[0].messages[1].tool_calls![0].function?.name, ).toBe("${shell}"); expect( result.conversations[0].messages[1].tool_calls![1].function?.name, ).toBe("${read_shell}"); expect( result.conversations[0].messages[1].tool_calls![2].function?.name, ).toBe("${write_shell}"); expect( result.conversations[0].messages[1].tool_calls![3].function?.name, ).toBe("someOtherName"); }); test("normalizes workDir paths to placeholder with forward slashes", async () => { const requestBody = JSON.stringify({ messages: [{ role: "user", content: "Read file" }], }); const responseBody = JSON.stringify({ choices: [ { message: { role: "assistant", tool_calls: [ { id: "tc1", type: "function", function: { name: "view", arguments: JSON.stringify({ path: workDir + "\\subdir\\file.txt", }), }, }, ], }, }, ], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); const args = result.conversations[0].messages[1].tool_calls![0].function!.arguments; expect(args).toBe(`{"path":"${workingDirPlaceholder}/subdir/file.txt"}`); }); test("removes prefix exchanges keeping only the longest conversation", async () => { const turn1Request = JSON.stringify({ messages: [{ role: "user", content: "Hello" }], }); const turn1Response = JSON.stringify({ choices: [{ message: { role: "assistant", content: "Hi" } }], }); const turn2Request = JSON.stringify({ messages: [ { role: "user", content: "Hello" }, { role: "assistant", content: "Hi" }, { role: "user", content: "How are you?" }, ], }); const turn2Response = JSON.stringify({ choices: [{ message: { role: "assistant", content: "Good!" } }], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody: turn1Request, responseBody: turn1Response, }, { url: "/chat/completions", requestBody: turn2Request, responseBody: turn2Response, }, ]); const result = await readYamlOutput(outputPath); expect(result.conversations).toHaveLength(1); expect(result.conversations[0].messages).toHaveLength(4); }); test("strips current_datetime from user messages", async () => { const requestBody = JSON.stringify({ messages: [ { role: "user", content: "2025-12-09 What time is it?", }, ], }); const responseBody = JSON.stringify({ choices: [{ message: { role: "assistant", content: "It's now" } }], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); expect(result.conversations[0].messages[0].content).toBe( "What time is it?", ); }); test("strips system_reminder from user messages", async () => { const requestBody = JSON.stringify({ messages: [ { role: "user", content: "What is 2+2?\n\n\nNo tables currently exist.\n", }, ], }); const responseBody = JSON.stringify({ choices: [{ message: { role: "assistant", content: "4" } }], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); expect(result.conversations[0].messages[0].content).toBe("What is 2+2?"); }); test("strips plan mode prefix from user messages", async () => { const requestBody = JSON.stringify({ messages: [ { role: "user", content: "[[PLAN]] Create a brief implementation plan.", }, ], }); const responseBody = JSON.stringify({ choices: [{ message: { role: "assistant", content: "Plan" } }], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); expect(result.conversations[0].messages[0].content).toBe( "Create a brief implementation plan.", ); }); test("normalizes task completion notification wording", async () => { const idleNotification = [ "", 'Agent "sdk-background-agent" (general-purpose) has finished processing and is now idle. Use read_agent with agent_id "sdk-background-agent" to read the results, or write_agent to send follow-up messages.', "", ].join("\n"); const fullNotification = [ "", 'Agent "sdk-background-agent" (general-purpose) has completed successfully. Use read_agent with agent_id "sdk-background-agent" to retrieve the full results.', "", ].join("\n"); const requestBody = JSON.stringify({ messages: [ { role: "user", content: idleNotification, }, ], }); const responseBody = JSON.stringify({ choices: [{ message: { role: "assistant", content: "Done" } }], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); expect(result.conversations[0].messages[0].content).toBe(fullNotification); }); test("strips agent_instructions from user messages", async () => { const requestBody = JSON.stringify({ messages: [ { role: "user", content: "\nYou are a helpful test agent.\n\n\n\n\nSay hello briefly.", }, ], }); const responseBody = JSON.stringify({ choices: [{ message: { role: "assistant", content: "Hello!" } }], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); expect(result.conversations[0].messages[0].content).toBe( "Say hello briefly.", ); }); test("strips agent_instructions containing skill-context from user messages", async () => { const requestBody = JSON.stringify({ messages: [ { role: "user", content: '\n\nSkill content here\n\nYou are a helpful agent.\n\n\nSay hello.', }, ], }); const responseBody = JSON.stringify({ choices: [{ message: { role: "assistant", content: "Hi!" } }], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); expect(result.conversations[0].messages[0].content).toBe("Say hello."); }); test("strips skill metadata frontmatter from skill-context user messages", async () => { const skillDir = path.join(workDir, ".test_skills", "test-skill"); const requestBody = JSON.stringify({ messages: [ { role: "user", content: ` Base directory for this skill: ${skillDir} --- name: test-skill description: A test skill that adds a marker to responses --- # Test Skill Instructions Always include PINEAPPLE_COCONUT_42. `, }, ], }); const responseBody = JSON.stringify({ choices: [{ message: { role: "assistant", content: "OK!" } }], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); expect(result.conversations[0].messages[0].content).toBe(` Base directory for this skill: ${workingDirPlaceholder}/.test_skills/test-skill # Test Skill Instructions Always include PINEAPPLE_COCONUT_42. `); }); test("applies tool result normalizers to tool response content", async () => { const requestBody = JSON.stringify({ messages: [ { role: "user", content: "Help me" }, { role: "assistant", tool_calls: [ { id: "tc1", type: "function", function: { name: "tool_alpha", arguments: "{}" }, }, { id: "tc2", type: "function", function: { name: "tool_beta", arguments: "{}" }, }, ], }, { role: "tool", tool_call_id: "tc1", content: "alpha result" }, { role: "tool", tool_call_id: "tc2", content: "beta result" }, ], }); const responseBody = JSON.stringify({ choices: [{ message: { role: "assistant", content: "Done" } }], }); const outputPath = await createProxy( [{ url: "/chat/completions", requestBody, responseBody }], { toolResultNormalizers: [ { toolName: "tool_alpha", normalizer: (r) => r.toUpperCase() }, { toolName: "tool_beta", normalizer: (r) => `[${r}]` }, ], }, ); const result = await readYamlOutput(outputPath); const toolMessages = result.conversations[0].messages.filter( (m) => m.role === "tool", ); expect(toolMessages[0].content).toBe("ALPHA RESULT"); expect(toolMessages[1].content).toBe("[beta result]"); }); test("removes the runtime-specific available-tools list", async () => { const requestBody = JSON.stringify({ messages: [ { role: "user", content: "Help me" }, { role: "assistant", tool_calls: [ { id: "tc1", type: "function", function: { name: "report_intent", arguments: "{}" }, }, ], }, { role: "tool", tool_call_id: "tc1", content: "Tool 'report_intent' does not exist. Available tools that can be called are bash, read_bash, view, read_agent, list_agents, write_agent, grep, glob, task.", }, ], }); const responseBody = JSON.stringify({ choices: [{ message: { role: "assistant", content: "Done" } }], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); const toolMessage = result.conversations[0].messages.find( (m) => m.role === "tool", ); expect(toolMessage?.content).toBe("Tool 'report_intent' does not exist."); }); test("normalizes interrupted tool execution results", async () => { const requestBody = JSON.stringify({ messages: [ { role: "user", content: "Run a slow analysis" }, { role: "assistant", tool_calls: [ { id: "tc1", type: "function", function: { name: "slow_analysis", arguments: '{"value":"test_abort"}', }, }, { id: "tc2", type: "function", function: { name: "powershell", arguments: '{"command":"sleep 100"}', }, }, { id: "tc3", type: "function", function: { name: "bash", arguments: '{"command":"sleep 100"}', }, }, ], }, { role: "tool", tool_call_id: "tc1", content: 'Failed to execute `slow_analysis` tool with arguments: {"value":"test_abort"} due to error: Error: Session aborted', }, { role: "tool", tool_call_id: "tc2", content: "", }, { role: "tool", tool_call_id: "tc3", content: "unknown attachedShellSession handle 9", }, ], }); const responseBody = JSON.stringify({ choices: [{ message: { role: "assistant", content: "Done" } }], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); const toolMessages = result.conversations[0].messages.filter( (m) => m.role === "tool", ); expect(toolMessages.map((message) => message.content)).toEqual([ "The execution of this tool, or a previous tool was interrupted.", "The execution of this tool, or a previous tool was interrupted.", "The execution of this tool, or a previous tool was interrupted.", ]); }); test("normalizes background agent IDs and removes runtime advisories", async () => { const stableResult = "Agent started in background with agent_id: background-agent. You'll be notified when it completes. Tell the user you're waiting and end your response, or continue unrelated work until notified."; const requestBody = JSON.stringify({ messages: [ { role: "user", content: "Help me" }, { role: "assistant", tool_calls: [ { id: "tc1", type: "function", function: { name: "task", arguments: "{}" }, }, ], }, { role: "tool", tool_call_id: "tc1", content: "Agent started in background with agent_id: 3e0c7565-6091-58cb-85bb-6cb14db23ef7. You'll be notified when it completes. Tell the user you're waiting and end your response, or continue unrelated work until notified. The agent supports multi-turn conversations.", }, ], }); const responseBody = JSON.stringify({ choices: [{ message: { role: "assistant", content: "Done" } }], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); const toolMessage = result.conversations[0].messages.find( (m) => m.role === "tool", ); expect(toolMessage?.content).toBe(stableResult); }); test("normalizes read_agent result metadata", async () => { const requestBody = JSON.stringify({ messages: [ { role: "user", content: "Help me" }, { role: "assistant", tool_calls: [ { id: "tc1", type: "function", function: { name: "read_agent", arguments: '{"agent_id":"read-file","wait":true}', }, }, ], }, { role: "tool", tool_call_id: "tc1", content: "Agent is idle (waiting for messages). agent_id: read-file, agent_type: explore, status: idle, description: Reading subagent-test.txt, elapsed: 1.25s, total_turns: 1\n\n[Turn 0]\nDone.", }, ], }); const responseBody = JSON.stringify({ choices: [{ message: { role: "assistant", content: "Done" } }], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); const toolMessage = result.conversations[0].messages.find( (m) => m.role === "tool", ); expect(toolMessage?.content).toBe( "Agent completed. agent_id: read-file, agent_type: explore, status: completed, description: Reading subagent-test.txt, elapsed: 0s, total_turns: 0, duration: 0s\n\nDone.", ); }); test("normalizes GitHub CLI proxy auth failures", async () => { const requestBody = JSON.stringify({ messages: [ { role: "user", content: "Summarize this issue" }, { role: "assistant", tool_calls: [ { id: "tc1", type: "function", function: { name: "web_fetch", arguments: "{}" }, }, ], }, { role: "tool", tool_call_id: "tc1", content: 'Post "https://api.github.com/graphql": tls: failed to verify certificate: x509: certificate signed by unknown authority\n', }, { role: "tool", tool_call_id: "tc1", content: "\u28fe\u28fdHTTP 401: Requires authentication (https://api.github.com/graphql)\nTry authenticating with: gh auth login\n", }, ], }); const responseBody = JSON.stringify({ choices: [{ message: { role: "assistant", content: "Done" } }], }); const outputPath = await createProxy([ { url: "/chat/completions", requestBody, responseBody }, ]); const result = await readYamlOutput(outputPath); const toolMessages = result.conversations[0].messages.filter( (m) => m.role === "tool", ); expect(toolMessages).toEqual([ { role: "tool", tool_call_id: "toolcall_0", content: "${gh_auth_required}\n", }, { role: "tool", tool_call_id: "toolcall_0", content: "${gh_auth_required}\n", }, ]); }); test("ignores non-chat-completion endpoints", async () => { const outputPath = await createProxy([ { url: "/models", requestBody: "{}", responseBody: "{}" }, { url: "/embeddings", requestBody: "{}", responseBody: "{}" }, ]); await expect(readFile(outputPath)).rejects.toThrow(/ENOENT/); }); describe("cache replay", () => { async function makeRequest( proxyUrl: string, requestPath: string, options?: { method?: string; body?: object }, ): Promise<{ status: number; body: string }> { return new Promise((resolve, reject) => { const url = new URL(proxyUrl); const req = http.request( { hostname: url.hostname, port: url.port, path: requestPath, method: options?.method ?? "POST", headers: { "content-type": "application/json" }, }, (res) => { const chunks: Buffer[] = []; res.on("data", (chunk: Buffer) => chunks.push(chunk)); res.on("end", () => { resolve({ status: res.statusCode || 500, body: Buffer.concat(chunks).toString("utf-8"), }); }); }, ); req.on("error", reject); if (options?.body) { req.write(JSON.stringify(options.body)); } req.end(); }); } test("returns cached response when request matches prefix", async () => { const cachePath = path.join(tempDir, "cache.yaml"); const cacheContent = yaml.stringify({ models: ["test-model"], conversations: [ { messages: [ { role: "system", content: "${system}" }, { role: "user", content: "Hello" }, { role: "assistant", content: "Hi there!" }, ], }, ], } satisfies NormalizedData); await writeFile(cachePath, cacheContent); const proxy = new ReplayingCapiProxy( "http://localhost:9999", cachePath, workDir, ); const proxyUrl = await proxy.start(); try { const response = await makeRequest(proxyUrl, "/chat/completions", { body: { model: "test-model", messages: [ { role: "system", content: "You are helpful" }, { role: "user", content: "Hello" }, ], }, }); expect(response.status).toBe(200); const parsed = JSON.parse(response.body) as ChatCompletion; expect(parsed.choices[0].message.content).toBe("Hi there!"); } finally { await proxy.stop(); } }); test("returns cached response with tool calls", async () => { const cachePath = path.join(tempDir, "cache.yaml"); const cacheContent = yaml.stringify({ models: ["test-model"], conversations: [ { messages: [ { role: "system", content: "${system}" }, { role: "user", content: "List files" }, { role: "assistant", tool_calls: [ { id: "toolcall_0", type: "function", function: { name: "list_files", arguments: '{"path":"."}' }, }, ], }, ], }, ], } satisfies NormalizedData); await writeFile(cachePath, cacheContent); const proxy = new ReplayingCapiProxy( "http://localhost:9999", cachePath, workDir, ); const proxyUrl = await proxy.start(); try { const response = await makeRequest(proxyUrl, "/chat/completions", { body: { model: "test-model", messages: [ { role: "system", content: "System prompt" }, { role: "user", content: "List files" }, ], }, }); expect(response.status).toBe(200); const parsed = JSON.parse(response.body) as ChatCompletion; expect(parsed.choices[0].message.tool_calls).toHaveLength(1); const toolCall = parsed.choices[0].message .tool_calls![0] as ChatCompletionMessageFunctionToolCall; expect(toolCall.function.name).toBe("list_files"); } finally { await proxy.stop(); } }); test("matches shell tool results with shell ID completion markers", async () => { const originalShellConfig = process.platform === "win32" ? ShellConfig.powerShell : ShellConfig.bash; const cachePath = path.join(tempDir, "cache.yaml"); const cacheContent = yaml.stringify({ models: ["test-model"], conversations: [ { messages: [ { role: "system", content: "${system}" }, { role: "user", content: "Run command" }, { role: "assistant", tool_calls: [ { id: "toolcall_0", type: "function", function: { name: "${shell}", arguments: '{"command":"echo ok"}', }, }, ], }, { role: "tool", tool_call_id: "toolcall_0", content: "ok\n", }, { role: "assistant", content: "Done" }, ], }, ], } satisfies NormalizedData); await writeFile(cachePath, cacheContent); const proxy = new ReplayingCapiProxy( "http://localhost:9999", cachePath, workDir, ); const proxyUrl = await proxy.start(); try { const response = await makeRequest(proxyUrl, "/chat/completions", { body: { model: "test-model", messages: [ { role: "system", content: "System prompt" }, { role: "user", content: "Run command" }, { role: "assistant", tool_calls: [ { id: "runtime-call-id", type: "function", function: { name: originalShellConfig.shellToolName, arguments: '{"command":"echo ok"}', }, }, ], }, { role: "tool", tool_call_id: "runtime-call-id", content: "ok\n", }, ], }, }); expect(response.status).toBe(200); expect( (JSON.parse(response.body) as ChatCompletion).choices[0].message .content, ).toBe("Done"); } finally { await proxy.stop(); } }); test("matches semantically equivalent interrupted tool results", async () => { const originalShellConfig = process.platform === "win32" ? ShellConfig.powerShell : ShellConfig.bash; const cachePath = path.join(tempDir, "cache.yaml"); const cacheContent = yaml.stringify({ models: ["test-model"], conversations: [ { messages: [ { role: "system", content: "${system}" }, { role: "user", content: "Run command" }, { role: "assistant", tool_calls: [ { id: "toolcall_0", type: "function", function: { name: "${shell}", arguments: '{"command":"sleep 100"}', }, }, ], }, { role: "tool", tool_call_id: "toolcall_0", content: "The execution of this tool, or a previous tool was interrupted.", }, { role: "assistant", content: "Ready for another request." }, ], }, ], } satisfies NormalizedData); await writeFile(cachePath, cacheContent); const proxy = new ReplayingCapiProxy( "http://localhost:9999", cachePath, workDir, ); const proxyUrl = await proxy.start(); try { const messages = [ { role: "system", content: "System prompt" }, { role: "user", content: "Run command" }, { role: "assistant", tool_calls: [ { id: "runtime-call-id", type: "function", function: { name: originalShellConfig.shellToolName, arguments: '{"command":"sleep 100"}', }, }, ], }, ]; const interruptedResponse = await makeRequest( proxyUrl, "/chat/completions", { body: { model: "test-model", messages: [ ...messages, { role: "tool", tool_call_id: "runtime-call-id", content: "Session aborted", }, ], }, }, ); expect(interruptedResponse.status).toBe(200); expect( (JSON.parse(interruptedResponse.body) as ChatCompletion).choices[0] .message.content, ).toBe("Ready for another request."); const unknownHandleResponse = await makeRequest( proxyUrl, "/chat/completions", { body: { model: "test-model", messages: [ ...messages, { role: "tool", tool_call_id: "runtime-call-id", content: "unknown attachedShellSession handle 9", }, ], }, }, ); expect(unknownHandleResponse.status).toBe(200); const meaningfulErrorResponse = await makeRequest( proxyUrl, "/chat/completions", { body: { model: "test-model", messages: [ ...messages, { role: "tool", tool_call_id: "runtime-call-id", content: "The command failed because the executable was missing.", }, ], }, }, ); expect(meaningfulErrorResponse.status).toBe(500); } finally { await proxy.stop(); } }); test("matches available-tools results after the built-in tool set changes", async () => { const cachePath = path.join(tempDir, "cache.yaml"); // Legacy snapshot recorded before write_agent was a built-in tool: the // enumeration frozen on disk still contains the older tool list. const cacheContent = yaml.stringify({ models: ["test-model"], conversations: [ { messages: [ { role: "system", content: "${system}" }, { role: "user", content: "Report intent" }, { role: "assistant", tool_calls: [ { id: "toolcall_0", type: "function", function: { name: "report_intent", arguments: "{}" }, }, ], }, { role: "tool", tool_call_id: "toolcall_0", content: "Tool 'report_intent' does not exist. Available tools that can be called are ${shell}, view, read_agent, list_agents, grep, glob, task.", }, { role: "assistant", content: "Done" }, ], }, ], } satisfies NormalizedData); await writeFile(cachePath, cacheContent); const proxy = new ReplayingCapiProxy( "http://localhost:9999", cachePath, workDir, ); const proxyUrl = await proxy.start(); try { const response = await makeRequest(proxyUrl, "/chat/completions", { body: { model: "test-model", messages: [ { role: "system", content: "System prompt" }, { role: "user", content: "Report intent" }, { role: "assistant", tool_calls: [ { id: "runtime-call-id", type: "function", function: { name: "report_intent", arguments: "{}" }, }, ], }, { role: "tool", tool_call_id: "runtime-call-id", // Newer runtime added write_agent to the built-in tool set. content: "Tool 'report_intent' does not exist. Available tools that can be called are bash, read_bash, view, read_agent, list_agents, write_agent, grep, glob, task.", }, ], }, }); expect(response.status).toBe(200); expect( (JSON.parse(response.body) as ChatCompletion).choices[0].message .content, ).toBe("Done"); } finally { await proxy.stop(); } }); test("expands workdir placeholder in cached response", async () => { const cachePath = path.join(tempDir, "cache.yaml"); const cacheContent = yaml.stringify({ models: ["test-model"], conversations: [ { messages: [ { role: "system", content: "${system}" }, { role: "user", content: "Read file" }, { role: "assistant", tool_calls: [ { id: "toolcall_0", type: "function", function: { name: "read_file", arguments: `{"path":"${workingDirPlaceholder}/test.txt"}`, }, }, ], }, ], }, ], } satisfies NormalizedData); await writeFile(cachePath, cacheContent); const proxy = new ReplayingCapiProxy( "http://localhost:9999", cachePath, workDir, ); const proxyUrl = await proxy.start(); try { const response = await makeRequest(proxyUrl, "/chat/completions", { body: { model: "test-model", messages: [ { role: "system", content: "System" }, { role: "user", content: "Read file" }, ], }, }); expect(response.status).toBe(200); const parsed = JSON.parse(response.body) as ChatCompletion; const toolCall = parsed.choices[0].message .tool_calls![0] as ChatCompletionMessageFunctionToolCall; const args = JSON.parse(toolCall.function.arguments) as { path: string; }; expect(args.path).toBe(workDir + "/test.txt"); } finally { await proxy.stop(); } }); test("matches multi-turn conversation", async () => { const cachePath = path.join(tempDir, "cache.yaml"); const cacheContent = yaml.stringify({ models: ["test-model"], conversations: [ { messages: [ { role: "system", content: "${system}" }, { role: "user", content: "Hello" }, { role: "assistant", content: "Hi!" }, { role: "user", content: "How are you?" }, { role: "assistant", content: "I am fine!" }, ], }, ], } satisfies NormalizedData); await writeFile(cachePath, cacheContent); const proxy = new ReplayingCapiProxy( "http://localhost:9999", cachePath, workDir, ); const proxyUrl = await proxy.start(); try { // First turn const response1 = await makeRequest(proxyUrl, "/chat/completions", { body: { model: "test-model", messages: [ { role: "system", content: "Be helpful" }, { role: "user", content: "Hello" }, ], }, }); expect( (JSON.parse(response1.body) as ChatCompletion).choices[0].message .content, ).toBe("Hi!"); // Second turn const response2 = await makeRequest(proxyUrl, "/chat/completions", { body: { model: "test-model", messages: [ { role: "system", content: "Be helpful" }, { role: "user", content: "Hello" }, { role: "assistant", content: "Hi!" }, { role: "user", content: "How are you?" }, ], }, }); expect( (JSON.parse(response2.body) as ChatCompletion).choices[0].message .content, ).toBe("I am fine!"); } finally { await proxy.stop(); } }); test("matches cached task completion notification wording variants", async () => { const cachePath = path.join(tempDir, "cache.yaml"); const unreadNotification = [ "", 'Agent "read-file" (explore) has completed successfully. Use read_agent with agent_id "read-file" to retrieve unread results.', "", ].join("\n"); const idleNotification = [ "", 'Agent "read-file" (explore) has finished processing and is now idle. Use read_agent with agent_id "read-file" to read the results, or write_agent to send follow-up messages.', "", ].join("\n"); const cacheContent = yaml.stringify({ models: ["test-model"], conversations: [ { messages: [ { role: "system", content: "${system}" }, { role: "user", content: "Hello" }, { role: "assistant", content: "Hi!" }, { role: "user", content: unreadNotification }, { role: "assistant", content: "Read agent completed." }, ], }, ], } satisfies NormalizedData); await writeFile(cachePath, cacheContent); const proxy = new ReplayingCapiProxy( "http://localhost:9999", cachePath, workDir, ); const proxyUrl = await proxy.start(); try { const response = await makeRequest(proxyUrl, "/chat/completions", { body: { model: "test-model", messages: [ { role: "system", content: "Be helpful" }, { role: "user", content: "Hello" }, { role: "assistant", content: "Hi!" }, { role: "user", content: idleNotification }, ], }, }); expect(response.status).toBe(200); expect( (JSON.parse(response.body) as ChatCompletion).choices[0].message .content, ).toBe("Read agent completed."); } finally { await proxy.stop(); } }); test("replays background agent calls with the runtime-generated ID", async () => { const cachePath = path.join(tempDir, "cache.yaml"); const startResult = "Agent started in background with agent_id: read-file. You'll be notified when it completes. Tell the user you're waiting and end your response, or continue unrelated work until notified."; const notification = '\nAgent "read-file" (explore) has completed successfully. Use read_agent with agent_id "read-file" to retrieve the full results.\n'; const cacheContent = yaml.stringify({ models: ["test-model"], conversations: [ { messages: [ { role: "system", content: "${system}" }, { role: "user", content: "Read the file" }, { role: "assistant", tool_calls: [ { id: "toolcall_0", type: "function", function: { name: "task", arguments: '{"agent_type":"explore","name":"read-file","prompt":"Read it","mode":"background"}', }, }, ], }, { role: "tool", tool_call_id: "toolcall_0", content: startResult, }, { role: "assistant", content: "Waiting." }, { role: "user", content: notification }, { role: "assistant", tool_calls: [ { id: "toolcall_1", type: "function", function: { name: "read_agent", arguments: '{"agent_id":"read-file","wait":true}', }, }, ], }, ], }, ], } satisfies NormalizedData); await writeFile(cachePath, cacheContent); const proxy = new ReplayingCapiProxy( "http://localhost:9999", cachePath, workDir, ); const proxyUrl = await proxy.start(); const runtimeAgentId = "3e0c7565-6091-58cb-85bb-6cb14db23ef7"; try { const response = await makeRequest(proxyUrl, "/chat/completions", { body: { model: "test-model", messages: [ { role: "system", content: "Be helpful" }, { role: "user", content: "Read the file" }, { role: "assistant", tool_calls: [ { id: "runtime-call-id", type: "function", function: { name: "task", arguments: '{"agent_type":"explore","name":"read-file","prompt":"Read it","mode":"background"}', }, }, ], }, { role: "tool", tool_call_id: "runtime-call-id", content: startResult.replace("read-file", runtimeAgentId), }, { role: "assistant", content: "Waiting." }, { role: "user", content: notification }, ], }, }); expect(response.status).toBe(200); const parsed = JSON.parse(response.body) as ChatCompletion; const toolCall = parsed.choices[0].message .tool_calls![0] as ChatCompletionMessageFunctionToolCall; expect(JSON.parse(toolCall.function.arguments)).toEqual({ agent_id: runtimeAgentId, wait: true, }); } finally { await proxy.stop(); } }); test("matches parallel tool results regardless of arrival order", async () => { const cachePath = path.join(tempDir, "cache.yaml"); const cacheContent = yaml.stringify({ models: ["test-model"], conversations: [ { messages: [ { role: "system", content: "${system}" }, { role: "user", content: "Lookup city and country" }, { role: "assistant", tool_calls: [ { id: "toolcall_0", type: "function", function: { name: "lookup_city", arguments: '{"city":"Paris"}', }, }, { id: "toolcall_1", type: "function", function: { name: "lookup_country", arguments: '{"country":"France"}', }, }, ], }, { role: "tool", tool_call_id: "toolcall_1", content: "COUNTRY_FRANCE", }, { role: "tool", tool_call_id: "toolcall_0", content: "CITY_PARIS", }, { role: "assistant", content: "Paris is in France." }, ], }, ], } satisfies NormalizedData); await writeFile(cachePath, cacheContent); const proxy = new ReplayingCapiProxy( "http://localhost:9999", cachePath, workDir, ); const proxyUrl = await proxy.start(); try { const response = await makeRequest(proxyUrl, "/chat/completions", { body: { model: "test-model", messages: [ { role: "system", content: "Be helpful" }, { role: "user", content: "Lookup city and country" }, { role: "assistant", tool_calls: [ { id: "city-id", type: "function", function: { name: "lookup_city", arguments: '{"city":"Paris"}', }, }, { id: "country-id", type: "function", function: { name: "lookup_country", arguments: '{"country":"France"}', }, }, ], }, { role: "tool", tool_call_id: "country-id", content: "COUNTRY_FRANCE", }, { role: "tool", tool_call_id: "city-id", content: "CITY_PARIS" }, ], }, }); expect(response.status).toBe(200); expect( (JSON.parse(response.body) as ChatCompletion).choices[0].message .content, ).toBe("Paris is in France."); } finally { await proxy.stop(); } }); test("returns streaming response when stream: true", async () => { const cachePath = path.join(tempDir, "cache.yaml"); const cacheContent = yaml.stringify({ models: ["test-model"], conversations: [ { messages: [ { role: "system", content: "${system}" }, { role: "user", content: "Hello" }, { role: "assistant", content: "Hi there!" }, ], }, ], } satisfies NormalizedData); await writeFile(cachePath, cacheContent); const proxy = new ReplayingCapiProxy( "http://localhost:9999", cachePath, workDir, ); const proxyUrl = await proxy.start(); try { const response = await makeRequest(proxyUrl, "/chat/completions", { body: { model: "test-model", messages: [ { role: "system", content: "You are helpful" }, { role: "user", content: "Hello" }, ], stream: true, }, }); expect(response.status).toBe(200); expect(response.body).toContain("data: "); expect(response.body).toContain("[DONE]"); // Parse the SSE chunk const dataLine = response.body .split("\n") .find((line) => line.startsWith("data: {")); expect(dataLine).toBeDefined(); const chunk = JSON.parse(dataLine!.slice(6)) as ChatCompletionChunk; expect(chunk.object).toBe("chat.completion.chunk"); expect(chunk.choices[0].delta.content).toBe("Hi there!"); } finally { await proxy.stop(); } }); test.each([false, true])( "defaults to Sonnet 5 without stored models (capture exists: %s)", async (captureExists) => { const cachePath = path.join(tempDir, "cache.yaml"); if (captureExists) { await writeFile( cachePath, yaml.stringify({ models: [], conversations: [], } satisfies NormalizedData), ); } const proxy = new ReplayingCapiProxy( "http://localhost:9999", cachePath, workDir, ); const proxyUrl = await proxy.start(); try { const response = await makeRequest(proxyUrl, "/models", { method: "GET", }); expect(response.status).toBe(200); const parsed = JSON.parse(response.body) as { data: Array<{ id: string }>; }; expect(parsed.data.map((model) => model.id)).toEqual(["claude-sonnet-5"]); } finally { await proxy.stop(); } }, ); test("returns cached models for /models endpoint", async () => { const cachePath = path.join(tempDir, "cache.yaml"); const cacheContent = yaml.stringify({ models: ["gpt-4o", "claude-sonnet-4"], conversations: [], } satisfies NormalizedData); await writeFile(cachePath, cacheContent); const proxy = new ReplayingCapiProxy( "http://localhost:9999", cachePath, workDir, ); const proxyUrl = await proxy.start(); try { const response = await makeRequest(proxyUrl, "/models", { method: "GET", }); expect(response.status).toBe(200); const parsed = JSON.parse(response.body) as { data: Array<{ id: string; name: string }>; }; expect(parsed.data).toHaveLength(2); expect(parsed.data[0].id).toBe("gpt-4o"); expect(parsed.data[1].id).toBe("claude-sonnet-4"); } finally { await proxy.stop(); } }); }); });