diff --git a/test/harness/replayingCapiProxy.test.ts b/test/harness/replayingCapiProxy.test.ts index 803226548..009a1e40a 100644 --- a/test/harness/replayingCapiProxy.test.ts +++ b/test/harness/replayingCapiProxy.test.ts @@ -509,6 +509,47 @@ Always include PINEAPPLE_COCONUT_42. expect(toolMessages[1].content).toBe("[beta result]"); }); + test("collapses the available-tools list to a stable placeholder", async () => { + const requestBody = JSON.stringify({ + messages: [ + { role: "user", content: "Help me" }, + { + role: "assistant", + tool_calls: [ + { + id: "tc1", + type: "function", + function: { name: "report_intent", arguments: "{}" }, + }, + ], + }, + { + role: "tool", + tool_call_id: "tc1", + content: + "Tool 'report_intent' does not exist. Available tools that can be called are bash, read_bash, view, read_agent, list_agents, write_agent, grep, glob, task.", + }, + ], + }); + const responseBody = JSON.stringify({ + choices: [{ message: { role: "assistant", content: "Done" } }], + }); + + const outputPath = await createProxy([ + { url: "/chat/completions", requestBody, responseBody }, + ]); + + const result = await readYamlOutput(outputPath); + const toolMessage = result.conversations[0].messages.find( + (m) => m.role === "tool", + ); + // The whole enumeration collapses so snapshots stay stable as the built-in + // tool set evolves (e.g. write_agent being added). + expect(toolMessage?.content).toBe( + "Tool 'report_intent' does not exist. Available tools that can be called are ${available_tools}.", + ); + }); + test("normalizes read_agent timing metadata", async () => { const requestBody = JSON.stringify({ messages: [ @@ -827,6 +868,85 @@ Always include PINEAPPLE_COCONUT_42. } }); + test("matches available-tools results after the built-in tool set changes", async () => { + const cachePath = path.join(tempDir, "cache.yaml"); + // Legacy snapshot recorded before write_agent was a built-in tool: the + // enumeration frozen on disk still contains the older tool list. + const cacheContent = yaml.stringify({ + models: ["test-model"], + conversations: [ + { + messages: [ + { role: "system", content: "${system}" }, + { role: "user", content: "Report intent" }, + { + role: "assistant", + tool_calls: [ + { + id: "toolcall_0", + type: "function", + function: { name: "report_intent", arguments: "{}" }, + }, + ], + }, + { + role: "tool", + tool_call_id: "toolcall_0", + content: + "Tool 'report_intent' does not exist. Available tools that can be called are ${shell}, view, read_agent, list_agents, grep, glob, task.", + }, + { role: "assistant", content: "Done" }, + ], + }, + ], + } satisfies NormalizedData); + await writeFile(cachePath, cacheContent); + + const proxy = new ReplayingCapiProxy( + "http://localhost:9999", + cachePath, + workDir, + ); + const proxyUrl = await proxy.start(); + + try { + const response = await makeRequest(proxyUrl, "/chat/completions", { + body: { + model: "test-model", + messages: [ + { role: "system", content: "System prompt" }, + { role: "user", content: "Report intent" }, + { + role: "assistant", + tool_calls: [ + { + id: "runtime-call-id", + type: "function", + function: { name: "report_intent", arguments: "{}" }, + }, + ], + }, + { + role: "tool", + tool_call_id: "runtime-call-id", + // Newer runtime added write_agent to the built-in tool set. + content: + "Tool 'report_intent' does not exist. Available tools that can be called are bash, read_bash, view, read_agent, list_agents, write_agent, grep, glob, task.", + }, + ], + }, + }); + + expect(response.status).toBe(200); + expect( + (JSON.parse(response.body) as ChatCompletion).choices[0].message + .content, + ).toBe("Done"); + } finally { + await proxy.stop(); + } + }); + test("expands workdir placeholder in cached response", async () => { const cachePath = path.join(tempDir, "cache.yaml"); const cacheContent = yaml.stringify({ diff --git a/test/harness/replayingCapiProxy.ts b/test/harness/replayingCapiProxy.ts index 740f8ab74..6a9271de2 100644 --- a/test/harness/replayingCapiProxy.ts +++ b/test/harness/replayingCapiProxy.ts @@ -133,6 +133,7 @@ export class ReplayingCapiProxy extends CapturingHttpProxy { this.state.storedData = yaml.parse(content) as NormalizedData; normalizeToolResultOrder(this.state.storedData.conversations); normalizeStoredUserMessages(this.state.storedData.conversations); + normalizeStoredToolMessages(this.state.storedData.conversations); } } @@ -1079,6 +1080,22 @@ function normalizeStoredUserMessages(conversations: NormalizedConversation[]) { } } +// Re-normalizes the built-in tool enumeration in stored tool results at load +// time. Snapshots recorded before normalizeAvailableToolNames collapsed the +// whole list (or recorded against an older tool set) still contain the literal +// enumeration on disk; the result normalizers only run against live requests, +// so without this the stored side would keep the stale list and never match a +// request whose tool set has since changed. +function normalizeStoredToolMessages(conversations: NormalizedConversation[]) { + for (const conversation of conversations) { + for (const message of conversation.messages) { + if (message.role === "tool" && typeof message.content === "string") { + message.content = normalizeAvailableToolNames(message.content); + } + } + } +} + function normalizeSkillContextFrontmatter(content: string): string { // Runtime versions may include or omit SKILL.md metadata in the prompt context. return content.replace( @@ -1169,41 +1186,21 @@ function normalizeReadAgentTimings(result: string): string { .replace(/\bduration: \d+(?:\.\d+)?s\b/g, "duration: 0s"); } -// Maps the platform-specific shell tool family names to stable placeholders. -// On Windows the runtime exposes powershell/read_powershell/stop_powershell/..., -// on Linux/macOS it exposes bash/read_bash/stop_bash/.... Ordered so that the -// prefixed names are handled explicitly; \b boundaries keep bare names from -// matching inside the prefixed ones. -const shellToolFamilyReplacements: ReadonlyArray = [ - [/\bread_powershell\b/g, "${read_shell}"], - [/\bstop_powershell\b/g, "${stop_shell}"], - [/\blist_powershell\b/g, "${list_shell}"], - [/\bwrite_powershell\b/g, "${write_shell}"], - [/\bpowershell\b/g, "${shell}"], - [/\bread_bash\b/g, "${read_shell}"], - [/\bstop_bash\b/g, "${stop_shell}"], - [/\blist_bash\b/g, "${list_shell}"], - [/\bwrite_bash\b/g, "${write_shell}"], - [/\bbash\b/g, "${shell}"], -]; - -function normalizeShellToolFamilyNames(text: string): string { - let result = text; - for (const [pattern, replacement] of shellToolFamilyReplacements) { - result = result.replace(pattern, replacement); - } - return result; -} +// Stable placeholder for the built-in tool enumeration the runtime emits when a +// nonexistent tool is called (see normalizeAvailableToolNames). +export const availableToolsPlaceholder = "${available_tools}"; // When a model calls a tool that doesn't exist (e.g., the removed report_intent // tool), the runtime replies with "Available tools that can be called are ." -// The shell tool family names in that list are platform-specific, so normalize -// them to placeholders to keep snapshots matching across Windows/Linux/macOS. +// That enumeration is both platform-specific (shell tool family names differ +// across OSes) and runtime-version-specific (built-in tools such as write_agent +// are added or removed over time), so any test that trips this path would break +// whenever the tool set changes. Collapse the whole list to a stable placeholder +// so snapshots keep matching as the built-in tool set evolves. function normalizeAvailableToolNames(result: string): string { return result.replace( - /(Available tools that can be called are )([^.]*)/g, - (_full, prefix: string, list: string) => - prefix + normalizeShellToolFamilyNames(list), + /(Available tools that can be called are )[^.]*/g, + (_full, prefix: string) => prefix + availableToolsPlaceholder, ); }