From 07221fa8ef2ac63a6db8d3dad94c526644fab3e7 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Wed, 13 May 2026 09:21:41 -0700 Subject: [PATCH] feat: record systemHash and toolsHash as fixture metadata --- src/__tests__/recorder.test.ts | 150 +++++++++++++++++++++++++++++++++ src/recorder.ts | 34 ++++++++ src/types.ts | 8 ++ 3 files changed, 192 insertions(+) diff --git a/src/__tests__/recorder.test.ts b/src/__tests__/recorder.test.ts index 98aac469..7491ac4b 100644 --- a/src/__tests__/recorder.test.ts +++ b/src/__tests__/recorder.test.ts @@ -4618,3 +4618,153 @@ describe("recorder Bedrock binary event stream progressive streaming", () => { expect(clientCT.toLowerCase()).toContain("application/vnd.amazon.eventstream"); }); }); + +// --------------------------------------------------------------------------- +// Fixture metadata recording (systemHash / toolsHash) +// --------------------------------------------------------------------------- + +describe("fixture metadata recording", () => { + let metaUpstream: ServerInstance | undefined; + let metaRecorder: ServerInstance | undefined; + let metaTmpDir: string | undefined; + + afterEach(async () => { + if (metaRecorder) { + await new Promise((resolve) => metaRecorder!.server.close(() => resolve())); + metaRecorder = undefined; + } + if (metaUpstream) { + await new Promise((resolve) => metaUpstream!.server.close(() => resolve())); + metaUpstream = undefined; + } + if (metaTmpDir) { + fs.rmSync(metaTmpDir, { recursive: true, force: true }); + metaTmpDir = undefined; + } + }); + + async function setupMetaRecorder(): Promise<{ recorderUrl: string; fixturePath: string }> { + metaUpstream = await createServer( + [ + { + match: { userMessage: "test metadata sys" }, + response: { content: "ok sys" }, + }, + { + match: { userMessage: "test metadata tools" }, + response: { content: "ok tools" }, + }, + { + match: { userMessage: "test no metadata" }, + response: { content: "ok none" }, + }, + { + match: { userMessage: "first hash probe" }, + response: { content: "ok diff" }, + }, + { + match: { userMessage: "second hash probe" }, + response: { content: "ok diff two" }, + }, + ], + { port: 0 }, + ); + metaTmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "aimock-meta-")); + metaRecorder = await createServer([], { + port: 0, + record: { + providers: { openai: metaUpstream.url }, + fixturePath: metaTmpDir, + }, + }); + return { recorderUrl: metaRecorder.url, fixturePath: metaTmpDir }; + } + + it("records systemHash when system message present", async () => { + const { recorderUrl, fixturePath } = await setupMetaRecorder(); + + await post(`${recorderUrl}/v1/chat/completions`, { + model: "gpt-4o", + messages: [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "test metadata sys" }, + ], + }); + + const files = fs.readdirSync(fixturePath).filter((f) => f.endsWith(".json")); + expect(files.length).toBeGreaterThanOrEqual(1); + const fixture = JSON.parse( + fs.readFileSync(path.join(fixturePath, files[0]), "utf-8"), + ) as FixtureFile; + expect(fixture.fixtures[0].metadata).toBeDefined(); + expect(fixture.fixtures[0].metadata!.systemHash).toMatch(/^[a-f0-9]{8}$/); + expect(fixture.fixtures[0].metadata!.toolsHash).toBeUndefined(); + }); + + it("records toolsHash when tools present", async () => { + const { recorderUrl, fixturePath } = await setupMetaRecorder(); + + await post(`${recorderUrl}/v1/chat/completions`, { + model: "gpt-4o", + messages: [{ role: "user", content: "test metadata tools" }], + tools: [{ type: "function", function: { name: "get_weather", parameters: {} } }], + }); + + const files = fs.readdirSync(fixturePath).filter((f) => f.endsWith(".json")); + expect(files.length).toBeGreaterThanOrEqual(1); + const fixture = JSON.parse( + fs.readFileSync(path.join(fixturePath, files[0]), "utf-8"), + ) as FixtureFile; + expect(fixture.fixtures[0].metadata).toBeDefined(); + expect(fixture.fixtures[0].metadata!.toolsHash).toMatch(/^[a-f0-9]{8}$/); + }); + + it("omits metadata when no system message or tools", async () => { + const { recorderUrl, fixturePath } = await setupMetaRecorder(); + + await post(`${recorderUrl}/v1/chat/completions`, { + model: "gpt-4o", + messages: [{ role: "user", content: "test no metadata" }], + }); + + const files = fs.readdirSync(fixturePath).filter((f) => f.endsWith(".json")); + expect(files.length).toBeGreaterThanOrEqual(1); + const fixture = JSON.parse( + fs.readFileSync(path.join(fixturePath, files[0]), "utf-8"), + ) as FixtureFile; + expect(fixture.fixtures[0].metadata).toBeUndefined(); + }); + + it("produces different systemHash for different system prompts", async () => { + const { recorderUrl, fixturePath } = await setupMetaRecorder(); + + // Use different user messages so the second request also proxies to upstream + // (same userMessage would match the first recorded fixture in memory). + await post(`${recorderUrl}/v1/chat/completions`, { + model: "gpt-4o", + messages: [ + { role: "system", content: "Generate a title." }, + { role: "user", content: "first hash probe" }, + ], + }); + + await post(`${recorderUrl}/v1/chat/completions`, { + model: "gpt-4o", + messages: [ + { role: "system", content: "Generate suggestions." }, + { role: "user", content: "second hash probe" }, + ], + }); + + const files = fs.readdirSync(fixturePath).filter((f) => f.endsWith(".json")); + expect(files.length).toBe(2); + const fixtures = files.map( + (f) => JSON.parse(fs.readFileSync(path.join(fixturePath, f), "utf-8")) as FixtureFile, + ); + const hash1 = fixtures[0].fixtures[0].metadata?.systemHash; + const hash2 = fixtures[1].fixtures[0].metadata?.systemHash; + expect(hash1).toBeDefined(); + expect(hash2).toBeDefined(); + expect(hash1).not.toBe(hash2); + }); +}); diff --git a/src/recorder.ts b/src/recorder.ts index e0648048..a0bca481 100644 --- a/src/recorder.ts +++ b/src/recorder.ts @@ -334,10 +334,14 @@ export async function proxyAndRecord( const matchRequest = defaults.requestTransform ? defaults.requestTransform(request) : request; const fixtureMatch = buildFixtureMatch(matchRequest); + // Build metadata (drift-detection hashes, not match criteria) + const metadata = buildFixtureMetadata(request); + // Build and save the fixture const fixture: Fixture = { match: fixtureMatch, response: fixtureResponse, + ...(metadata && { metadata }), }; // Check if the match is empty — warn but still save to disk. @@ -1214,3 +1218,33 @@ function buildFixtureMatch(request: ChatCompletionRequest): { return match; } + +/** + * Build optional metadata for drift detection. Contains 8-char SHA-256 + * hashes of the system prompt and tool definitions present in the request. + * Returns undefined when neither is present. + */ +function buildFixtureMetadata( + request: ChatCompletionRequest, +): { systemHash?: string; toolsHash?: string } | undefined { + const meta: { systemHash?: string; toolsHash?: string } = {}; + + const messages = request.messages ?? []; + const systemTexts = messages + .filter((m) => m.role === "system") + .map((m) => (typeof m.content === "string" ? m.content : JSON.stringify(m.content))) + .join("\n"); + if (systemTexts) { + meta.systemHash = crypto.createHash("sha256").update(systemTexts).digest("hex").slice(0, 8); + } + + if (request.tools && request.tools.length > 0) { + meta.toolsHash = crypto + .createHash("sha256") + .update(JSON.stringify(request.tools)) + .digest("hex") + .slice(0, 8); + } + + return Object.keys(meta).length > 0 ? meta : undefined; +} diff --git a/src/types.ts b/src/types.ts index 4b3ab11e..10f8b700 100644 --- a/src/types.ts +++ b/src/types.ts @@ -268,6 +268,10 @@ export interface Fixture { disconnectAfterMs?: number; streamingProfile?: StreamingProfile; chaos?: ChaosConfig; + metadata?: { + systemHash?: string; + toolsHash?: string; + }; } export type FixtureOpts = Omit; @@ -359,6 +363,10 @@ export interface FixtureFileEntry { disconnectAfterMs?: number; streamingProfile?: StreamingProfile; chaos?: ChaosConfig; + metadata?: { + systemHash?: string; + toolsHash?: string; + }; } // Request journal