diff --git a/CHANGELOG.md b/CHANGELOG.md index 49975ea7..a0ed806a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- `aimock.json` now honours `llm.latency`, `llm.chunkSize`, `llm.replaySpeed` and `llm.logLevel`, which the config loader previously accepted and ignored. Each behaves as its `llmock` CLI flag and `createMockSuite({ llm })` equivalent. `llm.replaySpeed` divides every delay source (recorded timings, streaming profiles, `latency`), so a fixture suite can replay recorded inter-chunk timings faster than they were recorded; a fixture's own `replaySpeed` still takes precedence. A non-positive `llm.replaySpeed` is ignored with a warning, matching the existing fixture-level guard (#294) + ## [1.36.0] - 2026-07-13 ### Added diff --git a/docs/aimock-cli/index.html b/docs/aimock-cli/index.html index 2ad382de..20900965 100644 --- a/docs/aimock-cli/index.html +++ b/docs/aimock-cli/index.html @@ -135,9 +135,9 @@

Config Fields

object LLMock configuration. Accepts fixtures, latency, - chunkSize, logLevel, validateOnLoad, - metrics, strict, chaos, - streamingProfile, record. + chunkSize, replaySpeed, logLevel, + validateOnLoad, metrics, strict, + chaos, streamingProfile, record. diff --git a/src/__tests__/config-loader.test.ts b/src/__tests__/config-loader.test.ts index 78a814a4..7f03af2e 100644 --- a/src/__tests__/config-loader.test.ts +++ b/src/__tests__/config-loader.test.ts @@ -1,9 +1,11 @@ -import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { loadConfig, startFromConfig } from "../config-loader.js"; import type { AimockConfig } from "../config-loader.js"; +import type { RecordedTimings } from "../types.js"; +import { Logger } from "../logger.js"; function makeTmpDir(): string { return mkdtempSync(join(tmpdir(), "config-loader-test-")); @@ -32,6 +34,47 @@ function writeFixtureFile(dir: string, name = "fixtures.json"): string { return filePath; } +// Content is long enough to split into several chunks so per-chunk delays are observable +function writeStreamFixtureFile( + dir: string, + recordedTimings?: RecordedTimings, + name = "stream-fixtures.json", +): string { + const filePath = join(dir, name); + writeFileSync( + filePath, + JSON.stringify({ + fixtures: [ + { + match: { userMessage: "hello" }, + response: { content: "Hello world from the streaming config test!" }, + ...(recordedTimings ? { recordedTimings } : {}), + }, + ], + }), + "utf-8", + ); + return filePath; +} + +// Streams a completion and counts SSE frames. Without stream: true there is no SSE and no +// chunk delays, so the timing assertions below would pass whether or not the options are wired. +async function streamChunkCount(url: string): Promise { + const resp = await fetch(`${url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4", + stream: true, + messages: [{ role: "user", content: "hello" }], + }), + }); + const body = await resp.text(); + return body + .split("\n\n") + .filter((frame) => frame.startsWith("data: ") && !frame.includes("[DONE]")).length; +} + describe("loadConfig", () => { let tmpDir: string; @@ -156,6 +199,108 @@ describe("startFromConfig", () => { expect(resp.status).toBe(500); }); + it("with llm.chunkSize, the response is split into more chunks", async () => { + const fixturePath = writeStreamFixtureFile(tmpDir); + + const small = await startFromConfig({ llm: { fixtures: fixturePath, chunkSize: 5 } }); + cleanups.push(() => small.llmock.stop()); + const smallChunks = await streamChunkCount(small.url); + + const large = await startFromConfig({ llm: { fixtures: fixturePath, chunkSize: 1000 } }); + cleanups.push(() => large.llmock.stop()); + const largeChunks = await streamChunkCount(large.url); + + expect(smallChunks).toBeGreaterThan(largeChunks); + }); + + it("with llm.latency, streamed chunks are delayed", async () => { + const fixturePath = writeStreamFixtureFile(tmpDir); + const config: AimockConfig = { + llm: { fixtures: fixturePath, chunkSize: 5, latency: 100 }, + }; + const { llmock, url } = await startFromConfig(config); + cleanups.push(() => llmock.stop()); + + const start = Date.now(); + const chunks = await streamChunkCount(url); + const elapsed = Date.now() - start; + + // 100ms per chunk across >= 5 chunks. Asserted as a lower bound so a slow + // machine can never fail it; without the passthrough latency is 0. + expect(chunks).toBeGreaterThanOrEqual(5); + expect(elapsed).toBeGreaterThanOrEqual(250); + }); + + it("with llm.replaySpeed, recorded timings replay faster", async () => { + // 500ms TTFT + 4 x 500ms inter-chunk = ~2500ms at 1x speed + const fixturePath = writeStreamFixtureFile(tmpDir, { + ttftMs: 500, + interChunkDelaysMs: [500, 500, 500, 500], + totalDurationMs: 2500, + }); + const config: AimockConfig = { + llm: { fixtures: fixturePath, chunkSize: 5, replaySpeed: 100 }, + }; + const { llmock, url } = await startFromConfig(config); + cleanups.push(() => llmock.stop()); + + const start = Date.now(); + await streamChunkCount(url); + const elapsed = Date.now() - start; + + // At 100x this lands in ~25ms; the bound is deliberately generous so only a + // dropped passthrough (which costs the full ~2500ms) can fail it. + expect(elapsed).toBeLessThan(1000); + }); + + it("with non-positive llm.replaySpeed, warns and ignores it", async () => { + const fixturePath = writeStreamFixtureFile(tmpDir); + // Spy on the logger the code path actually uses rather than console, so the + // assertion verifies the intended warn call without coupling to the + // "[aimock]" prefix or the console transport. + const warn = vi.spyOn(Logger.prototype, "warn").mockImplementation(() => {}); + + // 0 looks like "no delay" but hits the `speed > 0` guard in calculateDelay, + // which would replay at full recorded speed. + const { llmock } = await startFromConfig({ + llm: { fixtures: fixturePath, replaySpeed: 0 }, + }); + cleanups.push(() => llmock.stop()); + + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("llm.replaySpeed must be a positive number"), + ); + warn.mockRestore(); + }); + + it("with llm.logLevel debug, the server logs debug output that silent suppresses", async () => { + const fixturePath = writeFixtureFile(tmpDir); + // The server logger writes debug/info lines via console.log; capture them so + // we can assert the configured level actually reaches the running server. + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + // A matched request emits a "Fixture matched" debug line, but only when the + // configured logLevel is high enough — so it is a direct probe of the passthrough. + async function debugLineCount(logLevel: "debug" | "silent"): Promise { + log.mockClear(); + const { llmock, url } = await startFromConfig({ + llm: { fixtures: fixturePath, logLevel }, + }); + cleanups.push(() => llmock.stop()); + await fetch(`${url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: "gpt-4", messages: [{ role: "user", content: "hello" }] }), + }); + return log.mock.calls.filter((args) => String(args[1]).startsWith("Fixture matched")).length; + } + + expect(await debugLineCount("debug")).toBeGreaterThan(0); + expect(await debugLineCount("silent")).toBe(0); + + log.mockRestore(); + }); + it("with mcp tools config, MCPMock created and tools/list works", async () => { const config: AimockConfig = { mcp: { @@ -649,11 +794,8 @@ describe("startFromConfig", () => { { pattern: "stream", events: [ - { - kind: "status-update", - taskId: "t1", - status: { state: "working", message: { parts: [{ text: "streaming..." }] } }, - }, + { type: "status", state: "TASK_STATE_WORKING" }, + { type: "artifact", parts: [{ text: "streaming..." }], name: "out" }, ], delayMs: 0, }, @@ -670,6 +812,28 @@ describe("startFromConfig", () => { expect(cardRes.status).toBe(200); const card = await cardRes.json(); expect(card.name).toBe("stream-agent"); + + // Verify the streaming task actually streams the configured events over SSE + const streamRes = await fetch(`${url}/a2a`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + jsonrpc: "2.0", + method: "SendStreamingMessage", + params: { message: { parts: [{ text: "please stream" }] } }, + id: 1, + }), + }); + expect(streamRes.status).toBe(200); + expect(streamRes.headers.get("content-type")).toBe("text/event-stream"); + const streamBody = await streamRes.text(); + const events = streamBody + .split("\n\n") + .filter((frame) => frame.startsWith("data: ")) + .map((frame) => JSON.parse(frame.slice("data: ".length))); + expect(events.length).toBe(2); + expect(events[0].result.task.status.state).toBe("TASK_STATE_WORKING"); + expect(events[1].result.artifact.parts[0].text).toBe("streaming..."); }); it("with a2a custom path, mounts at specified path for tasks", async () => { diff --git a/src/config-loader.ts b/src/config-loader.ts index a02e4500..871904ca 100644 --- a/src/config-loader.ts +++ b/src/config-loader.ts @@ -90,6 +90,10 @@ export interface VectorConfig { export interface AimockConfig { llm?: { fixtures?: string; + latency?: number; + chunkSize?: number; + replaySpeed?: number; + logLevel?: "silent" | "warn" | "info" | "debug"; chaos?: ChaosConfig; record?: RecordConfig; }; @@ -115,10 +119,22 @@ export async function startFromConfig( ): Promise<{ llmock: LLMock; url: string }> { const logger = new Logger("info"); + // A non-positive replaySpeed fails calculateDelay's `speed > 0` check and applies the + // full delay rather than none. Mirrors the fixture-level guard in fixture-loader.ts. + let replaySpeed = config.llm?.replaySpeed; + if (replaySpeed != null && (!Number.isFinite(replaySpeed) || replaySpeed <= 0)) { + logger.warn(`llm.replaySpeed must be a positive number, got ${replaySpeed}. Ignoring.`); + replaySpeed = undefined; + } + // Load fixtures if specified const llmock = new LLMock({ port: overrides?.port ?? config.port ?? 0, host: overrides?.host ?? config.host ?? "127.0.0.1", + latency: config.llm?.latency, + chunkSize: config.llm?.chunkSize, + replaySpeed, + logLevel: config.llm?.logLevel, chaos: config.llm?.chaos, record: config.llm?.record, metrics: config.metrics,