From 4aed415a7132ac17c2070ba6dbf51ce77adf374e Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Thu, 14 May 2026 15:35:05 +0200 Subject: [PATCH] fix(playground): saved-replay sidebar + chat resilience (v0.2.3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Saved-replay sidebar reflects out-of-band fixture deletions within ~1 s via a Node fs.watch on .copilotkit/fixtures/ (vscode's FileSystemWatcher was unreliable for dot-prefixed paths on Windows). - Replay ▶ no longer drops the recorded conversation: the App shell unmounts the stale bundle on bundle-ready, queues play-fixture, and dispatches it after the new PlaygroundChat mounts its listener. - Clicking ▶ on a fixture whose file is gone no longer bricks the panel — load-fixture validates first, refreshes the sidebar, warns; runBundle recovers mid-session by falling back to record mode. - Refresh always reconciles the sidebar with disk at the start of every rebundle (guaranteed manual recovery path). - Empty vscode.lm streams from large tool budgets (Claude via Copilot Chat with ~80+ tools) auto-retry once without vscode.lm tools so the user gets a real response on the same query. If both attempts return empty, an in-chat message explains the cause. - LanguageModelDataPart text/json mimes are surfaced as text instead of being silently dropped. Unknown stream-part types are logged so future model changes leave a breadcrumb. --- CHANGELOG.md | 15 + package.json | 2 +- src/extension/activate.ts | 21 ++ .../__tests__/fixtures-dir-watcher.test.ts | 92 ++++++ .../__tests__/view-provider.test.ts | 310 +++++++++++++++++- .../__tests__/vscode-lm-factory.test.ts | 107 ++++++ .../playground/fixtures-dir-watcher.ts | 65 ++++ src/extension/playground/view-provider.ts | 72 +++- src/extension/playground/vscode-lm-factory.ts | 101 +++++- src/webview/playground/App.tsx | 39 ++- .../__tests__/App.replay-race.test.tsx | 119 +++++++ test-workspace/playground/v2/Tools.tsx | 18 +- 12 files changed, 925 insertions(+), 36 deletions(-) create mode 100644 src/extension/playground/__tests__/fixtures-dir-watcher.test.ts create mode 100644 src/extension/playground/fixtures-dir-watcher.ts create mode 100644 src/webview/playground/__tests__/App.replay-race.test.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index d1c32233d..07bb1df91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ ## [Unreleased] +## 0.2.3 — 2026-05-14 + +### Fixed + +- Playground saved-replay sidebar: deleting a fixture file out of band (Explorer, terminal, git pull) now refreshes the sidebar within ~1 s via a Node `fs.watch` on `.copilotkit/fixtures/`. The previous `vscode.workspace.createFileSystemWatcher` was unreliable for dot-prefixed paths on Windows; the entry would stick around until the next reload. +- Playground ▶ on a saved replay no longer drops the recorded conversation. The previous race posted `play-fixture` before the new bundle's `PlaygroundChat` had registered its replay listener; the App shell now unmounts the stale bundle synchronously on `bundle-ready`, queues the replay messages, and dispatches them after the new chat mounts (React runs child effects before parent effects on the same commit, so the listener is guaranteed to be attached). +- Playground recovery: clicking ▶ on a fixture whose file has been deleted no longer permanently bricks the panel ("Preparing chat surface…" + ENOENT loop). `load-fixture` now reads the file before committing `replayFixturePath`, refreshes the sidebar, and surfaces a one-shot warning. The `runBundle` pass also recovers when the active fixture vanishes mid-session by falling back to record mode instead of crashing. +- Playground Refresh button: always reconciles the saved-replays sidebar with disk at the start of every rebundle. Refresh is now a guaranteed manual recovery path even if the fs.watch misses an event. +- Playground chat with many `vscode.lm` tools: some models (notably Claude through Copilot Chat with ~80+ tools forwarded) silently reject requests by returning 200 OK with an empty stream. The chat now auto-retries once without the `vscode.lm` tools so the user gets a real response on the same query. If both attempts return empty, an actionable in-chat message explains the cause and points at the `copilotkit.playground.enableVscodeLmTools` setting. +- Playground stream translation: text content streamed as `LanguageModelDataPart` (text/plain or application/json mimes) is surfaced as `TEXT_MESSAGE_CONTENT` instead of being silently dropped. Newer Claude builds routed through Copilot Chat had been producing empty chats this way. + +### Added + +- Playground output channel now logs unknown `vscode.lm` stream-part types (constructor name + mime when applicable) so future model-side changes leave a breadcrumb instead of an empty chat. + ## 0.2.2 — 2026-05-05 ### Fixed diff --git a/package.json b/package.json index ef5ab911d..be43411e2 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "copilotkit-vscode-extension", "displayName": "CopilotKit", - "version": "0.2.2", + "version": "0.2.3", "description": "Preview generative-UI components, explore CopilotKit hooks, and inspect AG-UI agent runs — all without leaving your editor.", "categories": [ "Other", diff --git a/src/extension/activate.ts b/src/extension/activate.ts index 811b6fd8a..fea729a85 100644 --- a/src/extension/activate.ts +++ b/src/extension/activate.ts @@ -21,6 +21,7 @@ import { } from "./playground/view-provider"; import { scanPlayground } from "./playground/scanner"; import { PlaygroundFileWatcher } from "./playground/file-watcher"; +import { FixturesDirWatcher } from "./playground/fixtures-dir-watcher"; let activePlaygroundProvider: PlaygroundViewProvider | null = null; @@ -343,6 +344,26 @@ export function activate(context: vscode.ExtensionContext): void { runPlaygroundScan(); }); context.subscriptions.push(playgroundFileWatcher); + + // Reflect manual fixture changes (Explorer delete, git pull, etc.) + // in the saved-replays sidebar. We use a Node fs.watch on the + // fixtures directory rather than vscode.createFileSystemWatcher — + // the latter is unreliable for dot-prefixed paths like + // `.copilotkit/` on Windows. The webview-initiated save/delete + // round-trip already pushes a fresh list, so this watcher is for + // out-of-band edits only. + const fixturesDir = path.join(workspaceRoot, ".copilotkit", "fixtures"); + const fixturesDirWatcher = new FixturesDirWatcher( + fixturesDir, + () => { + playgroundOutputChannel.appendLine( + "[playground] fixtures dir changed — refreshing sidebar", + ); + playgroundProvider.refreshFixturesList(); + }, + (line) => playgroundOutputChannel.appendLine(line), + ); + context.subscriptions.push(fixturesDirWatcher); } runPlaygroundScan(); diff --git a/src/extension/playground/__tests__/fixtures-dir-watcher.test.ts b/src/extension/playground/__tests__/fixtures-dir-watcher.test.ts new file mode 100644 index 000000000..e1dd65f44 --- /dev/null +++ b/src/extension/playground/__tests__/fixtures-dir-watcher.test.ts @@ -0,0 +1,92 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("vscode", () => ({})); + +import { FixturesDirWatcher } from "../fixtures-dir-watcher"; + +describe("FixturesDirWatcher", () => { + let tmpRoot: string; + let watcher: FixturesDirWatcher | null = null; + + beforeEach(() => { + tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), "fixtures-watcher-")); + }); + + afterEach(() => { + watcher?.dispose(); + watcher = null; + fs.rmSync(tmpRoot, { recursive: true, force: true }); + }); + + it("fires onChange when a file inside the watched dir is deleted", async () => { + const fixturesDir = path.join(tmpRoot, ".copilotkit", "fixtures"); + fs.mkdirSync(fixturesDir, { recursive: true }); + const filePath = path.join(fixturesDir, "x.json"); + fs.writeFileSync(filePath, "{}"); + + const onChange = vi.fn(); + watcher = new FixturesDirWatcher(fixturesDir, onChange); + + fs.unlinkSync(filePath); + + await waitFor(() => onChange.mock.calls.length > 0); + expect(onChange).toHaveBeenCalled(); + }); + + it("fires onChange when a file is created in the watched dir", async () => { + const fixturesDir = path.join(tmpRoot, ".copilotkit", "fixtures"); + fs.mkdirSync(fixturesDir, { recursive: true }); + + const onChange = vi.fn(); + watcher = new FixturesDirWatcher(fixturesDir, onChange); + + fs.writeFileSync(path.join(fixturesDir, "new.json"), "{}"); + + await waitFor(() => onChange.mock.calls.length > 0); + expect(onChange).toHaveBeenCalled(); + }); + + it("is a no-op when the directory does not exist yet", () => { + const fixturesDir = path.join(tmpRoot, "absent", "fixtures"); + const log = vi.fn(); + const onChange = vi.fn(); + watcher = new FixturesDirWatcher(fixturesDir, onChange, log); + + // Nothing thrown, no attachment log line. + expect(log).not.toHaveBeenCalledWith( + expect.stringMatching(/attached/), + ); + expect(onChange).not.toHaveBeenCalled(); + }); + + it("dispose stops further events", async () => { + const fixturesDir = path.join(tmpRoot, ".copilotkit", "fixtures"); + fs.mkdirSync(fixturesDir, { recursive: true }); + + const onChange = vi.fn(); + watcher = new FixturesDirWatcher(fixturesDir, onChange); + watcher.dispose(); + watcher = null; + + fs.writeFileSync(path.join(fixturesDir, "after-dispose.json"), "{}"); + // Give the OS a moment to deliver any stray events. + await new Promise((r) => setTimeout(r, 100)); + expect(onChange).not.toHaveBeenCalled(); + }); +}); + +async function waitFor( + predicate: () => boolean, + { timeoutMs = 2000, intervalMs = 20 } = {}, +): Promise { + const start = Date.now(); + while (!predicate()) { + if (Date.now() - start > timeoutMs) { + throw new Error("waitFor timeout"); + } + await new Promise((r) => setTimeout(r, intervalMs)); + } +} diff --git a/src/extension/playground/__tests__/view-provider.test.ts b/src/extension/playground/__tests__/view-provider.test.ts index 3ddb27c71..83c65247a 100644 --- a/src/extension/playground/__tests__/view-provider.test.ts +++ b/src/extension/playground/__tests__/view-provider.test.ts @@ -1,7 +1,11 @@ import { describe, expect, it, vi } from "vitest"; // vscode is not available in the vitest runtime — shim the surface used by -// PlaygroundViewProvider (Uri.joinPath for renderHtml, nothing else). +// PlaygroundViewProvider (Uri.joinPath for renderHtml, plus the warning- +// message API used by load-fixture's stale-file recovery path). +const vscodeMocks = vi.hoisted(() => ({ + showWarningMessage: vi.fn(), +})); vi.mock("vscode", () => ({ Uri: { joinPath: (_base: unknown, ...parts: string[]) => ({ @@ -9,6 +13,9 @@ vi.mock("vscode", () => ({ fsPath: parts.join("/"), }), }, + window: { + showWarningMessage: vscodeMocks.showWarningMessage, + }, })); import { PlaygroundViewProvider } from "../view-provider"; @@ -258,6 +265,307 @@ describe("PlaygroundViewProvider — bundling", () => { }); }); +describe("PlaygroundViewProvider — missing-fixture recovery", () => { + it("load-fixture with a missing file does not set replayFixturePath and refreshes the sidebar", async () => { + const readFn = vi.fn().mockImplementation(() => { + throw new Error("ENOENT: no such file or directory"); + }); + const listFn = vi.fn().mockReturnValue([]); + const startRuntimeHost = vi.fn().mockResolvedValue({ + url: "http://127.0.0.1:22222", + stop: vi.fn().mockResolvedValue(undefined), + vscodeLmTools: { enabled: false, count: 0 }, + }); + const bundleFn = vi + .fn() + .mockResolvedValue({ success: true, code: "var __copilotkit_playground = {};" }); + const codegenFn = vi.fn().mockReturnValue({ + outDir: "/tmp/x", + entryPath: "/tmp/x/entry.tsx", + }); + + vscodeMocks.showWarningMessage.mockClear(); + + const provider = new PlaygroundViewProvider( + { fsPath: "/fake", scheme: "file" } as never, + { onRefresh: vi.fn(), onOpenSource: vi.fn() }, + makeDeps({ + startRuntimeHost, + bundle: bundleFn, + writeSources: codegenFn, + fixtureStore: { + list: listFn, + read: readFn, + save: vi.fn(), + delete: vi.fn(), + }, + }), + ); + const view = makeWebview(); + provider.resolveWebviewView(view as never, {} as never, {} as never); + view.send({ type: "ready" }); + + view.webview.postMessage.mockClear(); + listFn.mockClear(); + + view.send({ type: "load-fixture", filePath: "/ghost/x.json" }); + await new Promise((r) => setTimeout(r, 30)); + + // Fresh fixtures-list must be posted so the stale UI entry vanishes. + expect(view.webview.postMessage).toHaveBeenCalledWith({ + type: "fixtures-list", + fixtures: [], + }); + // User gets a one-shot warning. + expect(vscodeMocks.showWarningMessage).toHaveBeenCalledTimes(1); + + // CRITICAL: a subsequent rebundle must NOT try to read the ghost + // fixture (replayFixturePath should not have been committed) and + // must start the runtime in record mode rather than dying with + // ENOENT. + readFn.mockClear(); + startRuntimeHost.mockClear(); + provider.setScanResult({ + providers: [ + { + filePath: "/x/App.tsx", + loc: { line: 1, column: 0, endLine: 1, endColumn: 1 }, + importedName: "CopilotKitProvider", + importSource: "@copilotkit/react-core/v2", + props: {}, + }, + ], + componentsWithHooks: [], + hookSites: [], + warnings: [], + }); + await new Promise((r) => setTimeout(r, 30)); + + expect(readFn).not.toHaveBeenCalled(); + expect(startRuntimeHost).toHaveBeenCalledWith( + expect.objectContaining({ mode: "record" }), + ); + }); + + it("runBundle recovers when the active fixture vanishes mid-session", async () => { + // First read (during load-fixture) succeeds; second read (during the + // forced rebundle) throws to simulate the file being deleted between + // load-fixture and the next runBundle pass. + let allowRead = true; + const readFn = vi.fn().mockImplementation(() => { + if (allowRead) { + return { + metadata: { + name: "X", + createdAt: "2026-01-01T00:00:00.000Z", + modelId: "m", + modelVendor: "v", + version: 2 as const, + }, + calls: [], + }; + } + throw new Error("ENOENT"); + }); + const listFn = vi.fn().mockReturnValue([]); + const startRuntimeHost = vi.fn().mockResolvedValue({ + url: "http://127.0.0.1:22222", + stop: vi.fn().mockResolvedValue(undefined), + vscodeLmTools: { enabled: false, count: 0 }, + }); + const bundleFn = vi + .fn() + .mockResolvedValue({ success: true, code: "var __copilotkit_playground = {};" }); + const codegenFn = vi.fn().mockReturnValue({ + outDir: "/tmp/x", + entryPath: "/tmp/x/entry.tsx", + }); + + const provider = new PlaygroundViewProvider( + { fsPath: "/fake", scheme: "file" } as never, + { onRefresh: vi.fn(), onOpenSource: vi.fn() }, + makeDeps({ + startRuntimeHost, + bundle: bundleFn, + writeSources: codegenFn, + fixtureStore: { + list: listFn, + read: readFn, + save: vi.fn(), + delete: vi.fn(), + }, + }), + ); + const view = makeWebview(); + provider.resolveWebviewView(view as never, {} as never, {} as never); + view.send({ type: "ready" }); + + view.send({ type: "load-fixture", filePath: "/some/x.json" }); + await new Promise((r) => setTimeout(r, 30)); + + // Simulate the file being deleted out from under us. The next + // rebundle must NOT propagate ENOENT — it must clear the stale + // path, refresh fixtures-list, and start the runtime in record + // mode so the chat surface unsticks. + allowRead = false; + startRuntimeHost.mockClear(); + view.webview.postMessage.mockClear(); + + provider.setScanResult({ + providers: [ + { + filePath: "/x/App.tsx", + loc: { line: 1, column: 0, endLine: 1, endColumn: 1 }, + importedName: "CopilotKitProvider", + importSource: "@copilotkit/react-core/v2", + props: {}, + }, + ], + componentsWithHooks: [], + hookSites: [], + warnings: [], + }); + await new Promise((r) => setTimeout(r, 50)); + + expect(startRuntimeHost).toHaveBeenCalledWith( + expect.objectContaining({ mode: "record" }), + ); + // A bundle-ready must eventually fire — the chat must unstick. + expect(view.webview.postMessage).toHaveBeenCalledWith( + expect.objectContaining({ type: "bundle-ready" }), + ); + // And the sidebar gets a fresh list (entry gone). + expect(view.webview.postMessage).toHaveBeenCalledWith( + expect.objectContaining({ type: "fixtures-list" }), + ); + }); +}); + +describe("PlaygroundViewProvider — refreshFixturesList", () => { + it("posts a fresh fixtures-list to the webview", () => { + const listFn = vi + .fn() + .mockReturnValueOnce([]) + .mockReturnValueOnce([ + { + filePath: "/fake/.copilotkit/fixtures/a.json", + metadata: { + name: "A", + createdAt: "2026-01-01T00:00:00.000Z", + modelId: "m", + modelVendor: "v", + version: 2 as const, + }, + }, + ]); + const provider = new PlaygroundViewProvider( + { fsPath: "/fake", scheme: "file" } as never, + { onRefresh: vi.fn(), onOpenSource: vi.fn() }, + makeDeps({ + fixtureStore: { + list: listFn, + read: vi.fn(), + save: vi.fn(), + delete: vi.fn(), + }, + }), + ); + const view = makeWebview(); + provider.resolveWebviewView(view as never, {} as never, {} as never); + view.send({ type: "ready" }); + + view.webview.postMessage.mockClear(); + provider.refreshFixturesList(); + + expect(view.webview.postMessage).toHaveBeenCalledWith({ + type: "fixtures-list", + fixtures: expect.arrayContaining([ + expect.objectContaining({ metadata: expect.objectContaining({ name: "A" }) }), + ]), + }); + }); + + it("clears the active replay path if its backing file is gone", async () => { + // Pick a path that definitely does not exist on disk so the + // fs.existsSync check inside refreshFixturesList returns false. + const phantomPath = "/this/path/does/not/exist/x.json"; + const readFn = vi.fn().mockReturnValue({ + metadata: { + name: "X", + createdAt: "2026-01-01T00:00:00.000Z", + modelId: "m", + modelVendor: "v", + version: 2 as const, + }, + calls: [], + }); + const startRuntimeHost = vi.fn().mockResolvedValue({ + url: "http://127.0.0.1:22222", + stop: vi.fn().mockResolvedValue(undefined), + vscodeLmTools: { enabled: false, count: 0 }, + }); + const bundleFn = vi.fn().mockResolvedValue({ + code: "var __copilotkit_playground = {};", + success: true, + }); + const codegenFn = vi.fn().mockReturnValue({ + outDir: "/tmp/ignored", + entryPath: "/tmp/ignored/entry.tsx", + }); + + const provider = new PlaygroundViewProvider( + { fsPath: "/fake", scheme: "file" } as never, + { onRefresh: vi.fn(), onOpenSource: vi.fn() }, + makeDeps({ + startRuntimeHost, + bundle: bundleFn, + writeSources: codegenFn, + fixtureStore: { + list: vi.fn().mockReturnValue([]), + read: readFn, + save: vi.fn(), + delete: vi.fn(), + }, + }), + ); + const view = makeWebview(); + provider.resolveWebviewView(view as never, {} as never, {} as never); + view.send({ type: "ready" }); + + // Pretend the user clicked ▶ on a fixture that no longer exists. + view.send({ type: "load-fixture", filePath: phantomPath }); + await new Promise((r) => setTimeout(r, 30)); + + // Drop our stale view of the active fixture. + provider.refreshFixturesList(); + + // The next rebundle must NOT try to read the missing fixture and + // must start the runtime in "record" mode (the fallback). + startRuntimeHost.mockClear(); + readFn.mockClear(); + provider.setScanResult({ + providers: [ + { + filePath: "/x/App.tsx", + loc: { line: 1, column: 0, endLine: 1, endColumn: 1 }, + importedName: "CopilotKitProvider", + importSource: "@copilotkit/react-core/v2", + props: {}, + }, + ], + componentsWithHooks: [], + hookSites: [], + warnings: [], + }); + await new Promise((r) => setTimeout(r, 30)); + + expect(readFn).not.toHaveBeenCalled(); + expect(startRuntimeHost).toHaveBeenCalledWith( + expect.objectContaining({ mode: "record" }), + ); + }); +}); + describe("PlaygroundViewProvider — HTML bootstrap", () => { it("injects a nonce bootstrap script so bundle-loader can discover it", () => { const provider = new PlaygroundViewProvider( diff --git a/src/extension/playground/__tests__/vscode-lm-factory.test.ts b/src/extension/playground/__tests__/vscode-lm-factory.test.ts index d03af7979..1b1709bb6 100644 --- a/src/extension/playground/__tests__/vscode-lm-factory.test.ts +++ b/src/extension/playground/__tests__/vscode-lm-factory.test.ts @@ -264,3 +264,110 @@ describe("vscodeLmFactory — replay mode", () => { await expect(collect()).rejects.toThrow(); }); }); + +describe("vscodeLmFactory — empty-stream recovery", () => { + function makeMultiResponseModel(streams: unknown[][]): { + model: LanguageModelChat; + callCount: () => number; + } { + let idx = 0; + const model = { + id: "test-model", + family: "test", + name: "Test", + vendor: "test", + sendRequest: vi.fn(async () => { + const parts = streams[idx] ?? []; + idx++; + return { + stream: (async function* () { + for (const p of parts) yield p; + })(), + text: (async function* () {})(), + }; + }), + } as unknown as LanguageModelChat; + return { model, callCount: () => idx }; + } + + it("auto-retries without vscode.lm tools when the first stream is empty", async () => { + const { LanguageModelTextPart } = await import("vscode"); + const { model, callCount } = makeMultiResponseModel([ + [], // first attempt: empty + [new LanguageModelTextPart("retry worked")], // second attempt: text + ]); + const factory = vscodeLmFactory({ + model, + mode: "live", + vscodeLmTools: [ + { + name: "ghc_tool_1", + description: "x", + inputSchema: { type: "object" }, + } as unknown as import("vscode").LanguageModelToolInformation, + ], + }); + const chunks: unknown[] = []; + const ac = new AbortController(); + for await (const c of factory({ + input: minimalInput, + abortController: ac, + abortSignal: ac.signal, + })) { + chunks.push(c); + } + expect(callCount()).toBe(2); + expect(chunks).toEqual([ + { type: "TEXT_MESSAGE_CONTENT", delta: "retry worked" }, + ]); + }); + + it("yields a single actionable message when both attempts return empty", async () => { + const { model, callCount } = makeMultiResponseModel([[], []]); + const factory = vscodeLmFactory({ + model, + mode: "live", + vscodeLmTools: Array.from({ length: 25 }, (_, i) => ({ + name: `vscode_tool_${i}`, + description: "x", + inputSchema: { type: "object" }, + })) as unknown as import("vscode").LanguageModelToolInformation[], + }); + const chunks: Array<{ type: string; delta?: string }> = []; + const ac = new AbortController(); + for await (const c of factory({ + input: minimalInput, + abortController: ac, + abortSignal: ac.signal, + })) { + chunks.push(c as { type: string; delta?: string }); + } + expect(callCount()).toBe(2); + expect(chunks).toHaveLength(1); + expect(chunks[0]?.type).toBe("TEXT_MESSAGE_CONTENT"); + // The high-tool-count branch of the message mentions the tool counts + // so the user can act on it. + expect(chunks[0]?.delta).toMatch(/25/); + }); + + it("does not retry when there were no vscode.lm tools to drop", async () => { + const { model, callCount } = makeMultiResponseModel([[]]); + const factory = vscodeLmFactory({ + model, + mode: "live", + // No vscodeLmTools — empty stream is not recoverable by retry. + }); + const chunks: unknown[] = []; + const ac = new AbortController(); + for await (const c of factory({ + input: minimalInput, + abortController: ac, + abortSignal: ac.signal, + })) { + chunks.push(c); + } + expect(callCount()).toBe(1); + // Only the synthetic actionable message. + expect(chunks).toHaveLength(1); + }); +}); diff --git a/src/extension/playground/fixtures-dir-watcher.ts b/src/extension/playground/fixtures-dir-watcher.ts new file mode 100644 index 000000000..5d3c3f0ec --- /dev/null +++ b/src/extension/playground/fixtures-dir-watcher.ts @@ -0,0 +1,65 @@ +import * as fs from "node:fs"; +import * as vscode from "vscode"; + +/** + * Watches `/.copilotkit/fixtures/` for changes so the + * saved-replays sidebar stays in sync with the on-disk state. + * + * We use Node's `fs.watch` rather than `vscode.workspace.createFileSystemWatcher` + * because VS Code's watcher is unreliable for dot-prefixed directories + * across platforms (Windows in particular swallows events for paths under + * `.copilotkit/`). `fs.watch` runs in the extension host's Node runtime + * and sees every event the OS reports for the watched directory. + * + * If the fixtures directory doesn't exist yet, `attach()` is a no-op and + * a 5-second poll re-tries until the directory appears (e.g. after the + * user's first save). Once attached, the watcher is permanent. + */ +export class FixturesDirWatcher implements vscode.Disposable { + private watcher: fs.FSWatcher | null = null; + private retry: ReturnType | null = null; + + constructor( + private readonly fixturesDir: string, + private readonly onChange: () => void, + private readonly log: (line: string) => void = () => {}, + ) { + this.attach(); + if (!this.watcher) { + this.retry = setInterval(() => this.attach(), 5000); + } + } + + private attach(): void { + if (this.watcher) return; + if (!fs.existsSync(this.fixturesDir)) return; + try { + this.watcher = fs.watch(this.fixturesDir, (eventType, filename) => { + this.log( + `[fixtures-watcher] fs.watch ${eventType} ${filename ?? ""}`, + ); + this.onChange(); + }); + this.log(`[fixtures-watcher] attached to ${this.fixturesDir}`); + if (this.retry) { + clearInterval(this.retry); + this.retry = null; + } + } catch (err) { + this.log( + `[fixtures-watcher] fs.watch failed for ${this.fixturesDir}: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + } + } + + dispose(): void { + this.watcher?.close(); + this.watcher = null; + if (this.retry) { + clearInterval(this.retry); + this.retry = null; + } + } +} diff --git a/src/extension/playground/view-provider.ts b/src/extension/playground/view-provider.ts index fe61a2649..e0344b568 100644 --- a/src/extension/playground/view-provider.ts +++ b/src/extension/playground/view-provider.ts @@ -168,13 +168,26 @@ export class PlaygroundViewProvider implements vscode.WebviewViewProvider { return; } case "load-fixture": { - this.replayFixturePath = msg.filePath; + // Read the fixture BEFORE committing replayFixturePath — + // otherwise a stale sidebar entry (file deleted out from + // under us) would trap subsequent rebundles in a permanent + // "Preparing chat surface…" + ENOENT loop, because runBundle + // would keep trying to read the missing file. try { - this.replayFixtureName = - this.deps.fixtureStore.read(msg.filePath).metadata.name ?? - null; + const fixture = this.deps.fixtureStore.read(msg.filePath); + this.replayFixturePath = msg.filePath; + this.replayFixtureName = fixture.metadata.name ?? null; } catch { + this.replayFixturePath = null; this.replayFixtureName = null; + this.post({ + type: "fixtures-list", + fixtures: this.deps.fixtureStore.list(), + }); + void vscode.window.showWarningMessage( + "That fixture file no longer exists — it may have been deleted from disk.", + ); + return; } if (this.latestResult) { this.setScanResult(this.latestResult); @@ -234,6 +247,29 @@ export class PlaygroundViewProvider implements vscode.WebviewViewProvider { }); } + /** + * Pushes a fresh fixtures-list to the webview. Called from the + * activator when `.copilotkit/fixtures/*.json` changes on disk (e.g. + * the user deleted a saved replay in Explorer or git pulled new + * fixtures in). If the file backing the active replay session was + * removed, also drop our reference so the next rebundle falls back + * to record mode instead of throwing on a missing file. + */ + refreshFixturesList(): void { + if ( + this.replayFixturePath !== null && + !fs.existsSync(this.replayFixturePath) + ) { + this.replayFixturePath = null; + this.replayFixtureName = null; + } + if (!this.ready) return; + this.post({ + type: "fixtures-list", + fixtures: this.deps.fixtureStore.list(), + }); + } + async stopSession(): Promise { if (!this.currentSession) return; const { runtime } = this.currentSession; @@ -243,6 +279,14 @@ export class PlaygroundViewProvider implements vscode.WebviewViewProvider { private async runBundle(result: PlaygroundScanResult): Promise { const seq = ++this.bundleSeq; + // Sync the sidebar with disk on every rebundle. This is a cheap + // belt-and-suspenders so Refresh always reflects out-of-band file + // changes (manual delete, git pull) even if the fs.watch on + // `.copilotkit/fixtures/` missed an event. + this.post({ + type: "fixtures-list", + fixtures: this.deps.fixtureStore.list(), + }); let sources: PlaygroundSources | null = null; let session: { runtime: RuntimeHostHandle; @@ -291,9 +335,23 @@ export class PlaygroundViewProvider implements vscode.WebviewViewProvider { // 4. Start the runtime host (in-process). const recordedCalls: RecordedCall[] = []; - const replayFixture = this.replayFixturePath - ? this.deps.fixtureStore.read(this.replayFixturePath) - : null; + // Read the active replay fixture defensively. If the file vanished + // between load-fixture and this rebundle (manual delete, git pull, + // failed watcher event), drop the stale reference and fall back to + // record mode instead of letting ENOENT brick the panel. + let replayFixture: SavedFixture | null = null; + if (this.replayFixturePath) { + try { + replayFixture = this.deps.fixtureStore.read(this.replayFixturePath); + } catch { + this.replayFixturePath = null; + this.replayFixtureName = null; + this.post({ + type: "fixtures-list", + fixtures: this.deps.fixtureStore.list(), + }); + } + } const runtime = await this.deps.startRuntimeHost({ model, mode: replayFixture ? "replay" : "record", diff --git a/src/extension/playground/vscode-lm-factory.ts b/src/extension/playground/vscode-lm-factory.ts index ace614874..19b56e9f5 100644 --- a/src/extension/playground/vscode-lm-factory.ts +++ b/src/extension/playground/vscode-lm-factory.ts @@ -112,23 +112,60 @@ export function vscodeLmFactory( const vscodeLmToolNames = new Set(vscodeLmTools.map((t) => t.name)); const tools = [...userTools, ...vscodeLmTools]; - try { + // Single-attempt streaming helper. Used twice: first with the full + // tool set (user + vscode.lm); if that returns 0 chunks AND vscode.lm + // tools were forwarded, again with user tools only. Models that + // silently reject requests with too many tools (e.g. Claude via + // Copilot Chat with 80+ tools) recover gracefully without the user + // having to flip a setting. + const attempt = async function* ( + toolsForRequest: vscode.LanguageModelChatTool[], + label: string, + ): AsyncGenerator { log( - `[vscode-lm-factory] sendRequest model=${opts.model.id} userTools=${userTools.length} vscodeLmTools=${vscodeLmTools.length}`, + `[vscode-lm-factory] sendRequest ${label} model=${opts.model.id} userTools=${userTools.length} vscodeLmTools=${ + toolsForRequest.length - userTools.length + }`, ); const response = await opts.model.sendRequest( messages, - tools.length > 0 ? { tools } : {}, + toolsForRequest.length > 0 ? { tools: toolsForRequest } : {}, tokenSource.token, ); let chunkCount = 0; + let unknownPartCount = 0; for await (const part of response.stream) { if (ctx.abortSignal.aborted) break; + let translated = false; for (const chunk of translatePart(part)) { + translated = true; chunkCount++; if (opts.mode === "record") recordedChunks.push(chunk); yield chunk; } + if ( + !translated && + !(part instanceof vscode.LanguageModelToolCallPart) + ) { + // Part type we don't recognize (e.g. a thinking/reasoning part + // some vscode.lm providers stream, or a DataPart with a binary + // mime). Log it so we can extend translatePart instead of + // silently producing an empty chat. + unknownPartCount++; + const ctor = (part as { constructor?: { name?: string } }) + ?.constructor?.name; + const isDataPart = + typeof vscode.LanguageModelDataPart === "function" && + part instanceof vscode.LanguageModelDataPart; + const mime = isDataPart + ? (part as vscode.LanguageModelDataPart).mimeType + : undefined; + log( + `[vscode-lm-factory] dropping unknown stream part: ctor=${ctor ?? typeof part}${ + mime ? ` mime=${mime}` : "" + }`, + ); + } // For vscode.lm tool calls, also invoke server-side and emit a // TOOL_CALL_RESULT chunk so the chat history records the result // without the consumer needing to execute the tool itself. @@ -146,7 +183,40 @@ export function vscodeLmFactory( yield resultChunk; } } - log(`[vscode-lm-factory] stream complete chunks=${chunkCount}`); + log( + `[vscode-lm-factory] ${label} complete chunks=${chunkCount}${ + unknownPartCount > 0 ? ` unknownParts=${unknownPartCount}` : "" + }`, + ); + return chunkCount; + }; + + try { + // First attempt with the full tool surface. + let chunkCount = yield* attempt(tools, "attempt 1"); + + // If the model silently returned nothing AND we sent vscode.lm + // tools, retry without them. Keeps `enableVscodeLmTools=true` viable + // even when the active model can't cope with a large tool budget. + if (chunkCount === 0 && vscodeLmTools.length > 0) { + log( + `[vscode-lm-factory] attempt 1 empty — retrying without ${vscodeLmTools.length} vscode.lm tool(s)`, + ); + chunkCount = yield* attempt(userTools, "attempt 2 (no vscode.lm tools)"); + } + + // Still nothing → surface an actionable in-chat message so the user + // isn't staring at a frozen input. + if (chunkCount === 0) { + const totalTools = userTools.length + vscodeLmTools.length; + const message = + vscodeLmTools.length > 20 + ? `**The model returned no content.** This request forwarded **${totalTools} tools** (${vscodeLmTools.length} from VS Code language-model providers like GitHub Copilot, plus ${userTools.length} from your code). Retrying without the VS Code tools also returned nothing — the model may be rate-limited, refusing the request, or unavailable.\n\nTry a different model, or disable \`copilotkit.playground.enableVscodeLmTools\` in your VS Code settings and click **Refresh**.` + : `**The model returned no content.** Check the *CopilotKit Playground* output channel for details. The model may have hit a rate limit, refused the request, or returned a part type the playground doesn't recognize.`; + // Synthetic chunk — NOT pushed into recordedChunks so it never + // gets baked into a saved fixture. + yield { type: "TEXT_MESSAGE_CONTENT", delta: message }; + } } catch (err) { log( `[vscode-lm-factory] sendRequest threw: ${err instanceof Error ? err.message : String(err)}`, @@ -353,4 +423,27 @@ function* translatePart(part: unknown): Generator { yield { type: "TOOL_CALL_END", toolCallId }; return; } + // Some vscode.lm providers stream text content as `LanguageModelDataPart` + // with a text/* or application/json mime instead of plain `TextPart` + // (notably newer Claude builds routed through Copilot Chat). Surface + // those as text deltas — otherwise they'd be dropped and the user would + // see an empty chat. Binary mimes (image/*, etc.) are skipped here and + // logged at the call site so we can extend support if needed. + // + // Guarded with a typeof check because older VS Code builds (or the + // vitest vscode mock) may not define `LanguageModelDataPart`, and + // `part instanceof undefined` throws. + if ( + typeof vscode.LanguageModelDataPart === "function" && + part instanceof vscode.LanguageModelDataPart + ) { + const mime = part.mimeType || ""; + if (mime.startsWith("text/") || mime === "application/json") { + const text = new TextDecoder("utf-8").decode(part.data); + if (text.length > 0) { + yield { type: "TEXT_MESSAGE_CONTENT", delta: text }; + } + return; + } + } } diff --git a/src/webview/playground/App.tsx b/src/webview/playground/App.tsx index 0ffdb3b9b..c9c029eb4 100644 --- a/src/webview/playground/App.tsx +++ b/src/webview/playground/App.tsx @@ -11,6 +11,7 @@ import { onExtensionMessage, sendToExtension } from "./bridge"; import type { PlaygroundScanResult } from "../../extension/playground/types"; import type { MountErrorPayload } from "../../extension/playground/bridge-types"; import type { FixtureListEntry } from "../../extension/playground/fixture-store"; +import type { ReplayMessage } from "../../extension/playground/fixture-replay"; export function App(): React.JSX.Element { const [result, setResult] = React.useState({ @@ -44,6 +45,15 @@ export function App(): React.JSX.Element { Array<{ id: string; name: string; family: string; vendor: string }> >([]); const [selectedModelId, setSelectedModelId] = React.useState(""); + // The extension posts `play-fixture` immediately after `bundle-ready`, + // but executing the new bundle is async — the listener for the replay + // event lives inside the bundle (in PlaygroundChat) and only registers + // once the new bundle has mounted. Queue the messages here and dispatch + // them via the effect below, so the new PlaygroundChat is guaranteed to + // have attached its listener before the event fires. + const [pendingReplay, setPendingReplay] = React.useState< + ReplayMessage[] | null + >(null); // Collapse state for the side panels — persisted so users who don't // need them keep their full chat width across reloads. The classes @@ -78,6 +88,11 @@ export function App(): React.JSX.Element { else if (msg.type === "bundle-ready") { setStateBanner(null); setBundleError(null); + // Unmount the old bundle synchronously. Its runtime was stopped + // server-side before this rebundle, so leaving it mounted would + // both point at a dead URL and (on a load-fixture flow) catch + // the imminent replay event in the wrong listener. + setBundle(null); executePlaygroundBundle(msg.payload.code, msg.payload.css).then( (exports) => setBundle(exports), (err) => { @@ -134,18 +149,30 @@ export function App(): React.JSX.Element { // We use a window CustomEvent because PlaygroundChat is generated // into the rolldown'd bundle and doesn't share a React context with // this shell — a global event bus is the simplest cross-bundle - // wiring. - window.dispatchEvent( - new CustomEvent("copilotkit-playground-replay", { - detail: { messages: msg.messages }, - }), - ); + // wiring. Stash the messages and let the effect below dispatch + // once the bundle (and therefore the chat's replay listener) is + // mounted. + setPendingReplay(msg.messages); } }); sendToExtension({ type: "ready" }); return unsubscribe; }, []); + // Dispatch a queued replay only after the new bundle is mounted. + // React runs child useEffects before parent ones on the same commit, + // so by the time this fires the new PlaygroundChat has already + // registered its replay listener and will catch the event. + React.useEffect(() => { + if (!bundle || !pendingReplay) return; + window.dispatchEvent( + new CustomEvent("copilotkit-playground-replay", { + detail: { messages: pendingReplay }, + }), + ); + setPendingReplay(null); + }, [bundle, pendingReplay]); + // Forward "click tool name" events from the bundled chat surface to // the extension. The chat lives inside the rolldown'd bundle and // dispatches a CustomEvent on window since it can't postMessage to diff --git a/src/webview/playground/__tests__/App.replay-race.test.tsx b/src/webview/playground/__tests__/App.replay-race.test.tsx new file mode 100644 index 000000000..496197c17 --- /dev/null +++ b/src/webview/playground/__tests__/App.replay-race.test.tsx @@ -0,0 +1,119 @@ +/** @vitest-environment jsdom */ +import * as React from "react"; +import { act, render } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +// `vi.mock` is hoisted above imports, so we route the controllable +// state through `vi.hoisted` instead of a regular top-level closure. +const mocks = vi.hoisted(() => { + let deferredResolve: ((exports: unknown) => void) | null = null; + let deferredReject: ((err: unknown) => void) | null = null; + const executePlaygroundBundle = (): Promise => + new Promise((resolve, reject) => { + deferredResolve = resolve as (exports: unknown) => void; + deferredReject = reject; + }); + return { + executePlaygroundBundle, + resolve: (exports: unknown) => deferredResolve?.(exports), + reject: (err: unknown) => deferredReject?.(err), + }; +}); + +vi.mock("../bundle-loader", () => ({ + executePlaygroundBundle: mocks.executePlaygroundBundle, +})); + +vi.mock("../bridge", () => ({ + // Stub out the real vscode.postMessage path — sendToExtension is a + // fire-and-forget from the App's perspective for this race test. + sendToExtension: () => {}, + // Hand back a real `window message` listener so we can drive the App + // via window.postMessage(...). + onExtensionMessage: (handler: (msg: unknown) => void) => { + const listener = (e: MessageEvent): void => handler(e.data); + window.addEventListener("message", listener); + return () => window.removeEventListener("message", listener); + }, +})); + +import { App } from "../App"; + +describe("App play-fixture race", () => { + it("defers the replay CustomEvent until the new bundle is mounted", async () => { + const replayEvents: Array<{ messages: unknown[] }> = []; + const listener = (ev: Event): void => { + const detail = (ev as CustomEvent<{ messages: unknown[] }>).detail; + replayEvents.push(detail); + }; + window.addEventListener("copilotkit-playground-replay", listener); + + render(); + + // Give the App's initial useEffect a chance to attach the message + // listener before we post. + await act(async () => { + await Promise.resolve(); + }); + + // Step 1: extension delivers a new bundle, then immediately the + // play-fixture payload. This mirrors view-provider.ts ordering. + await act(async () => { + window.postMessage( + { type: "bundle-ready", payload: { code: "" } }, + "*", + ); + window.postMessage( + { + type: "play-fixture", + messages: [{ id: "1", role: "user", content: "hi" }], + }, + "*", + ); + // Flush the postMessage queue. + await new Promise((r) => setTimeout(r, 0)); + }); + + // executePlaygroundBundle was called but the promise hasn't resolved + // yet — the new ChatPlayground (and its replay listener) isn't + // mounted. The App MUST NOT have dispatched the replay event yet. + expect(replayEvents).toHaveLength(0); + + // Step 2: resolve the deferred bundle. The new ChatPlayground stub + // registers a listener in its mount effect — the App's effect runs + // AFTER child effects on the same commit, so the listener is + // guaranteed to be attached before the App dispatches the queued + // replay. + let chatSawReplay = 0; + function ChatPlayground(): React.JSX.Element | null { + React.useEffect(() => { + const onReplay = (): void => { + chatSawReplay += 1; + }; + window.addEventListener("copilotkit-playground-replay", onReplay); + return () => + window.removeEventListener( + "copilotkit-playground-replay", + onReplay, + ); + }, []); + return null; + } + function PlaygroundEntry(): React.JSX.Element | null { + return null; + } + + await act(async () => { + mocks.resolve({ PlaygroundEntry, ChatPlayground }); + // Let the bundle-promise microtask + React effects flush. + await new Promise((r) => setTimeout(r, 0)); + }); + + // Once the new bundle mounts, the queued replay should have fired + // exactly once — and the new chat must have caught it. + expect(replayEvents).toHaveLength(1); + expect(chatSawReplay).toBe(1); + + window.removeEventListener("copilotkit-playground-replay", listener); + }); +}); diff --git a/test-workspace/playground/v2/Tools.tsx b/test-workspace/playground/v2/Tools.tsx index 6d46e57b5..79dfc63fd 100644 --- a/test-workspace/playground/v2/Tools.tsx +++ b/test-workspace/playground/v2/Tools.tsx @@ -2,22 +2,6 @@ import { useFrontendTool } from "@copilotkit/react-core/v2"; import { z } from "zod"; export function Tools() { - useFrontendTool({ - name: "displayCurrentWeather", - description: "Display current weather for a city", - parameters: z.object({ city: z.string() }), - handler: async ({ city }: { city: string }) => ({ city, temp: 72, unit: "F" }), - render: (props: Record) => { - const args = (props.args ?? {}) as Record; - const result = (props.result ?? {}) as Record | undefined; - return ( -
- Weather for {String(args.city ?? "...")}: {String(result?.temp ?? "loading")}° - {String(result?.unit ?? "")} -
- ); - }, - }); - + return
v2 tools
; }