factory.e2e.test.ts

import { execFileSync } from "node:child_process";
import { existsSync, readFileSync, writeFileSync } from "node:fs";
import { copyFile, mkdir, rm } from "node:fs/promises";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { expect, it, vi } from "vitest";
import { approveAll, FactoryResumeError, RuntimeConnection } from "../../src/index.js";
import {
    createSdkTestContext,
    DEFAULT_GITHUB_TOKEN,
    getLegacyCliPathForTests,
} from "./harness/sdkTestContext.js";
import { retry } from "./harness/sdkTestHelper.js";

const __dirname = dirname(fileURLToPath(import.meta.url));
const factoryTestContext = await createSdkTestContext({
    copilotClientOptions: {
        connection: RuntimeConnection.forStdio({ path: await getLegacyCliPathForTests() }),
        env: {
            COPILOT_CLI_ENABLED_FEATURE_FLAGS: "EXTENSIONS,AGENT_FACTORIES",
        },
    },
});

async function setupFactoryExtension(workDir: string, onPermissionRequest = approveAll) {
    const { copilotClient, openAiEndpoint } = factoryTestContext;
    const extensionDir = join(workDir, ".github", "extensions", "factory-smoke");
    const readyFile = join(extensionDir, "ready");
    await rm(join(workDir, ".github"), { recursive: true, force: true });
    await mkdir(extensionDir, { recursive: true });
    await copyFile(
        join(__dirname, "fixtures", "factory-extension.mjs"),
        join(extensionDir, "extension.mjs")
    );
    execFileSync("git", ["init", "--quiet"], { cwd: workDir });

    await openAiEndpoint.setCopilotUserByToken(DEFAULT_GITHUB_TOKEN, {
        login: "factory-e2e-user",
        copilot_plan: "individual_pro",
        token_based_billing: true,
        is_mcp_enabled: true,
        endpoints: {
            api: openAiEndpoint.url,
            telemetry: "https://localhost:1/telemetry",
        },
        analytics_tracking_id: "e2e-test-tracking-id",
    });

    const session = await copilotClient.createSession({
        requestExtensions: true,
        extensionSdkPath: resolve(__dirname, "..", "..", "dist"),
        onPermissionRequest,
        onElicitationRequest: async () => ({
            action: "accept",
            content: { action: "approve" },
        }),
    });

    await retry(
        "wait for the factory extension to join the session",
        async () => {
            expect(existsSync(readyFile)).toBe(true);
        },
        300,
        100
    );

    return session;
}

it("runs an extension-authored factory across the SDK process boundary", async () => {
    const { workDir } = factoryTestContext;
    await using session = await setupFactoryExtension(workDir);

    const result = await session.factory.run("argument-echo", {
        args: { source: "sdk-e2e", count: 11 },
        notifyOnComplete: false,
    });

    expect(result).toMatchObject({
        status: "completed",
        result: { source: "sdk-e2e", count: 11 },
    });
});

// TODO(cli-1.0.81-2): the subagent request is rejected downstream under CLI 1.0.81-2, so the
// fixture reports didThrow: true. Re-enable once the runtime fix ships.
//
// The timeout is generous because the factory abandons its subagent once the runtime has
// accepted the request, so the run settles only after the runtime drains that work.
it.skip("forwards every declared subagent option to the runtime", async () => {
    if (!factoryTestContext) {
        throw new Error("Factory E2E requires the stdio transport");
    }
    const { workDir } = factoryTestContext;
    await using session = await setupFactoryExtension(workDir);

    const result = await session.factory.run("forwards-subagent-options");

    expect(result).toMatchObject({
        status: "completed",
        result: { didThrow: false },
    });
}, 60_000);

it("throws FactoryResumeError with not_found for an unknown run", async () => {
    if (!factoryTestContext) {
        throw new Error("Factory E2E requires the stdio transport");
    }
    const { workDir } = factoryTestContext;
    await using session = await setupFactoryExtension(workDir);

    const error = await session.factory
        .resume("00000000-0000-0000-0000-000000000000")
        .catch((caught: unknown) => caught);

    expect(error).toBeInstanceOf(FactoryResumeError);
    expect((error as FactoryResumeError).code).toBe("not_found");
});

it("throws FactoryResumeError with non_resumable for a completed run", async () => {
    if (!factoryTestContext) {
        throw new Error("Factory E2E requires the stdio transport");
    }
    const { workDir } = factoryTestContext;
    await using session = await setupFactoryExtension(workDir);

    const run = await session.factory.run("argument-echo", { notifyOnComplete: false });
    const error = await session.factory.resume(run.runId).catch((caught: unknown) => caught);

    expect(error).toBeInstanceOf(FactoryResumeError);
    expect((error as FactoryResumeError).code).toBe("non_resumable");
});

it("forwards factory runtime controls across the SDK process boundary", async () => {
    const { workDir } = factoryTestContext;
    await using session = await setupFactoryExtension(workDir);

    const suppressed = await session.factory.run("phased", {
        notifyOnComplete: false,
        logPhaseNames: false,
    });

    expect(suppressed).toMatchObject({
        status: "completed",
        result: "finished",
    });
    const progress = await session.factory.getRunProgress(suppressed.runId);
    expect(progress.records).toEqual(
        expect.arrayContaining([
            expect.objectContaining({ kind: "phase", text: "Collect" }),
            expect.objectContaining({ kind: "log", text: "Collected" }),
            expect.objectContaining({ kind: "phase", text: "Summarize" }),
            expect.objectContaining({ kind: "log", text: "Summarized" }),
        ])
    );

    const events = await session.getEvents();
    expect(
        events.some(
            (event) =>
                event.type === "system.notification" &&
                event.data.kind.type === "factory_completed" &&
                event.data.kind.runId === suppressed.runId
        )
    ).toBe(false);
    expect(
        events.filter(
            (event) => event.type === "session.info" && event.data.infoType === "factory_phase"
        )
    ).toEqual([]);
});

it("pages factory runs and returns cursor metadata", async () => {
    const { workDir } = factoryTestContext;
    await using session = await setupFactoryExtension(workDir);

    const first = await session.factory.run("argument-echo", {
        args: { ordinal: 1 },
        notifyOnComplete: false,
    });
    const second = await session.factory.run("argument-echo", {
        args: { ordinal: 2 },
        notifyOnComplete: false,
    });
    const third = await session.factory.run("argument-echo", {
        args: { ordinal: 3 },
        notifyOnComplete: false,
    });

    const newest = await session.factory.listRuns({ limit: 1 });
    expect(newest).toMatchObject({
        runs: [expect.objectContaining({ runId: third.runId })],
        hasMoreNewer: false,
        omittedOlder: 2,
    });
    expect(newest.oldestSeq).toBe(newest.newestSeq);
    expect(newest.oldestSeq).not.toBeNull();

    const older = await session.factory.listRuns({
        beforeSeq: newest.oldestSeq!,
        limit: 1,
    });
    expect(older).toMatchObject({
        runs: [expect.objectContaining({ runId: second.runId })],
        hasMoreNewer: true,
        omittedOlder: 1,
    });

    const oldest = await session.factory.listRuns({
        beforeSeq: older.oldestSeq!,
        limit: 1,
    });
    expect(oldest).toMatchObject({
        runs: [expect.objectContaining({ runId: first.runId })],
        hasMoreNewer: true,
        omittedOlder: 0,
    });
});

it("runs a factory when its session denies every permission request", async () => {
    if (!factoryTestContext) {
        throw new Error("Factory E2E requires the stdio transport");
    }
    const { workDir } = factoryTestContext;
    const denyPermissions = vi.fn(() => ({ kind: "reject" as const }));
    await using session = await setupFactoryExtension(workDir, denyPermissions);

    await expect(
        session.factory.run("argument-echo", { notifyOnComplete: false })
    ).resolves.toMatchObject({
        status: "completed",
    });
    expect(denyPermissions).not.toHaveBeenCalled();
});

it("resumes a failed factory when its session denies every permission request", async () => {
    if (!factoryTestContext) {
        throw new Error("Factory E2E requires the stdio transport");
    }
    const { workDir } = factoryTestContext;
    const denyPermissions = vi.fn(() => ({ kind: "reject" as const }));
    await using session = await setupFactoryExtension(workDir, denyPermissions);

    const failedRun = await session.factory.run("fails-once", { notifyOnComplete: false });
    expect(failedRun).toMatchObject({
        status: "error",
    });

    await expect(
        session.factory.resume(failedRun.runId, {
            notifyOnComplete: false,
            logPhaseNames: false,
        })
    ).resolves.toMatchObject({
        status: "completed",
        result: "resumed",
    });
    expect(denyPermissions).not.toHaveBeenCalled();
});

it("refuses a factory started through the context session from a factory body", async () => {
    if (!factoryTestContext) {
        throw new Error("Factory E2E requires the stdio transport");
    }
    const { workDir } = factoryTestContext;
    await using session = await setupFactoryExtension(workDir);

    const result = await session.factory.run("starts-from-context-session", {
        notifyOnComplete: false,
    });

    expect(result).toMatchObject({
        status: "completed",
        result: expect.stringContaining("factory.run and factory.resume"),
    });
    expect((result as { result: string }).result).toContain("factory body");
});

it("refuses a factory started through the module session from a factory body", async () => {
    if (!factoryTestContext) {
        throw new Error("Factory E2E requires the stdio transport");
    }
    const { workDir } = factoryTestContext;
    await using session = await setupFactoryExtension(workDir);

    const result = await session.factory.run("starts-from-module-session", {
        notifyOnComplete: false,
    });

    expect(result).toMatchObject({
        status: "completed",
        result: expect.stringContaining("factory.run and factory.resume"),
    });
    expect((result as { result: string }).result).toContain("factory body");
});

it("allows a module-level extension watcher to start a factory while another body is parked", async () => {
    if (!factoryTestContext) {
        throw new Error("Factory E2E requires the stdio transport");
    }
    const { workDir } = factoryTestContext;
    const extensionDir = join(workDir, ".github", "extensions", "factory-smoke");
    await using session = await setupFactoryExtension(workDir);

    const parked = session.factory.run("parked", { notifyOnComplete: false });
    await retry(
        "wait for the parked factory to enter its body",
        async () => {
            expect(existsSync(join(extensionDir, "entered"))).toBe(true);
        },
        100,
        100
    );

    writeFileSync(join(extensionDir, "start-b"), "start");
    const bResultFile = join(extensionDir, "b-result");
    await retry(
        "wait for the module-level watcher factory run to succeed",
        async () => {
            expect(existsSync(bResultFile)).toBe(true);
            expect(JSON.parse(readFileSync(bResultFile, "utf8"))).toMatchObject({
                status: "success",
                result: {
                    status: "completed",
                    result: { source: "module-watcher" },
                },
            });
        },
        100,
        100
    );

    writeFileSync(join(extensionDir, "release"), "release");
    await expect(parked).resolves.toMatchObject({
        status: "completed",
        result: "released",
    });
}, 60_000);

it("returns an array result from an extension-authored factory", async () => {
    if (!factoryTestContext) {
        throw new Error("Factory E2E requires the stdio transport");
    }
    const { workDir } = factoryTestContext;
    await using session = await setupFactoryExtension(workDir);

    const result = await session.factory.run("array-result", { notifyOnComplete: false });

    expect(result).toMatchObject({
        status: "completed",
        result: [1, "two", false],
    });
});

it("passes array factory arguments across the SDK process boundary", async () => {
    if (!factoryTestContext) {
        throw new Error("Factory E2E requires the stdio transport");
    }
    const { workDir } = factoryTestContext;
    await using session = await setupFactoryExtension(workDir);

    const args = [1, "two", false];
    const result = await session.factory.run("argument-echo", {
        args,
        notifyOnComplete: false,
    });

    expect(result).toMatchObject({
        status: "completed",
        result: args,
    });
});