From 1451b393017d22ffbd065a7fa4766c804c15938e Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Sun, 26 Apr 2026 08:55:13 -0700 Subject: [PATCH 1/6] perf: sandbox improvements, perf test infrastructure, benchmarks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sandbox fixes (both monty + hyperlight): - glob() returns workspace-relative paths (was absolute — caused retries) - glob() supports brace expansion: {db,services}/**/*.py - web_fetch() strips HTML to text via html.parser, caps at 20K chars - uv run --quiet suppresses spinner that polluted tool results - mcp_call always listed in tool reference (was missing at install time) Skill documentation: - Return types documented (glob→list, view→string, bash→dict) - No os.path/os.walk — explicit prohibition - One program, one bash call rule - Backend-specific MCP syntax (monty: mcp_call(), hyperlight: call_tool) Perf test infrastructure: - Input token tracking from copilot process logs - Context bloat measurement (tool_result_bytes) - Cost estimation at GPT-5.4 pricing - Failed/timed-out runs marked SKIPPED (not fake 100% reduction) - Partial result capture (tool.execution_partial_result events) - compare_results.py updated with context + cost metrics 13 benchmark prompts covering: - Cross-file analysis (config audit, test coverage, import maps) - MCP context bloat (same task with 0 vs 4 MCP servers) - Real-world tasks (docstring coverage, function index, env var audit) - Difficult cases that track where codeact still needs improvement README updated with benchmark results table. Unit tests for tool allow/deny configuration (14 tests). --- .github/plugin/marketplace.json | 18 + .gitignore | 21 + PLAN.md | 550 ++++++++ README.md | 252 +++- plugins/codeact/agents/codeact.agent.md.tmpl | 33 + plugins/codeact/hooks.json | 8 + plugins/codeact/hooks/pre-tool-use.ps1 | 75 ++ plugins/codeact/hooks/pre-tool-use.sh | 113 ++ .../instructions/codeact.instructions.md.tmpl | 39 + plugins/codeact/plugin.json | 12 + plugins/codeact/scripts/codeact | 85 ++ plugins/codeact/scripts/detect-backend.sh | 35 + .../codeact/scripts/install-instructions.ps1 | 101 ++ .../codeact/scripts/install-instructions.sh | 136 ++ plugins/codeact/scripts/mcp-bridge.py | 351 ++++++ plugins/codeact/scripts/preflight.ps1 | 61 + plugins/codeact/scripts/preflight.sh | 81 ++ .../codeact-install-hyperlight/SKILL.md | 20 + .../skills/codeact-install-hyperlight/run.sh | 5 + .../skills/codeact-install-monty/SKILL.md | 18 + .../skills/codeact-install-monty/run.sh | 5 + .../codeact/skills/codeact-install/SKILL.md | 36 + plugins/codeact/skills/codeact-install/run.sh | 6 + .../skills/hyperlight-codeact/SKILL.md | 90 ++ .../references/tool-patterns.md | 9 + .../hyperlight-codeact/scripts/codeact.py | 288 ++++- plugins/codeact/skills/monty-codeact/SKILL.md | 109 ++ .../monty-codeact/references/tool-patterns.md | 73 +- .../skills}/monty-codeact/scripts/codeact.py | 286 ++++- plugins/codeact/tests/compare_results.py | 230 ++++ .../codeact/tests/fixtures/setup-workspace.sh | 1083 ++++++++++++++++ .../tests/fixtures/user-tools/shout.py | 18 + .../tests/prompts/functional-natural.json | 38 + plugins/codeact/tests/prompts/functional.json | 64 + plugins/codeact/tests/prompts/perf.json | 104 ++ .../tests/results/perf-results-latest.json | 811 ++++++++++++ plugins/codeact/tests/run_tests.py | 1118 +++++++++++++++++ plugins/codeact/tests/unit/__init__.py | 0 .../codeact/tests/unit/test_tools_config.py | 144 +++ skills/hyperlight-codeact/SKILL.md | 151 --- skills/monty-codeact/SKILL.md | 155 --- 41 files changed, 6406 insertions(+), 426 deletions(-) create mode 100644 .github/plugin/marketplace.json create mode 100644 .gitignore create mode 100644 PLAN.md create mode 100644 plugins/codeact/agents/codeact.agent.md.tmpl create mode 100644 plugins/codeact/hooks.json create mode 100644 plugins/codeact/hooks/pre-tool-use.ps1 create mode 100755 plugins/codeact/hooks/pre-tool-use.sh create mode 100644 plugins/codeact/instructions/codeact.instructions.md.tmpl create mode 100644 plugins/codeact/plugin.json create mode 100755 plugins/codeact/scripts/codeact create mode 100755 plugins/codeact/scripts/detect-backend.sh create mode 100644 plugins/codeact/scripts/install-instructions.ps1 create mode 100755 plugins/codeact/scripts/install-instructions.sh create mode 100644 plugins/codeact/scripts/mcp-bridge.py create mode 100644 plugins/codeact/scripts/preflight.ps1 create mode 100755 plugins/codeact/scripts/preflight.sh create mode 100644 plugins/codeact/skills/codeact-install-hyperlight/SKILL.md create mode 100755 plugins/codeact/skills/codeact-install-hyperlight/run.sh create mode 100644 plugins/codeact/skills/codeact-install-monty/SKILL.md create mode 100755 plugins/codeact/skills/codeact-install-monty/run.sh create mode 100644 plugins/codeact/skills/codeact-install/SKILL.md create mode 100755 plugins/codeact/skills/codeact-install/run.sh create mode 100644 plugins/codeact/skills/hyperlight-codeact/SKILL.md rename {skills => plugins/codeact/skills}/hyperlight-codeact/references/tool-patterns.md (90%) rename {skills => plugins/codeact/skills}/hyperlight-codeact/scripts/codeact.py (72%) create mode 100644 plugins/codeact/skills/monty-codeact/SKILL.md rename {skills => plugins/codeact/skills}/monty-codeact/references/tool-patterns.md (59%) rename {skills => plugins/codeact/skills}/monty-codeact/scripts/codeact.py (67%) create mode 100644 plugins/codeact/tests/compare_results.py create mode 100755 plugins/codeact/tests/fixtures/setup-workspace.sh create mode 100644 plugins/codeact/tests/fixtures/user-tools/shout.py create mode 100644 plugins/codeact/tests/prompts/functional-natural.json create mode 100644 plugins/codeact/tests/prompts/functional.json create mode 100644 plugins/codeact/tests/prompts/perf.json create mode 100644 plugins/codeact/tests/results/perf-results-latest.json create mode 100644 plugins/codeact/tests/run_tests.py create mode 100644 plugins/codeact/tests/unit/__init__.py create mode 100644 plugins/codeact/tests/unit/test_tools_config.py delete mode 100644 skills/hyperlight-codeact/SKILL.md delete mode 100644 skills/monty-codeact/SKILL.md diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json new file mode 100644 index 0000000..cc02bb0 --- /dev/null +++ b/.github/plugin/marketplace.json @@ -0,0 +1,18 @@ +{ + "name": "copilot-skills", + "owner": { "name": "jsturtevant" }, + "metadata": { + "description": "Personal marketplace — codeact and friends", + "version": "0.1.0" + }, + "plugins": [ + { + "name": "codeact", + "description": "Collapse multi-step tool chains into sandboxed Python runs.", + "version": "0.1.0", + "source": "./plugins/codeact", + "license": "MIT", + "keywords": ["codeact", "codemode", "sandbox", "mcp", "tools"] + } + ] +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2571f8b --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.venv/ +venv/ + +# Codeact +# Per-plugin discovered-tools manifest (generated by /codeact-install) +.codeact-tools.json +# Per-plugin backend marker (generated by /codeact-install) +.codeact-backend +# Generated agent file (rendered from .tmpl at install time) +plugins/codeact/agents/codeact.agent.md +# Timestamped perf snapshots — keep only perf-results.json + perf-results-latest.json in git +plugins/codeact/tests/results/perf-results-2*.json + +# Editor / OS +.DS_Store +*.swp +.vscode/ diff --git a/PLAN.md b/PLAN.md new file mode 100644 index 0000000..0edb669 --- /dev/null +++ b/PLAN.md @@ -0,0 +1,550 @@ +# Plan: `codeact` Copilot CLI plugin + +## Scope locked + +- Plugin name: `codeact` +- Marketplace repo: `jsturtevant/copilot-skills`, manifest at `.github/plugin/marketplace.json` +- Backends: `monty` + `hyperlight`, default `auto` (KVM/mshv/Hyper-V detection, fallback to monty, override via `CODEACT_BACKEND`) +- Skills layout: two peer skills (`hyperlight-codeact`, `monty-codeact`), no umbrella +- Top-level `skills/` removed; everything under `plugins/codeact/` +- Reminder strategy: skill description + custom instructions snippet + PreToolUse deny (in enforcement mode) + custom agent +- Future submission target: `awesome-copilot` marketplace (not now) + +## Constraints discovered (Copilot CLI vs Claude Code) + +Copilot CLI hooks **cannot inject system prompts or modify user prompts**. Per https://docs.github.com/en/copilot/reference/hooks-configuration: + +| Hook | Output behavior | +|------|-----------------| +| SessionStart | Ignored | +| SessionEnd | Ignored | +| UserPromptSubmitted | Ignored (prompt modification not supported) | +| PreToolUse | Only `permissionDecision: "deny"` is processed | +| PostToolUse | Ignored | + +So the caveman trick (SessionStart stdout → system context, UserPromptSubmit `hookSpecificOutput.additionalContext` → per-turn reminder) is **not available** in Copilot CLI today. + +The only model-facing text channel a plugin controls is the **PreToolUse `permissionDecisionReason`** when denying. + +Custom instructions surfaces (per https://docs.github.com/en/copilot/reference/custom-instructions-support and https://docs.github.com/en/copilot/how-tos/copilot-cli/customize-copilot/add-custom-instructions) for Copilot CLI: +1. `.github/copilot-instructions.md` (repo-wide) +2. `.github/instructions/**/*.instructions.md` (path-specific, frontmatter `applyTo`, glob — e.g. `applyTo: "**"` matches everything) +3. `AGENTS.md` (agent instructions) +4. `$HOME/.copilot/copilot-instructions.md` (personal, applies globally) +5. `COPILOT_CUSTOM_INSTRUCTIONS_DIRS` env var — comma-separated directories scanned for `AGENTS.md` and `.github/instructions/**/*.instructions.md` + +**Install strategy:** drop a single self-owned file at `.github/instructions/codeact.instructions.md` with `applyTo: "**"`. Never modifies user's `copilot-instructions.md` or `AGENTS.md`. Easy to remove (one file). Global variant writes to `$HOME/.copilot/codeact.instructions.md` (or sets `COPILOT_CUSTOM_INSTRUCTIONS_DIRS` to plugin install dir). + +## Workaround layers (stack of activation surfaces) + +| Layer | Always-on? | User action | Strength | +|-------|-----------|-------------|----------| +| 1 Skill description matching | When prompt matches triggers | Install plugin | Low passive | +| 2 Custom agent (`/agent codeact`) | Per session, sticky | Invoke once per session | High but opt-in | +| 3 Repo `.github/instructions/codeact.instructions.md` (`applyTo: "**"`) | Yes, in that repo | Run `/codeact-install` once per repo | Medium, owned-file (no merge with user's instructions) | +| 4 Global `$HOME/.copilot/codeact.instructions.md` | Yes, all sessions | Run `/codeact-install --global` | Medium, set-and-forget | +| 5 PreToolUse `deny` reason | Yes, on tool calls | Set `CODEACT_MODE=nudge\|exclusive` | High self-correcting | +| 6 (future) MCP-in-sandbox | Yes when configured | Migrate MCP config | Highest for context savings | + +Default install wires layers 1+2+5(off). Recommended user step: run `/codeact-install` for layer 3 (or `--global` for layer 4). Power users opt into layer 5. + +## Config model: instructions+agent ARE the config + +Since Copilot CLI plugins can't inject runtime context, there is no in-memory "current backend" state to track. Instead: + +- **A self-owned path-specific instructions file IS the persisted configuration.** Switching backends = rewriting that one file. +- File location: `.github/instructions/codeact.instructions.md` (repo) or `$HOME/.copilot/codeact.instructions.md` (global). Frontmatter `applyTo: "**"` so it activates for every prompt. +- The file is owned wholly by codeact — the install/switch skills overwrite it atomically. No sentinel-bracketed merges needed because we never share a file with the user. +- The custom agent file `agents/codeact.agent.md` ships pre-templated; switch skills regenerate it with the chosen backend. +- `CODEACT_BACKEND` env var still wins at runtime for power users; skills set the *default* baked into the instructions file. + +## Discovery + preflight (always run on install/switch) + +Every management skill (`/codeact-install`, `/codeact-monty-backend`, `/codeact-hyperlight-backend`) runs the same pipeline before writing any config: + +1. **`scripts/preflight.sh `** — verify the chosen backend's runtime is actually usable. Exits non-zero with a diagnostic if any check fails. Checks per backend: + - **monty:** `python3 --version` ≥ 3.10, `uv --version` available (or fallback path documented), `pydantic-monty` installable from PyPI. + - **hyperlight:** `/dev/kvm` readable (Linux), or `mshv` / Hyper-V available (Windows). On macOS, fail with clear message "hyperlight unsupported on macOS, use monty". `python3 --version` ≤ 3.13 (Wasm guest constraint), `uv --version` available, `hyperlight-sandbox[wasm,python_guest]>=0.3.0` resolvable. + - **shared:** `bash` present, write access to target instructions path. +2. **`scripts/codeact --discover --backend `** — invokes the backend's `codeact.py --discover` (already exists) to enumerate registered tools. Output = JSON manifest of host tools + any MCP tools the backend would proxy in. +3. **`scripts/codeact --instructions --backend `** — LLM-ready Markdown reference for the same tool list (shape suitable for direct paste into instructions / agent). +4. **Template substitution** — `install-instructions.sh` reads the discovered manifest and substitutes: + - `{{BACKEND}}` → chosen backend name + - `{{TOOL_LIST}}` → comma-separated tool names from discovery + - `{{TOOL_REFERENCE}}` → full Markdown reference block + - `{{SYNTAX}}` → backend-specific syntax example (monty: plain calls; hyperlight: `call_tool(...)`) + - `{{CODEACT_DIR}}` → absolute plugin install path +5. **Atomic write** of `.github/instructions/codeact.instructions.md` and `agents/codeact.agent.md`. Both files now name the actual available tools, including any MCP servers configured at install time. + +If preflight fails, the skill aborts and prints the diagnostic. No partial config is written. + +Discovery is **re-runnable**: if user adds an MCP server later, they run `/codeact-install` again to refresh the tool list in the config files. + +## Management skills (slash-invoked) + +Three purpose-built skills, each a folder under `skills/` with `SKILL.md` + script. User types the slash prefix to invoke. + +| Slash invocation | Skill folder | Action | +|------------------|--------------|--------| +| `/codeact-install` | `skills/codeact-install/` | Run `detect-backend.sh` to pick best backend, then write `.github/instructions/codeact.instructions.md` (or `$HOME/.copilot/codeact.instructions.md` with `--global`). Regenerates `agents/codeact.agent.md`. Idempotent overwrite. | +| `/codeact-install-monty` | `skills/codeact-install-monty/` | Rewrite the instructions file + agent file with backend pinned to `monty`. | +| `/codeact-install-hyperlight` | `skills/codeact-install-hyperlight/` | Rewrite the instructions file + agent file with backend pinned to `hyperlight`. | + +Each `SKILL.md`: +- `name`: matches folder (e.g. `codeact-install`) +- `description`: short, includes "Use when user asks to install/configure codeact" so the matcher also fires on natural language +- `allowed-tools: shell` (so script runs without per-call confirmation — noted as security trade-off in skill body) +- Body: "Run `bash $SKILL_DIR/run.sh [--global]` from this skill's base directory." Skill scripts shell out to shared `scripts/install-instructions.sh` + `scripts/detect-backend.sh` in the plugin root. + +User flow: +``` +$ copilot +> /codeact-install + CodeAct: detected hyperlight (KVM available). Wrote + .github/instructions/codeact.instructions.md (applyTo: "**"). + Restart session to load. + +> /codeact-install-monty + +``` +copilot-skills/ +├── README.md +├── PLAN.md # this file +├── .github/plugin/marketplace.json +└── plugins/codeact/ + ├── plugin.json + ├── agents/ + │ ├── codeact.agent.md.tmpl # source template (with {{BACKEND}}, {{TOOL_LIST}}, etc.) + │ └── codeact.agent.md # rendered by switch skills (pre-rendered fallback committed) + ├── skills/ + │ ├── hyperlight-codeact/ # backend skill + │ │ ├── SKILL.md # description rewritten for trigger breadth + │ │ ├── references/tool-patterns.md + │ │ └── scripts/codeact.py + │ ├── monty-codeact/ # backend skill + │ │ ├── SKILL.md # description rewritten for trigger breadth + │ │ ├── references/tool-patterns.md + │ │ └── scripts/codeact.py + │ ├── codeact-install/ # /codeact-install management skill + │ │ ├── SKILL.md + │ │ └── run.sh + │ ├── codeact-install-monty/ # /codeact-install-monty + │ │ ├── SKILL.md + │ │ └── run.sh + │ └── codeact-install-hyperlight/ # /codeact-install-hyperlight + │ ├── SKILL.md + │ └── run.sh + ├── hooks.json + ├── hooks/ + │ ├── pre-tool-use.sh + │ └── pre-tool-use.ps1 + ├── instructions/ + │ └── codeact.instructions.md.tmpl # path-specific template, applyTo: "**", {{BACKEND}} substituted + └── scripts/ + ├── install-instructions.sh # accepts --backend [--global]; runs preflight + discovery + ├── install-instructions.ps1 + ├── detect-backend.sh # auto-pick backend + ├── preflight.sh # verify backend runtime usable, fail-fast diagnostics + ├── preflight.ps1 + └── codeact # thin dispatcher → backend script (also: --discover, --instructions) +``` + +## Component specs + +### `plugin.json` +```json +{ + "name": "codeact", + "description": "Collapse multi-step tool chains into one sandboxed Python run. Hyperlight + Monty backends.", + "version": "0.1.0", + "author": { "name": "jsturtevant" }, + "license": "MIT", + "repository": "https://github.com/jsturtevant/copilot-skills", + "keywords": ["codeact", "codemode", "sandbox", "tool-chaining", "mcp", "python"], + "agents": "agents/", + "skills": "skills/", + "hooks": "hooks.json" +} +``` + +No `commands` field — Copilot CLI uses skills as the slash-command surface. + +### Skill descriptions (discovery surface) + +Both `hyperlight-codeact/SKILL.md` and `monty-codeact/SKILL.md`: +```yaml +description: | + CodeAct via . Use when chaining 3+ tool calls, looping over files, + filtering/aggregating tool results, calling MCP tools in sequence, batch + operations, "for each", "find all then", "process all". Collapses N + round-trips into one sandboxed Python run via scripts/codeact.py. + Tools inside sandbox: view, create, edit, glob, grep, bash, sql, web_fetch, + github_api + any registered MCP tool. Trigger: "codeact", "chain tools", + "sandbox", "batch", "for each". +``` +Body content unchanged (current is solid). + +### Custom agent — `agents/codeact.agent.md` + +The whole point: agent has **no direct host tools**. Only `bash` to run the +codeact dispatcher. All file reads, edits, searches, MCP calls happen +*inside* the sandbox via the Python program. This forces codeact-or-nothing +without needing the PreToolUse exclusive-mode hook. + +```yaml +--- +name: codeact +description: Sandbox-only agent. All work happens inside one Python run via codeact dispatcher. +tools: ["bash"] +--- + +You have exactly one tool: `bash`. Use it only to invoke the codeact +dispatcher: + + bash {{CODEACT_DIR}}/scripts/codeact --auto --workspace . --code '' + +All file reads, edits, searches, shell commands, MCP calls happen *inside* +that Python program. Available sandbox functions: + +{{TOOL_LIST}} + +Backend: **{{BACKEND}}** (auto-detected at install; override with +`CODEACT_BACKEND=monty|hyperlight`). + +Hyperlight syntax: `call_tool("name", **kwargs)`. +Monty syntax: plain function calls — `view(path="...")`, `glob(pattern="...")`. + +If a task is genuinely a single read or single edit and writing Python +would be more verbose than the work itself, say so explicitly and ask the +user to switch agents. Do not try to work around the lack of direct tools. +``` + +Note: `{{TOOL_LIST}}` and `{{BACKEND}}` are substituted by `install-instructions.sh` +during install/switch, so the agent prompt always names the actually-discovered +sandbox tools (incl. MCP tools). + +### Custom instructions file (path-specific, owned by codeact) + +`instructions/codeact.instructions.md.tmpl` (template): +```markdown +--- +applyTo: "**" +--- + +## CodeAct (installed via `codeact` plugin, backend: {{BACKEND}}) + +When a task needs ≥3 tool calls, a loop over files, filtering/aggregation +of tool results, or chaining MCP tools, prefer one sandboxed Python run via +`bash {{CODEACT_DIR}}/scripts/codeact --code '...'` instead of serial direct +calls. + +Current backend: **{{BACKEND}}** (override with `CODEACT_BACKEND=monty|hyperlight`). + +### Available sandbox tools (discovered at install time) + +{{TOOL_LIST}} + +{{TOOL_REFERENCE}} + +See `{{CODEACT_DIR}}/skills/{{BACKEND}}-codeact/SKILL.md` for invocation syntax. +``` + +`scripts/install-instructions.sh`: +- `--backend ` (or auto via `detect-backend.sh` if omitted) +- `--global` → write to `$HOME/.copilot/codeact.instructions.md` +- default → write to `./.github/instructions/codeact.instructions.md` +- Pipeline: `preflight.sh ` → abort on failure → `codeact --discover --backend ` → `codeact --instructions --backend ` → substitute placeholders → atomic tmp+rename, 0644. +- Same pipeline regenerates `agents/codeact.agent.md` so the agent prompt names the same tool list. +- File is fully owned: overwrite without merge. Removal = `rm` the one file. + +### PreToolUse enforcement + +`hooks.json`: +```json +{ + "version": 1, + "hooks": { + "preToolUse": [ + { "type": "command", "bash": "./hooks/pre-tool-use.sh", "timeoutSec": 5 } + ] + } +} +``` + +`hooks/pre-tool-use.sh` driven by `CODEACT_MODE`: + +| `CODEACT_MODE` | Behavior | +|---|---| +| unset / `off` | Read input, exit 0. No interference. | +| `nudge` | Counter file `${XDG_RUNTIME_DIR:-/tmp}/codeact-$PPID.count`. Increment on each read-only tool (`view`, `glob`, `grep`, `rg`, `read_file`, `file_search`). After ≥3 in a row, deny next call with reason text. Reset counter on deny or on `bash` invoking `scripts/codeact.py`. | +| `exclusive` | Allow only `bash` calls whose args contain `scripts/codeact.py` (or `codeact `). Deny everything else with reason text. | + +Reason text (model-facing channel): +``` +CodeAct enforcement active (CODEACT_MODE=). Collapse this work into +one sandboxed Python run: + + bash plugins/codeact/skills/-codeact/scripts/codeact.py \ + --auto --workspace . --code '' + +Sandbox tools: view, create, edit, glob, grep, bash, sql, web_fetch, +github_api + MCP. Override backend with CODEACT_BACKEND=monty|hyperlight. +Disable enforcement: unset CODEACT_MODE. +``` + +Counter state per-PPID so concurrent sessions don't collide. Silent-fail on FS errors. PowerShell variant mirrors logic. + +### Backend auto-detect — `scripts/detect-backend.sh` + +Honors `CODEACT_BACKEND` if set. Otherwise: +- macOS → `monty` +- Linux → `/dev/kvm` or `/dev/mshv` readable → `hyperlight`, else `monty` +- Windows → Hyper-V available → `hyperlight`, else `monty` + +`scripts/codeact` = thin wrapper. Subcommands: +- (no flag) `--code '...'` → run sandboxed code via detected backend's `codeact.py` +- `--discover [--backend X]` → emit tools JSON manifest +- `--instructions [--backend X]` → emit LLM-ready tool reference Markdown + +### Preflight — `scripts/preflight.sh` + +Usage: `preflight.sh ` → exit 0 if usable, non-zero with human-readable diagnostic otherwise. Called automatically by install/switch skills before any config write. Can also be invoked directly by users to debug install issues. + +### Marketplace — `.github/plugin/marketplace.json` + +```json +{ + "name": "copilot-skills", + "owner": { "name": "jsturtevant" }, + "metadata": { + "description": "Personal marketplace — codeact and friends", + "version": "0.1.0" + }, + "plugins": [ + { + "name": "codeact", + "description": "Collapse multi-step tool chains into sandboxed Python runs.", + "version": "0.1.0", + "source": "./plugins/codeact", + "license": "MIT", + "keywords": ["codeact", "sandbox", "mcp"] + } + ] +} +``` + +User install: +```bash +copilot plugin marketplace add jsturtevant/copilot-skills +copilot plugin install codeact@copilot-skills +bash ~/.copilot/installed-plugins/copilot-skills/codeact/scripts/install-instructions.sh +``` + +## Build order + +1. Move `skills/hyperlight-codeact/` → `plugins/codeact/skills/hyperlight-codeact/`. Same for monty. Delete top-level `skills/`. +2. Rewrite both backend `SKILL.md` descriptions with expanded triggers. +3. Write `plugins/codeact/plugin.json`. +4. Write `agents/codeact.agent.md` template (with `{{BACKEND}}`, `{{TOOL_LIST}}` placeholders). +5. Write `instructions/codeact.instructions.md.tmpl` (path-specific, `applyTo: "**"`, all `{{...}}` placeholders). +6. Write `scripts/detect-backend.sh` + `scripts/codeact` dispatcher (with `--discover` and `--instructions` subcommands wrapping existing backend `codeact.py`). +7. Write `scripts/preflight.sh` + `scripts/preflight.ps1` (per-backend runtime checks). +8. Write `scripts/install-instructions.sh` (`--backend` + `--global` flags; pipeline = preflight → discover → instructions → substitute → atomic write of both instructions file and agent file). +9. Write management skills: + - `skills/codeact-install/{SKILL.md,run.sh}` (auto-detect backend + preflight + discover + write) + - `skills/codeact-install-monty/{SKILL.md,run.sh}` (preflight monty + discover + write) + - `skills/codeact-install-hyperlight/{SKILL.md,run.sh}` (preflight hyperlight + discover + write) + Each `run.sh` shells out to shared `../../scripts/install-instructions.sh`. +10. Write `hooks.json` + `hooks/pre-tool-use.sh` + `.ps1`. +11. Write `.github/plugin/marketplace.json`. +12. Rewrite `README.md`. +13. Local test: `copilot plugin install ./plugins/codeact`, verify `/skills list`, `/agent`, `/codeact-install` (preflight + discovery), inspect generated instructions file for actual tool list, switch skills, exercise PreToolUse counter, verify deny path. + +## Test harness + +### Design + +End-to-end tests using the real `copilot` CLI. Creates a temp workspace with known files, runs prompts in multiple arms, captures JSONL output, and compares metrics. + +### File layout + +``` +plugins/codeact/tests/ +├── run_tests.py # Main test runner: `all` | `functional` | `perf` subcommands +├── prompts/ +│ ├── functional.json # Functional test prompts + expected assertions +│ └── perf.json # Perf test prompts (multi-step tasks) +└── fixtures/ + └── setup-workspace.sh # Creates temp workspace with known file structure +``` + +### Temp workspace structure (created by `setup-workspace.sh`) + +``` +/tmp/codeact-test-XXXX/ +├── src/ +│ ├── app.py # 100-line file with 5 TODOs +│ ├── utils.py # Helper functions, some without docstrings +│ ├── models.py # Data models +│ ├── api.py # API endpoints with HTTP references +│ └── config.py # Configuration +├── config/ +│ ├── settings.json # Valid JSON +│ ├── database.json # Valid JSON +│ └── broken.json # Invalid JSON (for error-tolerant test) +├── tests/ +│ ├── test_app.py # Test file +│ └── test_utils.py # Test file +└── README.md +``` + +### Test arms (3-way comparison) + +| Arm | CLI flags | Purpose | +|-----|-----------|---------| +| `baseline` | `--no-custom-instructions` (no plugin) | Standard multi-tool behavior | +| `codeact` | `--plugin-dir ./plugins/codeact` | CodeAct with skill matching | +| `codeact-instruct` | `--plugin-dir ./plugins/codeact` + instructions file installed | CodeAct with always-on instructions | + +All arms run with: `--output-format json --yolo -s -p ""` + +### Functional test prompts (`prompts/functional.json`) + +Each prompt has assertions about what should appear in the output: + +```json +[ + { + "id": "multi-file-search", + "prompt": "Find all TODO comments across all Python files in src/ and list each with its file and line number. Use codeact.", + "assertions": { + "codeact_invoked": true, + "output_contains": ["TODO", "app.py"], + "min_todos_found": 3 + } + }, + { + "id": "batch-count", + "prompt": "Count lines of code in each Python file under src/ and show the top 3 largest. Use codeact to do this in one pass.", + "assertions": { + "codeact_invoked": true, + "output_contains": ["lines", "app.py"] + } + }, + { + "id": "json-validate", + "prompt": "Check all JSON files in config/ for valid syntax. Report which are valid and which have errors. Use codeact.", + "assertions": { + "codeact_invoked": true, + "output_contains": ["broken.json", "ERROR"] + } + }, + { + "id": "single-file-no-codeact", + "prompt": "Read README.md and tell me what it says.", + "assertions": { + "codeact_invoked": false, + "output_contains": ["README"] + } + } +] +``` + +### Performance test prompts (`prompts/perf.json`) + +Multi-step tasks where CodeAct should show significant token/request reduction: + +```json +[ + { + "id": "find-todos", + "prompt": "Find all TODO comments in every Python file under src/, show each with file path and line number, then count the total.", + "expected_baseline_tool_calls": ">=5", + "expected_codeact_tool_calls": "<=2" + }, + { + "id": "code-stats", + "prompt": "For each Python file in src/, count the number of functions, classes, and lines. Show a summary table.", + "expected_baseline_tool_calls": ">=6", + "expected_codeact_tool_calls": "<=2" + }, + { + "id": "batch-edit-check", + "prompt": "Find all files in src/ that import 'os' and list them with the line numbers where the import appears.", + "expected_baseline_tool_calls": ">=4", + "expected_codeact_tool_calls": "<=2" + } +] +``` + +### Metrics extracted by `run_tests.py` + +From JSONL output (`--output-format json`): + +| Metric | Source event | Field | +|--------|-------------|-------| +| Output tokens | `assistant.message` | `data.outputTokens` | +| Premium requests | `result` | `usage.premiumRequests` | +| API duration (ms) | `result` | `usage.totalApiDurationMs` | +| Session duration (ms) | `result` | `usage.sessionDurationMs` | +| Tool call count | `assistant.message` | `data.toolRequests[]` (length) | +| Tool names used | `assistant.message` | `data.toolRequests[].toolName` | +| CodeAct invoked? | tool calls | Any `bash` call with `codeact` in args | + +### `run_tests.py` output + +``` +┌─────────────────┬──────────┬─────────┬──────────────┬─────────┐ +│ Prompt │ Arm │ Tokens │ Tool Calls │ Requests│ +├─────────────────┼──────────┼─────────┼──────────────┼─────────┤ +│ find-todos │ baseline │ 1,250 │ 8 │ 12 │ +│ find-todos │ codeact │ 450 │ 1 │ 4 │ +│ find-todos │ Δ │ -64% │ -87% │ -67% │ +├─────────────────┼──────────┼─────────┼──────────────┼─────────┤ +│ code-stats │ baseline │ 2,100 │ 12 │ 18 │ +│ code-stats │ codeact │ 600 │ 1 │ 5 │ +│ code-stats │ Δ │ -71% │ -92% │ -72% │ +└─────────────────┴──────────┴─────────┴──────────────┴─────────┘ + +PASS: codeact arm shows ≥40% token reduction on all multi-step prompts. +PASS: codeact_invoked=true for all multi-step prompts. +PASS: codeact_invoked=false for single-file prompt. +``` + +### `run_tests.py all` workflow + +```bash +cd plugins/codeact +python3 tests/run_tests.py all # full run, auto-creates + cleans temp workspace +python3 tests/run_tests.py functional --prompts ... --workspace ... --plugin-dir ... +python3 tests/run_tests.py perf --prompts ... --workspace ... --plugin-dir ... +``` + +`all` does: preflight (copilot CLI, python3, plugin.json) → create temp workspace via `fixtures/setup-workspace.sh` → verify plugin loads → functional → perf → cleanup (skip with `--keep-workspace`). + +### Running + +```bash +cd plugins/codeact +python3 tests/run_tests.py all +``` + +Requires: `copilot` CLI authenticated, `python3`, `uv` (for backend deps). +Each perf prompt runs twice (baseline + codeact) so costs ~2x premium requests per prompt. + +## Future / parking lot + +- **Compression mode** (caveman-style output compression) gated by `config.compression.enabled`. Out of scope for v0.1. +- **MCP-in-sandbox** registry (`~/.config/codeact/mcp.json`) so MCP schemas don't bloat host context — separate proxy mode in `codeact.py --mcp `. +- **Submit to `awesome-copilot` marketplace** once stable. +- **File feature request** with GitHub for hook stdout context-injection (parity with Claude Code's `additionalContext`). + +## Tool customization (shipped) + +Both backends honour the same user-config layer (resolved from `CODEACT_CONFIG_DIR`, else `$XDG_CONFIG_HOME/codeact/`, else `~/.config/codeact/`): + +- **Allowlist / denylist** via env (`CODEACT_TOOLS`, `CODEACT_DISABLE`) or `config.json` (`enabled`, `disabled`). Env wins. +- **Custom tools** via drop-in Python files in `/tools/*.py`. Each file defines a callable (default `run`) plus an optional `TOOL` dict (`name`, `description`, `parameters`, `function`). Loaded as `implementation.type = "user"`. Trust = host process. +- Custom tools also subject to allow/deny filtering. +- After config changes, re-run `/codeact-install` to regenerate the discovered tool list baked into the instructions file + agent prompt. +- Tested by `tests/unit/test_tools_config.py` (14 cases covering both backends, run via `python3 tests/run_tests.py unit` or as part of `all`). Functional prompt `custom-tool-shout` exercises the full path through the real `copilot` CLI: workspace fixture drops `shout.py` into `/.codeact-config/tools/`, prompt sets `CODEACT_CONFIG_DIR` so the agent can call it via codeact. diff --git a/README.md b/README.md index 305a960..63a6b87 100644 --- a/README.md +++ b/README.md @@ -1,63 +1,241 @@ -# Copilot Skills +# CodeAct — Copilot CLI Plugin -Agent skills for GitHub Copilot CLI that implement the **CodeAct pattern** — collapse multi-step tool chains into a single sandboxed Python execution. +Collapse multi-step tool chains into a single sandboxed Python execution. Instead of N individual tool calls (model → tool → model → tool …), the agent writes one Python program that chains all the tools together and runs it in a single turn. + +## Before / After + +**Before (standard):** 8 tool calls, 12 API requests, ~1,250 output tokens +``` +model: "I'll search for TODOs" → grep → model: "Found matches in 3 files" +→ view file1 → model: "File 1 has..." → view file2 → model: "File 2 has..." +→ view file3 → model: "File 3 has..." → model: "Here's the summary..." +``` + +**After (codeact):** 1 tool call, 4 API requests, ~450 output tokens +``` +model: "I'll find all TODOs in one pass" → bash codeact.py --code ' + for f in glob(pattern="**/*.py"): + content = view(path=f) + for i, line in enumerate(content.split(chr(10))): + if "TODO" in line: + print(f"{f}:{i+1}: {line.strip()}") +' → model: "Here are all TODOs..." +``` ## Install +**From marketplace** (recommended): + +```bash +copilot plugin marketplace add jsturtevant/copilot-skills +copilot plugin install codeact@copilot-skills + +# First time use — run inside Copilot CLI +/codeact-install + +# Or run install script directly +bash ~/.copilot/installed-plugins/copilot-skills/codeact/scripts/install-instructions.sh + +# Global install (applies to all repos) +bash ~/.copilot/installed-plugins/copilot-skills/codeact/scripts/install-instructions.sh --global +``` + +**From local checkout** (development): + +```bash +copilot plugin install ./plugins/codeact +bash plugins/codeact/scripts/install-instructions.sh +``` + +## Backends + +| Backend | Runtime | Startup | Python support | Isolation | Requires | +|---------|---------|---------|---------------|-----------|----------| +| **monty** (default) | [Pydantic Monty](https://github.com/pydantic/monty) | <1μs | Subset (no classes, limited stdlib) | Interpreter-level | Python 3.10+ | +| **hyperlight** | [Hyperlight](https://github.com/hyperlight-dev/hyperlight) | ~680ms | Full CPython (Wasm) | Micro-VM | KVM/mshv/Hyper-V, Python ≤3.13 | + +Auto-detected at install. Override at install time: `--backend monty|hyperlight`. + +Switch backend: ```bash -# Install a specific skill -gh skill install jsturtevant/copilot-skills monty-codeact -gh skill install jsturtevant/copilot-skills hyperlight-codeact +# Use the management skills +/codeact-install-monty +/codeact-install-hyperlight -# Or browse and choose interactively -gh skill install jsturtevant/copilot-skills +# Or directly +bash plugins/codeact/scripts/install-instructions.sh --backend monty ``` -Then `/skills reload` in Copilot CLI to pick them up. +## Sandbox Tools + +Both backends register tools matching **Copilot CLI built-in tool names**: + +| Tool | What it does | Requires | +|------|-------------|----------| +| `view` | Read files / list directories | — | +| `create` | Create new files | — | +| `edit` | Surgical string replacement | — | +| `glob` | Find files by pattern | — | +| `bash` | Run shell commands | — | +| `sql` | SQLite queries | — | +| `grep` | Search file contents | `rg` | +| `web_fetch` | Fetch URLs | `curl` | +| `github_api` | GitHub REST API | `gh` | + +**Monty syntax:** `view(path="README.md")` — natural function calls +**Hyperlight syntax:** `call_tool("view", path="README.md")` — via wrapper -## Available Skills +### Customising tools -### monty-codeact +Both backends consult the same user config (defaults to `~/.config/codeact/`, +override with `CODEACT_CONFIG_DIR`). -**CodeAct with [Pydantic Monty](https://github.com/pydantic/monty)** — a minimal, secure Python interpreter written in Rust. +**Disable built-ins** (allowlist or denylist; env wins over config file): -- Sub-microsecond startup (<1μs) -- Tools called as natural Python functions: `view(path="README.md")` -- Lightweight: `pip install pydantic-monty` (~4.5MB) -- Auto-installs dependencies via `uv` if missing +```bash +# Denylist via env — drop bash + sql for this session +CODEACT_DISABLE=bash,sql copilot ... + +# Allowlist via env — only view/glob/grep registered +CODEACT_TOOLS=view,glob,grep copilot ... +``` + +Or persist in `~/.config/codeact/config.json`: -Best for: fast, lightweight tool chaining where full Python isn't needed. +```json +{ + "disabled": ["bash", "sql"], + "enabled": [] +} +``` + +After changing config, re-run `/codeact-install` so the instructions file ++ agent prompt reflect the new tool list. -### hyperlight-codeact +**Add your own tools** — drop a `.py` file in `~/.config/codeact/tools/`. +File stem becomes the tool name; the file must define a callable (default: +`run`) plus an optional `TOOL` metadata dict: -**CodeAct with [Hyperlight](https://github.com/hyperlight-dev/hyperlight)** — micro-VM sandbox using WebAssembly. +```python +# ~/.config/codeact/tools/shout.py +"""Shout text back in uppercase.""" -- Full CPython runtime inside a Wasm sandbox -- Tools called via `call_tool("view", path="README.md")` -- Stronger isolation (separate micro-VM per execution) -- Auto-installs dependencies via `uv` if missing +TOOL = { + "description": "Echo input text in uppercase.", + "parameters": { + "text": {"type": "string", "required": True}, + }, + # "name": "shout", # optional; defaults to filename stem + # "function": "run", # optional; defaults to "run" +} + +def run(text: str = "") -> str: + return text.upper() +``` -Best for: when you need full Python support or stronger sandbox isolation. +After adding, re-run `/codeact-install` to refresh discovery. Custom tools +also honor allowlist/denylist filters. -### Shared features +> **Trust:** custom tools run on the host with full process privileges +> (same as built-in `bash`). Only install code you trust. -Both skills discover and register tools that match **Copilot CLI built-in tool names**: +## Activation Layers -| Copilot CLI tool | Sandbox function | What it does | -|---|---|---| -| `view` | `view()` / `call_tool("view")` | Read files / list directories | -| `create` | `create()` / `call_tool("create")` | Create new files | -| `edit` | `edit()` / `call_tool("edit")` | Surgical string replacement | -| `glob` | `glob()` / `call_tool("glob")` | Find files by pattern | -| `grep` | `grep()` / `call_tool("grep")` | Search file contents (needs `rg`) | -| `bash` | `bash()` / `call_tool("bash")` | Run shell commands | -| `sql` | `sql()` / `call_tool("sql")` | SQLite queries | -| `web_fetch` | `web_fetch()` / `call_tool("web_fetch")` | Fetch URLs (needs `curl`) | -| `github_api` | `github_api()` / `call_tool("github_api")` | GitHub REST API (needs `gh`) | +| Layer | Always-on? | How | +|-------|-----------|-----| +| Skill description matching | When prompt matches | Install plugin | +| Custom agent (`/agent codeact`) | Per session | Type once | +| Repo instructions (`/codeact-install`) | In that repo | Run once | +| Global instructions (`/codeact-install --global`) | All sessions | Run once | +| PreToolUse enforcement (`CODEACT_MODE=nudge\|exclusive`) | Yes | Set env var | + +Layered-activation pattern borrowed from [caveman](https://github.com/JuliusBrussee/caveman) (skill + agent + always-on instructions + tool-call enforcement). + +### Copilot CLI limitations (vs Claude Code) + +Copilot CLI plugin [hooks](https://docs.github.com/en/copilot/reference/hooks-configuration) **cannot inject system prompts or modify user prompts** — only `PreToolUse` `permissionDecision: "deny"` reaches the model. So the always-on caveman trick (SessionStart → system context, UserPromptSubmit → per-turn reminder) isn't available, which is why this plugin leans on a self-owned `.github/instructions/codeact.instructions.md` plus a custom agent. Hook stdout-as-context (parity with Claude Code's `additionalContext`) would let plugins like this self-activate without writing files. + +## Testing (developers) + +End-to-end harness runs prompts through the real `copilot` CLI in a temp +workspace and compares **baseline vs codeact** arms for token / tool-call / +premium-request reduction. Unit tests for the tool-config layer (allow/deny +lists + custom tool loading) live under `tests/unit/` and run first via +`unittest discover` — fast, no `copilot` CLI needed. + +**Prerequisites:** authenticated `copilot` CLI, [`uv`](https://docs.astral.sh/uv/getting-started/installation/) (resolves Python + any script deps on demand — no manual `pip install`). Each perf prompt runs twice (baseline + codeact) so it consumes ~2× premium requests per prompt. + +All commands below run from the **repo root**. + +```bash +# Unit tests only (no copilot CLI required) +uv run plugins/codeact/tests/run_tests.py unit + +# Functional only — auto-creates + cleans up a temp workspace +uv run plugins/codeact/tests/run_tests.py functional + +# Perf only — baseline vs codeact comparison (auto workspace) +uv run plugins/codeact/tests/run_tests.py perf + +# Full run — unit + preflight + functional + perf + cleanup +uv run plugins/codeact/tests/run_tests.py all + +# Keep the temp workspace for inspection (works with all/functional/perf) +uv run plugins/codeact/tests/run_tests.py all --keep-workspace + +# Custom token-reduction threshold (default 40%; all + perf) +uv run plugins/codeact/tests/run_tests.py perf --min-token-reduction 30 +``` + +**Reuse an existing workspace** (skip auto-create, e.g. when iterating on a fixture): + +```bash +uv run plugins/codeact/tests/run_tests.py functional \ + --workspace /tmp/my-workspace + +uv run plugins/codeact/tests/run_tests.py perf \ + --workspace /tmp/my-workspace \ + --prompts plugins/codeact/tests/prompts/perf.json +``` + +Perf results are written to `plugins/codeact/tests/results/perf-results-.json` +(plus `plugins/codeact/tests/results/perf-results-latest.json` as a stable +pointer to the most recent run). + +**Compare two runs** (e.g. before/after a change): + +```bash +# Auto-compare: latest vs previous run (no args needed) +uv run plugins/codeact/tests/compare_results.py + +# Explicit files with labels +uv run plugins/codeact/tests/compare_results.py \ + plugins/codeact/tests/results/perf-results-20260424T101500Z.json \ + plugins/codeact/tests/results/perf-results-latest.json \ + --a-label before --b-label after + +# Custom plot output path +uv run plugins/codeact/tests/compare_results.py --out /tmp/diff.png +``` + +Without `uv`, falls back to `python3 tests/compare_results.py ...` and prints +the delta table only (plot needs `pip install matplotlib numpy`). ## Why CodeAct? -Instead of N individual tool calls (model → tool → model → tool …), the agent writes one Python program that chains all the tools together and runs it in a single turn. This cuts latency by ~50% and token usage by 85%+ for multi-step tasks. +Instead of N individual tool calls (model → tool → model → tool …), the agent writes one Python program that chains all the tools together and runs it in a single turn. Fewer turns means the conversation context — system prompt, tool definitions, prior messages — is replayed fewer times. With MCP servers loaded, each server's tool catalog adds to that context, so the savings compound. + +Each test runs the same prompt twice: once as a **baseline** (standard Copilot CLI, no plugin) and once with **codeact** (plugin loaded). Token counts are extracted from copilot process logs. Tests use a 30+ file Python project with handlers, services, middleware, configs, and tests. + +| Task | Turns | Input Tokens | Est. Cost Savings | +|------|:-----:|:------------:|:-----------------:| +| Test coverage + 4 MCP servers | 6 → 2 | 335K → 103K | **69%** | +| Full project function index | 4 → 2 | 130K → 57K | **57%** | +| Test coverage (no MCP) | 4 → 2 | 123K → 58K | **57%** | +| Docstring coverage | 3 → 2 | 86K → 56K | **49%** | +| MCP docs cross-ref + 4 servers | 3 → 3 | 167K → 88K | **49%** | + +Cost estimated at GPT-5.4 pricing ($2.50/M input, $15/M output). Run `uv run plugins/codeact/tests/run_tests.py perf --backend monty` to reproduce. For more on the pattern, see [CodeAct with Hyperlight](https://devblogs.microsoft.com/agent-framework/codeact-with-hyperlight/) from Microsoft. diff --git a/plugins/codeact/agents/codeact.agent.md.tmpl b/plugins/codeact/agents/codeact.agent.md.tmpl new file mode 100644 index 0000000..97aa467 --- /dev/null +++ b/plugins/codeact/agents/codeact.agent.md.tmpl @@ -0,0 +1,33 @@ +--- +name: codeact +description: Sandbox-only agent. All work in one Python run. Use for batch file ops, cross-referencing, looping, aggregation. +tools: ["bash"] +--- + +One tool: `bash`. Invoke the codeact dispatcher: + + bash {{CODEACT_DIR}}/scripts/codeact --auto --workspace . --code '' + +Sandbox functions: {{TOOL_LIST}} + +Return types (critical): +- `glob(pattern=...)` → **list of strings** like `["src/app.py", ...]` +- `view(path=...)` → **string** (file content) +- `mcp_call(server=..., tool=..., ...)` → **string** +- `bash(command=...)` → **dict** with stdout/stderr/returncode +- No `os.path`, no `os.walk` — use `glob()` and `view()`. + +{{SYNTAX}} + +{{BACKEND_LIMITATIONS}} + +Rules: +- **One bash call, one program.** Do NOT scout with view/glob first. Do NOT + retry with extra tool calls — fix the program instead. +- **MCP inside sandbox, not outside.** When `.mcp.json` is configured, use + `mcp_call(server="name", tool="tool_name", ...)` INSIDE the sandbox. +- Wrap file reads in try/except. +- Use double quotes in Python code (avoids shell quoting issues with `--code '...'`). +- For ≤5 files, tell user to switch agents. + +{{TOOL_REFERENCE}} diff --git a/plugins/codeact/hooks.json b/plugins/codeact/hooks.json new file mode 100644 index 0000000..76963d1 --- /dev/null +++ b/plugins/codeact/hooks.json @@ -0,0 +1,8 @@ +{ + "version": 1, + "hooks": { + "preToolUse": [ + { "type": "command", "bash": "./hooks/pre-tool-use.sh", "timeoutSec": 5 } + ] + } +} diff --git a/plugins/codeact/hooks/pre-tool-use.ps1 b/plugins/codeact/hooks/pre-tool-use.ps1 new file mode 100644 index 0000000..af7cf2d --- /dev/null +++ b/plugins/codeact/hooks/pre-tool-use.ps1 @@ -0,0 +1,75 @@ +# pre-tool-use.ps1 — Windows PreToolUse hook for codeact enforcement +$ErrorActionPreference = "SilentlyContinue" + +$Mode = if ($env:CODEACT_MODE) { $env:CODEACT_MODE } else { "off" } + +# Fast path +if ($Mode -eq "off" -or $Mode -eq "") { + $null = [Console]::In.ReadToEnd() + exit 0 +} + +$InputRaw = [Console]::In.ReadToEnd() +$InputObj = $InputRaw | ConvertFrom-Json + +$ToolName = if ($InputObj.toolName) { $InputObj.toolName } elseif ($InputObj.tool_name) { $InputObj.tool_name } else { "" } +$ToolArgs = if ($InputObj.toolInput) { $InputObj.toolInput | ConvertTo-Json -Compress } elseif ($InputObj.input) { $InputObj.input | ConvertTo-Json -Compress } else { "{}" } + +function Test-CodeActCall { + if ($ToolName -ne "bash" -and $ToolName -ne "shell") { return $false } + return $ToolArgs -match 'codeact\.py|scripts/codeact' +} + +# Read discovered tool list from install-time manifest +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$PluginDir = Split-Path -Parent $ScriptDir +$ToolsFile = Join-Path $PluginDir ".codeact-tools.json" +$InstalledTools = "view, create, edit, glob, bash, sql" +if (Test-Path $ToolsFile) { + try { + $manifest = Get-Content $ToolsFile -Raw | ConvertFrom-Json + $InstalledTools = ($manifest.tools | ForEach-Object { $_.name }) -join ', ' + } catch {} +} + +$DenyReason = @" +CodeAct enforcement active (CODEACT_MODE=$Mode). Collapse this work into one sandboxed Python run: + + bash plugins/codeact/scripts/codeact --code '' + +Sandbox tools: ${InstalledTools}. +Disable enforcement: unset CODEACT_MODE. +"@ + +function Send-Deny { + @{ permissionDecision = "deny"; permissionDecisionReason = $DenyReason } | ConvertTo-Json -Compress + exit 0 +} + +function Send-Allow { + Write-Output '{}' + exit 0 +} + +$CounterFile = "$env:TEMP\codeact-$PID.count" + +switch ($Mode) { + "nudge" { + if (Test-CodeActCall) { "0" | Set-Content $CounterFile; Send-Allow } + $readOnly = @("view", "glob", "grep", "rg", "read_file", "file_search") + if ($readOnly -contains $ToolName) { + $count = if (Test-Path $CounterFile) { [int](Get-Content $CounterFile) } else { 0 } + $count++ + "$count" | Set-Content $CounterFile + if ($count -ge 3) { "0" | Set-Content $CounterFile; Send-Deny } + } else { + "0" | Set-Content $CounterFile + } + Send-Allow + } + "exclusive" { + if (Test-CodeActCall) { Send-Allow } + Send-Deny + } + default { Send-Allow } +} diff --git a/plugins/codeact/hooks/pre-tool-use.sh b/plugins/codeact/hooks/pre-tool-use.sh new file mode 100755 index 0000000..d811e81 --- /dev/null +++ b/plugins/codeact/hooks/pre-tool-use.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# pre-tool-use.sh — PreToolUse hook for codeact enforcement +# Driven by CODEACT_MODE env var: +# unset / "off" → pass through (exit 0) +# "nudge" → count sequential read-only tool calls, deny after ≥3 +# "exclusive" → allow only bash calls invoking codeact, deny all else +# +# Input: JSON on stdin with tool call details +# Output: JSON on stdout with permissionDecision + reason (or empty for allow) + +set -uo pipefail + +MODE="${CODEACT_MODE:-off}" + +# Fast path: no enforcement +if [[ "$MODE" == "off" ]] || [[ -z "$MODE" ]]; then + cat > /dev/null # consume stdin + exit 0 +fi + +# Read tool call from stdin +INPUT=$(cat) + +# Extract tool name and arguments using jq (per Copilot docs best practice) +TOOL_NAME=$(echo "$INPUT" | jq -r '.toolName // .tool_name // ""' 2>/dev/null || echo "") +TOOL_ARGS=$(echo "$INPUT" | jq -c '.toolInput // .input // {}' 2>/dev/null || echo "{}") + +# Check if this is a codeact bash call +is_codeact_call() { + if [[ "$TOOL_NAME" != "bash" ]] && [[ "$TOOL_NAME" != "shell" ]]; then + return 1 + fi + echo "$TOOL_ARGS" | grep -qE 'codeact\.py|scripts/codeact' +} + +# Counter file for nudge mode (per-PPID to avoid session collisions) +COUNTER_DIR="${XDG_RUNTIME_DIR:-/tmp}" +COUNTER_FILE="$COUNTER_DIR/codeact-$PPID.count" + +# Read discovered tool list from install-time manifest +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PLUGIN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +TOOLS_FILE="$PLUGIN_DIR/.codeact-tools.json" +if [[ -f "$TOOLS_FILE" ]]; then + INSTALLED_TOOLS=$(jq -r '[.tools[].name] | join(", ")' "$TOOLS_FILE" 2>/dev/null || echo "view, create, edit, glob, bash, sql") +else + INSTALLED_TOOLS="view, create, edit, glob, bash, sql" +fi + +DENY_REASON="CodeAct enforcement active (CODEACT_MODE=${MODE}). Collapse this work into one sandboxed Python run: + + bash plugins/codeact/scripts/codeact --code '' + +Sandbox tools: ${INSTALLED_TOOLS}. +Disable enforcement: unset CODEACT_MODE." + +deny() { + jq -n --arg reason "$DENY_REASON" '{permissionDecision: "deny", permissionDecisionReason: $reason}' + exit 0 +} + +allow() { + echo '{}' + exit 0 +} + +case "$MODE" in + nudge) + # Reset counter on codeact invocation + if is_codeact_call; then + echo "0" > "$COUNTER_FILE" 2>/dev/null || true + allow + fi + + # Read-only tools increment counter + READ_ONLY_TOOLS="view glob grep rg read_file file_search" + IS_READ_ONLY=false + for t in $READ_ONLY_TOOLS; do + if [[ "$TOOL_NAME" == "$t" ]]; then + IS_READ_ONLY=true + break + fi + done + + if [[ "$IS_READ_ONLY" == "true" ]]; then + COUNT=$(cat "$COUNTER_FILE" 2>/dev/null || echo "0") + COUNT=$((COUNT + 1)) + echo "$COUNT" > "$COUNTER_FILE" 2>/dev/null || true + + if (( COUNT >= 3 )); then + echo "0" > "$COUNTER_FILE" 2>/dev/null || true + deny + fi + else + # Non-read-only, non-codeact tool: reset counter + echo "0" > "$COUNTER_FILE" 2>/dev/null || true + fi + + allow + ;; + + exclusive) + if is_codeact_call; then + allow + fi + deny + ;; + + *) + # Unknown mode, pass through + allow + ;; +esac diff --git a/plugins/codeact/instructions/codeact.instructions.md.tmpl b/plugins/codeact/instructions/codeact.instructions.md.tmpl new file mode 100644 index 0000000..689ea82 --- /dev/null +++ b/plugins/codeact/instructions/codeact.instructions.md.tmpl @@ -0,0 +1,39 @@ +--- +applyTo: "**" +--- + +## CodeAct — sandboxed Python (backend: {{BACKEND}}) + +Use codeact **instead of** chained tool calls when a task reads ≥8 files, +cross-references file sets, or needs ≥5 sequential tool calls. + +### Invoke + +```bash +{{CODEACT_DIR}}/scripts/codeact --auto --workspace . --code '' +``` + +### Critical rules + +1. **One bash call, one program.** Write a single Python program that globs, + reads, analyzes, and prints results. Do NOT scout with view/glob before + invoking codeact. Do NOT retry with extra tool calls if the first run + has a minor issue — fix the program instead. +2. **Return types** — getting these wrong causes retries: + - `glob` → **list of strings** like `["src/app.py", ...]` + - `view` → **string** (file content) + - `mcp_call` → **string** + - `bash` → **dict** with stdout/stderr/returncode +3. **No `os.path`, no `os.walk`** — use `glob` to find files, `view` to read them. +4. **Wrap file reads in try/except.** Print partial results as you go. +5. **Use double quotes in Python code** to avoid shell quoting conflicts with + `--code '...'`. Write `"string"` not `'string'` inside codeact programs. +6. Skip codeact for ≤5 files or single grep→view→done workflows. + +{{SYNTAX}} + +{{BACKEND_LIMITATIONS}} + +### Sandbox tools: {{TOOL_LIST}} + +{{TOOL_REFERENCE}} diff --git a/plugins/codeact/plugin.json b/plugins/codeact/plugin.json new file mode 100644 index 0000000..a5be78f --- /dev/null +++ b/plugins/codeact/plugin.json @@ -0,0 +1,12 @@ +{ + "name": "codeact", + "description": "Collapse multi-step tool chains into one sandboxed Python run. Hyperlight + Monty backends.", + "version": "0.1.0", + "author": { "name": "jsturtevant" }, + "license": "MIT", + "repository": "https://github.com/jsturtevant/copilot-skills", + "keywords": ["codeact", "codemode", "sandbox", "tool-chaining", "mcp", "python", "tools"], + "agents": "agents/", + "skills": "skills/", + "hooks": "hooks.json" +} diff --git a/plugins/codeact/scripts/codeact b/plugins/codeact/scripts/codeact new file mode 100755 index 0000000..d2265fd --- /dev/null +++ b/plugins/codeact/scripts/codeact @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# codeact — thin dispatcher to backend-specific codeact.py +# Subcommands: +# --code '' Run sandboxed code +# --discover [--backend X] Emit tools JSON manifest +# --instructions [--backend X] Emit LLM-ready tool reference +# All other flags forwarded to backend codeact.py +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PLUGIN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# Extract --backend from args (if present), otherwise auto-detect +BACKEND="" +REMAINING_ARGS=() +while [[ $# -gt 0 ]]; do + case "$1" in + --backend) + BACKEND="$2" + shift 2 + ;; + --backend=*) + BACKEND="${1#--backend=}" + shift + ;; + *) + REMAINING_ARGS+=("$1") + shift + ;; + esac +done + +if [[ -z "$BACKEND" ]]; then + # Prefer the backend selected at install time over re-running detection, + # so runtime always matches the syntax baked into the instructions file. + if [[ -f "$PLUGIN_DIR/.codeact-backend" ]]; then + BACKEND=$(<"$PLUGIN_DIR/.codeact-backend") + else + BACKEND=$(bash "$SCRIPT_DIR/detect-backend.sh") + fi +fi + +BACKEND_SCRIPT="$PLUGIN_DIR/skills/${BACKEND}-codeact/scripts/codeact.py" + +if [[ ! -f "$BACKEND_SCRIPT" ]]; then + echo "Error: Backend script not found: $BACKEND_SCRIPT" >&2 + echo "Available backends:" >&2 + ls "$PLUGIN_DIR/skills/" 2>/dev/null | grep -oP '(.+)-codeact' | sed 's/-codeact//' >&2 + exit 1 +fi + +# Check if uv is available for auto-install +HAS_UV=false +if command -v uv >/dev/null 2>&1; then + HAS_UV=true +fi + +# Build uv run prefix based on backend +case "$BACKEND" in + monty) + if [[ "$HAS_UV" == "true" ]]; then + exec uv run --quiet --with pydantic-monty python3 "$BACKEND_SCRIPT" "${REMAINING_ARGS[@]}" + else + exec python3 "$BACKEND_SCRIPT" "${REMAINING_ARGS[@]}" + fi + ;; + hyperlight) + if [[ "$HAS_UV" == "true" ]]; then + PY_MINOR=$(python3 -c 'import sys; print(sys.version_info.minor)') + UV_EXTRA=() + if (( PY_MINOR > 13 )); then + UV_EXTRA=(--python 3.13) + fi + exec uv run --quiet "${UV_EXTRA[@]}" \ + --with 'hyperlight-sandbox[wasm,python_guest]>=0.3.0' \ + python3 "$BACKEND_SCRIPT" "${REMAINING_ARGS[@]}" + else + exec python3 "$BACKEND_SCRIPT" "${REMAINING_ARGS[@]}" + fi + ;; + *) + echo "Error: Unknown backend '$BACKEND'" >&2 + exit 1 + ;; +esac diff --git a/plugins/codeact/scripts/detect-backend.sh b/plugins/codeact/scripts/detect-backend.sh new file mode 100755 index 0000000..33ad173 --- /dev/null +++ b/plugins/codeact/scripts/detect-backend.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# detect-backend.sh — auto-pick codeact backend at install time. +# macOS → monty +# Linux + /dev/kvm → hyperlight +# Linux + /dev/mshv → hyperlight +# Linux (neither) → monty +# Windows + Hyper-V → hyperlight (via PowerShell check) +# Windows (no HV) → monty +set -euo pipefail + +case "$(uname -s)" in + Darwin) + echo "monty" + ;; + Linux) + if [[ -r /dev/kvm ]] || [[ -r /dev/mshv ]]; then + echo "hyperlight" + else + echo "monty" + fi + ;; + MINGW*|MSYS*|CYGWIN*) + # Check for Hyper-V via PowerShell + if powershell.exe -NoProfile -Command \ + "(Get-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V).State -eq 'Enabled'" \ + 2>/dev/null | grep -qi true; then + echo "hyperlight" + else + echo "monty" + fi + ;; + *) + echo "monty" + ;; +esac diff --git a/plugins/codeact/scripts/install-instructions.ps1 b/plugins/codeact/scripts/install-instructions.ps1 new file mode 100644 index 0000000..528a99a --- /dev/null +++ b/plugins/codeact/scripts/install-instructions.ps1 @@ -0,0 +1,101 @@ +# install-instructions.ps1 — Windows variant of install-instructions.sh +param( + [string]$Backend = "", + [switch]$Global +) + +$ErrorActionPreference = "Stop" +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$PluginDir = Split-Path -Parent $ScriptDir + +# Auto-detect backend +if (-not $Backend) { + $Backend = if ((Get-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -ErrorAction SilentlyContinue).State -eq 'Enabled') { "hyperlight" } + else { "monty" } +} + +Write-Host "Backend: $Backend" + +# Preflight +& powershell.exe -File "$ScriptDir\preflight.ps1" -Backend $Backend +if ($LASTEXITCODE -ne 0) { throw "Preflight failed for backend $Backend" } + +# Discover tools — persist manifest for hook consumption +$toolsFile = Join-Path $PluginDir ".codeact-tools.json" +python3 "$PluginDir\skills\${Backend}-codeact\scripts\codeact.py" --discover --output $toolsFile +Write-Host "Wrote: $toolsFile" + +# Persist backend choice so runtime dispatch matches install +$backendMarker = Join-Path $PluginDir ".codeact-backend" +Set-Content -Path $backendMarker -Value $Backend -NoNewline +Write-Host "Wrote: $backendMarker" + +$toolList = python3 -c "import json; d=json.load(open(r'$toolsFile')); print(', '.join(t['name'] for t in d.get('tools', [])))" + +# Generate instructions reference +$toolRef = (python3 "$PluginDir\skills\${Backend}-codeact\scripts\codeact.py" --instructions | Out-String) + +# Backend-specific syntax block +if ($Backend -eq "monty") { + $syntax = @" +Tools are called as regular Python functions with keyword arguments: +``````python +content = view(path="README.md") +files = glob(pattern="**/*.py") +hits = grep(pattern="TODO", paths="src") +result = bash(command="git log --oneline -5") +`````` +"@ +} else { + $syntax = @" +Tools are called via call_tool() with keyword arguments: +``````python +content = call_tool("view", path="README.md") +files = call_tool("glob", pattern="**/*.py") +hits = call_tool("grep", pattern="TODO", paths="src") +result = call_tool("bash", command="git log --oneline -5") +`````` +"@ +} + +# Output paths +if ($Global) { + $instrDir = "$env:USERPROFILE\.copilot" + $instrFile = "$instrDir\codeact.instructions.md" +} else { + $instrDir = ".github\instructions" + $instrFile = "$instrDir\codeact.instructions.md" +} +$agentFile = Join-Path $PluginDir "agents\codeact.agent.md" +$agentTmpl = Join-Path $PluginDir "agents\codeact.agent.md.tmpl" + +if (-not (Test-Path $agentTmpl)) { throw "Agent template not found: $agentTmpl" } + +New-Item -ItemType Directory -Path $instrDir -Force | Out-Null + +function Substitute($templatePath) { + $content = Get-Content $templatePath -Raw + $content = $content -replace '\{\{BACKEND\}\}', $Backend + $content = $content -replace '\{\{CODEACT_DIR\}\}', $PluginDir + $content = $content -replace '\{\{TOOL_LIST\}\}', $toolList + # Literal replacement for multiline blocks (no regex metacharacter expansion) + $content = $content.Replace('{{TOOL_REFERENCE}}', $toolRef) + $content = $content.Replace('{{SYNTAX}}', $syntax) + return $content +} + +# Atomic-ish write: write temp then move +function AtomicWrite($targetPath, $content) { + $tmp = "$targetPath.tmp" + [System.IO.File]::WriteAllText($tmp, $content, [System.Text.UTF8Encoding]::new($false)) + Move-Item -Path $tmp -Destination $targetPath -Force +} + +AtomicWrite $instrFile (Substitute (Join-Path $PluginDir "instructions\codeact.instructions.md.tmpl")) +Write-Host "Wrote: $instrFile" + +AtomicWrite $agentFile (Substitute $agentTmpl) +Write-Host "Wrote: $agentFile" + +Write-Host "" +Write-Host "CodeAct installed (backend=$Backend). Restart session to load." diff --git a/plugins/codeact/scripts/install-instructions.sh b/plugins/codeact/scripts/install-instructions.sh new file mode 100755 index 0000000..8e6ab42 --- /dev/null +++ b/plugins/codeact/scripts/install-instructions.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# install-instructions.sh — Install codeact instructions + agent files +# Pipeline: preflight → discover → instructions → substitute → atomic write +# +# Usage: +# install-instructions.sh [--backend ] [--global] +# +# Options: +# --backend Force backend (monty|hyperlight). Default: auto-detect. +# --global Write to $HOME/.copilot/ instead of .github/instructions/ +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +PLUGIN_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +BACKEND="" +GLOBAL=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --backend) BACKEND="$2"; shift 2 ;; + --backend=*) BACKEND="${1#--backend=}"; shift ;; + --global) GLOBAL=true; shift ;; + *) echo "Unknown arg: $1" >&2; exit 1 ;; + esac +done + +# --- 1. Auto-detect backend if not specified --- +if [[ -z "$BACKEND" ]]; then + BACKEND=$(bash "$SCRIPT_DIR/detect-backend.sh") +fi +echo "Backend: $BACKEND" >&2 + +# --- 2. Preflight --- +bash "$SCRIPT_DIR/preflight.sh" "$BACKEND" + +# --- 3. Discover tools --- +TOOLS_FILE="$PLUGIN_DIR/.codeact-tools.json" +python3 "$PLUGIN_DIR/skills/${BACKEND}-codeact/scripts/codeact.py" --discover --output "$TOOLS_FILE" +echo "Wrote: $TOOLS_FILE" >&2 + +# --- 3a. Persist backend choice so runtime dispatch matches install --- +BACKEND_MARKER="$PLUGIN_DIR/.codeact-backend" +echo "$BACKEND" > "$BACKEND_MARKER" +echo "Wrote: $BACKEND_MARKER" >&2 + +TOOL_LIST=$(jq -r '[.tools[].name] | join(", ")' "$TOOLS_FILE") +# Always include mcp_call in the list so the model knows it exists +if ! echo "$TOOL_LIST" | grep -q "mcp_call"; then + TOOL_LIST="$TOOL_LIST, mcp_call" +fi + +# --- 4. Generate instructions reference --- +TOOL_REFERENCE=$(python3 "$PLUGIN_DIR/skills/${BACKEND}-codeact/scripts/codeact.py" --instructions) + +# --- 4b. Backend-specific syntax guide --- +case "$BACKEND" in + monty) + SYNTAX="Tools are called as regular Python functions: \`view(path=\"f.py\")\`, \`glob(pattern=\"**/*.py\")\`, etc." + BACKEND_LIMITATIONS="### Monty limitations (avoid retries) + +Monty runs a Python subset. These **will error**: +- \`f\"{x:<10}\"\` or any f-string format spec → use \`+\` with manual padding +- \`\"{:<10}\".format(x)\` → no \`str.format()\` +- \`class Foo:\` → no classes +- \`match x:\` → no match/case +- \`str.startswith()\` with tuple → use \`or\` +- Set comprehensions → use \`list\` + \`in\` +- \`os.path\`, \`os.walk\` → use \`glob()\` and \`view()\` instead + +**MCP from inside sandbox:** Use \`mcp_call(server=\"name\", tool=\"tool\", ...)\` to call MCP servers. Both \`mcp_call()\` and \`web_fetch()\` work inside the sandbox." + ;; + hyperlight) + SYNTAX="Tools are called via \`call_tool(name, **kwargs)\` — no import needed." + BACKEND_LIMITATIONS="**MCP from inside sandbox:** Use \`call_tool(\"mcp_call\", server=\"name\", tool=\"tool\", ...)\` to call MCP servers. Both \`mcp_call\` and \`web_fetch\` work inside the sandbox." + ;; +esac + +# --- 5. Determine output paths --- +if [[ "$GLOBAL" == "true" ]]; then + INSTRUCTIONS_DIR="$HOME/.copilot" + INSTRUCTIONS_FILE="$INSTRUCTIONS_DIR/codeact.instructions.md" +else + INSTRUCTIONS_DIR=".github/instructions" + INSTRUCTIONS_FILE="$INSTRUCTIONS_DIR/codeact.instructions.md" +fi +AGENT_FILE="$PLUGIN_DIR/agents/codeact.agent.md" + +# --- 6. Template substitution --- +CODEACT_DIR="$PLUGIN_DIR" + +substitute() { + local template="$1" + local content + content=$(cat "$template") + content="${content//\{\{BACKEND\}\}/$BACKEND}" + content="${content//\{\{CODEACT_DIR\}\}/$CODEACT_DIR}" + content="${content//\{\{TOOL_LIST\}\}/$TOOL_LIST}" + # TOOL_REFERENCE and SYNTAX contain newlines so use python for safe substitution + echo "$content" | python3 -c " +import sys +content = sys.stdin.read() +ref = '''$TOOL_REFERENCE''' +syntax = '''$SYNTAX''' +limits = '''$BACKEND_LIMITATIONS''' +content = content.replace('{{TOOL_REFERENCE}}', ref) +content = content.replace('{{SYNTAX}}', syntax) +content = content.replace('{{BACKEND_LIMITATIONS}}', limits) +print(content, end='') +" +} + +# --- 7. Atomic write --- +mkdir -p "$INSTRUCTIONS_DIR" + +# Instructions file +TMPFILE=$(mktemp "${INSTRUCTIONS_FILE}.XXXXXX") +substitute "$PLUGIN_DIR/instructions/codeact.instructions.md.tmpl" > "$TMPFILE" +mv "$TMPFILE" "$INSTRUCTIONS_FILE" +chmod 644 "$INSTRUCTIONS_FILE" +echo "Wrote: $INSTRUCTIONS_FILE" >&2 + +# Agent file (template is .tmpl, output strips .tmpl suffix) +AGENT_TMPL="$PLUGIN_DIR/agents/codeact.agent.md.tmpl" +if [[ ! -f "$AGENT_TMPL" ]]; then + echo "Error: agent template not found: $AGENT_TMPL" >&2 + exit 1 +fi +TMPFILE=$(mktemp "${AGENT_FILE}.XXXXXX") +substitute "$AGENT_TMPL" > "$TMPFILE" +mv "$TMPFILE" "$AGENT_FILE" +chmod 644 "$AGENT_FILE" +echo "Wrote: $AGENT_FILE" >&2 + +echo "" >&2 +echo "CodeAct installed (backend=$BACKEND). Restart session to load." >&2 diff --git a/plugins/codeact/scripts/mcp-bridge.py b/plugins/codeact/scripts/mcp-bridge.py new file mode 100644 index 0000000..a44c8ff --- /dev/null +++ b/plugins/codeact/scripts/mcp-bridge.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +"""MCP bridge — call MCP server tools from the codeact sandbox. + +Supports two MCP server types: + - HTTP/SSE: POST JSON-RPC to the server URL (streamable-http transport) + - stdio: spawn the server command, send JSON-RPC over stdin/stdout + +Usage: + python3 mcp-bridge.py --config .mcp.json --server microsoft-docs \ + --tool search --args '{"query": "Azure Functions"}' + + python3 mcp-bridge.py --config .mcp.json --list-servers + python3 mcp-bridge.py --config .mcp.json --server markitdown --list-tools +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import urllib.request +import urllib.error +from pathlib import Path +from typing import Any + + +def _load_mcp_config(config_path: str | None = None) -> dict[str, Any]: + """Load MCP server configuration from well-known locations.""" + search_paths = [ + Path(".mcp.json"), + Path(".vscode/mcp.json"), + Path(".github/copilot/mcp.json"), + ] + if config_path: + search_paths.insert(0, Path(config_path)) + + for p in search_paths: + if p.is_file(): + try: + cfg = json.loads(p.read_text()) + # Normalize: accept both "servers" and "mcpServers" + servers = cfg.get("servers") or cfg.get("mcpServers") or {} + return {"servers": servers, "source": str(p)} + except Exception: + continue + return {"servers": {}, "source": None} + + +def _call_http_server(url: str, tool_name: str, arguments: dict[str, Any], + timeout: int = 30) -> str: + """Call a tool on an HTTP/SSE MCP server using streamable-http transport.""" + payload = { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": { + "name": tool_name, + "arguments": arguments, + }, + } + data = json.dumps(payload).encode("utf-8") + req = urllib.request.Request( + url, + data=data, + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + }, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + content_type = resp.headers.get("Content-Type", "") + body = resp.read().decode("utf-8") + + # Direct JSON-RPC response + if "application/json" in content_type: + result = json.loads(body) + return _extract_result(result) + + # SSE stream — parse event lines + if "text/event-stream" in content_type: + return _parse_sse_result(body) + + # Unknown content type — return raw + return body + except urllib.error.URLError as exc: + return json.dumps({"error": str(exc)}) + + +def _call_stdio_server(command: str, args: list[str], tool_name: str, + arguments: dict[str, Any], env: dict[str, str] | None = None, + timeout: int = 30) -> str: + """Call a tool on a stdio MCP server (spawn, init, call, close).""" + init_msg = { + "jsonrpc": "2.0", "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": {"name": "codeact-bridge", "version": "1.0"}, + }, + } + call_msg = { + "jsonrpc": "2.0", "id": 2, + "method": "tools/call", + "params": {"name": tool_name, "arguments": arguments}, + } + + stdin_data = ( + json.dumps(init_msg) + "\n" + + json.dumps({"jsonrpc": "2.0", "method": "notifications/initialized"}) + "\n" + + json.dumps(call_msg) + "\n" + ) + + run_env = dict(os.environ) + if env: + run_env.update(env) + + try: + proc = subprocess.run( + [command] + args, + input=stdin_data, + capture_output=True, + text=True, + timeout=timeout, + env=run_env, + ) + except FileNotFoundError: + return json.dumps({"error": f"Command not found: {command}"}) + except subprocess.TimeoutExpired: + return json.dumps({"error": f"MCP server timed out after {timeout}s"}) + + # Parse JSON-RPC responses from stdout (one per line) + for line in proc.stdout.strip().split("\n"): + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + if msg.get("id") == 2: # Our tools/call response + return _extract_result(msg) + except json.JSONDecodeError: + continue + + # Fallback: return everything + return proc.stdout or proc.stderr or json.dumps({"error": "No response from MCP server"}) + + +def _list_tools_stdio(command: str, args: list[str], + env: dict[str, str] | None = None, + timeout: int = 15) -> list[dict[str, Any]]: + """List tools from a stdio MCP server.""" + init_msg = { + "jsonrpc": "2.0", "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": {"name": "codeact-bridge", "version": "1.0"}, + }, + } + list_msg = {"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}} + stdin_data = ( + json.dumps(init_msg) + "\n" + + json.dumps({"jsonrpc": "2.0", "method": "notifications/initialized"}) + "\n" + + json.dumps(list_msg) + "\n" + ) + + run_env = dict(os.environ) + if env: + run_env.update(env) + + try: + proc = subprocess.run( + [command] + args, + input=stdin_data, + capture_output=True, + text=True, + timeout=timeout, + env=run_env, + ) + except Exception: + return [] + + for line in proc.stdout.strip().split("\n"): + try: + msg = json.loads(line.strip()) + if msg.get("id") == 2 and "result" in msg: + return msg["result"].get("tools", []) + except json.JSONDecodeError: + continue + return [] + + +def _list_tools_http(url: str, timeout: int = 15) -> list[dict[str, Any]]: + """List tools from an HTTP MCP server.""" + payload = { + "jsonrpc": "2.0", "id": 1, + "method": "tools/list", "params": {}, + } + req = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers={ + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + }, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + body = resp.read().decode("utf-8") + content_type = resp.headers.get("Content-Type", "") + if "text/event-stream" in content_type: + # Parse SSE for the result + for line in body.split("\n"): + if line.startswith("data:"): + data_str = line[5:].strip() + try: + msg = json.loads(data_str) + if "result" in msg: + return msg["result"].get("tools", []) + except json.JSONDecodeError: + continue + return [] + result = json.loads(body) + if "result" in result: + return result["result"].get("tools", []) + except Exception: + pass + return [] + + +def _extract_result(msg: dict[str, Any]) -> str: + """Extract the text content from a JSON-RPC result.""" + if "error" in msg: + return json.dumps(msg["error"]) + result = msg.get("result", {}) + # MCP tools/call returns {"content": [{"type": "text", "text": "..."}]} + content = result.get("content", []) + if isinstance(content, list): + texts = [c.get("text", "") for c in content if c.get("type") == "text"] + if texts: + return "\n".join(texts) + # Fallback + return json.dumps(result) if isinstance(result, dict) else str(result) + + +def _parse_sse_result(body: str) -> str: + """Parse SSE stream for the tools/call result.""" + for line in body.split("\n"): + if line.startswith("data:"): + data_str = line[5:].strip() + try: + msg = json.loads(data_str) + if msg.get("id") and ("result" in msg or "error" in msg): + return _extract_result(msg) + except json.JSONDecodeError: + continue + return body # Return raw if no result found + + +def call_mcp(config: dict[str, Any], server_name: str, tool_name: str, + arguments: dict[str, Any], timeout: int = 30) -> str: + """Call an MCP tool. Main entry point.""" + servers = config.get("servers", {}) + if server_name not in servers: + return json.dumps({"error": f"Unknown MCP server: {server_name}", + "available": list(servers.keys())}) + + scfg = servers[server_name] + + # HTTP/SSE server + if scfg.get("type") == "http" or "url" in scfg: + url = scfg.get("url", "") + return _call_http_server(url, tool_name, arguments, timeout=timeout) + + # stdio server + command = scfg.get("command", "") + args = scfg.get("args", []) + env = scfg.get("env") + if not command: + return json.dumps({"error": f"MCP server {server_name} has no command or url"}) + + return _call_stdio_server(command, args, tool_name, arguments, + env=env, timeout=timeout) + + +def list_mcp_tools(config: dict[str, Any], server_name: str, + timeout: int = 15) -> list[dict[str, Any]]: + """List tools available on an MCP server.""" + servers = config.get("servers", {}) + if server_name not in servers: + return [] + scfg = servers[server_name] + if scfg.get("type") == "http" or "url" in scfg: + return _list_tools_http(scfg["url"], timeout=timeout) + command = scfg.get("command", "") + args = scfg.get("args", []) + env = scfg.get("env") + if not command: + return [] + return _list_tools_stdio(command, args, env=env, timeout=timeout) + + +def main(): + ap = argparse.ArgumentParser(description="MCP bridge for codeact sandbox") + ap.add_argument("--config", help="Path to .mcp.json (default: auto-discover)") + ap.add_argument("--server", help="MCP server name") + ap.add_argument("--tool", help="Tool name to call") + ap.add_argument("--args", default="{}", help="JSON arguments for the tool") + ap.add_argument("--timeout", type=int, default=30, help="Timeout in seconds") + ap.add_argument("--list-servers", action="store_true", help="List available MCP servers") + ap.add_argument("--list-tools", action="store_true", help="List tools on a server") + args = ap.parse_args() + + config = _load_mcp_config(args.config) + + if args.list_servers: + for name, scfg in config["servers"].items(): + stype = "http" if (scfg.get("type") == "http" or "url" in scfg) else "stdio" + print(f" {name} ({stype})") + return + + if not args.server: + ap.error("--server is required") + + if args.list_tools: + tools = list_mcp_tools(config, args.server, timeout=args.timeout) + for t in tools: + desc = t.get("description", "")[:60] + print(f" {t['name']}: {desc}") + return + + if not args.tool: + ap.error("--tool is required for calling") + + try: + arguments = json.loads(args.args) + except json.JSONDecodeError as exc: + print(json.dumps({"error": f"Invalid --args JSON: {exc}"})) + sys.exit(1) + + result = call_mcp(config, args.server, args.tool, arguments, + timeout=args.timeout) + print(result) + + +if __name__ == "__main__": + main() diff --git a/plugins/codeact/scripts/preflight.ps1 b/plugins/codeact/scripts/preflight.ps1 new file mode 100644 index 0000000..0daf466 --- /dev/null +++ b/plugins/codeact/scripts/preflight.ps1 @@ -0,0 +1,61 @@ +# preflight.ps1 — Windows preflight check for codeact backends +param( + [Parameter(Mandatory=$true)] + [string]$Backend +) + +$ErrorActionPreference = "Stop" + +function Fail($msg) { + Write-Error "PREFLIGHT FAIL ($Backend): $msg" + exit 1 +} + +function Warn($msg) { + Write-Warning "PREFLIGHT WARN ($Backend): $msg" +} + +# Shared checks +try { $null = Get-Command python3 -ErrorAction Stop } catch { Fail "python3 not found" } + +$pyVer = python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" +$pyParts = $pyVer -split '\.' +$pyMajor = [int]$pyParts[0] +$pyMinor = [int]$pyParts[1] + +if ($pyMajor -lt 3 -or ($pyMajor -eq 3 -and $pyMinor -lt 10)) { + Fail "Python $pyVer too old. Need >=3.10." +} + +$hasUv = $null -ne (Get-Command uv -ErrorAction SilentlyContinue) + +switch ($Backend) { + "monty" { + if (-not $hasUv) { Warn "uv not found. Will try pip fallback." } + try { + python3 -c "import pydantic_monty" 2>$null + } catch { + if ($hasUv) { + Write-Host "pydantic-monty not installed. Will auto-install via uv." -ForegroundColor Yellow + } else { + Warn "pydantic-monty not installed. Install: pip install pydantic-monty" + } + } + } + "hyperlight" { + if ($pyMajor -gt 3 -or ($pyMajor -eq 3 -and $pyMinor -gt 13)) { + Fail "Python $pyVer too new for hyperlight Wasm. Need <=3.13." + } + $hvEnabled = (Get-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V -ErrorAction SilentlyContinue).State -eq 'Enabled' + if (-not $hvEnabled) { + Fail "Hyper-V not enabled. Hyperlight needs hardware virtualization." + } + if (-not $hasUv) { Warn "uv not found. Will try pip fallback." } + } + default { + Fail "Unknown backend: $Backend. Expected 'monty' or 'hyperlight'." + } +} + +Write-Host "Preflight OK: $Backend (Python $pyVer, uv=$hasUv)" -ForegroundColor Green +exit 0 diff --git a/plugins/codeact/scripts/preflight.sh b/plugins/codeact/scripts/preflight.sh new file mode 100755 index 0000000..f34974b --- /dev/null +++ b/plugins/codeact/scripts/preflight.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# preflight.sh — verify a codeact backend runtime is usable +# Usage: preflight.sh +# Exit 0 if usable, non-zero with diagnostic on failure. +set -euo pipefail + +BACKEND="${1:?Usage: preflight.sh }" + +fail() { echo "PREFLIGHT FAIL ($BACKEND): $*" >&2; exit 1; } +warn() { echo "PREFLIGHT WARN ($BACKEND): $*" >&2; } + +# --- shared checks --- +command -v bash >/dev/null 2>&1 || fail "bash not found" +command -v python3 >/dev/null 2>&1 || fail "python3 not found" + +PY_VERSION=$(python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")') +PY_MAJOR=$(echo "$PY_VERSION" | cut -d. -f1) +PY_MINOR=$(echo "$PY_VERSION" | cut -d. -f2) + +if (( PY_MAJOR < 3 || (PY_MAJOR == 3 && PY_MINOR < 10) )); then + fail "Python $PY_VERSION too old. Need >=3.10." +fi + +HAS_UV=false +if command -v uv >/dev/null 2>&1; then + HAS_UV=true +fi + +case "$BACKEND" in + monty) + if [[ "$HAS_UV" != "true" ]]; then + warn "uv not found. Will try pip fallback for pydantic-monty install." + fi + # Check if pydantic-monty is importable or installable + if ! python3 -c "import pydantic_monty" 2>/dev/null; then + if [[ "$HAS_UV" == "true" ]]; then + echo "pydantic-monty not installed. Will auto-install via uv run --with." >&2 + else + warn "pydantic-monty not installed. Install with: pip install pydantic-monty" + fi + fi + ;; + + hyperlight) + # macOS check + if [[ "$(uname -s)" == "Darwin" ]]; then + fail "Hyperlight not supported on macOS. Use monty backend." + fi + + # Python version ceiling + if (( PY_MAJOR > 3 || (PY_MAJOR == 3 && PY_MINOR > 13) )); then + fail "Python $PY_VERSION too new for hyperlight Wasm backend. Need <=3.13. Use: uv run --python 3.13 ..." + fi + + # KVM/mshv check on Linux + if [[ "$(uname -s)" == "Linux" ]]; then + if [[ ! -r /dev/kvm ]] && [[ ! -r /dev/mshv ]]; then + fail "Neither /dev/kvm nor /dev/mshv readable. Hyperlight needs hardware virtualization." + fi + fi + + if [[ "$HAS_UV" != "true" ]]; then + warn "uv not found. Will try pip fallback for hyperlight-sandbox install." + fi + + if ! python3 -c "from hyperlight_sandbox import Sandbox" 2>/dev/null; then + if [[ "$HAS_UV" == "true" ]]; then + echo "hyperlight-sandbox not installed. Will auto-install via uv run --with." >&2 + else + warn "hyperlight-sandbox not installed. Install with: pip install 'hyperlight-sandbox[wasm,python_guest]>=0.3.0'" + fi + fi + ;; + + *) + fail "Unknown backend: $BACKEND. Expected 'monty' or 'hyperlight'." + ;; +esac + +echo "Preflight OK: $BACKEND (Python $PY_VERSION, uv=$HAS_UV)" >&2 +exit 0 diff --git a/plugins/codeact/skills/codeact-install-hyperlight/SKILL.md b/plugins/codeact/skills/codeact-install-hyperlight/SKILL.md new file mode 100644 index 0000000..1155250 --- /dev/null +++ b/plugins/codeact/skills/codeact-install-hyperlight/SKILL.md @@ -0,0 +1,20 @@ +--- +name: codeact-install-hyperlight +description: | + Switch codeact to the Hyperlight backend (micro-VM sandbox via WebAssembly, + stronger isolation). Use when user wants to switch to hyperlight, needs + full Python support, or stronger sandbox isolation. +--- + +# codeact-install-hyperlight + +Switch codeact to the **Hyperlight** backend (micro-VM sandbox). + +```bash +bash "$SKILL_DIR/run.sh" +``` + +Runs preflight for hyperlight (requires KVM/mshv/Hyper-V), rediscovers tools, +and rewrites the instructions and agent files with backend pinned to `hyperlight`. + +**Note:** Hyperlight is not supported on macOS. Use monty on macOS. diff --git a/plugins/codeact/skills/codeact-install-hyperlight/run.sh b/plugins/codeact/skills/codeact-install-hyperlight/run.sh new file mode 100755 index 0000000..0e09252 --- /dev/null +++ b/plugins/codeact/skills/codeact-install-hyperlight/run.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# run.sh — codeact-install-hyperlight skill entry point +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +exec bash "$SCRIPT_DIR/../../scripts/install-instructions.sh" --backend hyperlight "$@" diff --git a/plugins/codeact/skills/codeact-install-monty/SKILL.md b/plugins/codeact/skills/codeact-install-monty/SKILL.md new file mode 100644 index 0000000..c8f2be1 --- /dev/null +++ b/plugins/codeact/skills/codeact-install-monty/SKILL.md @@ -0,0 +1,18 @@ +--- +name: codeact-install-monty +description: | + Switch codeact to the Monty backend (Pydantic Monty — minimal Python interpreter + in Rust, sub-microsecond startup). Use when user wants to switch to monty, + use monty backend, or needs lightweight sandboxing. +--- + +# codeact-install-monty + +Switch codeact to the **Monty** backend (Pydantic Monty). + +```bash +bash "$SKILL_DIR/run.sh" +``` + +Runs preflight for monty, rediscovers tools, and rewrites the instructions +and agent files with backend pinned to `monty`. diff --git a/plugins/codeact/skills/codeact-install-monty/run.sh b/plugins/codeact/skills/codeact-install-monty/run.sh new file mode 100755 index 0000000..90177ac --- /dev/null +++ b/plugins/codeact/skills/codeact-install-monty/run.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# run.sh — codeact-install-monty skill entry point +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +exec bash "$SCRIPT_DIR/../../scripts/install-instructions.sh" --backend monty "$@" diff --git a/plugins/codeact/skills/codeact-install/SKILL.md b/plugins/codeact/skills/codeact-install/SKILL.md new file mode 100644 index 0000000..f1ef16d --- /dev/null +++ b/plugins/codeact/skills/codeact-install/SKILL.md @@ -0,0 +1,36 @@ +--- +name: codeact-install +description: | + Install or reconfigure codeact. Auto-detects best backend (monty or hyperlight), + runs preflight checks, discovers available tools, and writes configuration files. + Use when user asks to install, configure, set up, or reconfigure codeact. + Supports --global flag for system-wide install. +--- + +# codeact-install + +Install or reconfigure the codeact plugin. Detects the best backend, +verifies it works, discovers available tools, and writes configuration. + +## Usage + +Run the install script from this skill's directory: + +```bash +bash "$SKILL_DIR/run.sh" +``` + +For global install (applies to all repos): + +```bash +bash "$SKILL_DIR/run.sh" --global +``` + +The script will: +1. Auto-detect the best backend (monty or hyperlight) +2. Run preflight checks to verify the backend works +3. Discover available tools +4. Write `.github/instructions/codeact.instructions.md` (or `$HOME/.copilot/` with --global) +5. Update the codeact agent file + +After install, restart the session to load the new configuration. diff --git a/plugins/codeact/skills/codeact-install/run.sh b/plugins/codeact/skills/codeact-install/run.sh new file mode 100755 index 0000000..5446277 --- /dev/null +++ b/plugins/codeact/skills/codeact-install/run.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# run.sh — codeact-install skill entry point +# Forwards all args to shared install-instructions.sh +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +exec bash "$SCRIPT_DIR/../../scripts/install-instructions.sh" "$@" diff --git a/plugins/codeact/skills/hyperlight-codeact/SKILL.md b/plugins/codeact/skills/hyperlight-codeact/SKILL.md new file mode 100644 index 0000000..6afd624 --- /dev/null +++ b/plugins/codeact/skills/hyperlight-codeact/SKILL.md @@ -0,0 +1,90 @@ +--- +description: | + CodeAct via Hyperlight. Use when looping over many files (8+), cross-referencing + results from multiple sources, aggregating data across directories, or + chaining 5+ dependent tool calls. Collapses N round-trips into one sandboxed + Python run via scripts/codeact.py. NOT beneficial for <5 files or simple + grep-then-view — direct tool calls have less overhead at small scale. + ESPECIALLY valuable when MCP servers are loaded — fewer turns means the + MCP tool catalog context is replayed fewer times. + Run `scripts/codeact.py --discover` to see available sandbox tools. + Trigger: "codeact", "chain tools", "sandbox", "batch", "for each", + "hyperlight sandbox", "collapse tool calls", "run in sandbox", + "sandbox execution". +name: hyperlight-codeact +--- +# Hyperlight CodeAct + +Collapse multi-step tool chains into a single sandboxed Python execution +inside an isolated Hyperlight micro-VM (WebAssembly). + +## Syntax + +Use `call_tool(name, **kwargs)` — built-in global, no import needed: + +```python +content = call_tool('view', path='src/main.py') +files = call_tool('glob', pattern='**/*.py') +hits = call_tool('grep', pattern='TODO', paths='src') +result = call_tool('bash', command='git log --oneline -5') +call_tool('edit', path='config.json', old_str='"debug": false', new_str='"debug": true') +``` + +## Return types (critical — wrong assumptions cause retries) + +- `call_tool('glob', ...)` → **list of strings** like `["src/app.py", ...]` +- `call_tool('view', ...)` → **string** (file content) +- `call_tool('bash', ...)` → **dict** with `stdout`, `stderr`, `returncode` +- `call_tool('mcp_call', ...)` → **string** + +## Pattern + +```python +for f in call_tool('glob', pattern='src/**/*.py'): + try: + content = call_tool('view', path=f) + except Exception: + continue + # analyze content... + print(f"{f}: {result}") +``` + +## Rules + +- **One bash call, one program.** Do not scout with separate tool calls first. +- **Wrap file reads in try/except.** +- `glob()` returns workspace-relative paths — pass directly to `view()`. +- Brace expansion works: `call_tool('glob', pattern='src/{db,services}/**/*.py')` + +## Discover tools + +```bash +python3 scripts/codeact.py --discover # JSON manifest +python3 scripts/codeact.py --instructions # LLM-ready reference +``` + +Tools are auto-detected based on what's installed on the host. + +## Execute + +```bash +uv run --python 3.13 --with 'hyperlight-sandbox[wasm,python_guest]>=0.3.0' \ + python3 scripts/codeact.py --auto --workspace . --code '...' +``` + +Output: `{"stdout": "...", "stderr": "...", "exit_code": 0, "success": true}` + +## Trust model + +Sandboxed code can only reach the outside world through `call_tool()` bridges. +Tools run on the host with full process access. Use `--workspace` to restrict +file tools to a directory tree. + +## Prerequisites + +- Python 3.10–3.13 (Wasm backend has no wheels for 3.14+) +- `uv` (recommended) or `pip install 'hyperlight-sandbox[wasm,python_guest]>=0.3.0'` + +## References + +- [references/tool-patterns.md](references/tool-patterns.md) diff --git a/skills/hyperlight-codeact/references/tool-patterns.md b/plugins/codeact/skills/hyperlight-codeact/references/tool-patterns.md similarity index 90% rename from skills/hyperlight-codeact/references/tool-patterns.md rename to plugins/codeact/skills/hyperlight-codeact/references/tool-patterns.md index c7920de..45fdb34 100644 --- a/skills/hyperlight-codeact/references/tool-patterns.md +++ b/plugins/codeact/skills/hyperlight-codeact/references/tool-patterns.md @@ -35,6 +35,15 @@ Sandbox tools mirror Copilot CLI built-in tools: | `web_fetch` | `curl` | Fetch URLs | | `github_api` | `gh` CLI | GitHub REST API | +## Return types (critical — wrong assumptions cause retries) + +- `call_tool('glob', ...)` → `list[str]` e.g. `["src/app.py", "src/utils.py"]` +- `call_tool('view', ...)` → `str` (full file content) +- `call_tool('bash', ...)` → `dict` with `stdout`, `stderr`, `returncode` +- `call_tool('mcp_call', ...)` → `str` +- `call_tool('web_fetch', ...)` → `str` (HTML auto-stripped to text, capped at 20K) +- Brace expansion works: `call_tool('glob', pattern='src/{db,services}/**/*.py')` + ## Chaining Patterns ### Sequential: search -> read -> analyze diff --git a/skills/hyperlight-codeact/scripts/codeact.py b/plugins/codeact/skills/hyperlight-codeact/scripts/codeact.py similarity index 72% rename from skills/hyperlight-codeact/scripts/codeact.py rename to plugins/codeact/skills/hyperlight-codeact/scripts/codeact.py index 8704d73..3d60e21 100644 --- a/skills/hyperlight-codeact/scripts/codeact.py +++ b/plugins/codeact/skills/hyperlight-codeact/scripts/codeact.py @@ -202,6 +202,28 @@ def discover_tools() -> list[dict[str, Any]]: "implementation": {"type": "builtin"}, }) + # -- mcp_call (bridge to MCP servers when .mcp.json exists) -- + mcp_cfg = _load_mcp_config() + if mcp_cfg.get("servers"): + server_names = list(mcp_cfg["servers"].keys()) + tools.append({ + "name": "mcp_call", + "cli_equivalent": "MCP servers", + "description": ( + "Call a tool on an MCP server. Available servers: " + + ", ".join(server_names) + + ". Use call_tool('mcp_call', server='name', tool='tool_name', key=val) " + "to invoke. Returns the tool result as a string." + ), + "parameters": { + "server": {"type": "string", "required": True, + "description": f"MCP server name. One of: {', '.join(server_names)}"}, + "tool": {"type": "string", "required": True, + "description": "Tool name on the MCP server."}, + }, + "implementation": {"type": "builtin"}, + }) + return tools @@ -228,6 +250,99 @@ def discover_mcp_servers() -> list[dict[str, Any]]: return servers +# --------------------------------------------------------------------------- +# User config: enable/disable + custom tool loading +# --------------------------------------------------------------------------- + +def _user_config_dir() -> Path: + """Resolve the user config directory (CODEACT_CONFIG_DIR overrides).""" + override = os.environ.get("CODEACT_CONFIG_DIR") + if override: + return Path(override) + base = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config") + return Path(base) / "codeact" + + +def _split_csv(value: str | None) -> list[str]: + if not value: + return [] + return [s.strip() for s in value.split(",") if s.strip()] + + +def _load_user_tool(py_file: Path) -> dict[str, Any] | None: + """Load a single user tool .py file. Returns a tool def, or None on error.""" + import importlib.util + name = py_file.stem + try: + spec = importlib.util.spec_from_file_location(f"codeact_user_{name}", py_file) + if spec is None or spec.loader is None: + return None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + except Exception as exc: + print(f"⚠ failed to load custom tool {py_file}: {exc}", file=sys.stderr) + return None + + meta = getattr(mod, "TOOL", {}) or {} + func_name = meta.get("function", "run") + func = getattr(mod, func_name, None) + if not callable(func): + print(f"⚠ custom tool {py_file} has no callable '{func_name}'", file=sys.stderr) + return None + + return { + "name": meta.get("name", name), + "cli_equivalent": "user", + "description": meta.get("description", (mod.__doc__ or "User tool").strip()), + "parameters": meta.get("parameters", {}), + "implementation": { + "type": "user", + "module_path": str(py_file), + "function": func_name, + }, + } + + +def apply_user_config(tools: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Filter built-in tools per allow/deny config and append custom tools.""" + cfg_dir = _user_config_dir() + cfg_file = cfg_dir / "config.json" + cfg: dict[str, Any] = {} + if cfg_file.is_file(): + try: + cfg = json.loads(cfg_file.read_text()) + except Exception as exc: + print(f"⚠ invalid {cfg_file}: {exc}", file=sys.stderr) + + enabled = set(_split_csv(os.environ.get("CODEACT_TOOLS")) or cfg.get("enabled", [])) + disabled = set(_split_csv(os.environ.get("CODEACT_DISABLE")) or cfg.get("disabled", [])) + + filtered = [] + for t in tools: + n = t["name"] + if disabled and n in disabled: + continue + if enabled and n not in enabled: + continue + filtered.append(t) + + tools_dir = cfg_dir / "tools" + if tools_dir.is_dir(): + for py in sorted(tools_dir.glob("*.py")): + if py.name.startswith("_"): + continue + tdef = _load_user_tool(py) + if tdef is None: + continue + if disabled and tdef["name"] in disabled: + continue + if enabled and tdef["name"] not in enabled: + continue + filtered.append(tdef) + + return filtered + + # --------------------------------------------------------------------------- # Built-in host-side tool handlers # --------------------------------------------------------------------------- @@ -283,7 +398,25 @@ def _edit(path: str = "", old_str: str = "", new_str: str = "") -> str: def _glob(pattern: str = "**/*", paths: str = ".") -> list[str]: base = _check_workspace(Path(paths)) - return sorted(str(p) for p in base.glob(pattern) if p.is_file())[:200] + # Support brace expansion: {a,b} → run multiple globs and merge + if "{" in pattern and "}" in pattern: + prefix = pattern[:pattern.index("{")] + rest = pattern[pattern.index("{"):] + brace_end = rest.index("}") + 1 + alternatives = rest[1:brace_end-1].split(",") + suffix = rest[brace_end:] + expanded = [prefix + alt + suffix for alt in alternatives] + all_matches: list[str] = [] + for p in expanded: + all_matches.extend(str(m) for m in base.glob(p) if m.is_file()) + matches = sorted(set(all_matches))[:200] + else: + matches = sorted(str(p) for p in base.glob(pattern) if p.is_file())[:200] + # Return workspace-relative paths + if _WORKSPACE_ROOT is not None: + root = str(_WORKSPACE_ROOT) + "/" + matches = [m[len(root):] if m.startswith(root) else m for m in matches] + return matches def _bash(command: str = "", timeout: int = 30) -> dict[str, Any]: @@ -321,15 +454,56 @@ def _grep(pattern: str = "", paths: str = ".", glob: str = "", def _web_fetch(url: str = "", method: str = "GET", headers: dict[str, str] | None = None, - data: str = "") -> str: - cmd = ["curl", "-sS", "-X", method] + data: str = "", max_length: int = 20000) -> str: + cmd = ["curl", "-sS", "-L", "-X", method] for k, v in (headers or {}).items(): cmd += ["-H", f"{k}: {v}"] if data: cmd += ["-d", data] cmd.append(url) r = subprocess.run(cmd, capture_output=True, text=True, timeout=30) - return r.stdout + body = r.stdout + # Strip HTML to plain text so the sandbox doesn't waste tokens parsing tags + if " int(max_length): + body = body[:int(max_length)] + f"\n\n[truncated at {max_length} chars]" + return body + + +def _html_to_text(html: str) -> str: + """Convert HTML to plain text using stdlib html.parser.""" + from html.parser import HTMLParser + + class _Extractor(HTMLParser): + def __init__(self): + super().__init__() + self._parts: list[str] = [] + self._skip = False + + def handle_starttag(self, tag, attrs): + if tag in ("script", "style"): + self._skip = True + + def handle_endtag(self, tag): + if tag in ("script", "style"): + self._skip = False + + def handle_data(self, data): + if not self._skip: + self._parts.append(data) + + parser = _Extractor() + try: + parser.feed(html) + except Exception: + text = re.sub(r"<[^>]*>", " ", html) + return re.sub(r"\s+", " ", text).strip() + text = " ".join(parser._parts) + text = re.sub(r"[ \t]+", " ", text) + text = re.sub(r"\n\s*\n", "\n\n", text) + return text.strip() def _github_api(endpoint: str = "", method: str = "GET", @@ -346,6 +520,43 @@ def _github_api(endpoint: str = "", method: str = "GET", return r.stdout +# --------------------------------------------------------------------------- +# MCP bridge +# --------------------------------------------------------------------------- + +_MCP_CONFIG: dict[str, Any] | None = None + + +def _load_mcp_config() -> dict[str, Any]: + """Lazy-load MCP config from the bridge module.""" + global _MCP_CONFIG + if _MCP_CONFIG is not None: + return _MCP_CONFIG + bridge_path = Path(__file__).resolve().parent.parent.parent.parent / "scripts" / "mcp-bridge.py" + if not bridge_path.is_file(): + _MCP_CONFIG = {"servers": {}} + return _MCP_CONFIG + import importlib.util + spec = importlib.util.spec_from_file_location("mcp_bridge", bridge_path) + if spec is None or spec.loader is None: + _MCP_CONFIG = {"servers": {}} + return _MCP_CONFIG + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + _MCP_CONFIG = mod._load_mcp_config() + _MCP_CONFIG["_bridge_mod"] = mod + return _MCP_CONFIG + + +def _mcp_call(server: str = "", tool: str = "", **kwargs) -> str: + """Call an MCP server tool. Available when .mcp.json defines servers.""" + config = _load_mcp_config() + bridge = config.get("_bridge_mod") + if bridge is None: + return json.dumps({"error": "MCP bridge not available"}) + return bridge.call_mcp(config, server, tool, kwargs) + + _BUILTIN_HANDLERS: dict[str, Any] = { "view": _view, "create": _create, @@ -356,6 +567,7 @@ def _github_api(endpoint: str = "", method: str = "GET", "grep": _grep, "web_fetch": _web_fetch, "github_api": _github_api, + "mcp_call": _mcp_call, } @@ -394,6 +606,21 @@ def _py(**kw: Any) -> Any: return ns.get("result") return _py + if impl_type == "user": + import importlib.util + path = Path(impl["module_path"]) + func_name = impl.get("function", "run") + spec = importlib.util.spec_from_file_location( + f"codeact_user_{path.stem}", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"could not load user tool {path}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + fn = getattr(mod, func_name) + def _user(**kw: Any) -> Any: + return fn(**kw) + return _user + raise ValueError(f"Unknown implementation type: {impl_type}") @@ -402,50 +629,27 @@ def _py(**kw: Any) -> Any: # --------------------------------------------------------------------------- def build_instructions(tools: list[dict[str, Any]]) -> str: - """Generate a call_tool() reference block for LLM system prompts.""" - lines = [ - "## Sandbox Tool Reference", - "", - "Inside the sandbox, use `call_tool(name, **kwargs)` to invoke host tools.", - "It is a built-in global — no import needed.", - "All arguments must be keyword arguments.", - "", - "Tool names match Copilot CLI built-in tools.", - "", - ] + """Generate compact call_tool() reference for LLM prompts.""" + lines = ["### Sandbox tools (use `call_tool(name, **kwargs)` — no import needed)"] + lines.append("") for t in tools: sig_parts = [] for pname, pdef in t.get("parameters", {}).items(): if pdef.get("required"): - sig_parts.append(f"{pname}=<{pdef['type']}>") + sig_parts.append(f"{pname}=...") else: sig_parts.append(f"{pname}={pdef.get('default', '...')!r}") sig = ", ".join(sig_parts) - cli_eq = t.get("cli_equivalent", "") - label = f" (≈ CLI {cli_eq})" if cli_eq else "" - lines.append(f"### `call_tool(\"{t['name']}\", {sig})`{label}") - lines.append(f"{t.get('description', '')}") - lines.append("") - + desc = t.get("description", "").split(".")[0] + lines.append(f"- `call_tool(\"{t['name']}\", {sig})` — {desc}") + # Always document mcp_call even if no .mcp.json at install time tool_names = {t["name"] for t in tools} - lines.append("### Chaining example") - lines.append("```python") - if "grep" in tool_names and "view" in tool_names: - lines.append("# Find TODOs, then read the first matching file") - lines.append("hits = call_tool('grep', pattern='TODO', paths='src', glob='*.py')") - lines.append("first_file = hits.strip().split('\\n')[0].split(':')[0]") - lines.append("content = call_tool('view', path=first_file)") - lines.append("print(content[:200])") - elif "glob" in tool_names and "view" in tool_names: - lines.append("# List Python files, then read the first one") - lines.append("files = call_tool('glob', pattern='**/*.py')") - lines.append("if files:") - lines.append(" content = call_tool('view', path=files[0])") - lines.append(" print(content[:200])") - else: - lines.append("result1 = call_tool('view', path='README.md')") - lines.append("print(result1[:200])") - lines.append("```") + if "mcp_call" not in tool_names: + lines.append('- `call_tool("mcp_call", server=..., tool=..., **kwargs)` — Call an MCP server tool (available when .mcp.json is configured)') + lines.append("") + lines.append("Return types: `glob`→**list of strings**, `view`→**string**, " + "`bash`→**dict** (stdout/stderr/returncode), `mcp_call`→**string**") + lines.append("") return "\n".join(lines) # --------------------------------------------------------------------------- @@ -604,7 +808,7 @@ def main() -> None: # ---- discovery / instructions ---- if args.discover or args.instructions: - tools = discover_tools() + tools = apply_user_config(discover_tools()) mcp = discover_mcp_servers() if args.instructions: print(build_instructions(tools)) @@ -633,7 +837,7 @@ def main() -> None: elif args.manifest: config = json.loads(Path(args.manifest).read_text()) elif args.auto: - config["tools"] = discover_tools() + config["tools"] = apply_user_config(discover_tools()) code = args.code if args.code_file: diff --git a/plugins/codeact/skills/monty-codeact/SKILL.md b/plugins/codeact/skills/monty-codeact/SKILL.md new file mode 100644 index 0000000..d588b29 --- /dev/null +++ b/plugins/codeact/skills/monty-codeact/SKILL.md @@ -0,0 +1,109 @@ +--- +description: | + CodeAct via Monty. Use when looping over many files (8+), cross-referencing + results from multiple sources, aggregating data across directories, or + chaining 5+ dependent tool calls. Collapses N round-trips into one sandboxed + Python run via scripts/codeact.py. NOT beneficial for <5 files or simple + grep-then-view — direct tool calls have less overhead at small scale. + ESPECIALLY valuable when MCP servers are loaded — fewer turns means the + MCP tool catalog context is replayed fewer times. + Run `scripts/codeact.py --discover` to see available sandbox tools. + Sub-microsecond startup. + Trigger: "codeact", "chain tools", "sandbox", "batch", "for each", + "monty codeact", "monty sandbox", "run in monty", "pydantic monty". +name: monty-codeact +--- +# Monty CodeAct + +Collapse multi-step tool chains into a single sandboxed Python execution using +[Pydantic Monty](https://github.com/pydantic/monty) — a minimal, secure Python +interpreter written in Rust with sub-microsecond startup. + +## Syntax + +Tools are called as regular Python functions with keyword arguments: + +```python +content = view(path="src/main.py") +files = glob(pattern="**/*.py") +hits = grep(pattern="TODO", paths="src") +result = bash(command="git log --oneline -5") +edit(path="config.json", old_str='"debug": false', new_str='"debug": true') +``` + +Chain by sequencing: + +```python +for f in glob(pattern="**/*.py", paths="src"): + content = view(path=f) + if "TODO" in content: + print(f + ": " + str(content.count("TODO")) + " TODOs") +``` + +## Discover tools + +```bash +python3 scripts/codeact.py --discover # JSON manifest +python3 scripts/codeact.py --instructions # LLM-ready reference +``` + +Tools are auto-detected based on what's installed on the host. + +## Execute + +```bash +uv run --with pydantic-monty python3 scripts/codeact.py --auto --workspace . --code '...' +``` + +Output: `{"stdout": "...", "stderr": "...", "return_value": null, "success": true}` + +## Monty limitations + +Monty runs a subset of Python. **Will error on:** +- **Classes** — no `class` keyword at all +- **Match statements** — no `match`/`case` +- **f-string format specs** — `f"{x:<10}"`, `f"{x:>5}"`, `f"{x:.2f}"` all fail +- **`str.format()`** — `"{:<10}".format(x)` fails +- **`str.startswith()` with tuple** — use `or` instead +- **Set comprehensions** — build with list + `in` checks +- **Third-party imports** — only stdlib subset +- **Most stdlib** — only: json, re, datetime, sys, os.environ (no os.path, no os.walk) +- **Brace expansion in glob** — `glob(pattern="src/{db,services}/**/*.py")` fails. Use two separate `glob()` calls. + +**Sandbox tool return types** (getting these wrong causes retries): +- `glob(pattern=...)` → **list of strings** like `["src/app.py", "src/utils.py"]` +- `view(path=...)` → **string** (file content) +- `mcp_call(server=..., tool=..., ...)` → **string** +- `bash(command=...)` → **dict** with keys `stdout`, `stderr`, `returncode` + +**Key usage patterns:** +- **Do NOT scout first.** `glob()` and `view()` are in the sandbox — discover + files inside your codeact program, not with separate tool calls before it. +- **One program, one bash call.** Do not run multiple codeact invocations. + If the first one fails, fix the bug in the program, don't add a scouting step. +- **Wrap file reads in try/except** so one bad file doesn't abort the run. +- Use `for f in glob(pattern="**/*.py"):` to iterate files. No os.walk or os.path. + +**Output formatting workaround** (use instead of format specs): +```python +def pad(s, w): + s = str(s) + return s + " " * max(0, w - len(s)) +``` + +**Tips:** Use `chr(10)` for newlines. Use `import json` explicitly. +Use string concatenation (`+`) or simple f-strings (`f"count: {n}"`). + +## Trust model + +Sandboxed code can only reach the outside world through registered tool +functions. Use `--workspace` to restrict file tools to a directory tree. + +## Prerequisites + +- Python 3.10+ +- `uv` (recommended) or `pip install pydantic-monty` + +## References + +- [references/tool-patterns.md](references/tool-patterns.md) diff --git a/skills/monty-codeact/references/tool-patterns.md b/plugins/codeact/skills/monty-codeact/references/tool-patterns.md similarity index 59% rename from skills/monty-codeact/references/tool-patterns.md rename to plugins/codeact/skills/monty-codeact/references/tool-patterns.md index ed4277b..507d313 100644 --- a/skills/monty-codeact/references/tool-patterns.md +++ b/plugins/codeact/skills/monty-codeact/references/tool-patterns.md @@ -50,22 +50,67 @@ files = call_tool("glob", pattern="**/*.py") ## Monty-specific notes - Use `chr(10)` for newline character (backslash escapes in some contexts differ) -- Use string concatenation `+` or f-strings: `f"count: {n}"` +- Use string concatenation `+` or simple f-strings: `f"count: {n}"` +- **No f-string format specs** — `f"{x:<10}"`, `f"{x:>5}"`, `f"{x:.2f}"` all error +- **No `str.format()`** — `"{:<10}".format(x)` errors +- **No `os.path` or `os.walk`** — use `glob()` to find files, `view()` to read them +- For tabular output, use manual padding: + ```python + def pad(s, w): + s = str(s) + return s + " " * max(0, w - len(s)) + ``` - No classes, no match statements, no third-party imports -- Supported stdlib: `json`, `re`, `datetime`, `sys`, `os`, `typing`, `asyncio` +- Supported stdlib: `json`, `re`, `datetime`, `sys`, `os.environ` (no os.path), `typing`, `asyncio` - Sub-microsecond startup vs ~680ms for Hyperlight +### Return types (critical — wrong assumptions cause retries) +- `glob(pattern=...)` → `list[str]` e.g. `["src/app.py", "src/utils.py"]` +- `view(path=...)` → `str` (full file content) +- `bash(command=...)` → `dict` with `stdout`, `stderr`, `returncode` +- `mcp_call(server=..., tool=..., ...)` → `str` + ## Chaining Patterns ### Sequential: search -> read -> analyze ```python +# Do everything in one program — no scouting needed for f in glob(pattern="**/*.py", paths="src"): - content = view(path=f) + try: + content = view(path=f) + except Exception as e: + print(f + ": ERROR - " + str(e)) + continue lines = content.split(chr(10)) todos = [l for l in lines if "TODO" in l] if todos: - print(f + ": " + str(len(todos)) + " TODOs") + clean = f.replace("./", "") + print(clean + ": " + str(len(todos)) + " TODOs") +``` + +### Cross-file import analysis + +```python +import re +files = glob(pattern="src/**/*.py") +deps = {} +for f in files: + clean = f.replace("./", "") + try: + content = view(path=f) + except Exception: + continue + imports = [] + for line in content.split(chr(10)): + if line.startswith("from src.") or line.startswith("import src."): + m = re.match(r"(?:from|import)\s+(src\.\S+)", line) + if m: + imports.append(m.group(1)) + if imports: + deps[clean] = imports +for mod, imps in deps.items(): + print(mod + " -> " + ", ".join(imps)) ``` ### Fan-out / fan-in @@ -107,6 +152,26 @@ for f in files: print(f + ": ERROR - " + str(e)) ``` +### MCP server calls + +When `.mcp.json` is present, `mcp_call` bridges to MCP servers: + +```python +# Search Microsoft docs +result = mcp_call(server="microsoft-docs", tool="microsoft_docs_search", + query="Azure Functions") +print(result[:200]) + +# Chain: search docs then fetch a page +import json as _json +hits = _json.loads(mcp_call(server="microsoft-docs", + tool="microsoft_docs_search", + query="Azure Functions")) +url = hits["results"][0]["url"] +page = mcp_call(server="microsoft-docs", tool="microsoft_docs_fetch", url=url) +print(page[:500]) +``` + ## Custom Tool Definitions Add to the manifest JSON: diff --git a/skills/monty-codeact/scripts/codeact.py b/plugins/codeact/skills/monty-codeact/scripts/codeact.py similarity index 67% rename from skills/monty-codeact/scripts/codeact.py rename to plugins/codeact/skills/monty-codeact/scripts/codeact.py index f711b8c..553d170 100644 --- a/skills/monty-codeact/scripts/codeact.py +++ b/plugins/codeact/skills/monty-codeact/scripts/codeact.py @@ -148,12 +148,14 @@ def discover_tools() -> list[dict[str, Any]]: tools.append({ "name": "web_fetch", "cli_equivalent": "web_fetch", - "description": "Fetch a URL and return its content.", + "description": "Fetch a URL. HTML is auto-converted to plain text " + "and capped at max_length chars.", "parameters": { "url": {"type": "string", "required": True}, "method": {"type": "string", "required": False, "default": "GET"}, "headers": {"type": "object", "required": False}, "data": {"type": "string", "required": False}, + "max_length": {"type": "number", "required": False, "default": 20000}, }, "implementation": {"type": "builtin"}, }) @@ -171,6 +173,28 @@ def discover_tools() -> list[dict[str, Any]]: "implementation": {"type": "builtin"}, }) + # -- mcp_call (bridge to MCP servers when .mcp.json exists) -- + mcp_cfg = _load_mcp_config() + if mcp_cfg.get("servers"): + server_names = list(mcp_cfg["servers"].keys()) + tools.append({ + "name": "mcp_call", + "cli_equivalent": "MCP servers", + "description": ( + "Call a tool on an MCP server. Available servers: " + + ", ".join(server_names) + + ". Use mcp_call(server='name', tool='tool_name', key=val, ...) " + "to invoke. Returns the tool result as a string." + ), + "parameters": { + "server": {"type": "string", "required": True, + "description": f"MCP server name. One of: {', '.join(server_names)}"}, + "tool": {"type": "string", "required": True, + "description": "Tool name on the MCP server."}, + }, + "implementation": {"type": "builtin"}, + }) + return tools @@ -197,6 +221,99 @@ def discover_mcp_servers() -> list[dict[str, Any]]: return servers +# --------------------------------------------------------------------------- +# User config: enable/disable + custom tool loading +# --------------------------------------------------------------------------- + +def _user_config_dir() -> Path: + """Resolve the user config directory (CODEACT_CONFIG_DIR overrides).""" + override = os.environ.get("CODEACT_CONFIG_DIR") + if override: + return Path(override) + base = os.environ.get("XDG_CONFIG_HOME") or str(Path.home() / ".config") + return Path(base) / "codeact" + + +def _split_csv(value: str | None) -> list[str]: + if not value: + return [] + return [s.strip() for s in value.split(",") if s.strip()] + + +def _load_user_tool(py_file: Path) -> dict[str, Any] | None: + """Load a single user tool .py file. Returns a tool def, or None on error.""" + import importlib.util + name = py_file.stem + try: + spec = importlib.util.spec_from_file_location(f"codeact_user_{name}", py_file) + if spec is None or spec.loader is None: + return None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + except Exception as exc: + print(f"⚠ failed to load custom tool {py_file}: {exc}", file=sys.stderr) + return None + + meta = getattr(mod, "TOOL", {}) or {} + func_name = meta.get("function", "run") + func = getattr(mod, func_name, None) + if not callable(func): + print(f"⚠ custom tool {py_file} has no callable '{func_name}'", file=sys.stderr) + return None + + return { + "name": meta.get("name", name), + "cli_equivalent": "user", + "description": meta.get("description", (mod.__doc__ or "User tool").strip()), + "parameters": meta.get("parameters", {}), + "implementation": { + "type": "user", + "module_path": str(py_file), + "function": func_name, + }, + } + + +def apply_user_config(tools: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Filter built-in tools per allow/deny config and append custom tools.""" + cfg_dir = _user_config_dir() + cfg_file = cfg_dir / "config.json" + cfg: dict[str, Any] = {} + if cfg_file.is_file(): + try: + cfg = json.loads(cfg_file.read_text()) + except Exception as exc: + print(f"⚠ invalid {cfg_file}: {exc}", file=sys.stderr) + + enabled = set(_split_csv(os.environ.get("CODEACT_TOOLS")) or cfg.get("enabled", [])) + disabled = set(_split_csv(os.environ.get("CODEACT_DISABLE")) or cfg.get("disabled", [])) + + filtered = [] + for t in tools: + n = t["name"] + if disabled and n in disabled: + continue + if enabled and n not in enabled: + continue + filtered.append(t) + + tools_dir = cfg_dir / "tools" + if tools_dir.is_dir(): + for py in sorted(tools_dir.glob("*.py")): + if py.name.startswith("_"): + continue + tdef = _load_user_tool(py) + if tdef is None: + continue + if disabled and tdef["name"] in disabled: + continue + if enabled and tdef["name"] not in enabled: + continue + filtered.append(tdef) + + return filtered + + # --------------------------------------------------------------------------- # Built-in host-side tool handlers # --------------------------------------------------------------------------- @@ -251,7 +368,26 @@ def _edit(path="", old_str="", new_str=""): def _glob(pattern="**/*", paths="."): base = _check_workspace(Path(paths)) - return sorted(str(p) for p in base.glob(pattern) if p.is_file())[:200] + # Support brace expansion: {a,b} → run multiple globs and merge + if "{" in pattern and "}" in pattern: + import itertools + prefix = pattern[:pattern.index("{")] + rest = pattern[pattern.index("{"):] + brace_end = rest.index("}") + 1 + alternatives = rest[1:brace_end-1].split(",") + suffix = rest[brace_end:] + expanded = [prefix + alt + suffix for alt in alternatives] + all_matches: list[str] = [] + for p in expanded: + all_matches.extend(str(m) for m in base.glob(p) if m.is_file()) + matches = sorted(set(all_matches))[:200] + else: + matches = sorted(str(p) for p in base.glob(pattern) if p.is_file())[:200] + # Return workspace-relative paths so sandbox code doesn't need to strip prefixes + if _WORKSPACE_ROOT is not None: + root = str(_WORKSPACE_ROOT) + "/" + matches = [m[len(root):] if m.startswith(root) else m for m in matches] + return matches def _bash(command="", timeout=30): @@ -285,15 +421,57 @@ def _grep(pattern="", paths=".", glob="", context_lines=0): return r.stdout -def _web_fetch(url="", method="GET", headers=None, data=""): - cmd = ["curl", "-sS", "-X", method] +def _web_fetch(url="", method="GET", headers=None, data="", max_length=20000): + cmd = ["curl", "-sS", "-L", "-X", method] for k, v in (headers or {}).items(): cmd += ["-H", f"{k}: {v}"] if data: cmd += ["-d", data] cmd.append(url) r = subprocess.run(cmd, capture_output=True, text=True, timeout=30) - return r.stdout + body = r.stdout + # Strip HTML to plain text so the sandbox doesn't waste tokens parsing tags + if " int(max_length): + body = body[:int(max_length)] + f"\n\n[truncated at {max_length} chars]" + return body + + +def _html_to_text(html: str) -> str: + """Convert HTML to plain text using stdlib html.parser.""" + from html.parser import HTMLParser + + class _Extractor(HTMLParser): + def __init__(self): + super().__init__() + self._parts: list[str] = [] + self._skip = False + + def handle_starttag(self, tag, attrs): + if tag in ("script", "style"): + self._skip = True + + def handle_endtag(self, tag): + if tag in ("script", "style"): + self._skip = False + + def handle_data(self, data): + if not self._skip: + self._parts.append(data) + + parser = _Extractor() + try: + parser.feed(html) + except Exception: + # Fallback: brute-force strip if parser chokes on malformed HTML + text = re.sub(r"<[^>]*>", " ", html) + return re.sub(r"\s+", " ", text).strip() + text = " ".join(parser._parts) + text = re.sub(r"[ \t]+", " ", text) + text = re.sub(r"\n\s*\n", "\n\n", text) + return text.strip() def _github_api(endpoint="", method="GET", body=""): @@ -309,6 +487,44 @@ def _github_api(endpoint="", method="GET", body=""): return r.stdout +# --------------------------------------------------------------------------- +# MCP bridge +# --------------------------------------------------------------------------- + +_MCP_CONFIG: dict[str, Any] | None = None + + +def _load_mcp_config() -> dict[str, Any]: + """Lazy-load MCP config from the bridge module.""" + global _MCP_CONFIG + if _MCP_CONFIG is not None: + return _MCP_CONFIG + bridge_path = Path(__file__).resolve().parent.parent.parent.parent / "scripts" / "mcp-bridge.py" + if not bridge_path.is_file(): + _MCP_CONFIG = {"servers": {}} + return _MCP_CONFIG + import importlib.util + spec = importlib.util.spec_from_file_location("mcp_bridge", bridge_path) + if spec is None or spec.loader is None: + _MCP_CONFIG = {"servers": {}} + return _MCP_CONFIG + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + _MCP_CONFIG = mod._load_mcp_config() + # Stash the module for call_mcp + _MCP_CONFIG["_bridge_mod"] = mod + return _MCP_CONFIG + + +def _mcp_call(server="", tool="", **kwargs): + """Call an MCP server tool. Available when .mcp.json defines servers.""" + config = _load_mcp_config() + bridge = config.get("_bridge_mod") + if bridge is None: + return json.dumps({"error": "MCP bridge not available"}) + return bridge.call_mcp(config, server, tool, kwargs) + + _BUILTIN_HANDLERS: dict[str, Any] = { "view": _view, "create": _create, @@ -319,6 +535,7 @@ def _github_api(endpoint="", method="GET", body=""): "grep": _grep, "web_fetch": _web_fetch, "github_api": _github_api, + "mcp_call": _mcp_call, } @@ -356,6 +573,21 @@ def _py(**kw): return ns.get("result") return _py + if impl_type == "user": + import importlib.util + path = Path(impl["module_path"]) + func_name = impl.get("function", "run") + spec = importlib.util.spec_from_file_location( + f"codeact_user_{path.stem}", path) + if spec is None or spec.loader is None: + raise RuntimeError(f"could not load user tool {path}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + fn = getattr(mod, func_name) + def _user(**kw): + return fn(**kw) + return _user + raise ValueError(f"Unknown implementation type: {impl_type}") @@ -364,44 +596,24 @@ def _py(**kw): # --------------------------------------------------------------------------- def build_instructions(tools: list[dict[str, Any]]) -> str: - """Generate tool reference for LLM prompts.""" - lines = [ - "## Sandbox Tool Reference (Monty)", - "", - "Inside the sandbox, call tools directly as Python functions.", - "No call_tool() wrapper needed — just call them by name.", - "All arguments must be keyword arguments.", - "", - "Tool names match Copilot CLI built-in tools.", - "", - ] + """Generate compact tool reference for LLM prompts.""" + lines = ["### Sandbox tools (call as Python functions, keyword args only)"] + lines.append("") for t in tools: sig_parts = [] for pname, pdef in t.get("parameters", {}).items(): if pdef.get("required"): - sig_parts.append(f"{pname}=<{pdef['type']}>") + sig_parts.append(f"{pname}=...") else: sig_parts.append(f"{pname}={pdef.get('default', '...')!r}") sig = ", ".join(sig_parts) - cli_eq = t.get("cli_equivalent", "") - label = f" (= CLI {cli_eq})" if cli_eq else "" - lines.append(f"### `{t['name']}({sig})`{label}") - lines.append(f"{t.get('description', '')}") - lines.append("") - + desc = t.get("description", "").split(".")[0] # first sentence only + lines.append(f"- `{t['name']}({sig})` — {desc}") + # Always document mcp_call even if no .mcp.json at install time tool_names = {t["name"] for t in tools} - lines.append("### Chaining example") - lines.append("```python") - if "glob" in tool_names and "view" in tool_names: - lines.append("# List Python files, read the first one") - lines.append('files = glob(pattern="**/*.py")') - lines.append("if files:") - lines.append(" content = view(path=files[0])") - lines.append(" print(content[:200])") - else: - lines.append('content = view(path="README.md")') - lines.append("print(content[:200])") - lines.append("```") + if "mcp_call" not in tool_names: + lines.append('- `mcp_call(server=..., tool=..., **kwargs)` — Call an MCP server tool (available when .mcp.json is configured)') + lines.append("") return "\n".join(lines) @@ -498,7 +710,7 @@ def main() -> None: # ---- discovery / instructions ---- if args.discover or args.instructions: - tools = discover_tools() + tools = apply_user_config(discover_tools()) mcp = discover_mcp_servers() if args.instructions: print(build_instructions(tools)) @@ -532,7 +744,7 @@ def main() -> None: elif args.manifest: config = json.loads(Path(args.manifest).read_text()) elif args.auto: - config["tools"] = discover_tools() + config["tools"] = apply_user_config(discover_tools()) code = args.code if args.code_file: diff --git a/plugins/codeact/tests/compare_results.py b/plugins/codeact/tests/compare_results.py new file mode 100644 index 0000000..0b504e5 --- /dev/null +++ b/plugins/codeact/tests/compare_results.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "matplotlib>=3.8", +# "numpy>=1.26", +# ] +# /// +"""compare_results.py — Compare two perf-results JSON files from run_tests.py. + +Usage: + # Auto-compare: latest vs previous (no args needed) + uv run compare_results.py + + # Explicit files + uv run compare_results.py [--out plot.png] + + # With labels + uv run compare_results.py before.json after.json --a-label before --b-label after + +With no arguments, auto-discovers the two most recent perf-results-*.json files +in plugins/codeact/tests/results/ and compares them (older = A, newer = B). + +Both files are produced by `run_tests.py perf` (or `all`) and live in +plugins/codeact/tests/results/. Prints a side-by-side delta table and, if +matplotlib is available, writes a grouped bar chart to the path given by +--out (default: tests/results/compare-.png). + +Renders ASCII-only output if matplotlib isn't installed. +""" +from __future__ import annotations + +import argparse +import json +import sys +from datetime import datetime, timezone +from pathlib import Path + + +def load(path: Path) -> dict: + """Load a perf-results file. Tolerates the old (bare list) format.""" + raw = json.loads(path.read_text()) + if isinstance(raw, list): + return {"timestamp": "(unknown)", "results": raw} + return raw + + +def index_by_id(results: list[dict]) -> dict[str, dict]: + return {r["prompt_id"]: r for r in results} + + +def fmt_pct(v: float) -> str: + return f"{v:+.1f}%" + + +def print_table(a_label: str, b_label: str, a: dict, b: dict) -> list[dict]: + """Print delta table and return per-prompt comparison records.""" + a_idx = index_by_id(a["results"]) + b_idx = index_by_id(b["results"]) + common = sorted(set(a_idx) & set(b_idx)) + a_only = sorted(set(a_idx) - set(b_idx)) + b_only = sorted(set(b_idx) - set(a_idx)) + + print(f"\n{a_label}: {a.get('timestamp', '?')} ({len(a['results'])} prompts)") + print(f"{b_label}: {b.get('timestamp', '?')} ({len(b['results'])} prompts)") + if a_only: + print(f" only in {a_label}: {', '.join(a_only)}") + if b_only: + print(f" only in {b_label}: {', '.join(b_only)}") + + if not common: + print("\nNo overlapping prompt IDs to compare.") + return [] + + metrics = [ + ("token_reduction_pct", "tok red %"), + ("context_reduction_pct", "ctx red %"), + ("cost_reduction_pct", "cost red %"), + ("tool_reduction_pct", "tool red %"), + ("turn_reduction_pct", "turn red %"), + ("request_reduction_pct", "req red %"), + ("codeact_tokens", "codeact tok"), + ("codeact_tool_calls", "codeact tools"), + ("codeact_turns", "codeact turns"), + ] + header = f"{'prompt':<20} " + " ".join(f"{lbl:>14}" for _, lbl in metrics) * 1 + print("\n" + "=" * (20 + 16 * len(metrics) * 2)) + print(f"{'prompt':<20} " + " ".join(f"{lbl + ' (A)':>14} {lbl + ' (B)':>14} {'Δ':>10}" for _, lbl in metrics)) + print("-" * (20 + 42 * len(metrics))) + + records = [] + for pid in common: + ra, rb = a_idx[pid], b_idx[pid] + row = [f"{pid:<20}"] + rec = {"prompt_id": pid} + for key, _ in metrics: + va = ra.get(key, 0) or 0 + vb = rb.get(key, 0) or 0 + delta = vb - va + rec[f"{key}_a"] = va + rec[f"{key}_b"] = vb + rec[f"{key}_delta"] = delta + if isinstance(va, float) or isinstance(vb, float) or "pct" in key: + row.append(f"{va:>14.1f} {vb:>14.1f} {delta:>+10.1f}") + else: + row.append(f"{va:>14d} {vb:>14d} {delta:>+10d}") + print(" ".join(row)) + records.append(rec) + print("=" * (20 + 42 * len(metrics))) + return records + + +def plot(records: list[dict], a_label: str, b_label: str, out_path: Path) -> bool: + try: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except ImportError: + print("\nmatplotlib not installed — skipping plot. " + "Install with: pip install matplotlib", file=sys.stderr) + return False + + if not records: + return False + + pids = [r["prompt_id"] for r in records] + metrics = [ + ("token_reduction_pct", "Output token reduction %"), + ("cost_reduction_pct", "Estimated cost reduction %"), + ("tool_reduction_pct", "Tool-call reduction %"), + ("turn_reduction_pct", "API turn reduction %"), + ] + + fig, axes = plt.subplots(len(metrics), 1, figsize=(max(8, len(pids) * 1.2), 9), sharex=True) + if len(metrics) == 1: + axes = [axes] + + width = 0.38 + import numpy as np + x = np.arange(len(pids)) + + for ax, (key, title) in zip(axes, metrics): + a_vals = [r[f"{key}_a"] for r in records] + b_vals = [r[f"{key}_b"] for r in records] + ax.bar(x - width / 2, a_vals, width, label=a_label) + ax.bar(x + width / 2, b_vals, width, label=b_label) + ax.set_ylabel(title) + ax.axhline(0, color="black", linewidth=0.5) + ax.legend(loc="best") + ax.grid(True, axis="y", alpha=0.3) + + axes[-1].set_xticks(x) + axes[-1].set_xticklabels(pids, rotation=30, ha="right") + fig.suptitle(f"CodeAct perf comparison: {a_label} vs {b_label}") + fig.tight_layout() + out_path.parent.mkdir(parents=True, exist_ok=True) + fig.savefig(out_path, dpi=120) + print(f"\nPlot written to {out_path}") + return True + + +def _find_recent_results(n: int = 2) -> list[Path]: + """Find the N most recent perf-results-*.json files by filename timestamp.""" + results_dir = Path(__file__).parent / "results" + files = sorted(results_dir.glob("perf-results-2*.json"), reverse=True) + return files[:n] + + +def main() -> None: + ap = argparse.ArgumentParser( + description="Compare two perf-results files. " + "With no args, auto-compares the two most recent results.") + ap.add_argument("baseline", nargs="?", type=Path, default=None, + help="Older / reference results JSON (arm A). " + "Omit to auto-discover.") + ap.add_argument("candidate", nargs="?", type=Path, default=None, + help="Newer / candidate results JSON (arm B). " + "Omit to auto-discover.") + ap.add_argument("--out", type=Path, default=None, + help="Plot output path (default: tests/results/compare-.png)") + ap.add_argument("--a-label", default=None, help="Label for baseline arm") + ap.add_argument("--b-label", default=None, help="Label for candidate arm") + args = ap.parse_args() + + # Auto-discover if no files specified + if args.baseline is None and args.candidate is None: + recent = _find_recent_results(2) + if len(recent) < 2: + ap.error("Need at least 2 perf-results-*.json files in " + "tests/results/ for auto-compare. Run `run_tests.py perf` " + "at least twice, or specify files explicitly.") + args.candidate = recent[0] # newest + args.baseline = recent[1] # previous + print(f"Auto-discovered:") + print(f" previous: {args.baseline.name}") + print(f" latest: {args.candidate.name}") + elif args.baseline is not None and args.candidate is None: + # One file given — compare it against latest + recent = _find_recent_results(1) + if not recent: + ap.error("No perf-results-*.json found for auto-compare.") + args.candidate = recent[0] + print(f"Comparing against latest: {args.candidate.name}") + + # Default labels from filenames + if args.a_label is None: + args.a_label = args.baseline.stem.replace("perf-results-", "") + if args.b_label is None: + args.b_label = args.candidate.stem.replace("perf-results-", "") + + for p in (args.baseline, args.candidate): + if not p.is_file(): + ap.error(f"file not found: {p}") + + a = load(args.baseline) + b = load(args.candidate) + records = print_table(args.a_label, args.b_label, a, b) + + if not records: + sys.exit(0) + + out = args.out + if out is None: + ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + out = Path(__file__).parent / "results" / f"compare-{ts}.png" + plot(records, args.a_label, args.b_label, out) + + +if __name__ == "__main__": + main() diff --git a/plugins/codeact/tests/fixtures/setup-workspace.sh b/plugins/codeact/tests/fixtures/setup-workspace.sh new file mode 100755 index 0000000..27670e5 --- /dev/null +++ b/plugins/codeact/tests/fixtures/setup-workspace.sh @@ -0,0 +1,1083 @@ +#!/usr/bin/env bash +# setup-workspace.sh — Create a temp workspace for codeact tests +# Outputs the workspace path on stdout +set -euo pipefail + +WORKSPACE=$(mktemp -d /tmp/codeact-test-XXXX) + +# --- src/ directory with Python files --- +mkdir -p "$WORKSPACE/src" + +cat > "$WORKSPACE/src/app.py" << 'PYEOF' +"""Main application module.""" +import os +import sys +from pathlib import Path + + +# TODO: Add proper logging configuration +def main(): + """Entry point for the application.""" + config = load_config() + # TODO: Validate config before using + server = create_server(config) + server.run() + + +def load_config(): + """Load configuration from environment.""" + return { + "host": os.environ.get("APP_HOST", "0.0.0.0"), + "port": int(os.environ.get("APP_PORT", "8080")), + "debug": os.environ.get("APP_DEBUG", "false").lower() == "true", + } + + +def create_server(config): + """Create and configure the server.""" + # TODO: Support HTTPS configuration + return Server(config) + + +class Server: + def __init__(self, config): + self.config = config + self.routes = {} + + def add_route(self, path, handler): + self.routes[path] = handler + + def run(self): + # TODO: Implement graceful shutdown + print(f"Server running on {self.config['host']}:{self.config['port']}") + + def health_check(self): + return {"status": "ok"} + + +def parse_request(raw): + """Parse an HTTP request string.""" + lines = raw.split("\n") + method, path, _ = lines[0].split(" ") + return {"method": method, "path": path} + + +def format_response(status, body): + """Format an HTTP response.""" + return f"HTTP/1.1 {status}\r\nContent-Length: {len(body)}\r\n\r\n{body}" + + +# TODO: Add request rate limiting +def handle_request(request): + """Handle an incoming request.""" + if request["method"] == "GET": + return format_response("200 OK", '{"message": "hello"}') + return format_response("405 Method Not Allowed", "") + + +if __name__ == "__main__": + main() +PYEOF + +cat > "$WORKSPACE/src/utils.py" << 'PYEOF' +"""Utility functions.""" +import re +import json +from datetime import datetime + + +def slugify(text): + text = text.lower().strip() + text = re.sub(r'[^\w\s-]', '', text) + text = re.sub(r'[\s_-]+', '-', text) + return text + + +def timestamp(): + return datetime.now().isoformat() + + +def deep_merge(base, override): + result = base.copy() + for key, value in override.items(): + if key in result and isinstance(result[key], dict) and isinstance(value, dict): + result[key] = deep_merge(result[key], value) + else: + result[key] = value + return result + + +def truncate(text, max_length=100): + if len(text) <= max_length: + return text + return text[:max_length - 3] + "..." + + +def parse_csv_line(line): + return [field.strip() for field in line.split(",")] + + +def validate_email(email): + pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' + return bool(re.match(pattern, email)) + + +def chunk_list(lst, size): + return [lst[i:i + size] for i in range(0, len(lst), size)] +PYEOF + +cat > "$WORKSPACE/src/models.py" << 'PYEOF' +"""Data models.""" +import os +from dataclasses import dataclass, field +from typing import Optional, List + + +@dataclass +class User: + id: int + name: str + email: str + role: str = "user" + active: bool = True + + def display_name(self): + return f"{self.name} ({self.role})" + + +@dataclass +class Project: + id: int + name: str + owner: User + members: List[User] = field(default_factory=list) + description: Optional[str] = None + + def add_member(self, user): + if user not in self.members: + self.members.append(user) + + def member_count(self): + return len(self.members) + 1 # +1 for owner + + +@dataclass +class Task: + id: int + title: str + project: Project + assignee: Optional[User] = None + status: str = "open" + priority: int = 0 + + def assign(self, user): + self.assignee = user + + def close(self): + self.status = "closed" + + def is_overdue(self): + return False # TODO: implement with due dates +PYEOF + +cat > "$WORKSPACE/src/api.py" << 'PYEOF' +"""API endpoint handlers.""" +import os +import json + + +# TODO: Add authentication middleware +def get_users(db): + """Get all users from the database.""" + users = db.query("SELECT * FROM users WHERE active = 1") + return json.dumps(users) + + +def create_user(db, data): + """Create a new user.""" + required = ["name", "email"] + for field in required: + if field not in data: + return {"error": f"Missing field: {field}"}, 400 + db.insert("users", data) + return {"status": "created"}, 201 + + +def get_projects(db, user_id=None): + """Get projects, optionally filtered by user.""" + if user_id: + return db.query(f"SELECT * FROM projects WHERE owner_id = {user_id}") + return db.query("SELECT * FROM projects") + + +def health(db): + """Health check endpoint.""" + try: + db.query("SELECT 1") + return {"status": "healthy", "database": "connected"} + except Exception: + return {"status": "unhealthy", "database": "disconnected"} + + +def search(db, query, limit=10): + """Search across all resources.""" + results = [] + for table in ["users", "projects", "tasks"]: + rows = db.query(f"SELECT * FROM {table} WHERE name LIKE '%{query}%' LIMIT {limit}") + results.extend(rows) + return results +PYEOF + +cat > "$WORKSPACE/src/config.py" << 'PYEOF' +"""Configuration management.""" +import os +import json +from pathlib import Path + + +DEFAULT_CONFIG = { + "app": { + "name": "myapp", + "version": "1.0.0", + "debug": False, + }, + "server": { + "host": "0.0.0.0", + "port": 8080, + "workers": 4, + }, + "database": { + "url": "sqlite:///app.db", + "pool_size": 5, + }, + "logging": { + "level": "INFO", + "format": "%(asctime)s %(levelname)s %(message)s", + }, +} + + +def load_config(path=None): + """Load config from file, falling back to defaults.""" + config = DEFAULT_CONFIG.copy() + if path and Path(path).exists(): + with open(path) as f: + overrides = json.load(f) + config.update(overrides) + # Environment overrides + if os.environ.get("APP_DEBUG"): + config["app"]["debug"] = True + if os.environ.get("APP_PORT"): + config["server"]["port"] = int(os.environ["APP_PORT"]) + return config + + +def save_config(config, path): + """Save config to file.""" + Path(path).parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + json.dump(config, f, indent=2) +PYEOF + +# --- config/ directory with JSON files --- +mkdir -p "$WORKSPACE/config" + +cat > "$WORKSPACE/config/settings.json" << 'EOF' +{ + "app_name": "test-app", + "version": "2.1.0", + "features": { + "dark_mode": true, + "notifications": true, + "beta_features": false + } +} +EOF + +cat > "$WORKSPACE/config/database.json" << 'EOF' +{ + "host": "localhost", + "port": 5432, + "name": "testdb", + "pool_size": 10, + "ssl": true +} +EOF + +cat > "$WORKSPACE/config/broken.json" << 'EOF' +{ + "key": "value", + "missing_closing_bracket": [1, 2, 3 + "another_key": true +} +EOF + +# --- tests/ directory --- +mkdir -p "$WORKSPACE/tests" + +cat > "$WORKSPACE/tests/test_app.py" << 'PYEOF' +"""Tests for app module.""" +import pytest + + +def test_load_config(): + from src.app import load_config + config = load_config() + assert "host" in config + assert "port" in config + + +def test_parse_request(): + from src.app import parse_request + req = parse_request("GET /api/users HTTP/1.1\nHost: localhost") + assert req["method"] == "GET" + assert req["path"] == "/api/users" + + +def test_format_response(): + from src.app import format_response + resp = format_response("200 OK", "hello") + assert "200 OK" in resp + assert "hello" in resp +PYEOF + +cat > "$WORKSPACE/tests/test_utils.py" << 'PYEOF' +"""Tests for utils module.""" + + +def test_slugify(): + from src.utils import slugify + assert slugify("Hello World") == "hello-world" + assert slugify(" spaces ") == "spaces" + + +def test_truncate(): + from src.utils import truncate + assert truncate("short") == "short" + assert len(truncate("a" * 200)) <= 100 + + +def test_validate_email(): + from src.utils import validate_email + assert validate_email("user@example.com") + assert not validate_email("invalid") +PYEOF + +# --- README.md --- +cat > "$WORKSPACE/README.md" << 'EOF' +# Test Project + +A sample project for testing codeact functionality. + +## Structure + +- `src/` — Source code +- `config/` — Configuration files +- `tests/` — Test files + +## Setup + +```bash +pip install -r requirements.txt +python -m src.app +``` +EOF + +# --- src/handlers/ --- +mkdir -p "$WORKSPACE/src/handlers" + +cat > "$WORKSPACE/src/handlers/__init__.py" << 'PYEOF' +"""HTTP request handlers.""" +PYEOF + +cat > "$WORKSPACE/src/handlers/user_handler.py" << 'PYEOF' +"""User endpoint handlers.""" +from src.models import User +from src.services.user_service import get_user, list_users + + +def handle_get_users(request): + """List all active users.""" + users = list_users(active_only=True) + return {"users": users, "count": len(users)} + + +def handle_get_user(request, user_id): + user = get_user(user_id) + if not user: + return {"error": "Not found"}, 404 + return user + + +def handle_create_user(request): + """Create a new user from request data.""" + data = request.get("body", {}) + # TODO: Add input validation + if not data.get("email"): + return {"error": "Email required"}, 400 + user = User(id=0, name=data["name"], email=data["email"]) + return {"id": user.id, "status": "created"}, 201 + + +def handle_update_user(request, user_id): + data = request.get("body", {}) + # TODO: Validate update fields + return {"id": user_id, "status": "updated"} + + +def handle_delete_user(request, user_id): + """Delete a user by ID.""" + # TODO: Add soft-delete support + return {"status": "deleted"}, 204 +PYEOF + +cat > "$WORKSPACE/src/handlers/project_handler.py" << 'PYEOF' +"""Project endpoint handlers.""" +from src.models import Project +from src.services.project_service import get_project, list_projects + + +def handle_list_projects(request): + """Return all projects.""" + return {"projects": list_projects()} + + +def handle_get_project(request, project_id): + project = get_project(project_id) + if not project: + return {"error": "Not found"}, 404 + return project + + +def handle_create_project(request): + """Create a new project.""" + data = request.get("body", {}) + # TODO: Validate required fields + return {"status": "created"}, 201 + + +def handle_update_project(request, project_id): + data = request.get("body", {}) + return {"id": project_id, "status": "updated"} + + +def handle_archive_project(request, project_id): + """Archive a project instead of deleting.""" + # TODO: Notify project members on archive + return {"status": "archived"} +PYEOF + +cat > "$WORKSPACE/src/handlers/task_handler.py" << 'PYEOF' +"""Task management handlers.""" +from src.models import Task + + +def handle_list_tasks(request, project_id): + """List tasks for a project.""" + # TODO: Add pagination support + return {"tasks": [], "project_id": project_id} + + +def handle_get_task(request, task_id): + return {"task_id": task_id} + + +def handle_create_task(request, project_id): + data = request.get("body", {}) + # TODO: Validate priority range 0-5 + return {"status": "created"}, 201 + + +def handle_assign_task(request, task_id, user_id): + """Assign a task to a user.""" + return {"task_id": task_id, "assignee": user_id} + + +def handle_close_task(request, task_id): + # TODO: Check if task has open subtasks before closing + return {"task_id": task_id, "status": "closed"} +PYEOF + +cat > "$WORKSPACE/src/handlers/auth_handler.py" << 'PYEOF' +"""Authentication handlers.""" +import os +import json + + +def handle_login(request): + """Authenticate user and return token.""" + data = request.get("body", {}) + if not data.get("email") or not data.get("password"): + return {"error": "Credentials required"}, 401 + # TODO: Implement proper password hashing + return {"token": "fake-jwt-token"} + + +def handle_logout(request): + return {"status": "logged out"} + + +def handle_refresh_token(request): + """Refresh an expired JWT token.""" + # TODO: Validate refresh token expiry + return {"token": "new-fake-jwt-token"} + + +def handle_reset_password(request): + data = request.get("body", {}) + if not data.get("email"): + return {"error": "Email required"}, 400 + # TODO: Send actual reset email + return {"status": "reset email sent"} +PYEOF + +# --- src/middleware/ --- +mkdir -p "$WORKSPACE/src/middleware" + +cat > "$WORKSPACE/src/middleware/__init__.py" << 'PYEOF' +"""Request/response middleware.""" +PYEOF + +cat > "$WORKSPACE/src/middleware/auth.py" << 'PYEOF' +"""Authentication middleware.""" +import os + + +SECRET_KEY = os.environ.get("JWT_SECRET", "dev-secret") + + +def authenticate(request): + """Verify JWT token in Authorization header.""" + token = request.get("headers", {}).get("Authorization", "") + if not token.startswith("Bearer "): + return None + # TODO: Actually verify JWT signature + return {"user_id": 1, "role": "user"} + + +def require_role(role): + def decorator(handler): + def wrapper(request, *args, **kwargs): + user = authenticate(request) + if not user or user.get("role") != role: + return {"error": "Forbidden"}, 403 + return handler(request, *args, **kwargs) + return wrapper + return decorator + + +def hash_password(password): + # TODO: Use bcrypt instead of this placeholder + return f"hashed_{password}" +PYEOF + +cat > "$WORKSPACE/src/middleware/logging_mw.py" << 'PYEOF' +"""Request logging middleware.""" +import time +from datetime import datetime + + +def log_request(request): + """Log incoming request details.""" + method = request.get("method", "?") + path = request.get("path", "?") + timestamp = datetime.now().isoformat() + print(f"[{timestamp}] {method} {path}") + + +def log_response(request, response, duration_ms): + status = response[1] if isinstance(response, tuple) else 200 + print(f" -> {status} ({duration_ms}ms)") + + +def timing_middleware(handler): + def wrapper(request, *args, **kwargs): + start = time.time() + response = handler(request, *args, **kwargs) + duration = (time.time() - start) * 1000 + log_response(request, response, duration) + return response + return wrapper +PYEOF + +cat > "$WORKSPACE/src/middleware/cors.py" << 'PYEOF' +"""CORS handling middleware.""" + +ALLOWED_ORIGINS = ["http://localhost:3000", "https://app.example.com"] + + +def cors_headers(origin): + """Generate CORS headers for the given origin.""" + if origin in ALLOWED_ORIGINS: + return { + "Access-Control-Allow-Origin": origin, + "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE", + "Access-Control-Allow-Headers": "Content-Type, Authorization", + } + return {} + + +def cors_middleware(handler): + def wrapper(request, *args, **kwargs): + origin = request.get("headers", {}).get("Origin", "") + response = handler(request, *args, **kwargs) + headers = cors_headers(origin) + return response, headers + return wrapper +PYEOF + +cat > "$WORKSPACE/src/middleware/rate_limit.py" << 'PYEOF' +"""Rate limiting middleware.""" +import time + + +# In-memory store (not suitable for production) +_request_counts = {} + + +def check_rate_limit(client_ip, max_requests=100, window_seconds=60): + """Check if client has exceeded rate limit.""" + now = time.time() + key = f"{client_ip}:{int(now / window_seconds)}" + _request_counts[key] = _request_counts.get(key, 0) + 1 + return _request_counts[key] <= max_requests + + +def rate_limit_middleware(handler): + def wrapper(request, *args, **kwargs): + ip = request.get("client_ip", "unknown") + if not check_rate_limit(ip): + return {"error": "Too many requests"}, 429 + return handler(request, *args, **kwargs) + return wrapper + + +def reset_counts(): + # TODO: Add automatic cleanup of old window entries + _request_counts.clear() +PYEOF + +# --- src/services/ --- +mkdir -p "$WORKSPACE/src/services" + +cat > "$WORKSPACE/src/services/__init__.py" << 'PYEOF' +"""Business logic services.""" +PYEOF + +cat > "$WORKSPACE/src/services/user_service.py" << 'PYEOF' +"""User business logic.""" +from src.models import User + + +_users_db = [] + + +def get_user(user_id): + for u in _users_db: + if u.id == user_id: + return u + return None + + +def list_users(active_only=False): + """Return all users, optionally filtered by active status.""" + if active_only: + return [u for u in _users_db if u.active] + return list(_users_db) + + +def create_user(name, email, role="user"): + """Create and store a new user.""" + user = User(id=len(_users_db) + 1, name=name, email=email, role=role) + _users_db.append(user) + return user + + +def deactivate_user(user_id): + user = get_user(user_id) + if user: + user.active = False + return user + + +def search_users(query): + # TODO: Add fuzzy matching + return [u for u in _users_db if query.lower() in u.name.lower()] +PYEOF + +cat > "$WORKSPACE/src/services/project_service.py" << 'PYEOF' +"""Project business logic.""" +from src.models import Project + + +_projects_db = [] + + +def get_project(project_id): + """Fetch a single project by ID.""" + for p in _projects_db: + if p.id == project_id: + return p + return None + + +def list_projects(owner_id=None): + if owner_id: + return [p for p in _projects_db if p.owner.id == owner_id] + return list(_projects_db) + + +def create_project(name, owner, description=None): + """Create a new project.""" + project = Project( + id=len(_projects_db) + 1, + name=name, owner=owner, description=description, + ) + _projects_db.append(project) + return project + + +def archive_project(project_id): + # TODO: Cascade archive to project tasks + project = get_project(project_id) + return project +PYEOF + +cat > "$WORKSPACE/src/services/email_service.py" << 'PYEOF' +"""Email sending service.""" +import os + + +SMTP_HOST = os.environ.get("SMTP_HOST", "localhost") +SMTP_PORT = int(os.environ.get("SMTP_PORT", "587")) + + +def send_email(to, subject, body): + """Send an email message.""" + # TODO: Implement actual SMTP connection + print(f"Sending to {to}: {subject}") + return True + + +def send_welcome_email(user): + return send_email( + user.email, + "Welcome!", + f"Hello {user.name}, welcome to the platform.", + ) + + +def send_password_reset(email, reset_url): + # TODO: Add email template rendering + return send_email(email, "Password Reset", f"Reset here: {reset_url}") + + +def send_notification(user, message): + """Send a notification email.""" + return send_email(user.email, "Notification", message) +PYEOF + +cat > "$WORKSPACE/src/services/cache_service.py" << 'PYEOF' +"""In-memory cache service.""" +import time + + +_cache = {} + + +def get(key): + entry = _cache.get(key) + if entry is None: + return None + if entry["expires"] and time.time() > entry["expires"]: + del _cache[key] + return None + return entry["value"] + + +def set(key, value, ttl_seconds=300): + """Store a value with optional TTL.""" + _cache[key] = { + "value": value, + "expires": time.time() + ttl_seconds if ttl_seconds else None, + } + + +def delete(key): + _cache.pop(key, None) + + +def clear(): + """Clear the entire cache.""" + # TODO: Add cache statistics tracking before clear + _cache.clear() + + +def cache_size(): + return len(_cache) +PYEOF + +# --- src/db/ --- +mkdir -p "$WORKSPACE/src/db" + +cat > "$WORKSPACE/src/db/__init__.py" << 'PYEOF' +"""Database layer.""" +PYEOF + +cat > "$WORKSPACE/src/db/connection.py" << 'PYEOF' +"""Database connection management.""" +import os + + +DATABASE_URL = os.environ.get("DATABASE_URL", "sqlite:///app.db") +_pool = [] + + +def get_connection(): + """Get a database connection from the pool.""" + # TODO: Implement actual connection pooling + if _pool: + return _pool.pop() + return _create_connection() + + +def _create_connection(): + return {"url": DATABASE_URL, "active": True} + + +def release_connection(conn): + """Return a connection to the pool.""" + _pool.append(conn) +PYEOF + +cat > "$WORKSPACE/src/db/migrations.py" << 'PYEOF' +"""Schema migration utilities.""" +import json +from pathlib import Path + + +MIGRATIONS_DIR = Path("migrations") + + +def list_migrations(): + """List all available migration files.""" + if not MIGRATIONS_DIR.exists(): + return [] + return sorted(MIGRATIONS_DIR.glob("*.sql")) + + +def get_current_version(): + # TODO: Read from _schema_version table + return 0 + + +def apply_migration(migration_path): + """Apply a single migration file.""" + content = Path(migration_path).read_text() + print(f"Applying: {migration_path}") + return True + + +def migrate_up(): + """Run all pending migrations.""" + current = get_current_version() + pending = [m for m in list_migrations() if _version_from(m) > current] + # TODO: Wrap in transaction + for m in pending: + apply_migration(m) + return len(pending) + + +def _version_from(path): + return int(Path(path).stem.split("_")[0]) +PYEOF + +cat > "$WORKSPACE/src/db/queries.py" << 'PYEOF' +"""Common database queries.""" + + +def find_users_by_role(conn, role): + """Find all users with a given role.""" + return conn.execute( + "SELECT * FROM users WHERE role = ?", (role,) + ).fetchall() + + +def find_active_projects(conn): + return conn.execute( + "SELECT * FROM projects WHERE archived = 0" + ).fetchall() + + +def count_tasks_by_status(conn, project_id): + """Count tasks grouped by status for a project.""" + return conn.execute( + "SELECT status, COUNT(*) FROM tasks WHERE project_id = ? GROUP BY status", + (project_id,), + ).fetchall() + + +def search_all(conn, query): + results = {} + for table in ["users", "projects", "tasks"]: + results[table] = conn.execute( + f"SELECT * FROM {table} WHERE name LIKE ?", + (f"%{query}%",), + ).fetchall() + return results + + +def recent_activity(conn, limit=20): + """Get recent activity log entries.""" + return conn.execute( + "SELECT * FROM activity_log ORDER BY created_at DESC LIMIT ?", + (limit,), + ).fetchall() +PYEOF + +# --- Additional test files --- + +cat > "$WORKSPACE/tests/test_handlers.py" << 'PYEOF' +"""Tests for request handlers.""" + + +def test_handle_get_users(): + from src.handlers.user_handler import handle_get_users + result = handle_get_users({}) + assert "users" in result + assert "count" in result + + +def test_handle_login_missing_credentials(): + from src.handlers.auth_handler import handle_login + result = handle_login({"body": {}}) + assert result[1] == 401 + + +def test_handle_list_projects(): + from src.handlers.project_handler import handle_list_projects + result = handle_list_projects({}) + assert "projects" in result + + +def test_handle_create_task(): + from src.handlers.task_handler import handle_create_task + result = handle_create_task({"body": {"title": "Test"}}, project_id=1) + assert result[1] == 201 +PYEOF + +cat > "$WORKSPACE/tests/test_services.py" << 'PYEOF' +"""Tests for service layer.""" + + +def test_create_user(): + from src.services.user_service import create_user + user = create_user("Test", "test@example.com") + assert user.name == "Test" + assert user.email == "test@example.com" + + +def test_cache_set_get(): + from src.services.cache_service import set, get, clear + clear() + set("key", "value") + assert get("key") == "value" + + +def test_send_email(): + from src.services.email_service import send_email + assert send_email("test@test.com", "Subject", "Body") is True +PYEOF + +cat > "$WORKSPACE/tests/test_middleware.py" << 'PYEOF' +"""Tests for middleware.""" + + +def test_authenticate_no_token(): + from src.middleware.auth import authenticate + result = authenticate({}) + assert result is None + + +def test_cors_allowed_origin(): + from src.middleware.cors import cors_headers + headers = cors_headers("http://localhost:3000") + assert "Access-Control-Allow-Origin" in headers + + +def test_rate_limit_within_bounds(): + from src.middleware.rate_limit import check_rate_limit + assert check_rate_limit("127.0.0.1") is True +PYEOF + +# --- scripts/ directory --- +mkdir -p "$WORKSPACE/scripts" + +cat > "$WORKSPACE/scripts/seed_db.py" << 'PYEOF' +"""Seed the database with sample data.""" +from src.services.user_service import create_user +from src.services.project_service import create_project + + +def seed(): + """Create sample users and projects.""" + admin = create_user("Admin", "admin@example.com", role="admin") + user1 = create_user("Alice", "alice@example.com") + user2 = create_user("Bob", "bob@example.com") + # TODO: Add sample tasks and activity log entries + create_project("Alpha", admin, description="First project") + create_project("Beta", user1) + print("Database seeded.") + + +if __name__ == "__main__": + seed() +PYEOF + +cat > "$WORKSPACE/scripts/health_check.py" << 'PYEOF' +"""Health check script for monitoring.""" +import sys +from src.db.connection import get_connection + + +def check(): + """Run health checks and exit with status code.""" + conn = get_connection() + if not conn.get("active"): + print("FAIL: database unreachable") + sys.exit(1) + print("OK: all checks passed") + sys.exit(0) + + +if __name__ == "__main__": + check() +PYEOF + +# --- Additional config files --- + +cat > "$WORKSPACE/config/api_keys.json" << 'EOF' +{ + "stripe_api_key": "sk_test_placeholder", + "sendgrid_api_key": "SG.placeholder", + "sentry_dsn": "https://placeholder@sentry.io/1", + "redis_url": "redis://localhost:6379", + "s3_bucket": "my-app-uploads", + "cloudflare_token": "cf_placeholder" +} +EOF + +cat > "$WORKSPACE/config/features.json" << 'EOF' +{ + "enable_signup": true, + "enable_oauth": false, + "enable_webhooks": true, + "max_upload_size_mb": 50, + "rate_limit_per_minute": 100, + "maintenance_mode": false, + "beta_users_only": false, + "dark_mode": true +} +EOF + +echo "$WORKSPACE" diff --git a/plugins/codeact/tests/fixtures/user-tools/shout.py b/plugins/codeact/tests/fixtures/user-tools/shout.py new file mode 100644 index 0000000..534b33c --- /dev/null +++ b/plugins/codeact/tests/fixtures/user-tools/shout.py @@ -0,0 +1,18 @@ +"""Example custom CodeAct tool: shout(text) → uppercases text. + +Drop this file into ~/.config/codeact/tools/ to register a `shout` tool. +""" + +TOOL = { + "name": "shout", # optional; defaults to filename stem + "description": "Echo back input text in uppercase.", + "parameters": { + "text": {"type": "string", "required": True, + "description": "Text to shout."}, + }, + # "function": "run", # optional; defaults to "run" +} + + +def run(text: str = "") -> str: + return text.upper() diff --git a/plugins/codeact/tests/prompts/functional-natural.json b/plugins/codeact/tests/prompts/functional-natural.json new file mode 100644 index 0000000..817e8db --- /dev/null +++ b/plugins/codeact/tests/prompts/functional-natural.json @@ -0,0 +1,38 @@ +[ + { + "id": "multi-file-search", + "_comment": "Same intent as functional.json but no 'use codeact' hint. Cross-references TODOs against import lines, which is awkward in one bash one-liner — codeact should win naturally.", + "prompt": "For each Python file in src/, list its TODO comments AND the modules it imports. Show file -> {todos: [...], imports: [...]} as JSON.", + "assertions": { + "codeact_invoked": true, + "output_contains": ["app.py", "TODO"] + } + }, + { + "id": "batch-count", + "_comment": "Multi-dimensional aggregation per file (lines, TODOs, function defs) plus sort. Hard to do without a loop — natural fit for codeact.", + "prompt": "For each Python file under src/, count its total lines, number of TODO comments, and number of `def` function definitions. Show all three as a table sorted by total lines descending.", + "assertions": { + "codeact_invoked": true, + "output_contains": ["app.py", "lines"] + } + }, + { + "id": "json-validate", + "_comment": "Validate + aggregate (top-level key count for valid ones). Multi-file scan with per-file analysis — codeact fits.", + "prompt": "Check every JSON file in config/ for valid syntax. For each valid file also report the number of top-level keys. For invalid files report the parse error. Combine into one summary table.", + "assertions": { + "codeact_invoked": true, + "output_contains": ["broken.json"] + } + }, + { + "id": "single-file-read", + "_comment": "Negative test: trivial single-file read should NOT trigger codeact even when no hint is given.", + "prompt": "Read README.md and tell me what it says in one sentence.", + "assertions": { + "codeact_invoked": false, + "output_contains": ["README"] + } + } +] diff --git a/plugins/codeact/tests/prompts/functional.json b/plugins/codeact/tests/prompts/functional.json new file mode 100644 index 0000000..6448fda --- /dev/null +++ b/plugins/codeact/tests/prompts/functional.json @@ -0,0 +1,64 @@ +[ + { + "id": "multi-file-search", + "prompt": "Find all TODO comments across all Python files in src/ and list each with its file and line number. Use codeact to do this in a single sandbox run.", + "assertions": { + "codeact_invoked": true, + "output_contains": ["TODO", "app.py"], + "min_matches": 3 + } + }, + { + "id": "batch-count", + "prompt": "Count lines of code in each Python file under src/ and show the top 3 largest files. Use codeact to do this in one pass.", + "assertions": { + "codeact_invoked": true, + "output_contains": ["lines", "app.py"] + } + }, + { + "id": "json-validate", + "prompt": "Check all JSON files in config/ for valid syntax. Report which are valid and which have errors. Use codeact.", + "assertions": { + "codeact_invoked": true, + "output_contains": ["broken.json"] + } + }, + { + "id": "single-file-read", + "_comment": "Negative test: trivial single-file read should NOT trigger codeact. Codeact is for multi-step chains; one-shot reads should go through the agent's built-in `view` tool.", + "prompt": "Read README.md and tell me what it says in one sentence.", + "assertions": { + "codeact_invoked": false, + "output_contains": ["README"] + } + }, + { + "id": "custom-tool-shout", + "prompt": "Use codeact to call the `shout` tool with text=\"hello world\" and print the result.", + "files": [ + { + "path": ".codeact-config/tools/shout.py", + "from": "user-tools/shout.py" + } + ], + "assertions": { + "codeact_invoked": true, + "output_contains": ["HELLO WORLD"] + } + }, + { + "id": "tool-config-disable", + "prompt": "Use codeact to attempt to invoke the `bash` tool (e.g. run `echo hi`). The codeact config in this workspace disables the bash tool, so the call should fail. Wrap the call in try/except. On any exception, print exactly: TOOL_DISABLED_OK followed by the exception message. On success, print: TOOL_STILL_ENABLED followed by the result. Use whichever syntax the active codeact backend exposes.", + "files": [ + { + "path": ".codeact-config/config.json", + "content": "{\"disabled\": [\"bash\"]}" + } + ], + "assertions": { + "codeact_invoked": true, + "output_contains": ["TOOL_DISABLED_OK"] + } + } +] diff --git a/plugins/codeact/tests/prompts/perf.json b/plugins/codeact/tests/prompts/perf.json new file mode 100644 index 0000000..9d72d06 --- /dev/null +++ b/plugins/codeact/tests/prompts/perf.json @@ -0,0 +1,104 @@ +[ + { + "id": "config-and-code-audit", + "prompt": "Read all JSON files in config/ and all Python files in src/. For each JSON config key (at any nesting level), check if it appears as a string literal anywhere in the Python source code. List config keys that are never referenced.", + "description": "Read 5 JSON configs + 20+ Python source files + cross-reference keys \u2014 forces 25+ sequential view calls in baseline", + "min_token_reduction": 0 + }, + { + "id": "todo-in-context", + "prompt": "Find every TODO comment in every Python file across the entire project (src/, tests/, scripts/). For each TODO, show the file path, line number, the full TODO comment text, and the name of the function or class it appears in (or 'module-level' if not inside a function). Group results by directory.", + "description": "Glob all dirs + view each file + parse function context for each TODO \u2014 baseline needs sequential view+grep per file to get function context", + "min_token_reduction": 0 + }, + { + "id": "import-dependency-map", + "prompt": "Read every Python file under src/ and build an import dependency map: for each file, list which other src/ files it imports from. Then identify any circular import chains and any src/ modules that nothing else imports (dead modules). Show the full dependency map and findings.", + "description": "Read 20+ src files + parse imports + cross-reference all pairs + detect cycles \u2014 baseline needs sequential reads then cross-referencing", + "min_token_reduction": 0 + }, + { + "id": "handler-service-mismatch", + "prompt": "Read all files in src/handlers/ and src/services/. For each handler function, check if it calls the corresponding service function (e.g. handle_get_users should call list_users or get_user from user_service). List any handler functions that don't reference their expected service, and any service functions that no handler calls.", + "description": "Read handlers + services + cross-reference function calls \u2014 only ~13 files total so codeact overhead is proportionally higher", + "min_token_reduction": 0 + }, + { + "id": "test-coverage-gap", + "prompt": "Read all Python files in src/ and all test files in tests/. For each public function defined in src/ (not starting with underscore), check if there's a corresponding test function in tests/ that imports or references it. List all untested public functions with their file and line number.", + "description": "Read 20+ src files + 5 test files + cross-reference function names \u2014 heavy cross-file analysis baseline does sequentially" + }, + { + "id": "env-var-inventory", + "prompt": "Read every Python file in src/, tests/, and scripts/. Find every os.environ.get or os.getenv call and extract the variable name. Then read all JSON files in config/. Print two lists: env vars not in any config file, and config keys not referenced by os.environ. One item per line, no extra commentary.", + "description": "Scan 30+ Python files for env var refs + 5 JSON configs, then cross-reference both directions. Baseline must read every file then compare sets.", + "min_token_reduction": 0 + }, + { + "id": "docstring-coverage", + "prompt": "Read every Python file in src/. For each file, find all function and class definitions (lines starting with 'def ' or 'class '). Check if each has a docstring (triple-quoted string on the next non-empty line). List all functions and classes that are missing docstrings, grouped by file, with line numbers.", + "description": "Read 20+ src/ files + parse function defs + check next-line content. Per-function analysis across many files.", + "min_token_reduction": 40 + }, + { + "id": "middleware-chain-audit", + "prompt": "Read all files in src/middleware/, src/handlers/, and src/services/. For each middleware function, check which handlers or services import or reference it. List each middleware with its consumers. Then identify any middleware that nothing imports (dead middleware) and any handler that doesn't use any middleware.", + "description": "Read 3 directories (~20 files), extract imports and references, cross-reference middleware-to-consumer. Three-way cross-reference.", + "min_token_reduction": 0 + }, + { + "id": "full-project-function-index", + "prompt": "Read every Python file in the entire project. Build a complete index of all function definitions: for each function show the file path, line number, function name, number of parameters, and whether it has a return statement. Sort by file path then line number. Show the total count at the end.", + "description": "Read all 30+ Python files, parse every function def, count params, check for return. Heavy per-file parsing across entire project.", + "min_token_reduction": 40 + }, + { + "id": "config-audit-mcp-bloated", + "prompt": "Read all JSON files in config/ and all Python files in src/. For each JSON config key (at any nesting level), check if it appears as a string literal anywhere in the Python source code. List config keys that are never referenced.", + "description": "Same cross-reference task as config-and-code-audit but with 4 MCP servers loaded. Context bloat from MCP tool catalogs makes each turn expensive \u2014 codeact saves by needing fewer turns.", + "files": [ + { + "path": ".mcp.json", + "content": "{\n \"mcpServers\": {\n \"microsoft-docs\": {\n \"type\": \"http\",\n \"url\": \"https://learn.microsoft.com/api/mcp\"\n },\n \"playwright\": {\n \"command\": \"npx\",\n \"args\": [\"-y\", \"@playwright/mcp@latest\"]\n },\n \"markitdown\": {\n \"command\": \"uvx\",\n \"args\": [\"markitdown-mcp\"]\n },\n \"azure-mcp\": {\n \"command\": \"npx\",\n \"args\": [\"-y\", \"@azure/mcp@latest\", \"server\", \"start\"]\n }\n }\n}\n" + } + ] + }, + { + "id": "mcp-docs-cross-ref", + "prompt": "Search Microsoft docs for \"database connection pooling best practices\" using the microsoft-docs MCP server. Then read all Python files in src/db/ and src/services/. Cross-reference: for each best practice mentioned in the docs results, check if our code follows it or violates it. Show which files are relevant and what they do right or wrong.", + "description": "MCP search + 30+ file scan + cross-reference. High file count makes codeact valuable; MCP data processed inside sandbox.", + "disable_mcp_in_codeact": true, + "min_token_reduction": 0, + "files": [ + { + "path": ".mcp.json", + "content": "{\n \"mcpServers\": {\n \"microsoft-docs\": {\n \"type\": \"http\",\n \"url\": \"https://learn.microsoft.com/api/mcp\"\n }\n }\n}\n" + } + ] + }, + { + "id": "test-coverage-mcp-bloated", + "prompt": "Read all Python files in src/ and all test files in tests/. For each public function defined in src/ (not starting with underscore), check if there's a corresponding test function in tests/ that imports or references it. List all untested public functions with their file and line number.", + "description": "Same cross-reference task as test-coverage-gap but with 4 MCP servers loaded. Tests that MCP context bloat amplifies codeact savings on turn-heavy baseline workloads.", + "files": [ + { + "path": ".mcp.json", + "content": "{\n \"mcpServers\": {\n \"microsoft-docs\": {\n \"type\": \"http\",\n \"url\": \"https://learn.microsoft.com/api/mcp\"\n },\n \"playwright\": {\n \"command\": \"npx\",\n \"args\": [\"-y\", \"@playwright/mcp@latest\"]\n },\n \"markitdown\": {\n \"command\": \"uvx\",\n \"args\": [\"markitdown-mcp\"]\n },\n \"azure-mcp\": {\n \"command\": \"npx\",\n \"args\": [\"-y\", \"@azure/mcp@latest\", \"server\", \"start\"]\n }\n }\n}\n" + } + ], + "min_token_reduction": 40 + }, + { + "id": "mcp-docs-cross-ref-bloated", + "prompt": "Search Microsoft docs for \"database connection pooling best practices\" using the microsoft-docs MCP server. Then read all Python files in src/db/ and src/services/. Cross-reference: for each best practice mentioned in the docs results, check if our code follows it or violates it. Show which files are relevant and what they do right or wrong.", + "description": "Same cross-reference task as mcp-docs-cross-ref but with 4 MCP servers loaded. Context bloat from all 4 MCP tool catalogs makes each turn much more expensive \u2014 codeact saves by needing fewer turns.", + "disable_mcp_in_codeact": true, + "min_token_reduction": 0, + "files": [ + { + "path": ".mcp.json", + "content": "{\"mcpServers\": {\"microsoft-docs\": {\"type\": \"http\", \"url\": \"https://learn.microsoft.com/api/mcp\"}, \"playwright\": {\"command\": \"npx\", \"args\": [\"-y\", \"@playwright/mcp@latest\"]}, \"markitdown\": {\"command\": \"uvx\", \"args\": [\"markitdown-mcp\"]}, \"azure-mcp\": {\"command\": \"npx\", \"args\": [\"-y\", \"@azure/mcp@latest\", \"server\", \"start\"]}}}" + } + ] + } +] diff --git a/plugins/codeact/tests/results/perf-results-latest.json b/plugins/codeact/tests/results/perf-results-latest.json new file mode 100644 index 0000000..ab72551 --- /dev/null +++ b/plugins/codeact/tests/results/perf-results-latest.json @@ -0,0 +1,811 @@ +{ + "timestamp": "20260426T154117Z", + "min_token_reduction": 40, + "results": [ + { + "prompt_id": "config-and-code-audit", + "status": "valid", + "baseline_tokens": 7199, + "codeact_tokens": 2949, + "token_reduction_pct": 59.0, + "baseline_input_tokens": 196737, + "codeact_input_tokens": 116094, + "input_token_reduction_pct": 41.0, + "baseline_cost_est": 0.599827, + "codeact_cost_est": 0.33447, + "cost_reduction_pct": 44.2, + "baseline_requests": 6, + "codeact_requests": 6, + "request_reduction_pct": 0.0, + "baseline_tool_calls": 36, + "codeact_tool_calls": 4, + "tool_reduction_pct": 88.9, + "baseline_turns": 6, + "codeact_turns": 4, + "turn_reduction_pct": 33.3, + "baseline_context_bytes": 29401, + "codeact_context_bytes": 5283, + "context_reduction_pct": 82.0, + "codeact_invoked": true, + "baseline_tools": [ + "report_intent", + "glob", + "glob", + "report_intent", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "report_intent", + "bash", + "bash" + ], + "codeact_tools": [ + "report_intent", + "bash", + "bash", + "bash" + ], + "min_token_reduction": 0 + }, + { + "prompt_id": "todo-in-context", + "status": "valid", + "baseline_tokens": 2835, + "codeact_tokens": 3132, + "token_reduction_pct": -10.5, + "baseline_input_tokens": 81063, + "codeact_input_tokens": 115990, + "input_token_reduction_pct": -43.1, + "baseline_cost_est": 0.245182, + "codeact_cost_est": 0.336955, + "cost_reduction_pct": -37.4, + "baseline_requests": 6, + "codeact_requests": 6, + "request_reduction_pct": 0.0, + "baseline_tool_calls": 3, + "codeact_tool_calls": 4, + "tool_reduction_pct": -33.3, + "baseline_turns": 3, + "codeact_turns": 4, + "turn_reduction_pct": -33.3, + "baseline_context_bytes": 10624, + "codeact_context_bytes": 4748, + "context_reduction_pct": 55.3, + "codeact_invoked": true, + "baseline_tools": [ + "report_intent", + "grep", + "grep" + ], + "codeact_tools": [ + "report_intent", + "bash", + "bash", + "bash" + ], + "min_token_reduction": -50 + }, + { + "prompt_id": "import-dependency-map", + "status": "valid", + "baseline_tokens": 3969, + "codeact_tokens": 3685, + "token_reduction_pct": 7.2, + "baseline_input_tokens": 86083, + "codeact_input_tokens": 87685, + "input_token_reduction_pct": -1.9, + "baseline_cost_est": 0.274743, + "codeact_cost_est": 0.274487, + "cost_reduction_pct": 0.1, + "baseline_requests": 6, + "codeact_requests": 6, + "request_reduction_pct": 0.0, + "baseline_tool_calls": 26, + "codeact_tool_calls": 3, + "tool_reduction_pct": 88.5, + "baseline_turns": 3, + "codeact_turns": 3, + "turn_reduction_pct": 0.0, + "baseline_context_bytes": 22036, + "codeact_context_bytes": 4708, + "context_reduction_pct": 78.6, + "codeact_invoked": true, + "baseline_tools": [ + "report_intent", + "glob", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view" + ], + "codeact_tools": [ + "report_intent", + "bash", + "bash" + ], + "min_token_reduction": 0 + }, + { + "prompt_id": "handler-service-mismatch", + "status": "codeact_failed", + "baseline_tokens": 2167, + "codeact_tokens": 0, + "token_reduction_pct": 0, + "baseline_input_tokens": 80314, + "codeact_input_tokens": 0, + "input_token_reduction_pct": 0, + "baseline_cost_est": 0.23329, + "codeact_cost_est": 0.0, + "cost_reduction_pct": 0, + "baseline_requests": 6, + "codeact_requests": 0, + "request_reduction_pct": 0, + "baseline_tool_calls": 13, + "codeact_tool_calls": 0, + "tool_reduction_pct": 0, + "baseline_turns": 3, + "codeact_turns": 0, + "turn_reduction_pct": 0, + "baseline_context_bytes": 8066, + "codeact_context_bytes": 0, + "context_reduction_pct": 0, + "codeact_invoked": false, + "baseline_tools": [ + "report_intent", + "glob", + "glob", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view" + ], + "codeact_tools": [], + "min_token_reduction": -150 + }, + { + "prompt_id": "test-coverage-gap", + "status": "valid", + "baseline_tokens": 7120, + "codeact_tokens": 1783, + "token_reduction_pct": 75.0, + "baseline_input_tokens": 87460, + "codeact_input_tokens": 85060, + "input_token_reduction_pct": 2.7, + "baseline_cost_est": 0.32545, + "codeact_cost_est": 0.239395, + "cost_reduction_pct": 26.4, + "baseline_requests": 6, + "codeact_requests": 6, + "request_reduction_pct": 0.0, + "baseline_tool_calls": 28, + "codeact_tool_calls": 3, + "tool_reduction_pct": 89.3, + "baseline_turns": 3, + "codeact_turns": 3, + "turn_reduction_pct": 0.0, + "baseline_context_bytes": 25252, + "codeact_context_bytes": 4072, + "context_reduction_pct": 83.9, + "codeact_invoked": true, + "baseline_tools": [ + "report_intent", + "glob", + "glob", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view" + ], + "codeact_tools": [ + "report_intent", + "bash", + "bash" + ], + "min_token_reduction": null + }, + { + "prompt_id": "env-var-inventory", + "status": "valid", + "baseline_tokens": 3896, + "codeact_tokens": 2655, + "token_reduction_pct": 31.9, + "baseline_input_tokens": 89354, + "codeact_input_tokens": 114623, + "input_token_reduction_pct": -28.3, + "baseline_cost_est": 0.281825, + "codeact_cost_est": 0.326383, + "cost_reduction_pct": -15.8, + "baseline_requests": 6, + "codeact_requests": 6, + "request_reduction_pct": 0.0, + "baseline_tool_calls": 39, + "codeact_tool_calls": 4, + "tool_reduction_pct": 89.7, + "baseline_turns": 3, + "codeact_turns": 4, + "turn_reduction_pct": -33.3, + "baseline_context_bytes": 27535, + "codeact_context_bytes": 2968, + "context_reduction_pct": 89.2, + "codeact_invoked": true, + "baseline_tools": [ + "report_intent", + "glob", + "glob", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view" + ], + "codeact_tools": [ + "report_intent", + "bash", + "bash", + "bash" + ], + "min_token_reduction": 0 + }, + { + "prompt_id": "docstring-coverage", + "status": "valid", + "baseline_tokens": 7443, + "codeact_tokens": 1680, + "token_reduction_pct": 77.4, + "baseline_input_tokens": 86144, + "codeact_input_tokens": 84527, + "input_token_reduction_pct": 1.9, + "baseline_cost_est": 0.327005, + "codeact_cost_est": 0.236517, + "cost_reduction_pct": 27.7, + "baseline_requests": 6, + "codeact_requests": 6, + "request_reduction_pct": 0.0, + "baseline_tool_calls": 26, + "codeact_tool_calls": 3, + "tool_reduction_pct": 88.5, + "baseline_turns": 3, + "codeact_turns": 3, + "turn_reduction_pct": 0.0, + "baseline_context_bytes": 22036, + "codeact_context_bytes": 3291, + "context_reduction_pct": 85.1, + "codeact_invoked": true, + "baseline_tools": [ + "report_intent", + "glob", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view" + ], + "codeact_tools": [ + "report_intent", + "bash", + "bash" + ], + "min_token_reduction": 40 + }, + { + "prompt_id": "middleware-chain-audit", + "status": "valid", + "baseline_tokens": 2513, + "codeact_tokens": 4336, + "token_reduction_pct": -72.5, + "baseline_input_tokens": 181710, + "codeact_input_tokens": 181582, + "input_token_reduction_pct": 0.1, + "baseline_cost_est": 0.49197, + "codeact_cost_est": 0.518995, + "cost_reduction_pct": -5.5, + "baseline_requests": 6, + "codeact_requests": 6, + "request_reduction_pct": 0.0, + "baseline_tool_calls": 26, + "codeact_tool_calls": 6, + "tool_reduction_pct": 76.9, + "baseline_turns": 6, + "codeact_turns": 6, + "turn_reduction_pct": 0.0, + "baseline_context_bytes": 18935, + "codeact_context_bytes": 3819, + "context_reduction_pct": 79.8, + "codeact_invoked": true, + "baseline_tools": [ + "report_intent", + "glob", + "glob", + "glob", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "grep", + "grep", + "grep", + "grep", + "glob", + "view", + "view" + ], + "codeact_tools": [ + "report_intent", + "bash", + "bash", + "bash", + "bash", + "bash" + ], + "min_token_reduction": -50 + }, + { + "prompt_id": "full-project-function-index", + "status": "valid", + "baseline_tokens": 7381, + "codeact_tokens": 2886, + "token_reduction_pct": 60.9, + "baseline_input_tokens": 133231, + "codeact_input_tokens": 117157, + "input_token_reduction_pct": 12.1, + "baseline_cost_est": 0.443792, + "codeact_cost_est": 0.336182, + "cost_reduction_pct": 24.2, + "baseline_requests": 6, + "codeact_requests": 6, + "request_reduction_pct": 0.0, + "baseline_tool_calls": 34, + "codeact_tool_calls": 4, + "tool_reduction_pct": 88.2, + "baseline_turns": 4, + "codeact_turns": 4, + "turn_reduction_pct": 0.0, + "baseline_context_bytes": 38411, + "codeact_context_bytes": 13580, + "context_reduction_pct": 64.6, + "codeact_invoked": true, + "baseline_tools": [ + "report_intent", + "glob", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "bash" + ], + "codeact_tools": [ + "report_intent", + "bash", + "bash", + "bash" + ], + "min_token_reduction": 40 + }, + { + "prompt_id": "config-audit-mcp-bloated", + "status": "valid", + "baseline_tokens": 7419, + "codeact_tokens": 3920, + "token_reduction_pct": 47.2, + "baseline_input_tokens": 330421, + "codeact_input_tokens": 259584, + "input_token_reduction_pct": 21.4, + "baseline_cost_est": 0.937338, + "codeact_cost_est": 0.70776, + "cost_reduction_pct": 24.5, + "baseline_requests": 6, + "codeact_requests": 6, + "request_reduction_pct": 0.0, + "baseline_tool_calls": 59, + "codeact_tool_calls": 5, + "tool_reduction_pct": 91.5, + "baseline_turns": 6, + "codeact_turns": 5, + "turn_reduction_pct": 16.7, + "baseline_context_bytes": 24448, + "codeact_context_bytes": 5390, + "context_reduction_pct": 78.0, + "codeact_invoked": true, + "baseline_tools": [ + "report_intent", + "glob", + "glob", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep", + "grep" + ], + "codeact_tools": [ + "report_intent", + "bash", + "bash", + "bash", + "bash" + ], + "min_token_reduction": null + }, + { + "prompt_id": "mcp-docs-cross-ref", + "status": "codeact_failed", + "baseline_tokens": 2739, + "codeact_tokens": 0, + "token_reduction_pct": 0, + "baseline_input_tokens": 94527, + "codeact_input_tokens": 0, + "input_token_reduction_pct": 0, + "baseline_cost_est": 0.277402, + "codeact_cost_est": 0.0, + "cost_reduction_pct": 0, + "baseline_requests": 6, + "codeact_requests": 0, + "request_reduction_pct": 0, + "baseline_tool_calls": 13, + "codeact_tool_calls": 0, + "tool_reduction_pct": 0, + "baseline_turns": 3, + "codeact_turns": 0, + "turn_reduction_pct": 0, + "baseline_context_bytes": 33161, + "codeact_context_bytes": 0, + "context_reduction_pct": 0, + "codeact_invoked": false, + "baseline_tools": [ + "report_intent", + "microsoft-docs-microsoft_docs_search", + "glob", + "glob", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view" + ], + "codeact_tools": [], + "min_token_reduction": -150 + }, + { + "prompt_id": "test-coverage-mcp-bloated", + "status": "valid", + "baseline_tokens": 5368, + "codeact_tokens": 1749, + "token_reduction_pct": 67.4, + "baseline_input_tokens": 273135, + "codeact_input_tokens": 152280, + "input_token_reduction_pct": 44.2, + "baseline_cost_est": 0.763358, + "codeact_cost_est": 0.406935, + "cost_reduction_pct": 46.7, + "baseline_requests": 6, + "codeact_requests": 6, + "request_reduction_pct": 0.0, + "baseline_tool_calls": 33, + "codeact_tool_calls": 3, + "tool_reduction_pct": 90.9, + "baseline_turns": 5, + "codeact_turns": 3, + "turn_reduction_pct": 40.0, + "baseline_context_bytes": 25406, + "codeact_context_bytes": 4527, + "context_reduction_pct": 82.2, + "codeact_invoked": true, + "baseline_tools": [ + "report_intent", + "glob", + "glob", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "report_intent", + "view", + "view", + "view", + "view" + ], + "codeact_tools": [ + "report_intent", + "bash", + "bash" + ], + "min_token_reduction": 40 + }, + { + "prompt_id": "mcp-docs-cross-ref-bloated", + "status": "valid", + "baseline_tokens": 2909, + "codeact_tokens": 8213, + "token_reduction_pct": -182.3, + "baseline_input_tokens": 159118, + "codeact_input_tokens": 163021, + "input_token_reduction_pct": -2.5, + "baseline_cost_est": 0.44143, + "codeact_cost_est": 0.530748, + "cost_reduction_pct": -20.2, + "baseline_requests": 6, + "codeact_requests": 6, + "request_reduction_pct": 0.0, + "baseline_tool_calls": 13, + "codeact_tool_calls": 16, + "tool_reduction_pct": -23.1, + "baseline_turns": 3, + "codeact_turns": 5, + "turn_reduction_pct": -66.7, + "baseline_context_bytes": 33161, + "codeact_context_bytes": 16510, + "context_reduction_pct": 50.2, + "codeact_invoked": true, + "baseline_tools": [ + "report_intent", + "microsoft-docs-microsoft_docs_search", + "glob", + "glob", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view" + ], + "codeact_tools": [ + "report_intent", + "bash", + "bash", + "report_intent", + "glob", + "glob", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "view", + "web_search" + ], + "min_token_reduction": -150 + } + ] +} \ No newline at end of file diff --git a/plugins/codeact/tests/run_tests.py b/plugins/codeact/tests/run_tests.py new file mode 100644 index 0000000..3301668 --- /dev/null +++ b/plugins/codeact/tests/run_tests.py @@ -0,0 +1,1118 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = [] +# /// +"""run_tests.py — Test harness for codeact plugin. + +Runs prompts through copilot CLI, captures JSONL output, extracts metrics, +and compares baseline vs codeact arms. + +Usage: + # Full end-to-end run (creates temp workspace, runs all tests, cleans up) + python3 run_tests.py all + + # Sub-runs (caller supplies workspace + plugin dir) + python3 run_tests.py functional --prompts prompts/functional.json \\ + --workspace /tmp/test --plugin-dir ./plugins/codeact + + python3 run_tests.py perf --prompts prompts/perf.json \\ + --workspace /tmp/test --plugin-dir ./plugins/codeact \\ + --min-token-reduction 40 +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +@dataclass +class RunMetrics: + """Metrics extracted from a copilot CLI JSONL run.""" + + prompt_id: str = "" + arm: str = "" + output_tokens: int = 0 + input_tokens: int = 0 + api_turns: int = 0 + premium_requests: int = 0 + api_duration_ms: int = 0 + session_duration_ms: int = 0 + tool_calls: list[dict[str, Any]] = field(default_factory=list) + tool_names: list[str] = field(default_factory=list) + assistant_text: str = "" + codeact_invoked: bool = False + codeact_evidence: list[str] = field(default_factory=list) + success: bool = False + raw_events: list[dict[str, Any]] = field(default_factory=list) + raw_stdout: str = "" + raw_stderr: str = "" + # Context bloat: total bytes of tool results returned to the conversation. + # Each turn re-sends prior results as context, so this directly correlates + # with input token cost. Lower = less context replay per turn. + tool_result_bytes: int = 0 + + +# Module-level switch flipped by --verbose. +VERBOSE = False + +# Default cap for evidence / tool-call printout. Long enough to show the +# command up to the start of inline `--code` payloads, short enough to keep +# normal-mode output scannable. --verbose disables truncation entirely. +EVIDENCE_TRUNCATE = 300 + + +def _truncate(s: str, limit: int = EVIDENCE_TRUNCATE) -> str: + if VERBOSE or len(s) <= limit: + return s + return s[: limit - 3] + "..." + + +def run_copilot( + prompt: str, + workspace: str, + plugin_dir: str | None = None, + no_custom_instructions: bool = False, + timeout: int = 180, + log_label: str | None = None, + agent: str | None = None, + disable_mcp_servers: list[str] | None = None, + allow_tools: list[str] | None = None, + deny_tools: list[str] | None = None, +) -> tuple[list[dict[str, Any]], str, str]: + """Run a prompt through copilot CLI. Returns (parsed JSONL events, stdout, stderr). + + Stdout + stderr are also written to ``/.copilot-logs/