diff --git a/.claude/settings.local.json b/.claude/settings.local.json new file mode 100644 index 000000000..657af820e --- /dev/null +++ b/.claude/settings.local.json @@ -0,0 +1,30 @@ +{ + "permissions": { + "allow": [ + "Bash(bun test:*)", + "Bash(bun run:*)", + "Skill(update-config)", + "Skill(update-config:*)", + "mcp__plugin_context7_context7__resolve-library-id", + "Bash(powershell -Command \"Get-Command bun -ErrorAction SilentlyContinue | Select-Object Source\")", + "Bash(powershell -Command \"Test-Path ''$env:USERPROFILE\\\\.local\\\\share\\\\copilot-api\\\\github_token''\")", + "Bash(powershell -Command \"nssm status CopilotProxy 2>&1\")", + "Bash(powershell -Command \"nssm get CopilotProxy AppDirectory 2>&1; nssm get CopilotProxy Application 2>&1; nssm get CopilotProxy AppParameters 2>&1; nssm get CopilotProxy AppEnvironmentExtra 2>&1\")", + "Bash(powershell -Command \"if \\(Test-Path ''C:\\\\Users\\\\ttbasil\\\\Desktop\\\\Projects\\\\PublicProjects\\\\copilot-api\\\\logs''\\) { Get-ChildItem ''C:\\\\Users\\\\ttbasil\\\\Desktop\\\\Projects\\\\PublicProjects\\\\copilot-api\\\\logs'' } else { ''logs dir not found'' }\")", + "Bash(powershell -Command \"Get-WinEvent -LogName ''System'' -FilterXPath ''*[System[Provider[@Name=\"\"nssm\"\"] or Provider[@Name=\"\"Service Control Manager\"\"]] and System[TimeCreated[timediff\\(@SystemTime\\) <= 604800000]]]'' -MaxEvents 30 2>&1 | Select-Object TimeCreated, Id, Message | Format-List\")", + "Bash(powershell -Command \"Get-WinEvent -LogName ''System'' -FilterXPath ''*[System[Provider[@Name=\"\"nssm\"\"]]]'' -MaxEvents 20 2>&1 | Select-Object TimeCreated, Id, Message | Format-List\")", + "Bash(powershell -Command \"Get-WinEvent -LogName ''System'' -FilterXPath ''*[System[Provider[@Name=''''nssm'''']]]'' -MaxEvents 20 2>&1\")", + "Bash(powershell -Command \"nssm get CopilotProxy AppStdout 2>&1; nssm get CopilotProxy AppStderr 2>&1\")", + "Bash(powershell -Command \"Test-Path ''C:\\\\Users\\\\ttbasil\\\\.local\\\\share\\\\copilot-api\\\\github_token''\")", + "Bash(powershell -Command \"nssm stop CopilotProxy 2>&1\")", + "Bash(powershell -Command \"Start-Process -FilePath ''nssm'' -ArgumentList ''set CopilotProxy Application C:\\\\Users\\\\ttbasil\\\\Desktop\\\\Projects\\\\PublicProjects\\\\copilot-api\\\\start-service.bat'' -Verb RunAs -Wait -WindowStyle Hidden 2>&1\")", + "Bash(powershell -Command \"Start-Process -FilePath ''nssm'' -ArgumentList ''get CopilotProxy Application'' -Verb RunAs -Wait -RedirectStandardOutput ''C:\\\\Users\\\\ttbasil\\\\Desktop\\\\nssm_check.txt'' -WindowStyle Hidden; Get-Content ''C:\\\\Users\\\\ttbasil\\\\Desktop\\\\nssm_check.txt''\")", + "Bash(powershell -Command \"if \\(Test-Path ''C:\\\\Users\\\\ttbasil\\\\Desktop\\\\nssm_output.txt''\\) { Get-Content ''C:\\\\Users\\\\ttbasil\\\\Desktop\\\\nssm_output.txt'' } else { ''Output file not created'' }\")", + "Bash(npx eslint:*)", + "Bash(curl -s http://localhost:4141/v1/models)", + "Bash(python -m json.tool)", + "Bash(powershell -Command \"Remove-Item -Recurse -Force ''c:\\\\Users\\\\ttbasil\\\\Desktop\\\\Projects\\\\PublicProjects\\\\copilot-api\\\\test-output''\")", + "Bash(bunx eslint *)" + ] + } +} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 000000000..0958f1b86 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,26 @@ +name: Deploy to Server + +on: + push: + branches: [master] + workflow_dispatch: + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Deploy via SSH + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.SERVER_HOST }} + username: ${{ secrets.SERVER_USER }} + password: ${{ secrets.SERVER_PASSWORD }} + script: | + export PATH=/root/.bun/bin:$PATH + cd /root/projects/copilot-api + git pull origin master + bun install + bun run build + systemctl restart copilot-api + sleep 3 + systemctl is-active copilot-api diff --git a/.gitignore b/.gitignore index 577a4f199..9c9c6a76f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,10 @@ node_modules/ # local env *.local +.env + +# TLS certificates / private keys (self-signed, machine-local) +certs/ # aider .aider* @@ -11,4 +15,18 @@ node_modules/ .eslintcache # build output -dist/ \ No newline at end of file +dist/ + + +# Ignore pycharm, webstorm, vscode files +.idea/ +.vscode/ +# Ignore log files +*.log + +# git worktrees +.worktrees/ + + +# Codex logs +.codex-logs/ \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..fdb87167d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,94 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +A reverse-engineered proxy that exposes the GitHub Copilot API as an OpenAI-compatible and Anthropic-compatible HTTP server. It handles authentication with GitHub's device flow, manages Copilot token refresh, and translates between API formats. + +## Commands + +```sh +# Install dependencies +bun install + +# Development (watch mode) +bun run dev + +# Production +bun run start + +# Build (compiles to dist/ via tsdown) +bun run build + +# Lint +bun run lint # lint staged files +bun run lint:all # lint entire project + +# Type check +bun run typecheck + +# Find unused exports/dependencies +bun run knip +``` + +## Architecture + +### Entry Points +- `src/main.ts` — CLI entry point using `citty`. Registers subcommands: `start`, `auth`, `check-usage`, `debug` +- `src/start.ts` — Implements the `start` command: sets up state, authenticates, caches models/VSCode version, starts the Hono server via `srvx` +- `src/auth.ts` — Standalone GitHub OAuth device flow (without starting server) + +### HTTP Server (`src/server.ts`) +Built with **Hono**. Routes are mounted at both `/` and `/v1/` prefixes for OpenAI compatibility: +- `POST /v1/chat/completions` → OpenAI-compatible chat completions +- `GET /v1/models` → list available Copilot models +- `POST /v1/embeddings` → embeddings +- `POST /v1/messages` + `POST /v1/messages/count_tokens` → Anthropic-compatible endpoints +- `GET /usage` → Copilot quota/usage stats +- `GET /token` → current Copilot token + +### Anthropic ↔ OpenAI Translation (`src/routes/messages/`) +The messages route translates Anthropic API format to OpenAI format before forwarding to Copilot, then translates the response back: +- `non-stream-translation.ts` — bidirectional translation for non-streaming responses +- `stream-translation.ts` — translates OpenAI SSE chunks to Anthropic SSE event format +- `anthropic-types.ts` — TypeScript types for Anthropic request/response shapes + +### Global State (`src/lib/state.ts`) +A single mutable `state` object holds runtime configuration and tokens: +- `githubToken` / `copilotToken` — authentication tokens +- `accountType` — `individual` | `business` | `enterprise` (affects Copilot API base URL) +- `models` / `vsCodeVersion` — cached at startup +- `manualApprove`, `rateLimitSeconds`, `rateLimitWait`, `showToken` — feature flags + +### Authentication Flow (`src/lib/token.ts`) +1. Reads GitHub token from `~/.local/share/copilot-api/github_token` +2. If missing, triggers GitHub device-code OAuth flow +3. Exchanges GitHub token for a short-lived Copilot token via `src/services/github/get-copilot-token.ts` +4. Copilot token auto-refreshes on an interval (`refresh_in - 60` seconds) + +### API Config & Headers (`src/lib/api-config.ts`) +Constructs headers that impersonate VS Code's Copilot Chat extension (required by GitHub's Copilot API). Copilot base URL varies by `accountType`: +- individual: `https://api.githubcopilot.com` +- business/enterprise: `https://api.{accountType}.githubcopilot.com` + +### Services (`src/services/`) +- `copilot/create-chat-completions.ts` — core fetch to Copilot's `/chat/completions`; detects vision requests and agent vs. user calls via `X-Initiator` header +- `copilot/get-models.ts` / `create-embeddings.ts` — other Copilot API calls +- `github/` — device code OAuth, access token polling, Copilot token exchange, user info, usage stats + +### Middleware / Lib +- `src/lib/rate-limit.ts` — enforces `rateLimitSeconds` between requests; can wait or error +- `src/lib/approval.ts` — interactive CLI prompt for `--manual` mode +- `src/lib/tokenizer.ts` — token counting using `gpt-tokenizer` +- `src/lib/proxy.ts` — initializes HTTP proxy from env vars (`HTTP_PROXY`, etc.) via `proxy-from-env` + `undici` +- `src/lib/shell.ts` — generates shell `env VAR=value command` strings for `--claude-code` flag + +## Key Conventions + +- **Path alias**: `~/` maps to `src/` (configured in tsconfig/tsdown), so imports use `~/lib/state` instead of relative paths +- **Runtime**: Bun is required (not Node.js); uses Bun's native APIs and `@types/bun` +- **Logging**: `consola` throughout — use `consola.debug` for verbose-only output, `consola.info` for normal output +- **Error handling**: `HTTPError` in `src/lib/error.ts` wraps fetch response errors +- **Validation**: `zod` for request payload validation; `tiny-invariant` for runtime assertions +- **Pre-commit**: `lint-staged` runs ESLint auto-fix on all staged files via `simple-git-hooks` diff --git "a/C\357\200\272Usersttbasil.claudeclaude-notify-signalsstop" "b/C\357\200\272Usersttbasil.claudeclaude-notify-signalsstop" new file mode 100644 index 000000000..e69de29bb diff --git a/README.md b/README.md index 0d36c13c9..5d98ba22b 100644 --- a/README.md +++ b/README.md @@ -1,306 +1,455 @@ -# Copilot API Proxy +# Copilot API -> [!WARNING] -> This is a reverse-engineered proxy of GitHub Copilot API. It is not supported by GitHub, and may break unexpectedly. Use at your own risk. +A reverse-engineered proxy that turns the GitHub Copilot API into fully compatible **OpenAI** and **Anthropic** endpoints — letting you use Copilot with any tool that speaks either protocol, including [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview). -> [!WARNING] -> **GitHub Security Notice:** -> Excessive automated or scripted use of Copilot (including rapid or bulk requests, such as via automated tools) may trigger GitHub's abuse-detection systems. -> You may receive a warning from GitHub Security, and further anomalous activity could result in temporary suspension of your Copilot access. -> -> GitHub prohibits use of their servers for excessive automated bulk activity or any activity that places undue burden on their infrastructure. -> -> Please review: -> -> - [GitHub Acceptable Use Policies](https://docs.github.com/site-policy/acceptable-use-policies/github-acceptable-use-policies#4-spam-and-inauthentic-activity-on-github) -> - [GitHub Copilot Terms](https://docs.github.com/site-policy/github-terms/github-terms-for-additional-products-and-features#github-copilot) -> -> Use this proxy responsibly to avoid account restrictions. +## Features -[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/E1E519XS7W) +- **Triple API Compatibility** — OpenAI Chat Completions, OpenAI Responses API, and Anthropic Messages API, all backed by GitHub Copilot +- **Claude Code Integration** — Interactive model selector (`--claude-code`) copies a ready-to-paste launch command; full support for thinking blocks, typed tools, token counting, and auto-compaction +- **Automatic Endpoint Routing** — Models that only support `/responses` (e.g. gpt-5.4-mini) are transparently routed through the Responses API with bidirectional translation +- **Web Search** — Two-pass search via [Tavily](https://tavily.com) (free) or [Brave Search](https://brave.com/search/api/) — the proxy intercepts search tool calls, fetches live results, and injects them for the model +- **Smart Context Management** — Auto-switches to the largest-context model when token count exceeds the requested model's window; image stripping cascade on 413 errors to trigger compaction +- **Rate Limiting** — Interval-based and sliding-window burst limiting with configurable wait-or-reject behavior +- **Usage Dashboard** — Web UI showing Copilot quota, premium interactions, and detailed usage stats +- **Manual Approval Mode** — Interactively approve/deny each request (`--manual`) +- **Docker & npx** — Run anywhere: from source, via `npx copilot-api@latest`, or as a Docker container +- **Proxy Support** — HTTP/HTTPS proxy via environment variables with per-URL routing +- **Native HTTPS** — Serve over TLS with a self-signed cert (`TLS_CERT`/`TLS_KEY`) for secure LAN access — no reverse proxy needed ---- +## Demo -**Note:** If you are using [opencode](https://github.com/sst/opencode), you do not need this project. Opencode supports GitHub Copilot provider out of the box. +https://github.com/user-attachments/assets/7654b383-669d-4eb9-b23c-06d7aefee8c5 ---- +## Architecture -## Project Overview +### High-Level Request Flow -A reverse-engineered proxy for the GitHub Copilot API that exposes it as an OpenAI and Anthropic compatible service. This allows you to use GitHub Copilot with any tool that supports the OpenAI Chat Completions API or the Anthropic Messages API, including to power [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview). +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ Clients │ +│ Claude Code · Cursor · OpenAI SDK · Anthropic SDK · Any HTTP │ +└──────────┬──────────────────┬──────────────────┬────────────────────┘ + │ │ │ + ▼ ▼ ▼ +┌──────────────────┐ ┌───────────────┐ ┌─────────────────────┐ +│ POST /v1/messages│ │ POST /v1/chat │ │ POST /v1/responses │ +│ (Anthropic API) │ │ /completions │ │ (Responses API) │ +└────────┬─────────┘ │ (OpenAI API) │ └──────────┬──────────┘ + │ └───────┬───────┘ │ + ▼ │ ▼ +┌──────────────────┐ │ ┌──────────────────────┐ +│ Anthropic→OpenAI │ │ │ Responses↔CC │ +│ Translation │ │ │ Translation │ +│ (bidirectional) │ │ │ (bidirectional) │ +└────────┬─────────┘ │ └──────────┬───────────┘ + │ │ │ + ▼ ▼ ▼ +┌─────────────────────────────────────────────────────────────────────┐ +│ Middleware Pipeline │ +│ Rate Limiter → Burst Limiter → Manual Approval → Token Counter │ +│ → Model Selector → Web Search Interceptor → Image Validator │ +└──────────────────────────────┬──────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────┐ +│ Copilot Service Layer │ +│ ┌────────────────────────┐ ┌────────────────────────────────┐ │ +│ │ POST /chat/completions │ │ POST /responses │ │ +│ │ (default endpoint) │ │ (gpt-5.x, o-series models) │ │ +│ └────────────┬───────────┘ └────────────────┬───────────────┘ │ +│ └────────────────┬──────────────┘ │ +│ ▼ │ +│ api.githubcopilot.com │ +│ api.business.githubcopilot.com │ +│ api.enterprise.githubcopilot.com │ +└─────────────────────────────────────────────────────────────────────┘ +``` -## Features +### Translation Layers -- **OpenAI & Anthropic Compatibility**: Exposes GitHub Copilot as an OpenAI-compatible (`/v1/chat/completions`, `/v1/models`, `/v1/embeddings`) and Anthropic-compatible (`/v1/messages`) API. -- **Claude Code Integration**: Easily configure and launch [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) to use Copilot as its backend with a simple command-line flag (`--claude-code`). -- **Usage Dashboard**: A web-based dashboard to monitor your Copilot API usage, view quotas, and see detailed statistics. -- **Rate Limit Control**: Manage API usage with rate-limiting options (`--rate-limit`) and a waiting mechanism (`--wait`) to prevent errors from rapid requests. -- **Manual Request Approval**: Manually approve or deny each API request for fine-grained control over usage (`--manual`). -- **Token Visibility**: Option to display GitHub and Copilot tokens during authentication and refresh for debugging (`--show-token`). -- **Flexible Authentication**: Authenticate interactively or provide a GitHub token directly, suitable for CI/CD environments. -- **Support for Different Account Types**: Works with individual, business, and enterprise GitHub Copilot plans. +The proxy maintains three API protocol translators that convert between formats in real time, for both streaming and non-streaming responses: -## Demo +``` +┌─────────────────────────────────────────────────────────┐ +│ Anthropic Messages API │ +│ ┌─────────────────────────────────────────────────┐ │ +│ │ Request: Anthropic → OpenAI │ │ +│ │ • System blocks → system message │ │ +│ │ • Content blocks (text, image, doc, tool_result)│ │ +│ │ • Thinking blocks → reasoning_content │ │ +│ │ • Typed tools (bash, text_editor, web_search) │ │ +│ │ • Tool choice (auto/any/tool/none) │ │ +│ │ • Model name normalization │ │ +│ │ • Tool result compression (>20K chars) │ │ +│ │ • Image validation & stripping cascade │ │ +│ ├─────────────────────────────────────────────────┤ │ +│ │ Response: OpenAI → Anthropic │ │ +│ │ • SSE: message_start → content_block_start → │ │ +│ │ content_block_delta → content_block_stop → │ │ +│ │ message_delta → message_stop │ │ +│ │ • reasoning_content → thinking blocks │ │ +│ │ • Tool calls → tool_use content blocks │ │ +│ │ • Truncated tool call detection │ │ +│ │ • Deferred finish_reason (waits for usage) │ │ +│ │ • 10s keepalive pings, 90s stall timeout │ │ +│ └─────────────────────────────────────────────────┘ │ +├─────────────────────────────────────────────────────────┤ +│ Responses API ↔ Chat Completions │ +│ ┌─────────────────────────────────────────────────┐ │ +│ │ • Auto-routes models by supported_endpoints │ │ +│ │ • Claude models → Chat Completions translation │ │ +│ │ • gpt-5/o-series → Responses API translation │ │ +│ │ • Streaming event translation both directions │ │ +│ │ • JSON repair for truncated tool arguments │ │ +│ │ • Reasoning/thinking delta handling │ │ +│ └─────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────┘ +``` -https://github.com/user-attachments/assets/7654b383-669d-4eb9-b23c-06d7aefee8c5 +### Authentication Flow + +``` +┌──────────┐ Device Code ┌──────────┐ Poll Token ┌──────────┐ +│ Client │ ───────────────► │ GitHub │ ───────────────► │ GitHub │ +│ (CLI) │ ◄─────────────── │ OAuth │ ◄─────────────── │ Token │ +│ │ user_code + │ Device │ access_token │ (PAT) │ +│ │ verification_url│ Flow │ │ │ +└──────────┘ └──────────┘ └────┬─────┘ + │ + Stored at │ + ~/.local/share/copilot-api/ │ + github_token │ + │ + ▼ +┌──────────┐ Auto-refresh ┌──────────────────────────────────────────┐ +│ Copilot │ ◄────────────── │ GET /copilot_internal/v2/token │ +│ JWT │ (refresh_in │ Authorization: token │ +│ Token │ - 60 seconds) │ → returns JWT with expiry │ +└──────────┘ └──────────────────────────────────────────┘ +``` + +### Web Search Architecture + +``` +┌──────────────────────────────────────────────────────────────────┐ +│ Two-Pass Web Search Flow │ +│ │ +│ Client Request (with web_search tool) │ +│ │ │ +│ ▼ │ +│ ┌─────────────┐ Is it a web search? ┌────────────────────┐ │ +│ │ Interceptor │ ─────────────────────► │ Pass 1: Non-stream │ │ +│ │ Detection │ typed tool or │ call to Copilot │ │ +│ │ │ recognized name │ (asks what to │ │ +│ └──────────────┘ │ search) │ │ +│ └────────┬───────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────┐ │ +│ │ Execute Search │ │ +│ │ Tavily (preferred) │ │ +│ │ or Brave Search │ │ +│ │ 5s timeout, max 5 results │ │ +│ └────────────┬───────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌────────────────────────────┐ │ +│ │ Pass 2: Full call │ │ +│ │ Injects search results │ │ +│ │ tool_choice: "none" │ │ +│ │ (original stream mode) │ │ +│ └────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────┘ +``` + +### Project Structure + +``` +src/ +├── main.ts # CLI entry point (citty subcommands) +├── start.ts # Server startup, auth, caching +├── auth.ts # Standalone OAuth device flow +├── server.ts # Hono app, route registration, middleware +│ +├── routes/ +│ ├── completions/ # POST /v1/chat/completions +│ │ └── handler.ts +│ ├── responses/ # POST /v1/responses (Responses API) +│ │ └── handler.ts +│ ├── messages/ # POST /v1/messages (Anthropic API) +│ │ ├── handler.ts # request orchestration, retries, error handling +│ │ ├── non-stream-translation.ts # Anthropic ↔ OpenAI (non-streaming) +│ │ ├── stream-translation.ts # Anthropic ↔ OpenAI (SSE streaming) +│ │ ├── count-tokens.ts # /v1/messages/count_tokens +│ │ └── anthropic-types.ts # TypeScript types +│ ├── models/ # GET /v1/models +│ ├── embeddings/ # POST /v1/embeddings +│ ├── usage/ # GET /usage +│ └── token/ # GET /token +│ +├── services/ +│ ├── copilot/ +│ │ ├── create-chat-completions.ts # Core fetch to Copilot API +│ │ ├── create-embeddings.ts +│ │ ├── get-models.ts # Model list + context window helpers +│ │ └── responses-translation.ts # Responses ↔ Chat Completions translation +│ ├── github/ +│ │ ├── get-copilot-token.ts # JWT token exchange + auto-refresh +│ │ ├── get-copilot-usage.ts # Quota/usage stats +│ │ ├── get-device-code.ts # OAuth device flow +│ │ ├── get-user.ts # GitHub user info +│ │ └── poll-access-token.ts # OAuth polling +│ └── web-search/ +│ ├── interceptor.ts # Two-pass search orchestration +│ ├── brave.ts # Brave Search provider +│ ├── tavily.ts # Tavily provider +│ ├── system-prompt.ts # Search instruction injection +│ └── tool-definition.ts # Tool detection & definition +│ +└── lib/ + ├── api-config.ts # Copilot API URLs & VS Code impersonation headers + ├── error.ts # HTTPError, Anthropic error formatting + ├── model-selector.ts # Auto-switch to largest-context model + ├── rate-limit.ts # Interval + burst rate limiters + ├── request-logger.ts # Colored terminal logging middleware + ├── session-id.ts # Claude Code session ID extraction + ├── shell.ts # Cross-shell env var generation + ├── state.ts # Global mutable runtime state + ├── token.ts # Token persistence & refresh + ├── tokenizer.ts # gpt-tokenizer token counting + ├── proxy.ts # HTTP proxy support (undici) + ├── approval.ts # Interactive request approval + └── paths.ts # Data directory paths +``` ## Prerequisites -- Bun (>= 1.2.x) -- GitHub account with Copilot subscription (individual, business, or enterprise) +- [Bun](https://bun.sh) >= 1.2.x +- GitHub account with an active Copilot subscription (Individual, Business, or Enterprise) ## Installation -To install dependencies, run: - ```sh bun install ``` -## Using with Docker - -Build image +## Quick Start ```sh -docker build -t copilot-api . +# Via npx (no clone needed) +npx copilot-api@latest start + +# From source +bun run dev # development with watch mode +bun run start # production ``` -Run the container +On first run, the proxy triggers GitHub's device-code OAuth flow — follow the on-screen URL to authorize. -```sh -# Create a directory on your host to persist the GitHub token and related data -mkdir -p ./copilot-data +### Quick Start (Windows) -# Run the container with a bind mount to persist the token -# This ensures your authentication survives container restarts +The included `start.bat` handles everything automatically: -docker run -p 4141:4141 -v $(pwd)/copilot-data:/root/.local/share/copilot-api copilot-api -``` +1. Create a `.env` file in the project root (see [Environment Variables](#environment-variables)) +2. Double-click `start.bat` or run it from a terminal -> **Note:** -> The GitHub token and related data will be stored in `copilot-data` on your host. This is mapped to `/root/.local/share/copilot-api` inside the container, ensuring persistence across restarts. +The script will load env vars, build if needed, show the active search provider, start the server, and open the Usage Dashboard in your browser. -### Docker with Environment Variables +> Need HTTPS for access from another machine on your network? See [HTTPS / TLS (LAN Access)](#https--tls-lan-access). -You can pass the GitHub token directly to the container using environment variables: +## Environment Variables -```sh -# Build with GitHub token -docker build --build-arg GH_TOKEN=your_github_token_here -t copilot-api . +Create a `.env` file in the project root. It is gitignored. -# Run with GitHub token -docker run -p 4141:4141 -e GH_TOKEN=your_github_token_here copilot-api +```env +# Web Search (optional — pick one) +TAVILY_API_KEY=tvly-... # Preferred: free at tavily.com (1,000 req/mo) +BRAVE_API_KEY=BSA... # Alternative: brave.com/search/api -# Run with additional options -docker run -p 4141:4141 -e GH_TOKEN=your_token copilot-api start --verbose --port 4141 +# Proxy (optional) +HTTP_PROXY=http://proxy:8080 +HTTPS_PROXY=http://proxy:8080 + +# HTTPS / TLS (optional — enables HTTPS when both are set) +TLS_CERT=certs/server.crt # Path to certificate (PEM) or inline PEM +TLS_KEY=certs/server.key # Path to private key (PEM) or inline PEM +TLS_PASSPHRASE= # Private key passphrase (only if encrypted) ``` -### Docker Compose Example +> **Provider priority:** If both keys are set, Tavily is used. -```yaml -version: "3.8" -services: - copilot-api: - build: . - ports: - - "4141:4141" - environment: - - GH_TOKEN=your_github_token_here - restart: unless-stopped -``` +> **TLS:** Set **both** `TLS_CERT` and `TLS_KEY` to serve over HTTPS; leave both unset for plain HTTP (default). See [HTTPS / TLS](#https--tls-lan-access). -The Docker image includes: +## HTTPS / TLS (LAN Access) -- Multi-stage build for optimized image size -- Non-root user for enhanced security -- Health check for container monitoring -- Pinned base image version for reproducible builds +By default the proxy serves plain **HTTP**, which is fine for `localhost`. If another machine on your network must reach the proxy over **HTTPS** (some clients refuse plain HTTP), the server can terminate TLS itself — no reverse proxy required. -## Using with npx +HTTPS turns on automatically when both `TLS_CERT` and `TLS_KEY` are set (via `.env` or the environment). Each value may be a **file path** or an **inline PEM** string. If only one is set, the server exits with an error. When TLS is active the console prints `TLS enabled — server will listen over HTTPS` and the listening URL switches to `https://`. -You can run the project directly using npx: +### Quick Start (Windows) -```sh -npx copilot-api@latest start -``` +The repo ships batch files that wire this up end to end: -With options: +1. **Generate a self-signed certificate** — run `generate-cert.bat`. It writes `certs/server.crt` and `certs/server.key`, with the certificate valid for your PC's LAN IP and hostname (Subject Alternative Names). Edit the `LAN_IP` and `HOSTNAME` values at the top of the script if yours differ. +2. **Start the server** — run `start.bat` (port 4141), `start-openai.bat` (port 1515), or `start-controlled.bat` (port 3131). Each script points `TLS_CERT`/`TLS_KEY` at `certs/` and refuses to start if the certificate is missing. +3. **Connect from the other machine** at `https://:` (e.g. `https://192.168.0.105:4141`). -```sh -npx copilot-api@latest start --port 8080 -``` +> The `certs/` directory is gitignored — your private key is never committed. -For authentication only: +### Generating a certificate manually + +Any tool that produces a PEM cert/key pair works. With OpenSSL, the key detail is that the **Subject Alternative Name (SAN) must list the exact IP or hostname** the client connects to — modern HTTPS clients validate against the SAN, not the Common Name: ```sh -npx copilot-api@latest auth +openssl req -x509 -newkey rsa:2048 -sha256 -days 825 -nodes \ + -keyout certs/server.key -out certs/server.crt \ + -subj "/CN=YOUR-HOSTNAME" \ + -addext "subjectAltName=IP:192.168.0.105,IP:127.0.0.1,DNS:YOUR-HOSTNAME,DNS:localhost" ``` -## Command Structure +Verify the SANs landed correctly: -Copilot API now uses a subcommand structure with these main commands: +```sh +openssl x509 -in certs/server.crt -noout -subject -ext subjectAltName +``` -- `start`: Start the Copilot API server. This command will also handle authentication if needed. -- `auth`: Run GitHub authentication flow without starting the server. This is typically used if you need to generate a token for use with the `--github-token` option, especially in non-interactive environments. -- `check-usage`: Show your current GitHub Copilot usage and quota information directly in the terminal (no server required). -- `debug`: Display diagnostic information including version, runtime details, file paths, and authentication status. Useful for troubleshooting and support. +### Trusting the certificate on the client -## Command Line Options +A self-signed certificate is rejected until the **connecting machine trusts it**. Copy `certs/server.crt` to that machine and install it as a trusted root: -### Start Command Options +- **Windows** (admin terminal): `certutil -addstore -f Root server.crt` +- **Node-based clients:** point `NODE_EXTRA_CA_CERTS` at the `.crt` file +- **curl / OpenSSL tools:** pass `--cacert server.crt` or set `SSL_CERT_FILE` -The following command line options are available for the `start` command: - -| Option | Description | Default | Alias | -| -------------- | ----------------------------------------------------------------------------- | ---------- | ----- | -| --port | Port to listen on | 4141 | -p | -| --verbose | Enable verbose logging | false | -v | -| --account-type | Account type to use (individual, business, enterprise) | individual | -a | -| --manual | Enable manual request approval | false | none | -| --rate-limit | Rate limit in seconds between requests | none | -r | -| --wait | Wait instead of error when rate limit is hit | false | -w | -| --github-token | Provide GitHub token directly (must be generated using the `auth` subcommand) | none | -g | -| --claude-code | Generate a command to launch Claude Code with Copilot API config | false | -c | -| --show-token | Show GitHub and Copilot tokens on fetch and refresh | false | none | -| --proxy-env | Initialize proxy from environment variables | false | none | +### Firewall -### Auth Command Options +Windows Firewall blocks inbound connections by default. Allow the port on the machine running the proxy: -| Option | Description | Default | Alias | -| ------------ | ------------------------- | ------- | ----- | -| --verbose | Enable verbose logging | false | -v | -| --show-token | Show GitHub token on auth | false | none | +```sh +netsh advfirewall firewall add rule name="copilot-api 4141" dir=in action=allow protocol=TCP localport=4141 +``` -### Debug Command Options +> **DHCP note:** The IP is baked into the certificate's SANs. If your LAN IP changes, update `LAN_IP` in `generate-cert.bat` and the start scripts, regenerate the certificate, and re-trust it on the client. A DHCP reservation (or connecting by hostname) avoids this. -| Option | Description | Default | Alias | -| ------ | ------------------------- | ------- | ----- | -| --json | Output debug info as JSON | false | none | +### nginx / public reverse proxy -## API Endpoints +If you expose `copilot-api` through nginx, raise nginx's request body limit for the API location. Codex `/v1/responses` turns can include long conversation history, tool output, and images. With nginx's default body limit, nginx returns a raw HTML `413 Request Entity Too Large` before this server receives the request, so the `/v1/responses` handler cannot emit the `response.failed` / `context_length_exceeded` event that Codex uses to auto-compact. -The server exposes several endpoints to interact with the Copilot API. It provides OpenAI-compatible endpoints and now also includes support for Anthropic-compatible endpoints, allowing for greater flexibility with different tools and services. +Use the sample in `deploy/nginx/copilot-api.conf.example`, or add the equivalent directive to your `server` or `location` block: -### OpenAI Compatible Endpoints - -These endpoints mimic the OpenAI API structure. +```nginx +client_max_body_size 256m; +``` -| Endpoint | Method | Description | -| --------------------------- | ------ | --------------------------------------------------------- | -| `POST /v1/chat/completions` | `POST` | Creates a model response for the given chat conversation. | -| `GET /v1/models` | `GET` | Lists the currently available models. | -| `POST /v1/embeddings` | `POST` | Creates an embedding vector representing the input text. | +After changing nginx config, run: -### Anthropic Compatible Endpoints +```sh +nginx -t +sudo systemctl reload nginx +``` -These endpoints are designed to be compatible with the Anthropic Messages API. +## Command Structure -| Endpoint | Method | Description | -| -------------------------------- | ------ | ------------------------------------------------------------ | -| `POST /v1/messages` | `POST` | Creates a model response for a given conversation. | -| `POST /v1/messages/count_tokens` | `POST` | Calculates the number of tokens for a given set of messages. | +| Command | Description | +|---|---| +| `start` | Start the proxy server (handles auth if needed) | +| `auth` | Run GitHub OAuth flow without starting the server | +| `check-usage` | Show Copilot quota/usage in the terminal | +| `debug` | Display version, runtime, paths, and auth status | -### Usage Monitoring Endpoints +### Start Command Options -New endpoints for monitoring your Copilot usage and quotas. +| Option | Alias | Default | Description | +|---|---|---|---| +| `--port` | `-p` | `4141` | Port to listen on | +| `--verbose` | `-v` | `false` | Enable verbose logging | +| `--account-type` | `-a` | `individual` | `individual`, `business`, or `enterprise` | +| `--manual` | — | `false` | Require interactive approval for each request | +| `--rate-limit` | `-r` | — | Minimum seconds between requests | +| `--wait` | `-w` | `false` | Queue requests instead of rejecting when rate limited | +| `--burst-count` | — | — | Max requests in burst window | +| `--burst-window` | — | — | Burst window duration in seconds | +| `--github-token` | `-g` | — | Provide a pre-existing GitHub token (skip OAuth) | +| `--claude-code` | `-c` | `false` | Interactive Claude Code setup wizard | +| `--show-token` | — | `false` | Display tokens in logs for debugging | +| `--proxy-env` | — | `false` | Use `HTTP_PROXY`/`HTTPS_PROXY` from environment | -| Endpoint | Method | Description | -| ------------ | ------ | ------------------------------------------------------------ | -| `GET /usage` | `GET` | Get detailed Copilot usage statistics and quota information. | -| `GET /token` | `GET` | Get the current Copilot token being used by the API. | +### Auth Command Options -## Example Usage +| Option | Alias | Default | Description | +|---|---|---|---| +| `--verbose` | `-v` | `false` | Verbose logging | +| `--show-token` | — | `false` | Show token after auth | -Using with npx: +### Debug Command Options -```sh -# Basic usage with start command -npx copilot-api@latest start +| Option | Default | Description | +|---|---|---| +| `--json` | `false` | Output as JSON | -# Run on custom port with verbose logging -npx copilot-api@latest start --port 8080 --verbose +## API Endpoints -# Use with a business plan GitHub account -npx copilot-api@latest start --account-type business +### OpenAI Compatible -# Use with an enterprise plan GitHub account -npx copilot-api@latest start --account-type enterprise +| Endpoint | Method | Description | +|---|---|---| +| `/v1/chat/completions` | POST | Chat completions (streaming & non-streaming) | +| `/v1/responses` | POST | OpenAI Responses API | +| `/v1/models` | GET | List available models (with context window metadata) | +| `/v1/embeddings` | POST | Generate embedding vectors | -# Enable manual approval for each request -npx copilot-api@latest start --manual +### Anthropic Compatible -# Set rate limit to 30 seconds between requests -npx copilot-api@latest start --rate-limit 30 +| Endpoint | Method | Description | +|---|---|---| +| `/v1/messages` | POST | Anthropic Messages API (full protocol translation) | +| `/v1/messages/count_tokens` | POST | Token counting with model-specific scaling | -# Wait instead of error when rate limit is hit -npx copilot-api@latest start --rate-limit 30 --wait +### Utility -# Provide GitHub token directly -npx copilot-api@latest start --github-token ghp_YOUR_TOKEN_HERE +| Endpoint | Method | Description | +|---|---|---| +| `/usage` | GET | Copilot quota and usage statistics | +| `/token` | GET | Current Copilot JWT token | -# Run only the auth flow -npx copilot-api@latest auth +> All OpenAI endpoints are also available without the `/v1/` prefix. The Responses API is available at both `/responses` and `/v1/responses`. -# Run auth flow with verbose logging -npx copilot-api@latest auth --verbose +## Web Search -# Show your Copilot usage/quota in the terminal (no server needed) -npx copilot-api@latest check-usage +The proxy performs real-time web searches using a two-pass architecture: -# Display debug information for troubleshooting -npx copilot-api@latest debug +1. **Pass 1** — Copilot determines what to search (non-streaming call) +2. **Search** — Proxy fetches results from Tavily or Brave (5s timeout, max 5 results) +3. **Pass 2** — Copilot generates a response using the injected search results -# Display debug information in JSON format -npx copilot-api@latest debug --json +Each web search uses 2–3 internal Copilot API calls. -# Initialize proxy from environment variables (HTTP_PROXY, HTTPS_PROXY, etc.) -npx copilot-api@latest start --proxy-env -``` +### Setup -## Using the Usage Viewer +**Tavily (Recommended, Free)** — Sign up at [app.tavily.com](https://app.tavily.com), add `TAVILY_API_KEY` to `.env` -After starting the server, a URL to the Copilot Usage Dashboard will be displayed in your console. This dashboard is a web interface for monitoring your API usage. +**Brave Search** — Sign up at [brave.com/search/api](https://brave.com/search/api/), add `BRAVE_API_KEY` to `.env` -1. Start the server. For example, using npx: - ```sh - npx copilot-api@latest start - ``` -2. The server will output a URL to the usage viewer. Copy and paste this URL into your browser. It will look something like this: - `https://ericc-ch.github.io/copilot-api?endpoint=http://localhost:4141/usage` - - If you use the `start.bat` script on Windows, this page will open automatically. +### Trigger Conditions -The dashboard provides a user-friendly interface to view your Copilot usage data: +- **Path 1 (zero-cost):** Client sends a typed Anthropic web search tool (`type: "web_search_20250305"`) +- **Path 2 (preflight):** Client sends a tool with a recognized name and the last user message appears to need real-time info -- **API Endpoint URL**: The dashboard is pre-configured to fetch data from your local server endpoint via the URL query parameter. You can change this URL to point to any other compatible API endpoint. -- **Fetch Data**: Click the "Fetch" button to load or refresh the usage data. The dashboard will automatically fetch data on load. -- **Usage Quotas**: View a summary of your usage quotas for different services like Chat and Completions, displayed with progress bars for a quick overview. -- **Detailed Information**: See the full JSON response from the API for a detailed breakdown of all available usage statistics. -- **URL-based Configuration**: You can also specify the API endpoint directly in the URL using a query parameter. This is useful for bookmarks or sharing links. For example: - `https://ericc-ch.github.io/copilot-api?endpoint=http://your-api-server/usage` +Recognized names: `web_search`, `internet_search`, `brave_search`, `bing_search`, `google_search`, `find_online`, `internet_research` ## Using with Claude Code -This proxy can be used to power [Claude Code](https://docs.anthropic.com/en/claude-code), an experimental conversational AI assistant for developers from Anthropic. - -There are two ways to configure Claude Code to use this proxy: - -### Interactive Setup with `--claude-code` flag - -To get started, run the `start` command with the `--claude-code` flag: +### Interactive Setup ```sh npx copilot-api@latest start --claude-code ``` -You will be prompted to select a primary model and a "small, fast" model for background tasks. After selecting the models, a command will be copied to your clipboard. This command sets the necessary environment variables for Claude Code to use the proxy. +Select a primary model and a small/fast model. A ready-to-paste launch command is copied to your clipboard. -Paste and run this command in a new terminal to launch Claude Code. +### Manual Setup -### Manual Configuration with `settings.json` - -Alternatively, you can configure Claude Code by creating a `.claude/settings.json` file in your project's root directory. This file should contain the environment variables needed by Claude Code. This way you don't need to run the interactive setup every time. - -Here is an example `.claude/settings.json` file: +Create `.claude/settings.json` in your project root: ```json { @@ -315,37 +464,119 @@ Here is an example `.claude/settings.json` file: "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1" }, "permissions": { - "deny": [ - "WebSearch" - ] + "deny": ["WebSearch"] } } ``` -You can find more options here: [Claude Code settings](https://docs.anthropic.com/en/docs/claude-code/settings#environment-variables) +More options: [Claude Code settings](https://docs.anthropic.com/en/docs/claude-code/settings#environment-variables) · [IDE integrations](https://docs.anthropic.com/en/docs/claude-code/ide-integrations) -You can also read more about IDE integration here: [Add Claude Code to your IDE](https://docs.anthropic.com/en/docs/claude-code/ide-integrations) +## Advanced Features -## Running from Source +### Automatic Endpoint Routing + +Models declare their `supported_endpoints`. When a model doesn't support `/chat/completions` (e.g. some gpt-5.x variants), the proxy automatically routes through the Responses API with full translation. Claude models go the opposite direction — they're translated from Responses API to Chat Completions. + +### Context Overflow Auto-Switch + +When estimated token count exceeds the requested model's context window, the proxy auto-switches to the largest available model. This prevents context-window errors without client-side changes. + +### Image Handling + +- **413 Stripping Cascade:** On payload-too-large errors, the proxy retries by progressively stripping images: older images first → all images → trigger compaction +- **Proactive Trimming:** Set `IMAGE_CONTEXT_TRIMMING_ENABLED=1` to auto-trim processed images beyond a message threshold +- **Validation:** Rejects PNG images smaller than 4×4 pixels + +### Large Edit Guidance + +When file-edit tools (Edit, Write, MultiEdit) are present and the model's max output is under 32K tokens, the proxy injects a system message warning about output limits — helping models plan chunked edits instead of overflowing. + +### Empty Response Recovery + +If Copilot returns an empty response (common with some model backends), the proxy retries up to 2 times and falls back to a synthetic response explaining the failure. + +### Truncated Tool Call Detection + +When a model's output is cut off mid-tool-call, the proxy detects the truncation and returns an explanatory text block with `end_turn` instead of a malformed tool_use block. + +### Token Counting & Compaction Scaling + +Token counts include overhead estimates for typed tools (bash: 700, text_editor: 700, etc.), custom tools, and attachments. Counts are scaled per model family (Claude ×1.2, Grok ×1.03, others dynamically) to ensure accurate compaction triggers in Claude Code. + +## Docker + +### Build & Run + +```sh +docker build -t copilot-api . -The project can be run from source in several ways: +mkdir -p ./copilot-data +docker run -p 4141:4141 -v $(pwd)/copilot-data:/root/.local/share/copilot-api copilot-api +``` -### Development Mode +### With Environment Variables ```sh -bun run dev +docker run -p 4141:4141 \ + -e GH_TOKEN=your_github_token \ + -e TAVILY_API_KEY=tvly-... \ + copilot-api ``` -### Production Mode +### Docker Compose + +```yaml +version: "3.8" +services: + copilot-api: + build: . + ports: + - "4141:4141" + environment: + - GH_TOKEN=your_github_token_here + - TAVILY_API_KEY=tvly-your-key-here + restart: unless-stopped +``` + +The Docker image features multi-stage builds, a non-root user, health checks, and pinned base images. + +## Using with npx + +```sh +npx copilot-api@latest start # basic +npx copilot-api@latest start --port 8080 # custom port +npx copilot-api@latest start --account-type business # business plan +npx copilot-api@latest auth # auth only +npx copilot-api@latest check-usage # quota info +npx copilot-api@latest debug --json # diagnostics +``` + +## Usage Dashboard + +After starting the server, the console displays a URL to the web-based usage dashboard: + +``` +https://ericc-ch.github.io/copilot-api?endpoint=http://localhost:4141/usage +``` + +The dashboard shows usage quotas (Chat, Completions, Premium), detailed statistics, and supports custom endpoints via the `?endpoint=` parameter. On Windows, `start.bat` opens it automatically. + +## Running from Source ```sh -bun run start +bun install # install dependencies +bun run dev # development (watch mode) +bun run start # production +bun run build # compile to dist/ +bun run typecheck # type check +bun run lint:all # lint all files +bun run knip # find unused exports/dead code ``` -## Usage Tips +## Tips -- To avoid hitting GitHub Copilot's rate limits, you can use the following flags: - - `--manual`: Enables manual approval for each request, giving you full control over when requests are sent. - - `--rate-limit `: Enforces a minimum time interval between requests. For example, `copilot-api start --rate-limit 30` will ensure there's at least a 30-second gap between requests. - - `--wait`: Use this with `--rate-limit`. It makes the server wait for the cooldown period to end instead of rejecting the request with an error. This is useful for clients that don't automatically retry on rate limit errors. -- If you have a GitHub business or enterprise plan account with Copilot, use the `--account-type` flag (e.g., `--account-type business`). See the [official documentation](https://docs.github.com/en/enterprise-cloud@latest/copilot/managing-copilot/managing-github-copilot-in-your-organization/managing-access-to-github-copilot-in-your-organization/managing-github-copilot-access-to-your-organizations-network#configuring-copilot-subscription-based-network-routing-for-your-enterprise-or-organization) for more details. +- **Rate limiting:** `--rate-limit 30` enforces a 30s gap. Add `--wait` to queue instead of reject. Use `--burst-count` and `--burst-window` for sliding-window limits. +- **Business/Enterprise:** Always pass `--account-type business` or `enterprise` — it changes the Copilot API base URL. +- **Web search cost:** Each search uses 2–3 internal API calls. Monitor your quota. +- **Token persistence:** Stored at `~/.local/share/copilot-api/github_token`. Use `auth` to regenerate. +- **Proxy:** Set `HTTP_PROXY`/`HTTPS_PROXY` and pass `--proxy-env` to route through a corporate proxy. diff --git a/bun.lock b/bun.lock index 20e895e7f..9ece87578 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "copilot-api", diff --git a/deploy/copilot-api-watchdog.service b/deploy/copilot-api-watchdog.service new file mode 100644 index 000000000..c5c82c9a8 --- /dev/null +++ b/deploy/copilot-api-watchdog.service @@ -0,0 +1,15 @@ +[Unit] +Description=Copilot API token-expiry watchdog +After=copilot-api.service +Wants=copilot-api.service + +[Service] +Type=simple +User=root +ExecStart=/root/projects/copilot-api/deploy/token-watchdog.sh +Restart=always +RestartSec=5 +Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +[Install] +WantedBy=multi-user.target diff --git a/deploy/nginx/copilot-api.conf.example b/deploy/nginx/copilot-api.conf.example new file mode 100644 index 000000000..836ae959a --- /dev/null +++ b/deploy/nginx/copilot-api.conf.example @@ -0,0 +1,35 @@ +# Example nginx reverse proxy for public copilot-api access. +# +# The important line is client_max_body_size. Codex /v1/responses requests can +# contain long conversation history and image/tool output payloads. nginx's +# default body limit is too small for that traffic and returns a raw HTML 413 +# before copilot-api can trigger Codex auto-compaction. + +server { + listen 8443 ssl; + server_name 157-173-124-192.sslip.io; + + ssl_certificate /etc/letsencrypt/live/157-173-124-192.sslip.io/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/157-173-124-192.sslip.io/privkey.pem; + include /etc/letsencrypt/options-ssl-nginx.conf; + ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; + + client_max_body_size 256m; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + + location / { + proxy_pass http://127.0.0.1:3131; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ''; + + proxy_buffering off; + proxy_cache off; + chunked_transfer_encoding on; + } +} diff --git a/deploy/token-watchdog.sh b/deploy/token-watchdog.sh new file mode 100644 index 000000000..4b8dd2c3d --- /dev/null +++ b/deploy/token-watchdog.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Watchdog for copilot-api. +# +# Follows the systemd journal for the copilot-api service and restarts it when +# the upstream Copilot "token expired" error appears. Restarting forces the app +# to run setupCopilotToken() again, fetching a fresh Copilot token. +# +# Safety: +# * COOLDOWN - minimum seconds between restarts (debounces error bursts). +# * MAX_PER_HOUR - hard cap on restarts per rolling hour (avoids restart loops +# when the underlying GitHub token is truly dead). +set -uo pipefail + +SERVICE="${SERVICE:-copilot-api.service}" +# Extended-regex of log lines that indicate a dead/stale Copilot token and +# warrant a restart. Covers two failure modes: +# 1. Upstream 401 on a request: "...token expired: unauthorized: token expired" +# 2. The in-app periodic refresh failing (src/lib/token.ts rethrows, producing +# "Failed to refresh Copilot token" + an unhandled promise rejection). +PATTERN="${PATTERN:-token expired: unauthorized: token expired|Failed to refresh Copilot token|Unhandled promise rejection: Failed to get Copilot token}" +COOLDOWN="${COOLDOWN:-120}" +MAX_PER_HOUR="${MAX_PER_HOUR:-6}" + +STATE_DIR="${STATE_DIR:-/var/lib/copilot-api-watchdog}" +LAST_RESTART_FILE="$STATE_DIR/last_restart" +HISTORY_FILE="$STATE_DIR/history" +mkdir -p "$STATE_DIR" + +log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } + +# Prune restart history to the last hour and return the remaining count. +restarts_last_hour() { + local now=$1 cutoff ts count=0 + cutoff=$((now - 3600)) + local tmp="$STATE_DIR/.history.tmp" + : > "$tmp" + if [[ -f "$HISTORY_FILE" ]]; then + while read -r ts; do + [[ -z "$ts" ]] && continue + if (( ts >= cutoff )); then + echo "$ts" >> "$tmp" + count=$((count + 1)) + fi + done < "$HISTORY_FILE" + fi + mv "$tmp" "$HISTORY_FILE" + echo "$count" +} + +log "Watchdog started; following journal for $SERVICE (cooldown=${COOLDOWN}s, cap=${MAX_PER_HOUR}/h)" + +# -n0 => start at the tail, only react to errors that happen from now on. +journalctl -u "$SERVICE" -f -n0 -o cat 2>/dev/null | while IFS= read -r line; do + if [[ "$line" =~ $PATTERN ]]; then + now=$(date +%s) + + last=0 + [[ -f "$LAST_RESTART_FILE" ]] && last=$(cat "$LAST_RESTART_FILE" 2>/dev/null || echo 0) + if (( now - last < COOLDOWN )); then + continue + fi + + count=$(restarts_last_hour "$now") + if (( count >= MAX_PER_HOUR )); then + log "Detected token failure but restart cap reached (${count}/${MAX_PER_HOUR} in last hour); backing off" + echo "$now" > "$LAST_RESTART_FILE" + continue + fi + + log "Detected token failure -> restarting $SERVICE" + if systemctl restart "$SERVICE"; then + echo "$now" > "$LAST_RESTART_FILE" + echo "$now" >> "$HISTORY_FILE" + log "Restarted $SERVICE successfully" + else + log "ERROR: failed to restart $SERVICE" + fi + fi +done diff --git a/deploy/wd-selftest.sh b/deploy/wd-selftest.sh new file mode 100644 index 000000000..3bf778bed --- /dev/null +++ b/deploy/wd-selftest.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Functional test for token-watchdog.sh using stubbed journalctl/systemctl. +set -uo pipefail + +WD=/root/projects/copilot-api/deploy/token-watchdog.sh +sed -i 's/\r$//' "$WD"; chmod +x "$WD" + +TESTROOT=/tmp/wdtest +rm -rf "$TESTROOT"; mkdir -p "$TESTROOT/bin" + +MATCH='10:45:00 AM x POST /v1/messages claude 401 221ms IDE token expired: unauthorized: token expired' +MATCH2=' ERROR Failed to refresh Copilot token: Failed to get Copilot token' + +# Stub journalctl: emit a 401 line, then (after >cooldown) a "Failed to refresh" line. +cat > "$TESTROOT/bin/journalctl" < "$TESTROOT/bin/systemctl" <<'EOF' +#!/usr/bin/env bash +echo "[$(date '+%H:%M:%S')] STUB systemctl $*" >> /tmp/wdtest/actions.log +exit 0 +EOF +chmod +x "$TESTROOT/bin/"* +: > "$TESTROOT/actions.log" + +echo "=== TEST 1: two patterns + cooldown (expect 2 restarts: 401 trigger, suppress dup, refresh-fail trigger) ===" +PATH="$TESTROOT/bin:$PATH" STATE_DIR="$TESTROOT/state1" COOLDOWN=2 MAX_PER_HOUR=6 \ + timeout 15 bash "$WD" 2>&1 | sed 's/^/ /' +echo " --- systemctl calls ---" +cat "$TESTROOT/actions.log" | sed 's/^/ /' +c1=$(grep -c 'restart' "$TESTROOT/actions.log") +echo " restart count = $c1 (expected 2)" + +: > "$TESTROOT/actions.log" +echo +echo "=== TEST 2: hourly cap (MAX_PER_HOUR=1, expect only 1 restart) ===" +PATH="$TESTROOT/bin:$PATH" STATE_DIR="$TESTROOT/state2" COOLDOWN=1 MAX_PER_HOUR=1 \ + timeout 15 bash "$WD" 2>&1 | sed 's/^/ /' +echo " --- systemctl calls ---" +cat "$TESTROOT/actions.log" | sed 's/^/ /' +c2=$(grep -c 'restart' "$TESTROOT/actions.log") +echo " restart count = $c2 (expected 1)" + +echo +if [[ "$c1" == "2" && "$c2" == "1" ]]; then + echo "RESULT: PASS" +else + echo "RESULT: FAIL (test1=$c1 want 2, test2=$c2 want 1)" +fi +rm -rf "$TESTROOT" diff --git a/docs/superpowers/plans/2026-03-16-tool-parity.md b/docs/superpowers/plans/2026-03-16-tool-parity.md new file mode 100644 index 000000000..0406689c1 --- /dev/null +++ b/docs/superpowers/plans/2026-03-16-tool-parity.md @@ -0,0 +1,1349 @@ +# Tool Parity Implementation Plan + +> **For agentic workers:** REQUIRED: Use superpowers:subagent-driven-development (if subagents available) or superpowers:executing-plans to implement this plan. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Fix 11 gaps in copilot-api's Anthropic↔OpenAI translation layer to achieve full tool parity with Claude Code v2.1.76. + +**Architecture:** Four surgical in-place edits — no new files, no new abstractions. `anthropic-types.ts` gets new interfaces and a discriminator helper; `non-stream-translation.ts` uses them to filter typed tools, handle array tool-result content, and process new content block types; `count-tokens-handler.ts` imports the discriminator and splits token overhead correctly; `create-chat-completions.ts` gets a single `strict?: boolean` field. + +**Tech Stack:** Bun runtime, TypeScript, `bun:test` for tests. Run tests with `bun test`. Run type-checking with `bun run typecheck`. Lint with `bun run lint:all`. + +--- + +## File Map + +| File | Change | +|------|--------| +| `src/routes/messages/anthropic-types.ts` | Add `AnthropicCustomTool`, `AnthropicTypedTool`, `isTypedTool`; add `AnthropicDocumentBlock`, `AnthropicRedactedThinkingBlock`, `AnthropicServerToolUseBlock`, `AnthropicWebSearchToolResultBlock`; update `AnthropicTool`, `AnthropicThinkingBlock`, `AnthropicToolResultBlock`, content union types, `tool_choice` | +| `src/routes/messages/non-stream-translation.ts` | Update `translateAnthropicToolsToOpenAI`, `mapContent`, `handleUserMessage`, `handleAssistantMessage`, `translateModelName`; add `mapToolResultContent` | +| `src/routes/messages/count-tokens-handler.ts` | Import `isTypedTool`; add `ANTHROPIC_TYPED_TOOL_TOKEN_OVERHEAD`; replace tools block logic | +| `src/services/copilot/create-chat-completions.ts` | Add `strict?: boolean` to `Tool.function` interface | +| `tests/anthropic-request.test.ts` | Add new test cases for all 11 gaps | + +--- + +## Chunk 1: Type Definitions (`anthropic-types.ts`) + +### Task 1: Split `AnthropicTool` into `AnthropicCustomTool` | `AnthropicTypedTool` union with `isTypedTool` helper + +**Files:** +- Modify: `src/routes/messages/anthropic-types.ts` +- Test: `tests/anthropic-request.test.ts` + +- [ ] **Step 1.1: Write the failing test for typed-tool filtering** + +Add to `tests/anthropic-request.test.ts` inside the `"Anthropic to OpenAI translation logic"` describe block: + +```typescript +test("should filter out Anthropic typed tools (no input_schema) from tools array", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [{ role: "user", content: "Hello" }], + max_tokens: 100, + tools: [ + // Custom tool — should be kept + { name: "Bash", description: "Run shell commands", input_schema: { type: "object", properties: {}, additionalProperties: false } }, + // Anthropic-typed tool — should be filtered + { type: "bash_20250124", name: "bash" } as unknown as Parameters[0]["tools"][0], + ], + } + const result = translateToOpenAI(anthropicPayload) + // Only the custom "Bash" tool survives + expect(result.tools).toHaveLength(1) + expect(result.tools?.[0].function.name).toBe("Bash") +}) +``` + +- [ ] **Step 1.2: Run the failing test** + +```bash +cd /c/Users/ttbasil/Desktop/Projects/PublicProjects/copilot-api && bun test tests/anthropic-request.test.ts --test-name-pattern "Anthropic typed tools" +``` + +Expected: FAIL — the current translation returns both tools unfiltered, so `toHaveLength(1)` fails at runtime. The test uses `as unknown as ...` double-cast to bypass TypeScript, so the failure is a runtime assertion error, not a compile error. + +- [ ] **Step 1.3: Replace `AnthropicTool` in `anthropic-types.ts`** + +In `src/routes/messages/anthropic-types.ts`, replace the existing `AnthropicTool` interface (lines 83–87): + +```typescript +// Before: +export interface AnthropicTool { + name: string + description?: string + input_schema: Record +} +``` + +With: + +```typescript +// Custom tool (has input_schema) — what Claude Code and standard clients send +export interface AnthropicCustomTool { + name: string + description?: string + input_schema: Record + strict?: boolean + cache_control?: { type: "ephemeral"; ttl?: number } + defer_loading?: boolean + input_examples?: unknown[] + eager_input_streaming?: boolean +} + +// Anthropic-typed tool (versioned type string, no input_schema) +// Examples: bash_20250124, text_editor_20250728, computer_20251124, web_search_20250305 +export interface AnthropicTypedTool { + type: string + name: string + [key: string]: unknown +} + +export type AnthropicTool = AnthropicCustomTool | AnthropicTypedTool + +// Discriminant: typed tools never have input_schema; custom tools always do. +// Using presence of input_schema is more robust than checking for type, +// since a future custom tool definition could include a type field. +export function isTypedTool(tool: AnthropicTool): tool is AnthropicTypedTool { + return !("input_schema" in tool) +} +``` + +- [ ] **Step 1.4: Run the test to verify it passes** + +```bash +bun test tests/anthropic-request.test.ts --test-name-pattern "Anthropic typed tools" +``` + +Expected: PASS + +- [ ] **Step 1.5: Run typecheck to catch any regressions** + +```bash +bun run typecheck +``` + +Expected: May see errors in `non-stream-translation.ts` where `tool.input_schema` is now on a union type — those are fixed in later tasks. `count-tokens-handler.ts` will not error here because `tool.name` is present on both union members. Note any errors but do not fix them yet. + +- [ ] **Step 1.6: Commit** + +> Note: The pre-commit hook runs ESLint (lint-staged) on staged files — not `typecheck`. Downstream TypeScript errors in `non-stream-translation.ts` will not block the commit; they are fixed in later tasks. + +```bash +git add src/routes/messages/anthropic-types.ts tests/anthropic-request.test.ts +git commit -m "feat: split AnthropicTool into CustomTool|TypedTool union with isTypedTool discriminator" +``` + +--- + +### Task 2: Add new content block types to `anthropic-types.ts` + +**Files:** +- Modify: `src/routes/messages/anthropic-types.ts` + +- [ ] **Step 2.1: Write failing tests for new content block types** + +Add to `tests/anthropic-request.test.ts`: + +```typescript +test("should handle document blocks in user messages", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What does this PDF say?" }, + { + type: "document", + source: { type: "base64", media_type: "application/pdf", data: "JVBERi0x" }, + }, + ], + }, + ], + max_tokens: 100, + } + // Should not throw; document block is converted to placeholder text + expect(() => translateToOpenAI(anthropicPayload)).not.toThrow() + const result = translateToOpenAI(anthropicPayload) + expect(isValidChatCompletionRequest(result)).toBe(true) + // Placeholder text must appear in the message content + const userMsg = result.messages.find((m) => m.role === "user") + expect(typeof userMsg?.content).toBe("string") + expect(userMsg?.content as string).toContain("[Document: PDF content not displayable]") +}) + +test("should handle redacted_thinking blocks in assistant messages", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Think hard about this." }, + { + role: "assistant", + content: [ + { type: "redacted_thinking", data: "EncryptedThinkingData==" }, + { type: "text", text: "Here is my answer." }, + ], + }, + { role: "user", content: "Follow up." }, + ], + max_tokens: 100, + } + expect(() => translateToOpenAI(anthropicPayload)).not.toThrow() + const result = translateToOpenAI(anthropicPayload) + // The redacted_thinking block is stripped; only the text block survives + const assistantMsg = result.messages.find((m) => m.role === "assistant") + expect(assistantMsg?.content).toContain("Here is my answer.") + // redacted_thinking data must NOT appear as raw base64 + expect(assistantMsg?.content as string).not.toContain("EncryptedThinkingData==") +}) +``` + +- [ ] **Step 2.2: Run the failing tests** + +```bash +bun test tests/anthropic-request.test.ts --test-name-pattern "document blocks|redacted_thinking" +``` + +Expected: The `document` test FAILS — `mapContent` silently drops the unknown `document` block type, so `toContain("[Document: PDF content not displayable]")` fails. The `redacted_thinking` test **PASSES** vacuously — the existing `mapContent` filter (`type === "text" || type === "thinking"`) already silently drops `redacted_thinking` blocks, so the output coincidentally matches the desired behavior. The value of adding the `redacted_thinking` type is type-system coverage (enforced by `bun run typecheck`), not new runtime behavior. + +- [ ] **Step 2.3a: Update `AnthropicThinkingBlock` to add `signature` field** + +Find the existing `AnthropicThinkingBlock` interface in `src/routes/messages/anthropic-types.ts` and replace it: + +```typescript +// Before: +export interface AnthropicThinkingBlock { + type: "thinking" + thinking: string +} + +// After: +export interface AnthropicThinkingBlock { + type: "thinking" + thinking: string + signature?: string // Used by Claude Code extended thinking +} +``` + +- [ ] **Step 2.3b: Add new content block interfaces** + +After the (now-updated) `AnthropicThinkingBlock`, add the following four new interfaces: + +```typescript +// New: redacted thinking (redact-thinking-2026-02-12 beta) +export interface AnthropicRedactedThinkingBlock { + type: "redacted_thinking" + data: string +} + +// New: document block (PDFs sent via Read tool) +// source union covers all Anthropic-documented source types; handler emits a +// placeholder string regardless, so media_type is intentionally wide. +export interface AnthropicDocumentBlock { + type: "document" + title?: string + source: + | { type: "base64"; media_type: string; data: string } + | { type: "url"; url: string } + | { type: "text"; data: string } + cache_control?: { type: "ephemeral"; ttl?: number } +} + +// New: server-side tool use block in assistant messages +// Appears in multi-turn histories from real Anthropic API with web_search server tool +export interface AnthropicServerToolUseBlock { + type: "server_tool_use" + id: string + name: string + input: Record +} + +// New: web search tool result block in user messages +// Appears in multi-turn histories from real Anthropic API with web_search server tool +export interface AnthropicWebSearchToolResultBlock { + type: "web_search_tool_result" + tool_use_id: string + content: unknown +} +``` + +- [ ] **Step 2.4: Update `AnthropicToolResultBlock` content type** + +Replace the existing `AnthropicToolResultBlock`: + +```typescript +export interface AnthropicToolResultBlock { + type: "tool_result" + tool_use_id: string + content: string | Array + is_error?: boolean +} +``` + +- [ ] **Step 2.5: Update `tool_choice` to accept `disable_parallel_tool_use`** + +In `AnthropicMessagesPayload`, replace the `tool_choice` field: + +```typescript +// Before: +tool_choice?: { + type: "auto" | "any" | "tool" | "none" + name?: string +} + +// After: +tool_choice?: { + type: "auto" | "any" | "tool" | "none" + name?: string + disable_parallel_tool_use?: boolean // parsed but not forwarded — no OpenAI equivalent +} +``` + +- [ ] **Step 2.6: Update content block union types** + +Replace the existing `AnthropicUserContentBlock` and `AnthropicAssistantContentBlock` type aliases. `AnthropicToolResultBlock` is already in the user union — this update adds `AnthropicDocumentBlock` and `AnthropicWebSearchToolResultBlock` to it, and adds `AnthropicRedactedThinkingBlock` and `AnthropicServerToolUseBlock` to the assistant union: + +```typescript +export type AnthropicUserContentBlock = + | AnthropicTextBlock + | AnthropicImageBlock + | AnthropicDocumentBlock + | AnthropicToolResultBlock + | AnthropicWebSearchToolResultBlock + +export type AnthropicAssistantContentBlock = + | AnthropicTextBlock + | AnthropicToolUseBlock + | AnthropicThinkingBlock + | AnthropicRedactedThinkingBlock + | AnthropicServerToolUseBlock +``` + +- [ ] **Step 2.7: Run the tests** + +```bash +bun test tests/anthropic-request.test.ts --test-name-pattern "document blocks|redacted_thinking" +``` + +Expected: FAIL — the `document` test fails because `mapContent` does not yet produce `"[Document: PDF content not displayable]"` placeholder text (the assertion `toContain("[Document: ...]")` will fail). Check that TypeScript is clean in `anthropic-types.ts`: + +```bash +bun run typecheck 2>&1 | head -40 +``` + +The only errors should be in `non-stream-translation.ts` (where `tool.input_schema` and the new union members cause TS errors) — not in `anthropic-types.ts` itself and not in `count-tokens-handler.ts` (which only accesses `tool.name`, present on both union members). + +- [ ] **Step 2.8: Commit** + +```bash +git add src/routes/messages/anthropic-types.ts tests/anthropic-request.test.ts +git commit -m "feat: add new Anthropic content block types (document, redacted_thinking, server_tool_use, web_search_tool_result)" +``` + +--- + +## Chunk 2: OpenAI Tool Type (`create-chat-completions.ts`) + +### Task 3: Add `strict` to OpenAI `Tool` interface + +**Files:** +- Modify: `src/services/copilot/create-chat-completions.ts` +- Test: `tests/anthropic-request.test.ts` + +- [ ] **Step 3.1: Write failing test for `strict` forwarding** + +Add to `tests/anthropic-request.test.ts`: + +```typescript +test("should forward strict:true from custom tool definitions to OpenAI", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [{ role: "user", content: "Hello" }], + max_tokens: 100, + tools: [ + { + name: "getWeather", + description: "Get weather", + input_schema: { type: "object", properties: { location: { type: "string" } }, required: ["location"] }, + strict: true, + }, + ], + } + const result = translateToOpenAI(anthropicPayload) + expect(result.tools?.[0].function.strict).toBe(true) +}) + +test("should not add strict field when not provided", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [{ role: "user", content: "Hello" }], + max_tokens: 100, + tools: [ + { + name: "getWeather", + description: "Get weather", + input_schema: { type: "object", properties: {} }, + }, + ], + } + const result = translateToOpenAI(anthropicPayload) + expect(result.tools?.[0].function.strict).toBeUndefined() +}) +``` + +- [ ] **Step 3.2: Run the failing tests** + +```bash +bun test tests/anthropic-request.test.ts --test-name-pattern "strict" +``` + +Expected: FAIL — `result.tools?.[0].function.strict` doesn't exist yet. + +- [ ] **Step 3.3: Add `strict` to the `Tool` interface** + +In `src/services/copilot/create-chat-completions.ts`, find the `Tool` interface (around line 153) and update: + +```typescript +export interface Tool { + type: "function" + function: { + name: string + description?: string + parameters: Record + strict?: boolean // Structured Outputs — forwarded from Anthropic custom tool definitions + } +} +``` + +- [ ] **Step 3.4: Run the tests** + +```bash +bun test tests/anthropic-request.test.ts --test-name-pattern "strict" +``` + +Expected: The first test (`strict: true` → `.toBe(true)`) still FAILS — the translation doesn't forward `strict` yet. The second test (`strict` not provided → `.toBeUndefined()`) **PASSES** vacuously — the field isn't forwarded, so it's `undefined` which satisfies the assertion. Both tests pass after Task 4's translation update. + +- [ ] **Step 3.5: Commit the type-only change** + +> Note: One strict-forwarding test still fails here — the translation logic is updated in Task 4. Committing with a red test is intentional in this TDD workflow; the pre-commit hook only runs ESLint, not the test suite. + +```bash +git add src/services/copilot/create-chat-completions.ts +git commit -m "feat: add strict field to OpenAI Tool.function interface for Structured Outputs" +``` + +--- + +## Chunk 3: Translation Functions (`non-stream-translation.ts`) + +### Task 4: Update `translateAnthropicToolsToOpenAI` — filter typed tools, forward `strict`, strip extra fields + +**Files:** +- Modify: `src/routes/messages/non-stream-translation.ts` + +- [ ] **Step 4.1: Update imports in `non-stream-translation.ts`** + +> **Prerequisite:** Tasks 1 and 2 (Chunk 1) must be committed before this step — the import list includes types added by those tasks. + +At the top of `src/routes/messages/non-stream-translation.ts`, replace the existing import block from `"./anthropic-types"` with: + +```typescript +import { + type AnthropicAssistantContentBlock, + type AnthropicAssistantMessage, + type AnthropicCustomTool, + type AnthropicDocumentBlock, + type AnthropicMessage, + type AnthropicMessagesPayload, + type AnthropicRedactedThinkingBlock, + type AnthropicResponse, + type AnthropicServerToolUseBlock, + type AnthropicTextBlock, + type AnthropicThinkingBlock, + type AnthropicTool, + type AnthropicToolResultBlock, + type AnthropicToolUseBlock, + type AnthropicUserContentBlock, + type AnthropicUserMessage, + type AnthropicWebSearchToolResultBlock, + isTypedTool, +} from "./anthropic-types" +``` + +- [ ] **Step 4.2: Replace `translateAnthropicToolsToOpenAI`** + +Find the existing `translateAnthropicToolsToOpenAI` function and replace it entirely: + +```typescript +function translateAnthropicToolsToOpenAI( + anthropicTools: Array | undefined, +): Array | undefined { + if (!anthropicTools) { + return undefined + } + + return anthropicTools + .filter((tool): tool is AnthropicCustomTool => !isTypedTool(tool)) + .map((tool) => ({ + type: "function", + function: { + name: tool.name, + description: tool.description, + parameters: tool.input_schema, + // Forward strict for Structured Outputs; strip all other extra fields + // (cache_control, defer_loading, input_examples, eager_input_streaming) + ...(tool.strict !== undefined ? { strict: tool.strict } : {}), + }, + })) +} +``` + +- [ ] **Step 4.3: Run the tests that Task 4 makes pass** + +```bash +bun test tests/anthropic-request.test.ts --test-name-pattern "Anthropic typed tools|forward strict" +``` + +Expected: All 2 targeted tests PASS. (The `"should not add strict field"` test passed vacuously from Task 3; the `"Anthropic typed tools"` test was failing since Task 1. Both should be green now.) + +- [ ] **Step 4.4: Run the full test suite** + +```bash +bun test +``` + +Expected: All existing tests pass; new tests from Tasks 1–3 pass. + +- [ ] **Step 4.5: Commit** + +```bash +git add src/routes/messages/non-stream-translation.ts +git commit -m "feat: filter typed tools and forward strict field in translateAnthropicToolsToOpenAI" +``` + +--- + +### Task 5: Update `translateModelName` — generalized claude-4+ normalization + +**Files:** +- Modify: `src/routes/messages/non-stream-translation.ts` +- Test: `tests/anthropic-request.test.ts` + +- [ ] **Step 5.1: Write failing tests for model name normalization** + +Add to `tests/anthropic-request.test.ts`: + +```typescript +describe("translateModelName normalization", () => { + // Helper: call translateToOpenAI with just the model and extract the model name + function getTranslatedModel(model: string): string { + const result = translateToOpenAI({ + model, + messages: [{ role: "user", content: "Hi" }], + max_tokens: 10, + }) + return result.model + } + + test("normalizes claude-sonnet-4-6 to claude-sonnet-4", () => { + expect(getTranslatedModel("claude-sonnet-4-6")).toBe("claude-sonnet-4") + }) + + test("normalizes claude-haiku-4-5 to claude-haiku-4 (was missing before)", () => { + expect(getTranslatedModel("claude-haiku-4-5")).toBe("claude-haiku-4") + }) + + test("normalizes claude-opus-4-6 to claude-opus-4", () => { + expect(getTranslatedModel("claude-opus-4-6")).toBe("claude-opus-4") + }) + + test("does NOT change claude-sonnet-3-5 (stable 3.x name)", () => { + expect(getTranslatedModel("claude-sonnet-3-5")).toBe("claude-sonnet-3-5") + }) + + test("does NOT change claude-haiku-3-5 (stable 3.x name)", () => { + expect(getTranslatedModel("claude-haiku-3-5")).toBe("claude-haiku-3-5") + }) + + test("normalizes long versioned names like claude-sonnet-4-6-20251231", () => { + expect(getTranslatedModel("claude-sonnet-4-6-20251231")).toBe("claude-sonnet-4") + }) + + test("does NOT change non-claude models", () => { + expect(getTranslatedModel("gpt-4o")).toBe("gpt-4o") + expect(getTranslatedModel("grok-2")).toBe("grok-2") + }) +}) +``` + +- [ ] **Step 5.2: Run the failing tests** + +```bash +bun test tests/anthropic-request.test.ts --test-name-pattern "translateModelName" +``` + +Expected: The `haiku-4-5` test FAILS; others may pass already. + +- [ ] **Step 5.3: Replace `translateModelName` in `non-stream-translation.ts`** + +Find the existing `translateModelName` function and **replace it entirely** with: + +```typescript +function translateModelName(model: string): string { + // Normalize claude-{family}-4-{minor}[-extra] → claude-{family}-4 + // Only applies to generation 4+ where minor version numbers are subagent-build-specific. + // Known limitation: multi-word family names like claude-sonnet-mini-4 won't match + // ([a-z]+ does not cross hyphens), but no such models currently exist. + // + // Test cases: + // claude-sonnet-4-6 → claude-sonnet-4 ✓ + // claude-haiku-4-5 → claude-haiku-4 ✓ + // claude-opus-4-6 → claude-opus-4 ✓ + // claude-sonnet-4-6-20251231 → claude-sonnet-4 ✓ + // claude-sonnet-3-5 → claude-sonnet-3-5 ✓ (unchanged) + // claude-haiku-3-5 → claude-haiku-3-5 ✓ (unchanged) + return model.replace(/^(claude-[a-z]+-4)-\d+.*$/, "$1") +} +``` + +- [ ] **Step 5.4: Run the tests** + +```bash +bun test tests/anthropic-request.test.ts --test-name-pattern "translateModelName" +``` + +Expected: All 7 model name tests PASS. + +- [ ] **Step 5.5: Commit** + +```bash +git add src/routes/messages/non-stream-translation.ts tests/anthropic-request.test.ts +git commit -m "feat: generalize translateModelName to handle all claude-4+ variants including haiku-4-5" +``` + +--- + +### Task 6: Update `mapContent` — add document, server_tool_use cases; update signature + +**Files:** +- Modify: `src/routes/messages/non-stream-translation.ts` + +- [ ] **Step 6.1: Write failing tests for document and server_tool_use in mapContent** + +Add to `tests/anthropic-request.test.ts`: + +```typescript +test("document block in user message produces placeholder text", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Summarize this PDF." }, + { + type: "document", + title: "My Report", + source: { type: "base64", media_type: "application/pdf", data: "JVBERi0x" }, + }, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const userMsg = result.messages.find((m) => m.role === "user") + expect(typeof userMsg?.content).toBe("string") + const content = userMsg?.content as string + expect(content).toContain("Summarize this PDF.") + expect(content).toContain("[Document: PDF content not displayable]") +}) + +test("server_tool_use block in assistant message is serialized to text", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Search for something." }, + { + role: "assistant", + content: [ + { type: "server_tool_use", id: "srv_1", name: "web_search", input: { query: "test" } }, + { type: "text", text: "I searched for you." }, + ], + }, + { role: "user", content: "Thanks." }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const assistantMsg = result.messages.find((m) => m.role === "assistant") + const content = assistantMsg?.content as string + expect(content).toContain("[Server tool use:") + expect(content).toContain("web_search") + expect(content).toContain("I searched for you.") +}) +``` + +- [ ] **Step 6.2: Run the failing tests** + +```bash +bun test tests/anthropic-request.test.ts --test-name-pattern "placeholder|server_tool_use" +``` + +Expected: FAIL — `document` and `server_tool_use` blocks fall through to `// No default`. + +- [ ] **Step 6.3: Update `mapContent` in `non-stream-translation.ts`** + +Find the `mapContent` function. The signature needs to accept the new union members (which it already does since we're reusing the existing `AnthropicUserContentBlock | AnthropicAssistantContentBlock` union — the new block types are already in those unions from Task 2). + +**Update the no-image text path** (the `if (!hasImage)` block) — replace it: + +```typescript +if (!hasImage) { + return content + .filter( + (block) => + block.type === "text" + || block.type === "thinking" + || block.type === "document" // user messages: PDF → placeholder + || block.type === "server_tool_use", // assistant messages: serialise to JSON + ) + .map((block) => { + if (block.type === "text") return (block as AnthropicTextBlock).text + if (block.type === "thinking") + return (block as AnthropicThinkingBlock).thinking + if (block.type === "document") + return "[Document: PDF content not displayable]" + // server_tool_use + return `[Server tool use: ${JSON.stringify(block)}]` + }) + .join("\n\n") +} +``` + +**Add cases to the image-path switch statement** (after the existing `"image"` case): + +```typescript +case "document": { + contentParts.push({ + type: "text", + text: "[Document: PDF content not displayable]", + }) + break +} +case "server_tool_use": { + contentParts.push({ + type: "text", + text: `[Server tool use: ${JSON.stringify(block)}]`, + }) + break +} +// redacted_thinking: silently skip — opaque binary data, no OpenAI equivalent +// Keep the existing: // No default +``` + +- [ ] **Step 6.4: Run the tests** + +```bash +bun test tests/anthropic-request.test.ts --test-name-pattern "placeholder|server_tool_use" +``` + +Expected: Both tests PASS. + +- [ ] **Step 6.5: Run full suite** + +```bash +bun test +``` + +Expected: All tests pass. + +- [ ] **Step 6.6: Commit** + +```bash +git add src/routes/messages/non-stream-translation.ts tests/anthropic-request.test.ts +git commit -m "feat: add document and server_tool_use handling to mapContent" +``` + +--- + +### Task 7: Update `handleAssistantMessage` — strip `redacted_thinking`, include `server_tool_use` in Branch 1 + +**Files:** +- Modify: `src/routes/messages/non-stream-translation.ts` +- Test: `tests/anthropic-request.test.ts` + +- [ ] **Step 7.1: Write failing tests** + +Add to `tests/anthropic-request.test.ts`: + +```typescript +test("redacted_thinking block is stripped from assistant message (Branch 2, no tool calls)", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Think hard." }, + { + role: "assistant", + content: [ + { type: "redacted_thinking", data: "EncryptedBinaryData==" }, + { type: "text", text: "My considered answer." }, + ], + }, + { role: "user", content: "Follow up." }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const assistantMsg = result.messages.find((m) => m.role === "assistant") + // Content must contain the text but NOT the redacted data + expect(assistantMsg?.content).toContain("My considered answer.") + expect(assistantMsg?.content).not.toContain("EncryptedBinaryData==") +}) + +test("server_tool_use block is serialized in assistant message with tool calls (Branch 1)", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Do something." }, + { + role: "assistant", + content: [ + { type: "server_tool_use", id: "srv_1", name: "web_search", input: { query: "test" } }, + { type: "text", text: "Let me also call a tool." }, + { type: "tool_use", id: "call_1", name: "Bash", input: { command: "ls" } }, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const assistantMsg = result.messages.find((m) => m.role === "assistant") + // Branch 1: has tool_use, so content is the text + server_tool_use serialized + expect(assistantMsg?.content).toContain("Let me also call a tool.") + expect(assistantMsg?.content).toContain("[Server tool use:") + expect(assistantMsg?.tool_calls).toHaveLength(1) + expect(assistantMsg?.tool_calls?.[0].function.name).toBe("Bash") +}) +``` + +- [ ] **Step 7.2: Run the failing tests** + +```bash +bun test tests/anthropic-request.test.ts --test-name-pattern "redacted_thinking|Branch 1" +``` + +Expected: FAIL. + +- [ ] **Step 7.3: Update `handleAssistantMessage` in `non-stream-translation.ts`** + +> **Prerequisite:** Tasks 1 and 2 (Chunk 1) must be committed. Task 2 adds `AnthropicRedactedThinkingBlock` to `AnthropicAssistantContentBlock` — without that, the `.filter((b) => b.type !== "redacted_thinking")` call is a TypeScript error (the discriminant isn't in the union yet). Task 2 also adds `AnthropicServerToolUseBlock` to `AnthropicAssistantContentBlock`, which is needed for Branch 1's `serverToolUseBlocks` filter. + +Find the `handleAssistantMessage` function. It has two branches based on `toolUseBlocks.length > 0`. + +**Update Branch 1** (the `toolUseBlocks.length > 0` branch) — add `serverToolUseBlocks` and update `allTextContent`: + +```typescript +const serverToolUseBlocks = message.content.filter( + (block): block is AnthropicServerToolUseBlock => block.type === "server_tool_use", +) + +const allTextContent = [ + ...textBlocks.map((b) => b.text), + ...thinkingBlocks.map((b) => b.thinking), + ...serverToolUseBlocks.map((b) => `[Server tool use: ${JSON.stringify(b)}]`), +] + .filter(Boolean) // strip empty strings from text/thinking .map() to avoid spurious \n\n + .join("\n\n") + +return [ + { + role: "assistant", + content: allTextContent || null, // null not "" when all content is empty (tool-only response) + tool_calls: toolUseBlocks.map((toolUse) => ({ + id: toolUse.id, + type: "function", + function: { + name: toolUse.name, + arguments: JSON.stringify(toolUse.input), + }, + })), + }, +] +``` + +**Update Branch 2** (the ternary else — content IS an array but has no `tool_use` blocks, currently lines 171–176 in the source) — filter `redacted_thinking` before calling `mapContent`. At this point in `handleAssistantMessage`, the content is always an array (the early return at lines 129–136 handles the non-array case). Remove the `Array.isArray` guard: + +```typescript +// Branch 2 — no custom tool_use blocks +// filter out redacted_thinking (opaque binary, no OpenAI equivalent) +// server_tool_use and document are handled by mapContent's updated switch/filter +const visibleContent = (message.content as Array).filter( + (b) => b.type !== "redacted_thinking", +) + +return [ + { + role: "assistant", + content: mapContent(visibleContent), + }, +] +``` + +- [ ] **Step 7.4: Run the tests** + +```bash +bun test tests/anthropic-request.test.ts --test-name-pattern "redacted_thinking|Branch 1" +``` + +Expected: Both PASS. + +- [ ] **Step 7.5: Run the full suite** + +```bash +bun test +``` + +Expected: All tests pass. + +- [ ] **Step 7.6: Commit** + +```bash +git add src/routes/messages/non-stream-translation.ts tests/anthropic-request.test.ts +git commit -m "feat: strip redacted_thinking and serialize server_tool_use in handleAssistantMessage" +``` + +--- + +### Task 8: Update `handleUserMessage` — array `tool_result` content + `web_search_tool_result` blocks + +**Files:** +- Modify: `src/routes/messages/non-stream-translation.ts` +- Test: `tests/anthropic-request.test.ts` + +- [ ] **Step 8.1: Write failing tests** + +> **Prerequisite:** Task 2 (Chunk 1) must be committed. `AnthropicWebSearchToolResultBlock` (added in Task 2, Step 2.3b) has `content: unknown`, so the test literal's `content` array shape is valid without any cast. + +Add to `tests/anthropic-request.test.ts`: + +```typescript +test("tool_result with array content containing image is translated to vision ContentPart", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Take a screenshot." }, + { + role: "assistant", + content: [{ type: "tool_use", id: "call_sc", name: "computer", input: { action: "screenshot" } }], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "call_sc", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgo=", + }, + }, + ], + }, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const toolMsg = result.messages.find((m) => m.role === "tool") + // Should be an array with an image_url part + expect(Array.isArray(toolMsg?.content)).toBe(true) + const parts = toolMsg?.content as Array<{ type: string }> + expect(parts[0].type).toBe("image_url") +}) + +test("tool_result with array content containing text is translated to string", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Run a command." }, + { + role: "assistant", + content: [{ type: "tool_use", id: "call_b", name: "Bash", input: { command: "ls" } }], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "call_b", + content: [ + { type: "text", text: "file1.txt\nfile2.txt" }, + ], + }, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const toolMsg = result.messages.find((m) => m.role === "tool") + expect(typeof toolMsg?.content).toBe("string") + expect(toolMsg?.content).toContain("file1.txt") +}) + +test("web_search_tool_result block is serialized as user message", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Search the web." }, + { + role: "user", + content: [ + { + type: "web_search_tool_result", + tool_use_id: "srv_ws_1", + content: [{ type: "web_search_result", url: "https://example.com", title: "Example", encrypted_content: "abc" }], + }, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const webResultMsg = result.messages.find( + (m) => m.role === "user" && typeof m.content === "string" && (m.content as string).includes("[Web search result:") + ) + expect(webResultMsg).toBeDefined() +}) +``` + +- [ ] **Step 8.2: Run the failing tests** + +```bash +bun test tests/anthropic-request.test.ts --test-name-pattern "tool_result with array|web_search_tool_result" +``` + +Expected: FAIL. + +- [ ] **Step 8.3: Add `mapToolResultContent` function to `non-stream-translation.ts`** + +After the `mapContent` function, add: + +```typescript +// Handles tool_result content which may be a string or array of content blocks +function mapToolResultContent( + content: AnthropicToolResultBlock["content"], +): string | Array | null { + if (typeof content === "string") { + return content + } + // content is an array of text/image/document blocks — reuse mapContent logic + return mapContent(content) +} +``` + +- [ ] **Step 8.4: Update `handleUserMessage` in `non-stream-translation.ts`** + +Replace the entire `handleUserMessage` function with: + +```typescript +function handleUserMessage(message: AnthropicUserMessage): Array { + const newMessages: Array = [] + + if (Array.isArray(message.content)) { + const toolResultBlocks = message.content.filter( + (block): block is AnthropicToolResultBlock => block.type === "tool_result", + ) + const webSearchResultBlocks = message.content.filter( + (block): block is AnthropicWebSearchToolResultBlock => + block.type === "web_search_tool_result", + ) + const otherBlocks = message.content.filter( + (block) => + block.type !== "tool_result" && + block.type !== "web_search_tool_result", + // document blocks remain here intentionally — mapContent handles them + ) + + // Tool results must come first to maintain protocol: tool_use -> tool_result -> user + for (const block of toolResultBlocks) { + newMessages.push({ + role: "tool", + tool_call_id: block.tool_use_id, + content: mapToolResultContent(block.content), + }) + } + + // Web search result blocks → serialize as user message + if (webSearchResultBlocks.length > 0) { + const text = webSearchResultBlocks + .map((b) => `[Web search result: ${JSON.stringify(b.content)}]`) + .join("\n\n") + newMessages.push({ role: "user", content: text }) + } + + if (otherBlocks.length > 0) { + newMessages.push({ + role: "user", + content: mapContent(otherBlocks), + }) + } + } else { + newMessages.push({ + role: "user", + content: mapContent(message.content), + }) + } + + return newMessages +} +``` + +- [ ] **Step 8.5: Run the tests** + +```bash +bun test tests/anthropic-request.test.ts --test-name-pattern "tool_result with array|web_search_tool_result" +``` + +Expected: All 3 PASS. + +- [ ] **Step 8.6: Run full suite** + +```bash +bun test +``` + +Expected: All tests pass. + +- [ ] **Step 8.7: Commit** + +```bash +git add src/routes/messages/non-stream-translation.ts tests/anthropic-request.test.ts +git commit -m "feat: handle array tool_result content and web_search_tool_result blocks in handleUserMessage" +``` + +--- + +## Chunk 4: Token Counting (`count-tokens-handler.ts`) + +### Task 9: Fix token overhead for Anthropic-typed tools + +**Files:** +- Modify: `src/routes/messages/count-tokens-handler.ts` +- Test: `tests/anthropic-request.test.ts` *(token counting uses a different handler — add a focused unit test)* + +- [ ] **Step 9.1: Verify `isTypedTool` discriminator (sanity check from Task 1)** + +Add to `tests/anthropic-request.test.ts` *(or confirm they already pass if you added them during Task 1)*: + +```typescript +// Import the token-counting logic for direct testing +import { isTypedTool } from "../src/routes/messages/anthropic-types" + +describe("isTypedTool discriminator", () => { + test("returns true for a typed tool (no input_schema)", () => { + const typedTool = { type: "bash_20250124", name: "bash" } + expect(isTypedTool(typedTool)).toBe(true) + }) + + test("returns false for a custom tool (has input_schema)", () => { + const customTool = { name: "Bash", description: "Run shell commands", input_schema: {} } + expect(isTypedTool(customTool)).toBe(false) + }) + + test("returns false for custom tool even if it has extra fields", () => { + const customTool = { name: "Bash", input_schema: {}, strict: true, cache_control: { type: "ephemeral" } } + expect(isTypedTool(customTool)).toBe(false) + }) +}) +``` + +- [ ] **Step 9.2: Confirm discriminator tests pass** + +```bash +bun test tests/anthropic-request.test.ts --test-name-pattern "isTypedTool" +``` + +Expected: All 3 PASS (the function was added in Task 1 — these confirm it's correctly exported and behaves as expected). This is a sanity check, not a red-first TDD step. + +- [ ] **Step 9.3: Update `count-tokens-handler.ts`** + +Open `src/routes/messages/count-tokens-handler.ts`. + +**Add import** at the top (after existing imports): + +```typescript +import { isTypedTool } from "./anthropic-types" +``` + +**Add the overhead table** before the `handleCountTokens` function: + +```typescript +// Token overhead for Anthropic-typed tools (per Anthropic pricing docs). +// Custom tools use the existing flat +346 for the entire tools array. +// Typed tools add per-tool overhead on top. +const ANTHROPIC_TYPED_TOOL_TOKEN_OVERHEAD: Record = { + "text_editor_20250728": 700, + "text_editor_20250429": 700, + "text_editor_20250124": 700, + "text_editor_20241022": 700, + "bash_20250124": 700, + "bash_20241022": 700, + // computer_use and web_search: overhead included in beta pricing, not additive +} +``` + +**Replace the existing tools block** inside `handleCountTokens` (the `if (anthropicPayload.tools && ...)` section): + +```typescript +if (anthropicPayload.tools && anthropicPayload.tools.length > 0) { + let mcpToolExist = false + if (anthropicBeta?.startsWith("claude-code")) { + mcpToolExist = anthropicPayload.tools.some( + (tool) => !isTypedTool(tool) && tool.name.startsWith("mcp__"), + ) + } + if (!mcpToolExist) { + if (anthropicPayload.model.startsWith("claude")) { + const hasCustomTools = anthropicPayload.tools.some((t) => !isTypedTool(t)) + const typedTools = anthropicPayload.tools.filter(isTypedTool) + + // Preserve existing flat +346 for the custom tools array (unchanged behavior) + if (hasCustomTools) { + tokenCount.input = tokenCount.input + 346 + } + // Add per-typed-tool overhead for Anthropic-typed tools (new) + for (const tool of typedTools) { + tokenCount.input = + tokenCount.input + (ANTHROPIC_TYPED_TOOL_TOKEN_OVERHEAD[tool.type] ?? 0) + } + } else if (anthropicPayload.model.startsWith("grok")) { + tokenCount.input = tokenCount.input + 480 + } + } +} +``` + +- [ ] **Step 9.4: Run typecheck** + +```bash +bun run typecheck +``` + +Expected: 0 errors. + +- [ ] **Step 9.5: Run full test suite** + +```bash +bun test +``` + +Expected: All tests pass. + +- [ ] **Step 9.6: Commit** + +```bash +git add src/routes/messages/count-tokens-handler.ts tests/anthropic-request.test.ts +git commit -m "feat: fix token overhead calculation to distinguish typed vs custom tools" +``` + +--- + +## Chunk 5: Final Verification + +### Task 10: Typecheck, lint, and full regression pass + +- [ ] **Step 10.1: Run typecheck — must be clean** + +```bash +bun run typecheck +``` + +Expected output: no errors. If any exist, fix them before proceeding. + +- [ ] **Step 10.2: Run lint — must be clean** + +```bash +bun run lint:all +``` + +Expected: no errors (auto-fixable issues are fixed; no unfixable warnings). + +- [ ] **Step 10.3: Run the full test suite** + +```bash +bun test +``` + +Expected: All tests PASS. Verify the new tests cover all 11 gaps: + +| Gap | Test | +|-----|------| +| Typed tool filtering | "should filter out Anthropic-typed tools" | +| tool_result array (image) | "tool_result with array content containing image" | +| tool_result array (text) | "tool_result with array content containing text" | +| strict forwarded | "should forward strict:true", "should not add strict field" | +| disable_parallel_tool_use | Parsed by type system — covered by TypeScript compile | +| server_tool_use block | "server_tool_use block in assistant message is serialized" | +| document block | "document block in user message produces placeholder text" | +| redacted_thinking block | "redacted_thinking block is stripped", "redacted_thinking in handleAssistantMessage" | +| token overhead | "isTypedTool discriminator" (typed tool filtering is the mechanism) | +| cache_control/defer_loading stripped | Covered by typed tool / custom tool separation tests | +| model name normalization | All 7 "translateModelName normalization" tests | + +- [ ] **Step 10.4: Build — make sure the project compiles** + +```bash +bun run build +``` + +Expected: Build succeeds with output in `dist/`. + +- [ ] **Step 10.5: Final commit (if any unstaged changes remain)** + +```bash +git status +# If there are unstaged changes (should be none if each task was committed properly): +git add src/routes/messages/anthropic-types.ts src/routes/messages/non-stream-translation.ts src/routes/messages/count-tokens-handler.ts src/services/copilot/create-chat-completions.ts tests/anthropic-request.test.ts +git commit -m "chore: verify tool parity — all 11 gaps fixed, typecheck and build clean" +``` + +--- + +### Task 11: Summary commit with changelog note + +- [ ] **Step 11.1: Verify git log shows all work** + +```bash +git log --oneline -10 +``` + +Expected: See all the commits from Tasks 1–10 cleanly stacked. + +- [ ] **Step 11.2: Done — no PR needed unless upstream integration is required** + +All 11 gaps are fixed across 4 files with full test coverage. The proxy now handles: +- **Claude Code v2.1.76** — all 24+ tools pass through correctly; extra fields (`cache_control`, `defer_loading`, `strict`, etc.) handled +- **Computer use workflows** — screenshot `tool_result` images translate to `image_url` +- **Multi-turn histories** — `server_tool_use`, `web_search_tool_result`, `redacted_thinking` no longer crash +- **Other Anthropic clients** — `bash_20250124`, `text_editor_*`, `computer_*` typed tools silently filtered +- **Model normalization** — all current and future `claude-4+` variants normalize correctly + +--- + +## Quick Reference + +| Command | Purpose | +|---------|---------| +| `bun test` | Run all tests | +| `bun test tests/anthropic-request.test.ts` | Run request-translation tests only | +| `bun run typecheck` | TypeScript type check (no emit) | +| `bun run lint:all` | ESLint entire project | +| `bun run build` | Compile to `dist/` via tsdown | +| `bun run dev` | Start in watch mode | diff --git a/docs/superpowers/plans/2026-03-16-web-search.md b/docs/superpowers/plans/2026-03-16-web-search.md new file mode 100644 index 000000000..9038799a4 --- /dev/null +++ b/docs/superpowers/plans/2026-03-16-web-search.md @@ -0,0 +1,1490 @@ +# Web Search Implementation Plan + +> **For agentic workers:** REQUIRED: Use superpowers:subagent-driven-development (if subagents available) or superpowers:executing-plans to implement this plan. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add transparent server-side web search via Brave Search API — when a client requests web search (typed tool or natural language), the proxy performs the search itself and injects results before forwarding to Copilot, replicating the real Anthropic API's server-side behavior. + +**Architecture:** Detection runs on the raw Anthropic payload (before `translateToOpenAI`) using `isTypedTool` + `WEB_SEARCH_TOOL_NAMES`. A new `web-search-detection.ts` module handles typed-tool and natural-language detection; a new `interceptor.ts` runs the non-streaming first pass + Brave call + second pass; `brave.ts` is a pure HTTP client for the Brave Search API. + +**Tech Stack:** TypeScript, Bun, `bun:test`, `fetch` (with `AbortController` for timeouts), `fetch-event-stream` (already in use), `consola` for logging. + +--- + +## File Map + +| Action | File | Responsibility | +|--------|------|----------------| +| Create | `src/services/web-search/types.ts` | `BraveSearchResult`, `BraveSearchError` types only | +| Create | `src/services/web-search/tool-definition.ts` | `WEB_SEARCH_TOOL_NAMES` set + `WEB_SEARCH_FUNCTION_TOOL` constant | +| Create | `src/services/web-search/brave.ts` | `searchBrave()` — HTTP call to Brave Search API, returns results or throws | +| Create | `src/services/web-search/interceptor.ts` | `webSearchInterceptor()` — tool-call loop (first pass → search → second pass) | +| Create | `src/routes/messages/web-search-detection.ts` | `detectWebSearchIntent()` + `stripWebSearchTypedTools()` — Anthropic-layer detection | +| Modify | `src/lib/state.ts` | Add `braveApiKey?: string` + `isWebSearchEnabled()` | +| Modify | `src/start.ts` | Read `BRAVE_API_KEY` env var, store in state, log startup message | +| Modify | `src/routes/messages/handler.ts` | Restructure to branch on web search intent; move `manualApprove` before translation | +| Create | `tests/web-search.test.ts` | All web search tests (detection, interceptor, Brave client) | + +--- + +## Chunk 1: Types, Constants, and Brave Client + +### Task 1: Shared types + +**Files:** +- Create: `src/services/web-search/types.ts` + +- [ ] **Step 1: Create `src/services/web-search/types.ts`** + +```typescript +export interface BraveSearchResult { + title: string + url: string + description: string +} + +export class BraveSearchError extends Error { + constructor(public readonly reason: string) { + super(`Brave search failed: ${reason}`) + this.name = "BraveSearchError" + } +} +``` + +- [ ] **Step 2: Run typecheck to verify no errors** + +```bash +bun run typecheck +``` + +Expected: no errors. + +- [ ] **Step 3: Commit** + +```bash +git add src/services/web-search/types.ts +git commit -m "feat: add BraveSearchResult and BraveSearchError types" +``` + +--- + +### Task 2: Tool name set and injected function tool constant + +**Files:** +- Create: `src/services/web-search/tool-definition.ts` +- Create: `tests/web-search.test.ts` (first tests) + +The `Tool` type is imported from `~/services/copilot/create-chat-completions` — that is the correct import path (not a local alias for something else). The `parameters` field is required (not optional) on `Tool`. + +- [ ] **Step 1: Write the failing tests** + +Create `tests/web-search.test.ts`: + +```typescript +import { describe, test, expect } from "bun:test" + +import { isTypedTool, type AnthropicTool } from "~/routes/messages/anthropic-types" +import { + WEB_SEARCH_TOOL_NAMES, + WEB_SEARCH_FUNCTION_TOOL, +} from "~/services/web-search/tool-definition" + +describe("WEB_SEARCH_TOOL_NAMES", () => { + test("contains web_search", () => { + expect(WEB_SEARCH_TOOL_NAMES.has("web_search")).toBe(true) + }) + + test("contains internet_research", () => { + expect(WEB_SEARCH_TOOL_NAMES.has("internet_research")).toBe(true) + }) + + test("does NOT contain search (too generic)", () => { + expect(WEB_SEARCH_TOOL_NAMES.has("search")).toBe(false) + }) +}) + +describe("Typed tool detection guard", () => { + test("typed tool named web_search — no input_schema — matches", () => { + const tool: AnthropicTool = { type: "web_search_20260101", name: "web_search" } + expect(isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name)).toBe(true) + }) + + test("typed tool named internet_research matches", () => { + const tool: AnthropicTool = { type: "internet_research_20260101", name: "internet_research" } + expect(isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name)).toBe(true) + }) + + test("future versioned type — detected by name, not type string", () => { + const tool: AnthropicTool = { type: "web_search_20260101", name: "web_search" } + // tool.type changed, but tool.name is still "web_search" — still detected + expect(isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name)).toBe(true) + }) + + test("custom tool named web_search WITH input_schema — NOT matched", () => { + const tool: AnthropicTool = { + name: "web_search", + input_schema: { type: "object", properties: {} }, + } + // isTypedTool returns false because input_schema is present + expect(isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name)).toBe(false) + }) + + test("custom tool named search WITH input_schema — NOT matched", () => { + const tool: AnthropicTool = { + name: "search", + input_schema: { type: "object", properties: {} }, + } + expect(isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name)).toBe(false) + }) +}) + +describe("WEB_SEARCH_FUNCTION_TOOL", () => { + test("type is function", () => { + expect(WEB_SEARCH_FUNCTION_TOOL.type).toBe("function") + }) + + test("function name is web_search", () => { + expect(WEB_SEARCH_FUNCTION_TOOL.function.name).toBe("web_search") + }) + + test("has parameters with query property", () => { + const params = WEB_SEARCH_FUNCTION_TOOL.function.parameters as { + properties: { query: unknown } + required: string[] + } + expect(params.properties.query).toBeDefined() + expect(params.required).toContain("query") + }) +}) +``` + +- [ ] **Step 2: Run tests to confirm they fail (module not found)** + +```bash +bun test tests/web-search.test.ts +``` + +Expected: error — `Cannot find module '~/services/web-search/tool-definition'` + +- [ ] **Step 3: Create `src/services/web-search/tool-definition.ts`** + +```typescript +import type { Tool } from "~/services/copilot/create-chat-completions" + +export const WEB_SEARCH_TOOL_NAMES = new Set([ + "web_search", + "internet_search", + "brave_search", + "bing_search", + "google_search", + "find_online", + "internet_research", +]) + +export const WEB_SEARCH_FUNCTION_TOOL: Tool = { + type: "function", + function: { + name: "web_search", + description: + "Search the web for current information. Use this when you need up-to-date facts, recent events, or information beyond your training data.", + parameters: { + type: "object", + properties: { + query: { + type: "string", + description: "The search query", + }, + }, + required: ["query"], + }, + }, +} +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +```bash +bun test tests/web-search.test.ts +``` + +Expected: all tests in the `WEB_SEARCH_TOOL_NAMES`, `Typed tool detection guard`, and `WEB_SEARCH_FUNCTION_TOOL` describe blocks pass. + +- [ ] **Step 5: Run typecheck** + +```bash +bun run typecheck +``` + +Expected: no errors. + +- [ ] **Step 6: Commit** + +```bash +git add src/services/web-search/tool-definition.ts tests/web-search.test.ts +git commit -m "feat: add WEB_SEARCH_TOOL_NAMES and WEB_SEARCH_FUNCTION_TOOL constants" +``` + +--- + +### Task 3: Brave Search API client + +**Files:** +- Create: `src/services/web-search/brave.ts` +- Modify: `tests/web-search.test.ts` (add Brave client tests) + +The Brave Web Search API endpoint is: +``` +GET https://api.search.brave.com/res/v1/web/search?q=&count=5 +Headers: + Accept: application/json + Accept-Encoding: gzip + X-Subscription-Token: +``` + +Successful response shape (only fields we use): +```typescript +{ + web?: { + results?: Array<{ + title: string + url: string + description?: string + }> + } +} +``` + +- [ ] **Step 1: Add Brave client tests to `tests/web-search.test.ts`** + +Append these `describe` blocks at the end of the file: + +```typescript +import { BraveSearchError, type BraveSearchResult } from "~/services/web-search/types" +import * as braveModule from "~/services/web-search/brave" + +describe("searchBrave — result formatting", () => { + test("formats top 5 results as BraveSearchResult[]", async () => { + // Mock global fetch for this test + const mockResponse = { + web: { + results: [ + { title: "Result 1", url: "https://example.com/1", description: "Desc 1" }, + { title: "Result 2", url: "https://example.com/2", description: "Desc 2" }, + { title: "Result 3", url: "https://example.com/3", description: "Desc 3" }, + { title: "Result 4", url: "https://example.com/4", description: "Desc 4" }, + { title: "Result 5", url: "https://example.com/5", description: "Desc 5" }, + ], + }, + } + const originalFetch = globalThis.fetch + globalThis.fetch = async () => + new Response(JSON.stringify(mockResponse), { status: 200 }) + + try { + const results = await braveModule.searchBrave("test query", "fake-api-key") + expect(results).toHaveLength(5) + expect(results[0]).toEqual({ + title: "Result 1", + url: "https://example.com/1", + description: "Desc 1", + }) + expect(results[4]?.url).toBe("https://example.com/5") + } finally { + globalThis.fetch = originalFetch + } + }) + + test("returns empty array when web.results is empty", async () => { + const mockResponse = { web: { results: [] } } + const originalFetch = globalThis.fetch + globalThis.fetch = async () => + new Response(JSON.stringify(mockResponse), { status: 200 }) + + try { + const results = await braveModule.searchBrave("nothing here", "fake-api-key") + expect(results).toHaveLength(0) + } finally { + globalThis.fetch = originalFetch + } + }) + + test("returns empty array when web key is absent", async () => { + const mockResponse = {} + const originalFetch = globalThis.fetch + globalThis.fetch = async () => + new Response(JSON.stringify(mockResponse), { status: 200 }) + + try { + const results = await braveModule.searchBrave("nothing", "fake-api-key") + expect(results).toHaveLength(0) + } finally { + globalThis.fetch = originalFetch + } + }) + + test("uses empty string for missing description field", async () => { + const mockResponse = { + web: { results: [{ title: "T", url: "https://u.com" }] }, + } + const originalFetch = globalThis.fetch + globalThis.fetch = async () => + new Response(JSON.stringify(mockResponse), { status: 200 }) + + try { + const results = await braveModule.searchBrave("q", "fake-api-key") + expect(results[0]?.description).toBe("") + } finally { + globalThis.fetch = originalFetch + } + }) +}) + +describe("searchBrave — error handling", () => { + test("throws BraveSearchError on non-200 response", async () => { + const originalFetch = globalThis.fetch + globalThis.fetch = async () => + new Response("Forbidden", { status: 403 }) + + try { + await expect(braveModule.searchBrave("query", "bad-key")).rejects.toBeInstanceOf(BraveSearchError) + } finally { + globalThis.fetch = originalFetch + } + }) + + test("BraveSearchError reason includes status code on non-200", async () => { + const originalFetch = globalThis.fetch + globalThis.fetch = async () => + new Response("Forbidden", { status: 403 }) + + try { + // Use expect.assertions to ensure the catch handler runs (prevents silent pass if no throw) + expect.assertions(2) + await braveModule.searchBrave("query", "bad-key").catch((e: BraveSearchError) => { + expect(e).toBeInstanceOf(BraveSearchError) + expect(e.reason).toContain("403") + }) + } finally { + globalThis.fetch = originalFetch + } + }) + + test("throws BraveSearchError on network failure", async () => { + const originalFetch = globalThis.fetch + globalThis.fetch = async () => { + throw new Error("network error") + } + + try { + await expect(braveModule.searchBrave("query", "key")).rejects.toBeInstanceOf(BraveSearchError) + } finally { + globalThis.fetch = originalFetch + } + }) +}) +``` + +- [ ] **Step 2: Run the new tests to confirm they fail** + +```bash +bun test tests/web-search.test.ts +``` + +Expected: `Cannot find module '~/services/web-search/brave'` + +- [ ] **Step 3: Create `src/services/web-search/brave.ts`** + +```typescript +import { BraveSearchError, type BraveSearchResult } from "./types" + +const BRAVE_SEARCH_URL = "https://api.search.brave.com/res/v1/web/search" +const TIMEOUT_MS = 5000 +const MAX_RESULTS = 5 + +export async function searchBrave( + query: string, + apiKey: string, +): Promise { + const controller = new AbortController() + const timeoutId = setTimeout(() => { + controller.abort() + }, TIMEOUT_MS) + + try { + const url = new URL(BRAVE_SEARCH_URL) + url.searchParams.set("q", query) + url.searchParams.set("count", String(MAX_RESULTS)) + + const response = await fetch(url.toString(), { + headers: { + Accept: "application/json", + "Accept-Encoding": "gzip", + "X-Subscription-Token": apiKey, + }, + signal: controller.signal, + }) + + if (!response.ok) { + throw new BraveSearchError(`HTTP ${response.status}`) + } + + const data = (await response.json()) as { + web?: { + results?: Array<{ + title: string + url: string + description?: string + }> + } + } + + return (data.web?.results ?? []).map((r) => ({ + title: r.title, + url: r.url, + description: r.description ?? "", + })) + } catch (error) { + if (error instanceof BraveSearchError) { + throw error + } + const reason = + error instanceof Error ? error.message : "unknown network error" + throw new BraveSearchError(reason) + } finally { + clearTimeout(timeoutId) + } +} +``` + +- [ ] **Step 4: Run the Brave client tests to verify they pass** + +```bash +bun test tests/web-search.test.ts +``` + +Expected: all tests pass including the new `searchBrave` describe blocks. + +- [ ] **Step 5: Run typecheck** + +```bash +bun run typecheck +``` + +Expected: no errors. + +- [ ] **Step 6: Commit** + +```bash +git add src/services/web-search/brave.ts tests/web-search.test.ts +git commit -m "feat: add Brave Search API client with 5s timeout" +``` + +--- + +## Chunk 2: Interceptor + +### Task 4: Web search interceptor (tool-call loop) + +**Files:** +- Create: `src/services/web-search/interceptor.ts` +- Modify: `tests/web-search.test.ts` (add interceptor tests) + +The interceptor receives an OpenAI `ChatCompletionsPayload` with the `web_search` function tool already injected. It knows nothing about Anthropic types. Its job: +1. Call Copilot non-streaming (first pass) +2. If `finish_reason !== "tool_calls"` or no `web_search` call → return as-is +3. If `web_search` call found → call Brave → build messages → call Copilot again (second pass, original stream flag) + +The `formatSearchResults` helper is a pure function that converts `BraveSearchResult[]` into the plain-text string injected as the tool result. It also handles the zero-results case and the failure case. + +- [ ] **Step 1: Add interceptor tests to `tests/web-search.test.ts`** + +Append these imports and describe blocks at the end of the file. Tests use `spyOn` from `bun:test` (Bun's ESM-compatible mocking API) — never `@ts-ignore` namespace reassignment, which is read-only in native ESM. + +```typescript +import { spyOn, beforeEach, afterEach, mock } from "bun:test" + +import { webSearchInterceptor } from "~/services/web-search/interceptor" +import type { ChatCompletionsPayload, ChatCompletionResponse, Message } from "~/services/copilot/create-chat-completions" +import * as createChatCompletionsModule from "~/services/copilot/create-chat-completions" +import { state } from "~/lib/state" + +// Helper: build a minimal non-streaming ChatCompletionResponse +function makeCopilotResponse( + finishReason: "stop" | "tool_calls", + toolCalls?: Array<{ id: string; name: string; arguments: string }>, +): ChatCompletionResponse { + return { + id: "resp-1", + object: "chat.completion", + created: 0, + model: "gpt-4o", + choices: [ + { + index: 0, + logprobs: null, + finish_reason: finishReason, + message: { + role: "assistant", + content: finishReason === "stop" ? "Here is my answer." : null, + tool_calls: toolCalls?.map((tc) => ({ + id: tc.id, + type: "function" as const, + function: { name: tc.name, arguments: tc.arguments }, + })), + }, + }, + ], + } +} + +function makePayload(stream = false): ChatCompletionsPayload { + return { + model: "gpt-4o", + stream, + messages: [{ role: "user", content: "What is the weather today?" }], + tools: [ + { + type: "function", + function: { + name: "web_search", + description: "Search the web", + parameters: { type: "object", properties: { query: { type: "string" } }, required: ["query"] }, + }, + }, + ], + } +} + +describe("webSearchInterceptor — no search path", () => { + afterEach(() => { + mock.restore() + }) + + test("returns response as-is when finish_reason is stop", async () => { + const stopResponse = makeCopilotResponse("stop") + const createSpy = spyOn(createChatCompletionsModule, "createChatCompletions") + .mockResolvedValue(stopResponse) + + const result = await webSearchInterceptor(makePayload()) + + expect(result).toEqual(stopResponse) + expect(createSpy).toHaveBeenCalledTimes(1) + }) + + test("returns response as-is when tool_calls is for a different tool", async () => { + const otherToolResponse = makeCopilotResponse("tool_calls", [ + { id: "tc-1", name: "bash", arguments: '{"command":"ls"}' }, + ]) + const createSpy = spyOn(createChatCompletionsModule, "createChatCompletions") + .mockResolvedValue(otherToolResponse) + + const result = await webSearchInterceptor(makePayload()) + + expect(result).toEqual(otherToolResponse) + expect(createSpy).toHaveBeenCalledTimes(1) + }) +}) + +describe("webSearchInterceptor — search path", () => { + beforeEach(() => { + // The interceptor guards on state.braveApiKey before calling searchBrave. + // Set a fake key so the guard passes and the spy is reached. + state.braveApiKey = "test-key" + }) + + afterEach(() => { + state.braveApiKey = undefined + mock.restore() + }) + + test("calls Brave and makes a second Copilot call when web_search is triggered", async () => { + const firstResponse = makeCopilotResponse("tool_calls", [ + { id: "tc-ws", name: "web_search", arguments: '{"query":"latest AI news"}' }, + ]) + const finalResponse = makeCopilotResponse("stop") + const braveResults = [ + { title: "AI News", url: "https://ainews.com", description: "Latest AI developments" }, + ] + + const createSpy = spyOn(createChatCompletionsModule, "createChatCompletions") + .mockResolvedValueOnce(firstResponse) + .mockResolvedValueOnce(finalResponse) + spyOn(braveModule, "searchBrave").mockResolvedValue(braveResults) + + const result = await webSearchInterceptor(makePayload()) + + expect(createSpy).toHaveBeenCalledTimes(2) + expect(result).toEqual(finalResponse) + // First pass must be non-streaming + expect(createSpy.mock.calls[0]?.[0]?.stream).toBe(false) + }) + + test("second pass uses original stream flag", async () => { + const firstResponse = makeCopilotResponse("tool_calls", [ + { id: "tc-ws", name: "web_search", arguments: '{"query":"news"}' }, + ]) + const finalResponse = makeCopilotResponse("stop") + + const createSpy = spyOn(createChatCompletionsModule, "createChatCompletions") + .mockResolvedValueOnce(firstResponse) + .mockResolvedValueOnce(finalResponse) + spyOn(braveModule, "searchBrave").mockResolvedValue([]) + + // streaming request + await webSearchInterceptor(makePayload(true)) + + expect(createSpy).toHaveBeenCalledTimes(2) + expect(createSpy.mock.calls[0]?.[0]?.stream).toBe(false) // first pass always non-streaming + expect(createSpy.mock.calls[1]?.[0]?.stream).toBe(true) // second pass uses original stream=true + }) + + test("injects stub tool results for non-search tool_calls alongside web_search", async () => { + const firstResponse = makeCopilotResponse("tool_calls", [ + { id: "tc-ws", name: "web_search", arguments: '{"query":"q"}' }, + { id: "tc-bash", name: "bash", arguments: '{"command":"ls"}' }, + ]) + const finalResponse = makeCopilotResponse("stop") + + const createSpy = spyOn(createChatCompletionsModule, "createChatCompletions") + .mockResolvedValueOnce(firstResponse) + .mockResolvedValueOnce(finalResponse) + spyOn(braveModule, "searchBrave").mockResolvedValue([]) + + await webSearchInterceptor(makePayload()) + + // Messages in second call should include tool results for BOTH tool_calls + const secondCallMessages = createSpy.mock.calls[1]?.[0]?.messages as Message[] + const toolMessages = secondCallMessages.filter((m) => m.role === "tool") + expect(toolMessages).toHaveLength(2) + const toolIds = toolMessages.map((m) => m.tool_call_id) + expect(toolIds).toContain("tc-ws") + expect(toolIds).toContain("tc-bash") + }) + + test("injects failure message when Brave throws BraveSearchError", async () => { + const firstResponse = makeCopilotResponse("tool_calls", [ + { id: "tc-ws", name: "web_search", arguments: '{"query":"q"}' }, + ]) + const finalResponse = makeCopilotResponse("stop") + + const createSpy = spyOn(createChatCompletionsModule, "createChatCompletions") + .mockResolvedValueOnce(firstResponse) + .mockResolvedValueOnce(finalResponse) + spyOn(braveModule, "searchBrave").mockRejectedValue(new BraveSearchError("HTTP 429")) + + await webSearchInterceptor(makePayload()) + + expect(createSpy).toHaveBeenCalledTimes(2) + const secondCallMessages = createSpy.mock.calls[1]?.[0]?.messages as Message[] + const toolMsg = secondCallMessages.find((m) => m.role === "tool") + expect(toolMsg?.content).toContain("Web search failed") + expect(toolMsg?.content).toContain("training data") + }) + + test("injects failure message when query JSON.parse fails", async () => { + const firstResponse = makeCopilotResponse("tool_calls", [ + { id: "tc-ws", name: "web_search", arguments: "INVALID_JSON" }, + ]) + const finalResponse = makeCopilotResponse("stop") + + const createSpy = spyOn(createChatCompletionsModule, "createChatCompletions") + .mockResolvedValueOnce(firstResponse) + .mockResolvedValueOnce(finalResponse) + spyOn(braveModule, "searchBrave").mockResolvedValue([]) + + await webSearchInterceptor(makePayload()) + + expect(createSpy).toHaveBeenCalledTimes(2) + const secondCallMessages = createSpy.mock.calls[1]?.[0]?.messages as Message[] + const toolMsg = secondCallMessages.find((m) => m.role === "tool") + expect(toolMsg?.content).toContain("Web search failed") + }) + + test("passes tool_choice through to second Copilot call unchanged", async () => { + const firstResponse = makeCopilotResponse("tool_calls", [ + { id: "tc-ws", name: "web_search", arguments: '{"query":"q"}' }, + ]) + const finalResponse = makeCopilotResponse("stop") + + const createSpy = spyOn(createChatCompletionsModule, "createChatCompletions") + .mockResolvedValueOnce(firstResponse) + .mockResolvedValueOnce(finalResponse) + spyOn(braveModule, "searchBrave").mockResolvedValue([]) + + const payloadWithChoice: ChatCompletionsPayload = { + ...makePayload(), + tool_choice: { type: "function", function: { name: "web_search" } }, + } + + await webSearchInterceptor(payloadWithChoice) + + // tool_choice must be forwarded to the second (synthesis) call unchanged + expect(createSpy.mock.calls[1]?.[0]?.tool_choice).toEqual({ + type: "function", + function: { name: "web_search" }, + }) + }) +}) +``` + +- [ ] **Step 2: Run the new tests to confirm they fail** + +```bash +bun test tests/web-search.test.ts 2>&1 | head -20 +``` + +Expected: `Cannot find module '~/services/web-search/interceptor'` + +- [ ] **Step 3: Create `src/services/web-search/interceptor.ts`** + +```typescript +import consola from "consola" + +import { + createChatCompletions, + type ChatCompletionsPayload, + type ChatCompletionResponse, + type Message, +} from "~/services/copilot/create-chat-completions" +import { state } from "~/lib/state" + +import { searchBrave } from "./brave" +import { BraveSearchError, type BraveSearchResult } from "./types" + +export async function webSearchInterceptor( + payload: ChatCompletionsPayload, +): ReturnType { + // First pass: always non-streaming so we can inspect finish_reason + const firstPassPayload: ChatCompletionsPayload = { ...payload, stream: false } + const firstResponse = (await createChatCompletions( + firstPassPayload, + )) as ChatCompletionResponse + + const choice = firstResponse.choices[0] + if (!choice || choice.finish_reason !== "tool_calls" || !choice.message.tool_calls) { + return firstResponse + } + + const webSearchCall = choice.message.tool_calls.find( + (tc) => tc.function.name === "web_search", + ) + if (!webSearchCall) { + // tool_calls but no web_search call — return first response as-is + return firstResponse + } + + // Parse query + let toolResultContent: string | undefined + try { + const args = JSON.parse(webSearchCall.function.arguments) as { query: string } + const query = args.query + + let results: BraveSearchResult[] + try { + if (!state.braveApiKey) throw new BraveSearchError("BRAVE_API_KEY not set") + results = await searchBrave(query, state.braveApiKey) + } catch (error) { + const reason = error instanceof BraveSearchError ? error.reason : String(error) + consola.warn(`Web search failed: ${reason}`) + toolResultContent = `Web search failed: ${reason}\nPlease answer based on your training data and let the user know that web search is currently unavailable.` + } + + if (toolResultContent === undefined) { + toolResultContent = formatSearchResults(query, results!) + } + } catch { + consola.warn("Web search: failed to parse tool call arguments") + toolResultContent = + "Web search failed: could not parse search query.\nPlease answer based on your training data and let the user know that web search is currently unavailable." + } + + // Build messages for second pass + const assistantMessage: Message = { + role: "assistant", + content: choice.message.content ?? null, + tool_calls: choice.message.tool_calls, + } + + // Inject a tool result for every tool_call in the assistant message. + // Non-search tool calls get an empty stub so Copilot's second pass has + // a complete result set (required — partial results cause rejection). + const toolResultMessages: Message[] = choice.message.tool_calls.map((tc) => ({ + role: "tool", + tool_call_id: tc.id, + // toolResultContent is always set before reaching this point (both try branches assign it) + content: tc.id === webSearchCall.id ? (toolResultContent ?? "") : "", + })) + + const secondPassMessages: Message[] = [ + ...payload.messages, + assistantMessage, + ...toolResultMessages, + ] + + // Second pass: use original stream flag + return createChatCompletions({ + ...payload, + messages: secondPassMessages, + }) +} + +function formatSearchResults(query: string, results: BraveSearchResult[]): string { + if (results.length === 0) { + return `No results found for: "${query}"` + } + + const lines = [`Web search results for: "${query}"`, ""] + for (const [i, result] of results.entries()) { + lines.push(`${i + 1}. Title: ${result.title}`) + lines.push(` URL: ${result.url}`) + lines.push(` Snippet: ${result.description}`) + lines.push("") + } + + return lines.join("\n").trimEnd() +} +``` + +- [ ] **Step 4: Run the interceptor tests to verify they pass** + +```bash +bun test tests/web-search.test.ts +``` + +Expected: all tests pass. + +- [ ] **Step 5: Typecheck deferred to Task 5** + +The interceptor references `state.braveApiKey`, which is not added to the `State` interface until Task 5. Running `bun run typecheck` here will fail with `Property 'braveApiKey' does not exist`. Proceed to commit; the typecheck is covered in Task 5 Step 3 once the state field is defined. + +- [ ] **Step 6: Commit** + +```bash +git add src/services/web-search/interceptor.ts tests/web-search.test.ts +git commit -m "feat: add web search interceptor (tool-call loop)" +``` + +> ⚠️ **Typecheck window**: After this commit, `bun run typecheck` will fail because `interceptor.ts` references `state.braveApiKey` which does not yet exist on the `State` interface. This is resolved in Task 5 Step 1. Do not run `bun run typecheck` between Tasks 4 and 5. + +--- + +## Chunk 3: Detection, State, Startup, Handler Wiring + +### Task 5: State and startup changes + +**Files:** +- Modify: `src/lib/state.ts` +- Modify: `src/start.ts` + +- [ ] **Step 1: Add `braveApiKey` to `State` and `isWebSearchEnabled()` to `src/lib/state.ts`** + +Current `State` interface ends at `lastRequestTimestamp?: number`. Add after it: + +```typescript + braveApiKey?: string +``` + +And add this function after the `state` export: + +```typescript +export function isWebSearchEnabled(): boolean { + return !!state.braveApiKey +} +``` + +- [ ] **Step 2: Add env var reading to `src/start.ts`** + +In `runServer`, after `state.showToken = options.showToken` (around line 47), add: + +```typescript + const braveApiKey = process.env.BRAVE_API_KEY + if (braveApiKey) { + state.braveApiKey = braveApiKey + consola.info("Web search enabled (Brave) — note: each web search request uses 2-3 Copilot API calls") + } +``` + +- [ ] **Step 3: Run typecheck** + +```bash +bun run typecheck +``` + +Expected: no errors. + +- [ ] **Step 4: Run all tests to verify nothing broke** + +```bash +bun test +``` + +Expected: all existing tests pass, no new failures. + +- [ ] **Step 5: Commit** + +```bash +git add src/lib/state.ts src/start.ts +git commit -m "feat: add braveApiKey to state and isWebSearchEnabled() helper" +``` + +--- + +### Task 6: Web search detection module + +**Files:** +- Create: `src/routes/messages/web-search-detection.ts` +- Modify: `tests/web-search.test.ts` (add detection tests) + +This module has two exports: +1. `detectWebSearchIntent(payload)` — Path 1 (zero cost typed tool check) then Path 2 (preflight Copilot call) +2. `stripWebSearchTypedTools(payload)` — returns new payload without web search typed tools + +For Path 2's preflight model: use `state.models?.data`, pick the first model whose ID is different from `payload.model`. If there's only one model or `state.models` is unavailable, fall back to `payload.model`. The preflight uses `stream: false` and `max_tokens: 5`. + +The last user message for Path 2 is extracted by scanning `payload.messages` from the end for the first `role: "user"` entry. If the message content is a string, use it directly. If it's an array of content blocks, join all `type: "text"` block texts. + +- [ ] **Step 1: Add detection tests to `tests/web-search.test.ts`** + +Append at the end of the file: + +```typescript +import { + detectWebSearchIntent, + stripWebSearchTypedTools, +} from "~/routes/messages/web-search-detection" +import * as stateModule from "~/lib/state" + +function makeAnthropicPayload( + tools?: AnthropicMessagesPayload["tools"], + lastUserContent = "Tell me about yourself", +): AnthropicMessagesPayload { + return { + model: "claude-opus-4", + max_tokens: 1024, + messages: [{ role: "user", content: lastUserContent }], + tools, + } +} + +function makePreflightResponse(answer: "yes" | "no"): ChatCompletionResponse { + return { + id: "preflight-1", + object: "chat.completion", + created: 0, + model: "gpt-4o-mini", + choices: [ + { + index: 0, + logprobs: null, + finish_reason: "stop", + message: { role: "assistant", content: answer }, + }, + ], + } +} + +describe("stripWebSearchTypedTools", () => { + test("removes typed web_search tool from tools array", () => { + const payload = makeAnthropicPayload([ + { type: "web_search_20250305", name: "web_search" }, + { name: "bash", description: "Run bash", input_schema: { type: "object", properties: {}, required: [] } }, + ]) + const stripped = stripWebSearchTypedTools(payload) + expect(stripped.tools).toHaveLength(1) + expect(stripped.tools?.[0]).toMatchObject({ name: "bash" }) + }) + + test("keeps non-search typed tools (e.g. bash_20250124)", () => { + const payload = makeAnthropicPayload([ + { type: "web_search_20250305", name: "web_search" }, + { type: "bash_20250124", name: "bash" }, + ]) + const stripped = stripWebSearchTypedTools(payload) + expect(stripped.tools).toHaveLength(1) + expect(stripped.tools?.[0]).toMatchObject({ name: "bash" }) + }) + + test("returns payload unchanged when no web search tools present", () => { + const payload = makeAnthropicPayload([ + { name: "my_tool", description: "A tool", input_schema: { type: "object", properties: {} } }, + ]) + const stripped = stripWebSearchTypedTools(payload) + expect(stripped.tools).toHaveLength(1) + }) + + test("does not mutate original payload", () => { + const payload = makeAnthropicPayload([ + { type: "web_search_20250305", name: "web_search" }, + ]) + const originalToolsLength = payload.tools?.length + stripWebSearchTypedTools(payload) + expect(payload.tools?.length).toBe(originalToolsLength) + }) +}) + +describe("detectWebSearchIntent — Path 1 (typed tool)", () => { + afterEach(() => { + mock.restore() + }) + + test("returns true immediately when typed web_search tool present (no preflight call)", async () => { + const payload = makeAnthropicPayload([ + { type: "web_search_20250305", name: "web_search" }, + ]) + + // Set up spy to verify Path 1 short-circuits before any preflight call + const createSpy = spyOn(createChatCompletionsModule, "createChatCompletions") + + const result = await detectWebSearchIntent(payload) + + expect(result).toBe(true) + expect(createSpy).not.toHaveBeenCalled() + }) +}) + +describe("detectWebSearchIntent — Path 2 (natural language preflight)", () => { + afterEach(() => { + mock.restore() + }) + + test("returns true when preflight responds yes", async () => { + const payload = makeAnthropicPayload(undefined, "What happened in the news today?") + + spyOn(createChatCompletionsModule, "createChatCompletions") + .mockResolvedValue(makePreflightResponse("yes")) + + const result = await detectWebSearchIntent(payload) + + expect(result).toBe(true) + }) + + test("returns false when preflight responds no", async () => { + const payload = makeAnthropicPayload(undefined, "Write me a poem") + + spyOn(createChatCompletionsModule, "createChatCompletions") + .mockResolvedValue(makePreflightResponse("no")) + + const result = await detectWebSearchIntent(payload) + + expect(result).toBe(false) + }) + + test("returns false (and logs warning) when preflight call throws", async () => { + const payload = makeAnthropicPayload(undefined, "Search for something") + + spyOn(createChatCompletionsModule, "createChatCompletions") + .mockRejectedValue(new Error("network failure")) + + const result = await detectWebSearchIntent(payload) + + expect(result).toBe(false) + // No exception propagated — graceful fallback + }) +}) +``` + +- [ ] **Step 2: Run the new tests to confirm they fail** + +```bash +bun test tests/web-search.test.ts 2>&1 | head -20 +``` + +Expected: `Cannot find module '~/routes/messages/web-search-detection'` + +- [ ] **Step 3: Create `src/routes/messages/web-search-detection.ts`** + +```typescript +import consola from "consola" + +import { + createChatCompletions, + type ChatCompletionResponse, +} from "~/services/copilot/create-chat-completions" +import { state } from "~/lib/state" +import { WEB_SEARCH_TOOL_NAMES } from "~/services/web-search/tool-definition" + +import { isTypedTool, type AnthropicMessagesPayload } from "./anthropic-types" + +/** + * Returns true if this request should trigger a web search. + * + * Path 1: Zero-cost — checks if any typed tool in the request has a name + * in WEB_SEARCH_TOOL_NAMES. Short-circuits to true without an API call. + * + * Path 2: Only fires when Path 1 is false. Sends a lightweight preflight + * classification request to Copilot asking whether the last user message + * requires real-time web data. Falls back to false on any failure. + */ +export async function detectWebSearchIntent( + payload: AnthropicMessagesPayload, +): Promise { + // Path 1: typed tool detection (free) + const hasWebSearchTypedTool = payload.tools?.some( + (tool) => isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name), + ) ?? false + + if (hasWebSearchTypedTool) { + return true + } + + // Path 2: natural language preflight (costs one Copilot API call) + const lastUserMessage = getLastUserMessageText(payload) + if (!lastUserMessage) { + return false + } + + try { + const preflightModel = getPreflightModel(payload.model) + const response = (await createChatCompletions({ + model: preflightModel, + stream: false, + max_tokens: 5, + messages: [ + { + role: "system", + content: + 'You are a classifier. Answer only "yes" or "no". No explanation.', + }, + { + role: "user", + content: `Does this message require searching the web for current or real-time information?\nMessage: "${lastUserMessage}"`, + }, + ], + })) as ChatCompletionResponse + + const answer = response.choices[0]?.message.content?.trim().toLowerCase() ?? "" + return answer === "yes" + } catch (error) { + consola.warn( + "Web search preflight classification failed, treating as no-search-needed:", + error, + ) + return false + } +} + +/** + * Returns a new payload with all typed web search tools removed. + * Does not mutate the input. + */ +export function stripWebSearchTypedTools( + payload: AnthropicMessagesPayload, +): AnthropicMessagesPayload { + return { + ...payload, + tools: payload.tools?.filter( + (tool) => !(isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name)), + ), + } +} + +function getLastUserMessageText(payload: AnthropicMessagesPayload): string { + for (let i = payload.messages.length - 1; i >= 0; i--) { + const msg = payload.messages[i] + if (msg?.role !== "user") continue + if (typeof msg.content === "string") return msg.content + if (Array.isArray(msg.content)) { + return msg.content + .filter((block): block is { type: "text"; text: string } => block.type === "text") + .map((block) => block.text) + .join(" ") + } + } + return "" +} + +function getPreflightModel(requestModel: string): string { + const models = state.models?.data ?? [] + const alternative = models.find((m) => m.id !== requestModel) + return alternative?.id ?? requestModel +} +``` + +- [ ] **Step 4: Run the detection tests to verify they pass** + +```bash +bun test tests/web-search.test.ts +``` + +Expected: all detection tests pass. + +- [ ] **Step 5: Run typecheck** + +```bash +bun run typecheck +``` + +Expected: no errors. + +- [ ] **Step 6: Commit** + +```bash +git add src/routes/messages/web-search-detection.ts tests/web-search.test.ts +git commit -m "feat: add web search detection and stripping module" +``` + +--- + +### Task 7: Wire everything into handler.ts + +**Files:** +- Modify: `src/routes/messages/handler.ts` +- Modify: `tests/web-search.test.ts` (add end-to-end disabled test) + +This task restructures the handler to move `manualApprove` before translation and branch on web search intent. The current handler code is: + +```typescript +// Current (before change): +const anthropicPayload = await c.req.json() +// ...debug log... +const openAIPayload = translateToOpenAI(anthropicPayload) +// ...debug log... +if (state.manualApprove) { + await awaitApproval() +} +const response = await createChatCompletions(openAIPayload) +``` + +The new structure: + +```typescript +// New (after change): +const anthropicPayload = await c.req.json() +// ...debug log... + +if (state.manualApprove) { + await awaitApproval() +} + +let response: Awaited> + +if (isWebSearchEnabled() && await detectWebSearchIntent(anthropicPayload)) { + const cleanedPayload = stripWebSearchTypedTools(anthropicPayload) + const openAIPayload = translateToOpenAI(cleanedPayload) + consola.debug("Translated OpenAI request payload (web search):", JSON.stringify(openAIPayload)) + openAIPayload.tools = [...(openAIPayload.tools ?? []), WEB_SEARCH_FUNCTION_TOOL] + response = await webSearchInterceptor(openAIPayload) +} else { + const openAIPayload = translateToOpenAI(anthropicPayload) + consola.debug("Translated OpenAI request payload:", JSON.stringify(openAIPayload)) + response = await createChatCompletions(openAIPayload) +} +``` + +Add these imports to the handler: +```typescript +import { isWebSearchEnabled } from "~/lib/state" +import { detectWebSearchIntent, stripWebSearchTypedTools } from "./web-search-detection" +import { webSearchInterceptor } from "~/services/web-search/interceptor" +import { WEB_SEARCH_FUNCTION_TOOL } from "~/services/web-search/tool-definition" +``` + +- [ ] **Step 1: Add end-to-end disabled test to `tests/web-search.test.ts`** + +Append at the end: + +```typescript +describe("isWebSearchEnabled", () => { + test("returns false when braveApiKey is not set", () => { + const originalKey = stateModule.state.braveApiKey + stateModule.state.braveApiKey = undefined + try { + expect(stateModule.isWebSearchEnabled()).toBe(false) + } finally { + stateModule.state.braveApiKey = originalKey + } + }) + + test("returns true when braveApiKey is set", () => { + const originalKey = stateModule.state.braveApiKey + stateModule.state.braveApiKey = "test-key" + try { + expect(stateModule.isWebSearchEnabled()).toBe(true) + } finally { + stateModule.state.braveApiKey = originalKey + } + }) +}) +``` + +- [ ] **Step 2: Run the new tests to verify they pass** + +```bash +bun test tests/web-search.test.ts +``` + +Expected: the `isWebSearchEnabled` tests pass (the helper is already implemented from Task 5). + +- [ ] **Step 3: Rewrite `src/routes/messages/handler.ts`** + +Replace the full file contents with the restructured version. Preserve all existing debug logging. The file after rewrite: + +```typescript +import type { Context } from "hono" + +import consola from "consola" +import { streamSSE } from "hono/streaming" + +import { awaitApproval } from "~/lib/approval" +import { checkRateLimit } from "~/lib/rate-limit" +import { isWebSearchEnabled, state } from "~/lib/state" +import { + createChatCompletions, + type ChatCompletionChunk, + type ChatCompletionResponse, +} from "~/services/copilot/create-chat-completions" +import { webSearchInterceptor } from "~/services/web-search/interceptor" +import { WEB_SEARCH_FUNCTION_TOOL } from "~/services/web-search/tool-definition" + +import { + type AnthropicMessagesPayload, + type AnthropicStreamState, +} from "./anthropic-types" +import { + translateToAnthropic, + translateToOpenAI, +} from "./non-stream-translation" +import { translateChunkToAnthropicEvents } from "./stream-translation" +import { + detectWebSearchIntent, + stripWebSearchTypedTools, +} from "./web-search-detection" + +export async function handleCompletion(c: Context) { + await checkRateLimit(state) + + const anthropicPayload = await c.req.json() + consola.debug("Anthropic request payload:", JSON.stringify(anthropicPayload)) + + if (state.manualApprove) { + await awaitApproval() + } + + let response: Awaited> + + if (isWebSearchEnabled() && (await detectWebSearchIntent(anthropicPayload))) { + const cleanedPayload = stripWebSearchTypedTools(anthropicPayload) + const openAIPayload = translateToOpenAI(cleanedPayload) + openAIPayload.tools = [...(openAIPayload.tools ?? []), WEB_SEARCH_FUNCTION_TOOL] + consola.debug( + "Translated OpenAI request payload (web search):", + JSON.stringify(openAIPayload), + ) + response = await webSearchInterceptor(openAIPayload) + } else { + const openAIPayload = translateToOpenAI(anthropicPayload) + consola.debug( + "Translated OpenAI request payload:", + JSON.stringify(openAIPayload), + ) + response = await createChatCompletions(openAIPayload) + } + + if (isNonStreaming(response)) { + consola.debug( + "Non-streaming response from Copilot:", + JSON.stringify(response).slice(-400), + ) + const anthropicResponse = translateToAnthropic(response) + consola.debug( + "Translated Anthropic response:", + JSON.stringify(anthropicResponse), + ) + return c.json(anthropicResponse) + } + + consola.debug("Streaming response from Copilot") + return streamSSE(c, async (stream) => { + const streamState: AnthropicStreamState = { + messageStartSent: false, + contentBlockIndex: 0, + contentBlockOpen: false, + toolCalls: {}, + } + + for await (const rawEvent of response) { + consola.debug("Copilot raw stream event:", JSON.stringify(rawEvent)) + if (rawEvent.data === "[DONE]") { + break + } + + if (!rawEvent.data) { + continue + } + + const chunk = JSON.parse(rawEvent.data) as ChatCompletionChunk + const events = translateChunkToAnthropicEvents(chunk, streamState) + + for (const event of events) { + consola.debug("Translated Anthropic event:", JSON.stringify(event)) + await stream.writeSSE({ + event: event.type, + data: JSON.stringify(event), + }) + } + } + }) +} + +const isNonStreaming = ( + response: Awaited>, +): response is ChatCompletionResponse => Object.hasOwn(response, "choices") +``` + +- [ ] **Step 4: Run all tests** + +```bash +bun test +``` + +Expected: all tests pass. + +- [ ] **Step 5: Run typecheck** + +```bash +bun run typecheck +``` + +Expected: no errors. + +- [ ] **Step 6: Run build** + +```bash +bun run build +``` + +Expected: completes without errors. + +- [ ] **Step 7: Commit** + +```bash +git add src/routes/messages/handler.ts tests/web-search.test.ts +git commit -m "feat: wire web search into message handler" +``` + +--- + +## Final Verification + +- [ ] **Run full test suite** + +```bash +bun test +``` + +Expected: all tests pass. + +- [ ] **Run typecheck** + +```bash +bun run typecheck +``` + +Expected: no errors. + +- [ ] **Run build** + +```bash +bun run build +``` + +Expected: `dist/` built without errors. diff --git a/docs/superpowers/plans/2026-03-19-burst-rate-limit.md b/docs/superpowers/plans/2026-03-19-burst-rate-limit.md new file mode 100644 index 000000000..1acb22771 --- /dev/null +++ b/docs/superpowers/plans/2026-03-19-burst-rate-limit.md @@ -0,0 +1,448 @@ +# Burst Rate Limiting Implementation Plan + +> **For agentic workers:** REQUIRED: Use superpowers:subagent-driven-development (if subagents available) or superpowers:executing-plans to implement this plan. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a sliding-window burst rate limiter that allows at most N requests within X seconds, throttling excess requests until a slot opens, leaving the existing per-request gap limiter (`--rate-limit`) untouched. + +**Architecture:** A new `checkBurstLimit(state)` function in `src/lib/rate-limit.ts` tracks request timestamps in `state.burstRequestTimestamps` (process-wide, shared across all clients of this proxy — appropriate since a single GitHub Copilot token is shared). It prunes expired entries on every call and sleeps until a slot opens if the window is full (retry loop, not FIFO queue — ordering under concurrent load is not guaranteed). Two new CLI flags (`--burst-count`, `--burst-window`) configure it, both required together. The gap limiter (`checkRateLimit`) runs first so its sleep delay happens before the burst timestamp is recorded, keeping timestamps close to actual outbound dispatch time. + +**Tech Stack:** Bun runtime, TypeScript, `bun:test` for tests, `consola` for logging, `citty` for CLI arg parsing. + +--- + +## Chunk 1: State and core logic + +### Task 1: Extend State with burst limiting fields + +**Files:** +- Modify: `src/lib/state.ts` + +- [ ] **Step 1: Add the three new fields to the `State` interface and `state` object** + + In `src/lib/state.ts`, add to the `State` interface after the existing rate-limit comment block: + + ```typescript + // Burst rate limiting configuration + burstCount?: number + burstWindowSeconds?: number + burstRequestTimestamps: number[] + ``` + + And in the `state` object literal, add: + + ```typescript + burstRequestTimestamps: [], + ``` + + (`burstCount` and `burstWindowSeconds` default to `undefined` automatically since they are optional — no explicit initialisation needed beyond the interface.) + +- [ ] **Step 2: Run typecheck to verify no type errors** + + ```bash + bun run typecheck + ``` + + Expected: no errors. + +- [ ] **Step 3: Commit** + + ```bash + git add src/lib/state.ts + git commit -m "feat: add burst rate limit fields to State" + ``` + +--- + +### Task 2: Implement `checkBurstLimit` + +**Files:** +- Modify: `src/lib/rate-limit.ts` +- Create: `tests/burst-rate-limit.test.ts` + +- [ ] **Step 1: Write the failing tests** + + Create `tests/burst-rate-limit.test.ts`: + + ```typescript + import { describe, test, expect, beforeEach } from "bun:test" + import type { State } from "~/lib/state" + import { checkBurstLimit } from "~/lib/rate-limit" + + function makeState(overrides: Partial = {}): State { + return { + accountType: "individual", + manualApprove: false, + rateLimitWait: false, + showToken: false, + burstRequestTimestamps: [], + ...overrides, + } + } + + describe("checkBurstLimit", () => { + test("returns immediately when burst limiting is not configured", async () => { + const state = makeState() + const start = Date.now() + await checkBurstLimit(state) + expect(Date.now() - start).toBeLessThan(50) + expect(state.burstRequestTimestamps).toHaveLength(0) + }) + + test("returns immediately when only burstCount is set (not configured)", async () => { + const state = makeState({ burstCount: 3 }) + const start = Date.now() + await checkBurstLimit(state) + expect(Date.now() - start).toBeLessThan(50) + }) + + test("returns immediately when only burstWindowSeconds is set (not configured)", async () => { + const state = makeState({ burstWindowSeconds: 10 }) + const start = Date.now() + await checkBurstLimit(state) + expect(Date.now() - start).toBeLessThan(50) + }) + + test("records timestamp and proceeds when under the burst limit", async () => { + const state = makeState({ burstCount: 3, burstWindowSeconds: 10 }) + await checkBurstLimit(state) + expect(state.burstRequestTimestamps).toHaveLength(1) + await checkBurstLimit(state) + expect(state.burstRequestTimestamps).toHaveLength(2) + await checkBurstLimit(state) + expect(state.burstRequestTimestamps).toHaveLength(3) + }) + + test("prunes expired timestamps before checking the limit", async () => { + const state = makeState({ burstCount: 2, burstWindowSeconds: 1 }) + // Inject 2 timestamps that are already expired (2 seconds ago) + const expired = Date.now() - 2000 + state.burstRequestTimestamps = [expired, expired] + // Should proceed immediately (expired entries pruned → window is empty) + const start = Date.now() + await checkBurstLimit(state) + expect(Date.now() - start).toBeLessThan(50) + expect(state.burstRequestTimestamps).toHaveLength(1) + }) + + test("waits when the window is full and proceeds once a slot opens", async () => { + // Use a very short window so the test doesn't take long + const state = makeState({ burstCount: 1, burstWindowSeconds: 0.1 }) + // Fill the window with a timestamp right now + state.burstRequestTimestamps = [Date.now()] + const start = Date.now() + // This call should wait ~100ms for the slot to expire + await checkBurstLimit(state) + const elapsed = Date.now() - start + expect(elapsed).toBeGreaterThanOrEqual(80) // at least 80ms wait + expect(elapsed).toBeLessThan(500) // but not excessively long + expect(state.burstRequestTimestamps).toHaveLength(1) + }) + }) + ``` + +- [ ] **Step 2: Run tests to verify they fail** + + ```bash + bun test tests/burst-rate-limit.test.ts + ``` + + Expected: all tests fail because `checkBurstLimit` does not exist yet. + +- [ ] **Step 3: Implement `checkBurstLimit` in `src/lib/rate-limit.ts`** + + Add this function to the end of `src/lib/rate-limit.ts` (after `checkRateLimit`): + + ```typescript + export async function checkBurstLimit(state: State): Promise { + if (state.burstCount === undefined || state.burstWindowSeconds === undefined) + return + + const windowMs = state.burstWindowSeconds * 1000 + + // eslint-disable-next-line no-constant-condition + while (true) { + const now = Date.now() + + state.burstRequestTimestamps = state.burstRequestTimestamps.filter( + (ts) => ts > now - windowMs, + ) + + if (state.burstRequestTimestamps.length < state.burstCount) { + // Slot is free — record this request synchronously (no await between + // the length check and push, preventing interleaving in async handlers). + state.burstRequestTimestamps.push(now) + return + } + + // Window is full — wait until the oldest slot expires. + const waitMs = Math.max( + 0, + state.burstRequestTimestamps[0] + windowMs - now, + ) + // Use ms for short waits, seconds for long ones — avoids misleading "1s" for a 100ms wait. + const waitLabel = + waitMs < 1000 ? `${waitMs}ms` : `${(waitMs / 1000).toFixed(1)}s` + consola.warn( + `Burst limit reached. Waiting ${waitLabel} before proceeding...`, + ) + await sleep(waitMs) + consola.debug("Burst limit wait completed, re-checking...") + // Loop back with a fresh Date.now() — do not push unconditionally. + } + } + ``` + +- [ ] **Step 4: Run tests to verify they pass** + + ```bash + bun test tests/burst-rate-limit.test.ts + ``` + + Expected: all 6 tests pass. + +- [ ] **Step 5: Run full test suite to check for regressions** + + ```bash + bun test + ``` + + Expected: all tests pass. + +- [ ] **Step 6: Commit** + + ```bash + git add src/lib/rate-limit.ts tests/burst-rate-limit.test.ts + git commit -m "feat: implement checkBurstLimit sliding-window rate limiter" + ``` + +--- + +## Chunk 2: CLI wiring and handler integration + +### Task 3: Add CLI flags and validation to `src/start.ts` + +**Files:** +- Modify: `src/start.ts` + +- [ ] **Step 1: Add `burstCount` and `burstWindowSeconds` to `RunServerOptions`** + + In `src/start.ts`, find the `RunServerOptions` interface and add two new optional fields after `rateLimitWait`: + + ```typescript + burstCount?: number + burstWindowSeconds?: number + ``` + +- [ ] **Step 2: Add state assignment in `runServer`** + + In `runServer`, find these two consecutive lines (around line 46–47): + + ```typescript + state.rateLimitSeconds = options.rateLimit + state.rateLimitWait = options.rateLimitWait + ``` + + Add two new lines **immediately after** `state.rateLimitWait = ...` (before `state.showToken`): + + ```typescript + state.burstCount = options.burstCount + state.burstWindowSeconds = options.burstWindowSeconds + ``` + +- [ ] **Step 3: Add `--burst-count` and `--burst-window` CLI args** + + In the `args` block of the `start` command (after the existing `wait` arg), add: + + ```typescript + "burst-count": { + type: "string", + description: + "Max requests allowed within the burst window (positive integer). Must be used with --burst-window.", + }, + "burst-window": { + type: "string", + description: + "Burst window duration in seconds (positive number). Must be used with --burst-count.", + }, + ``` + +- [ ] **Step 4: Add parsing and validation in the `run()` callback** + + In the `run({ args })` callback, after the existing `rateLimit` parsing block and before the `return runServer(...)` call, add: + + ```typescript + const rawBurstCount = args["burst-count"] + const rawBurstWindow = args["burst-window"] + + let burstCount: number | undefined + let burstWindowSeconds: number | undefined + + if (rawBurstCount !== undefined && rawBurstWindow !== undefined) { + const parsedCount = Number(rawBurstCount) + if (!Number.isInteger(parsedCount) || parsedCount < 1) { + consola.error( + `--burst-count must be a positive integer (got: ${rawBurstCount})`, + ) + process.exit(1) + } + + const parsedWindow = Number(rawBurstWindow) + if (!(parsedWindow > 0)) { + consola.error( + `--burst-window must be a positive number greater than 0 (got: ${rawBurstWindow})`, + ) + process.exit(1) + } + + burstCount = parsedCount + burstWindowSeconds = parsedWindow + } else if (rawBurstCount !== undefined || rawBurstWindow !== undefined) { + const missing = rawBurstCount === undefined ? "--burst-count" : "--burst-window" + consola.error( + `--burst-count and --burst-window must both be provided (missing: ${missing})`, + ) + process.exit(1) + } + ``` + + Then pass `burstCount` and `burstWindowSeconds` into `runServer(...)`: + + ```typescript + return runServer({ + // ... existing fields ... + burstCount, + burstWindowSeconds, + }) + ``` + +- [ ] **Step 5: Run typecheck** + + ```bash + bun run typecheck + ``` + + Expected: no errors. + + > **Note:** Citty args with `type: "string"` and no `default` produce `string | undefined`. The ESLint rule `@typescript-eslint/no-unnecessary-condition` may flag `=== undefined` checks depending on how citty types the arg. If lint fails on this in Task 5, add `// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition` before the `rawBurstCount !== undefined` / `rawBurstWindow !== undefined` checks (consistent with the existing pattern on line ~214 of `start.ts` for `rateLimitRaw`). + +- [ ] **Step 6: Commit** + + ```bash + git add src/start.ts + git commit -m "feat: add --burst-count and --burst-window CLI flags with validation" + ``` + +--- + +### Task 4: Wire `checkBurstLimit` into request handlers + +**Files:** +- Modify: `src/routes/chat-completions/handler.ts` +- Modify: `src/routes/messages/handler.ts` + +- [ ] **Step 1: Update the chat-completions handler** + + In `src/routes/chat-completions/handler.ts`, find the existing import: + + ```typescript + import { checkRateLimit } from "~/lib/rate-limit" + ``` + + Replace it with: + + ```typescript + import { checkBurstLimit, checkRateLimit } from "~/lib/rate-limit" + ``` + + Then find the line `await checkRateLimit(state)` in `handleCompletion` and add the burst check immediately **after** it (gap limiter runs first so its sleep occurs before the burst timestamp is recorded): + + ```typescript + await checkRateLimit(state) + await checkBurstLimit(state) + ``` + +- [ ] **Step 2: Update the messages handler** + + In `src/routes/messages/handler.ts`, find the existing import: + + ```typescript + import { checkRateLimit } from "~/lib/rate-limit" + ``` + + Replace it with: + + ```typescript + import { checkBurstLimit, checkRateLimit } from "~/lib/rate-limit" + ``` + + Then find `await checkRateLimit(state)` inside `handleCompletion` (it is on the **second line** of that function, around line 61). Add the burst check immediately **after** it (gap limiter runs first): + + ```typescript + await checkRateLimit(state) + await checkBurstLimit(state) + ``` + + Note: `handleCompletion` in this file is more complex than the chat-completions version — it immediately delegates to `handleNonStreaming` or `streamSSE`. Both limiters run at the top before any of that delegation. + +- [ ] **Step 3: Run typecheck** + + ```bash + bun run typecheck + ``` + + Expected: no errors. + +- [ ] **Step 4: Run full test suite** + + ```bash + bun test + ``` + + Expected: all tests pass. + +- [ ] **Step 5: Smoke-test the CLI flag help output** + + ```bash + bun run src/main.ts start --help + ``` + + Expected: `--burst-count` and `--burst-window` appear in the help text. + +- [ ] **Step 6: Commit** + + ```bash + git add src/routes/chat-completions/handler.ts src/routes/messages/handler.ts + git commit -m "feat: wire checkBurstLimit into chat-completions and messages handlers" + ``` + +--- + +## Chunk 3: Final lint and typecheck + +### Task 5: Full lint and typecheck pass + +**Files:** (no changes — verification only) + +- [ ] **Step 1: Run linter on the whole project** + + ```bash + bun run lint:all + ``` + + Expected: no errors. + +- [ ] **Step 2: Run typecheck** + + ```bash + bun run typecheck + ``` + + Expected: no errors. + +- [ ] **Step 3: Run full test suite one final time** + + ```bash + bun test + ``` + + Expected: all tests pass. diff --git a/docs/superpowers/plans/2026-03-23-413-image-stripping-retry.md b/docs/superpowers/plans/2026-03-23-413-image-stripping-retry.md new file mode 100644 index 000000000..ee383ad43 --- /dev/null +++ b/docs/superpowers/plans/2026-03-23-413-image-stripping-retry.md @@ -0,0 +1,503 @@ +# 413 Image Stripping with Progressive Retry — Implementation Plan + +> **For agentic workers:** REQUIRED: Use superpowers:subagent-driven-development (if subagents available) or superpowers:executing-plans to implement this plan. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Intercept 413 Request Entity Too Large errors from Copilot, progressively strip base64 images from the conversation, retry, and fall back to triggering Claude Code auto-compaction so the model never stops. + +**Architecture:** New `image-stripping.ts` module with `stripImages` utility and `fetchWithImageStripping` cascade wrapper. Handler passes `fetchCopilotResponse` as a parameter to avoid circular imports. Cascade: keep-last-image → strip-all-images → throw CompactionNeededError. + +**Tech Stack:** TypeScript, Hono, Bun runtime. No new dependencies. + +**Spec:** `docs/superpowers/specs/2026-03-23-413-image-stripping-retry-design.md` + +--- + +## File Structure + +| File | Action | Responsibility | +|------|--------|----------------| +| `src/routes/messages/image-stripping.ts` | Create | `CompactionNeededError` class, `stripImages()` internal utility, `fetchWithImageStripping()` cascade wrapper | +| `src/routes/messages/handler.ts` | Modify | Import and wire up `fetchWithImageStripping` + `CompactionNeededError` in both streaming and non-streaming paths | + +--- + +## Chunk 1: Image Stripping Module + +### Task 1: Create `CompactionNeededError` and `stripImages` utility + +**Files:** +- Create: `src/routes/messages/image-stripping.ts` + +- [ ] **Step 1: Create `image-stripping.ts` with `CompactionNeededError` class** + +Create the file with the custom error class: + +```typescript +import consola from "consola" + +import { HTTPError } from "~/lib/error" + +import type { AnthropicMessagesPayload } from "./anthropic-types" + +/** + * Thrown when the 413 retry cascade is exhausted (all images stripped, + * request still too large). Signals the handler to return an + * `invalid_request_error` that triggers Claude Code auto-compaction. + */ +export class CompactionNeededError extends Error { + constructor() { + super("Request too large even after stripping all images") + this.name = "CompactionNeededError" + } +} +``` + +- [ ] **Step 2: Add the `stripImages` function** + +Append to `image-stripping.ts`. This walks the Anthropic payload and replaces base64 image blocks with text placeholders: + +```typescript +/** + * Deep-clones the payload and replaces base64 image blocks with text + * placeholders. When `keepLast` is true and 2+ images exist, the last + * image (most recent in conversation order) is preserved. + * + * Returns the cloned (possibly mutated) payload and the count of images + * actually replaced. + */ +function stripImages( + payload: AnthropicMessagesPayload, + keepLast: boolean, +): { payload: AnthropicMessagesPayload; strippedCount: number } { + // Deep-clone to avoid mutating the original + const cloned = structuredClone(payload) + + // Collect references to all base64 image blocks in conversation order. + // Each entry holds the parent array and the index within that array so + // we can replace the block in-place after deciding which ones to keep. + const imageRefs: Array<{ parent: Array; index: number }> = [] + + for (const message of cloned.messages) { + if (message.role !== "user") continue + if (typeof message.content === "string") continue + + for (let i = 0; i < message.content.length; i++) { + const block = message.content[i] + + // AnthropicImageBlock.source.type is always "base64" per current + // type definitions, so narrowing on type === "image" is sufficient. + if (block.type === "image") { + imageRefs.push({ + parent: message.content as Array, + index: i, + }) + } + + // Walk nested tool_result content arrays + if (block.type === "tool_result" && Array.isArray(block.content)) { + for (let j = 0; j < block.content.length; j++) { + const nested = block.content[j] + if (nested.type === "image") { + imageRefs.push({ + parent: block.content as Array, + index: j, + }) + } + } + } + } + } + + // Determine which images to strip + const toStrip = + keepLast && imageRefs.length > 1 + ? imageRefs.slice(0, -1) // keep the last one + : imageRefs // strip all + + const placeholder = { + type: "text" as const, + text: "[Image removed to reduce request size]", + } + + for (const ref of toStrip) { + ref.parent[ref.index] = placeholder + } + + return { payload: cloned, strippedCount: toStrip.length } +} +``` + +- [ ] **Step 3: Run typecheck to verify no type errors** + +Run: `cd c:\Users\ttbasil\Desktop\Projects\PublicProjects\copilot-api && bun run typecheck` +Expected: No errors related to `image-stripping.ts` + +- [ ] **Step 4: Run lint to verify code style** + +Run: `cd c:\Users\ttbasil\Desktop\Projects\PublicProjects\copilot-api && bun run lint:all` +Expected: No lint errors in `image-stripping.ts`. Fix any issues found. + +- [ ] **Step 5: Commit** + +```bash +git add src/routes/messages/image-stripping.ts +git commit -m "feat: add stripImages utility and CompactionNeededError class + +Implements the image stripping algorithm that walks Anthropic message +payloads and replaces base64 image blocks with text placeholders. +Supports keepLast mode to preserve the most recent image." +``` + +--- + +### Task 2: Add `fetchWithImageStripping` cascade wrapper + +**Files:** +- Modify: `src/routes/messages/image-stripping.ts` + +- [ ] **Step 1: Add the `fetchWithImageStripping` function** + +Append to `image-stripping.ts` after the `stripImages` function. This wraps any fetch function with the 413 progressive retry cascade: + +```typescript +/** + * Wraps a Copilot fetch function with progressive 413 retry logic. + * + * Cascade: + * 1. Try original request + * 2. On 413 with 2+ images: strip older images, keep last, retry + * 3. On 413: strip ALL images, retry + * 4. On 413 with no images left: throw CompactionNeededError + * + * Non-413 HTTPErrors and non-HTTP errors propagate immediately. + */ +export async function fetchWithImageStripping( + fetchFn: (payload: AnthropicMessagesPayload) => Promise, + anthropicPayload: AnthropicMessagesPayload, +): Promise { + // Stage 1: Try original request + try { + return await fetchFn(anthropicPayload) + } catch (error) { + if (!is413(error)) throw error + } + + // Stage 2: Strip older images, keep most recent + const stage2 = stripImages(anthropicPayload, true) + if (stage2.strippedCount > 0) { + consola.warn( + `Request too large (413), retrying with older images stripped (keeping last image). Removed ${stage2.strippedCount} image(s).`, + ) + try { + return await fetchFn(stage2.payload) + } catch (error) { + if (!is413(error)) throw error + } + } + + // Stage 3: Strip ALL images (always from original payload) + const stage3 = stripImages(anthropicPayload, false) + if (stage3.strippedCount > 0) { + consola.warn( + `Still too large (413), retrying with all images stripped. Removed ${stage3.strippedCount} image(s).`, + ) + try { + return await fetchFn(stage3.payload) + } catch (error) { + if (!is413(error)) throw error + } + } + + // Stage 4: No images left, request is still too large — trigger compaction + consola.warn( + "Still too large (413) even without images, triggering auto-compaction", + ) + throw new CompactionNeededError() +} + +function is413(error: unknown): boolean { + return error instanceof HTTPError && error.response.status === 413 +} +``` + +- [ ] **Step 2: Run typecheck** + +Run: `cd c:\Users\ttbasil\Desktop\Projects\PublicProjects\copilot-api && bun run typecheck` +Expected: No type errors + +- [ ] **Step 3: Run lint** + +Run: `cd c:\Users\ttbasil\Desktop\Projects\PublicProjects\copilot-api && bun run lint:all` +Expected: No lint errors. Fix any issues found. + +- [ ] **Step 4: Commit** + +```bash +git add src/routes/messages/image-stripping.ts +git commit -m "feat: add fetchWithImageStripping 413 retry cascade + +Wraps fetchCopilotResponse with progressive 413 handling: +keep-last-image retry → strip-all-images retry → CompactionNeededError. +Accepts fetch function as parameter to avoid circular imports." +``` + +--- + +## Chunk 2: Handler Integration + +### Task 3: Integrate into `handleNonStreaming` + +**Files:** +- Modify: `src/routes/messages/handler.ts` + +- [ ] **Step 1: Add imports to handler.ts** + +Add the import at the top of `handler.ts`, after the existing `./stream-translation` import: + +```typescript +import { + CompactionNeededError, + fetchWithImageStripping, +} from "./image-stripping" +``` + +- [ ] **Step 2: Modify `handleNonStreaming` catch block** + +In `handleNonStreaming` (around line 94-125), replace the try/catch block. Change this: + +```typescript + try { + response = await fetchCopilotResponse(anthropicPayload) + } catch (error) { + // Re-throw HTTPErrors so they bubble up to the route-level forwardError + // handler, which returns the raw Copilot error JSON with the original + // HTTP status code. Returning an Anthropic-formatted error with type + // "invalid_request_error" causes Claude Code to auto-compact and retry + // in a loop when the prompt exceeds Copilot's token limit — each retry + // adds more context, making the prompt even larger. + if (error instanceof HTTPError) throw error + + consola.error("Copilot connection error (fetch-level):", error) + return c.json( + { + type: "error", + error: { + type: "api_error", + message: + error instanceof Error ? + error.message + : "An unexpected error occurred.", + }, + }, + 500, + ) + } +``` + +To this: + +```typescript + try { + response = await fetchWithImageStripping( + fetchCopilotResponse, + anthropicPayload, + ) + } catch (error) { + // 413 cascade exhausted — all images stripped, still too large. + // Return invalid_request_error to trigger Claude Code auto-compaction. + // This is safe because images are already gone and compaction will + // reduce the text content, producing a convergently smaller request. + if (error instanceof CompactionNeededError) { + return c.json( + { + type: "error", + error: { + type: "invalid_request_error", + message: + "Request too large. Conversation context exceeds model limit.", + }, + }, + 413, + ) + } + + // Re-throw non-413 HTTPErrors so they bubble up to the route-level + // forwardError handler, which returns the raw Copilot error JSON with + // the original HTTP status code. + if (error instanceof HTTPError) throw error + + consola.error("Copilot connection error (fetch-level):", error) + return c.json( + { + type: "error", + error: { + type: "api_error", + message: + error instanceof Error ? + error.message + : "An unexpected error occurred.", + }, + }, + 500, + ) + } +``` + +- [ ] **Step 3: Run typecheck** + +Run: `cd c:\Users\ttbasil\Desktop\Projects\PublicProjects\copilot-api && bun run typecheck` +Expected: No type errors + +- [ ] **Step 4: Run lint** + +Run: `cd c:\Users\ttbasil\Desktop\Projects\PublicProjects\copilot-api && bun run lint:all` +Expected: No lint errors. Fix any issues found. + +- [ ] **Step 5: Commit** + +```bash +git add src/routes/messages/handler.ts +git commit -m "feat: integrate 413 image stripping into non-streaming path + +handleNonStreaming now uses fetchWithImageStripping to progressively +strip images on 413 errors. Falls back to invalid_request_error to +trigger Claude Code auto-compaction when all images are stripped." +``` + +--- + +### Task 4: Integrate into `handleStreaming` + +**Files:** +- Modify: `src/routes/messages/handler.ts` + +- [ ] **Step 1: Replace `fetchCopilotResponse` in the streaming retry loop** + +In `handleStreaming` (around line 177-191), change the `fetchCopilotResponse` call inside the inner `try` of the retry loop. The full inner `try` block changes from: + +```typescript + for (let attempt = 1; attempt <= MAX_FETCH_RETRIES; attempt++) { + try { + copilotResponse = await fetchCopilotResponse(anthropicPayload) + break + } catch (error) { + lastError = error + if (error instanceof HTTPError) throw error + const isRetriable = + error instanceof Error && RETRIABLE_ERROR_NAMES.has(error.name) + if (!isRetriable || attempt === MAX_FETCH_RETRIES) throw error + consola.warn( + `Copilot fetch attempt ${attempt}/${MAX_FETCH_RETRIES} failed (${error.message}), retrying…`, + ) + } + } +``` + +To: + +```typescript + for (let attempt = 1; attempt <= MAX_FETCH_RETRIES; attempt++) { + try { + copilotResponse = await fetchWithImageStripping( + fetchCopilotResponse, + anthropicPayload, + ) + break + } catch (error) { + lastError = error + if (error instanceof HTTPError) throw error + const isRetriable = + error instanceof Error && RETRIABLE_ERROR_NAMES.has(error.name) + if (!isRetriable || attempt === MAX_FETCH_RETRIES) throw error + consola.warn( + `Copilot fetch attempt ${attempt}/${MAX_FETCH_RETRIES} failed (${error.message}), retrying…`, + ) + } + } +``` + +**Error flow after this change:** +- A 413 `HTTPError` from Copilot is caught *inside* `fetchWithImageStripping`, which runs the image-stripping cascade. It never reaches the `if (error instanceof HTTPError) throw error` in the retry loop. +- Non-413 `HTTPError`s (401, 429, 500…) are re-thrown by `fetchWithImageStripping`, hit `if (error instanceof HTTPError) throw error` in the retry loop, and escape to the outer catch — unchanged behavior. +- `CompactionNeededError` (thrown when the cascade is exhausted) is not an `HTTPError`, so it passes the `instanceof HTTPError` check, is not in `RETRIABLE_ERROR_NAMES`, and is thrown to the outer catch on the first attempt — correct behavior. +- Network errors (TimeoutError, ECONNRESET) from inside `fetchWithImageStripping` propagate out, hit the `isRetriable` check, and are retried by the outer loop — unchanged behavior. + +- [ ] **Step 2: Add `CompactionNeededError` handling in the outer catch block** + +In the outer catch block of `handleStreaming` (around line 213-236), add `CompactionNeededError` handling right after the `pingTimer` cleanup (line 214-215) and before the existing `HTTPError` check. Insert this block: + +After: +```typescript + clearInterval(pingTimer) + pingTimer = undefined +``` + +Add: +```typescript + + // 413 cascade exhausted — all images stripped, still too large. + // Emit invalid_request_error to trigger Claude Code auto-compaction. + if (error instanceof CompactionNeededError) { + const errorEvent = translateErrorToAnthropicErrorEvent( + "Request too large. Conversation context exceeds model limit.", + "invalid_request_error", + ) + await stream.writeSSE({ + event: errorEvent.type, + data: JSON.stringify(errorEvent), + }) + return + } +``` + +This goes before the existing: +```typescript + if (error instanceof HTTPError) { +``` + +- [ ] **Step 3: Run typecheck** + +Run: `cd c:\Users\ttbasil\Desktop\Projects\PublicProjects\copilot-api && bun run typecheck` +Expected: No type errors + +- [ ] **Step 4: Run lint** + +Run: `cd c:\Users\ttbasil\Desktop\Projects\PublicProjects\copilot-api && bun run lint:all` +Expected: No lint errors. Fix any issues found. + +- [ ] **Step 5: Commit** + +```bash +git add src/routes/messages/handler.ts +git commit -m "feat: integrate 413 image stripping into streaming path + +handleStreaming now uses fetchWithImageStripping inside the network +retry loop. CompactionNeededError triggers invalid_request_error SSE +event for Claude Code auto-compaction." +``` + +--- + +### Task 5: Verify build and final checks + +**Files:** None (verification only) + +- [ ] **Step 1: Run full typecheck** + +Run: `cd c:\Users\ttbasil\Desktop\Projects\PublicProjects\copilot-api && bun run typecheck` +Expected: Zero errors + +- [ ] **Step 2: Run full lint** + +Run: `cd c:\Users\ttbasil\Desktop\Projects\PublicProjects\copilot-api && bun run lint:all` +Expected: Zero errors + +- [ ] **Step 3: Run build** + +Run: `cd c:\Users\ttbasil\Desktop\Projects\PublicProjects\copilot-api && bun run build` +Expected: Build completes successfully, output in `dist/` + +- [ ] **Step 4: Run knip (unused exports check)** + +Run: `cd c:\Users\ttbasil\Desktop\Projects\PublicProjects\copilot-api && bun run knip` +Expected: No new unused exports from `image-stripping.ts`. Both exports (`CompactionNeededError`, `fetchWithImageStripping`) should be consumed by `handler.ts`. `stripImages` and `is413` are module-private (not exported) and should not be flagged. diff --git a/docs/superpowers/plans/2026-04-14-full-anthropic-api-parity.md b/docs/superpowers/plans/2026-04-14-full-anthropic-api-parity.md new file mode 100644 index 000000000..f4c27ab81 --- /dev/null +++ b/docs/superpowers/plans/2026-04-14-full-anthropic-api-parity.md @@ -0,0 +1,1374 @@ +# Full Anthropic API Parity — Implementation Plan + +> **For agentic workers:** REQUIRED: Use superpowers:subagent-driven-development (if subagents available) or superpowers:executing-plans to implement this plan. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Achieve maximum compatibility with the full Anthropic Messages API so the copilot-api proxy transparently handles any request from Claude Code v2.1.107+ without errors or data loss. + +**Architecture:** Incremental extension of the existing translation pipeline. Types are added in `anthropic-types.ts`, translation logic updated in `non-stream-translation.ts`, and guard fixes applied to 4 image-related files. Anthropic-specific features with no OpenAI equivalent are accepted in types and serialized to text during translation. + +**Tech Stack:** TypeScript, Hono, Bun runtime. No new dependencies. + +**Spec:** `docs/superpowers/specs/2026-04-14-full-anthropic-api-parity-design.md` + +--- + +## File Structure + +| File | Action | Responsibility | +|------|--------|----------------| +| `src/routes/messages/anthropic-types.ts` | Modify | Add 7 request payload fields, fix 4 existing types, add 7 new block interfaces, update 4 union types, add 2 streaming delta types | +| `src/routes/messages/non-stream-translation.ts` | Modify | Add `isServerToolResultBlock()` helper, update `handleUserMessage()` routing, update `handleAssistantMessage()` Branch 1, rewrite `mapContent()` Path A as for-loop, add cases + default to Path B | +| `src/routes/messages/count-tokens-handler.ts` | Modify | Add token overhead entries for new server tool types | +| `src/routes/messages/attachment-overhead.ts` | Modify | Fix `estimateImageTokens()` for URL images, add `search_result` and `container_upload` token estimates | +| `src/routes/messages/image-stripping.ts` | Modify | Add `source.type === "base64"` guards before accessing `.data` | +| `src/routes/messages/image-validation.ts` | Modify | Add `source.type === "base64"` guard in `getInvalidImageReason()` | +| `tests/anthropic-request.test.ts` | Modify | Add 9 test cases covering new block types, translation, and helpers | + +**No changes to:** `handler.ts`, `utils.ts`, `stream-translation.ts`, `web-search-detection.ts` + +--- + +## Chunk 1: Type System Foundation + +### Task 1: Extend `AnthropicMessagesPayload` with New Request Fields + +**Files:** +- Modify: `src/routes/messages/anthropic-types.ts:3-27` + +- [ ] **Step 1: Add 7 new optional fields to `AnthropicMessagesPayload`** + +Add after the existing `service_tier` field (line 26): + +```typescript +export interface AnthropicMessagesPayload { + // ... existing fields through service_tier ... + output_config?: { + effort?: "low" | "medium" | "high" | "max" + format?: { type: "json_schema"; schema: Record } + } + speed?: "standard" | "fast" + cache_control?: { type: "ephemeral"; ttl?: number } + container?: Record + mcp_servers?: Array> + context_management?: Record + inference_geo?: string +} +``` + +All fields are optional and have no OpenAI equivalent. `translateToOpenAI()` selectively picks known fields, so these are automatically excluded — no translation code changes needed. + +- [ ] **Step 2: Run typecheck to verify no regressions** + +Run: `bun run typecheck` +Expected: PASS (new optional fields don't break existing code) + +- [ ] **Step 3: Commit** + +```bash +git add src/routes/messages/anthropic-types.ts +git commit -m "feat: add new Anthropic request payload fields (output_config, speed, cache_control, container, mcp_servers, context_management, inference_geo)" +``` + +--- + +### Task 2: Fix `AnthropicImageBlock` — Add URL Source Variant + +**Files:** +- Modify: `src/routes/messages/anthropic-types.ts:49-56` + +- [ ] **Step 1: Update `AnthropicImageBlock` source to be a discriminated union** + +Replace the current source type (line 51-55): + +```typescript +export interface AnthropicImageBlock { + type: "image" + source: + | { type: "base64"; media_type: "image/jpeg" | "image/png" | "image/gif" | "image/webp"; data: string } + | { type: "url"; url: string } +} +``` + +> **WARNING**: This is a breaking change. Code that accesses `block.source.data` or `block.source.media_type` without checking `source.type === "base64"` will get TypeScript errors. The fixes for those files are in Tasks 7-10 (Chunk 2: image guard fixes). Do NOT run typecheck until Task 10 is complete. + +- [ ] **Step 2: Commit (typecheck deferred to after Task 10)** + +> **Note on compile safety**: This commit introduces type errors that are resolved by Tasks 7-10. If you prefer every commit to be compilable, defer this commit and squash it with Task 10's commit instead. + +```bash +git add src/routes/messages/anthropic-types.ts +git commit -m "feat: add URL source variant to AnthropicImageBlock" +``` + +--- + +### Task 3: Fix Existing Type Fields — Text Citations, Tool Use, Custom Tool + +**Files:** +- Modify: `src/routes/messages/anthropic-types.ts:29-33` (AnthropicTextBlock) +- Modify: `src/routes/messages/anthropic-types.ts:80-85` (AnthropicToolUseBlock) +- Modify: `src/routes/messages/anthropic-types.ts:143-152` (AnthropicCustomTool) + +- [ ] **Step 1: Add `citations` to `AnthropicTextBlock`** + +```typescript +export interface AnthropicTextBlock { + type: "text" + text: string + cache_control?: { type: "ephemeral"; ttl?: number } + citations?: Array // pass-through, not interpreted by proxy +} +``` + +The current `AnthropicTextBlock` has `cache_control` but does NOT have `citations`. Add the `citations` field. + +- [ ] **Step 2: Add `cache_control` and `caller` to `AnthropicToolUseBlock`** + +```typescript +export interface AnthropicToolUseBlock { + type: "tool_use" + id: string + name: string + input: Record + cache_control?: { type: "ephemeral"; ttl?: number } + caller?: Record +} +``` + +- [ ] **Step 3: Add `allowed_callers` to `AnthropicCustomTool`** + +```typescript +export interface AnthropicCustomTool { + name: string + description?: string + input_schema: Record + strict?: boolean + cache_control?: { type: "ephemeral"; ttl?: number } + defer_loading?: boolean + input_examples?: Array + eager_input_streaming?: boolean + allowed_callers?: Array +} +``` + +- [ ] **Step 4: Commit** + +```bash +git add src/routes/messages/anthropic-types.ts +git commit -m "feat: add citations, cache_control, caller, and allowed_callers to existing Anthropic types" +``` + +--- + +### Task 4: Add New Content Block Types + +**Files:** +- Modify: `src/routes/messages/anthropic-types.ts` + +- [ ] **Step 1: Add `AnthropicSearchResultBlock` and `AnthropicContainerUploadBlock`** + +Add after `AnthropicWebSearchToolResultBlock` (after line 114): + +```typescript +export interface AnthropicSearchResultBlock { + type: "search_result" + source: string + title: string + content: string + cache_control?: { type: "ephemeral"; ttl?: number } + citations?: Array + search_result_index?: number + start_block_index?: number + end_block_index?: number +} + +export interface AnthropicContainerUploadBlock { + type: "container_upload" + file_id: string + cache_control?: { type: "ephemeral"; ttl?: number } +} +``` + +- [ ] **Step 2: Add `ServerToolResultBase` and 5 server tool result block interfaces** + +Add after the new blocks from Step 1: + +```typescript +interface ServerToolResultBase { + tool_use_id: string + content: unknown + cache_control?: { type: "ephemeral"; ttl?: number } +} + +export interface AnthropicWebFetchToolResultBlock extends ServerToolResultBase { + type: "web_fetch_tool_result" +} + +export interface AnthropicCodeExecutionToolResultBlock extends ServerToolResultBase { + type: "code_execution_tool_result" +} + +export interface AnthropicBashCodeExecutionToolResultBlock extends ServerToolResultBase { + type: "bash_code_execution_tool_result" +} + +export interface AnthropicTextEditorCodeExecutionToolResultBlock extends ServerToolResultBase { + type: "text_editor_code_execution_tool_result" +} + +export interface AnthropicToolSearchToolResultBlock extends ServerToolResultBase { + type: "tool_search_tool_result" +} +``` + +- [ ] **Step 3: Add the exported `AnthropicServerToolResultBlock` union type** + +Add after the individual interfaces: + +```typescript +export type AnthropicServerToolResultBlock = + | AnthropicWebSearchToolResultBlock + | AnthropicWebFetchToolResultBlock + | AnthropicCodeExecutionToolResultBlock + | AnthropicBashCodeExecutionToolResultBlock + | AnthropicTextEditorCodeExecutionToolResultBlock + | AnthropicToolSearchToolResultBlock +``` + +This union MUST be exported — `non-stream-translation.ts` imports it for the `isServerToolResultBlock` type guard (Task 11). + +- [ ] **Step 4: Commit** + +```bash +git add src/routes/messages/anthropic-types.ts +git commit -m "feat: add search_result, container_upload, and 5 server tool result block types" +``` + +--- + +### Task 5: Update Union Types + +**Files:** +- Modify: `src/routes/messages/anthropic-types.ts:116-129` + +- [ ] **Step 1: Update `AnthropicUserContentBlock` union** + +Replace the current union (lines 116-121): + +```typescript +export type AnthropicUserContentBlock = + | AnthropicTextBlock + | AnthropicImageBlock + | AnthropicDocumentBlock + | AnthropicToolResultBlock + | AnthropicSearchResultBlock + | AnthropicContainerUploadBlock + | AnthropicServerToolResultBlock +``` + +Note: `AnthropicWebSearchToolResultBlock` is now part of `AnthropicServerToolResultBlock`, so it's included implicitly. + +- [ ] **Step 2: Update `AnthropicAssistantContentBlock` union** + +Replace the current union (lines 123-129): + +```typescript +export type AnthropicAssistantContentBlock = + | AnthropicTextBlock + | AnthropicToolUseBlock + | AnthropicThinkingBlock + | AnthropicRedactedThinkingBlock + | AnthropicServerToolUseBlock + | AnthropicServerToolResultBlock +``` + +- [ ] **Step 3: Commit** + +```bash +git add src/routes/messages/anthropic-types.ts +git commit -m "feat: update content block unions with new types" +``` + +--- + +### Task 6: Add Streaming Type Extensions + +**Files:** +- Modify: `src/routes/messages/anthropic-types.ts:221-229` (delta types) +- Modify: `src/routes/messages/anthropic-types.ts:210-219` (content_block_start) + +- [ ] **Step 1: Add `citations_delta` and `compaction_delta` to `AnthropicContentBlockDeltaEvent`** + +Add to the `delta` union (after line 228): + +```typescript +export interface AnthropicContentBlockDeltaEvent { + type: "content_block_delta" + index: number + delta: + | { type: "text_delta"; text: string } + | { type: "input_json_delta"; partial_json: string } + | { type: "thinking_delta"; thinking: string } + | { type: "signature_delta"; signature: string } + | { type: "citations_delta"; citation: unknown } + | { type: "compaction_delta"; content: unknown } +} +``` + +- [ ] **Step 2: Add `server_tool_use` and server tool result types to `AnthropicContentBlockStartEvent`** + +Add to the `content_block` union (after line 218): + +```typescript +export interface AnthropicContentBlockStartEvent { + type: "content_block_start" + index: number + content_block: + | { type: "text"; text: string } + | (Omit & { + input: Record + }) + | { type: "thinking"; thinking: string } + | { type: "server_tool_use"; id: string; name: string; input: Record } + | AnthropicServerToolResultBlock +} +``` + +These are **type-only changes**. `stream-translation.ts` does NOT need behavioral changes — it translates OpenAI chunks to Anthropic events, and new streaming types only appear in pure Anthropic API responses. + +- [ ] **Step 3: Commit** + +```bash +git add src/routes/messages/anthropic-types.ts +git commit -m "feat: add streaming type extensions (citations_delta, compaction_delta, server blocks)" +``` + +--- + +## Chunk 2: Image Guard Fixes + +After Task 2 (URL source variant), `source.data` and `source.media_type` are conditionally available. These 4 fixes resolve the resulting compile errors. **All fixes follow the same pattern**: add an early-return guard that skips URL-based images, since they have no base64 data to measure, strip, or validate. + +### Task 7: Fix `image-validation.ts` — Guard `getInvalidImageReason()` + +**Files:** +- Modify: `src/routes/messages/image-validation.ts:51-68` + +- [ ] **Step 1: Add URL source guard at the top of `getInvalidImageReason()`** + +Add as the first line inside the function (before line 54): + +```typescript +function getInvalidImageReason( + block: AnthropicImageBlock, +): InvalidImage | undefined { + if (block.source.type !== "base64") return undefined // URL images: can't validate dimensions + // ... rest of function unchanged (source is now narrowed to base64) ... +} +``` + +The guard narrows `source` to the `base64` variant, so all subsequent accesses to `source.data` and `source.media_type` are type-safe. + +- [ ] **Step 2: Run typecheck on this file** + +Run: `bun run typecheck` +Expected: `image-validation.ts` errors resolved (other files may still error) + +- [ ] **Step 3: Commit** + +```bash +git add src/routes/messages/image-validation.ts +git commit -m "fix: add base64 source guard in image validation for URL image support" +``` + +--- + +### Task 8: Fix `image-stripping.ts` — Guard `collectToolResultImages()` and `collectImageRefs()` + +**Files:** +- Modify: `src/routes/messages/image-stripping.ts:107-111` (collectToolResultImages) +- Modify: `src/routes/messages/image-stripping.ts:153-157` (collectImageRefs) + +- [ ] **Step 1: Add guard in `collectToolResultImages()`** + +At line 107, after `if (nested.type === "image") {`, add a guard before accessing `nested.source.data`: + +```typescript +if (nested.type === "image") { + if (nested.source.type !== "base64") continue // URL images: no base64 data to strip + refs.push({ + parent: content as Array, + index: j, + base64Length: nested.source.data.length, + messageIndex, + processed: false, + }) +} +``` + +- [ ] **Step 2: Add guard in `collectImageRefs()`** + +At line 153, after `if (block.type === "image") {`, add a guard: + +```typescript +if (block.type === "image") { + if (block.source.type !== "base64") continue // URL images: no base64 data to strip + const ref = { + parent: message.content as Array, + index: i, + base64Length: block.source.data.length, + messageIndex, + processed: false, + } + imageRefs.push(ref) + pendingRefs.push(ref) +} +``` + +- [ ] **Step 3: Run typecheck** + +Run: `bun run typecheck` +Expected: `image-stripping.ts` errors resolved + +- [ ] **Step 4: Commit** + +```bash +git add src/routes/messages/image-stripping.ts +git commit -m "fix: add base64 source guards in image stripping for URL image support" +``` + +--- + +### Task 9: Fix `attachment-overhead.ts` — Guard `estimateImageTokens()` + +**Files:** +- Modify: `src/routes/messages/attachment-overhead.ts:45-50` + +- [ ] **Step 1: Add URL source guard at the top of `estimateImageTokens()`** + +```typescript +function estimateImageTokens(block: AnthropicImageBlock): number { + if (block.source.type !== "base64") return MIN_IMAGE_TOKENS // URL images: use minimum estimate + return Math.max( + MIN_IMAGE_TOKENS, + Math.ceil(block.source.data.length / IMAGE_BASE64_CHARS_PER_TOKEN), + ) +} +``` + +- [ ] **Step 2: Run typecheck on this file** + +Run: `bun run typecheck` +Expected: `attachment-overhead.ts` errors resolved (other files may still error) + +- [ ] **Step 3: Commit** + +```bash +git add src/routes/messages/attachment-overhead.ts +git commit -m "fix: add base64 source guard in attachment overhead for URL image support" +``` + +--- + +### Task 10: Fix `non-stream-translation.ts` — Guard `mapContent()` Path B Image Case + +**Files:** +- Modify: `src/routes/messages/non-stream-translation.ts:425-433` + +- [ ] **Step 1: Add URL source handling in the `case "image"` branch of Path B** + +The current code at line 429 accesses `block.source.media_type` and `block.source.data` without a guard: + +```typescript +case "image": { + if (block.source.type === "url") { + contentParts.push({ + type: "image_url", + image_url: { url: block.source.url }, + }) + } else { + contentParts.push({ + type: "image_url", + image_url: { + url: `data:${block.source.media_type};base64,${block.source.data}`, + }, + }) + } + break +} +``` + +For URL-based images, use the URL directly. For base64, use the existing data URI encoding. + +- [ ] **Step 2: Run typecheck — ALL image guard errors should now be resolved** + +Run: `bun run typecheck` +Expected: PASS — zero type errors + +- [ ] **Step 3: Run existing tests** + +Run: `bun test` +Expected: All existing tests pass (no behavioral regressions) + +- [ ] **Step 4: Commit** + +```bash +git add src/routes/messages/non-stream-translation.ts +git commit -m "fix: handle URL image source in mapContent Path B" +``` + +--- + +## Chunk 3: Translation Logic Updates + +### Task 11: Add `isServerToolResultBlock` Helper and Update Imports + +**Files:** +- Modify: `src/routes/messages/non-stream-translation.ts:13-32` (imports) +- Modify: `src/routes/messages/non-stream-translation.ts` (add helper function) + +- [ ] **Step 1: Update imports to include new types** + +Add to the import block (around line 13): + +```typescript +import { + type AnthropicAssistantContentBlock, + type AnthropicAssistantMessage, + type AnthropicContainerUploadBlock, // NEW + type AnthropicCustomTool, + type AnthropicMessage, + type AnthropicMessagesPayload, + type AnthropicRedactedThinkingBlock, + type AnthropicResponse, + type AnthropicSearchResultBlock, // NEW + type AnthropicServerToolResultBlock, // NEW + type AnthropicServerToolUseBlock, + type AnthropicSystemBlock, + type AnthropicTextBlock, + type AnthropicThinkingBlock, + type AnthropicTool, + type AnthropicToolResultBlock, + type AnthropicToolUseBlock, + type AnthropicUserContentBlock, + type AnthropicUserMessage, + isTypedTool, +} from "./anthropic-types" +``` + +Remove `AnthropicWebSearchToolResultBlock` — after Task 12, nothing references it by name (the filter now uses `isServerToolResultBlock` which operates on the union type). + +- [ ] **Step 2: Add `isServerToolResultBlock` helper function** + +Add at module level, after the imports and before `translateToOpenAI()` (around line 35): + +```typescript +/** + * Type guard for server tool result blocks. Matches web_search_tool_result, + * web_fetch_tool_result, code_execution_tool_result, etc. + * Explicitly excludes plain "tool_result" (which has its own handler). + */ +function isServerToolResultBlock( + block: AnthropicUserContentBlock | AnthropicAssistantContentBlock, +): block is AnthropicServerToolResultBlock { + return block.type.endsWith("_tool_result") && block.type !== "tool_result" +} +``` + +- [ ] **Step 3: Commit** + +```bash +git add src/routes/messages/non-stream-translation.ts +git commit -m "feat: add isServerToolResultBlock helper and update imports" +``` + +--- + +### Task 12: Update `handleUserMessage()` — Replace Web Search Filter with Server Tool Result Filter + +**Files:** +- Modify: `src/routes/messages/non-stream-translation.ts:107-189` + +There are **3 code locations** that must change together. All are inside `handleUserMessage()`. + +- [ ] **Step 1: Replace `webSearchResultBlocks` filter with `serverToolResultBlocks`** + +Change lines 116-119 from: + +```typescript +const webSearchResultBlocks = message.content.filter( + (block): block is AnthropicWebSearchToolResultBlock => + block.type === "web_search_tool_result", +) +``` + +To: + +```typescript +const serverToolResultBlocks = message.content.filter(isServerToolResultBlock) +``` + +- [ ] **Step 2: Update `otherBlocks` filter to exclude all server tool results** + +Change lines 120-124 from: + +```typescript +const otherBlocks = message.content.filter( + (block) => + block.type !== "tool_result" && block.type !== "web_search_tool_result", + // document blocks remain here intentionally — mapContent handles them +) +``` + +To: + +```typescript +const otherBlocks = message.content.filter( + (block) => + block.type !== "tool_result" && !isServerToolResultBlock(block), + // document blocks remain here intentionally — mapContent handles them +) +``` + +- [ ] **Step 3: Update serialization block** + +Change lines 174-180 from: + +```typescript +// Web search result blocks → serialize as user message +if (webSearchResultBlocks.length > 0) { + const text = webSearchResultBlocks + .map((b) => `[Web search result: ${JSON.stringify(b.content)}]`) + .join("\n\n") + newMessages.push({ role: "user", content: text }) +} +``` + +To: + +```typescript +// Server tool result blocks → serialize as user message +if (serverToolResultBlocks.length > 0) { + const text = serverToolResultBlocks + .map((b) => `[${b.type}: ${JSON.stringify(b.content)}]`) + .join("\n\n") + newMessages.push({ role: "user", content: text }) +} +``` + +Note: This changes the serialization format from `[Web search result: ...]` to `[web_search_tool_result: ...]`. The downstream model receives this as plain text — the exact format doesn't matter, only that the content is preserved. + +> **IMPORTANT**: The existing test at `tests/anthropic-request.test.ts:527` asserts `m.content.includes("[Web search result:")`. This test WILL FAIL after this change. Update the test assertion to match the new format: change `"[Web search result:"` to `"web_search_tool_result"`. See Task 17 Step 9 for the replacement test that validates the new routing. + +- [ ] **Step 4: Update existing test for changed serialization format** + +In `tests/anthropic-request.test.ts`, find the test at line 497 (`"web_search_tool_result block is serialized as user message"`). Update the assertion at line 527: + +From: +```typescript +&& m.content.includes("[Web search result:"), +``` + +To: +```typescript +&& m.content.includes("web_search_tool_result"), +``` + +- [ ] **Step 5: Commit** + +```bash +git add src/routes/messages/non-stream-translation.ts tests/anthropic-request.test.ts +git commit -m "feat: replace web search filter with generic server tool result filter in handleUserMessage" +``` + +--- + +### Task 13: Update `handleAssistantMessage()` Branch 1 — Include Server Tool Results + +**Files:** +- Modify: `src/routes/messages/non-stream-translation.ts:191-260` + +- [ ] **Step 1: Add server tool result extraction in Branch 1** + +After the `serverToolUseBlocks` filter (line 214), add: + +```typescript +const serverToolResultBlocks = message.content.filter(isServerToolResultBlock) +``` + +- [ ] **Step 2: Include server tool results in `allTextContent`** + +Update the `allTextContent` array (lines 230-237) to include server tool results: + +```typescript +const allTextContent = [ + ...textBlocks.map((b) => b.text), + ...serverToolUseBlocks.map( + (b) => `[Server tool use: ${JSON.stringify(b)}]`, + ), + ...serverToolResultBlocks.map( + (b) => `[${b.type}: ${JSON.stringify(b.content)}]`, + ), +] + .filter(Boolean) + .join("\n\n") +``` + +Branch 2 (no tool_use blocks) passes `visibleBlocks` through `mapContent()`. New block types flow through to `mapContent()` automatically — no changes needed in Branch 2 beyond the `mapContent()` updates in Task 14. + +- [ ] **Step 3: Commit** + +```bash +git add src/routes/messages/non-stream-translation.ts +git commit -m "feat: include server tool results in handleAssistantMessage Branch 1" +``` + +--- + +### Task 14: Rewrite `mapContent()` — Path A (No Images) and Path B (Has Images) + +**Files:** +- Modify: `src/routes/messages/non-stream-translation.ts:386-456` + +This is the most complex change. `mapContent()` has two code paths that BOTH silently drop unknown block types. + +- [ ] **Step 1: Rewrite Path A (no images) as a for-loop** + +Replace lines 399-414: + +```typescript +return content + .filter( + (block) => + block.type === "text" + || block.type === "document" + || block.type === "server_tool_use", + ) + .map((block) => { + if (block.type === "text") return block.text + if (block.type === "document") + return "[Document: PDF content not displayable]" + return `[Server tool use: ${JSON.stringify(block)}]` + }) + .join("\n\n") +``` + +With: + +```typescript +const parts: string[] = [] +for (const block of content) { + switch (block.type) { + case "text": + parts.push(block.text) + break + case "document": + parts.push("[Document: PDF content not displayable]") + break + case "server_tool_use": + parts.push(`[Server tool use: ${JSON.stringify(block)}]`) + break + case "search_result": + parts.push( + `[Search: ${(block as AnthropicSearchResultBlock).title}]\nSource: ${(block as AnthropicSearchResultBlock).source}\n${(block as AnthropicSearchResultBlock).content}`, + ) + break + case "container_upload": + parts.push( + `[Container upload: ${(block as AnthropicContainerUploadBlock).file_id}]`, + ) + break + default: + // Catch-all: server tool results and future unknown types + if ( + "content" in block + && block.type !== "thinking" + && block.type !== "redacted_thinking" + ) { + parts.push( + `[${block.type}: ${JSON.stringify((block as { content: unknown }).content)}]`, + ) + } + break + } +} +return parts.filter(Boolean).join("\n\n") +``` + +**Why type casts**: `mapContent()` receives `Array`. The combined union is too wide for TypeScript's switch narrowing to resolve block-specific fields like `title`, `source`, `file_id`. The casts are safe because the switch case already narrows the `type` discriminant. + +- [ ] **Step 2: Add new cases + default to Path B (has images)** + +After the existing `case "server_tool_use"` block (line 449), add before the closing of the switch. **This is additive** — do NOT replace the `case "text"`, `case "image"`, `case "document"`, or `case "server_tool_use"` blocks. Those remain unchanged (including the URL source guard added in Task 10). Only add the new cases below and replace the trailing comments with a `default` case: + +```typescript +case "search_result": { + const sr = block as AnthropicSearchResultBlock + contentParts.push({ + type: "text", + text: `[Search: ${sr.title}]\nSource: ${sr.source}\n${sr.content}`, + }) + break +} +case "container_upload": { + contentParts.push({ + type: "text", + text: `[Container upload: ${(block as AnthropicContainerUploadBlock).file_id}]`, + }) + break +} +default: { + // Catch-all for server tool results and future unknown types + if ( + "content" in block + && block.type !== "thinking" + && block.type !== "redacted_thinking" + ) { + contentParts.push({ + type: "text", + text: `[${block.type}: ${JSON.stringify((block as { content: unknown }).content)}]`, + }) + } + break +} +``` + +Note: Remove the existing `// redacted_thinking: silently skip` and `// No default` comments at lines 452-453 of the current switch. + +- [ ] **Step 3: Run typecheck and tests** + +Run: `bun run typecheck && bun test` +Expected: PASS — all type errors resolved, all existing tests pass + +- [ ] **Step 4: Commit** + +```bash +git add src/routes/messages/non-stream-translation.ts +git commit -m "feat: rewrite mapContent to handle search_result, container_upload, and unknown block types" +``` + +--- + +## Chunk 4: Token Counting and Attachment Overhead + +### Task 15: Add Token Overhead for New Server Tool Types + +**Files:** +- Modify: `src/routes/messages/count-tokens-handler.ts:27-35` + +- [ ] **Step 1: Add new entries to `ANTHROPIC_TYPED_TOOL_TOKEN_OVERHEAD`** + +Extend the existing object (currently lines 27-35). Add new entries AFTER the existing `bash_20241022: 700` entry and the comment about computer_use/web_search: + +```typescript +const ANTHROPIC_TYPED_TOOL_TOKEN_OVERHEAD: Record = { + // --- existing --- + text_editor_20250728: 700, + text_editor_20250429: 700, + text_editor_20250124: 700, + text_editor_20241022: 700, + bash_20250124: 700, + bash_20241022: 700, + // computer_use and web_search: overhead included in beta pricing, not additive + // --- new (estimates for compaction heuristic, not billing) --- + web_fetch_20250910: 500, + web_fetch_20260209: 500, + web_fetch_20260309: 500, + code_execution_20250522: 500, + code_execution_20250825: 500, + code_execution_20260120: 500, + advisor_20260301: 500, + tool_search_tool_bm25_20251119: 200, + tool_search_tool_regex_20251119: 200, + tool_search_tool_bm25: 200, + tool_search_tool_regex: 200, + mcp_toolset: 300, +} +``` + +**Important**: Do NOT add `computer_*` or `web_search_*` entries — the existing comment states their overhead is included in beta pricing. These values are estimates for the compaction heuristic (not billing), so being approximate is fine. + +- [ ] **Step 2: Run typecheck** + +Run: `bun run typecheck` +Expected: PASS + +- [ ] **Step 3: Commit** + +```bash +git add src/routes/messages/count-tokens-handler.ts +git commit -m "feat: add token overhead estimates for new server tool types" +``` + +--- + +### Task 16: Add Attachment Overhead for New Block Types + +**Files:** +- Modify: `src/routes/messages/attachment-overhead.ts` + +- [ ] **Step 1: Add import for new types** + +Add to the import block at line 1: + +```typescript +import type { + AnthropicImageBlock, + AnthropicDocumentBlock, + AnthropicMessagesPayload, + AnthropicSearchResultBlock, // NEW +} from "./anthropic-types" +``` + +Note: Do NOT import `AnthropicContainerUploadBlock` — `getContainerUploadBlocksFromContent` only checks `block.type === "container_upload"` and returns a count, so the type is not needed and importing it would trigger a lint/knip unused-import warning. + +- [ ] **Step 2: Add `estimateSearchResultTokens` helper** + +Add after `estimateImageTokens` (after line 50): + +```typescript +function estimateSearchResultTokens(block: AnthropicSearchResultBlock): number { + return Math.max(500, Math.ceil(block.content.length / 4)) +} +``` + +- [ ] **Step 3: Add `getSearchResultBlocksFromContent` and `getContainerUploadBlocksFromContent` helpers** + +Add after `getImageBlocksFromContent` (after line 100). Follow the same pattern as existing helpers: + +```typescript +function getSearchResultBlocksFromContent( + content: NonNullable, +): Array { + if (typeof content === "string") return [] + + const results: Array = [] + + for (const block of content) { + if (block.type === "search_result") { + results.push(block as AnthropicSearchResultBlock) + } + // Note: search_result blocks do NOT appear inside tool_result.content + // (tool_result.content is typed as string | Array), + // so we don't need to check nested content here. + } + + return results +} + +function getContainerUploadBlocksFromContent( + content: NonNullable, +): number { + if (typeof content === "string") return 0 + + let count = 0 + for (const block of content) { + if (block.type === "container_upload") count++ + } + return count +} +``` + +Note: `getContainerUploadBlocksFromContent` returns a count (not an array) because we only need a fixed-overhead count, not the block contents. + +- [ ] **Step 4: Add new block types to `estimateAdditionalAttachmentTokens`** + +Inside the `for (const message of payload.messages)` loop (lines 107-119), add after the existing image loop: + +```typescript +for (const message of payload.messages) { + if (message.role !== "user") continue + + for (const document of getDocumentBlocksFromContent(message.content)) { + tokens += estimateDocumentTokens(document) + } + + for (const image of getImageBlocksFromContent(message.content)) { + tokens += estimateImageTokens(image) + } + + // NEW: search_result blocks + for (const searchResult of getSearchResultBlocksFromContent(message.content)) { + tokens += estimateSearchResultTokens(searchResult) + } + + // NEW: container_upload blocks (fixed overhead — just a file ID reference) + tokens += getContainerUploadBlocksFromContent(message.content) * 100 +} +``` + +- [ ] **Step 5: Run typecheck and tests** + +Run: `bun run typecheck && bun test` +Expected: PASS + +- [ ] **Step 6: Commit** + +```bash +git add src/routes/messages/attachment-overhead.ts +git commit -m "feat: add token overhead estimates for search_result and container_upload blocks" +``` + +--- + +## Chunk 5: Tests and Verification + +### Task 17: Add Tests for New Block Types and Translation + +**Files:** +- Modify: `tests/anthropic-request.test.ts` + +All tests follow the existing pattern: construct an `AnthropicMessagesPayload`, call `translateToOpenAI()`, and assert on the resulting OpenAI payload. Add a new `describe` block. + +- [ ] **Step 1: Add imports for new types** + +Update the import at line 4 to include new types: + +```typescript +import { + isTypedTool, + type AnthropicMessagesPayload, + type AnthropicTool, + type AnthropicSearchResultBlock, + type AnthropicContainerUploadBlock, +} from "~/routes/messages/anthropic-types" +``` + +- [ ] **Step 2: Add test — `search_result` block in user message produces formatted text** + +```typescript +describe("New Anthropic content block types (API parity)", () => { + test("search_result block in user message produces formatted text", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What did you find?" }, + { + type: "search_result", + source: "https://example.com/article", + title: "Example Article", + content: "This is the search result content.", + } as unknown as AnthropicSearchResultBlock & { type: "search_result" }, + ] as AnthropicMessagesPayload["messages"][0]["content"], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const userMsg = result.messages.find((m) => m.role === "user") + const content = typeof userMsg?.content === "string" ? userMsg.content : "" + expect(content).toContain("[Search: Example Article]") + expect(content).toContain("Source: https://example.com/article") + expect(content).toContain("This is the search result content.") + }) + + test("container_upload block in user message produces placeholder text", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "I uploaded a file." }, + { + type: "container_upload", + file_id: "file_abc123", + } as unknown as AnthropicContainerUploadBlock & { type: "container_upload" }, + ] as AnthropicMessagesPayload["messages"][0]["content"], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const userMsg = result.messages.find((m) => m.role === "user") + const content = typeof userMsg?.content === "string" ? userMsg.content : "" + expect(content).toContain("[Container upload: file_abc123]") + }) +``` + +- [ ] **Step 3: Add test — `web_fetch_tool_result` in user message serialized like web_search_tool_result** + +```typescript + test("web_fetch_tool_result block in user message is serialized as user text", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Fetch this page." }, + { + role: "user", + content: [ + { + type: "web_fetch_tool_result", + tool_use_id: "srv_wf_1", + content: { url: "https://example.com", text: "Page content" }, + } as any, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const resultMsg = result.messages.find( + (m) => + m.role === "user" + && typeof m.content === "string" + && m.content.includes("web_fetch_tool_result"), + ) + expect(resultMsg).toBeDefined() + expect(resultMsg?.content).toContain("example.com") + }) +``` + +- [ ] **Step 4: Add test — `code_execution_tool_result` in assistant message Branch 1** + +```typescript + test("code_execution_tool_result in assistant message with tool calls (Branch 1) appears in text", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Run some code." }, + { + role: "assistant", + content: [ + { type: "text", text: "Here are the results." }, + { + type: "code_execution_tool_result", + tool_use_id: "srv_ce_1", + content: { stdout: "Hello World", exit_code: 0 }, + } as any, + { + type: "tool_use", + id: "call_1", + name: "Bash", + input: { command: "echo done" }, + }, + ], + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "call_1", content: "done" }, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const assistantMsg = result.messages.find((m) => m.role === "assistant") + expect(assistantMsg?.content).toContain("Here are the results.") + expect(assistantMsg?.content).toContain("code_execution_tool_result") + expect(assistantMsg?.content).toContain("Hello World") + expect(assistantMsg?.tool_calls).toHaveLength(1) + }) +``` + +- [ ] **Step 5: Add test — `web_fetch_tool_result` in assistant message Branch 2 (no tool calls)** + +```typescript + test("web_fetch_tool_result in assistant message without tool calls (Branch 2) is serialized via mapContent", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Tell me what you fetched." }, + { + role: "assistant", + content: [ + { type: "text", text: "I fetched a page." }, + { + type: "web_fetch_tool_result", + tool_use_id: "srv_wf_2", + content: { url: "https://example.com", text: "Result text" }, + } as any, + ], + }, + { role: "user", content: "Thanks." }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const assistantMsg = result.messages.find((m) => m.role === "assistant") + const content = + typeof assistantMsg?.content === "string" ? assistantMsg.content : "" + expect(content).toContain("I fetched a page.") + expect(content).toContain("web_fetch_tool_result") + }) +``` + +- [ ] **Step 6: Add test — new payload fields are NOT passed through to OpenAI** + +```typescript + test("new Anthropic payload fields (output_config, speed, etc.) are not forwarded to OpenAI", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [{ role: "user", content: "Hello" }], + max_tokens: 100, + output_config: { effort: "high" }, + speed: "fast", + cache_control: { type: "ephemeral" }, + container: { skills: [] }, + mcp_servers: [{ url: "http://localhost:3000" }], + context_management: {}, + inference_geo: "us", + } + const result = translateToOpenAI(anthropicPayload) + const resultStr = JSON.stringify(result) + expect(resultStr).not.toContain("output_config") + expect(resultStr).not.toContain("speed") + expect(resultStr).not.toContain("cache_control") + expect(resultStr).not.toContain("container") + expect(resultStr).not.toContain("mcp_servers") + expect(resultStr).not.toContain("context_management") + expect(resultStr).not.toContain("inference_geo") + }) +``` + +- [ ] **Step 7: Add test — URL-based image source is handled without crash** + +```typescript + test("URL-based image source in user message does not crash", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What's in this image?" }, + { + type: "image", + source: { type: "url", url: "https://example.com/image.png" }, + } as any, + ], + }, + ], + max_tokens: 100, + } + expect(() => translateToOpenAI(anthropicPayload)).not.toThrow() + const result = translateToOpenAI(anthropicPayload) + const userMsg = result.messages.find((m) => m.role === "user") + expect(Array.isArray(userMsg?.content)).toBe(true) + const parts = userMsg?.content as Array<{ type: string; image_url?: { url: string } }> + const imagePart = parts.find((p) => p.type === "image_url") + expect(imagePart?.image_url?.url).toBe("https://example.com/image.png") + }) +``` + +- [ ] **Step 8: Add test — mapContent with image + search_result mix** + +```typescript + test("mapContent with image + search_result mix handles both correctly", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Look at this." }, + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgo=", + }, + }, + { + type: "search_result", + source: "https://example.com", + title: "Mixed Test", + content: "Search result in image context.", + } as any, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const userMsg = result.messages.find((m) => m.role === "user") + expect(Array.isArray(userMsg?.content)).toBe(true) + const parts = userMsg?.content as Array<{ type: string; text?: string }> + expect(parts.some((p) => p.type === "image_url")).toBe(true) + expect(parts.some((p) => p.type === "text" && p.text?.includes("[Search: Mixed Test]"))).toBe(true) + }) +``` + +- [ ] **Step 9: Add test — `isServerToolResultBlock` matches correctly** + +Note: `isServerToolResultBlock` is a module-private function so we can't import it directly. Instead, test its behavior through `handleUserMessage` — verify that `web_search_tool_result` goes through the server tool result path (not the `otherBlocks` → `mapContent` path) and that plain `tool_result` does NOT go through the server tool result path. + +```typescript + test("web_search_tool_result is routed through server tool result path (not mapContent)", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { + role: "user", + content: [ + { + type: "web_search_tool_result", + tool_use_id: "srv_1", + content: [{ type: "web_search_result", url: "https://example.com", title: "Test" }], + }, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + // Should produce a user message with serialized content (not a tool message) + const userMsg = result.messages.find( + (m) => m.role === "user" && typeof m.content === "string", + ) + expect(userMsg).toBeDefined() + expect(userMsg?.content).toContain("web_search_tool_result") + // Should NOT produce a tool message (tool_result routing is separate) + const toolMsg = result.messages.find((m) => m.role === "tool") + expect(toolMsg).toBeUndefined() + }) +}) // close describe block +``` + +- [ ] **Step 10: Run all tests** + +Run: `bun test` +Expected: All tests pass including the 9 new ones + +- [ ] **Step 11: Commit tests** + +```bash +git add tests/anthropic-request.test.ts +git commit -m "test: add tests for new Anthropic content block types and API parity" +``` + +--- + +### Task 18: Final Verification + +- [ ] **Step 1: Run full typecheck** + +Run: `bun run typecheck` +Expected: Zero type errors + +- [ ] **Step 2: Run full test suite** + +Run: `bun test` +Expected: All tests pass + +- [ ] **Step 3: Run linter** + +Run: `bun run lint:all` +Expected: No lint errors (fix any that appear) + +- [ ] **Step 4: Run knip (unused exports check)** + +Run: `bun run knip` +Expected: No new unused exports (the new types should be used by translation code) + +- [ ] **Step 5: Final commit if any lint fixes were needed** + +```bash +git add src/routes/messages/ tests/ +git commit -m "chore: lint fixes for API parity changes" +``` diff --git a/docs/superpowers/specs/2026-03-16-tool-parity-design.md b/docs/superpowers/specs/2026-03-16-tool-parity-design.md new file mode 100644 index 000000000..3393f460d --- /dev/null +++ b/docs/superpowers/specs/2026-03-16-tool-parity-design.md @@ -0,0 +1,567 @@ +# Tool Parity for copilot-api + +**Date:** 2026-03-16 +**Status:** Approved +**Scope:** Full tool parity for the Anthropic ↔ OpenAI translation layer in copilot-api, targeting Claude Code v2.1.76 (March 2026) + +--- + +## Background + +copilot-api is a reverse-engineered proxy that exposes GitHub Copilot as an OpenAI-compatible and Anthropic-compatible HTTP server. The Anthropic translation layer (`src/routes/messages/`) translates Anthropic Messages API requests to OpenAI format and back. + +The translation layer was built against an earlier subset of the Anthropic API. As of Claude Code v2.1.76, multiple gaps exist that cause silent data loss, runtime errors, or incorrect behavior. + +--- + +## Research Summary + +### How Claude Code v2.1.76 Uses Tools + +Claude Code sends all its built-in tools as **custom tools** (with `input_schema`) in the `tools` array on every API call. Key findings: + +- **Always sent (24 tools):** `Agent`, `Bash`, `Edit`, `EnterPlanMode`, `EnterWorktree`, `ExitPlanMode`, `ExitWorktree`, `Glob`, `Grep`, `LSP`, `NotebookEdit`, `Read`, `Skill`, `TaskOutput`, `TaskStop`, `TodoWrite`, `WebFetch`, `WebSearch`, `Write`, `AskUserQuestion`, `CronCreate`, `CronDelete`, `CronList`, `Brief` +- **Conditionally sent:** `TaskCreate`, `TaskGet`, `TaskList`, `TaskUpdate` (interactive mode), `TeamCreate`, `TeamDelete`, `SendMessage` (agent teams feature) +- **Never sent to model:** `ListMcpResourcesTool`, `ReadMcpResourceTool`, `StructuredOutput` (excluded from API payload by Claude Code itself) +- **Conditional/feature-gated:** `ToolSearch` (requires `ENABLE_TOOL_SEARCH` env var) + +### New Tool Definition Fields (v2.1.76) + +Claude Code injects extra fields onto tool definitions that the current proxy does not handle: + +| Field | When Present | OpenAI Equivalent | +|---|---|---| +| `strict: true` | `structured-outputs-2025-12-15` beta active | `function.strict: true` — **forward** | +| `cache_control` | Prompt caching enabled | None — **strip** | +| `defer_loading: true` | ToolSearch enabled | None — **strip** | +| `input_examples` | `tool-examples-2025-10-29` beta | None — **strip** | +| `eager_input_streaming: true` | Internal flag | None — **strip** | + +### New Content Block Types (2025-2026) + +| Block Type | Where | Handling | +|---|---|---| +| `document` | User messages (PDFs via Read tool) | Convert to text placeholder | +| `redacted_thinking` | Assistant messages | Strip (no OpenAI equivalent) | +| `thinking` with `signature` | Assistant messages | Strip `signature` field (already handled by collapsing to text) | +| `server_tool_use` | Assistant messages (multi-turn from real Anthropic API) | Serialize to JSON text | +| `web_search_tool_result` | User messages (multi-turn from real Anthropic API) | Serialize to JSON text | +| `tool_result` with array `content` | User messages (computer use screenshots, image output) | Convert image → `image_url`, text → concat, document → placeholder | + +### Anthropic-Typed Tools + +Non-Claude-Code Anthropic clients may send typed tools (`bash_20250124`, `text_editor_20250728`, `computer_20251124`, `web_search_20250305`, etc.). These have no `input_schema` and cannot be forwarded to Copilot as-is. **Strategy: filter and continue** — strip typed tools from the request, let the call proceed with only custom tools. + +--- + +## Gaps Being Fixed + +| # | Gap | Severity | +|---|-----|----------| +| 1 | Anthropic-typed tools (no `input_schema`) crash translation | 🔴 Critical | +| 2 | `tool_result.content` as array drops images/documents | 🔴 High | +| 3 | `strict` field not forwarded to OpenAI | 🟡 Medium | +| 4 | `disable_parallel_tool_use` in `tool_choice` not parsed | 🟡 Medium | +| 5 | `server_tool_use` / `web_search_tool_result` blocks cause runtime errors | 🟡 Medium | +| 6 | `document` content block type unknown | 🟡 Medium | +| 7 | `redacted_thinking` block type unknown | 🟡 Medium | +| 8 | Token count overhead wrong for typed Anthropic tools | 🟡 Medium | +| 9 | `cache_control`, `defer_loading`, `input_examples`, `eager_input_streaming` on tool defs not stripped | 🟢 Low | +| 10 | `thinking` block missing `signature` field in type definition | 🟢 Low | +| 11 | Model name normalization incomplete (haiku-4-5, future variants) | 🟢 Low | + +--- + +## Architecture + +### Approach: Surgical in-place fixes (Option A) + +Modify 4 existing files with targeted additions. No new files, no new abstractions. Follows existing code patterns. + +**Files changed:** +1. `src/routes/messages/anthropic-types.ts` — type definitions +2. `src/routes/messages/non-stream-translation.ts` — request/response translation +3. `src/routes/messages/count-tokens-handler.ts` — token overhead calculation +4. `src/services/copilot/create-chat-completions.ts` — OpenAI `Tool` type + +--- + +## Detailed Design + +### File 1: `anthropic-types.ts` + +#### 1.1 `AnthropicTool` — Split into union + +```typescript +// Custom tool (has input_schema) — what Claude Code sends +export interface AnthropicCustomTool { + name: string + description?: string + input_schema: Record + strict?: boolean + cache_control?: { type: "ephemeral"; ttl?: number } + defer_loading?: boolean + input_examples?: unknown[] + eager_input_streaming?: boolean +} + +// Anthropic-typed tool (versioned type string, no input_schema) +// e.g. bash_20250124, text_editor_20250728, computer_20251124, web_search_20250305 +export interface AnthropicTypedTool { + type: string + name: string + [key: string]: unknown +} + +export type AnthropicTool = AnthropicCustomTool | AnthropicTypedTool +``` + +**Detection helper (used in non-stream-translation.ts):** + +The discriminant is the **presence of `input_schema`**: custom tools always have it; typed tools never do. Using `!('input_schema' in tool)` as the typed-tool discriminant is more robust than `'type' in tool`, because a future custom tool definition could hypothetically include a `type` field. + +```typescript +export function isTypedTool(tool: AnthropicTool): tool is AnthropicTypedTool { + return !('input_schema' in tool) +} +``` + +#### 1.2 `tool_choice` — Add `disable_parallel_tool_use` + +`disable_parallel_tool_use` has no OpenAI equivalent and is silently ignored in the translation (not forwarded). It must be parsed in the type so it doesn't cause TypeScript errors, but the `translateAnthropicToolChoiceToOpenAI` function does not need to do anything with it. + +```typescript +tool_choice?: { + type: "auto" | "any" | "tool" | "none" + name?: string + disable_parallel_tool_use?: boolean // ← new; parsed but not forwarded (no OpenAI equivalent) +} +``` + +#### 1.3 `AnthropicToolResultBlock` — Array content + +```typescript +export interface AnthropicToolResultBlock { + type: "tool_result" + tool_use_id: string + content: string | Array + is_error?: boolean +} +``` + +#### 1.4 New `AnthropicDocumentBlock` + +Documents can arrive with different source types (base64 PDF, URL, plain text). The interface uses a wide union to avoid TypeScript errors if non-PDF documents arrive, since the handler emits a placeholder regardless of source type. The optional `title` field is included for completeness; it is not used by the translation layer but ensures the interface accurately reflects the wire format. + +```typescript +export interface AnthropicDocumentBlock { + type: "document" + title?: string + source: + | { type: "base64"; media_type: string; data: string } + | { type: "url"; url: string } + | { type: "text"; data: string } + cache_control?: { type: "ephemeral"; ttl?: number } +} +``` + +#### 1.5 `AnthropicThinkingBlock` — Add `signature` + +```typescript +export interface AnthropicThinkingBlock { + type: "thinking" + thinking: string + signature?: string // ← new +} +``` + +#### 1.6 New `AnthropicRedactedThinkingBlock` + +```typescript +export interface AnthropicRedactedThinkingBlock { + type: "redacted_thinking" + data: string +} +``` + +#### 1.7 Server tool blocks (multi-turn passthrough) + +```typescript +export interface AnthropicServerToolUseBlock { + type: "server_tool_use" + id: string + name: string + input: Record +} + +export interface AnthropicWebSearchToolResultBlock { + type: "web_search_tool_result" + tool_use_id: string + content: unknown +} +``` + +#### 1.8 Update content block union types + +```typescript +export type AnthropicUserContentBlock = + | AnthropicTextBlock + | AnthropicImageBlock + | AnthropicDocumentBlock // ← new + | AnthropicToolResultBlock + | AnthropicWebSearchToolResultBlock // ← new (multi-turn) + +export type AnthropicAssistantContentBlock = + | AnthropicTextBlock + | AnthropicToolUseBlock + | AnthropicThinkingBlock + | AnthropicRedactedThinkingBlock // ← new + | AnthropicServerToolUseBlock // ← new (multi-turn) +``` + +--- + +### File 2: `non-stream-translation.ts` + +#### 2.1 Tool filtering — `translateAnthropicToolsToOpenAI` + +```typescript +function translateAnthropicToolsToOpenAI( + anthropicTools: Array | undefined, +): Array | undefined { + if (!anthropicTools) return undefined + + return anthropicTools + .filter((tool): tool is AnthropicCustomTool => !isTypedTool(tool)) + .map((tool) => ({ + type: "function", + function: { + name: tool.name, + description: tool.description, + parameters: tool.input_schema, + ...(tool.strict !== undefined ? { strict: tool.strict } : {}), + // cache_control, defer_loading, input_examples, eager_input_streaming are stripped + }, + })) +} +``` + +#### 2.2 Tool result array content — `handleUserMessage` + +When `block.content` is an array: +- `text` blocks → concatenate as string (or `image_url` parts if images are present) +- `image` blocks → `image_url` data URI +- `document` blocks → text placeholder `"[Document: PDF content not displayable]"` + +```typescript +function mapToolResultContent( + content: AnthropicToolResultBlock["content"], +): string | Array | null { + if (typeof content === "string") return content + return mapContent(content) // reuse existing mapContent logic +} +``` + +In `handleUserMessage`, change: +```typescript +// Before: +content: mapContent(block.content), +// After: +content: mapToolResultContent(block.content), +``` + +#### 2.3 Document block handling in `mapContent` + +The canonical placeholder string is **`"[Document: PDF content not displayable]"`** — used consistently in all code paths. + +**Type signature update required:** Update `mapContent`'s parameter type to accept the new union members added in §1.8: + +```typescript +function mapContent( + content: + | string + | Array, +): string | Array | null +``` + +Since `AnthropicUserContentBlock` now includes `AnthropicDocumentBlock` and `AnthropicWebSearchToolResultBlock`, and `AnthropicAssistantContentBlock` includes `AnthropicServerToolUseBlock` and `AnthropicRedactedThinkingBlock`, the switch statements below will typecheck correctly without extra casts. + +The no-image text path in `mapContent` is the **single authoritative filter** that handles content for both user and assistant messages. It must include `document` and `server_tool_use` — these are mutually exclusive by context but the same code path handles both: + +```typescript +const hasImage = content.some((block) => block.type === "image") +if (!hasImage) { + return content + .filter((block) => + block.type === "text" + || block.type === "thinking" + || block.type === "document" // user messages: PDFs → placeholder + || block.type === "server_tool_use", // assistant messages: server tool → JSON + ) + .map((block) => { + if (block.type === "text") return (block as AnthropicTextBlock).text + if (block.type === "thinking") return (block as AnthropicThinkingBlock).thinking + if (block.type === "document") return "[Document: PDF content not displayable]" + return `[Server tool use: ${JSON.stringify(block)}]` // server_tool_use + }) + .join("\n\n") +} +``` + +In the image path, add to the switch statement: +```typescript +case "document": { + contentParts.push({ type: "text", text: "[Document: PDF content not displayable]" }) + break +} +case "server_tool_use": { + contentParts.push({ type: "text", text: `[Server tool use: ${JSON.stringify(block)}]` }) + break +} +// redacted_thinking: silently skip — no OpenAI equivalent, opaque binary data +``` + +#### 2.4 `handleAssistantMessage` — new block types + +**`redacted_thinking` blocks:** Strip before any processing. This only needs to happen in the **no-tool-use branch** (branch 2), because the tool-use branch (branch 1) already naturally excludes `redacted_thinking` through its explicit `textBlocks`, `thinkingBlocks`, and `toolUseBlocks` filters — `redacted_thinking` matches none of them and is already silently dropped with no change required. + +```typescript +// Branch 2 only (no custom tool_use blocks): +// Filter out redacted_thinking before calling mapContent +const assistantContent = + typeof message.content === "string" + ? message.content + : message.content.filter((b) => b.type !== "redacted_thinking") +return [{ role: "assistant", content: mapContent(assistantContent) }] +``` + +**`server_tool_use` blocks:** Handled in **both** branches, but through different mechanisms: + +- **Branch 2** (no custom `tool_use` blocks): `server_tool_use` flows through `mapContent` via the updated switch and no-image filter in §2.3. No additional changes needed in Branch 2. + +- **Branch 1** (has custom `tool_use` blocks): Branch 1 constructs `allTextContent` directly from explicit filtered arrays (`textBlocks`, `thinkingBlocks`) rather than calling `mapContent`. Therefore, `server_tool_use` blocks would be silently dropped without an explicit filter. Add a `serverToolUseBlocks` filter to Branch 1: + +```typescript +// Branch 1 — after the existing textBlocks and thinkingBlocks filters, add: +const serverToolUseBlocks = message.content.filter( + (block): block is AnthropicServerToolUseBlock => block.type === "server_tool_use", +) +// Update allTextContent to include server tool use serialization. +// The complete updated return statement in Branch 1: +const allTextContent = [ + ...textBlocks.map((b) => b.text), + ...thinkingBlocks.map((b) => b.thinking), + ...serverToolUseBlocks.map((b) => `[Server tool use: ${JSON.stringify(b)}]`), +] + .filter(Boolean) // strip empty strings produced by text/thinking .map() calls + .join("\n\n") + +return [ + { + role: "assistant", + content: allTextContent || null, // preserve existing || null: send null not "" when empty + tool_calls: toolUseBlocks.map((toolUse) => ({ + id: toolUse.id, + type: "function", + function: { + name: toolUse.name, + arguments: JSON.stringify(toolUse.input), + }, + })), + }, +] +``` + +> Note: `.filter(Boolean)` removes empty strings that can be produced by `textBlocks.map(b => b.text)` or `thinkingBlocks.map(b => b.thinking)` when blocks contain empty string content. Without it, `""` entries would create spurious `\n\n` separators in the joined string. The `|| null` on `content` is preserved from the existing code — it converts an empty `allTextContent` to `null`, which is the correct value for an assistant message with no text content (only tool calls). + +#### 2.5 `handleUserMessage` — web_search_tool_result blocks + +`web_search_tool_result` blocks must be excluded from `otherBlocks` to avoid double-processing. `document` blocks are **intentionally left in `otherBlocks`** — they route through `mapContent` correctly and should be passed as a user message (they represent a PDF a user is asking about). + +```typescript +const toolResultBlocks = message.content.filter( + (block): block is AnthropicToolResultBlock => block.type === "tool_result", +) +const webSearchResultBlocks = message.content.filter( + (block): block is AnthropicWebSearchToolResultBlock => + block.type === "web_search_tool_result", +) +const otherBlocks = message.content.filter( + (block) => + block.type !== "tool_result" && + block.type !== "web_search_tool_result", // ← new exclusion; document blocks remain +) + +// tool_result blocks → role: "tool" messages (existing logic, now using mapToolResultContent) +for (const block of toolResultBlocks) { + newMessages.push({ + role: "tool", + tool_call_id: block.tool_use_id, + content: mapToolResultContent(block.content), + }) +} + +// web_search_tool_result blocks → role: "user" message with serialized content +if (webSearchResultBlocks.length > 0) { + const text = webSearchResultBlocks + .map((b) => `[Web search result: ${JSON.stringify(b.content)}]`) + .join("\n\n") + newMessages.push({ role: "user", content: text }) +} + +// remaining otherBlocks → existing logic (now handles document blocks via mapContent) +if (otherBlocks.length > 0) { + newMessages.push({ role: "user", content: mapContent(otherBlocks) }) +} +``` + +#### 2.6 `translateModelName` — generalized normalization + +**Replace** the existing `translateModelName` function body entirely with: + +```typescript +function translateModelName(model: string): string { + // Normalize claude-{family}-4-{minor}[-extra] → claude-{family}-4 + // Only applies to generation 4+ where minor version numbers are subagent-build-specific. + // Known limitation: multi-word family names like claude-sonnet-mini-4 won't match + // (the [a-z]+ pattern does not cross hyphens), but no such models currently exist. + // + // Test cases: + // claude-sonnet-4-6 → claude-sonnet-4 ✓ + // claude-haiku-4-5 → claude-haiku-4 ✓ (was missing in old code) + // claude-opus-4-6 → claude-opus-4 ✓ + // claude-sonnet-4-6-20251231 → claude-sonnet-4 ✓ + // claude-sonnet-3-5 → claude-sonnet-3-5 ✓ (unchanged — 3.x is stable) + // claude-haiku-3-5 → claude-haiku-3-5 ✓ (unchanged — 3.x is stable) + return model.replace(/^(claude-[a-z]+-4)-\d+.*$/, "$1") +} +``` + +--- + +### File 3: `count-tokens-handler.ts` + +**Add import** at the top of the file: +```typescript +import { isTypedTool } from "./anthropic-types" +``` + +The current code adds `+346` **once** for the entire custom tools array (a flat overhead regardless of how many tools). This existing behavior is **preserved** for custom tools. The only change is: when typed tools are present, add their specific per-tool overhead on top. + +```typescript +// Anthropic-typed tool token overhead (per Anthropic pricing docs) +// Only versioned typed tools have specific overhead; custom tools use the existing flat +346 +const ANTHROPIC_TYPED_TOOL_TOKEN_OVERHEAD: Record = { + "text_editor_20250728": 700, + "text_editor_20250429": 700, + "text_editor_20250124": 700, + "text_editor_20241022": 700, + "bash_20250124": 700, + "bash_20241022": 700, + // computer_use and web_search: overhead included in beta pricing, not additive +} + +// In handleCountTokens — replace the existing tools block: +if (anthropicPayload.tools && anthropicPayload.tools.length > 0) { + let mcpToolExist = false + if (anthropicBeta?.startsWith("claude-code")) { + mcpToolExist = anthropicPayload.tools.some( + (tool) => !isTypedTool(tool) && tool.name.startsWith("mcp__"), + ) + } + if (!mcpToolExist) { + if (anthropicPayload.model.startsWith("claude")) { + const hasCustomTools = anthropicPayload.tools.some((t) => !isTypedTool(t)) + const typedTools = anthropicPayload.tools.filter(isTypedTool) + + // Preserve existing flat +346 for custom tools (unchanged behavior) + if (hasCustomTools) { + tokenCount.input += 346 + } + // Add per-typed-tool overhead for Anthropic-typed tools (new) + for (const tool of typedTools) { + tokenCount.input += ANTHROPIC_TYPED_TOOL_TOKEN_OVERHEAD[tool.type] ?? 0 + } + } else if (anthropicPayload.model.startsWith("grok")) { + tokenCount.input += 480 // grok flat overhead unchanged + } + } +} +``` + +--- + +### File 4: `create-chat-completions.ts` + +Add `strict?: boolean` to the `Tool.function` interface: + +```typescript +export interface Tool { + type: "function" + function: { + name: string + description?: string + parameters: Record + strict?: boolean // ← new: Structured Outputs (forward from Anthropic) + } +} +``` + +--- + +## Data Flow Summary + +``` +Claude Code → copilot-api proxy → GitHub Copilot + +[Anthropic Request] +tools: [ + { name: "Bash", input_schema: {...}, cache_control: {...}, strict: true }, // custom tool + { type: "bash_20250124", name: "bash" }, // typed tool (non-CC Anthropic client) +] +tool_choice: { type: "auto", disable_parallel_tool_use: true } +messages: [ + { role: "user", content: [ + { type: "tool_result", tool_use_id: "x", content: [{ type: "image", source: {...} }] }, + { type: "document", source: { type: "base64", media_type: "application/pdf", ... } } + ]} +] + +↓ translateToOpenAI() + +[OpenAI Request] +tools: [ + { type: "function", function: { name: "Bash", parameters: {...}, strict: true } }, + // typed tool "bash_20250124" → stripped (no input_schema, Copilot can't implement) + // cache_control, defer_loading, input_examples, eager_input_streaming → stripped +] +tool_choice: "auto" +// disable_parallel_tool_use → acknowledged in type, not forwarded (no OpenAI equivalent) +messages: [ + // tool_result with image array → role:"tool" with ContentPart array (vision-capable) + { role: "tool", tool_call_id: "x", content: [{ type: "image_url", image_url: { url: "data:image/jpeg;base64,..." } }] }, + // document block → role:"user" with placeholder text + { role: "user", content: "[Document: PDF content not displayable]" } +] +``` + +> **Note on tool role content:** OpenAI tool messages technically accept string content only in the base spec. However, GitHub Copilot's vision-capable models accept `ContentPart` arrays (including `image_url`) in tool messages, matching the pattern used for user messages with images. This is consistent with how the existing `image` block handling works in `handleUserMessage`. If a Copilot model rejects this, the fallback would be to extract only the text parts, but that would lose the image data entirely. + +--- + +## Testing + +- Unit tests for `translateAnthropicToolsToOpenAI` with typed tools present +- Unit tests for `mapContent`/`mapToolResultContent` with array tool result content containing images +- Unit tests for `translateModelName` with all claude model variants +- Unit tests for `handleAssistantMessage` with `redacted_thinking` blocks +- Integration smoke test: send a Claude Code-style payload with all new fields, verify it reaches Copilot without error + +--- + +## Out of Scope + +- Streaming path changes: no new block types appear in Anthropic streaming chunks that aren't already handled. The `input_json_delta` and `text_delta` handling in `stream-translation.ts` is correct as-is. +- `compaction_delta` streaming event: not yet documented in the official Anthropic API; leave for future work. +- MCP resource tools (`ListMcpResourcesTool`, `ReadMcpResourceTool`): these are never sent by Claude Code to the model and are not in the `tools` array, so no proxy changes needed. +- `pause_turn` stop reason: only appears from Anthropic server-side tools (web search), which we don't proxy. No change needed. diff --git a/docs/superpowers/specs/2026-03-16-web-search-design.md b/docs/superpowers/specs/2026-03-16-web-search-design.md new file mode 100644 index 000000000..e25b32da2 --- /dev/null +++ b/docs/superpowers/specs/2026-03-16-web-search-design.md @@ -0,0 +1,389 @@ +# Web Search Design Spec + +**Date:** 2026-03-16 +**Project:** copilot-api +**Status:** Approved + +--- + +## Goal + +Implement a transparent server-side web search capability in copilot-api using the Brave Search API. When a client (e.g. Claude Code) requests web search via the Anthropic `web_search` typed tool or via natural language intent, the proxy performs the search itself and injects results into the conversation before forwarding to Copilot — replicating the server-side behavior of the real Anthropic API. + +--- + +## Background + +The Anthropic API supports a `web_search_20250305` typed tool (and future versioned variants). When present, Anthropic handles the search server-side: the model emits a `server_tool_use` block, Anthropic runs the search, and injects a `web_search_tool_result` block — all transparently. The client never sees the loop. + +Copilot has no equivalent capability. Currently, copilot-api silently filters typed tools including `web_search_*`, so the model answers from training data only with no indication that search was unavailable. + +This design adds a web search interceptor that replicates the Anthropic server-side behavior using Brave Search, so clients get current web data regardless of whether they are routed through Copilot. + +--- + +## Architecture + +### Activation + +Web search is **opt-out**: automatically enabled whenever `BRAVE_API_KEY` is set in the environment. No CLI flag required. If the key is absent, behavior falls back to the existing silent-filter path. + +### High-Level Flow + +Detection and stripping happen **on the Anthropic payload** (`AnthropicMessagesPayload`) in `handler.ts`, **before** `translateToOpenAI` is called. This is the correct layer because: + +- `translateAnthropicToolsToOpenAI` already strips all typed tools — there is nothing to detect or strip in the translated OpenAI payload. +- `isTypedTool` is defined for Anthropic types and has no meaning on the already-translated OpenAI payload. + +``` +Client (Claude Code) + │ POST /v1/messages { tools: [web_search_20250305, Bash, ...], messages: [...] } + ▼ +handler.ts + │ isWebSearchEnabled() → true + │ detectWebSearchIntent(anthropicPayload) → true/false + │ if false → translateToOpenAI → createChatCompletions directly, return + │ if true: + │ stripWebSearchTypedTools(anthropicPayload) → cleanedPayload + │ translateToOpenAI(cleanedPayload) → openAIPayload (no typed tools) + │ inject web_search function tool into openAIPayload.tools + │ → webSearchInterceptor(openAIPayload) + ▼ +interceptor.ts + │ call Copilot (non-streaming) + │ + ├─ finish_reason: "stop" → return response as-is + └─ finish_reason: "tool_calls", name: "web_search" + │ extract query from arguments + ▼ + brave.ts + │ GET https://api.search.brave.com/res/v1/web/search?q= + │ returns top 5 { title, url, description } results + ▼ + interceptor.ts + │ append assistant tool_call message + tool result message + │ re-call Copilot (respects original stream flag) + │ return final response + ▼ +handler.ts + │ translateToAnthropic(response) — or stream translation for streaming + │ return to client + ▼ +Client receives answer with current web data baked in +``` + +--- + +## Detection + +Detection runs on the **Anthropic payload** in `handler.ts`, before translation. Two independent paths. Either one triggers the web search flow. Path 1 is checked first (zero cost); Path 2 only fires when Path 1 is false. + +### Path 1: Typed Tool Detection + +Match any typed tool (no `input_schema`) whose `name` is in the known web search names set: + +```typescript +const WEB_SEARCH_TOOL_NAMES = new Set([ + "web_search", + "internet_search", + "brave_search", + "bing_search", + "google_search", + "find_online", + "internet_research", +]) + +const hasWebSearchTool = payload.tools?.some( + (tool) => isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name) +) +``` + +Detection uses `tool.name` (stable across versions) rather than `tool.type` (e.g. `web_search_20250305`), so future versioned variants like `web_search_20260101` are automatically handled. + +**Note on `"search"` exclusion:** The generic name `"search"` is intentionally excluded from `WEB_SEARCH_TOOL_NAMES`. Users commonly define custom tools named `"search"` for local codebase search, database search, etc. Including it would cause false positives. The remaining names are specific enough to be unambiguous. + +**Note on typed-tool guard:** The `isTypedTool(tool)` guard ensures that a custom tool (one with `input_schema`) named `"web_search"` is never mistakenly triggered. Only Anthropic-managed server tools (no `input_schema`) trigger the web search path. + +### Path 2: Natural Language Detection + +Only triggered when Path 1 is false. Sends a lightweight preflight classification request to Copilot using only the **last user message** in the conversation (the most recent turn, not the full history — sufficient for intent detection and keeps the preflight call cheap): + +``` +System: You are a classifier. Answer only "yes" or "no". No explanation. +User: Does this message require searching the web for current or real-time information? +Message: "" +``` + +- Response `"yes"` (case-insensitive) → trigger web search +- Response `"no"` or any other value → skip web search +- If the preflight call itself fails → log a warning, treat as "no", continue without search (never block the main request) + +**Performance cost:** Path 2 fires an extra Copilot round-trip for every request that lacks a typed web search tool. For Claude Code sessions — which send many requests for Bash, file reads, and coding tasks — this means nearly every request pays a preflight cost. This is acceptable for users who want natural language search detection; however, to manage cost, the implementation should use a small/fast model for the preflight call rather than the model in the original request. The preflight call always uses `stream: false` and a minimal `max_tokens: 5`. + +**Known limitation:** The last-message-only approach can fail to detect web search intent in multi-turn conversations where the intent was established earlier (e.g., "What about news from last week?" after a prior research exchange). This is acceptable for v1; the classifier will miss some cases and the model will fall back to training data. + +--- + +## Stripping Typed Web Search Tools + +When Path 1 is true, the web search typed tools are stripped from the Anthropic payload **before** `translateToOpenAI` is called: + +```typescript +function stripWebSearchTypedTools( + payload: AnthropicMessagesPayload, +): AnthropicMessagesPayload { + return { + ...payload, + tools: payload.tools?.filter( + (tool) => !(isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name)), + ), + } +} +``` + +This produces a clean Anthropic payload where `translateToOpenAI` can proceed normally — any remaining typed tools (e.g., `bash_20250124`) are already handled by `translateAnthropicToolsToOpenAI`'s existing filter. + +--- + +## The Injected Function Tool + +After translation, the interceptor adds this OpenAI function tool to `openAIPayload.tools`: + +```typescript +{ + type: "function", + function: { + name: "web_search", + description: "Search the web for current information. Use this when you need up-to-date facts, recent events, or information beyond your training data.", + parameters: { + type: "object", + properties: { + query: { + type: "string", + description: "The search query" + } + }, + required: ["query"] + } + } +} +``` + +**`tool_choice` passthrough:** If the original Anthropic request had `tool_choice: { type: "tool", name: "web_search" }`, the translated OpenAI `tool_choice` will be `{ type: "function", function: { name: "web_search" } }`. Since the injected function tool is also named `"web_search"`, this continues to work correctly — no adjustment to `tool_choice` is needed. + +**`tool_choice: web_search` forces a second pass:** When a client sends `tool_choice: { type: "tool", name: "web_search" }`, Copilot's first pass will always call `web_search` (the model is forced to). The `stop` branch of step 3 is therefore unreachable in this case — the interceptor will always proceed to the Brave search and second pass. This is correct behavior: the client explicitly requested a web search, so the proxy always performs one. + +--- + +## The Tool-Call Loop + +The interceptor handles at most **one search per request** — no recursive loop. If Copilot calls `web_search` again in the final pass, the response is returned as-is. + +**Step-by-step:** + +1. Receive translated OpenAI payload (typed web search tools already stripped, function tool injected) +2. Call Copilot **non-streaming** (regardless of original `stream` flag) +3. Inspect `finish_reason` on the first response: + - Not `"tool_calls"` → return response as-is (model answered without searching) + - `"tool_calls"` with no `web_search` call → return response as-is (model called a different tool) + - `"tool_calls"` with one or more calls — if any is `name === "web_search"` → proceed to step 4 + - `"tool_calls"` with multiple calls including `web_search` → execute the web search; append all tool_call messages and the web_search result; for any other tool call IDs in the same assistant message, append a stub `{ role: "tool", tool_call_id: "", content: "" }` message so Copilot's second pass receives a complete tool result for each tool_call in the assistant turn (required to avoid Copilot rejecting a partial result set) +4. Parse `query` from the `web_search` tool call's `function.arguments` (JSON). If `JSON.parse` fails, inject a failure message as tool result (treat as API failure) and proceed to step 7. +5. Call Brave Search API → top 5 results or `BraveSearchError` (timeout: 5 seconds; treat timeout as network failure) +6. Format results as plain text (see below) +7. Append to messages: + - `{ role: "assistant", tool_calls: [...] }` — Copilot's full tool_use response (all tool calls, not just web_search) + - `{ role: "tool", tool_call_id: "...", content: "" }` — result for the web_search call +8. Re-call Copilot with the **original `stream` flag** — streaming clients get a streamed final response +9. Return final response to `handler.ts` + +**Formatted search results (injected as tool result content):** + +``` +Web search results for: "" + +1. Title: + URL: <url> + Snippet: <description> + +2. ... + +[up to 5 results] +``` + +Plain text — no JSON wrapping — so Copilot's model reads it naturally. + +**Zero results:** +``` +No results found for: "<query>" +``` + +**Brave API failure or JSON parse error:** +``` +Web search failed: <reason> +Please answer based on your training data and let the user know that web search is currently unavailable. +``` + +--- + +## Return Type + +The interceptor returns `ReturnType<typeof createChatCompletions>`, which resolves to: + +```typescript +Promise<ChatCompletionResponse | AsyncIterableIterator<ServerSentEvent>> +``` + +where `ServerSentEvent` is from `fetch-event-stream`. `handler.ts` already discriminates these two cases using `isNonStreaming(response)` (`Object.hasOwn(response, "choices")`). The interceptor's return value plugs directly into the existing handler branch — no new type alias is needed. + +--- + +## Error Handling + +| Failure | Behavior | +|---------|----------| +| `BRAVE_API_KEY` missing | `isWebSearchEnabled()` → false, silent passthrough (existing behavior) | +| Brave API non-200 | Inject failure message as tool result, re-submit to Copilot, which informs user gracefully | +| Brave API network error/timeout (5s limit) | Same as non-200 | +| Brave returns 0 results | Inject "No results found" as tool result | +| `JSON.parse` failure on tool arguments | Inject failure message as tool result, continue to second pass | +| Preflight classification call fails | Log warning, treat as "no search needed", continue normally | +| Copilot second pass fails | Propagate via existing error handling in `handler.ts` | + +The request **always completes** from the client's perspective — a Brave failure never produces a 5xx to the client. + +--- + +## New Files + +### `src/services/web-search/types.ts` +Shared types only. No logic. +- `BraveSearchResult` — `{ title: string; url: string; description: string }` +- `BraveSearchError` — typed error class with `reason: string` + +### `src/services/web-search/brave.ts` +Single responsibility: call Brave Search API, return structured results. +- `searchBrave(query: string, apiKey: string): Promise<BraveSearchResult[]>` +- Always fetches top 5 results (`count=5` query param, hardcoded) +- Applies a 5-second `AbortController` timeout +- Throws `BraveSearchError` on non-200, network failure, or timeout +- No knowledge of Copilot or the tool-call loop + +### `src/services/web-search/tool-definition.ts` +Constants only. No logic. +- `WEB_SEARCH_TOOL_NAMES: Set<string>` — known web search tool names +- `WEB_SEARCH_FUNCTION_TOOL: Tool` — the injected OpenAI function tool definition + +### `src/services/web-search/interceptor.ts` +Single responsibility: tool-call loop. +- `webSearchInterceptor(payload: ChatCompletionsPayload): ReturnType<typeof createChatCompletions>` +- Receives payload that already has the function tool injected +- Calls `brave.ts` and `createChatCompletions` +- No knowledge of Anthropic types + +--- + +## Modified Files + +### `src/lib/state.ts` +- Add `braveApiKey?: string` field to `State` interface +- Add `isWebSearchEnabled(): boolean` helper — `return !!state.braveApiKey` + +### `src/start.ts` +- Read `process.env.BRAVE_API_KEY` at startup, store in `state.braveApiKey` +- Log `consola.info("Web search enabled (Brave)")` if key is present + +### `src/routes/messages/handler.ts` +Detection, stripping, and injection all happen here, on the Anthropic payload, before translation. + +**Current handler order (before this change):** +1. `checkRateLimit` +2. `c.req.json()` — parse payload +3. `translateToOpenAI` +4. `manualApprove` (if enabled) +5. `createChatCompletions` + +**New handler order (after this change):** +1. `checkRateLimit` +2. `c.req.json()` — parse payload +3. `manualApprove` (if enabled) — moved before translation so approval fires before any Copilot call +4. `detectWebSearchIntent` + branch (web search or direct) +5. `translateToOpenAI` inside each branch +6. `createChatCompletions` / `webSearchInterceptor` + +`manualApprove` is deliberately moved before translation — it fires once for the user-visible request before any Copilot call is made, which is its intended purpose. This is a harmless ordering change since `translateToOpenAI` is pure (no I/O). + +```typescript +// Simplified handler pseudocode after changes: + +const anthropicPayload = await c.req.json<AnthropicMessagesPayload>() + +if (state.manualApprove) { + await awaitApproval() +} + +let response: Awaited<ReturnType<typeof createChatCompletions>> + +if (isWebSearchEnabled() && await detectWebSearchIntent(anthropicPayload)) { + const cleanedPayload = stripWebSearchTypedTools(anthropicPayload) + const openAIPayload = translateToOpenAI(cleanedPayload) + openAIPayload.tools = [...(openAIPayload.tools ?? []), WEB_SEARCH_FUNCTION_TOOL] + response = await webSearchInterceptor(openAIPayload) +} else { + const openAIPayload = translateToOpenAI(anthropicPayload) + response = await createChatCompletions(openAIPayload) +} +``` + +The rest of the handler (streaming/non-streaming Anthropic translation via `isNonStreaming`) is unchanged. + +**Note:** The `manualApprove` prompt fires once before the branch. The interceptor's internal Copilot calls (first pass + second pass) bypass manual approval and rate limiting intentionally — they are implementation details of the web search loop, not new user-visible requests. Users with tight rate limits should be aware that web search requests consume 2–3 Copilot API calls total. + +### `src/routes/messages/web-search-detection.ts` (new file) +A dedicated module for web search detection and stripping. It has access to both Anthropic types and `createChatCompletions`, and keeps `non-stream-translation.ts` focused on pure translation and `handler.ts` thin. + +Exports: +- `detectWebSearchIntent(payload: AnthropicMessagesPayload): Promise<boolean>` — Path 1 (typed tool check, zero cost) then Path 2 (preflight Copilot call if Path 1 is false) +- `stripWebSearchTypedTools(payload: AnthropicMessagesPayload): AnthropicMessagesPayload` — returns new payload with web search typed tools removed + +`WEB_SEARCH_TOOL_NAMES` (from `tool-definition.ts`) and `isTypedTool` (from `anthropic-types.ts`) are used internally by both functions. + +The Path 2 preflight call uses `createChatCompletions` directly with a hardcoded small model (use the first available model from `state.models` that is not the request model, falling back to the request model if only one model is available), `stream: false`, and `max_tokens: 5`. + +--- + +## Testing + +New test file: `tests/web-search.test.ts` + +| Test | What it verifies | +|------|-| +| Typed tool detection — `web_search` name | `WEB_SEARCH_TOOL_NAMES` match | +| Typed tool detection — `internet_research` name | Variant name match | +| Typed tool detection — `web_search_20260101` type (future version) | `tool.name` used, not `tool.type` | +| Typed tool detection — custom tool named `web_search` (has `input_schema`) | `isTypedTool` guard prevents false positive | +| Typed tool detection — custom tool named `search` (has `input_schema`) | `"search"` excluded from set, no false positive | +| Natural language — preflight returns "yes" | Search triggered | +| Natural language — preflight returns "no" | Search skipped | +| Natural language — preflight call fails | Warning logged, search skipped, request continues | +| Interceptor — Copilot returns `stop` | No search, response returned as-is | +| Interceptor — Copilot returns `tool_calls` for `web_search` | Brave called, results injected, second Copilot call made | +| Interceptor — Copilot returns `tool_calls` for different tool | No search, response returned as-is | +| Interceptor — Copilot returns `tool_calls` for `web_search` + another tool | Web search executed; stub result injected for non-search tool; second pass made | +| Interceptor — `tool_choice: { type: "function", function: { name: "web_search" } }` | Injected function tool satisfies the tool_choice correctly | +| Brave API — successful response | Top 5 results formatted correctly | +| Brave API — non-200 response | `BraveSearchError` thrown, failure message injected | +| Brave API — zero results | "No results found" message injected | +| Brave API — malformed JSON in tool arguments | Failure message injected, second pass made | +| End-to-end — `BRAVE_API_KEY` absent | `isWebSearchEnabled()` false, interceptor never called | + +--- + +## Out of Scope + +- Page content fetching (snippets only) +- Multiple searches per request (max 1 loop) +- Streaming the first Copilot pass (always buffered for tool-call detection) +- Caching search results +- User-facing configuration beyond `BRAVE_API_KEY` env var +- Disabling Path 2 natural language detection independently of Path 1 diff --git a/docs/superpowers/specs/2026-03-17-transparent-web-search-and-model-switching-design.md b/docs/superpowers/specs/2026-03-17-transparent-web-search-and-model-switching-design.md new file mode 100644 index 000000000..8b8013f59 --- /dev/null +++ b/docs/superpowers/specs/2026-03-17-transparent-web-search-and-model-switching-design.md @@ -0,0 +1,356 @@ +# Transparent Web Search + Context-Aware Model Switching — Design Spec + +**Date:** 2026-03-17 +**Status:** Approved + +## Problem + +Two related failures when AI agents (Claude Code, Claude Desktop, custom clients) use the proxy: + +### 1. Web search never fires for AI agents + +The proxy's web search only triggered when the client explicitly declared a `web_search` tool in the request. AI agents (especially Claude Code) never do this — they use their own built-in `WebFetch` tool, which bypasses the proxy entirely. Result: the proxy's Tavily integration is invisible to agents. + +### 2. Token overflow crashes the proxy + +When a long conversation (e.g. accumulated tool results, large system prompts) exceeds the context window of the Copilot-hosted model, Copilot returns `model_max_prompt_tokens_exceeded`. The proxy currently crashes with an unhelpful error (`Response {}`), and the client receives a double-wrapped stringified error JSON instead of the real structured error. + +--- + +## Solution Overview + +Four coordinated changes: + +1. **Always-on web search** — when web search is enabled, inject `web_search` into every request unconditionally; let Copilot decide when to call it +2. **Fix streaming in interceptor** — the interceptor currently forces `stream: false` on the first pass; when Copilot doesn't call the tool, streaming must be preserved by re-issuing the request with the original `stream` flag +3. **Context-aware model switching** — before sending any request, check if the prompt exceeds the model's context window and auto-switch to the largest-context available model +4. **Upstream error forwarding** — forward Copilot's JSON error body directly (already approved in companion spec `2026-03-17-upstream-error-forwarding-design.md`) + +--- + +## Section 1: Always-On Web Search Injection + +### Current flow (messages route handler) + +``` +detectWebSearchIntent(payload) ← preflight Copilot call (Path 2) or typed-tool check (Path 1) + → true: prepareWebSearchPayload → webSearchInterceptor + → false: createChatCompletions directly +``` + +### New flow + +``` +if (isWebSearchEnabled()): + prepareWebSearchPayload → webSearchInterceptor +else: + createChatCompletions directly +``` + +Detection is removed entirely. When `TAVILY_API_KEY` or `BRAVE_API_KEY` is set, every request gets `WEB_SEARCH_FUNCTION_TOOL` injected into the OpenAI `tools` array. Copilot calls the tool when it judges the question needs real-time data; otherwise it ignores the tool. See Section 2 for how streaming is preserved on non-search requests. + +### `web-search-detection.ts` — deleted + +The entire file is removed. It contained: +- `detectWebSearchIntent()` — no longer needed +- `stripWebSearchTypedTools()` — no longer needed (typed web search tools from clients are already stripped by `translateAnthropicToolsToOpenAI` in `non-stream-translation.ts`, which filters to custom tools only) +- `getPreflightModel()` — no longer needed +- `getLastUserMessageText()` — no longer needed + +The imports of `detectWebSearchIntent` and `stripWebSearchTypedTools` in `handler.ts` are removed. + +Note on `stripWebSearchTypedTools`: `translateAnthropicToolsToOpenAI` filters to `!isTypedTool(tool)`, which strips all Anthropic typed tools (including `web_search_20250305`) before the OpenAI payload is built. This was already true on the non-web-search branch in the old flow, so removal of `stripWebSearchTypedTools` is safe. + +Note on `tool_choice` pointing to a stripped typed tool (e.g. `tool_choice: { type: "tool", name: "web_search_20250305" }`): this produces a malformed OpenAI request regardless of this change — the typed tool is stripped but the `tool_choice` is translated to a named function choice. This pre-existing edge case is not made worse by this change and is out of scope. + +### handler.ts changes (messages route) + +The `response` variable is declared with `let` at the outer scope so it can be assigned in either branch. The web search branch declares `openAIPayload` with `let` so the model-switch guard (Section 3) can reassign it. + +```typescript +// Before +if (isWebSearchEnabled() && (await detectWebSearchIntent(anthropicPayload))) { + const cleanedPayload = stripWebSearchTypedTools(anthropicPayload) + const openAIPayload = prepareWebSearchPayload(translateToOpenAI(cleanedPayload)) + response = await webSearchInterceptor(openAIPayload) +} else { + const openAIPayload = translateToOpenAI(anthropicPayload) + response = await createChatCompletions(openAIPayload) +} + +// After +if (isWebSearchEnabled()) { + let openAIPayload = prepareWebSearchPayload(translateToOpenAI(anthropicPayload)) + // model-switch guard inserted here (see Section 3) + consola.debug("Translated OpenAI request payload (web search):", JSON.stringify(openAIPayload)) + response = await webSearchInterceptor(openAIPayload) +} else { + let openAIPayload = translateToOpenAI(anthropicPayload) + // model-switch guard inserted here (see Section 3) + consola.debug("Translated OpenAI request payload:", JSON.stringify(openAIPayload)) + response = await createChatCompletions(openAIPayload) +} +``` + +The `consola.debug` calls are placed **after** the model-switch guard so they log the final payload (including any model change). Note: the current handler has these debug calls at the end of each branch in the same position — they are retained, not removed. + +--- + +## Section 2: Fix Streaming in webSearchInterceptor + +### The problem + +`webSearchInterceptor` currently forces `stream: false` on the first pass unconditionally: + +```typescript +const firstPassPayload: ChatCompletionsPayload = { ...payload, stream: false } +``` + +When Copilot doesn't call the web search tool (`finish_reason !== "tool_calls"`), the interceptor returns `firstResponse` — which was fetched non-streaming. If the client requested streaming, they get a non-streaming response. This is a silent regression introduced by always routing through the interceptor. + +### The fix + +Change the interceptor to use the **client's original `stream` flag on the first pass**. If Copilot doesn't call the tool, the streaming response is returned directly as-is. If Copilot does call the tool, the first pass response is a non-streaming response (used to extract the tool call arguments), so the first pass must be forced non-streaming **only when the response will be consumed as data** (i.e. only when Copilot calls the tool). + +This requires two passes through the first-pass logic: + +**Option A (recommended): Two-step first pass** +1. Always send first pass with `stream: false` (non-streaming) to inspect `finish_reason` +2. If `finish_reason !== "tool_calls"` → **re-issue** the request with the original `stream` flag and return that response +3. If `finish_reason === "tool_calls"` → proceed with tool execution and second pass as before + +This adds one extra Copilot call only when the client requests streaming and Copilot doesn't call the tool (total: 2 calls — one non-streaming inspection + one streaming re-issue). For non-streaming non-search requests, there is zero overhead: 1 call total (the non-streaming inspection is returned directly). This is acceptable because web search is opt-in (requires an API key). + +**Option B: Single streaming first pass with tool-call sniffing** +Send first pass with original `stream` flag. If streaming, buffer SSE chunks looking for `finish_reason: tool_calls`. If tool call detected, switch to non-streaming second pass. Complex and fragile — rejected. + +### Change to `src/services/web-search/interceptor.ts` + +There are **two** early-exit points in the current interceptor that both return `firstResponse` (non-streaming) — both must be updated. + +**First early exit** (lines ~55–58, when `finish_reason !== "tool_calls"`): + +```typescript +// CURRENT +const choice = firstResponse.choices.at(0) +if (!choice || choice.finish_reason !== "tool_calls" || !choice.message.tool_calls) { + return firstResponse // always non-streaming — BUG when client wanted streaming +} +``` + +**Change to:** + +```typescript +// NEW +const choice = firstResponse.choices.at(0) +if (!choice || choice.finish_reason !== "tool_calls" || !choice.message.tool_calls) { + if (payload.stream) { + // Strip WEB_SEARCH_FUNCTION_TOOL before re-issuing so Copilot cannot call it again + // in the streaming response (a tool_call chunk would reach the client raw and corrupt + // the Anthropic translation layer). + const streamPayload: ChatCompletionsPayload = { + ...payload, + tools: payload.tools?.filter(t => t.function.name !== WEB_SEARCH_TOOL_NAME), + } + return createChatCompletions(streamPayload) + } + return firstResponse // non-streaming: return first pass directly (no extra call) +} +``` + +**Second early exit** (lines ~63–65, when tool_calls were made but none was the web_search tool): + +```typescript +// CURRENT +const webSearchCall = choice.message.tool_calls.find( + (tc) => tc.function.name === WEB_SEARCH_TOOL_NAME, +) +if (!webSearchCall) { + return firstResponse // always non-streaming — same streaming BUG +} +``` + +**Change to:** + +```typescript +// NEW +const webSearchCall = choice.message.tool_calls.find( + (tc) => tc.function.name === WEB_SEARCH_TOOL_NAME, +) +if (!webSearchCall) { + if (payload.stream) { + // Strip WEB_SEARCH_FUNCTION_TOOL before re-issuing (same reason as first early exit) + const streamPayload: ChatCompletionsPayload = { + ...payload, + tools: payload.tools?.filter(t => t.function.name !== WEB_SEARCH_TOOL_NAME), + } + return createChatCompletions(streamPayload) + } + return firstResponse +} +``` + +When `payload.stream` is falsy, `firstResponse` is returned directly — 1 Copilot call total. When `payload.stream` is truthy, a second request is issued with the original payload — 2 Copilot calls total. For actual web search (tool was called): first non-streaming pass + second pass with original stream flag = 2 Copilot calls + 1 Tavily call (unchanged). + +### `tool_choice: "none"` behaviour + +When the client sends `tool_choice: "none"`, the first pass non-streaming inspection call will return `finish_reason: "stop"` (Copilot won't call any tool). The interceptor then re-issues with original stream flag (if streaming) and returns. Correct behaviour. + +--- + +## Section 3: Context-Aware Model Switching + +### New file: `src/lib/model-selector.ts` + +Pure function, no side effects, easy to unit test. + +```typescript +export interface ModelSelectionResult { + model: string + switched: boolean + reason?: string +} + +export function selectModelForTokenCount( + requestedModelId: string, + models: ModelsResponse, + estimatedTokens: number, +): ModelSelectionResult +``` + +**Logic:** +1. Find the requested model in `models.data` by `id` +2. If model not found → return `{ model: requestedModelId, switched: false }` (can't decide without capability data) +3. Read `capabilities.limits.max_context_window_tokens` (preferred) or `capabilities.limits.max_prompt_tokens` as fallback. If neither is set → return `{ model: requestedModelId, switched: false }` +4. If `estimatedTokens <= contextWindow` → return `{ model: requestedModelId, switched: false }` (within limits) +5. Find the model with the largest `max_context_window_tokens` across all `models.data`. If it is the same as the requested model → return `{ model: requestedModelId, switched: false, reason: "already largest context model" }` +6. Return `{ model: largestContextModel.id, switched: true, reason: "prompt [N] tokens exceeds [requestedModel] context window [M], switching to [newModel] [L]" }` + +Note: token estimation uses the original model's tokenizer. If the fallback model uses a different tokenizer the estimate may be slightly off — this is acceptable since the guard is a best-effort heuristic. + +### Integration in handler.ts (messages route) + +Inserted after `prepareWebSearchPayload`/`translateToOpenAI`, before routing to interceptor or `createChatCompletions`. The `openAIPayload` variable is `let`-scoped (shown in Section 1). The guard is identical in both the web-search and non-web-search branches: + +```typescript +// Model-switch guard (inserted in both branches, after openAIPayload is assigned) +// Retain the existing consola.debug log of openAIPayload immediately before this block. +if (state.models) { + try { + const modelForCount = state.models.data.find(m => m.id === openAIPayload.model) + if (modelForCount) { + const { input: estimatedTokens } = await getTokenCount(openAIPayload, modelForCount) + const result = selectModelForTokenCount(openAIPayload.model, state.models, estimatedTokens) + if (result.switched) { + consola.warn(`Context overflow: ${result.reason}`) + openAIPayload = { ...openAIPayload, model: result.model } + } + } + } catch { + consola.debug("Token count estimation failed, skipping model switch") + } +} +``` + +For `getTokenCount`, the model object is found as: `state.models.data.find(m => m.id === openAIPayload.model)`. If not found, the try/catch handles the failure gracefully. + +### Integration in chat-completions/handler.ts + +The `chat-completions/handler.ts` existing handler calls `getTokenCount` with `selectedModel` for logging (lines 29–38). Change `selectedModel` from `const` to `let`, then insert the model-switch guard inside the same try block, after the existing `consola.info` call: + +```typescript +// Change: const → let +let selectedModel = state.models?.data.find( + (model) => model.id === payload.model, +) + +try { + if (selectedModel) { + const tokenCount = await getTokenCount(payload, selectedModel) + consola.info("Current token count:", tokenCount) + // NEW: model-switch guard + // state.models is non-null here — selectedModel was found from it + // Use non-null assertion (!) since TypeScript cannot narrow through the optional-chain above + const result = selectModelForTokenCount(payload.model, state.models!, tokenCount.input) + if (result.switched) { + consola.warn(`Context overflow: ${result.reason}`) + payload = { ...payload, model: result.model } + // Update selectedModel so max_tokens defaulting below uses the switched model + selectedModel = state.models!.data.find(m => m.id === result.model) ?? selectedModel + } + } else { + consola.warn("No model selected, skipping token count calculation") + } +} catch (error) { + consola.warn("Failed to calculate token count:", error) +} +``` + +Note: `payload` is already declared with `let` in `chat-completions/handler.ts` (line 21: `let payload = await c.req.json<ChatCompletionsPayload>()`), so reassigning `payload` is valid. + +--- + +## Section 4: Upstream Error Forwarding + +See companion spec `2026-03-17-upstream-error-forwarding-design.md`. Both changes from that spec are included in this implementation: + +1. Remove premature `consola.error(..., response)` in `create-chat-completions.ts` +2. Forward Copilot's parsed JSON error body directly in `error.ts` + +--- + +## Required New Imports + +The following imports must be added to files that don't already have them: + +| File | Import to add | +|------|--------------| +| `src/lib/model-selector.ts` | `import type { ModelsResponse } from "~/services/copilot/get-models"` | +| `src/routes/messages/handler.ts` | `import { getTokenCount } from "~/lib/tokenizer"` | +| `src/routes/messages/handler.ts` | `import { selectModelForTokenCount } from "~/lib/model-selector"` | +| `src/routes/chat-completions/handler.ts` | `import { selectModelForTokenCount } from "~/lib/model-selector"` | + +Remove from `src/routes/messages/handler.ts`: +- `import { detectWebSearchIntent, stripWebSearchTypedTools } from "./web-search-detection"` (entire import, file is deleted) + +--- + +## Files Changed + +| File | Change | +|------|--------| +| `src/routes/messages/handler.ts` | Remove detection branch; add model-switch guard; use `let` for `openAIPayload` | +| `src/routes/messages/web-search-detection.ts` | **Deleted** | +| `src/routes/chat-completions/handler.ts` | Add model-switch guard + `selectedModel` update after existing token count | +| `src/services/web-search/interceptor.ts` | Fix streaming: re-issue with original `stream` flag when Copilot doesn't call tool | +| `src/lib/model-selector.ts` | **New** — pure `selectModelForTokenCount` helper | +| `src/lib/error.ts` | Forward upstream JSON errors directly | +| `src/services/copilot/create-chat-completions.ts` | Remove premature `Response {}` log | + +## Tests + +| File | Change | +|------|--------| +| `tests/model-selector.test.ts` | **New** — unit tests: overflow detection, model switching, no-op within limits, missing model, missing capability data | +| `tests/web-search.test.ts` | Remove detection/preflight tests; add always-on injection tests; add streaming preservation tests for interceptor | + +--- + +## Edge Cases + +- **Web search disabled** (`TAVILY_API_KEY` and `BRAVE_API_KEY` both unset): `isWebSearchEnabled()` returns false, flow is identical to current behaviour — no overhead at all. +- **Copilot doesn't call web_search, streaming request**: interceptor fires non-streaming first pass (inspection), then re-issues with `stream: true`. Client gets streaming response. Total: 2 Copilot calls. +- **Copilot doesn't call web_search, non-streaming request**: interceptor fires non-streaming first pass, returns it directly. Total: 1 Copilot call. No overhead vs. direct path. +- **Copilot calls web_search**: first non-streaming pass → tool execution → second pass with original stream flag. Total: 2 Copilot calls + 1 Tavily call (unchanged from current behaviour). +- **`tool_choice: "none"`**: Copilot returns `finish_reason: "stop"` on first pass. Interceptor re-issues with original stream flag if streaming. Web search silently skipped. Correct behaviour. +- **All models have same or smaller context window**: `selectModelForTokenCount` returns `switched: false` — prompt sent as-is, Copilot may reject, but error is now forwarded properly. +- **Token count estimation fails**: try/catch swallows the error, original model is used, no request is broken. +- **Typed web_search tools from client** (e.g. `web_search_20250305`): already stripped by `translateAnthropicToolsToOpenAI` — no behaviour change. + +--- + +## Non-Goals + +- No changes to Brave/Tavily provider implementations +- No new CLI flags +- No changes to streaming translation (`stream-translation.ts`) diff --git a/docs/superpowers/specs/2026-03-17-upstream-error-forwarding-design.md b/docs/superpowers/specs/2026-03-17-upstream-error-forwarding-design.md new file mode 100644 index 000000000..b7e3b4962 --- /dev/null +++ b/docs/superpowers/specs/2026-03-17-upstream-error-forwarding-design.md @@ -0,0 +1,134 @@ +# Upstream Error Forwarding — Design Spec + +**Date:** 2026-03-17 +**Status:** Approved + +## Problem + +When Copilot returns an HTTP error (e.g. `model_max_prompt_tokens_exceeded`), the proxy: + +1. Logs `Response {}` in `create-chat-completions.ts` before the response body has been read — the `Response` object serializes to `{}`, producing useless output in the server log. +2. Re-wraps Copilot's JSON error body inside its own envelope in `forwardError`, producing double-wrapped stringified JSON: + ```json + { "error": { "message": "{\"error\":{\"message\":\"prompt token count...\",\"code\":\"model_max_prompt_tokens_exceeded\"}}", "type": "error" } } + ``` + The client receives a string where it expects an object. The `code` field is lost. The agent cannot programmatically detect or handle the error type. + +## Solution + +Two minimal changes, no new abstractions. + +### 1. `src/services/copilot/create-chat-completions.ts` + +Remove the premature log of the raw `Response` object before its body has been read. Body reading happens downstream in `forwardError`, so this log is both incorrect (logs `{}`) and redundant. + +**Before:** +```ts +consola.error("Failed to create chat completions", response) +throw new HTTPError("Failed to create chat completions", response) +``` + +**After:** +```ts +throw new HTTPError("Failed to create chat completions", response) +``` + +### 2. `src/lib/error.ts` + +The current `forwardError` function has two log calls and one response path for `HTTPError`: + +```ts +export async function forwardError(c: Context, error: unknown) { + consola.error("Error occurred:", error) // ← keep: logs the HTTPError object itself (not the unread Response) + + if (error instanceof HTTPError) { + const errorText = await error.response.text() + let errorJson: unknown + try { + errorJson = JSON.parse(errorText) + } catch { + errorJson = errorText // ← currently stores string; will change to null (see below) + } + consola.error("HTTP error:", errorJson) // ← keep, but update to handle null sentinel + return c.json( + { + error: { + message: errorText, + type: "error", + }, + }, + error.response.status as ContentfulStatusCode, + ) + } + // ... non-HTTPError path unchanged +} +``` + +The top-level `consola.error("Error occurred:", error)` logs the `HTTPError` object itself — not the unread `Response` — so it remains useful and is kept unchanged. + +**Change:** Forward Copilot's parsed JSON error object directly when it is a non-null object, preserving the upstream error shape and all fields (including `code`). Use `null` as the parse-failure sentinel (rather than the raw string) to cleanly distinguish "parsed successfully" from "parse failed". Fall back to the existing envelope shape for non-JSON responses and for the rare case where upstream returns a valid JSON primitive (number, boolean, string) — these are treated as unstructured and wrapped. + +**Before (HTTPError branch only):** +```ts +const errorText = await error.response.text() +let errorJson: unknown +try { + errorJson = JSON.parse(errorText) +} catch { + errorJson = errorText +} +consola.error("HTTP error:", errorJson) +return c.json( + { + error: { + message: errorText, + type: "error", + }, + }, + error.response.status as ContentfulStatusCode, +) +``` + +**After (HTTPError branch only):** +```ts +const errorText = await error.response.text() +let errorJson: unknown +try { + errorJson = JSON.parse(errorText) +} catch { + errorJson = null +} +consola.error("HTTP error:", errorJson ?? errorText) + +if (errorJson !== null && typeof errorJson === "object") { + return c.json(errorJson as Record<string, unknown>, error.response.status as ContentfulStatusCode) +} +return c.json( + { error: { message: errorText, type: "error" } }, + error.response.status as ContentfulStatusCode, +) +``` + +Note: the fallback envelope retains `type: "error"` (unchanged from before) to preserve the existing contract for non-JSON upstream errors. + +## Result + +When Copilot returns: +```json +{ "error": { "message": "prompt token count of 55059 exceeds the limit of 12288", "code": "model_max_prompt_tokens_exceeded" } } +``` + +The client now receives exactly that — structured JSON with the `code` field intact — enabling programmatic error handling in the agent. + +## Edge Cases + +- **Non-JSON upstream response** (e.g. plain text `"Bad Gateway"`): `JSON.parse` throws, sentinel is `null`, falls through to the envelope path — same behavior as before. +- **JSON primitive upstream response** (e.g. `true`, `42`): parses successfully but fails the `typeof === "object"` guard, falls through to the envelope path. This is intentional: JSON primitives are not valid API error responses and are treated as unstructured. +- **Non-`HTTPError` thrown** (e.g. network error, invariant failure): the non-HTTPError path in `forwardError` is unchanged — returns 500 with `error.message`. + +## Scope + +- 2 files modified (`create-chat-completions.ts`, `error.ts`) +- Net: ~1 line removed, ~5 lines added (~6 lines total touched) +- No new files, no new tests, no new abstractions +- Existing contract for non-JSON upstream errors is preserved (`type: "error"` unchanged) diff --git a/docs/superpowers/specs/2026-03-19-burst-rate-limit-design.md b/docs/superpowers/specs/2026-03-19-burst-rate-limit-design.md new file mode 100644 index 000000000..4101e7aac --- /dev/null +++ b/docs/superpowers/specs/2026-03-19-burst-rate-limit-design.md @@ -0,0 +1,172 @@ +# Burst Rate Limiting — Design Spec + +**Date:** 2026-03-19 +**Status:** Draft + +--- + +## Problem + +The existing rate limiter enforces a minimum gap in seconds between every single request (`--rate-limit`). This is too blunt: it penalises requests that are already naturally spaced apart (e.g. an LLM call that only fires after a preceding operation). GitHub Copilot's actual blocking behaviour is triggered by **flooding** — too many requests in a short window — not by any individual request's timing. + +--- + +## Goal + +Add a new, independent rate-limiting mode: **burst limiting**. Within a rolling time window of X seconds, allow at most N requests through freely. Any request that would exceed N waits until the oldest slot in the window ages out. Requests that arrive slowly and never saturate the window are never delayed. + +--- + +## Behaviour + +- Configured via two CLI flags (both required together): `--burst-count <N>` and `--burst-window <seconds>` +- When a new request arrives, `checkBurstLimit` is called: + 1. Compute `now = Date.now()` for this iteration. + 2. Prune all timestamps older than `now - windowMs` from `state.burstRequestTimestamps`. + 3. If `state.burstRequestTimestamps.length < burstCount` → push `now` synchronously (no `await` between check and push), return immediately. + 4. Otherwise → compute `waitMs = Math.max(0, state.burstRequestTimestamps[0] + windowMs - now)` using the **same `now` from step 1** (do not call `Date.now()` again within this iteration), sleep, then **loop back to step 1** with a fresh `now`. Do not push unconditionally after sleeping. +- The function **always waits** — it never throws a 429. There is no separate `--wait` flag for burst mode. +- Both burst limiting and the existing per-request gap limiter (`--rate-limit`) can be active simultaneously. Burst is checked first. +- `checkBurstLimit` is applied to: + - `src/routes/chat-completions/handler.ts` + - `src/routes/messages/handler.ts` (`POST /v1/messages` — actual completions) +- `checkBurstLimit` is **not** applied to: + - `src/routes/messages/count-tokens-handler.ts` (`POST /v1/messages/count_tokens` — no upstream Copilot completion call) + - `src/routes/embeddings/handler.ts` +- `checkBurstLimit` assumes pre-validated state (both fields defined and in range). It must not be called unless both `burstCount` and `burstWindowSeconds` are set and valid. + +--- + +## Concurrency & Ordering + +`state.burstRequestTimestamps` is a shared mutable array. Because Bun's event loop is single-threaded, two requests cannot execute JavaScript simultaneously — there is no true data race. However, two `async` handlers can interleave at `await` points. The check-and-push in step 3 is **synchronous with no `await` between them**, which prevents two concurrent handlers from both passing the length check before either records a timestamp. + +`state.burstRequestTimestamps` is always maintained in **insertion order** (oldest first). Entries are only ever appended via `push`; old entries are pruned via `filter` (which preserves order). This guarantees `state.burstRequestTimestamps[0]` is always the oldest in-window entry after pruning. + +--- + +## State Changes (`src/lib/state.ts`) + +New fields added to the `State` interface: + +| Field | Type | Default | Purpose | +|---|---|---|---| +| `burstCount` | `number \| undefined` | `undefined` | Max requests allowed within the window | +| `burstWindowSeconds` | `number \| undefined` | `undefined` | Rolling window duration in seconds | +| `burstRequestTimestamps` | `number[]` | `[]` | Timestamps (ms, from `Date.now()`) of in-window requests, oldest first | + +All three fields must be initialised in the `state` object literal: `burstCount: undefined`, `burstWindowSeconds: undefined`, `burstRequestTimestamps: []`. + +--- + +## CLI Changes (`src/start.ts`) + +Two new `citty` args added to the `start` command (type `"string"`, consistent with the existing `rate-limit` arg): + +| Flag | Description | +|---|---| +| `--burst-count` | Max requests within the window (positive integer ≥ 1) | +| `--burst-window` | Window duration in seconds (positive number > 0) | + +Two new fields added to the `RunServerOptions` interface as **parsed numbers** (matching the existing `rateLimit?: number` pattern — raw CLI strings are parsed before being passed into `runServer`): + +```typescript +burstCount?: number // parsed from --burst-count +burstWindowSeconds?: number // parsed from --burst-window +``` + +**Validation** (applied in the `run()` callback of the `start` command, before calling `runServer`, in this order): + +1. If both flags are absent → pass `burstCount: undefined, burstWindowSeconds: undefined` to `runServer`. No warning. +2. If exactly one flag is present → `consola.warn("Burst limiting disabled: --burst-count and --burst-window must both be provided (missing: --<flag-name>)")`, pass both as `undefined`. +3. If both flags are present: parse `--burst-count` with `Number(raw)` and check `Number.isInteger(parsed) && parsed >= 1`. Using `Number()` rather than `parseInt` is intentional here: it catches non-integer floats like `"2.5"` which `parseInt` would silently truncate to `2`. If invalid → `consola.error("--burst-count must be a positive integer (got: <value>)")` and `process.exit(1)`. +4. If both flags are present and count is valid: parse `--burst-window` with `Number(raw)` and check `parsed > 0`. If invalid → `consola.error("--burst-window must be a positive number greater than 0 (got: <value>)")` and `process.exit(1)`. + +Rules 3 and 4 are only reached when both flags are present. The process exits immediately on the first parse failure — if rule 3 fails, rule 4 is not evaluated and `--burst-window` is not validated. + +**State assignment** (in `runServer`, alongside existing state assignments): + +```typescript +state.burstCount = options.burstCount +state.burstWindowSeconds = options.burstWindowSeconds +``` + +--- + +## Rate Limit Logic (`src/lib/rate-limit.ts`) + +`sleep` is imported from `"./utils"` (already present in this file). `consola` is already imported in this file. No new imports are needed. + +New exported function: + +```typescript +export async function checkBurstLimit(state: State): Promise<void> +``` + +Algorithm: + +``` +1. If state.burstCount is undefined OR state.burstWindowSeconds is undefined → return + +2. const windowMs = state.burstWindowSeconds * 1000 + +3. Loop forever: + a. const now = Date.now() + b. state.burstRequestTimestamps = state.burstRequestTimestamps.filter( + ts => ts > now - windowMs + ) + c. If state.burstRequestTimestamps.length < state.burstCount: + state.burstRequestTimestamps.push(now) // synchronous: no await between check and push + return + d. Else: + const waitMs = Math.max( + 0, + state.burstRequestTimestamps[0] + windowMs - now // use same `now` from step 3a + ) + const waitSeconds = Math.ceil(waitMs / 1000) + consola.warn(`Burst limit reached. Waiting ${waitSeconds}s before proceeding...`) + await sleep(waitMs) + consola.debug("Burst limit wait completed, re-checking...") + // Note: consola.debug is intentional here (not consola.info) — the loop + // has not yet confirmed a slot is free, so this is an intermediate state, + // not a "proceeding with request" event. + continue // back to step 3a — now gets a fresh Date.now() +``` + +The existing `checkRateLimit` function is **not modified**. + +--- + +## Integration Points + +`checkBurstLimit(state)` is called as the **first** thing in: + +- `src/routes/chat-completions/handler.ts` — before `checkRateLimit(state)`. Add `checkBurstLimit` to the existing `~/lib/rate-limit` import. +- `src/routes/messages/handler.ts` — before `checkRateLimit(state)`. Add `checkBurstLimit` to the existing `~/lib/rate-limit` import. + +Not called in: +- `src/routes/messages/count-tokens-handler.ts` +- `src/routes/embeddings/handler.ts` + +No changes to routing, middleware, or any other files. + +--- + +## Files Changed + +| File | Change | +|---|---| +| `src/lib/state.ts` | Add `burstCount`, `burstWindowSeconds`, `burstRequestTimestamps` to `State` interface and `state` object | +| `src/lib/rate-limit.ts` | Add `checkBurstLimit` function (no new imports needed) | +| `src/start.ts` | Add `--burst-count` and `--burst-window` CLI args; add `burstCount` and `burstWindowSeconds` to `RunServerOptions`; add validation in `run()` callback; add state assignment in `runServer` | +| `src/routes/chat-completions/handler.ts` | Call `checkBurstLimit(state)` before `checkRateLimit(state)` | +| `src/routes/messages/handler.ts` | Call `checkBurstLimit(state)` before `checkRateLimit(state)` | + +--- + +## Non-Goals + +- No `--burst-wait` flag (always waits, never errors) +- No changes to the existing `--rate-limit` / `--wait` behaviour +- No changes to embeddings, count_tokens, or any other endpoints +- No UI or config-file support (CLI flags only, consistent with the rest of the project) diff --git a/docs/superpowers/specs/2026-03-23-413-image-stripping-retry-design.md b/docs/superpowers/specs/2026-03-23-413-image-stripping-retry-design.md new file mode 100644 index 000000000..3449c2179 --- /dev/null +++ b/docs/superpowers/specs/2026-03-23-413-image-stripping-retry-design.md @@ -0,0 +1,164 @@ +# 413 Image Stripping with Progressive Retry + +## Problem + +When Claude Code sends requests containing base64-encoded screenshots/images, the total request body can exceed GitHub Copilot's size limit. Copilot responds with `413 Request Entity Too Large`. The proxy currently surfaces this as an `api_error` SSE event, which causes Claude Code to stop working entirely rather than compacting and continuing. This is unacceptable for long-running unattended tasks (e.g., executing a large plan file overnight). + +## Solution + +Intercept 413 errors inside the proxy and progressively strip images from the conversation history before retrying. If stripping all images still isn't enough, trigger Claude Code's auto-compaction as a final fallback. The model never stops — it either gets a successful response with reduced image context, or compacts and continues. + +## Translation Boundary + +Image stripping operates on the **Anthropic-format payload** (`AnthropicMessagesPayload`) *before* it is passed to `fetchCopilotResponse`. This is correct because `fetchCopilotResponse` internally calls `translateToOpenAI()` which converts `AnthropicImageBlock` (with `source.data` base64) into OpenAI `image_url` content parts. By stripping images from the Anthropic payload before translation, the translated OpenAI payload will naturally be smaller. + +## Retry Cascade + +``` +Stage 1: Send original request + └─ 413? → Are there 2+ base64 images? + ├─ yes → Stage 2: Strip older base64 images, keep last, retry + │ └─ 413? → Stage 3: Strip ALL base64 images, retry + │ └─ 413? → Stage 4: throw CompactionNeededError + └─ no (0-1 images) → Stage 3: Strip ALL base64 images (if any), retry + └─ 413? → Stage 4: throw CompactionNeededError +``` + +Only 413 errors trigger this cascade. All other HTTP errors (401, 429, 500, etc.) propagate immediately as before. + +## Image Stripping Logic + +### Where images appear + +1. **User message content arrays** — `AnthropicImageBlock` items with `type: "image"` +2. **Tool result content arrays** — `AnthropicToolResultBlock.content` can contain `AnthropicImageBlock` items + +Document blocks are already replaced with placeholder text by the translation layer and are not a concern. + +### Which images to strip + +Only `base64`-sourced images (`source.type === "base64"`) are stripped. These contain the actual image data inline and are the cause of oversized requests. URL-sourced images (if they exist in future payloads) are tiny references and should not be stripped. + +### Definition of "most recent image" + +"Most recent" means the **last base64 image block encountered** when walking the `messages` array in order (index 0 → last). Within each message, content blocks are walked in array order. Tool result content arrays nested inside user messages are walked inline at their position within the parent message's content array. This produces a single flat ordering of all base64 image blocks across the entire conversation. + +### Algorithm: `stripImages(payload, keepLast)` + +1. Deep-clone the payload to avoid mutating the original +2. Walk all messages in array order. For each user message, walk its content array in order. When encountering a `tool_result` block with an array content, walk that nested array inline. Collect references to every block where `type === "image"` and `source.type === "base64"`. +3. If `keepLast` is true and more than one image was found, exclude the last collected image from the removal list. If only one image exists, it is kept (nothing to strip). +4. Replace each targeted image block with `{ type: "text", text: "[Image removed to reduce request size]" }` +5. Return `{ payload, strippedCount }` — the count of images actually replaced + +### Cascade short-circuit logic + +`fetchWithImageStripping` uses `strippedCount` to skip unnecessary stages: +- If Stage 2 `strippedCount === 0` (0 or 1 images total, nothing stripped with `keepLast: true`), skip directly to Stage 3 with the **original** payload +- If Stage 3 `strippedCount === 0` (no base64 images at all), skip directly to Stage 4 (throw `CompactionNeededError`) +- Stage 3 always operates on the **original** payload with `keepLast: false`, not on the Stage 2 result. This is simpler and avoids edge cases around residual state from Stage 2. + +### Placement + +New file: `src/routes/messages/image-stripping.ts` + +Contains: +- `stripImages(payload, keepLast)` — the image removal utility +- `fetchWithImageStripping(fetchFn, anthropicPayload)` — the 413 retry cascade wrapper +- `CompactionNeededError` — custom error class for the final fallback stage + +## Dependency Structure + +To avoid circular imports between `handler.ts` and `image-stripping.ts`, `fetchWithImageStripping` accepts `fetchCopilotResponse` **as a parameter** rather than importing it: + +```typescript +async function fetchWithImageStripping( + fetchFn: (payload: AnthropicMessagesPayload) => ReturnType<typeof fetchCopilotResponse>, + anthropicPayload: AnthropicMessagesPayload, +): ReturnType<typeof fetchCopilotResponse> +``` + +`handler.ts` calls `fetchWithImageStripping(fetchCopilotResponse, anthropicPayload)`. No exports from `handler.ts` are needed. The dependency flows one way: `handler.ts` → `image-stripping.ts`. + +## Integration into Handler + +### Non-streaming path (`handleNonStreaming`) + +- Replace `fetchCopilotResponse(anthropicPayload)` with `fetchWithImageStripping(fetchCopilotResponse, anthropicPayload)` +- Add a **separate** catch clause for `CompactionNeededError` before the existing `HTTPError` re-throw: + ```typescript + try { + response = await fetchWithImageStripping(fetchCopilotResponse, anthropicPayload) + } catch (error) { + if (error instanceof CompactionNeededError) { + return c.json({ + type: "error", + error: { + type: "invalid_request_error", + message: "Request too large. Conversation context exceeds model limit.", + }, + }, 413) + } + if (error instanceof HTTPError) throw error + // ... existing network error handling + } + ``` + +### Streaming path (`handleStreaming`) + +The existing network retry loop in `handleStreaming` immediately re-throws `HTTPError` instances (including 413). `fetchWithImageStripping` catches 413 `HTTPError`s internally before they reach this re-throw. The integration: + +- Replace `fetchCopilotResponse(anthropicPayload)` inside the network retry loop with `fetchWithImageStripping(fetchCopilotResponse, anthropicPayload)` +- Inside `fetchWithImageStripping`, a 413 `HTTPError` is caught and triggers the image-stripping cascade. Only non-413 `HTTPError`s are re-thrown (reaching the existing `if (error instanceof HTTPError) throw error` in the retry loop) +- If the cascade is exhausted, `fetchWithImageStripping` throws `CompactionNeededError` (not an `HTTPError`), which propagates out of the retry loop to the outer catch +- Add `CompactionNeededError` handling in the outer catch block: + ```typescript + } catch (error) { + // ... existing cleanup ... + if (error instanceof CompactionNeededError) { + const errorEvent = translateErrorToAnthropicErrorEvent( + "Request too large. Conversation context exceeds model limit.", + "invalid_request_error", + ) + await stream.writeSSE({ event: errorEvent.type, data: JSON.stringify(errorEvent) }) + return + } + // ... existing HTTPError / generic error handling ... + } + ``` + +### Why `invalid_request_error` is safe in the final fallback + +The existing code deliberately avoids `invalid_request_error` to prevent retry loops where Claude Code auto-compacts and retries, each retry adding more context. The concern is valid for *general* errors, but the final fallback here is safe because: + +1. By the time we reach Stage 4, all base64 images have been stripped. The request is pure text. +2. Claude Code's compaction reduces the conversation text (summarizes older messages). The compacted request will be smaller. +3. After compaction, the images that caused the original 413 are gone from the conversation history — Claude Code does not re-attach previously removed images. The compacted request will not re-introduce them. +4. If compaction produces a request that is *still* too large, Claude Code will compact again (further reducing text). This converges because text gets shorter with each compaction round. + +This is fundamentally different from the scenario the existing comments warn about, where the error itself causes Claude Code to add retry metadata that inflates the prompt. + +## Logging + +Each stage logs via `consola.warn`: +- `"Request too large (413), retrying with older images stripped (keeping last image)"` +- `"Still too large (413), retrying with all images stripped"` +- `"Still too large (413) even without images, triggering auto-compaction"` + +## Files Changed + +### New +- `src/routes/messages/image-stripping.ts` (~100-120 lines) + +### Modified +- `src/routes/messages/handler.ts`: + - Import `fetchWithImageStripping` and `CompactionNeededError` from `image-stripping.ts` + - `handleNonStreaming`: swap fetch call, add `CompactionNeededError` catch before `HTTPError` re-throw + - `handleStreaming`: swap fetch call inside retry loop, add `CompactionNeededError` catch in outer catch block + +### Unchanged +- `src/lib/error.ts` +- `src/routes/messages/non-stream-translation.ts` +- `src/routes/messages/stream-translation.ts` — `translateErrorToAnthropicErrorEvent(message, errorType = "api_error")` already accepts an optional second argument; the `CompactionNeededError` handler passes `"invalid_request_error"` as the second argument, requiring no signature change +- `src/routes/messages/count-tokens-handler.ts` +- `src/services/copilot/create-chat-completions.ts` diff --git a/docs/superpowers/specs/2026-04-14-full-anthropic-api-parity-design.md b/docs/superpowers/specs/2026-04-14-full-anthropic-api-parity-design.md new file mode 100644 index 000000000..ac89491fd --- /dev/null +++ b/docs/superpowers/specs/2026-04-14-full-anthropic-api-parity-design.md @@ -0,0 +1,549 @@ +# Full Anthropic API Parity — Design Spec + +**Date:** 2026-04-14 +**Status:** Draft (v3 — revised after two critical reviews) + +## Problem + +The copilot-api proxy's Anthropic translation layer supports core API features (text, image, tool_use, tool_result, thinking, basic streaming) but is missing many newer Anthropic API features introduced through 2025-2026. Claude Code v2.1.107+ sends requests that may include these newer features, causing potential failures or silent data loss when the proxy encounters unknown block types or request fields. + +## Goal + +Achieve maximum compatibility with the full Anthropic Messages API as documented, so the proxy can transparently handle any request from Claude Code or other Anthropic API clients without errors or data loss. + +## Approach + +Incremental extension of the existing architecture. The proxy already has a clean pattern: types in `anthropic-types.ts`, translation in `non-stream-translation.ts` and `stream-translation.ts`. We extend these files to cover all missing features. + +**Key constraint**: The proxy translates Anthropic format → OpenAI format → Copilot. Many Anthropic-specific features (server tools, citations, cache_control) have no OpenAI equivalent. For these, the strategy is: **accept in types, serialize to text in translation, never error on unknown**. + +## Gap Analysis + +### Missing Request Payload Fields + +| Field | Type | Notes | +|-------|------|-------| +| `output_config` | `{ effort?, format? }` | Reasoning effort + JSON schema output | +| `speed` | `"standard" \| "fast"` | Inference speed mode | +| `cache_control` | `{ type: "ephemeral", ttl? }` | Top-level cache control | +| `container` | `{ skills? }` | Persistent sandboxed environments | +| `mcp_servers` | `Array<MCPServerConfig>` | MCP server connections | +| `context_management` | `object` | Cross-request context control | +| `inference_geo` | `string` | Geographic region selection | + +### Missing/Incomplete Content Block Fields + +| Block | Missing Field | Notes | +|-------|--------------|-------| +| `AnthropicImageBlock` | `source.type: "url"` | Only `base64` is typed; API also supports URL sources | +| `AnthropicTextBlock` | `citations` | Text blocks can carry citation arrays | +| `AnthropicToolUseBlock` | `cache_control`, `caller` | `caller` indicates which tool initiated the call | +| `AnthropicCustomTool` | `allowed_callers` | Restricts which callers can invoke a tool | + +### Missing User Content Block Types + +| Block Type | Description | +|------------|-------------| +| `search_result` | Grounding with search results (`content`, `source`, `title`) | +| `container_upload` | Sandbox file uploads (`file_id`) | + +### Missing Assistant Content Block Types (multi-turn pass-through) + +These appear in conversation history when clients replay Anthropic API responses. They can also appear in **user messages** when clients forward prior assistant results. + +| Block Type | Description | +|------------|-------------| +| `web_fetch_tool_result` | Web fetch server tool results | +| `code_execution_tool_result` | Python code execution results | +| `bash_code_execution_tool_result` | Bash code execution results | +| `text_editor_code_execution_tool_result` | Text editor execution results | +| `tool_search_tool_result` | Tool search BM25/regex results | + +### Missing Streaming Features + +| Feature | Description | +|---------|-------------| +| `citations_delta` | Citation appended to current text block | +| `compaction_delta` | Compacted content delta | +| `server_tool_use` in content_block_start | Streaming server tool blocks | +| Fully-formed result blocks in content_block_start | Server tool results arrive complete | + +### Stop Reasons — Already Present + +`pause_turn` and `refusal` are already in `AnthropicResponse.stop_reason`. `mapOpenAIStopReasonToAnthropic()` maps `content_filter` → `"end_turn"` — intentional and adequate. **No changes needed.** + +### Missing Server Tool Types + +| Tool Type | Name | +|-----------|------| +| `web_fetch_*` | `web_fetch` | +| `code_execution_*` | `code_execution` | +| `advisor_20260301` | advisor | +| `tool_search_tool_*` | tool search | +| `mcp_toolset` | MCP connector | + +### Beta Headers + +The proxy does not need to interpret `anthropic-beta` headers — it just needs to not break when they're present. Typed tools carry their version in the `type` field. + +--- + +## Detailed Design + +### 1. Type Additions (`anthropic-types.ts`) + +#### 1.1 Request Payload + +Extend `AnthropicMessagesPayload` with new fields. All are **stripped** during translation — see §2.1. + +```typescript +export interface AnthropicMessagesPayload { + // ... existing fields ... + output_config?: { + effort?: "low" | "medium" | "high" | "max" + format?: { type: "json_schema"; schema: Record<string, unknown> } + } + speed?: "standard" | "fast" + cache_control?: { type: "ephemeral"; ttl?: number } // number matches existing codebase + container?: Record<string, unknown> + mcp_servers?: Array<Record<string, unknown>> + context_management?: Record<string, unknown> + inference_geo?: string +} +``` + +#### 1.2 Existing Type Fixes + +**`AnthropicImageBlock`** — add URL source variant: +```typescript +export interface AnthropicImageBlock { + type: "image" + source: + | { type: "base64"; media_type: "image/jpeg" | "image/png" | "image/gif" | "image/webp"; data: string } + | { type: "url"; url: string } +} +``` + +> **BREAKING CHANGE**: This makes `source.data` conditional. Files that access `source.data` without checking `source.type === "base64"` will get TypeScript errors. Known impact sites: +> - `attachment-overhead.ts` `estimateImageTokens()` — accesses `block.source.data.length` +> - `image-stripping.ts` `collectToolResultImages()` — accesses `nested.source.data.length` +> - `image-stripping.ts` `collectImageRefs()` — accesses `block.source.data.length` +> - `image-validation.ts` `getInvalidImageReason()` — accesses `block.source.data` +> +> **Fix for all**: Add `if (block.source.type !== "base64") return ...` guard before accessing `.data`. URL-based images have no base64 data to measure/strip/validate, so early-return is correct. + +**`AnthropicTextBlock`** — add optional citations: +```typescript +export interface AnthropicTextBlock { + type: "text" + text: string + cache_control?: { type: "ephemeral"; ttl?: number } + citations?: Array<unknown> // pass-through, not interpreted by proxy +} +``` + +**`AnthropicToolUseBlock`** — add cache_control and caller: +```typescript +export interface AnthropicToolUseBlock { + type: "tool_use" + id: string + name: string + input: Record<string, unknown> + cache_control?: { type: "ephemeral"; ttl?: number } + caller?: Record<string, unknown> // pass-through +} +``` + +**`AnthropicCustomTool`** — add allowed_callers: +```typescript +export interface AnthropicCustomTool { + // ... existing fields ... + allowed_callers?: Array<string> +} +``` + +#### 1.3 New User Content Blocks + +```typescript +export interface AnthropicSearchResultBlock { + type: "search_result" + source: string // URL + title: string + content: string // text content of the search result + cache_control?: { type: "ephemeral"; ttl?: number } + citations?: Array<unknown> + search_result_index?: number + start_block_index?: number + end_block_index?: number +} + +export interface AnthropicContainerUploadBlock { + type: "container_upload" + file_id: string + cache_control?: { type: "ephemeral"; ttl?: number } +} +``` + +#### 1.4 New Server Tool Result Blocks + +These share a common shape. They can appear in **both** user and assistant messages. + +```typescript +interface ServerToolResultBase { + tool_use_id: string + content: unknown + cache_control?: { type: "ephemeral"; ttl?: number } +} + +export interface AnthropicWebFetchToolResultBlock extends ServerToolResultBase { + type: "web_fetch_tool_result" +} +export interface AnthropicCodeExecutionToolResultBlock extends ServerToolResultBase { + type: "code_execution_tool_result" +} +export interface AnthropicBashCodeExecutionToolResultBlock extends ServerToolResultBase { + type: "bash_code_execution_tool_result" +} +export interface AnthropicTextEditorCodeExecutionToolResultBlock extends ServerToolResultBase { + type: "text_editor_code_execution_tool_result" +} +export interface AnthropicToolSearchToolResultBlock extends ServerToolResultBase { + type: "tool_search_tool_result" +} +``` + +#### 1.5 Updated Union Types + +The helper union **must be exported** so `non-stream-translation.ts` can import it for type narrowing. + +```typescript +export type AnthropicServerToolResultBlock = + | AnthropicWebSearchToolResultBlock + | AnthropicWebFetchToolResultBlock + | AnthropicCodeExecutionToolResultBlock + | AnthropicBashCodeExecutionToolResultBlock + | AnthropicTextEditorCodeExecutionToolResultBlock + | AnthropicToolSearchToolResultBlock + +export type AnthropicUserContentBlock = + | AnthropicTextBlock + | AnthropicImageBlock + | AnthropicDocumentBlock + | AnthropicToolResultBlock + | AnthropicSearchResultBlock + | AnthropicContainerUploadBlock + | AnthropicServerToolResultBlock + +export type AnthropicAssistantContentBlock = + | AnthropicTextBlock + | AnthropicToolUseBlock + | AnthropicThinkingBlock + | AnthropicRedactedThinkingBlock + | AnthropicServerToolUseBlock + | AnthropicServerToolResultBlock +``` + +#### 1.6 Stop Reason — Already Complete + +No changes needed. + +#### 1.7 Updated Streaming Types + +Add to `AnthropicContentBlockDeltaEvent.delta`: +```typescript + | { type: "citations_delta"; citation: unknown } + | { type: "compaction_delta"; content: unknown } +``` + +Add to `AnthropicContentBlockStartEvent.content_block`: +```typescript + | { type: "server_tool_use"; id: string; name: string; input: Record<string, unknown> } + | AnthropicServerToolResultBlock // fully-formed, no subsequent deltas +``` + +These are **type-only changes** in `anthropic-types.ts`. No behavioral changes in `stream-translation.ts`. + +### 2. Request Translation (`non-stream-translation.ts`) + +#### 2.1 Payload Translation — `translateToOpenAI()` + +All new Anthropic-only fields have **no OpenAI equivalent**. `translateToOpenAI()` selectively picks known fields, so unknown fields are automatically excluded — **no code change needed** in this function. + +#### 2.2 `handleUserMessage()` — New Block Routing + +**Current routing** (3 groups): +1. `tool_result` blocks → extracted, sent as OpenAI `tool` messages +2. `web_search_tool_result` blocks → extracted, serialized as user text +3. Everything else → `otherBlocks` → `mapContent()` + +**Change**: Replace group 2 with a broader server tool result filter. **Three code locations must change together**: + +```typescript +// NEW: helper function (add at module level) +function isServerToolResultBlock( + block: AnthropicUserContentBlock, +): block is AnthropicServerToolResultBlock { + // Matches web_search_tool_result, web_fetch_tool_result, code_execution_tool_result, etc. + // Excludes plain "tool_result" (which has its own handler). + return block.type.endsWith("_tool_result") && block.type !== "tool_result" +} + +// CHANGE 1: Replace webSearchResultBlocks filter +const serverToolResultBlocks = message.content.filter(isServerToolResultBlock) + +// CHANGE 2: Update otherBlocks filter to exclude ALL server tool results +const otherBlocks = message.content.filter( + (block) => block.type !== "tool_result" && !isServerToolResultBlock(block), +) + +// CHANGE 3: Update serialization (replaces current webSearchResultBlocks handler) +if (serverToolResultBlocks.length > 0) { + const text = serverToolResultBlocks + .map((b) => `[${b.type}: ${JSON.stringify(b.content)}]`) + .join("\n\n") + newMessages.push({ role: "user", content: text }) +} +``` + +The `isServerToolResultBlock` helper returns a **type guard** (`is AnthropicServerToolResultBlock`), so `b.content` is type-safe inside the map. + +#### 2.3 `handleAssistantMessage()` — Two Branches Need Updating + +**Branch 1** (has tool_use blocks): Currently extracts `textBlocks`, `toolUseBlocks`, `serverToolUseBlocks`. New `*_tool_result` blocks are silently dropped. + +**Fix**: Add server tool result extraction with type guard: +```typescript +const serverToolResultBlocks = message.content.filter( + (block): block is AnthropicServerToolResultBlock => + block.type.endsWith("_tool_result") && block.type !== "tool_result", +) + +const allTextContent = [ + ...textBlocks.map((b) => b.text), + ...serverToolUseBlocks.map((b) => `[Server tool use: ${JSON.stringify(b)}]`), + ...serverToolResultBlocks.map((b) => `[${b.type}: ${JSON.stringify(b.content)}]`), +] + .filter(Boolean) + .join("\n\n") +``` + +Note: The type guard narrows to `AnthropicServerToolResultBlock` so `b.content` is typed. + +**Branch 2** (no tool_use blocks): Passes `visibleBlocks` through `mapContent()`. The `visibleBlocks` filter already strips `thinking` and `redacted_thinking`. New block types pass through to `mapContent()` — see §2.4. + +#### 2.4 `mapContent()` — Rewrite Both Paths + +`mapContent()` has two code paths that BOTH silently drop unknown block types. + +**Path A (no images)** — Current code uses a chained `.filter().map().join()` with explicit type checks. Adding new block types to this chain creates type-narrowing issues because the filter predicate doesn't narrow the type. + +**Recommended fix**: Rewrite Path A as a for-loop with explicit type handling, matching Path B's pattern: + +```typescript +if (!hasImage) { + const parts: string[] = [] + for (const block of content) { + switch (block.type) { + case "text": + parts.push(block.text) + break + case "document": + parts.push("[Document: PDF content not displayable]") + break + case "server_tool_use": + parts.push(`[Server tool use: ${JSON.stringify(block)}]`) + break + case "search_result": + parts.push(`[Search: ${(block as AnthropicSearchResultBlock).title}]\nSource: ${(block as AnthropicSearchResultBlock).source}\n${(block as AnthropicSearchResultBlock).content}`) + break + case "container_upload": + parts.push(`[Container upload: ${(block as AnthropicContainerUploadBlock).file_id}]`) + break + default: + // Catch-all: server tool results and future unknown types + if ("content" in block && block.type !== "thinking" && block.type !== "redacted_thinking") { + parts.push(`[${block.type}: ${JSON.stringify((block as { content: unknown }).content)}]`) + } + break + } + } + return parts.filter(Boolean).join("\n\n") +} +``` + +**Path B (has images)** — Uses a `switch` statement. Add new cases: +```typescript +case "search_result": { + const sr = block as AnthropicSearchResultBlock + contentParts.push({ + type: "text", + text: `[Search: ${sr.title}]\nSource: ${sr.source}\n${sr.content}`, + }) + break +} +case "container_upload": { + contentParts.push({ + type: "text", + text: `[Container upload: ${(block as AnthropicContainerUploadBlock).file_id}]`, + }) + break +} +default: { + // Catch-all for server tool results and future unknown types + if ("content" in block && block.type !== "thinking" && block.type !== "redacted_thinking") { + contentParts.push({ + type: "text", + text: `[${block.type}: ${JSON.stringify((block as { content: unknown }).content)}]`, + }) + } +} +``` + +**Why type casts**: `mapContent()` receives `Array<AnthropicUserContentBlock | AnthropicAssistantContentBlock>`. TypeScript's switch narrowing works on discriminated unions, but the union is large and some cases need explicit casts to access block-specific fields like `title`, `source`, `file_id`. This is consistent with existing code (e.g., the `server_tool_use` case stringifies the whole block). + +### 3. Token Counting (`count-tokens-handler.ts`) + +Add estimated token overheads for new server tool types. The existing code has a comment: `// computer_use and web_search: overhead included in beta pricing, not additive`. Respect this — do **not** add entries for `computer_*` or `web_search_*`. Only add entries for tool types NOT mentioned in the comment. + +```typescript +const ANTHROPIC_TYPED_TOOL_TOKEN_OVERHEAD: Record<string, number> = { + // --- existing (from Anthropic pricing docs) --- + text_editor_20250728: 700, + text_editor_20250429: 700, + text_editor_20250124: 700, + text_editor_20241022: 700, + bash_20250124: 700, + bash_20241022: 700, + // computer_use and web_search: overhead included in beta pricing, not additive + // --- new (estimates for compaction heuristic, not billing) --- + web_fetch_20250910: 500, + web_fetch_20260209: 500, + web_fetch_20260309: 500, + code_execution_20250522: 500, + code_execution_20250825: 500, + code_execution_20260120: 500, + advisor_20260301: 500, + tool_search_tool_bm25_20251119: 200, + tool_search_tool_regex_20251119: 200, + tool_search_tool_bm25: 200, + tool_search_tool_regex: 200, + mcp_toolset: 300, +} +``` + +### 4. Image-Related Files — Type Guard Fixes + +Adding the URL source variant to `AnthropicImageBlock` (§1.2) causes compile errors in files that access `source.data` without checking `source.type`. All fixes follow the same pattern: add an early-return guard. + +**`attachment-overhead.ts` — `estimateImageTokens()`**: +```typescript +function estimateImageTokens(block: AnthropicImageBlock): number { + if (block.source.type !== "base64") return MIN_IMAGE_TOKENS // URL images: use minimum + return Math.max(MIN_IMAGE_TOKENS, Math.ceil(block.source.data.length / IMAGE_BASE64_CHARS_PER_TOKEN)) +} +``` + +**`image-stripping.ts` — `collectToolResultImages()` and `collectImageRefs()`**: +Add `if (nested.source.type !== "base64") continue` before accessing `nested.source.data.length`. Same for `block.source.data.length` in the main loop. URL-based images have no base64 data to strip. + +**`image-validation.ts` — `getInvalidImageReason()`**: +```typescript +if (block.source.type !== "base64") return undefined // URL images: can't validate dimensions +``` + +### 5. Attachment Overhead (`attachment-overhead.ts`) + +Add token estimates for new content block types by extending `estimateAdditionalAttachmentTokens()`: + +```typescript +// Add new helper functions: +function estimateSearchResultTokens(block: AnthropicSearchResultBlock): number { + return Math.max(500, Math.ceil(block.content.length / 4)) +} + +// Add to the message iteration loop inside estimateAdditionalAttachmentTokens(): +for (const message of payload.messages) { + if (message.role !== "user") continue + // ... existing document and image loops ... + + // NEW: search_result blocks + for (const block of getBlocksOfType(message.content, "search_result")) { + tokens += estimateSearchResultTokens(block as AnthropicSearchResultBlock) + } + // NEW: container_upload blocks (fixed overhead — just a file ID reference) + for (const block of getBlocksOfType(message.content, "container_upload")) { + tokens += 100 + } +} +``` + +The `getBlocksOfType` helper follows the same pattern as existing `getDocumentBlocksFromContent` / `getImageBlocksFromContent`. + +### 6. Web Search Detection — NO CHANGE + +Do NOT add `web_fetch` to `WEB_SEARCH_TOOL_NAMES`. Web fetch and web search are different tools — adding it would incorrectly route web fetch requests through the Brave Search interceptor pipeline. + +### 7. Handler / Utils / Stream Translation — NO CHANGES + +- `handler.ts`: Unchanged. `translateToOpenAI()` selectively picks known fields. +- `utils.ts`: Unchanged. Stop reason mapping already correct. +- `stream-translation.ts`: Unchanged. New streaming types are defined in `anthropic-types.ts`. + +--- + +## Files Changed + +| File | Changes | +|------|---------| +| `src/routes/messages/anthropic-types.ts` | Add new block types, fix existing types (image URL source, text citations, tool_use cache_control/caller), export `AnthropicServerToolResultBlock` union, extend content block and streaming type unions | +| `src/routes/messages/non-stream-translation.ts` | Add `isServerToolResultBlock()` helper; update `handleUserMessage()` (3 changes: filter, otherBlocks exclusion, serialization); update `handleAssistantMessage()` Branch 1 to include server tool results; rewrite `mapContent()` Path A as for-loop, add cases + default catch-all to Path B | +| `src/routes/messages/count-tokens-handler.ts` | Add typed-tool token overhead estimates for new server tool types (excluding computer/web_search per existing comment) | +| `src/routes/messages/attachment-overhead.ts` | Fix `estimateImageTokens()` for URL-based images; add `search_result` and `container_upload` token estimates | +| `src/routes/messages/image-stripping.ts` | Add `source.type === "base64"` guards before accessing `.data` in `collectToolResultImages()` and `collectImageRefs()` | +| `src/routes/messages/image-validation.ts` | Add `source.type === "base64"` guard in `getInvalidImageReason()` | +| `src/routes/messages/handler.ts` | **No changes** | +| `src/routes/messages/utils.ts` | **No changes** | +| `src/routes/messages/stream-translation.ts` | **No changes** (streaming types are in anthropic-types.ts) | +| `src/routes/messages/web-search-detection.ts` | **No changes** | + +## Testing Strategy + +### Unit Tests (`tests/anthropic-request.test.ts`) + +Add tests following the existing patterns: + +1. **`search_result` block in user message** — verify `mapContent()` produces `[Search: {title}]\nSource: {source}\n{content}` +2. **`container_upload` block in user message** — verify produces `[Container upload: {file_id}]` +3. **`web_fetch_tool_result` block in user message** — verify serialized like existing `web_search_tool_result` test +4. **`code_execution_tool_result` in assistant message (Branch 1, with tool calls)** — verify it appears in `allTextContent` +5. **`web_fetch_tool_result` in assistant message (Branch 2, no tool calls)** — verify serialized through `mapContent()` default catch-all +6. **New payload fields** (`output_config`, `speed`, etc.) — verify `translateToOpenAI()` doesn't include them in output +7. **URL-based image source in user message** — verify `mapContent()` handles without crash +8. **`mapContent()` with image + search_result mix** — verify the image path's default case serializes the search_result +9. **`isServerToolResultBlock` matches `web_search_tool_result` but NOT `tool_result`** — unit test the helper + +### Compile & Lint + +```sh +bun run typecheck # Verify no type errors from image URL source change +bun test # Regression test +bun run lint:all # Lint check +``` + +## Risks and Mitigations + +| Risk | Mitigation | +|------|------------| +| `AnthropicImageBlock` URL source causes compile errors in 3 files | §4 specifies exact guard pattern for each file | +| `mapContent()` Path A rewrite could change behavior for existing block types | The for-loop produces identical output for text/document/server_tool_use. Test existing behavior first. | +| `isServerToolResultBlock` matches future unknown `*_tool_result` types | This is intentional — future-proofing. Unknown types serialize as JSON, which is safe. | +| Token overhead estimates are approximate | Only affects compaction timing, not billing. Over-estimating is safer (triggers compaction earlier). | +| `default` catch-all in `mapContent()` could serialize unexpected blocks | Guards with `"content" in block` and excludes thinking/redacted_thinking. Blocks without `content` are silently skipped. | + +## Non-Goals + +- Implementing server tool execution (the proxy only translates) +- Supporting Anthropic Batch API or Files API +- Implementing prompt caching semantics +- Adding `reasoning_effort` to `ChatCompletionsPayload` (out of scope) +- Modifying the web search detection/interception pipeline diff --git a/generate-cert.bat b/generate-cert.bat new file mode 100644 index 000000000..3434f28c8 --- /dev/null +++ b/generate-cert.bat @@ -0,0 +1,64 @@ +@echo off +setlocal +TITLE Generate self-signed TLS certificate + +echo ================================================ +echo Generate self-signed TLS certificate +echo ================================================ +echo. + +:: --- Configure the names/addresses the cert is valid for --------------- +:: The laptop connecting over HTTPS must reach the proxy using one of these. +:: Edit these if your LAN IP or hostname changes. +set "LAN_IP=192.168.0.105" +set "HOSTNAME=HPE-5CG5083GTB" + +:: SAN (Subject Alternative Name) list. A modern HTTPS client validates the +:: server against this list, NOT the CN, so the connecting address MUST appear +:: here. Add more IP:/DNS: entries comma-separated if needed. +set "SAN=IP:%LAN_IP%,IP:127.0.0.1,DNS:%HOSTNAME%,DNS:localhost" + +where openssl >nul 2>nul +if errorlevel 1 ( + echo ERROR: openssl not found on PATH. + echo Install it ^(e.g. "winget install ShiningLight.OpenSSL.Light"^) and retry. + pause + exit /b 1 +) + +if not exist certs mkdir certs + +echo Generating 2048-bit key + self-signed cert ^(valid 825 days^)... +echo Subject CN : %HOSTNAME% +echo SANs : %SAN% +echo. + +openssl req -x509 -newkey rsa:2048 -sha256 -days 825 -nodes ^ + -keyout certs\server.key -out certs\server.crt ^ + -subj "/CN=%HOSTNAME%" ^ + -addext "subjectAltName=%SAN%" ^ + -addext "basicConstraints=critical,CA:TRUE" ^ + -addext "keyUsage=critical,digitalSignature,keyCertSign" + +if errorlevel 1 ( + echo. + echo ERROR: certificate generation failed. + pause + exit /b 1 +) + +echo. +echo ================================================ +echo Done. Files written to: +echo certs\server.crt ^(certificate - copy this to the laptop and trust it^) +echo certs\server.key ^(private key - keep on this PC only^) +echo ================================================ +echo. +echo Next steps: +echo 1. Copy certs\server.crt to the laptop and install it as a +echo Trusted Root Certification Authority. +echo 2. Start the proxy with start.bat ^(now HTTPS^). +echo 3. On the laptop, connect to: https://%LAN_IP%:4141 +echo. +pause +endlocal diff --git a/opencode.json b/opencode.json index b7590f5ff..e41fd259f 100644 --- a/opencode.json +++ b/opencode.json @@ -16,5 +16,73 @@ "enabled": true } }, - "$schema": "https://opencode.ai/config.json" + "$schema": "https://opencode.ai/config.json", + "provider": { + "copilot-anthropic": { + "npm": "@ai-sdk/openai-compatible", + "name": "Copilot (Anthropic)", + "options": { + "baseURL": "http://localhost:4141/v1" + }, + "models": { + "claude-opus-4.6": { + "name": "Claude Opus 4.6", + "limit": { + "context": 200000, + "output": 32000 + }, + "attachment": true, + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + } + }, + "claude-sonnet-4.6": { + "name": "Claude Sonnet 4.6", + "limit": { + "context": 200000, + "output": 32000 + }, + "attachment": true, + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + } + }, + "claude-haiku-4.5": { + "name": "Claude Haiku 4.5", + "limit": { + "context": 200000, + "output": 64000 + }, + "attachment": true, + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + } + } + } + }, + "copilot-openai": { + "npm": "@ai-sdk/openai-compatible", + "name": "Copilot (OpenAI)", + "options": { + "baseURL": "http://localhost:1515/v1" + }, + "models": { + "gpt-5.4": { + "name": "GPT 5.4", + "limit": { + "context": 400000, + "output": 128000 + }, + "attachment": true, + "modalities": { + "input": ["text", "image"], + "output": ["text"] + } + } + } + } + } } \ No newline at end of file diff --git a/src/lib/context-vars.ts b/src/lib/context-vars.ts new file mode 100644 index 000000000..41d506989 --- /dev/null +++ b/src/lib/context-vars.ts @@ -0,0 +1,8 @@ +// Augment Hono's ContextVariableMap so c.set/c.get for "tokenCount" is type-safe +declare module "hono" { + interface ContextVariableMap { + tokenCount: number | undefined + } +} + +export {} diff --git a/src/lib/error.ts b/src/lib/error.ts index c39c22596..bf2218005 100644 --- a/src/lib/error.ts +++ b/src/lib/error.ts @@ -17,20 +17,40 @@ export async function forwardError(c: Context, error: unknown) { if (error instanceof HTTPError) { const errorText = await error.response.text() + const contentType = error.response.headers.get("content-type") let errorJson: unknown try { errorJson = JSON.parse(errorText) } catch { - errorJson = errorText + errorJson = null + } + const errorMessage = extractUpstreamErrorMessage( + errorJson, + errorText, + contentType, + ) + consola.error("HTTP error:", errorJson ?? errorMessage) + + // Detect context-window-exceeded errors from upstream and return an + // Anthropic-formatted "invalid_request_error" so Claude Code triggers + // auto-compaction instead of just displaying a generic API error. + // The raw Copilot error JSON uses a non-Anthropic format that Claude + // Code doesn't recognize as retriable. + if (isContextWindowError(errorMessage, error.response.status)) { + consola.debug( + `Context window exceeded — extracted message: "${errorMessage}"`, + ) + return sendAnthropicContextWindowError(c, errorMessage, { status: 400 }) + } + + if (errorJson !== null && typeof errorJson === "object") { + return c.json( + errorJson as Record<string, unknown>, + error.response.status as ContentfulStatusCode, + ) } - consola.error("HTTP error:", errorJson) return c.json( - { - error: { - message: errorText, - type: "error", - }, - }, + { error: { message: errorMessage, type: "error" } }, error.response.status as ContentfulStatusCode, ) } @@ -45,3 +65,282 @@ export async function forwardError(c: Context, error: unknown) { 500, ) } + +/** Extracts the error message from a parsed Copilot error response. */ +export function extractUpstreamErrorMessage( + errorJson: unknown, + fallback: string, + contentType?: string | null, +): string { + if (errorJson !== null && typeof errorJson === "object") { + const obj = errorJson as Record<string, unknown> + // Copilot format: { error: { message: "..." } } + if (typeof obj.error === "object" && obj.error !== null) { + const err = obj.error as Record<string, unknown> + if (typeof err.message === "string") return err.message + } + // Direct message field + if (typeof obj.message === "string") return obj.message + } + + return summarizeUpstreamErrorBody(fallback, contentType) +} + +export function summarizeUpstreamErrorBody( + body: string, + contentType?: string | null, +): string { + const trimmed = body.trim() + if (!trimmed) return "Upstream request failed." + if (!looksLikeHtml(trimmed, contentType)) return trimmed + + const title = extractHtmlText(trimmed, /<title[^>]*>([\s\S]*?)<\/title>/i) + const headline = extractHtmlText( + trimmed, + /<strong[^>]*>([\s\S]*?)<\/strong>/i, + ) + const paragraph = extractHtmlText(trimmed, /<p[^>]*>([\s\S]*?)<\/p>/i) + + const parts = [title, headline ?? paragraph].filter(Boolean) + if (parts.length > 0) { + return `Upstream returned an HTML error page: ${parts.join(" - ")}` + } + + return "Upstream returned an HTML error page." +} + +function looksLikeHtml(body: string, contentType?: string | null): boolean { + return ( + contentType?.toLowerCase().includes("text/html") === true + || /^\s*<!doctype html/i.test(body) + || /^\s*<html[\s>]/i.test(body) + ) +} + +function extractHtmlText(html: string, pattern: RegExp): string | undefined { + const match = html.match(pattern) + if (!match?.[1]) return undefined + + const decoded = match[1] + .replaceAll(/<[^>]+>/g, " ") + .replaceAll(" ", " ") + .replaceAll("·", "·") + .replaceAll("—", "-") + .replaceAll("&", "&") + .replaceAll(/\s+/g, " ") + .trim() + + return decoded.length > 0 ? decoded : undefined +} + +/** + * Detects whether an error message indicates the input exceeds the model's + * context window. + */ +export function isContextWindowError( + message: string, + statusCode?: number, +): boolean { + if (statusCode === 413) { + return !isOpaquePayloadParseError(message) + } + + const lower = message.toLowerCase() + return ( + lower.includes("exceeds the context window") + || lower.includes("context_length_exceeded") + || lower.includes("maximum context length") + || lower.includes("input exceeds") + || lower.includes("exceeds the limit") + || lower.includes("model_max_prompt_tokens_exceeded") + ) +} + +function isOpaquePayloadParseError(message: string): boolean { + return message.trim().toLowerCase() === "failed to parse request" +} + +/** + * Builds a complete Anthropic-shaped error response for context-window errors. + * Matches the real Anthropic API response shape exactly, including `request_id`, + * so Claude Code recognizes it as a genuine `invalid_request_error` and triggers + * reactive auto-compaction. + */ +export function buildAnthropicContextWindowErrorResponse( + upstreamMessage: string, + modelLimit?: number, +) { + const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 10)}` + return { + requestId, + body: { + type: "error", + request_id: requestId, + error: { + type: "invalid_request_error", + message: formatAnthropicContextWindowError(upstreamMessage, modelLimit), + }, + }, + } +} + +export function sendAnthropicContextWindowError( + c: Context, + upstreamMessage: string, + options: { + modelLimit?: number + status?: ContentfulStatusCode + } = {}, +) { + const { requestId, body } = buildAnthropicContextWindowErrorResponse( + upstreamMessage, + options.modelLimit, + ) + c.header("request-id", requestId) + return c.json(body, options.status ?? 400) +} + +export function sendAnthropicInvalidRequestError( + c: Context, + message: string, + status: ContentfulStatusCode = 400, +) { + const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 10)}` + c.header("request-id", requestId) + return c.json( + { + type: "error", + request_id: requestId, + error: { + type: "invalid_request_error", + message, + }, + }, + status, + ) +} + +/** + * Converts a context-window error message into the exact format the real + * Anthropic API uses: `"prompt is too long: {N} tokens > {M} maximum"`. + * + * Claude Code pattern-matches on this format to trigger reactive compaction. + * + * @param upstreamMessage - The raw error message from Copilot (used to extract token counts) + * @param modelLimit - The model's actual max_prompt_tokens limit (used when regex extraction fails) + */ +export function formatAnthropicContextWindowError( + upstreamMessage: string, + modelLimit?: number, +): string { + // Try to extract "prompt token count of X exceeds the limit of Y" from Copilot + const match = upstreamMessage.match( + /(\d[\d,]*)\s+exceeds\s+the\s+limit\s+of\s+(\d[\d,]*)/i, + ) + if (match) { + const actual = match[1].replaceAll(",", "") + const limit = match[2].replaceAll(",", "") + return `prompt is too long: ${actual} tokens > ${limit} maximum` + } + + // Use the model's actual limit if available, otherwise fall back to defaults + const limit = modelLimit ?? 935_000 + // Estimate actual tokens as ~1.5× the limit (we know it exceeded) + const actual = Math.round(limit * 1.5) + return `prompt is too long: ${actual} tokens > ${limit} maximum` +} + +// ─── Responses API (OpenAI/Codex) context-window errors ────────────────────── +// +// The OpenAI Codex CLI only recognizes a context-window overflow — and +// therefore only triggers automatic conversation compaction — when it parses a +// streamed `response.failed` SSE event whose `response.error.code` equals +// `context_length_exceeded`. This was verified against the Codex source: +// - codex-rs/codex-api/src/sse/responses.rs → `is_context_window_error()` +// matches strictly on `error.code == "context_length_exceeded"`, producing +// `ApiError::ContextWindowExceeded`. +// - codex-rs/codex-client/src/transport.rs → `stream()` rejects any non-2xx +// HTTP status as `TransportError::Http` *before* the SSE parser runs, so a +// plain HTTP 400/413 can never reach the detector. The signal MUST be a +// 200 OK stream carrying the `response.failed` event. +// - `ApiError::ContextWindowExceeded` → `CodexErr::ContextWindowExceeded` +// (api_bridge.rs) is what the turn loop reacts to by auto-compacting +// (core/src/compact.rs, core/src/session/turn.rs). + +/** OpenAI error `code` Codex pattern-matches to trigger auto-compaction. */ +export const CONTEXT_LENGTH_EXCEEDED_CODE = "context_length_exceeded" + +const DEFAULT_CONTEXT_WINDOW_MESSAGE = + "Your input exceeds the context window of this model. " + + "Please reduce the size of your input or start a new conversation and try again." + +/** + * Chooses the message to surface for a context-window error. Prefers a real, + * informative upstream message (e.g. one containing genuine token counts) and + * otherwise uses a generic message. Never fabricates token numbers. + */ +export function contextWindowErrorMessage(upstreamMessage?: string): string { + const trimmed = upstreamMessage?.trim() + if ( + trimmed + && /exceeds the (?:limit|context)|context.?length|maximum context|too long|input exceeds/i.test( + trimmed, + ) + ) { + return trimmed + } + return DEFAULT_CONTEXT_WINDOW_MESSAGE +} + +/** + * Builds the `response.failed` SSE event payload that the OpenAI Responses API + * emits (over a 200 OK stream) when the prompt exceeds the context window. + * Codex reads the JSON `type` and `response.error.code` fields from the event + * `data`; the `code` is what drives compaction. + */ +export function buildResponsesContextWindowFailedEvent( + upstreamMessage?: string, +): { + type: "response.failed" + response: Record<string, unknown> +} { + const id = `resp_${Date.now()}_${Math.random().toString(36).slice(2, 10)}` + return { + type: "response.failed", + response: { + id, + object: "response", + created_at: Math.floor(Date.now() / 1000), + status: "failed", + error: { + code: CONTEXT_LENGTH_EXCEEDED_CODE, + message: contextWindowErrorMessage(upstreamMessage), + }, + usage: null, + metadata: {}, + }, + } +} + +/** + * Builds the OpenAI-shaped JSON error body for a context-window overflow on a + * non-streaming request (returned with HTTP 400). Mirrors the real OpenAI error + * shape so OpenAI-compatible clients recognize the `code`. + */ +export function buildOpenAIContextWindowErrorBody(upstreamMessage?: string): { + error: { + message: string + type: "invalid_request_error" + param: null + code: string + } +} { + return { + error: { + message: contextWindowErrorMessage(upstreamMessage), + type: "invalid_request_error", + param: null, + code: CONTEXT_LENGTH_EXCEEDED_CODE, + }, + } +} diff --git a/src/lib/model-resolver.ts b/src/lib/model-resolver.ts new file mode 100644 index 000000000..99eafeacb --- /dev/null +++ b/src/lib/model-resolver.ts @@ -0,0 +1,85 @@ +import type { ModelsResponse } from "~/services/copilot/get-models" + +/** + * Canonicalizes a model id for fuzzy matching by lowercasing and treating + * `.` and `-` as interchangeable separators. Copilot publishes Claude/Gemini/ + * GPT models with dotted version suffixes (e.g. `claude-opus-4.8`), but clients + * frequently request the all-hyphen form (`claude-opus-4-8`). Collapsing the + * separator lets the two forms compare equal. + * + * `claude-opus-4.8` and `claude-opus-4-8` both canonicalize to `claude-opus-4-8`. + */ +function canonicalize(modelId: string): string { + return modelId.toLowerCase().replaceAll(".", "-") +} + +/** + * Anthropic publishes its models with a trailing `-YYYYMMDD` release-date + * stamp (e.g. `claude-haiku-4-5-20251001`, `claude-sonnet-4-5-20250929`). + * Copilot's catalog carries the undated form (`claude-haiku-4.5`), so the + * stamp must be stripped before matching. + * + * The pattern is deliberately narrow — exactly eight consecutive trailing + * digits — so it never touches Copilot's own dated ids, whose dates use + * hyphen-separated components and therefore end in only two digits + * (`gpt-4.1-2025-04-14`, `gpt-4o-2024-11-20`). + */ +const ANTHROPIC_DATE_SUFFIX = /-\d{8}$/ + +/** + * Attempts to match a requested id against the catalog, first by exact id, + * then by separator-insensitive {@link canonicalize} comparison. Returns the + * real catalog id on success, or `undefined` when nothing matches. + */ +function matchCatalogId( + requestedId: string, + models: ModelsResponse, +): string | undefined { + // Exact match takes precedence over any fuzzy rewriting. + if (models.data.some((m) => m.id === requestedId)) return requestedId + + // Fall back to canonical (separator-insensitive) matching. + const target = canonicalize(requestedId) + return models.data.find((m) => canonicalize(m.id) === target)?.id +} + +/** + * Resolves a requested model id to an id that actually exists in Copilot's + * model catalog. + * + * Resolution order: + * 1. Exact match — returned verbatim. This guarantees legitimately + * hyphenated ids (e.g. `gpt-4-0125-preview`, `gpt-4`) are never rewritten. + * 2. Canonical match — the requested id is compared against each available + * model using {@link canonicalize}, so `claude-opus-4-8` resolves to the + * real `claude-opus-4.8`. The first catalog entry that canonicalizes + * equal wins (catalog order, mirroring `Array.find`). + * 3. Date-stamped match — Anthropic's trailing `-YYYYMMDD` stamp is stripped + * and steps 1–2 are retried, so `claude-haiku-4-5-20251001` resolves to + * `claude-haiku-4.5`. Runs last so a real exact/canonical match always + * wins before the stamp is removed. + * + * When nothing matches — or the catalog is unavailable — the original id is + * returned unchanged so the upstream API produces its normal error. + */ +export function resolveModelId( + requestedId: string, + models: ModelsResponse | undefined, +): string { + if (!requestedId || !models) return requestedId + + // 1 & 2. Exact, then canonical matching against the id as requested. + const direct = matchCatalogId(requestedId, models) + if (direct) return direct + + // 3. Strip Anthropic's trailing release-date stamp and retry. + if (ANTHROPIC_DATE_SUFFIX.test(requestedId)) { + const stripped = matchCatalogId( + requestedId.replace(ANTHROPIC_DATE_SUFFIX, ""), + models, + ) + if (stripped) return stripped + } + + return requestedId +} diff --git a/src/lib/model-selector.ts b/src/lib/model-selector.ts new file mode 100644 index 000000000..e1bf80695 --- /dev/null +++ b/src/lib/model-selector.ts @@ -0,0 +1,69 @@ +import type { ModelsResponse } from "~/services/copilot/get-models" + +import { getModelContextWindow } from "~/services/copilot/get-models" + +export interface ModelSelectionResult { + model: string + switched: boolean + reason?: string +} + +// These models support ~935k tokens (1M context) despite reporting lower limits. +const MODELS_WITH_1M_CONTEXT = new Set([ + "claude-opus-4.6", + "claude-opus-4.7", + "claude-opus-4.8", + "claude-sonnet-4.6", + "gemini-3.1-pro-preview", + "gemini-3.5-flash", + "gpt-5.4", + "gpt-5.5", +]) + +export function selectModelForTokenCount( + requestedModelId: string, + models: ModelsResponse, + estimatedTokens: number, +): ModelSelectionResult { + const requestedModel = models.data.find((m) => m.id === requestedModelId) + if (!requestedModel) { + return { model: requestedModelId, switched: false } + } + + if (MODELS_WITH_1M_CONTEXT.has(requestedModelId)) { + return { model: requestedModelId, switched: false } + } + + const contextWindow = getModelContextWindow(requestedModel) + if (contextWindow === undefined) { + return { model: requestedModelId, switched: false } + } + + if (estimatedTokens <= contextWindow) { + return { model: requestedModelId, switched: false } + } + + // Find the model with the largest context window + const largestModel = models.data.reduce< + (typeof models.data)[number] | undefined + >((best, m) => { + const win = getModelContextWindow(m) ?? 0 + const bestWin = best ? (getModelContextWindow(best) ?? 0) : 0 + return win > bestWin ? m : best + }, undefined) + + if (!largestModel || largestModel.id === requestedModelId) { + return { model: requestedModelId, switched: false } + } + + const largestWindow = getModelContextWindow(largestModel) ?? 0 + if (estimatedTokens > largestWindow) { + return { model: requestedModelId, switched: false } + } + + return { + model: largestModel.id, + switched: true, + reason: `prompt ${estimatedTokens} tokens exceeds ${requestedModelId} limit ${contextWindow}, switching to ${largestModel.id} (${largestWindow})`, + } +} diff --git a/src/lib/rate-limit.ts b/src/lib/rate-limit.ts index e41f58297..789ae062b 100644 --- a/src/lib/rate-limit.ts +++ b/src/lib/rate-limit.ts @@ -44,3 +44,82 @@ export async function checkRateLimit(state: State) { consola.info("Rate limit wait completed, proceeding with request") return } + +const burstQueues = new Map<string, Promise<void>>() + +export async function checkBurstLimit(state: State, model?: string) { + if (state.burstCount === undefined || state.burstWindowSeconds === undefined) + return + + const key = state.burstScope === "model" && model ? model : "__global__" + + const prev = burstQueues.get(key) ?? Promise.resolve() + const ticket = prev.then(() => acquireBurstSlot(state, key)) + burstQueues.set( + key, + ticket.catch(() => {}), + ) + return ticket +} + +function getTimestamps(state: State, key: string): Array<number> { + if (key === "__global__") return state.burstRequestTimestamps + let ts = state.burstPerModelTimestamps.get(key) + if (!ts) { + ts = [] + state.burstPerModelTimestamps.set(key, ts) + } + return ts +} + +function setTimestamps(state: State, key: string, ts: Array<number>) { + if (key === "__global__") { + state.burstRequestTimestamps = ts + } else { + state.burstPerModelTimestamps.set(key, ts) + } +} + +async function acquireBurstSlot(state: State, key: string) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const windowMs = state.burstWindowSeconds! * 1000 + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const maxBurst = state.burstCount! + const minSpacingMs = state.burstMinSpacingMs + const label = key === "__global__" ? "" : ` [${key}]` + + while (true) { + const now = Date.now() + + const filtered = getTimestamps(state, key).filter( + (ts) => ts > now - windowMs, + ) + setTimestamps(state, key, filtered) + + if (filtered.length < maxBurst) { + if (minSpacingMs > 0) { + const last = filtered.at(-1) + const elapsed = last !== undefined ? now - last : Infinity + if (elapsed < minSpacingMs) { + const gap = minSpacingMs - elapsed + const gapLabel = + gap < 1000 ? `${gap}ms` : `${(gap / 1000).toFixed(1)}s` + consola.debug(`${label} Spacing requests: waiting ${gapLabel}`) + await sleep(gap) + } + } + getTimestamps(state, key).push(Date.now()) + return + } + + const oldest = filtered[0] ?? now + const waitMs = Math.max(0, oldest + windowMs - now) + const waitLabel = + waitMs < 1000 ? `${waitMs}ms` : `${(waitMs / 1000).toFixed(1)}s` + consola.warn( + `${label} Burst limit reached. Waiting ${waitLabel} before proceeding...`, + ) + await sleep(waitMs) + consola.debug(`${label} Burst limit wait completed, re-checking...`) + } +} diff --git a/src/lib/request-logger.test.ts b/src/lib/request-logger.test.ts new file mode 100644 index 000000000..d1d539cea --- /dev/null +++ b/src/lib/request-logger.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, test } from "bun:test" +import { Hono } from "hono" + +import "~/lib/context-vars" + +import { + extractBodyFields, + formatDuration, + requestLogger, +} from "./request-logger" + +function makeApp( + handler: (c: import("hono").Context) => Response | Promise<Response>, +) { + const app = new Hono() + app.use(requestLogger) + app.post("/test", handler) + return app +} + +describe("formatDuration", () => { + test("formats sub-second durations as Xms", () => { + expect(formatDuration(0)).toBe("0ms") + expect(formatDuration(42)).toBe("42ms") + expect(formatDuration(999)).toBe("999ms") + }) + + test("formats >= 1000ms as X.Xs", () => { + expect(formatDuration(1000)).toBe("1.0s") + expect(formatDuration(1500)).toBe("1.5s") + expect(formatDuration(45200)).toBe("45.2s") + }) +}) + +// Helper: create a Hono Context from a raw Request so we can unit-test +// extractBodyFields without spinning up a full server. +async function makeContext(req: Request): Promise<import("hono").Context> { + let capturedCtx!: import("hono").Context + const app = new Hono() + app.all("/*", (c) => { + capturedCtx = c + return c.text("ok") + }) + await app.request(req) + return capturedCtx +} + +describe("extractBodyFields", () => { + test("extracts model and stream from valid JSON body", async () => { + const body = JSON.stringify({ model: "gpt-5.4", stream: true }) + const ctx = await makeContext( + new Request("http://x", { + method: "POST", + body, + headers: { "content-type": "application/json" }, + }), + ) + const result = await extractBodyFields(ctx) + expect(result.model).toBe("gpt-5.4") + expect(result.stream).toBe(true) + }) + + test("returns empty object for non-JSON body", async () => { + const ctx = await makeContext( + new Request("http://x", { method: "POST", body: "not json" }), + ) + const result = await extractBodyFields(ctx) + expect(result.model).toBeUndefined() + expect(result.stream).toBeUndefined() + }) + + test("returns empty object for empty body", async () => { + const ctx = await makeContext(new Request("http://x", { method: "GET" })) + const result = await extractBodyFields(ctx) + expect(result.model).toBeUndefined() + }) +}) + +describe("requestLogger middleware", () => { + test("calls next() and passes through the response", async () => { + const app = makeApp((c) => c.json({ ok: true }, 200)) + const res = await app.request("/test", { + method: "POST", + body: JSON.stringify({ model: "gpt-5.4" }), + headers: { "content-type": "application/json" }, + }) + expect(res.status).toBe(200) + const body = (await res.json()) as { ok: boolean } + expect(body.ok).toBe(true) + }) + + test("does not interfere with handler reading req.json()", async () => { + let parsed: unknown + const app = makeApp(async (c) => { + parsed = await c.req.json() + return c.json({ ok: true }) + }) + await app.request("/test", { + method: "POST", + body: JSON.stringify({ model: "test-model", stream: false }), + headers: { "content-type": "application/json" }, + }) + expect((parsed as { model: string }).model).toBe("test-model") + }) + + test("passes through non-2xx responses unchanged", async () => { + // Verifies the middleware does not break error responses — + // status and body are forwarded to the client as-is. + const app = makeApp((c) => + c.json({ error: { message: "model not found" } }, 404), + ) + const res = await app.request("/test", { + method: "POST", + body: JSON.stringify({ model: "bad-model" }), + headers: { "content-type": "application/json" }, + }) + expect(res.status).toBe(404) + const body = (await res.json()) as { error: { message: string } } + expect(body.error.message).toBe("model not found") + }) + + test("reads tokenCount set by handler via context", async () => { + // Verifies the c.set("tokenCount", n) / c.get("tokenCount") round-trip works + // across the middleware boundary, which is the mechanism the logger uses to + // display token counts in the log line. + let readBack: number | undefined + const app = new Hono() + // Observer middleware wraps requestLogger: runs before and after it + app.use(async (c, next) => { + await next() // runs requestLogger + handler + readBack = c.get("tokenCount") + }) + app.use(requestLogger) + app.post("/verify", (c) => { + c.set("tokenCount", 999) + return c.json({ ok: true }) + }) + const res = await app.request("/verify", { + method: "POST", + body: JSON.stringify({ model: "gpt-4" }), + headers: { "content-type": "application/json" }, + }) + expect(res.status).toBe(200) + expect(readBack).toBe(999) + }) +}) diff --git a/src/lib/request-logger.ts b/src/lib/request-logger.ts new file mode 100644 index 000000000..34267417c --- /dev/null +++ b/src/lib/request-logger.ts @@ -0,0 +1,139 @@ +import type { Context, MiddlewareHandler } from "hono" + +import { extractSessionId } from "~/lib/session-id" + +// ─── Pure helpers (exported for testing) ───────────────────────────────────── + +export function formatDuration(ms: number): string { + if (ms < 1000) return `${ms}ms` + return `${(ms / 1000).toFixed(1)}s` +} + +/** + * Extracts `model`, `stream`, and `sessionId` from the Hono request context. + * + * Uses `c.req.json()` instead of `c.req.raw.clone()` so that Hono's internal + * body-cache is shared between the middleware and any subsequent handler that + * also calls `c.req.json()`. Reading the raw request body via `.clone()` in + * production bypasses this cache and creates a `ReadableStream.tee()` on the + * live TCP-socket-backed body, which can corrupt the stream that the handler + * later tries to read — especially for large request bodies. + * + * `sessionId` is extracted from `metadata.user_id` (Anthropic protocol) via + * {@link extractSessionId}, which parses the JSON-encoded value that Claude + * Code sends and returns a stable 8-char hex hash. + */ +export async function extractBodyFields( + c: Context, +): Promise<{ model?: string; stream?: boolean; sessionId?: string }> { + try { + const parsed = await c.req.json<Record<string, unknown>>() + const metadata = parsed.metadata as Record<string, unknown> | undefined + const sessionId = extractSessionId( + typeof metadata?.user_id === "string" ? metadata.user_id : undefined, + ) + return { + model: typeof parsed.model === "string" ? parsed.model : undefined, + stream: typeof parsed.stream === "boolean" ? parsed.stream : undefined, + sessionId, + } + } catch { + return {} + } +} + +// ─── ANSI helpers ───────────────────────────────────────────────────────────── + +const R = "\x1b[0m" // reset +const DIM = "\x1b[2m" +const RED = "\x1b[31m" +const GREEN = "\x1b[32m" +const YELLOW = "\x1b[33m" +const BLUE = "\x1b[34m" +const CYAN = "\x1b[36m" + +function colorStatus(status: number): string { + const s = String(status) + return status < 400 ? `${GREEN}${s}${R}` : `${RED}${s}${R}` +} + +/** + * Pads a string to the given width. ANSI escape codes are excluded from the + * width calculation so coloured strings align correctly in the terminal. + */ +function pad(str: string, width: number): string { + // eslint-disable-next-line no-control-regex + const visible = str.replaceAll(/\x1b\[[0-9;]*m/g, "") + const diff = width - visible.length + return diff > 0 ? str + " ".repeat(diff) : str +} + +// ─── Middleware ─────────────────────────────────────────────────────────────── + +export const requestLogger: MiddlewareHandler = async (c, next) => { + const start = Date.now() + const method = c.req.method + const path = c.req.path + + // Extract body fields AFTER recording start time but BEFORE next() so the + // duration still covers the full round-trip. Because we use c.req.json() + // (Hono's cached body reader) the result is shared with the handler — + // no double-read, no stream tee. + const { model, stream, sessionId } = await extractBodyFields(c) + + await next() + + const duration = Date.now() - start + const status = c.res.status + const ok = status < 400 + + // Token count stored by handler via c.set("tokenCount", n) + // Type is known via ContextVariableMap augmentation in src/lib/context-vars.ts + const tokenCount = c.get("tokenCount") + + // On error: try to extract message from response body without consuming it + let errorMsg = "" + if (!ok) { + try { + const cloned = c.res.clone() + const text = await cloned.text() + const parsed = JSON.parse(text) as Record<string, unknown> + const err = parsed.error as Record<string, unknown> | undefined + errorMsg = + typeof err?.message === "string" ? err.message : text.slice(0, 120) + } catch { + // ignore + } + } + + const now = new Date() + const timeStr = pad(`${DIM}${now.toLocaleTimeString()}${R}`, 11) + const prefix = ok ? `${CYAN}◀${R}` : `${RED}✕${R}` + const methodStr = pad(`${DIM}${method}${R}`, 4) + const pathStr = pad(`${CYAN}${path}${R}`, 27) + const modelStr = pad(model ? `${YELLOW}${model}${R}` : "", 26) + const sessionStr = pad(sessionId ? `${DIM}${sessionId}${R}` : "", 8) + const streamStr = pad(stream === true ? `${BLUE}stream${R}` : "", 6) + const statusStr = pad(colorStatus(status), 3) + const durationStr = pad(formatDuration(duration), 6) + const tokenStr = tokenCount !== undefined ? `${DIM}in:${tokenCount}${R}` : "" + const errorStr = errorMsg ? `${RED}${errorMsg}${R}` : "" + + const line = [ + timeStr, + prefix, + methodStr, + pathStr, + modelStr, + sessionStr, + streamStr, + statusStr, + durationStr, + ].join(" ") + + // Append optional trailing fields only when present (no trailing whitespace) + const trailing = [tokenStr, errorStr].filter(Boolean).join(" ") + + // Use process.stdout directly to avoid consola adding its own timestamp + process.stdout.write(`${trailing ? `${line} ${trailing}` : line}\n`) +} diff --git a/src/lib/session-id.ts b/src/lib/session-id.ts new file mode 100644 index 000000000..b2f56c5df --- /dev/null +++ b/src/lib/session-id.ts @@ -0,0 +1,42 @@ +import { createHash } from "node:crypto" + +/** + * Extracts a compact session identifier from the Anthropic `metadata.user_id` field. + * + * Claude Code sets `metadata.user_id` to a JSON-encoded string like: + * `{"device_id":"...","session_id":"..."}` + * + * This function: + * 1. Attempts to JSON.parse the string and extract `session_id` or `device_id` + * 2. If parsing fails (plain UUID or arbitrary string), uses the raw value + * 3. Returns the first 8 hex chars of a SHA-256 hash for a compact, stable identifier + * + * The hash ensures a consistent 8-char hex string regardless of input format, + * and avoids leaking raw identifiers into logs. + */ +export function extractSessionId( + userId: string | undefined, +): string | undefined { + if (!userId || userId.length === 0) return undefined + + let key: string + + // Try to parse as JSON (Claude Code sends {"session_id":"...","device_id":"..."}) + if (userId.startsWith("{")) { + try { + const parsed = JSON.parse(userId) as Record<string, unknown> + // Prefer session_id, fall back to device_id, then the full string + key = + (typeof parsed.session_id === "string" && parsed.session_id) + || (typeof parsed.device_id === "string" && parsed.device_id) + || userId + } catch { + key = userId + } + } else { + key = userId + } + + // Hash to get a stable, compact identifier + return createHash("sha256").update(key).digest("hex").slice(0, 8) +} diff --git a/src/lib/state.ts b/src/lib/state.ts index 5ba4dc1d1..99a6d6569 100644 --- a/src/lib/state.ts +++ b/src/lib/state.ts @@ -15,6 +15,18 @@ export interface State { // Rate limiting configuration rateLimitSeconds?: number lastRequestTimestamp?: number + + // Burst rate limiting configuration + burstCount?: number + burstWindowSeconds?: number + burstMinSpacingMs: number + burstScope: "global" | "model" + burstRequestTimestamps: Array<number> + burstPerModelTimestamps: Map<string, Array<number>> + + // Web search configuration + braveApiKey?: string + tavilyApiKey?: string } export const state: State = { @@ -22,4 +34,12 @@ export const state: State = { manualApprove: false, rateLimitWait: false, showToken: false, + burstRequestTimestamps: [], + burstMinSpacingMs: 0, + burstScope: "global", + burstPerModelTimestamps: new Map(), +} + +export function isWebSearchEnabled(): boolean { + return Boolean(state.braveApiKey) || Boolean(state.tavilyApiKey) } diff --git a/src/lib/tokenizer.ts b/src/lib/tokenizer.ts index 8c3eda736..f9175f639 100644 --- a/src/lib/tokenizer.ts +++ b/src/lib/tokenizer.ts @@ -46,6 +46,15 @@ const calculateToolCallsTokens = ( /** * Calculate tokens for content parts */ +/** + * Estimates image tokens based on Anthropic's formula: tokens ≈ (w × h) / 750. + * Since we only have the base64 data URL at this point (not original dimensions), + * we use a conservative fixed estimate per image. Anthropic caps images at + * 1568 px on the longest edge, so the maximum is ~1,600 tokens. Most + * screenshots fall in the 1,200–1,600 range. Using 1,600 as a safe ceiling. + */ +const TOKENS_PER_IMAGE = 1_600 + const calculateContentPartsTokens = ( contentParts: Array<ContentPart>, encoder: Encoder, @@ -53,7 +62,12 @@ const calculateContentPartsTokens = ( let tokens = 0 for (const part of contentParts) { if (part.type === "image_url") { - tokens += encoder.encode(part.image_url.url).length + 85 + // Use a fixed per-image token estimate based on Anthropic's vision + // pricing (tokens = width*height / 750, max ~1,600 for full-size + // images). Previously this tokenized the entire base64 data URL as + // text, producing ~76K tokens per screenshot — 50× the real cost — + // which triggered premature compaction in Claude Code. + tokens += TOKENS_PER_IMAGE } else if (part.text) { tokens += encoder.encode(part.text).length } diff --git a/src/lib/tool-call-repair.ts b/src/lib/tool-call-repair.ts new file mode 100644 index 000000000..4c9bcf266 --- /dev/null +++ b/src/lib/tool-call-repair.ts @@ -0,0 +1,137 @@ +import type { Message } from "~/services/copilot/create-chat-completions" + +/** + * Repairs orphaned tool calls/results in an OpenAI-format message array so the + * upstream (Copilot, and its internal Anthropic re-validation for Claude + * models) does not reject the request. + * + * Anthropic enforces a strict invariant: every `tool_result` must reference a + * `tool_use` in the immediately-preceding assistant message, and every + * `tool_use` must be followed by its `tool_result`. Editing conversation + * history breaks this — context compaction can drop an assistant tool_use + * turn, and interrupted/parallel tool calls can leave a result without its + * call (or a call without its result). Forwarding either shape yields: + * + * "unexpected tool_use_id found in tool_result blocks: toolu_xxx. Each + * tool_result block must have a corresponding tool_use block in the previous + * message." (code: invalid_request_body) + * + * Two complementary repairs, both operating on `role:"tool"` runs that follow + * an `assistant` message bearing `tool_calls`: + * - {@link removeOrphanedResults}: drop `tool` messages whose `tool_call_id` + * is not among the assistant's `tool_calls` ids. These are stale/partial + * results from a real parallel batch — their content is throwaway and + * deleting them keeps the valid siblings adjacent to their assistant turn. + * - {@link insertMissingResults}: synthesize empty placeholder `tool` messages + * for assistant `tool_calls` ids that have no result, so no call dangles. + * + * A trailing `tool` message that does NOT follow an assistant turn with + * tool_calls (the common case after context compaction drops the assistant + * tool_use turn) is *converted* to a plain user message rather than deleted — + * Claude Code frequently bundles real user text into that same message, so + * dropping it would silently lose the user's instruction. Converting away + * from the `tool` role satisfies the invariant while preserving content. + * + * Mutates `messages` in place. + */ + +interface RepairRange { + messages: Array<Message> + start: number + end: number + callIds: Set<string> +} + +function removeOrphanedResults(range: RepairRange): number { + const { messages, start, callIds } = range + let j = range.end + for (let k = j - start - 2; k >= 0; k--) { + const id = messages[start + 1 + k].tool_call_id + if (id && !callIds.has(id)) { + messages.splice(start + 1 + k, 1) + j-- + } + } + return j +} + +function insertMissingResults(range: RepairRange): number { + const { messages, start, end, callIds } = range + const existingIds = new Set( + messages + .slice(start + 1, end) + .map((m) => m.tool_call_id) + .filter(Boolean), + ) + const missing = [...callIds].filter((id) => !existingIds.has(id)) + if (missing.length > 0) { + const placeholders = missing.map((id) => ({ + role: "tool" as const, + tool_call_id: id, + content: "", + })) + messages.splice(start + 1, 0, ...placeholders) + return end + missing.length + } + return end +} + +/** + * Converts an orphaned `tool` message (no preceding assistant tool_calls) into + * a plain user message in place, preserving its content so bundled user text + * is not lost. Empty-content orphans are dropped entirely (nothing to keep). + * + * Returns true if the message was converted (caller should advance past it), + * false if it was removed (caller should re-check the same index). + */ +function convertOrphanedResultToUser( + messages: Array<Message>, + index: number, +): boolean { + const msg = messages[index] + const hasContent = + typeof msg.content === "string" ? + msg.content.trim().length > 0 + : Array.isArray(msg.content) && msg.content.length > 0 + + if (!hasContent) { + messages.splice(index, 1) + return false + } + + messages[index] = { role: "user", content: msg.content } + return true +} + +export function repairOrphanedToolCalls(messages: Array<Message>): void { + let i = 0 + while (i < messages.length) { + const msg = messages[i] + if (msg.role === "assistant" && msg.tool_calls?.length) { + const callIds = new Set(msg.tool_calls.map((tc) => tc.id)) + + let j = i + 1 + while (j < messages.length && messages[j].role === "tool") j++ + + const range: RepairRange = { messages, start: i, end: j, callIds } + j = removeOrphanedResults(range) + range.end = j + j = insertMissingResults(range) + + i = j + continue + } + + if (msg.role === "tool" && msg.tool_call_id) { + const prev = i > 0 ? messages[i - 1] : null + if (!prev || prev.role !== "assistant" || !prev.tool_calls?.length) { + // Orphan with no owning assistant turn — preserve any bundled user + // text by converting to a user message instead of discarding it. + if (convertOrphanedResultToUser(messages, i)) i++ + continue + } + } + + i++ + } +} diff --git a/src/main.ts b/src/main.ts index 4f6ca784b..56539998f 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,12 +1,23 @@ #!/usr/bin/env node import { defineCommand, runMain } from "citty" +import consola from "consola" import { auth } from "./auth" import { checkUsage } from "./check-usage" import { debug } from "./debug" import { start } from "./start" +// Surface unhandled promise rejections as visible errors instead of silent exits. +// Without this, Bun exits the process without printing any useful diagnostic info. +process.on("unhandledRejection", (reason) => { + consola.error("Unhandled promise rejection:", reason) +}) + +process.on("uncaughtException", (error) => { + consola.error("Uncaught exception:", error) +}) + const main = defineCommand({ meta: { name: "copilot-api", diff --git a/src/routes/chat-completions/handler.ts b/src/routes/chat-completions/handler.ts index 04a5ae9ed..a5134485c 100644 --- a/src/routes/chat-completions/handler.ts +++ b/src/routes/chat-completions/handler.ts @@ -4,15 +4,33 @@ import consola from "consola" import { streamSSE, type SSEMessage } from "hono/streaming" import { awaitApproval } from "~/lib/approval" -import { checkRateLimit } from "~/lib/rate-limit" +import { resolveModelId } from "~/lib/model-resolver" +import { selectModelForTokenCount } from "~/lib/model-selector" +import { checkBurstLimit, checkRateLimit } from "~/lib/rate-limit" import { state } from "~/lib/state" import { getTokenCount } from "~/lib/tokenizer" import { isNullish } from "~/lib/utils" import { createChatCompletions, + createResponsesCompletion, type ChatCompletionResponse, type ChatCompletionsPayload, } from "~/services/copilot/create-chat-completions" +import { requiresResponsesApi } from "~/services/copilot/responses-translation" + +/** + * Returns the payload with its `model` normalized to a real Copilot model id + * (e.g. `claude-opus-4-8` → `claude-opus-4.8`). Returns the original payload + * unchanged when the id already matches or no canonical match exists. + */ +function withResolvedModel( + payload: ChatCompletionsPayload, +): ChatCompletionsPayload { + const resolvedModel = resolveModelId(payload.model, state.models) + if (resolvedModel === payload.model) return payload + consola.debug(`[model-resolver] '${payload.model}' → '${resolvedModel}'`) + return { ...payload, model: resolvedModel } +} export async function handleCompletion(c: Context) { await checkRateLimit(state) @@ -20,16 +38,39 @@ export async function handleCompletion(c: Context) { let payload = await c.req.json<ChatCompletionsPayload>() consola.debug("Request payload:", JSON.stringify(payload).slice(-400)) + // Normalize the requested model id (e.g. `claude-opus-4-8` → + // `claude-opus-4.8`) to a real Copilot model before lookup or forwarding. + payload = withResolvedModel(payload) + + await checkBurstLimit(state, payload.model) + // Find the selected model - const selectedModel = state.models?.data.find( + let selectedModel = state.models?.data.find( (model) => model.id === payload.model, ) - // Calculate and display token count try { if (selectedModel) { const tokenCount = await getTokenCount(payload, selectedModel) - consola.info("Current token count:", tokenCount) + c.set("tokenCount", tokenCount.input) + consola.debug("Token count:", tokenCount) + // Context-overflow guard: auto-switch to largest-context model if needed + // state.models is non-null here — selectedModel was found from it + if (state.models) { + const result = selectModelForTokenCount( + payload.model, + state.models, + tokenCount.input, + ) + if (result.switched) { + consola.warn(`Context overflow: ${result.reason}`) + payload = { ...payload, model: result.model } + // Update selectedModel so max_tokens defaulting below uses the switched model + selectedModel = + state.models.data.find((m) => m.id === result.model) + ?? selectedModel + } + } } else { consola.warn("No model selected, skipping token count calculation") } @@ -47,7 +88,15 @@ export async function handleCompletion(c: Context) { consola.debug("Set max_tokens to:", JSON.stringify(payload.max_tokens)) } - const response = await createChatCompletions(payload) + const hasTools = Array.isArray(payload.tools) && payload.tools.length > 0 + const usesResponsesApi = + (selectedModel !== undefined && requiresResponsesApi(selectedModel)) + || (hasTools && payload.model.startsWith("gpt-5")) + + const response = + usesResponsesApi ? + await createResponsesCompletion(payload) + : await createChatCompletions(payload) if (isNonStreaming(response)) { consola.debug("Non-streaming response:", JSON.stringify(response)) @@ -58,11 +107,16 @@ export async function handleCompletion(c: Context) { return streamSSE(c, async (stream) => { for await (const chunk of response) { consola.debug("Streaming chunk:", JSON.stringify(chunk)) + // Stop iterating once we see [DONE] — don't wait for the upstream HTTP + // connection to close, which can hang if the Copilot API keeps it open. + if (chunk.data === "[DONE]") break await stream.writeSSE(chunk as SSEMessage) } }) } const isNonStreaming = ( - response: Awaited<ReturnType<typeof createChatCompletions>>, + response: + | Awaited<ReturnType<typeof createChatCompletions>> + | Awaited<ReturnType<typeof createResponsesCompletion>>, ): response is ChatCompletionResponse => Object.hasOwn(response, "choices") diff --git a/src/routes/embeddings/route.ts b/src/routes/embeddings/route.ts index 4c4fc7b8a..05b95d60a 100644 --- a/src/routes/embeddings/route.ts +++ b/src/routes/embeddings/route.ts @@ -1,6 +1,8 @@ import { Hono } from "hono" import { forwardError } from "~/lib/error" +import { resolveModelId } from "~/lib/model-resolver" +import { state } from "~/lib/state" import { createEmbeddings, type EmbeddingRequest, @@ -11,6 +13,8 @@ export const embeddingRoutes = new Hono() embeddingRoutes.post("/", async (c) => { try { const paylod = await c.req.json<EmbeddingRequest>() + // Normalize the requested model id to a real Copilot model before forwarding. + paylod.model = resolveModelId(paylod.model, state.models) const response = await createEmbeddings(paylod) return c.json(response) diff --git a/src/routes/messages/anthropic-types.ts b/src/routes/messages/anthropic-types.ts index 881fffcc8..e1ed3a4a8 100644 --- a/src/routes/messages/anthropic-types.ts +++ b/src/routes/messages/anthropic-types.ts @@ -1,10 +1,12 @@ +import type { ToolNameMap } from "./tool-name-mapping" + // Anthropic API Types export interface AnthropicMessagesPayload { model: string messages: Array<AnthropicMessage> max_tokens: number - system?: string | Array<AnthropicTextBlock> + system?: string | Array<AnthropicSystemBlock> metadata?: { user_id?: string } @@ -17,32 +19,76 @@ export interface AnthropicMessagesPayload { tool_choice?: { type: "auto" | "any" | "tool" | "none" name?: string + disable_parallel_tool_use?: boolean // parsed but not forwarded — no OpenAI equivalent } thinking?: { type: "enabled" budget_tokens?: number } service_tier?: "auto" | "standard_only" + output_config?: { + effort?: "low" | "medium" | "high" | "max" + format?: { type: "json_schema"; schema: Record<string, unknown> } + } + speed?: "standard" | "fast" + cache_control?: { type: "ephemeral"; ttl?: number } + container?: Record<string, unknown> + mcp_servers?: Array<Record<string, unknown>> + context_management?: Record<string, unknown> + inference_geo?: string } export interface AnthropicTextBlock { type: "text" text: string + cache_control?: { type: "ephemeral"; ttl?: number } + citations?: Array<unknown> // pass-through, not interpreted by proxy +} + +/** + * Catch-all for system-level blocks with unknown `type` values (e.g. + * cache_control-only blocks, future block types). Allows `system` arrays + * to contain non-text entries without losing type safety on known blocks. + */ +export interface AnthropicGenericSystemBlock { + type: string + [key: string]: unknown } +export type AnthropicSystemBlock = + | AnthropicTextBlock + | AnthropicGenericSystemBlock + export interface AnthropicImageBlock { type: "image" - source: { - type: "base64" - media_type: "image/jpeg" | "image/png" | "image/gif" | "image/webp" - data: string - } + source: + | { + type: "base64" + media_type: "image/jpeg" | "image/png" | "image/gif" | "image/webp" + data: string + } + | { type: "url"; url: string } +} + +// New: document block (PDFs sent via Read tool) +// source union covers all Anthropic-documented source types; handler emits a +// placeholder string regardless, so media_type is intentionally wide. +export interface AnthropicDocumentBlock { + type: "document" + title?: string + source: + | { type: "base64"; media_type: string; data: string } + | { type: "url"; url: string } + | { type: "text"; data: string } + cache_control?: { type: "ephemeral"; ttl?: number } } export interface AnthropicToolResultBlock { type: "tool_result" tool_use_id: string - content: string + content: + | string + | Array<AnthropicTextBlock | AnthropicImageBlock | AnthropicDocumentBlock> is_error?: boolean } @@ -51,22 +97,109 @@ export interface AnthropicToolUseBlock { id: string name: string input: Record<string, unknown> + cache_control?: { type: "ephemeral"; ttl?: number } + caller?: Record<string, unknown> } export interface AnthropicThinkingBlock { type: "thinking" thinking: string + signature?: string // Used by Claude Code extended thinking +} + +// New: redacted thinking (redact-thinking-2026-02-12 beta) +export interface AnthropicRedactedThinkingBlock { + type: "redacted_thinking" + data: string +} + +// New: server-side tool use block in assistant messages +// Appears in multi-turn histories from real Anthropic API with web_search server tool +export interface AnthropicServerToolUseBlock { + type: "server_tool_use" + id: string + name: string + input: Record<string, unknown> +} + +// New: web search tool result block in user messages +// Appears in multi-turn histories from real Anthropic API with web_search server tool +export interface AnthropicWebSearchToolResultBlock { + type: "web_search_tool_result" + tool_use_id: string + content: unknown +} + +export interface AnthropicSearchResultBlock { + type: "search_result" + source: string + title: string + content: string + cache_control?: { type: "ephemeral"; ttl?: number } + citations?: Array<unknown> + search_result_index?: number + start_block_index?: number + end_block_index?: number +} + +export interface AnthropicContainerUploadBlock { + type: "container_upload" + file_id: string + cache_control?: { type: "ephemeral"; ttl?: number } +} + +interface ServerToolResultBase { + tool_use_id: string + content: unknown + cache_control?: { type: "ephemeral"; ttl?: number } +} + +interface AnthropicWebFetchToolResultBlock extends ServerToolResultBase { + type: "web_fetch_tool_result" +} + +interface AnthropicCodeExecutionToolResultBlock extends ServerToolResultBase { + type: "code_execution_tool_result" +} + +interface AnthropicBashCodeExecutionToolResultBlock + extends ServerToolResultBase { + type: "bash_code_execution_tool_result" +} + +interface AnthropicTextEditorCodeExecutionToolResultBlock + extends ServerToolResultBase { + type: "text_editor_code_execution_tool_result" } +interface AnthropicToolSearchToolResultBlock extends ServerToolResultBase { + type: "tool_search_tool_result" +} + +export type AnthropicServerToolResultBlock = + | AnthropicWebSearchToolResultBlock + | AnthropicWebFetchToolResultBlock + | AnthropicCodeExecutionToolResultBlock + | AnthropicBashCodeExecutionToolResultBlock + | AnthropicTextEditorCodeExecutionToolResultBlock + | AnthropicToolSearchToolResultBlock + export type AnthropicUserContentBlock = | AnthropicTextBlock | AnthropicImageBlock + | AnthropicDocumentBlock | AnthropicToolResultBlock + | AnthropicSearchResultBlock + | AnthropicContainerUploadBlock + | AnthropicServerToolResultBlock export type AnthropicAssistantContentBlock = | AnthropicTextBlock | AnthropicToolUseBlock | AnthropicThinkingBlock + | AnthropicRedactedThinkingBlock + | AnthropicServerToolUseBlock + | AnthropicServerToolResultBlock export interface AnthropicUserMessage { role: "user" @@ -80,10 +213,34 @@ export interface AnthropicAssistantMessage { export type AnthropicMessage = AnthropicUserMessage | AnthropicAssistantMessage -export interface AnthropicTool { +// Custom tool (has input_schema) — what Claude Code and standard clients send +export interface AnthropicCustomTool { name: string description?: string input_schema: Record<string, unknown> + strict?: boolean + cache_control?: { type: "ephemeral"; ttl?: number } + defer_loading?: boolean + input_examples?: Array<unknown> + eager_input_streaming?: boolean + allowed_callers?: Array<string> +} + +// Anthropic-typed tool (versioned type string, no input_schema) +// Examples: bash_20250124, text_editor_20250728, computer_20251124, web_search_20250305 +interface AnthropicTypedTool { + type: string + name: string + [key: string]: unknown +} + +export type AnthropicTool = AnthropicCustomTool | AnthropicTypedTool + +// Discriminant: typed tools never have input_schema; custom tools always do. +// Using presence of input_schema is more robust than checking for type, +// since a future custom tool definition could include a type field. +export function isTypedTool(tool: AnthropicTool): tool is AnthropicTypedTool { + return !("input_schema" in tool) } export interface AnthropicResponse { @@ -134,6 +291,13 @@ export interface AnthropicContentBlockStartEvent { input: Record<string, unknown> }) | { type: "thinking"; thinking: string } + | { + type: "server_tool_use" + id: string + name: string + input: Record<string, unknown> + } + | AnthropicServerToolResultBlock } export interface AnthropicContentBlockDeltaEvent { @@ -144,6 +308,8 @@ export interface AnthropicContentBlockDeltaEvent { | { type: "input_json_delta"; partial_json: string } | { type: "thinking_delta"; thinking: string } | { type: "signature_delta"; signature: string } + | { type: "citations_delta"; citation: unknown } + | { type: "compaction_delta"; content: unknown } } export interface AnthropicContentBlockStopEvent { @@ -194,13 +360,47 @@ export type AnthropicStreamEventData = // State for streaming translation export interface AnthropicStreamState { messageStartSent: boolean + /** Whether the terminal message_stop event has been emitted. */ + messageStopSent: boolean contentBlockIndex: number contentBlockOpen: boolean + /** Whether the currently open content block is a thinking block. */ + thinkingBlockOpen: boolean + /** Whether any visible text content has been emitted in this response. */ + hasEmittedText: boolean toolCalls: { [openAIToolIndex: number]: { id: string name: string anthropicBlockIndex: number + /** Accumulated JSON argument fragments for truncation detection. */ + accumulatedArgs: string } } + /** Maps OpenAI-safe function names back to the original Anthropic tool names. */ + toolNameMap?: ToolNameMap + /** Whether the original request included thinking: { type: "enabled" }. */ + thinkingEnabled: boolean + /** + * Last usage data seen from any upstream chunk. With `stream_options: + * { include_usage: true }`, the OpenAI API sends usage in the final chunk + * (often after the finish_reason chunk). We accumulate it here so the + * `message_delta` event can include accurate `input_tokens`. + */ + lastSeenUsage?: { + prompt_tokens: number + completion_tokens: number + prompt_tokens_details?: { cached_tokens: number } + } + /** + * Deferred finish_reason — set when we see `finish_reason` but want to + * delay emitting `message_delta` + `message_stop` until after the usage + * chunk arrives (or the stream ends). + */ + deferredFinishReason?: + | "stop" + | "length" + | "tool_calls" + | "content_filter" + | null } diff --git a/src/routes/messages/attachment-overhead.ts b/src/routes/messages/attachment-overhead.ts new file mode 100644 index 000000000..c6ec47d2e --- /dev/null +++ b/src/routes/messages/attachment-overhead.ts @@ -0,0 +1,168 @@ +import type { + AnthropicImageBlock, + AnthropicDocumentBlock, + AnthropicMessagesPayload, + AnthropicSearchResultBlock, +} from "./anthropic-types" + +// Approximation for text-like attachments where we only have raw characters. +// This intentionally rounds up a little so Claude Code compacts early rather +// than waiting until the proxy receives an oversized attachment payload. +const CHARS_PER_TOKEN = 4 + +// Anthropic's PDF docs show roughly 1k tokens for a small 3-page text-only +// PDF and much more for visually rich PDFs. We do not have extracted text or +// page counts here, so we use a size-based estimate with a conservative floor. +const MIN_PDF_TOKENS = 2_500 +const PDF_BASE64_CHARS_PER_TOKEN = 80 + +// URL/file-backed documents still consume meaningful context once expanded by +// Claude Code, but we do not know their exact size at count-time. +const DEFAULT_REMOTE_DOCUMENT_TOKENS = 2_500 +const MIN_IMAGE_TOKENS = 1_600 +const IMAGE_BASE64_CHARS_PER_TOKEN = 120 + +function estimateDocumentTokens(block: AnthropicDocumentBlock): number { + const source = block.source + + if (source.type === "text") { + return Math.max(1, Math.ceil(source.data.length / CHARS_PER_TOKEN)) + } + + if (source.type === "base64") { + if (source.media_type === "application/pdf") { + return Math.max( + MIN_PDF_TOKENS, + Math.ceil(source.data.length / PDF_BASE64_CHARS_PER_TOKEN), + ) + } + + return Math.max(1, Math.ceil(source.data.length / CHARS_PER_TOKEN)) + } + + return DEFAULT_REMOTE_DOCUMENT_TOKENS +} + +function estimateImageTokens(block: AnthropicImageBlock): number { + if (block.source.type !== "base64") return MIN_IMAGE_TOKENS // URL images: use minimum estimate + + return Math.max( + MIN_IMAGE_TOKENS, + Math.ceil(block.source.data.length / IMAGE_BASE64_CHARS_PER_TOKEN), + ) +} + +function estimateSearchResultTokens(block: AnthropicSearchResultBlock): number { + return Math.max(500, Math.ceil(block.content.length / 4)) +} + +function getDocumentBlocksFromContent( + content: NonNullable<AnthropicMessagesPayload["messages"][number]["content"]>, +): Array<AnthropicDocumentBlock> { + if (typeof content === "string") return [] + + const documents: Array<AnthropicDocumentBlock> = [] + + for (const block of content) { + if (block.type === "document") { + documents.push(block) + continue + } + + if (block.type === "tool_result" && Array.isArray(block.content)) { + for (const nested of block.content) { + if (nested.type === "document") { + documents.push(nested) + } + } + } + } + + return documents +} + +function getImageBlocksFromContent( + content: NonNullable<AnthropicMessagesPayload["messages"][number]["content"]>, +): Array<AnthropicImageBlock> { + if (typeof content === "string") return [] + + const images: Array<AnthropicImageBlock> = [] + + for (const block of content) { + if (block.type === "image") { + images.push(block) + continue + } + + if (block.type === "tool_result" && Array.isArray(block.content)) { + for (const nested of block.content) { + if (nested.type === "image") { + images.push(nested) + } + } + } + } + + return images +} + +function getSearchResultBlocksFromContent( + content: NonNullable<AnthropicMessagesPayload["messages"][number]["content"]>, +): Array<AnthropicSearchResultBlock> { + if (typeof content === "string") return [] + + const results: Array<AnthropicSearchResultBlock> = [] + + for (const block of content) { + if (block.type === "search_result") { + results.push(block) + } + // Note: search_result blocks do NOT appear inside tool_result.content + // (tool_result.content is typed as string | Array<Text|Image|Document>), + // so we don't need to check nested content here. + } + + return results +} + +function getContainerUploadBlocksFromContent( + content: NonNullable<AnthropicMessagesPayload["messages"][number]["content"]>, +): number { + if (typeof content === "string") return 0 + + let count = 0 + for (const block of content) { + if (block.type === "container_upload") count++ + } + return count +} + +export function estimateAdditionalAttachmentTokens( + payload: AnthropicMessagesPayload, +): number { + let tokens = 0 + + for (const message of payload.messages) { + if (message.role !== "user") continue + + for (const document of getDocumentBlocksFromContent(message.content)) { + tokens += estimateDocumentTokens(document) + } + + for (const image of getImageBlocksFromContent(message.content)) { + tokens += estimateImageTokens(image) + } + + // search_result blocks + for (const searchResult of getSearchResultBlocksFromContent( + message.content, + )) { + tokens += estimateSearchResultTokens(searchResult) + } + + // container_upload blocks (fixed overhead — just a file ID reference) + tokens += getContainerUploadBlocksFromContent(message.content) * 100 + } + + return tokens +} diff --git a/src/routes/messages/count-tokens-handler.ts b/src/routes/messages/count-tokens-handler.ts index 2ec849cb8..e37c22f03 100644 --- a/src/routes/messages/count-tokens-handler.ts +++ b/src/routes/messages/count-tokens-handler.ts @@ -2,12 +2,90 @@ import type { Context } from "hono" import consola from "consola" +import { resolveModelId } from "~/lib/model-resolver" import { state } from "~/lib/state" import { getTokenCount } from "~/lib/tokenizer" +import { + type Model, + getModelContextWindow, +} from "~/services/copilot/get-models" -import { type AnthropicMessagesPayload } from "./anthropic-types" +import { type AnthropicMessagesPayload, isTypedTool } from "./anthropic-types" +import { estimateAdditionalAttachmentTokens } from "./attachment-overhead" +import { getSessionId, hasStrippedImages } from "./image-stripping" import { translateToOpenAI } from "./non-stream-translation" +// The default context window size Claude Code assumes for unknown models. +// This is the effective window Claude Code uses for compaction thresholds +// when it doesn't recognize the model (typically ~200K for Claude models). +const CLAUDE_CODE_DEFAULT_WINDOW = 200_000 +const NON_CLAUDE_COMPACTION_SAFETY_MARGIN = 1.08 +const GEMINI_CONTEXT_SAFETY_BUFFER_TOKENS = 8_000 + +// Models that support ~935k input tokens (1M context variant). +const MODELS_WITH_1M_CONTEXT = new Set([ + "claude-opus-4.6", + "claude-opus-4.7", + "claude-opus-4.8", + "claude-sonnet-4.6", + "gemini-3.1-pro-preview", + "gemini-3.5-flash", + "gpt-5.4", + "gpt-5.5", +]) +const EFFECTIVE_1M_WINDOW = 935_000 + +// Token overhead for Anthropic-typed tools (per Anthropic pricing docs). +// Custom tools use the existing flat +346 for the entire tools array. +// Typed tools add per-tool overhead on top. +const ANTHROPIC_TYPED_TOOL_TOKEN_OVERHEAD: Record<string, number> = { + text_editor_20250728: 700, + text_editor_20250429: 700, + text_editor_20250124: 700, + text_editor_20241022: 700, + bash_20250124: 700, + bash_20241022: 700, + // computer_use and web_search: overhead included in beta pricing, not additive + // --- new (estimates for compaction heuristic, not billing) --- + web_fetch_20250910: 500, + web_fetch_20260209: 500, + web_fetch_20260309: 500, + code_execution_20250522: 500, + code_execution_20250825: 500, + code_execution_20260120: 500, + advisor_20260301: 500, + tool_search_tool_bm25_20251119: 200, + tool_search_tool_regex_20251119: 200, + tool_search_tool_bm25: 200, + tool_search_tool_regex: 200, + mcp_toolset: 300, +} + +function applyToolTokenOverhead( + tokenCount: { input: number; output: number }, + payload: AnthropicMessagesPayload, +): void { + if (!payload.tools) return + + if (payload.model.startsWith("claude")) { + const hasCustomTools = payload.tools.some((t) => !isTypedTool(t)) + // Preserve existing flat +346 for the custom tools array (unchanged behavior) + if (hasCustomTools) { + tokenCount.input = tokenCount.input + 346 + } + // Add per-typed-tool overhead for Anthropic-typed tools (new) + for (const tool of payload.tools) { + if (isTypedTool(tool)) { + tokenCount.input = + tokenCount.input + + (ANTHROPIC_TYPED_TOOL_TOKEN_OVERHEAD[tool.type] ?? 0) + } + } + } else if (payload.model.startsWith("grok")) { + tokenCount.input = tokenCount.input + 480 + } +} + /** * Handles token counting for Anthropic messages */ @@ -17,6 +95,33 @@ export async function handleCountTokens(c: Context) { const anthropicPayload = await c.req.json<AnthropicMessagesPayload>() + // Normalize the requested model id (e.g. `claude-opus-4-8` → + // `claude-opus-4.8`) so the model lookup and per-model scaling below + // operate on the real Copilot model rather than falling back to the + // default window. + anthropicPayload.model = resolveModelId( + anthropicPayload.model, + state.models, + ) + + // If images were stripped from this session's recent messages request, + // force compaction by returning a very high token count. The flag is + // per-session so Session A stripping images won't cause Session B to + // compact. The flag stays set until a messages request from this + // session arrives with zero images (meaning compaction succeeded). + const sessionId = getSessionId(anthropicPayload) + if (hasStrippedImages(sessionId)) { + const compactionTarget = + MODELS_WITH_1M_CONTEXT.has(anthropicPayload.model) ? EFFECTIVE_1M_WINDOW + : CLAUDE_CODE_DEFAULT_WINDOW + consola.debug( + `[${sessionId}] Images were recently stripped — returning ${compactionTarget} tokens to trigger compaction`, + ) + return c.json({ + input_tokens: compactionTarget, + }) + } + const openAIPayload = translateToOpenAI(anthropicPayload) const selectedModel = state.models?.data.find( @@ -24,47 +129,105 @@ export async function handleCountTokens(c: Context) { ) if (!selectedModel) { - consola.warn("Model not found, returning default token count") + const compactionTarget = + MODELS_WITH_1M_CONTEXT.has(anthropicPayload.model) ? EFFECTIVE_1M_WINDOW + : CLAUDE_CODE_DEFAULT_WINDOW + consola.warn( + `Model '${anthropicPayload.model}' not found in cached models, returning ${compactionTarget} tokens to trigger compaction`, + ) return c.json({ - input_tokens: 1, + input_tokens: compactionTarget, }) } const tokenCount = await getTokenCount(openAIPayload, selectedModel) + tokenCount.input += estimateAdditionalAttachmentTokens(anthropicPayload) if (anthropicPayload.tools && anthropicPayload.tools.length > 0) { let mcpToolExist = false if (anthropicBeta?.startsWith("claude-code")) { - mcpToolExist = anthropicPayload.tools.some((tool) => - tool.name.startsWith("mcp__"), + mcpToolExist = anthropicPayload.tools.some( + (tool) => !isTypedTool(tool) && tool.name.startsWith("mcp__"), ) } if (!mcpToolExist) { - if (anthropicPayload.model.startsWith("claude")) { - // https://docs.anthropic.com/en/docs/agents-and-tools/tool-use/overview#pricing - tokenCount.input = tokenCount.input + 346 - } else if (anthropicPayload.model.startsWith("grok")) { - tokenCount.input = tokenCount.input + 480 - } + applyToolTokenOverhead(tokenCount, anthropicPayload) } } let finalTokenCount = tokenCount.input + tokenCount.output - if (anthropicPayload.model.startsWith("claude")) { - finalTokenCount = Math.round(finalTokenCount * 1.15) + if (MODELS_WITH_1M_CONTEXT.has(anthropicPayload.model)) { + // 1M context models: no scaling needed — the real limit (~935k) is far + // above Claude Code's default window so compaction fires naturally. + } else if (anthropicPayload.model.startsWith("claude")) { + // Legacy Claude models: scale 1.2× so compaction fires before the + // conservative 168k Copilot limit. + finalTokenCount = Math.round(finalTokenCount * 1.2) } else if (anthropicPayload.model.startsWith("grok")) { finalTokenCount = Math.round(finalTokenCount * 1.03) + } else { + // For non-Claude/Grok models (GPT, Gemini, etc.), dynamically scale + // token counts based on the model's actual prompt token limit. Claude + // Code uses an internal "effective window" (typically ~200K) for compaction + // thresholds. If the model's actual prompt limit is smaller, we scale + // up so Claude Code's proactive compaction fires before the model rejects. + // + // Example: GPT-5-mini has a 128K prompt limit, Claude Code assumes ~200K. + // Scale factor: 200_000 / 128_000 ≈ 1.56. At 120K actual tokens, + // we report ~187K, which crosses Claude Code's ~190K threshold. + finalTokenCount = scaleTokensForModel(finalTokenCount, selectedModel) } - consola.info("Token count:", finalTokenCount) + consola.debug("Token count:", finalTokenCount) return c.json({ input_tokens: finalTokenCount, }) } catch (error) { consola.error("Error counting tokens:", error) + // Return a high token count on error so Claude Code's context-window + // compaction kicks in. Returning 1 would make Claude Code think the + // context is nearly empty, so it would never compact and the next + // completion request would be rejected by Copilot for exceeding the + // prompt token limit. return c.json({ - input_tokens: 1, + input_tokens: 200_000, }) } } + +/** + * Scales token counts for non-Claude/Grok models so Claude Code's proactive + * compaction fires before the model's actual prompt token limit is exhausted. + * + * Claude Code tracks context usage against an internal "effective window" + * (typically ~200K for Claude models). When using models with smaller prompt + * limits (e.g. GPT-5-mini at 128K), the raw token count won't trigger + * compaction in time — Claude Code would wait until ~190K tokens, but the + * model rejects at 128K. + * + * The scale factor maps the model's actual prompt limit to Claude Code's + * expected window: `scale = CLAUDE_CODE_DEFAULT_WINDOW / modelPromptLimit`. + * Only scales up (never down) to avoid artificially shrinking large windows. + */ +export function scaleTokensForModel(tokenCount: number, model: Model): number { + const modelWindow = getModelContextWindow(model) + if (!modelWindow || modelWindow >= CLAUDE_CODE_DEFAULT_WINDOW) { + // Model has a large enough context window — no scaling needed. + return tokenCount + } + const safeModelWindow = + model.id.startsWith("gemini") ? + Math.max(1, modelWindow - GEMINI_CONTEXT_SAFETY_BUFFER_TOKENS) + : modelWindow + const effectiveWindow = Math.round( + CLAUDE_CODE_DEFAULT_WINDOW * NON_CLAUDE_COMPACTION_SAFETY_MARGIN, + ) + const scale = effectiveWindow / safeModelWindow + consola.debug( + `Scaling token count for ${model.id}: ${tokenCount} × ${scale.toFixed(2)} ` + + `(model window: ${modelWindow}, safe window: ${safeModelWindow}, ` + + `effective: ${effectiveWindow})`, + ) + return Math.round(tokenCount * scale) +} diff --git a/src/routes/messages/handler.ts b/src/routes/messages/handler.ts index 85dbf6243..8066d6d99 100644 --- a/src/routes/messages/handler.ts +++ b/src/routes/messages/handler.ts @@ -1,26 +1,156 @@ +/* eslint-disable max-lines */ +import type { ServerSentEventMessage } from "fetch-event-stream" import type { Context } from "hono" +import type { SSEStreamingApi } from "hono/streaming" import consola from "consola" import { streamSSE } from "hono/streaming" import { awaitApproval } from "~/lib/approval" -import { checkRateLimit } from "~/lib/rate-limit" -import { state } from "~/lib/state" +import { + extractUpstreamErrorMessage, + HTTPError, + isContextWindowError, + formatAnthropicContextWindowError, + sendAnthropicContextWindowError, + sendAnthropicInvalidRequestError, +} from "~/lib/error" +import { resolveModelId } from "~/lib/model-resolver" +import { checkBurstLimit, checkRateLimit } from "~/lib/rate-limit" +import { isWebSearchEnabled, state } from "~/lib/state" import { createChatCompletions, + createResponsesCompletion, type ChatCompletionChunk, type ChatCompletionResponse, } from "~/services/copilot/create-chat-completions" +import { + getModelContextWindow, + getModelMaxOutput, +} from "~/services/copilot/get-models" +import { requiresResponsesApi } from "~/services/copilot/responses-translation" +import { + prepareWebSearchPayload, + webSearchInterceptor, +} from "~/services/web-search/interceptor" import { type AnthropicMessagesPayload, + type AnthropicResponse, + type AnthropicStreamEventData, type AnthropicStreamState, } from "./anthropic-types" +import { + CompactionNeededError, + fetchWithImageStripping, + type ImageStrippingResult, + updateImageFlag, +} from "./image-stripping" +import { findInvalidEmbeddedImage } from "./image-validation" +import { applyLargeEditGuidance } from "./large-edit-guidance" import { translateToAnthropic, translateToOpenAI, } from "./non-stream-translation" -import { translateChunkToAnthropicEvents } from "./stream-translation" +import { + findTruncatedToolCalls, + flushDeferredFinish, + isEmptyStreamResponse, + translateChunkToAnthropicEvents, + translateErrorToAnthropicErrorEvent, +} from "./stream-translation" +import { + createToolNameMapFromAnthropicPayload, + type ToolNameMap, +} from "./tool-name-mapping" +import { toAnthropicMessageId } from "./utils" +import { + detectWebSearchIntent, + stripWebSearchTypedTools, +} from "./web-search-detection" + +// Interval at which SSE ping events are sent to keep the downstream +// connection alive while waiting for Copilot to start responding or +// between chunks during slow generation (e.g. large file writes). +// Must be shorter than both the network's TCP idle timeout (~5 min on +// enterprise firewalls) and Claude Code's stream inactivity detector +// (~45s). 10 seconds gives comfortable headroom for both. +const PING_INTERVAL_MS = 10_000 + +// Maximum time to wait for the next upstream chunk inside pipeStreamToClient +// before assuming the stream is stalled. When the Copilot API finishes +// streaming a large tool call (e.g. 6000+ line Write), it sometimes never +// sends the chunk containing `finish_reason` — the HTTP body remains open +// and `reader.read()` blocks indefinitely. Models like Gemini 3 Pro can +// have long pauses (60-90s) between reasoning chunks while doing deep +// internal processing. The downstream stays alive via PING_INTERVAL_MS, +// so the only constraint here is how long we wait for a genuinely stalled +// upstream. 90s accommodates long reasoning phases while still recovering +// from truly dead connections within a reasonable time. +const STREAM_STALL_TIMEOUT_MS = 90_000 + +// Maximum number of times to retry a timed-out upstream fetch before giving up. +// Each attempt gets a fresh TCP connection, resetting the firewall idle timer. +// Retry is safe because we only retry before the first byte arrives — if Copilot +// hasn't started generating yet, the request is idempotent. +const MAX_FETCH_RETRIES = 3 + +// Maximum number of times to retry when the model returns an empty response +// (finish_reason "stop" with no content or tool calls). Some models +// (notably Gemini) occasionally do this after their reasoning phase completes +// without producing output. Retrying typically succeeds on the next attempt. +const MAX_EMPTY_RESPONSE_RETRIES = 2 + +// Error name/code patterns that indicate a retriable network failure +// (firewall idle timeout, connection reset) vs. a non-retriable one (4xx, auth). +const RETRIABLE_ERROR_NAMES = new Set([ + "TimeoutError", + "ECONNRESET", + "FailedToOpenSocket", + "ConnectionRefused", +]) + +/** + * Classifies whether a thrown fetch error is a transient network failure that + * is safe to retry (before any response byte has arrived). + * + * Bun/undici surface these markers inconsistently: an inactivity abort sets + * `error.name = "TimeoutError"`, while a raw socket reset arrives as a plain + * `Error` with the marker on `error.code` (e.g. `ECONNRESET`). Checking only + * `.name` (the previous behavior) let connection resets fall through to a hard + * 500 instead of being retried. We match against both. + */ +export function isRetriableFetchError(error: unknown): error is Error { + if (!(error instanceof Error)) return false + const code = (error as { code?: unknown }).code + return ( + RETRIABLE_ERROR_NAMES.has(error.name) + || (typeof code === "string" && RETRIABLE_ERROR_NAMES.has(code)) + ) +} + +const MODELS_WITH_1M_CONTEXT = new Set([ + "claude-opus-4.6", + "claude-opus-4.7", + "claude-opus-4.8", + "claude-sonnet-4.6", + "gemini-3.1-pro-preview", + "gemini-3.5-flash", + "gpt-5.4", + "gpt-5.5", +]) +const EFFECTIVE_1M_LIMIT = 935_000 + +/** + * Looks up the model's max_prompt_tokens limit from cached models. + * Used to produce accurate "prompt is too long: N tokens > M maximum" + * errors even when the upstream error doesn't contain token numbers. + */ +function lookupModelLimit(modelId: string): number | undefined { + if (MODELS_WITH_1M_CONTEXT.has(modelId)) return EFFECTIVE_1M_LIMIT + const model = state.models?.data.find((m) => m.id === modelId) + return model ? getModelContextWindow(model) : undefined +} export async function handleCompletion(c: Context) { await checkRateLimit(state) @@ -28,64 +158,1755 @@ export async function handleCompletion(c: Context) { const anthropicPayload = await c.req.json<AnthropicMessagesPayload>() consola.debug("Anthropic request payload:", JSON.stringify(anthropicPayload)) - const openAIPayload = translateToOpenAI(anthropicPayload) + // Normalize the requested model id (e.g. `claude-opus-4-8` → `claude-opus-4.8`) + // to a real Copilot model before any downstream lookup or forwarding. + const resolvedModel = resolveModelId(anthropicPayload.model, state.models) + if (resolvedModel !== anthropicPayload.model) { + consola.debug( + `[model-resolver] '${anthropicPayload.model}' → '${resolvedModel}'`, + ) + anthropicPayload.model = resolvedModel + } + + await checkBurstLimit(state, anthropicPayload.model) + + const invalidImage = findInvalidEmbeddedImage(anthropicPayload) + if (invalidImage) { + return sendAnthropicInvalidRequestError( + c, + `Embedded ${invalidImage.mediaType} image is too small to process reliably ` + + `(${invalidImage.width}x${invalidImage.height}). ` + + "This guard only blocks tiny placeholder-style images and does not " + + "affect normal screenshots or UI attachments. Use an image at least " + + "4x4 pixels.", + ) + } + + // Check if compaction has removed images from this session's conversation. + // This clears the per-session image-stripped flag so count_tokens stops + // returning the inflated 200K value for this session. + updateImageFlag(anthropicPayload) + + if (state.manualApprove) { + await awaitApproval() + } + + // NOTE: We intentionally do NOT pre-flight reject requests based on local + // token estimation. Returning an Anthropic-formatted "invalid_request_error" + // causes Claude Code to auto-compact and retry in a loop — each retry adds + // more context, making the prompt even larger. Instead we let the request + // through to Copilot and rely on forwardError to return the raw Copilot + // error JSON (not Anthropic format), which Claude Code won't retry. + + // Structured output requests (e.g. title generation) use output_config.format. + // The Copilot Chat Completions API does not enforce response_format, so the + // model may return free text instead of JSON. Handle these specially: force + // non-streaming, validate/repair the JSON, and emit as SSE if the original + // request was streaming. + if (anthropicPayload.output_config?.format) { + return handleStructuredOutput(c, anthropicPayload) + } + + // For non-streaming requests just fetch and translate synchronously — + // no SSE connection needed, so no ping mechanism required. + if (!anthropicPayload.stream) { + return handleNonStreaming(c, anthropicPayload) + } + + if (looksLikeCompactionRequest(anthropicPayload)) { + try { + const result = await fetchCompactionResponse(anthropicPayload) + const toolNameMap = + createToolNameMapFromAnthropicPayload(anthropicPayload) + return streamSSE(c, async (stream) => { + await emitNonStreamingAsSSE(stream, result.response, { + imageTokenOverhead: estimateTokensForStrippedImages( + result.strippedBase64Chars, + ), + toolNameMap, + }) + }) + } catch (error) { + const contextWindowMessage = await extractContextWindowMessage(error) + if (error instanceof CompactionNeededError || contextWindowMessage) { + const modelLimit = lookupModelLimit(anthropicPayload.model) + return sendAnthropicContextWindowError(c, contextWindowMessage ?? "", { + status: 400, + modelLimit, + }) + } + throw error + } + } + + // Attempt upstream fetch BEFORE opening the SSE connection so context-window + // errors surface as HTTP-level responses (400 + invalid_request_error) that + // Claude Code recognizes for auto-compaction. SSE error events inside an + // already-committed HTTP 200 stream do NOT trigger compaction. + try { + const strippingResult = await prefetchCopilotResponse(anthropicPayload) + return streamSSE(c, (stream) => + handleStreaming(stream, anthropicPayload, strippingResult), + ) + } catch (error) { + // Context window errors (HTTP 400 with "exceeds the context window") + // or CompactionNeededError (413 cascade exhausted): return 400 with + // Anthropic-formatted invalid_request_error in the exact format + // Claude Code expects: "prompt is too long: N tokens > M maximum". + const contextWindowMessage = await extractContextWindowMessage(error) + if (error instanceof CompactionNeededError || contextWindowMessage) { + const modelLimit = lookupModelLimit(anthropicPayload.model) + consola.debug( + `[context-window] Upstream error: "${contextWindowMessage}", modelLimit=${modelLimit}`, + ) + return sendAnthropicContextWindowError(c, contextWindowMessage ?? "", { + status: 400, + modelLimit, + }) + } + + // All other errors: let route-level forwardError handle them + throw error + } +} + +/** + * Estimates the token cost of stripped base64 image data. + * Used to inflate response `input_tokens` so Claude Code sees the true + * context size and triggers compaction when images accumulate. + * + * Per Anthropic's docs, images cost ~(width*height)/750 tokens, with a + * practical maximum of ~1,600 tokens per image. Since we don't know the + * original dimensions, we use 1,600 as a conservative ceiling. + * + * A typical screenshot is ~200KB base64 (~267,000 chars). Dividing by + * a generous 200K-chars-per-image gives us a rough image count, then we + * multiply by the per-image token cost. + */ +function estimateTokensForStrippedImages(base64Chars: number): number { + if (base64Chars <= 0) return 0 + // Estimate number of images from total base64 chars. + // A typical screenshot is 150K-300K base64 chars; use 200K as average. + const estimatedImages = Math.max(1, Math.round(base64Chars / 200_000)) + return estimatedImages * 1_600 +} + +/** + * Inflates `input_tokens` in `message_start` and `message_delta` SSE events + * to account for base64 images that were stripped before sending to Copilot. + * Mutates the event in-place. + */ +function inflateEventInputTokens( + event: AnthropicStreamEventData, + overhead: number, +): void { + if (event.type === "message_start") { + event.message.usage.input_tokens += overhead + } + if ( + event.type === "message_delta" + && event.usage?.input_tokens !== undefined + ) { + event.usage.input_tokens += overhead + } +} + +async function handleNonStreaming( + c: Context, + anthropicPayload: AnthropicMessagesPayload, +) { + if (looksLikeCompactionRequest(anthropicPayload)) { + return handleNonStreamingCompaction(c, anthropicPayload) + } + + let result: ImageStrippingResult< + Awaited<ReturnType<typeof createChatCompletions>> + > + + try { + result = await fetchWithImageStripping( + fetchCopilotResponse, + anthropicPayload, + ) + } catch (error) { + // 413 cascade exhausted — all images stripped, still too large. + // Return invalid_request_error to trigger Claude Code auto-compaction. + // This is safe because images are already gone and compaction will + // reduce the text content, producing a convergently smaller request. + if (error instanceof CompactionNeededError) { + const modelLimit = lookupModelLimit(anthropicPayload.model) + return sendAnthropicContextWindowError(c, "", { + status: 400, + modelLimit, + }) + } + + // Re-throw non-413 HTTPErrors so they bubble up to the route-level + // forwardError handler, which returns the raw Copilot error JSON with + // the original HTTP status code. + if (error instanceof HTTPError) throw error + + consola.error("Copilot connection error (fetch-level):", error) + return c.json( + { + type: "error", + error: { + type: "api_error", + message: + error instanceof Error ? + error.message + : "An unexpected error occurred.", + }, + }, + 500, + ) + } + + if (!isNonStreaming(result.response)) { + // Payload said non-streaming but Copilot returned a stream — treat as error. + consola.error("Expected non-streaming response but got stream") + return c.json( + { + type: "error", + error: { type: "api_error", message: "Unexpected streaming response." }, + }, + 500, + ) + } + consola.debug( - "Translated OpenAI request payload:", + "Non-streaming response from Copilot:", + JSON.stringify(result.response).slice(-400), + ) + + // Detect empty non-streaming responses: some models (notably Gemini) + // return finish_reason "stop" with empty/null content and 0 output tokens. + // Returning this as a valid response causes Claude Code to see "end_turn" + // with empty content and stop the session. Returning overloaded_error would + // cause Claude Code to retry the exact same request in an infinite loop. + // Instead, return a synthetic valid response with explanatory text so the + // conversation can move forward. + if (isEmptyNonStreamingResponse(result.response)) { + consola.debug( + "Empty non-streaming response detected — returning synthetic fallback", + ) + return c.json(buildSyntheticFallbackJson(anthropicPayload, result.response)) + } + + if (shouldUsePlainTextCompactionFallback(anthropicPayload, result.response)) { + consola.debug( + "Non-streaming compaction response lacked usable text — retrying with tools stripped", + ) + const fallbackResponse = await fetchPlainTextCompactionResponse( + anthropicPayload, + result.response, + ) + result = { + ...result, + response: fallbackResponse, + } + } + + const toolNameMap = createToolNameMapFromAnthropicPayload(anthropicPayload) + const anthropicResponse = translateToAnthropic( + result.response as ChatCompletionResponse, + toolNameMap, + ) + + // Inflate input_tokens to account for images stripped before sending. + // Copilot reports prompt_tokens based on the smaller (stripped) payload, + // but Claude Code uses this value to track context usage and decide when + // to compact. Without inflation, it never sees the true cost of images + // in the conversation and never compacts. + if (result.strippedBase64Chars > 0) { + anthropicResponse.usage.input_tokens += estimateTokensForStrippedImages( + result.strippedBase64Chars, + ) + } + + consola.debug( + "Translated Anthropic response:", + JSON.stringify(anthropicResponse), + ) + return c.json(anthropicResponse) +} + +/** + * Handles structured output requests (output_config.format present). + * + * Claude Code uses structured output for lightweight tasks like title + * generation — it sends output_config.format.type = "json_schema" and + * expects the response text to be valid JSON matching the schema. + * + * The Copilot Chat Completions API silently ignores `response_format`, so + * the model may return free text instead of JSON. To work around this: + * + * 1. Force the upstream request to **non-streaming** so we get the full + * response text before committing anything to the client. + * 2. Validate the response text as JSON. If invalid, try to extract a + * JSON object from the free-text response. + * 3. If the original Anthropic request was streaming, emit the validated + * response as a proper SSE event sequence; otherwise return as JSON. + */ +async function handleStructuredOutput( + c: Context, + anthropicPayload: AnthropicMessagesPayload, +) { + const wasStreaming = anthropicPayload.stream + + // Force non-streaming so we can validate the full response + const nonStreamingPayload: AnthropicMessagesPayload = { + ...anthropicPayload, + stream: false, + } + + const toolNameMap = createToolNameMapFromAnthropicPayload(nonStreamingPayload) + const openAIPayload = translateToOpenAI(nonStreamingPayload, toolNameMap) + consola.debug( + "Translated OpenAI request payload (structured output):", JSON.stringify(openAIPayload), ) - if (state.manualApprove) { - await awaitApproval() + const selectedModel = state.models?.data.find( + (m) => m.id === openAIPayload.model, + ) + clampMaxTokens(openAIPayload, selectedModel) + + consola.debug( + `[structured-output] model=${openAIPayload.model} found=${selectedModel !== undefined}`, + ) + + let response: ChatCompletionResponse + try { + const result = await createChatCompletions(openAIPayload) + // Non-streaming should return ChatCompletionResponse directly + response = result as ChatCompletionResponse + } catch (error) { + consola.error("[structured-output] Upstream fetch failed:", error) + if (error instanceof HTTPError) throw error + return c.json( + { + type: "error", + error: { + type: "api_error", + message: + error instanceof Error ? + error.message + : "Structured output fetch failed.", + }, + }, + 500, + ) + } + + // Extract the response text + const rawText = response.choices[0]?.message.content ?? "" + + // Try to validate/repair the JSON + const repairedText = repairJsonResponse(rawText) + consola.debug( + `[structured-output] raw=${JSON.stringify(rawText).slice(0, 200)} repaired=${JSON.stringify(repairedText).slice(0, 200)}`, + ) + + // Replace the response content with the repaired text + if (response.choices[0]) { + response.choices[0].message.content = repairedText + } + + if (wasStreaming) { + // Emit as SSE event sequence + return streamSSE(c, async (stream) => { + await emitNonStreamingAsSSE(stream, response, { toolNameMap }) + }) + } + + // Non-streaming: translate and return + const anthropicResponse = translateToAnthropic(response, toolNameMap) + consola.debug( + "Translated Anthropic response (structured output):", + JSON.stringify(anthropicResponse), + ) + return c.json(anthropicResponse) +} + +/** + * Attempts to ensure the response text is valid JSON. + * + * When the Copilot API ignores response_format, the model may wrap JSON + * in markdown code fences, add explanatory text around it, or return + * entirely free-form text. This function tries progressively more + * aggressive extraction strategies: + * + * 1. If the text is already valid JSON, return as-is. + * 2. Strip markdown code fences and try again. + * 3. Extract the first JSON object literal from the text. + * 4. If all else fails, return the original text unchanged (Claude Code + * will fall back to its default title). + */ +function repairJsonResponse(text: string): string { + const trimmed = text.trim() + + // 1. Already valid JSON + if (isValidJson(trimmed)) return trimmed + + // 2. Markdown code fence: ```json ... ``` or ``` ... ``` + // eslint-disable-next-line regexp/no-super-linear-backtracking, regexp/optimal-quantifier-concatenation + const fenceMatch = trimmed.match(/```(?:json)?\s*\n?([\s\S]*?)\n?\s*```/) + if (fenceMatch?.[1] && isValidJson(fenceMatch[1].trim())) { + return fenceMatch[1].trim() + } + + // 3. Extract first JSON object from anywhere in the text + const jsonMatch = trimmed.match(/\{[\s\S]*\}/) + if (jsonMatch?.[0] && isValidJson(jsonMatch[0])) { + return jsonMatch[0] + } + + // 4. Give up — return original + return text +} + +function isValidJson(text: string): boolean { + try { + JSON.parse(text) + return true + } catch { + return false + } +} + +async function handleNonStreamingCompaction( + c: Context, + anthropicPayload: AnthropicMessagesPayload, +) { + try { + return c.json(await fetchNonStreamingAnthropicResponse(anthropicPayload)) + } catch (error) { + if (error instanceof CompactionNeededError) { + const modelLimit = lookupModelLimit(anthropicPayload.model) + return sendAnthropicContextWindowError(c, "", { + status: 400, + modelLimit, + }) + } + if (error instanceof HTTPError) throw error + + consola.error("Copilot connection error (fetch-level):", error) + return c.json( + { + type: "error", + error: { + type: "api_error", + message: + error instanceof Error ? + error.message + : "An unexpected error occurred.", + }, + }, + 500, + ) + } +} + +export function looksLikeCompactionRequest( + payload: AnthropicMessagesPayload, +): boolean { + const latestUserMessage = [...payload.messages] + .reverse() + .find((message) => message.role === "user") + + const fragments: Array<string> = [] + if (latestUserMessage) { + if (typeof latestUserMessage.content === "string") { + fragments.push(latestUserMessage.content) + } else { + for (const block of latestUserMessage.content) { + if (block.type === "text") { + fragments.push(block.text) + } + } + } + } + + const text = stripSystemReminders(fragments.join("\n")).toLowerCase() + + const looksLikeResumeScaffold = + containsAny(text, [ + "continue the conversation from where it left off", + "continue from the latest state above", + "resume directly", + "pick up the last task as if the break never happened", + ]) + && containsAny(text, [ + "this session is being continued from a previous conversation", + "older context was compacted to fit the model context window", + "conversation continuation summary:", + ]) + + if (looksLikeResumeScaffold) { + return false + } + + if ( + containsAny(text, [ + "<command-name>/compact</command-name>", + "<command-message>compact</command-message>", + ]) + ) { + return true + } + + const asksToCompactConversation = + containsAny(text, ["summarize", "summarise", "compact"]) + && containsAny(text, ["conversation", "chat", "session"]) + + const asksToGenerateSummary = + containsAny(text, ["create", "generate", "write"]) + && containsAny(text, [ + "conversation continuation summary", + "conversation summary", + "compact summary", + "summary for continuation", + ]) + + return asksToCompactConversation || asksToGenerateSummary +} + +function containsAny(text: string, candidates: Array<string>): boolean { + return candidates.some((candidate) => text.includes(candidate)) +} + +/** + * Strips `<system-reminder>...</system-reminder>` blocks from text. + * + * Claude Code injects system-reminder tags into user messages containing + * skill descriptions, CLAUDE.md content, MCP server instructions, and other + * metadata. These injected blocks often contain words like "compact", + * "summarize", "conversation", "session" that cause false positives in + * `looksLikeCompactionRequest`. By stripping them before pattern matching, + * we only inspect the user's actual message text. + */ +function stripSystemReminders(text: string): string { + return text.replaceAll(/<system-reminder>[\s\S]*?<\/system-reminder>/gi, "") +} + +function hasUsableNonStreamingText(response: ChatCompletionResponse): boolean { + if (response.choices.length === 0) return false + const choice = response.choices[0] + return ( + typeof choice.message.content === "string" + && choice.message.content.trim().length > 0 + ) +} + +export function shouldUsePlainTextCompactionFallback( + payload: AnthropicMessagesPayload, + response: ChatCompletionResponse, +): boolean { + if (!looksLikeCompactionRequest(payload)) return false + if (!payload.tools || payload.tools.length === 0) return false + if (response.choices.length === 0) return false + + const choice = response.choices[0] + return ( + !hasUsableNonStreamingText(response) + || choice.finish_reason === "tool_calls" + || (choice.message.tool_calls !== undefined + && choice.message.tool_calls.length > 0) + ) +} + +function buildPlainTextCompactionPayload( + payload: AnthropicMessagesPayload, +): AnthropicMessagesPayload { + const systemInstruction = + "Respond with plain text only. Do not call tools. Produce only the " + + "requested conversation summary or compaction text." + + let mergedSystem: AnthropicMessagesPayload["system"] + if (typeof payload.system === "string") { + mergedSystem = [ + { type: "text", text: payload.system }, + { type: "text", text: systemInstruction }, + ] + } else if (Array.isArray(payload.system)) { + mergedSystem = [ + ...payload.system, + { + type: "text", + text: systemInstruction, + }, + ] + } else { + mergedSystem = systemInstruction + } + + return { + ...payload, + system: mergedSystem, + tools: undefined, + tool_choice: { type: "none" }, + } +} + +function extractCompactionFragments( + payload: AnthropicMessagesPayload, +): Array<string> { + const fragments: Array<string> = [] + + for (const message of payload.messages.slice(-12)) { + if (typeof message.content === "string") { + fragments.push(`[${message.role}] ${message.content}`) + continue + } + for (const block of message.content) { + if (block.type === "text" && block.text.trim().length > 0) { + fragments.push(`[${message.role}] ${block.text}`) + } + } + } + + return fragments +} + +function clampCompactionFragment(text: string, maxLength: number): string { + const normalized = text.replaceAll(/\s+/g, " ").trim() + if (normalized.length <= maxLength) return normalized + const headLength = Math.max(80, Math.floor((maxLength - 5) / 2)) + const tailLength = Math.max(40, maxLength - headLength - 5) + return `${normalized.slice(0, headLength)} ... ${normalized.slice(-tailLength)}` +} + +function buildCompactionSummaryText(payload: AnthropicMessagesPayload): string { + const fragments = extractCompactionFragments(payload) + .slice(-8) + .map((fragment) => clampCompactionFragment(fragment, 400)) + + return ( + "Conversation continuation summary:\n" + + fragments.join("\n") + + "\n\nContinue from the latest state above. Older context was compacted " + + "to fit the model context window." + ) +} + +export function buildSyntheticCompactionResponse( + payload: AnthropicMessagesPayload, + usage?: ChatCompletionResponse["usage"], +): ChatCompletionResponse { + const content = buildCompactionSummaryText(payload) + + return { + id: `chatcmpl_compact_${Date.now()}`, + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: payload.model, + choices: [ + { + index: 0, + message: { + role: "assistant", + content, + }, + finish_reason: "stop", + logprobs: null, + }, + ], + usage: usage ?? { + prompt_tokens: 0, + completion_tokens: 1, + total_tokens: 1, + }, } +} + +async function fetchNonStreamingAnthropicResponse( + anthropicPayload: AnthropicMessagesPayload, +): Promise<AnthropicResponse> { + const result = await fetchWithImageStripping( + fetchCopilotResponse, + anthropicPayload, + ) - const response = await createChatCompletions(openAIPayload) + if (!isNonStreaming(result.response)) { + throw new Error("Unexpected streaming response.") + } + + consola.debug( + "Non-streaming response from Copilot:", + JSON.stringify(result.response).slice(-400), + ) - if (isNonStreaming(response)) { + let response = result.response + if (isEmptyNonStreamingResponse(response)) { consola.debug( - "Non-streaming response from Copilot:", - JSON.stringify(response).slice(-400), + "Empty non-streaming response detected — returning synthetic fallback", ) - const anthropicResponse = translateToAnthropic(response) + return buildSyntheticFallbackJson(anthropicPayload, response) + } + + if (shouldUsePlainTextCompactionFallback(anthropicPayload, response)) { consola.debug( - "Translated Anthropic response:", - JSON.stringify(anthropicResponse), + "Non-streaming compaction response lacked usable text — retrying with tools stripped", + ) + response = await fetchPlainTextCompactionResponse( + anthropicPayload, + response, + ) + } + + const toolNameMap = createToolNameMapFromAnthropicPayload(anthropicPayload) + const anthropicResponse = translateToAnthropic(response, toolNameMap) + if (result.strippedBase64Chars > 0) { + anthropicResponse.usage.input_tokens += estimateTokensForStrippedImages( + result.strippedBase64Chars, + ) + } + + consola.debug( + "Translated Anthropic response:", + JSON.stringify(anthropicResponse), + ) + return anthropicResponse +} + +async function fetchCompactionResponse( + anthropicPayload: AnthropicMessagesPayload, +): Promise<{ response: ChatCompletionResponse; strippedBase64Chars: number }> { + const result = await fetchWithImageStripping(fetchCopilotResponse, { + ...anthropicPayload, + stream: false, + }) + + if (!isNonStreaming(result.response)) { + throw new Error("Unexpected streaming response during compaction.") + } + + let response = result.response + if (isEmptyNonStreamingResponse(response)) { + response = buildSyntheticCompactionResponse( + anthropicPayload, + response.usage, ) - return c.json(anthropicResponse) + } else if (shouldUsePlainTextCompactionFallback(anthropicPayload, response)) { + response = await fetchPlainTextCompactionResponse( + anthropicPayload, + response, + ) + } + + return { + response, + strippedBase64Chars: result.strippedBase64Chars, } +} + +async function fetchPlainTextCompactionResponse( + payload: AnthropicMessagesPayload, + originalResponse: ChatCompletionResponse, +): Promise<ChatCompletionResponse> { + try { + const fallbackPayload = buildPlainTextCompactionPayload(payload) + const fallbackResult = await fetchWithImageStripping( + fetchCopilotResponse, + fallbackPayload, + ) - consola.debug("Streaming response from Copilot") - return streamSSE(c, async (stream) => { - const streamState: AnthropicStreamState = { - messageStartSent: false, - contentBlockIndex: 0, - contentBlockOpen: false, - toolCalls: {}, + if (!isNonStreaming(fallbackResult.response)) { + return buildSyntheticCompactionResponse(payload, originalResponse.usage) } + if (isEmptyNonStreamingResponse(fallbackResult.response)) { + return buildSyntheticCompactionResponse( + payload, + fallbackResult.response.usage, + ) + } + if (!hasUsableNonStreamingText(fallbackResult.response)) { + return buildSyntheticCompactionResponse( + payload, + fallbackResult.response.usage, + ) + } + return fallbackResult.response + } catch (error) { + consola.warn("Plain-text compaction fallback failed:", error) + return buildSyntheticCompactionResponse(payload, originalResponse.usage) + } +} - for await (const rawEvent of response) { - consola.debug("Copilot raw stream event:", JSON.stringify(rawEvent)) - if (rawEvent.data === "[DONE]") { - break +/** + * Attempts the upstream Copilot fetch with retry logic BEFORE the SSE stream + * is opened. Context-window errors and CompactionNeededError propagate to + * the caller so they can be returned as HTTP-level errors (not SSE events). + */ +async function prefetchCopilotResponse( + anthropicPayload: AnthropicMessagesPayload, +): Promise< + ImageStrippingResult<Awaited<ReturnType<typeof fetchCopilotResponse>>> +> { + let strippingResult: + | ImageStrippingResult<Awaited<ReturnType<typeof fetchCopilotResponse>>> + | undefined + let lastError: unknown + + for (let attempt = 1; attempt <= MAX_FETCH_RETRIES; attempt++) { + try { + strippingResult = await fetchWithImageStripping( + fetchCopilotResponse, + anthropicPayload, + ) + break + } catch (error) { + lastError = error + // HTTPErrors (including context window 400s) propagate immediately + if (error instanceof HTTPError) throw error + // CompactionNeededError propagates immediately + if (error instanceof CompactionNeededError) throw error + const isRetriable = isRetriableFetchError(error) + if (!isRetriable || attempt === MAX_FETCH_RETRIES) throw error + consola.warn( + `Copilot fetch attempt ${attempt}/${MAX_FETCH_RETRIES} failed (${error.message}), retrying…`, + ) + } + } + + if (!strippingResult) throw lastError + return strippingResult +} + +/** + * If the error is an HTTPError with a context-window-exceeded message, + * returns that message string. Otherwise returns undefined. + * Reads and clones the response so the body is still available for + * downstream error handling. + */ +async function extractContextWindowMessage( + error: unknown, +): Promise<string | undefined> { + if (!(error instanceof HTTPError)) return undefined + try { + const cloned = error.response.clone() + const text = await cloned.text() + consola.debug( + `[context-window] Raw upstream error (status=${error.response.status}): ${text.slice(0, 500)}`, + ) + if (isContextWindowError(text)) { + // Try to extract the inner message from JSON + try { + const parsed = JSON.parse(text) as Record<string, unknown> + const errObj = parsed.error as Record<string, unknown> | undefined + if (typeof errObj?.message === "string") { + consola.debug( + `[context-window] Extracted inner message: ${errObj.message}`, + ) + return errObj.message + } + } catch { + // not JSON + } + return text + } + return undefined + } catch { + return undefined + } +} + +async function handleStreaming( + stream: SSEStreamingApi, + anthropicPayload: AnthropicMessagesPayload, + strippingResult: ImageStrippingResult< + Awaited<ReturnType<typeof fetchCopilotResponse>> + >, +): Promise<void> { + try { + const { response: copilotResponse, strippedBase64Chars } = strippingResult + const imageTokenOverhead = + estimateTokensForStrippedImages(strippedBase64Chars) + const toolNameMap = createToolNameMapFromAnthropicPayload(anthropicPayload) + + if (isNonStreaming(copilotResponse)) { + // Shouldn't happen for a streaming payload, but handle gracefully by + // emitting a proper Anthropic SSE event sequence from the non-streaming + // response. Sending just a single "message_start" with the full response + // body causes Claude Code to miss tool call details entirely. + consola.debug( + "Non-streaming response from Copilot (unexpected for streaming request):", + JSON.stringify(copilotResponse).slice(-400), + ) + + // Detect empty non-streaming response and treat like an empty stream — + // retry transparently instead of sending empty content to Claude Code. + if (isEmptyNonStreamingResponse(copilotResponse)) { + consola.debug( + "Empty non-streaming response detected in streaming path — retrying", + ) + const thinkingEnabled = anthropicPayload.thinking?.type === "enabled" + await retryEmptyResponse(stream, anthropicPayload, { + thinkingEnabled, + imageTokenOverhead, + toolNameMap, + }) + return } - if (!rawEvent.data) { + await emitNonStreamingAsSSE(stream, copilotResponse, { + imageTokenOverhead, + toolNameMap, + }) + return + } + + const thinkingEnabled = anthropicPayload.thinking?.type === "enabled" + const hadContent = await pipeStreamToClient(stream, copilotResponse, { + thinkingEnabled, + imageTokenOverhead, + toolNameMap, + }) + + // When the model returns an empty response (reasoning completed but no + // output), retry the request transparently. Since no message_start was + // sent to the client, the SSE connection is clean and we can pipe a new + // response without protocol violations. + if (!hadContent) { + await retryEmptyResponse(stream, anthropicPayload, { + thinkingEnabled, + imageTokenOverhead, + toolNameMap, + }) + } + } catch (error) { + // Errors here occur during stream piping (after the initial fetch + // succeeded). The SSE connection is already committed to HTTP 200, + // so we can only emit SSE error events — not HTTP-level errors. + // Context window errors and CompactionNeededError are already handled + // in handleCompletion (before the SSE stream starts). + consola.error("Error during stream piping:", error) + await emitStreamingError(stream, error, anthropicPayload.model) + } +} + +/** + * Clamps `max_tokens` on the OpenAI payload to the model's actual + * `max_output_tokens` limit. This prevents the upstream API from + * truncating the response mid-tool-call when the client (e.g. Claude Code) + * requests more output tokens than the model supports. + * + * When no `selectedModel` is provided, the function looks up the model + * from `state.models` by `payload.model`. Mutates the payload in-place. + */ +function clampMaxTokens( + payload: import("~/services/copilot/create-chat-completions").ChatCompletionsPayload, + selectedModel?: import("~/services/copilot/get-models").Model, +): void { + const model = + selectedModel ?? state.models?.data.find((m) => m.id === payload.model) + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- some models lack capabilities at runtime + const modelMaxOutput = model?.capabilities?.limits?.max_output_tokens + if ( + modelMaxOutput + && payload.max_tokens + && payload.max_tokens > modelMaxOutput + ) { + consola.debug( + `Clamping max_tokens from ${payload.max_tokens} to model limit ${modelMaxOutput}`, + ) + payload.max_tokens = modelMaxOutput + } +} + +async function fetchCopilotResponse( + anthropicPayload: AnthropicMessagesPayload, +): ReturnType<typeof createChatCompletions> { + if (isWebSearchEnabled() && (await detectWebSearchIntent(anthropicPayload))) { + const cleanedPayload = stripWebSearchTypedTools(anthropicPayload) + const openAIPayload = prepareWebSearchPayload( + translateToOpenAI(cleanedPayload), + ) + clampMaxTokens(openAIPayload) + consola.debug( + "Translated OpenAI request payload (web search):", + JSON.stringify(openAIPayload), + ) + return webSearchInterceptor(openAIPayload) + } + + const openAIPayload = translateToOpenAI(anthropicPayload) + consola.debug( + "Translated OpenAI request payload:", + JSON.stringify(openAIPayload), + ) + + const selectedModel = state.models?.data.find( + (m) => m.id === openAIPayload.model, + ) + clampMaxTokens(openAIPayload, selectedModel) + applyLargeEditGuidance( + openAIPayload, + selectedModel ? getModelMaxOutput(selectedModel) : undefined, + ) + consola.debug( + `[routing] model=${openAIPayload.model} found=${selectedModel !== undefined} requiresResponses=${selectedModel !== undefined && requiresResponsesApi(selectedModel)} endpoints=${JSON.stringify(selectedModel?.supported_endpoints)}`, + ) + if (selectedModel !== undefined && requiresResponsesApi(selectedModel)) { + // createResponsesCompletion returns AsyncIterable<SSEMessage> for streaming, + // which is structurally compatible with AsyncGenerator<ServerSentEventMessage> + // at runtime — both support for-await-of. Cast to align with the return type. + return createResponsesCompletion(openAIPayload) as ReturnType< + typeof createChatCompletions + > + } + + return createChatCompletions(openAIPayload) +} + +/** + * Retries the upstream fetch when the model returned an empty response + * (no content, no tool calls). Since no `message_start` was sent to the + * client yet, the SSE connection is clean and we can transparently pipe + * a fresh response. After all retries are exhausted, sends an error event. + */ +async function retryEmptyResponse( + stream: SSEStreamingApi, + anthropicPayload: AnthropicMessagesPayload, + ctx: { + thinkingEnabled: boolean + imageTokenOverhead: number + toolNameMap: ToolNameMap + }, +): Promise<void> { + for ( + let emptyRetry = 1; + emptyRetry <= MAX_EMPTY_RESPONSE_RETRIES; + emptyRetry++ + ) { + consola.debug( + `Empty response retry ${emptyRetry}/${MAX_EMPTY_RESPONSE_RETRIES}`, + ) + const retryResult = await fetchWithImageStripping( + fetchCopilotResponse, + anthropicPayload, + ) + const { response: retryResponse } = retryResult + + if (isNonStreaming(retryResponse)) { + // Non-streaming retry can also be empty — treat as another empty attempt + // and continue to the next retry rather than emitting empty content. + if (isEmptyNonStreamingResponse(retryResponse)) { + consola.debug( + `Empty non-streaming response on retry ${emptyRetry} — continuing`, + ) continue } + await emitNonStreamingAsSSE(stream, retryResponse, { + imageTokenOverhead: ctx.imageTokenOverhead, + toolNameMap: ctx.toolNameMap, + }) + return + } + + const retryHadContent = await pipeStreamToClient(stream, retryResponse, { + thinkingEnabled: ctx.thinkingEnabled, + imageTokenOverhead: ctx.imageTokenOverhead, + toolNameMap: ctx.toolNameMap, + }) + if (retryHadContent) return + } + + // All retries returned empty — the model persistently refuses to generate + // output for this conversation state. Sending overloaded_error here would + // cause Claude Code to retry the exact same (doomed) request in an infinite + // loop until it gives up and stops the session. Instead, emit a synthetic + // valid assistant response. This allows the conversation to move forward: + // Claude Code sees the model "said something" and can proceed to the next + // turn naturally. + consola.debug( + "All empty response retries exhausted — emitting synthetic fallback response", + ) + await emitSyntheticFallbackResponse(stream, anthropicPayload) +} + +// eslint-disable-next-line complexity +async function pipeStreamToClient( + stream: SSEStreamingApi, + response: AsyncGenerator<ServerSentEventMessage, void, unknown>, + options: { + thinkingEnabled: boolean + imageTokenOverhead?: number + toolNameMap?: ToolNameMap + }, +): Promise<boolean> { + const { thinkingEnabled, imageTokenOverhead = 0, toolNameMap } = options + const streamState: AnthropicStreamState = { + messageStartSent: false, + messageStopSent: false, + contentBlockIndex: 0, + contentBlockOpen: false, + thinkingBlockOpen: false, + hasEmittedText: false, + toolCalls: {}, + toolNameMap, + thinkingEnabled, + } + + // Ping while waiting between chunks to keep the connection alive. + const schedulePing = () => + setTimeout(() => { + consola.debug("Sending SSE ping between chunks") + stream + .writeSSE({ event: "ping", data: JSON.stringify({ type: "ping" }) }) + .catch(() => {}) + }, PING_INTERVAL_MS) + + let chunkTimer: ReturnType<typeof setTimeout> | undefined = schedulePing() + + try { + // Instead of `for await (const rawEvent of response)` which blocks + // indefinitely when the upstream never closes, we manually iterate with + // a stall timeout. This lets us break out and synthesize proper + // termination events when the Copilot API hangs after a large tool call. + for (;;) { + clearTimeout(chunkTimer) + chunkTimer = schedulePing() + + const rawEvent = await nextWithTimeout(response, streamState) + + // Timeout or natural end of stream + if (rawEvent === undefined) break + + consola.debug("Copilot raw stream event:", JSON.stringify(rawEvent)) + if (rawEvent.data === "[DONE]") break + if (!rawEvent.data) continue const chunk = JSON.parse(rawEvent.data) as ChatCompletionChunk + + // Detect empty responses before sending message_start: some models + // (notably Gemini) return a single chunk with finish_reason "stop", + // no content, and no tool calls after completing their reasoning phase. + // If we haven't sent message_start yet, we can safely signal the + // caller to retry instead of sending an empty turn to Claude Code. + if (!streamState.messageStartSent && isEmptyStreamResponse(chunk)) { + consola.debug( + "Empty response detected from model — signaling for retry", + ) + return false + } + const events = translateChunkToAnthropicEvents(chunk, streamState) for (const event of events) { + // Inflate input_tokens to account for images stripped before sending. + if (imageTokenOverhead > 0) { + inflateEventInputTokens(event, imageTokenOverhead) + } consola.debug("Translated Anthropic event:", JSON.stringify(event)) await stream.writeSSE({ event: event.type, data: JSON.stringify(event), }) } + + // Once finish_reason has been deferred AND we've consumed the usage + // chunk (or exhausted the stream), flush the deferred message_delta + + // message_stop. We continue reading after the finish_reason chunk to + // capture the usage-only chunk that arrives with `stream_options`. + if (streamState.messageStopSent) break + if ( + streamState.deferredFinishReason !== undefined + && streamState.lastSeenUsage + ) { + await emitDeferredFinish(stream, streamState, imageTokenOverhead) + break + } + } + + // If the stream ended (DONE / timeout) before usage arrived, flush + // the deferred finish with whatever usage we have (possibly 0). + if (streamState.deferredFinishReason !== undefined) { + await emitDeferredFinish(stream, streamState, imageTokenOverhead) + } + + await handleIncompleteStream(stream, streamState) + } catch (error) { + consola.error("Stream error from Copilot:", error) + + if (streamState.contentBlockOpen) { + await stream.writeSSE({ + event: "content_block_stop", + data: JSON.stringify({ + type: "content_block_stop", + index: streamState.contentBlockIndex, + }), + }) } + + const errorMessage = + error instanceof Error ? + error.message + : "An unexpected error occurred during streaming." + const errorEvent = translateErrorToAnthropicErrorEvent(errorMessage) + await stream.writeSSE({ + event: errorEvent.type, + data: JSON.stringify(errorEvent), + }) + } finally { + clearTimeout(chunkTimer) + } + return true +} + +/** + * Flushes deferred `message_delta` + `message_stop` events to the SSE stream. + * Applies image token overhead inflation if needed. + */ +async function emitDeferredFinish( + stream: SSEStreamingApi, + streamState: AnthropicStreamState, + imageTokenOverhead: number, +): Promise<void> { + const finishEvents = flushDeferredFinish(streamState) + for (const event of finishEvents) { + if (imageTokenOverhead > 0) { + inflateEventInputTokens(event, imageTokenOverhead) + } + consola.debug("Deferred finish event:", JSON.stringify(event)) + await stream.writeSSE({ + event: event.type, + data: JSON.stringify(event), + }) + } +} + +/** + * Pulls the next value from an async iterator with a stall timeout. + * + * Returns the next yielded value, or `undefined` if either: + * - The iterator is done (natural end of stream), OR + * - The iterator has been stalled for STREAM_STALL_TIMEOUT_MS while the + * stream has already started (messageStartSent === true). + * + * The stall timeout is the key fix for the Write tool hang: when the Copilot + * API finishes streaming a large tool call but never sends `finish_reason`, + * `response.next()` blocks forever on `reader.read()`. Racing it against a + * 90-second timeout lets us break out and synthesize the missing termination + * events (via handleIncompleteStream). The downstream stays alive via + * periodic ping events, so the timeout can be generous enough to accommodate + * models like Gemini that pause for 60-90s during deep reasoning. + */ +async function nextWithTimeout( + iter: AsyncGenerator<ServerSentEventMessage, void, unknown>, + streamState: AnthropicStreamState, +): Promise<ServerSentEventMessage | undefined> { + // Before the stream has started we don't apply a stall timeout — + // the initial response from Copilot can take a long time (model + // thinking) and is covered by the ping keepalive + upstream inactivity + // abort instead. + if (!streamState.messageStartSent) { + const result = await iter.next() + return result.done ? undefined : result.value + } + + // Race the next chunk against a stall timeout. + const stallTimeout = new Promise<"timeout">((resolve) => + setTimeout(() => resolve("timeout"), STREAM_STALL_TIMEOUT_MS), + ) + + const result = await Promise.race([iter.next(), stallTimeout]) + + if (result === "timeout") { + consola.debug( + `Upstream stream stalled for ${STREAM_STALL_TIMEOUT_MS / 1000}s after ` + + `message_start — synthesizing termination events`, + ) + return undefined + } + + return result.done ? undefined : result.value +} + +/** + * Handles the case where the upstream stream ended without a proper Anthropic + * termination sequence (message_delta + message_stop). + * + * Two scenarios: + * 1. Stream never produced any content → emit a synthetic error event. + * 2. Stream started (message_start sent) but ended without finish_reason → + * synthesize the missing termination events so Claude Code can proceed. + */ +async function handleIncompleteStream( + stream: SSEStreamingApi, + state: AnthropicStreamState, +): Promise<void> { + if (!state.messageStartSent) { + // No usable chunks arrived at all. + consola.debug( + "Copilot stream ended without producing any content — emitting error event", + ) + const errorEvent = translateErrorToAnthropicErrorEvent( + "The model returned an empty response. This may indicate the model is unavailable or does not support this request.", + ) + await stream.writeSSE({ + event: errorEvent.type, + data: JSON.stringify(errorEvent), + }) + return + } + + if (state.messageStopSent) { + return // Stream ended normally, nothing to do. + } + + // The upstream stream started but ended without a chunk containing + // finish_reason — no message_delta / message_stop was ever sent. + // Some models (notably Gemini) can terminate the stream abruptly after + // emitting content or tool-call chunks. Without a proper termination + // sequence Claude Code sees the SSE connection close with no indication + // of completion and treats the turn as abandoned / silently dead. + consola.debug( + "Copilot stream ended without finish_reason — synthesizing message_delta/message_stop", + ) + + if (state.contentBlockOpen) { + await stream.writeSSE({ + event: "content_block_stop", + data: JSON.stringify({ + type: "content_block_stop", + index: state.contentBlockIndex, + }), + }) + } + + // Check if any tool calls have truncated (invalid) JSON arguments. + // This happens when the output token limit was hit mid-tool-call — the + // accumulated argument fragments don't form valid JSON. In this case, + // emit an explanatory text block and use "end_turn" instead of "tool_use" + // so Claude Code reads the feedback instead of executing a broken tool. + const hasToolCalls = Object.keys(state.toolCalls).length > 0 + const truncated = hasToolCalls ? findTruncatedToolCalls(state) : [] + + if (truncated.length > 0) { + const toolName = truncated[0].name + consola.debug( + `Truncated tool call "${toolName}" detected during stream recovery`, + ) + const nextIndex = state.contentBlockIndex + 1 + await stream.writeSSE({ + event: "content_block_start", + data: JSON.stringify({ + type: "content_block_start", + index: nextIndex, + content_block: { type: "text", text: "" }, + }), + }) + await stream.writeSSE({ + event: "content_block_delta", + data: JSON.stringify({ + type: "content_block_delta", + index: nextIndex, + delta: { + type: "text_delta", + text: + `[Output truncated: the model's response was cut off while generating` + + ` tool call "${toolName}". The output exceeded the token limit.` + + ` Please retry with a smaller output, e.g. write the file in smaller chunks.]`, + }, + }), + }) + await stream.writeSSE({ + event: "content_block_stop", + data: JSON.stringify({ type: "content_block_stop", index: nextIndex }), + }) + } + + // Use "end_turn" when tool calls are truncated to prevent Claude Code + // from trying to execute broken tool calls. Use "tool_use" only when + // tool calls have valid (non-truncated) JSON arguments. + let stopReason: string = "end_turn" + if (hasToolCalls && truncated.length === 0) { + stopReason = "tool_use" + } + + await stream.writeSSE({ + event: "message_delta", + data: JSON.stringify({ + type: "message_delta", + delta: { + stop_reason: stopReason, + stop_sequence: null, + }, + usage: { + input_tokens: 0, + output_tokens: 0, + }, + }), + }) + await stream.writeSSE({ + event: "message_stop", + data: JSON.stringify({ type: "message_stop" }), }) } const isNonStreaming = ( response: Awaited<ReturnType<typeof createChatCompletions>>, ): response is ChatCompletionResponse => Object.hasOwn(response, "choices") + +/** + * Detects whether a non-streaming response is effectively empty. + * + * Some models (notably Gemini) return finish_reason "stop" with empty or null + * content and 0 completion tokens after their reasoning phase completes without + * producing output. Without this guard, the empty response gets translated to + * a valid Anthropic message with stop_reason "end_turn" and empty content, + * causing Claude Code to treat the model's turn as complete and stop the session. + */ +export function isEmptyNonStreamingResponse( + response: ChatCompletionResponse, +): boolean { + // No choices at all: upstream produced nothing usable. Treat as empty so + // the caller emits the synthetic fallback instead of a degenerate + // { content: [], stop_reason: null } body. + if (response.choices.length === 0) return true + const choice = response.choices[0] + // A normal completed turn finishes with "stop". Anything that finished with + // a real reason other than "stop" (tool_calls/length/content_filter) is not + // "empty" — let the normal translation path handle it. + // + // The remaining cases we DO treat as empty: finish_reason === "stop" with no + // content, and finish_reason null/undefined (a degenerate shape some models + // emit via Copilot) with no content. Both would otherwise translate to an + // empty content array, which Anthropic clients reject. + // + // ChoiceNonStreaming types finish_reason as non-null, but Copilot violates + // that at runtime (the bug this guard exists for), so widen the type to + // include the nullish shapes we actually observe before comparing. + const finishReason = choice.finish_reason as + | "stop" + | "length" + | "tool_calls" + | "content_filter" + | null + | undefined + if ( + finishReason !== "stop" + && finishReason !== null + && finishReason !== undefined + ) { + return false + } + if (choice.message.tool_calls && choice.message.tool_calls.length > 0) { + return false + } + // Content is empty if null, undefined, or empty string + const content = choice.message.content + return !content || content.trim() === "" +} + +/** + * Maps an HTTP status code to the corresponding Anthropic error type. + * Claude Code uses the error type to decide whether a request can be retried. + */ +function mapStatusToAnthropicErrorType(status: number): string { + if (status === 429) return "rate_limit_error" + if (status >= 400 && status < 500) return "invalid_request_error" + if (status >= 500) return "api_error" + return "api_error" +} + +/** + * Emits an SSE error event for a streaming request. + * + * Generally uses "api_error" to prevent Claude Code from retrying in a loop + * (the HTTP response is already committed to status 200). However, when the + * upstream explicitly says the input exceeds the model's context window, we + * use "invalid_request_error" so Claude Code triggers auto-compaction — + * compaction reduces the conversation context, which fixes the root cause. + */ +async function emitStreamingError( + stream: SSEStreamingApi, + error: unknown, + modelId?: string, +): Promise<void> { + const { errorMessage, errorType } = await extractStreamingErrorDetails(error) + + const contextWindowError = isContextWindowError(errorMessage) + const effectiveErrorType = + contextWindowError ? "invalid_request_error" : errorType + const modelLimit = modelId ? lookupModelLimit(modelId) : undefined + const effectiveMessage = + contextWindowError ? + formatAnthropicContextWindowError(errorMessage, modelLimit) + : errorMessage + + const errorEvent = translateErrorToAnthropicErrorEvent( + effectiveMessage, + effectiveErrorType, + ) + await stream.writeSSE({ + event: errorEvent.type, + data: JSON.stringify(errorEvent), + }) +} + +/** + * Extracts a meaningful error message and Anthropic-compatible error type + * from a Copilot error. For HTTPErrors this reads the response body to + * get the Copilot-provided message; for network errors it falls back to + * the generic Error message. + */ +async function extractStreamingErrorDetails(error: unknown): Promise<{ + errorMessage: string + errorType: string +}> { + if (error instanceof HTTPError) { + const errorType = mapStatusToAnthropicErrorType(error.response.status) + try { + const cloned = error.response.clone() + const text = await cloned.text() + let parsed: unknown + try { + parsed = JSON.parse(text) as Record<string, unknown> + } catch { + parsed = null + } + return { + errorMessage: extractUpstreamErrorMessage( + parsed, + text, + error.response.headers.get("content-type"), + ), + errorType, + } + } catch { + return { errorMessage: error.message, errorType } + } + } + + return { + errorMessage: + error instanceof Error ? + error.message + : "An unexpected error occurred during streaming.", + errorType: "api_error", + } +} + +/** + * Emits a proper Anthropic SSE event sequence from a non-streaming Copilot + * response. This is needed when Copilot unexpectedly returns a non-streaming + * body for a streaming request — sending the full response as a single + * "message_start" event causes Claude Code to miss all tool call input details. + */ +type EmitNonStreamingAsSSEOptions = { + imageTokenOverhead?: number + toolNameMap?: ToolNameMap +} + +async function emitNonStreamingAsSSE( + stream: SSEStreamingApi, + response: ChatCompletionResponse, + { imageTokenOverhead = 0, toolNameMap }: EmitNonStreamingAsSSEOptions = {}, +): Promise<void> { + const anthropicResponse = translateToAnthropic(response, toolNameMap) + + // 1. message_start (without content, stop_reason, stop_sequence) + await stream.writeSSE({ + event: "message_start", + data: JSON.stringify({ + type: "message_start", + message: { + id: anthropicResponse.id, + type: "message", + role: "assistant", + content: [], + model: anthropicResponse.model, + stop_reason: null, + stop_sequence: null, + usage: { + input_tokens: + anthropicResponse.usage.input_tokens + imageTokenOverhead, + output_tokens: 0, + ...(anthropicResponse.usage.cache_read_input_tokens !== undefined && { + cache_read_input_tokens: + anthropicResponse.usage.cache_read_input_tokens, + }), + }, + }, + }), + }) + + // 2. Emit each content block as start + delta + stop + for (let i = 0; i < anthropicResponse.content.length; i++) { + const block = anthropicResponse.content[i] + const blockIndex = i + + if (block.type === "text") { + await stream.writeSSE({ + event: "content_block_start", + data: JSON.stringify({ + type: "content_block_start", + index: blockIndex, + content_block: { type: "text", text: "" }, + }), + }) + await stream.writeSSE({ + event: "content_block_delta", + data: JSON.stringify({ + type: "content_block_delta", + index: blockIndex, + delta: { type: "text_delta", text: block.text }, + }), + }) + } else if (block.type === "tool_use") { + await stream.writeSSE({ + event: "content_block_start", + data: JSON.stringify({ + type: "content_block_start", + index: blockIndex, + content_block: { + type: "tool_use", + id: block.id, + name: block.name, + input: {}, + }, + }), + }) + const inputJson = JSON.stringify(block.input) + if (inputJson !== "{}") { + await stream.writeSSE({ + event: "content_block_delta", + data: JSON.stringify({ + type: "content_block_delta", + index: blockIndex, + delta: { type: "input_json_delta", partial_json: inputJson }, + }), + }) + } + } + + await stream.writeSSE({ + event: "content_block_stop", + data: JSON.stringify({ type: "content_block_stop", index: blockIndex }), + }) + } + + // 3. message_delta + message_stop + await stream.writeSSE({ + event: "message_delta", + data: JSON.stringify({ + type: "message_delta", + delta: { + stop_reason: anthropicResponse.stop_reason, + stop_sequence: anthropicResponse.stop_sequence, + }, + usage: { + output_tokens: anthropicResponse.usage.output_tokens, + }, + }), + }) + await stream.writeSSE({ + event: "message_stop", + data: JSON.stringify({ type: "message_stop" }), + }) +} + +// -- Synthetic fallback for persistent empty responses ----------------------- +// +// When a model (notably Gemini) persistently returns empty output for a given +// conversation state, retrying the same request is futile. Returning an error +// (overloaded_error) causes Claude Code to retry the same doomed request in +// an infinite loop until it exhausts its retry budget and stops the session. +// +// The solution: emit a *valid* assistant response with text explaining that the +// model produced no output. This is saved to conversation history and allows +// Claude Code to proceed — it will see the assistant's "turn" is complete and +// can issue the next turn (which has a different conversation state and often +// succeeds). + +const FALLBACK_TEXT = + "I apologize, but I was unable to generate a response for this turn. " + + "Let me try a different approach." + +/** + * Emits a synthetic Anthropic SSE event sequence that represents a valid + * assistant message containing a fallback text. This unblocks the + * conversation so Claude Code can proceed to the next turn. + */ +async function emitSyntheticFallbackResponse( + stream: SSEStreamingApi, + anthropicPayload: AnthropicMessagesPayload, +): Promise<void> { + const msgId = `msg_fallback_${Date.now()}` + const model = anthropicPayload.model + + await stream.writeSSE({ + event: "message_start", + data: JSON.stringify({ + type: "message_start", + message: { + id: msgId, + type: "message", + role: "assistant", + content: [], + model, + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 0, output_tokens: 0 }, + }, + }), + }) + await stream.writeSSE({ + event: "content_block_start", + data: JSON.stringify({ + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "" }, + }), + }) + await stream.writeSSE({ + event: "content_block_delta", + data: JSON.stringify({ + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: FALLBACK_TEXT }, + }), + }) + await stream.writeSSE({ + event: "content_block_stop", + data: JSON.stringify({ type: "content_block_stop", index: 0 }), + }) + await stream.writeSSE({ + event: "message_delta", + data: JSON.stringify({ + type: "message_delta", + delta: { stop_reason: "end_turn", stop_sequence: null }, + usage: { output_tokens: 1 }, + }), + }) + await stream.writeSSE({ + event: "message_stop", + data: JSON.stringify({ type: "message_stop" }), + }) +} + +/** + * Builds a non-streaming Anthropic JSON response with fallback text. + * Used by the non-streaming handler when the model returns empty. + */ +function buildSyntheticFallbackJson( + anthropicPayload: AnthropicMessagesPayload, + response: ChatCompletionResponse, +): import("./anthropic-types").AnthropicResponse { + return { + id: toAnthropicMessageId(response.id), + type: "message", + role: "assistant", + model: anthropicPayload.model, + content: [{ type: "text", text: FALLBACK_TEXT }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { + input_tokens: response.usage?.prompt_tokens ?? 0, + output_tokens: 1, + }, + } +} diff --git a/src/routes/messages/image-stripping.ts b/src/routes/messages/image-stripping.ts new file mode 100644 index 000000000..5fe7db184 --- /dev/null +++ b/src/routes/messages/image-stripping.ts @@ -0,0 +1,341 @@ +import consola from "consola" + +import { HTTPError } from "~/lib/error" +import { extractSessionId } from "~/lib/session-id" + +import type { + AnthropicDocumentBlock, + AnthropicImageBlock, + AnthropicMessagesPayload, + AnthropicTextBlock, +} from "./anthropic-types" + +/** + * Thrown when the 413 retry cascade is exhausted (all images stripped, + * request still too large). Signals the handler to return an + * `invalid_request_error` that triggers Claude Code auto-compaction. + */ +export class CompactionNeededError extends Error { + constructor() { + super("Request too large even after stripping all images") + this.name = "CompactionNeededError" + } +} + +/** + * Tracks which sessions have had images proactively stripped. + * Keyed by session ID (8-char hex hash from metadata.user_id). + * When a session's images are stripped, its ID is added here. + * When that session later sends a `/v1/messages` with zero images + * (meaning compaction removed them), its ID is removed. + * + * This is per-session to prevent cross-session contamination: + * Session A stripping images should NOT cause Session B to compact. + */ +const sessionsWithStrippedImages = new Set<string>() + +/** + * Returns true if the given session has had images stripped and + * `count_tokens` should return an inflated value to trigger compaction. + * Returns false for unknown sessions or sessions without the flag. + */ +export function hasStrippedImages(sessionId: string | undefined): boolean { + if (!sessionId) return false + return sessionsWithStrippedImages.has(sessionId) +} + +/** + * Extracts the session ID from an Anthropic payload's metadata. + * Convenience wrapper so callers don't need to import extractSessionId separately. + */ +export function getSessionId( + payload: AnthropicMessagesPayload, +): string | undefined { + return extractSessionId(payload.metadata?.user_id) +} + +/** + * Called by the messages handler at the start of every `/v1/messages` + * request. If the incoming payload has no base64 images AND this + * session previously had images stripped, compaction has succeeded + * and the per-session flag is cleared so `count_tokens` stops + * returning the inflated 200K value. + */ +export function updateImageFlag(payload: AnthropicMessagesPayload): void { + const sessionId = getSessionId(payload) + if (!sessionId || !sessionsWithStrippedImages.has(sessionId)) return + + const hasBase64Images = payload.messages.some((msg) => { + if (msg.role !== "user") return false + if (typeof msg.content === "string") return false + return msg.content.some((block) => { + if (block.type === "image" && block.source.type === "base64") return true + if (block.type === "tool_result" && Array.isArray(block.content)) { + return block.content.some( + (nested) => + nested.type === "image" && nested.source.type === "base64", + ) + } + return false + }) + }) + + if (!hasBase64Images) { + consola.debug( + `[${sessionId}] No images in conversation — compaction succeeded, clearing image flag.`, + ) + sessionsWithStrippedImages.delete(sessionId) + } +} + +/** Reference to a single image block within its parent array. */ +type ImageRef = { + parent: Array<unknown> + index: number + base64Length: number + messageIndex: number + processed: boolean +} + +/** Collects image refs from a tool_result's nested content array. */ +function collectToolResultImages( + content: Array< + AnthropicTextBlock | AnthropicImageBlock | AnthropicDocumentBlock + >, + refs: Array<ImageRef>, + messageIndex: number, +): void { + for (let j = 0; j < content.length; j++) { + const nested = content[j] + if (nested.type === "image") { + if (nested.source.type !== "base64") continue // URL images: no base64 data to strip + refs.push({ + parent: content as Array<unknown>, + index: j, + base64Length: nested.source.data.length, + messageIndex, + processed: false, + }) + } + } +} + +function parseImageTrimmingMessageThreshold(): number { + const raw = process.env.IMAGE_CONTEXT_TRIMMING_BEFORE_MESSAGES + if (!raw) return 6 + const parsed = Number.parseInt(raw, 10) + return Number.isFinite(parsed) && parsed >= 0 ? parsed : 6 +} + +function isImageContextTrimmingEnabled(): boolean { + const raw = process.env.IMAGE_CONTEXT_TRIMMING_ENABLED?.trim().toLowerCase() + return raw === "1" || raw === "true" || raw === "yes" || raw === "on" +} + +function messageHasAssistantText( + message: AnthropicMessagesPayload["messages"][number], +): boolean { + if (message.role !== "assistant") return false + if (typeof message.content === "string") { + return message.content.trim().length > 0 + } + + return message.content.some( + (block) => block.type === "text" && block.text.trim().length > 0, + ) +} + +function collectImageRefs(payload: AnthropicMessagesPayload): Array<ImageRef> { + const imageRefs: Array<ImageRef> = [] + const pendingRefs: Array<ImageRef> = [] + + for (const [messageIndex, message] of payload.messages.entries()) { + if (message.role === "user" && typeof message.content !== "string") { + for (let i = 0; i < message.content.length; i++) { + const block = message.content[i] + + if (block.type === "image" && block.source.type === "base64") { + const ref = { + parent: message.content as Array<unknown>, + index: i, + base64Length: block.source.data.length, + messageIndex, + processed: false, + } + imageRefs.push(ref) + pendingRefs.push(ref) + } + + if (block.type === "tool_result" && Array.isArray(block.content)) { + const before = imageRefs.length + collectToolResultImages(block.content, imageRefs, messageIndex) + pendingRefs.push(...imageRefs.slice(before)) + } + } + } + + if (messageHasAssistantText(message)) { + for (const ref of pendingRefs) { + ref.processed = true + } + pendingRefs.length = 0 + } + } + + return imageRefs +} + +function trimProcessedImages(payload: AnthropicMessagesPayload): { + payload: AnthropicMessagesPayload + trimmedCount: number +} { + if (!isImageContextTrimmingEnabled()) { + return { payload, trimmedCount: 0 } + } + + const threshold = parseImageTrimmingMessageThreshold() + const cloned = structuredClone(payload) + const imageRefs = collectImageRefs(cloned) + const lastMessageIndex = cloned.messages.length - 1 + const placeholder = { + type: "text" as const, + text: "[Processed image trimmed to reduce request size]", + } + + let trimmedCount = 0 + for (const ref of imageRefs) { + const laterMessages = lastMessageIndex - ref.messageIndex + if (!ref.processed || laterMessages < threshold) continue + ref.parent[ref.index] = placeholder + trimmedCount += 1 + } + + return { + payload: trimmedCount > 0 ? cloned : payload, + trimmedCount, + } +} + +/** + * Deep-clones the payload and replaces base64 image blocks with text + * placeholders. When `keepLast` is true and 2+ images exist, the last + * image (most recent in conversation order) is preserved. + * + * Returns the cloned (possibly mutated) payload, the count of images + * actually replaced, and the total base64 character count removed. + */ +function stripImages( + payload: AnthropicMessagesPayload, + keepLast: boolean, +): { + payload: AnthropicMessagesPayload + strippedCount: number + strippedBase64Chars: number +} { + // Deep-clone to avoid mutating the original + const cloned = structuredClone(payload) + const imageRefs = collectImageRefs(cloned) + + // Determine which images to strip + const toStrip = + keepLast && imageRefs.length > 1 ? imageRefs.slice(0, -1) : imageRefs + + const placeholder = { + type: "text" as const, + text: "[Image removed to reduce request size]", + } + + let strippedBase64Chars = 0 + for (const ref of toStrip) { + strippedBase64Chars += ref.base64Length + ref.parent[ref.index] = placeholder + } + + return { payload: cloned, strippedCount: toStrip.length, strippedBase64Chars } +} + +/** Result of fetchWithImageStripping, includes stripped image metadata. */ +export interface ImageStrippingResult<T> { + response: T + /** Total base64 characters removed from the payload before sending. */ + strippedBase64Chars: number +} + +/** + * Wraps a Copilot fetch function with reactive 413 retry logic. + * + * All images are sent as-is on the first attempt, letting Claude Code + * manage its own context (including deciding when to compact and which + * images to keep). The proxy only intervenes when Copilot's HTTP body + * size limit rejects the request with 413. + * + * Returns the response along with metadata about how many base64 + * characters were stripped, so the caller can inflate usage tokens. + * + * Cascade: + * 1. Send payload unchanged (all images intact) + * 2. On 413: strip older images (keep most recent), retry + * 3. On 413: strip ALL images, retry + * 4. On 413 with no images left: throw CompactionNeededError + * + * Non-413 HTTPErrors and non-HTTP errors propagate immediately. + */ +export async function fetchWithImageStripping<T>( + fetchFn: (payload: AnthropicMessagesPayload) => Promise<T>, + anthropicPayload: AnthropicMessagesPayload, +): Promise<ImageStrippingResult<T>> { + const proactivelyTrimmed = trimProcessedImages(anthropicPayload) + const sessionId = getSessionId(proactivelyTrimmed.payload) + if (proactivelyTrimmed.trimmedCount > 0) { + consola.debug( + `${sessionId ? `[${sessionId}] ` : ""}Trimmed ${proactivelyTrimmed.trimmedCount} processed image(s) before upstream request.`, + ) + } + + // Stage 1: Try with all images intact — let the model see everything. + try { + const response = await fetchFn(proactivelyTrimmed.payload) + return { response, strippedBase64Chars: 0 } + } catch (error) { + if (!is413(error)) throw error + } + + // Stage 2: Strip older images, keep the most recent one + const stage2 = stripImages(proactivelyTrimmed.payload, true) + if (stage2.strippedCount > 0) { + consola.debug( + `${sessionId ? `[${sessionId}] ` : ""}413 — stripped ${stage2.strippedCount} older image(s) (keeping last), retrying.`, + ) + try { + const response = await fetchFn(stage2.payload) + return { response, strippedBase64Chars: stage2.strippedBase64Chars } + } catch (error) { + if (!is413(error)) throw error + } + } + + // Stage 3: Strip ALL images + const stage3 = stripImages(proactivelyTrimmed.payload, false) + if (stage3.strippedCount > 0) { + if (sessionId) sessionsWithStrippedImages.add(sessionId) + consola.warn( + `${sessionId ? `[${sessionId}] ` : ""}413 — stripped all ${stage3.strippedCount} image(s), retrying.`, + ) + try { + const response = await fetchFn(stage3.payload) + return { response, strippedBase64Chars: stage3.strippedBase64Chars } + } catch (error) { + if (!is413(error)) throw error + } + } + + // Stage 4: No images left, request is still too large — trigger compaction + consola.warn( + "Still too large (413) even without images, triggering auto-compaction", + ) + throw new CompactionNeededError() +} + +function is413(error: unknown): boolean { + return error instanceof HTTPError && error.response.status === 413 +} diff --git a/src/routes/messages/image-validation.ts b/src/routes/messages/image-validation.ts new file mode 100644 index 000000000..3c0cc1388 --- /dev/null +++ b/src/routes/messages/image-validation.ts @@ -0,0 +1,95 @@ +import type { + AnthropicImageBlock, + AnthropicMessagesPayload, + AnthropicToolResultBlock, +} from "./anthropic-types" + +const MIN_IMAGE_DIMENSION = 4 + +type InvalidImage = { + mediaType: string + width: number + height: number +} + +export function findInvalidEmbeddedImage( + payload: AnthropicMessagesPayload, +): InvalidImage | undefined { + for (const message of payload.messages) { + if (message.role !== "user" || typeof message.content === "string") continue + + for (const block of message.content) { + if (block.type === "image") { + const invalid = getInvalidImageReason(block) + if (invalid) return invalid + } + + if (block.type === "tool_result" && Array.isArray(block.content)) { + const invalid = findInvalidImageInToolResult(block) + if (invalid) return invalid + } + } + } + + return undefined +} + +function findInvalidImageInToolResult( + block: AnthropicToolResultBlock, +): InvalidImage | undefined { + if (typeof block.content === "string") return undefined + + for (const nested of block.content) { + if (nested.type !== "image") continue + const invalid = getInvalidImageReason(nested) + if (invalid) return invalid + } + + return undefined +} + +function getInvalidImageReason( + block: AnthropicImageBlock, +): InvalidImage | undefined { + if (block.source.type !== "base64") return undefined // URL images: can't validate dimensions + + const size = + block.source.media_type === "image/png" ? + readPngSize(block.source.data) + : undefined + + if (!size) return undefined + if (size.width >= MIN_IMAGE_DIMENSION && size.height >= MIN_IMAGE_DIMENSION) { + return undefined + } + + return { + mediaType: block.source.media_type, + width: size.width, + height: size.height, + } +} +function readPngSize( + base64: string, +): { width: number; height: number } | undefined { + let bytes: Uint8Array + try { + bytes = Uint8Array.from(Buffer.from(base64, "base64")) + } catch { + return undefined + } + + if (bytes.length < 24) return undefined + + const pngSignature = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a] + if (!pngSignature.every((value, index) => bytes[index] === value)) { + return undefined + } + + const width = + (bytes[16] << 24) | (bytes[17] << 16) | (bytes[18] << 8) | bytes[19] + const height = + (bytes[20] << 24) | (bytes[21] << 16) | (bytes[22] << 8) | bytes[23] + + return { width: width >>> 0, height: height >>> 0 } +} diff --git a/src/routes/messages/large-edit-guidance.ts b/src/routes/messages/large-edit-guidance.ts new file mode 100644 index 000000000..9c9b7655f --- /dev/null +++ b/src/routes/messages/large-edit-guidance.ts @@ -0,0 +1,89 @@ +import type { ChatCompletionsPayload } from "~/services/copilot/create-chat-completions" + +const FILE_EDIT_TOOL_NAMES = new Set([ + "Edit", + "MultiEdit", + "NotebookEdit", + "Write", +]) + +const GUIDANCE_MARKER = "File-editing budget:" +const STRONG_GUIDANCE_MARKER = "High-risk large edit detected:" + +const RISKY_REQUEST_PATTERNS = [ + /\bcomplete\b/i, + /\bcomprehensive\b/i, + /\bentire\b/i, + /\bfull file\b/i, + /\bwhole file\b/i, + /\bone giant\b/i, + /\bone shot\b/i, + /\bsingle (?:write|edit|tool call|operation)\b/i, + /\bdo not split\b/i, + /\bexactly once\b/i, + /\bthousands? of lines\b/i, + /\b\d{4,}\s+lines\b/i, + /\brewrite the file\b/i, + /\bwrite the complete\b/i, +] + +function hasFileEditTool(payload: ChatCompletionsPayload): boolean { + return ( + payload.tools?.some((tool) => FILE_EDIT_TOOL_NAMES.has(tool.function.name)) + ?? false + ) +} + +function alreadyHasGuidance(payload: ChatCompletionsPayload): boolean { + return payload.messages.some( + (message) => + message.role === "system" + && typeof message.content === "string" + && (message.content.includes(GUIDANCE_MARKER) + || message.content.includes(STRONG_GUIDANCE_MARKER)), + ) +} + +function isRiskyLargeEditRequest(payload: ChatCompletionsPayload): boolean { + return payload.messages.some((message) => { + if (message.role !== "user") return false + if (typeof message.content !== "string") return false + const content = message.content + return RISKY_REQUEST_PATTERNS.some((pattern) => pattern.test(content)) + }) +} + +function buildGuidance(modelMaxOutput: number, strong: boolean): string { + const base = + `${GUIDANCE_MARKER} this model can emit about ${modelMaxOutput.toLocaleString()} output tokens in one turn. ` + + "When using Write/Edit/MultiEdit-style tools, never attempt one giant file rewrite. " + if (!strong) { + return ( + base + + "For large file creation or large edits, split the work into multiple smaller tool calls " + + "(for example staged edits, chunked appends, or a script written in pieces) so each tool call stays well below the output limit." + ) + } + + return ( + `${STRONG_GUIDANCE_MARKER} the user request looks likely to overflow a single tool call on this model. ` + + base + + "Do not satisfy this request with one massive Write/Edit/MultiEdit call, even if the user asks for a complete file in one step. " + + "Instead, choose a chunked strategy: create a scaffold first, then append or patch the file in multiple sequential tool calls, " + + "or generate a helper script and write that script in smaller pieces." + ) +} + +export function applyLargeEditGuidance( + payload: ChatCompletionsPayload, + modelMaxOutput: number | undefined, +): void { + if (!modelMaxOutput || modelMaxOutput > 32_000) return + if (!hasFileEditTool(payload) || alreadyHasGuidance(payload)) return + const strongGuidance = isRiskyLargeEditRequest(payload) + + payload.messages.unshift({ + role: "system", + content: buildGuidance(modelMaxOutput, strongGuidance), + }) +} diff --git a/src/routes/messages/non-stream-translation.ts b/src/routes/messages/non-stream-translation.ts index dc41e6382..9bbbb1a92 100644 --- a/src/routes/messages/non-stream-translation.ts +++ b/src/routes/messages/non-stream-translation.ts @@ -1,3 +1,6 @@ +import consola from "consola" + +import { repairOrphanedToolCalls } from "~/lib/tool-call-repair" import { type ChatCompletionResponse, type ChatCompletionsPayload, @@ -11,9 +14,14 @@ import { import { type AnthropicAssistantContentBlock, type AnthropicAssistantMessage, + type AnthropicCustomTool, type AnthropicMessage, type AnthropicMessagesPayload, + type AnthropicRedactedThinkingBlock, type AnthropicResponse, + type AnthropicServerToolResultBlock, + type AnthropicServerToolUseBlock, + type AnthropicSystemBlock, type AnthropicTextBlock, type AnthropicThinkingBlock, type AnthropicTool, @@ -21,58 +29,279 @@ import { type AnthropicToolUseBlock, type AnthropicUserContentBlock, type AnthropicUserMessage, + isTypedTool, } from "./anthropic-types" -import { mapOpenAIStopReasonToAnthropic } from "./utils" +import { + createToolNameMapFromAnthropicPayload, + toAnthropicToolName, + toOpenAIToolName, + type ToolNameMap, +} from "./tool-name-mapping" +import { mapOpenAIStopReasonToAnthropic, toAnthropicMessageId } from "./utils" + +const MAX_TOOL_RESULT_CHARS = 20_000 +const TOOL_RESULT_HEAD_CHARS = 5_000 +const TOOL_RESULT_MIDDLE_CHARS = 3_000 +const TOOL_RESULT_TAIL_CHARS = 5_000 + +/** + * Type guard for server tool result blocks. Matches web_search_tool_result, + * web_fetch_tool_result, code_execution_tool_result, etc. + * Explicitly excludes plain "tool_result" (which has its own handler). + */ +function isServerToolResultBlock( + block: AnthropicUserContentBlock | AnthropicAssistantContentBlock, +): block is AnthropicServerToolResultBlock { + return block.type.endsWith("_tool_result") && block.type !== "tool_result" +} // Payload translation export function translateToOpenAI( payload: AnthropicMessagesPayload, + toolNameMap: ToolNameMap = createToolNameMapFromAnthropicPayload(payload), ): ChatCompletionsPayload { + const messages = translateAnthropicMessagesToOpenAI( + payload.messages, + payload.system, + toolNameMap, + ) + + // When structured output is requested (e.g. title generation), reinforce + // the JSON constraint directly in the messages. The Copilot Chat + // Completions API ignores the `response_format` parameter, so the model + // only obeys the instruction if it appears prominently in the prompt. + if (payload.output_config?.format) { + enforceJsonOutput(messages) + } + return { model: translateModelName(payload.model), - messages: translateAnthropicMessagesToOpenAI( - payload.messages, - payload.system, - ), + messages, max_tokens: payload.max_tokens, stop: payload.stop_sequences, stream: payload.stream, + // Request usage data in the final streaming chunk so Claude Code can + // track actual input_tokens for proactive context-window compaction. + // Without this, streaming chunks have no usage → input_tokens defaults + // to 0 → Claude Code never knows the context is filling up. + stream_options: payload.stream ? { include_usage: true } : undefined, temperature: payload.temperature, top_p: payload.top_p, user: payload.metadata?.user_id, - tools: translateAnthropicToolsToOpenAI(payload.tools), - tool_choice: translateAnthropicToolChoiceToOpenAI(payload.tool_choice), + tools: translateAnthropicToolsToOpenAI(payload.tools, toolNameMap), + tool_choice: translateAnthropicToolChoiceToOpenAI( + payload.tool_choice, + toolNameMap, + ), + response_format: translateOutputConfig( + payload.output_config, + payload.model, + ), + ...buildReasoningFromThinking(payload.thinking), + } +} + +/** + * Maps an Anthropic `thinking` config onto the Responses-API `reasoning` + * control. Setting `summary: "auto"` is what makes Copilot stream + * `reasoning_summary_text.delta` events in real time during the model's + * thinking phase; without it the model reasons silently and the thinking text + * only surfaces (all at once) at the end of the turn. + * + * The `budget_tokens` hint is translated to a coarse effort level so the + * upstream allocates a comparable amount of reasoning. + */ +function buildReasoningFromThinking( + thinking: AnthropicMessagesPayload["thinking"], +): { reasoning?: { effort: string; summary: string } } { + if (thinking?.type !== "enabled") return {} + + const budget = thinking.budget_tokens + let effort = "medium" + if (typeof budget === "number") { + if (budget <= 8_000) effort = "low" + else if (budget >= 24_000) effort = "high" } + + return { reasoning: { effort, summary: "auto" } } } function translateModelName(model: string): string { - // Subagent requests use a specific model number which Copilot doesn't support - if (model.startsWith("claude-sonnet-4-")) { - return model.replace(/^claude-sonnet-4-.*/, "claude-sonnet-4") - } else if (model.startsWith("claude-opus-")) { - return model.replace(/^claude-opus-4-.*/, "claude-opus-4") + // Normalize claude-{family}-4-{minor}[-extra] → claude-{family}-4 + // Only applies to generation 4+ where minor version numbers are subagent-build-specific. + // eslint-disable-next-line regexp/no-super-linear-backtracking, regexp/optimal-quantifier-concatenation + return model.replace(/^(claude-[a-z]+-4)-\d+.*$/, "$1") +} + +/** + * Translates Anthropic's `output_config.format` to OpenAI's `response_format`. + * + * Claude Code sends `output_config.format.type = "json_schema"` for structured + * output requests like title generation. The Copilot Chat Completions API does + * not support `json_schema` structured outputs, so we downgrade to `json_object` + * which instructs the model to produce valid JSON. The system prompt already + * describes the expected JSON shape, so this is sufficient. + * + * Exception — Claude models: Copilot's Claude backend does not merely ignore + * `response_format`, it rejects the request with HTTP 422 Unprocessable Entity. + * For Claude we therefore omit `response_format` entirely and rely solely on + * the system-prompt JSON enforcement (`enforceJsonOutput`) plus the + * free-text → JSON repair in the structured-output handler. + */ +function translateOutputConfig( + outputConfig: AnthropicMessagesPayload["output_config"], + model: string, +): ChatCompletionsPayload["response_format"] { + if (!outputConfig?.format) return undefined + // Copilot rejects response_format for Claude models (422). JSON is still + // enforced via the system prompt, so dropping it here is safe. + if (model.startsWith("claude")) return undefined + return { type: "json_object" } +} + +/** + * Appends a JSON enforcement instruction to the system message when structured + * output is requested. The Copilot Chat Completions API does not support the + * `response_format` parameter, so models ignore the JSON constraint unless it + * is spelled out in the prompt itself. Without this, title generation requests + * (which use `output_config.format`) produce free-form text answers instead of + * the expected `{"title": "..."}` JSON, causing Claude Code to fall back to + * "Conversation continuation summary". + */ +function enforceJsonOutput(messages: Array<Message>): void { + const enforcement = + "\n\nIMPORTANT: You MUST respond with valid JSON only. " + + "Do not include any text, explanation, or markdown outside the JSON object. " + + "Your entire response must be a single JSON object." + + const systemMsg = messages.find((m) => m.role === "system") + if (systemMsg && typeof systemMsg.content === "string") { + systemMsg.content += enforcement + } else { + // No system message — add one + messages.unshift({ role: "system", content: enforcement.trim() }) } - return model } function translateAnthropicMessagesToOpenAI( anthropicMessages: Array<AnthropicMessage>, - system: string | Array<AnthropicTextBlock> | undefined, + system: string | Array<AnthropicSystemBlock> | undefined, + toolNameMap: ToolNameMap, ): Array<Message> { const systemMessages = handleSystemPrompt(system) const otherMessages = anthropicMessages.flatMap((message) => message.role === "user" ? handleUserMessage(message) - : handleAssistantMessage(message), + : handleAssistantMessage(message, toolNameMap), ) - return [...systemMessages, ...otherMessages] + const combined = [...systemMessages, ...otherMessages] + + // Drop/patch tool messages whose tool_use was removed from history (context + // compaction, interrupted or parallel tool calls). Copilot re-validates the + // Anthropic invariant for Claude models and rejects any tool_result that has + // no corresponding tool_use in the previous message. Run this BEFORE the + // merge so any orphan converted to a user message is then coalesced with an + // adjacent user turn (Copilot expects strictly alternating roles). + repairOrphanedToolCalls(combined) + + const merged = mergeConsecutiveSameRoleMessages(combined) + + // Final invariant: the conversation must end with a user (or tool) turn. + // A trailing assistant message is an "assistant prefill" — native Anthropic + // allows it, but Copilot's Claude backend rejects it: + // "This model does not support assistant message prefill. The conversation + // must end with a user message." + // Claude Code's multi-agent / workflow ("ultracode") mode emits such prefills + // (e.g. priming a structured-output reply). Normalize by appending a minimal + // user continuation so the request ends with a user turn, keeping the + // assistant prefill intact as the prior turn. + return ensureConversationEndsWithUser(merged) +} + +/** + * Appends a minimal user continuation when the translated conversation ends + * with an assistant message that has no pending tool_calls (an "assistant + * prefill"). Copilot's Claude backend requires the conversation to end with a + * user message; native Anthropic permits the prefill, so the proxy bridges the + * gap here. A trailing assistant message that *does* carry tool_calls is left + * untouched — appending a user turn would orphan those calls. + */ +function ensureConversationEndsWithUser( + messages: Array<Message>, +): Array<Message> { + const last = messages.at(-1) + const hasPendingToolCalls = (last?.tool_calls?.length ?? 0) > 0 + if (last?.role === "assistant" && !hasPendingToolCalls) { + messages.push({ role: "user", content: "Please continue." }) + } + return messages +} + +/** + * Merges consecutive messages with the same role into a single message. + * + * The Anthropic API allows consecutive user messages (e.g. after compaction, + * Claude Code sends a summary user message followed by the actual task as + * another user message). The OpenAI/Copilot API expects strictly alternating + * user/assistant roles — consecutive same-role messages confuse the model, + * causing it to echo the summary instead of acting on the task. + */ +function mergeConsecutiveSameRoleMessages( + messages: Array<Message>, +): Array<Message> { + if (messages.length <= 1) return messages + + let previous: Message = messages[0] + const merged: Array<Message> = [previous] + + for (let i = 1; i < messages.length; i++) { + const current = messages[i] + + // `tool` messages are always standalone — each maps 1:1 to a tool_call and + // must keep its own tool_call_id; never coalesce them. + if (current.role !== previous.role || current.role === "tool") { + merged.push(current) + previous = current + continue + } + + // Same role (user↔user or assistant↔assistant): collapse into `previous`, + // merging text AND preserving tool_calls. Naively copying only `.content` + // would drop an assistant's tool_calls and orphan its tool results — + // "messages with role 'tool' must be a response to a preceeding message + // with 'tool_calls'." + const prevText = extractTextContent(previous.content) + const currText = extractTextContent(current.content) + const combinedText = [prevText, currText].filter(Boolean).join("\n\n") + previous.content = combinedText.length > 0 ? combinedText : previous.content + + if (current.tool_calls && current.tool_calls.length > 0) { + previous.tool_calls = [ + ...(previous.tool_calls ?? []), + ...current.tool_calls, + ] + } + } + + return merged +} + +function extractTextContent( + content: string | Array<ContentPart> | null, +): string { + if (content === null) return "" + if (typeof content === "string") return content + return content + .filter((part): part is TextPart => part.type === "text") + .map((part) => part.text) + .join("\n\n") } function handleSystemPrompt( - system: string | Array<AnthropicTextBlock> | undefined, + system: string | Array<AnthropicSystemBlock> | undefined, ): Array<Message> { if (!system) { return [] @@ -81,37 +310,86 @@ function handleSystemPrompt( if (typeof system === "string") { return [{ role: "system", content: system }] } else { - const systemText = system.map((block) => block.text).join("\n\n") - return [{ role: "system", content: systemText }] + const systemText = system + .filter((block): block is AnthropicTextBlock => block.type === "text") + .map((block) => block.text) + .join("\n\n") + return systemText ? [{ role: "system", content: systemText }] : [] } } function handleUserMessage(message: AnthropicUserMessage): Array<Message> { const newMessages: Array<Message> = [] + const deferredUserContents: Array<string | Array<ContentPart>> = [] if (Array.isArray(message.content)) { const toolResultBlocks = message.content.filter( (block): block is AnthropicToolResultBlock => block.type === "tool_result", ) + const serverToolResultBlocks = message.content.filter((block) => + isServerToolResultBlock(block), + ) const otherBlocks = message.content.filter( - (block) => block.type !== "tool_result", + (block) => + block.type !== "tool_result" && !isServerToolResultBlock(block), + // document blocks remain here intentionally — mapContent handles them ) // Tool results must come first to maintain protocol: tool_use -> tool_result -> user for (const block of toolResultBlocks) { + const toolResult = translateToolResultForOpenAI(block.content) newMessages.push({ role: "tool", tool_call_id: block.tool_use_id, - content: mapContent(block.content), + content: toolResult.toolContent, }) + if (toolResult.followUpUserContent) { + deferredUserContents.push(toolResult.followUpUserContent) + } } - if (otherBlocks.length > 0) { - newMessages.push({ - role: "user", - content: mapContent(otherBlocks), - }) + const otherContent = otherBlocks.length > 0 ? mapContent(otherBlocks) : null + + const combinedDeferredUserContent = mergeMessageContents([ + ...deferredUserContents, + otherContent, + ]) + if (combinedDeferredUserContent) { + // When a user message contains both tool results and additional text + // (e.g. Claude Code's Skill tool returns tool_result + text blocks in + // the same user message), avoid emitting a standalone "user" message + // between the tool result and the next assistant message. Gemini + // returns empty responses when it sees user → assistant(+tool_calls) + // inside a tool-calling loop — it expects tool → assistant only. + // Appending the extra text to the last tool result is safe for all + // models; the OpenAI tool message content field accepts any text. + if (toolResultBlocks.length > 0 && deferredUserContents.length === 0) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- guarded by length > 0 + const lastToolMsg = newMessages.at(-1)! + const existingContent = + typeof lastToolMsg.content === "string" ? + lastToolMsg.content + : JSON.stringify(lastToolMsg.content) + const extraText = + typeof combinedDeferredUserContent === "string" ? + combinedDeferredUserContent + : JSON.stringify(combinedDeferredUserContent) + lastToolMsg.content = existingContent + "\n\n" + extraText + } else { + newMessages.push({ + role: "user", + content: combinedDeferredUserContent, + }) + } + } + + // Server tool result blocks → serialize as user message + if (serverToolResultBlocks.length > 0) { + const text = serverToolResultBlocks + .map((b) => `[${b.type}: ${JSON.stringify(b.content)}]`) + .join("\n\n") + newMessages.push({ role: "user", content: text }) } } else { newMessages.push({ @@ -125,6 +403,7 @@ function handleUserMessage(message: AnthropicUserMessage): Array<Message> { function handleAssistantMessage( message: AnthropicAssistantMessage, + toolNameMap: ToolNameMap, ): Array<Message> { if (!Array.isArray(message.content)) { return [ @@ -143,15 +422,40 @@ function handleAssistantMessage( (block): block is AnthropicTextBlock => block.type === "text", ) - const thinkingBlocks = message.content.filter( - (block): block is AnthropicThinkingBlock => block.type === "thinking", + const serverToolUseBlocks = message.content.filter( + (block): block is AnthropicServerToolUseBlock => + block.type === "server_tool_use", + ) + + const serverToolResultBlocks = message.content.filter((block) => + isServerToolResultBlock(block), + ) + + // Strip thinking + redacted_thinking — Copilot doesn't understand them and + // they massively inflate the prompt token count (thinking blocks from Claude + // Code's internal reasoning can be thousands of tokens each). + const visibleBlocks = message.content.filter( + ( + block, + ): block is Exclude< + typeof block, + AnthropicRedactedThinkingBlock | AnthropicThinkingBlock + > => block.type !== "redacted_thinking" && block.type !== "thinking", ) - // Combine text and thinking blocks, as OpenAI doesn't have separate thinking blocks + // Combine text and server_tool_use blocks for Branch 1 (tool_calls path) + // OpenAI doesn't have separate server_tool_use blocks const allTextContent = [ ...textBlocks.map((b) => b.text), - ...thinkingBlocks.map((b) => b.thinking), - ].join("\n\n") + ...serverToolUseBlocks.map( + (b) => `[Server tool use: ${JSON.stringify(b)}]`, + ), + ...serverToolResultBlocks.map( + (b) => `[${b.type}: ${JSON.stringify(b.content)}]`, + ), + ] + .filter(Boolean) + .join("\n\n") return toolUseBlocks.length > 0 ? [ @@ -162,7 +466,7 @@ function handleAssistantMessage( id: toolUse.id, type: "function", function: { - name: toolUse.name, + name: toOpenAIToolName(toolUse.name, toolNameMap), arguments: JSON.stringify(toolUse.input), }, })), @@ -171,11 +475,173 @@ function handleAssistantMessage( : [ { role: "assistant", - content: mapContent(message.content), + content: mapContent(visibleBlocks), }, ] } +// Handles tool_result content which may be a string or array of content blocks +function mapToolResultContent( + content: AnthropicToolResultBlock["content"], +): string | Array<ContentPart> | null { + if (typeof content === "string") { + return content + } + // Safe cast: AnthropicToolResultBlock content array is Array<TextBlock|ImageBlock|DocumentBlock>, + // all of which are members of AnthropicUserContentBlock — mapContent handles them correctly. + return mapContent( + content as Array< + AnthropicUserContentBlock | AnthropicAssistantContentBlock + >, + ) +} + +function translateToolResultForOpenAI( + content: AnthropicToolResultBlock["content"], +): { + toolContent: string | Array<ContentPart> | null + followUpUserContent?: string | Array<ContentPart> | null +} { + if (typeof content === "string") { + return { toolContent: compressToolResultText(content) } + } + + const hasImage = content.some((block) => block.type === "image") + if (!hasImage) { + const mappedContent = mapToolResultContent(content) + return { + toolContent: + typeof mappedContent === "string" ? + compressToolResultText(mappedContent) + : mappedContent, + } + } + + const textContent = content + .filter((block): block is AnthropicTextBlock => block.type === "text") + .map((block) => block.text) + .join("\n\n") + + return { + toolContent: + textContent + || "[Non-text tool result forwarded in the following user message.]", + followUpUserContent: mapContent( + content as Array< + AnthropicUserContentBlock | AnthropicAssistantContentBlock + >, + ), + } +} + +function compressToolResultText(text: string): string { + if (text.length <= MAX_TOOL_RESULT_CHARS) { + return text + } + + const omittedChars = + text.length + - TOOL_RESULT_HEAD_CHARS + - TOOL_RESULT_MIDDLE_CHARS + - TOOL_RESULT_TAIL_CHARS + const head = text.slice(0, TOOL_RESULT_HEAD_CHARS).trimEnd() + const middleStart = Math.max( + TOOL_RESULT_HEAD_CHARS, + Math.floor((text.length - TOOL_RESULT_MIDDLE_CHARS) / 2), + ) + const middle = text + .slice(middleStart, middleStart + TOOL_RESULT_MIDDLE_CHARS) + .trim() + const tail = text.slice(-TOOL_RESULT_TAIL_CHARS).trimStart() + const lineCount = text.split("\n").length + + return [ + `[Tool result condensed by proxy: kept the first ${TOOL_RESULT_HEAD_CHARS.toLocaleString()}, ` + + `middle ${TOOL_RESULT_MIDDLE_CHARS.toLocaleString()}, and last ` + + `${TOOL_RESULT_TAIL_CHARS.toLocaleString()} characters ` + + `out of ${text.length.toLocaleString()} total; omitted ` + + `${omittedChars.toLocaleString()} characters across ${lineCount.toLocaleString()} lines ` + + `to avoid prompt overflow while preserving the latest tool findings.]`, + "[If you need to stay within context, compact older conversation state before discarding this fresh tool result. If more detail is required, ask for a focused rerun or narrower command output.]", + "", + "=== BEGIN TOOL RESULT HEAD ===", + head, + "=== END TOOL RESULT HEAD ===", + "", + "=== BEGIN TOOL RESULT MIDDLE SAMPLE ===", + middle, + "=== END TOOL RESULT MIDDLE SAMPLE ===", + "", + "=== BEGIN TOOL RESULT TAIL ===", + tail, + "=== END TOOL RESULT TAIL ===", + ].join("\n") +} + +function mergeMessageContents( + contents: Array<string | Array<ContentPart> | null | undefined>, +): string | Array<ContentPart> | null { + const filtered = contents.filter( + (content): content is string | Array<ContentPart> => + content !== null + && content !== undefined + && (!(typeof content === "string") || content.length > 0), + ) + + if (filtered.length === 0) return null + if (filtered.every((content) => typeof content === "string")) { + return filtered.join("\n\n") + } + + const merged: Array<ContentPart> = [] + for (const content of filtered) { + if (typeof content === "string") { + merged.push({ type: "text", text: content }) + continue + } + merged.push(...content) + } + return merged +} + +/** + * Serializes a content block to a plain-text representation. + * Used by both paths of mapContent for non-image/non-text blocks. + * Returns null for blocks that should be silently skipped. + */ +function serializeBlockToText( + block: AnthropicUserContentBlock | AnthropicAssistantContentBlock, +): string | null { + switch (block.type) { + case "text": { + return block.text + } + case "document": { + return "[Document: PDF content not displayable]" + } + case "server_tool_use": { + return `[Server tool use: ${JSON.stringify(block)}]` + } + case "search_result": { + return `[Search: ${block.title}]\nSource: ${block.source}\n${block.content}` + } + case "container_upload": { + return `[Container upload: ${block.file_id}]` + } + default: { + // Catch-all: server tool results and future unknown types + if ( + "content" in block + && (block.type as string) !== "thinking" + && (block.type as string) !== "redacted_thinking" + ) { + return `[${block.type}: ${JSON.stringify((block as { content: unknown }).content)}]` + } + return null + } + } +} + function mapContent( content: | string @@ -191,38 +657,32 @@ function mapContent( const hasImage = content.some((block) => block.type === "image") if (!hasImage) { return content - .filter( - (block): block is AnthropicTextBlock | AnthropicThinkingBlock => - block.type === "text" || block.type === "thinking", - ) - .map((block) => (block.type === "text" ? block.text : block.thinking)) + .map((block) => serializeBlockToText(block)) + .filter(Boolean) .join("\n\n") } const contentParts: Array<ContentPart> = [] for (const block of content) { - switch (block.type) { - case "text": { - contentParts.push({ type: "text", text: block.text }) - - break - } - case "thinking": { - contentParts.push({ type: "text", text: block.thinking }) - - break - } - case "image": { + if (block.type === "image") { + if (block.source.type === "url") { + contentParts.push({ + type: "image_url", + image_url: { url: block.source.url }, + }) + } else { contentParts.push({ type: "image_url", image_url: { url: `data:${block.source.media_type};base64,${block.source.data}`, }, }) - - break } - // No default + } else { + const text = serializeBlockToText(block) + if (text) { + contentParts.push({ type: "text", text }) + } } } return contentParts @@ -230,22 +690,33 @@ function mapContent( function translateAnthropicToolsToOpenAI( anthropicTools: Array<AnthropicTool> | undefined, + toolNameMap: ToolNameMap, ): Array<Tool> | undefined { if (!anthropicTools) { return undefined } - return anthropicTools.map((tool) => ({ - type: "function", - function: { - name: tool.name, - description: tool.description, - parameters: tool.input_schema, - }, - })) + + const customTools: Array<Tool> = anthropicTools + .filter((tool): tool is AnthropicCustomTool => !isTypedTool(tool)) + .map((tool) => ({ + type: "function", + function: { + name: toOpenAIToolName(tool.name, toolNameMap), + description: tool.description, + parameters: tool.input_schema, + // Forward strict for Structured Outputs; strip all other extra fields + // (cache_control, defer_loading, input_examples, eager_input_streaming) + ...(tool.strict !== undefined ? { strict: tool.strict } : {}), + }, + })) + // Return undefined (not []) when all tools are typed — an empty tools array with an active + // tool_choice would produce a malformed OpenAI request. + return customTools.length > 0 ? customTools : undefined } function translateAnthropicToolChoiceToOpenAI( anthropicToolChoice: AnthropicMessagesPayload["tool_choice"], + toolNameMap: ToolNameMap, ): ChatCompletionsPayload["tool_choice"] { if (!anthropicToolChoice) { return undefined @@ -262,7 +733,9 @@ function translateAnthropicToolChoiceToOpenAI( if (anthropicToolChoice.name) { return { type: "function", - function: { name: anthropicToolChoice.name }, + function: { + name: toOpenAIToolName(anthropicToolChoice.name, toolNameMap), + }, } } return undefined @@ -280,6 +753,7 @@ function translateAnthropicToolChoiceToOpenAI( export function translateToAnthropic( response: ChatCompletionResponse, + toolNameMap?: ToolNameMap, ): AnthropicResponse { // Merge content from all choices const allTextBlocks: Array<AnthropicTextBlock> = [] @@ -291,7 +765,10 @@ export function translateToAnthropic( // Process all choices to extract text and tool use blocks for (const choice of response.choices) { const textBlocks = getAnthropicTextBlocks(choice.message.content) - const toolUseBlocks = getAnthropicToolUseBlocks(choice.message.tool_calls) + const toolUseBlocks = getAnthropicToolUseBlocks( + choice.message.tool_calls, + toolNameMap, + ) allTextBlocks.push(...textBlocks) allToolUseBlocks.push(...toolUseBlocks) @@ -304,25 +781,97 @@ export function translateToAnthropic( // Note: GitHub Copilot doesn't generate thinking blocks, so we don't include them in responses + // Some models (notably Gemini) intermittently return a non-tool_calls + // finish_reason ("stop", or even null — a degenerate shape) even when they + // emitted tool calls. Correct this to "tool_calls" whenever tool-use blocks + // are present, so Claude Code executes the pending tool calls instead of + // treating the turn as done. + // + // The one exception is "length": a length-truncated turn with tool calls is + // handled specially below (the arguments may be incomplete), so preserve it + // here rather than masking it as "tool_calls". + const correctedStopReason = + allToolUseBlocks.length > 0 && stopReason !== "length" ? + "tool_calls" + : stopReason + + // Guard: detect truncated tool calls when finish_reason is "length". + // When the output hits the token limit mid-tool-call, the JSON arguments are + // incomplete. safeParseJson silently returns {} for these, which would cause + // Claude Code to execute a tool with empty/wrong input. Instead, replace the + // broken tool use blocks with an explanatory text block and return "end_turn" + // so Claude Code reads the feedback and adjusts its strategy. + if (correctedStopReason === "length" && allToolUseBlocks.length > 0) { + const hasTruncated = response.choices.some((choice) => + choice.message.tool_calls?.some((tc) => { + if (!tc.function.arguments) return false + try { + JSON.parse(tc.function.arguments) + return false + } catch { + return true + } + }), + ) + + if (hasTruncated) { + const toolName = allToolUseBlocks[0].name + consola.debug( + `[non-stream] Truncated tool call detected for "${toolName}" — ` + + `replacing with explanatory text`, + ) + return { + id: toAnthropicMessageId(response.id), + type: "message", + role: "assistant", + model: response.model, + content: [ + ...allTextBlocks, + { + type: "text", + text: + `[Output truncated: the response exceeded the maximum output token limit` + + ` while generating tool call "${toolName}".` + + ` Please retry with a smaller output, e.g. write the file in smaller chunks.]`, + }, + ], + stop_reason: "end_turn", + stop_sequence: null, + usage: buildAnthropicUsage(response.usage), + } + } + } + + // Backstop: a completed non-streaming message must never carry + // stop_reason: null. Anthropic clients (Claude Code, strict SDK callers) + // treat null stop_reason on a non-stream response as a protocol violation. + // Degenerate upstream payloads (empty choices array, or a null + // finish_reason from models like Gemini) would otherwise map straight + // through to null here. When tool-use blocks are present, default to + // "tool_use" so the client still executes the tools; otherwise "end_turn". return { - id: response.id, + id: toAnthropicMessageId(response.id), type: "message", role: "assistant", model: response.model, content: [...allTextBlocks, ...allToolUseBlocks], - stop_reason: mapOpenAIStopReasonToAnthropic(stopReason), + stop_reason: + mapOpenAIStopReasonToAnthropic(correctedStopReason) + ?? (allToolUseBlocks.length > 0 ? "tool_use" : "end_turn"), stop_sequence: null, - usage: { - input_tokens: - (response.usage?.prompt_tokens ?? 0) - - (response.usage?.prompt_tokens_details?.cached_tokens ?? 0), - output_tokens: response.usage?.completion_tokens ?? 0, - ...(response.usage?.prompt_tokens_details?.cached_tokens - !== undefined && { - cache_read_input_tokens: - response.usage.prompt_tokens_details.cached_tokens, - }), - }, + usage: buildAnthropicUsage(response.usage), + } +} + +function buildAnthropicUsage(usage: ChatCompletionResponse["usage"]) { + return { + input_tokens: + (usage?.prompt_tokens ?? 0) + - (usage?.prompt_tokens_details?.cached_tokens ?? 0), + output_tokens: usage?.completion_tokens ?? 0, + ...(usage?.prompt_tokens_details?.cached_tokens !== undefined && { + cache_read_input_tokens: usage.prompt_tokens_details.cached_tokens, + }), } } @@ -344,6 +893,7 @@ function getAnthropicTextBlocks( function getAnthropicToolUseBlocks( toolCalls: Array<ToolCall> | undefined, + toolNameMap?: ToolNameMap, ): Array<AnthropicToolUseBlock> { if (!toolCalls) { return [] @@ -351,7 +901,16 @@ function getAnthropicToolUseBlocks( return toolCalls.map((toolCall) => ({ type: "tool_use", id: toolCall.id, - name: toolCall.function.name, - input: JSON.parse(toolCall.function.arguments) as Record<string, unknown>, + name: toAnthropicToolName(toolCall.function.name, toolNameMap), + input: safeParseJson(toolCall.function.arguments), })) } + +function safeParseJson(json: string): Record<string, unknown> { + if (!json) return {} + try { + return JSON.parse(json) as Record<string, unknown> + } catch { + return {} + } +} diff --git a/src/routes/messages/stream-translation.ts b/src/routes/messages/stream-translation.ts index 55094448f..43e0dab01 100644 --- a/src/routes/messages/stream-translation.ts +++ b/src/routes/messages/stream-translation.ts @@ -4,7 +4,75 @@ import { type AnthropicStreamEventData, type AnthropicStreamState, } from "./anthropic-types" -import { mapOpenAIStopReasonToAnthropic } from "./utils" +import { toAnthropicToolName } from "./tool-name-mapping" +import { mapOpenAIStopReasonToAnthropic, toAnthropicMessageId } from "./utils" + +/** + * Generates a human-readable description for a tool call so the user can see + * what the model is doing. Models like Gemini go straight to tool calls + * without any explanatory text — this provides that missing context. + * + * When arguments are available (e.g. Gemini sends name + args in one chunk), + * the description extracts key details like file paths and commands. + * Otherwise falls back to just the tool name. + */ +function describeToolCall(name: string, rawArgs: string | undefined): string { + // Try to parse arguments for richer descriptions + const args = parseToolArgs(rawArgs) + + if (args) { + // Bash tool — show the command (truncated if very long) + if (typeof args.command === "string") { + const cmd = args.command + return `Running: ${cmd.length > 120 ? cmd.slice(0, 120) + "…" : cmd}` + } + // File-based tools — show the path with an action verb + if (typeof args.file_path === "string") { + return describeFileToolCall(name, args.file_path) + } + // Search tools — show the pattern + if (typeof args.pattern === "string") { + return describeSearchToolCall(name, args.pattern) + } + // WebFetch — show the URL + if (typeof args.prompt === "string" && typeof args.url === "string") { + return `Fetching: ${args.url}` + } + } + + return `Using tool: ${name}` +} + +/** Safely parses JSON tool arguments, returning undefined on failure. */ +function parseToolArgs( + rawArgs: string | undefined, +): Record<string, unknown> | undefined { + if (!rawArgs) return undefined + try { + return JSON.parse(rawArgs) as Record<string, unknown> + } catch { + return undefined + } +} + +/** Returns a description for file-path-based tools (Read, Write, Edit, etc.). */ +function describeFileToolCall(name: string, filePath: string): string { + const lower = name.toLowerCase() + if (lower.includes("read") || name === "Read") return `Reading: ${filePath}` + if (lower.includes("write") || name === "Write") return `Writing: ${filePath}` + if (lower.includes("edit") || name === "Edit") return `Editing: ${filePath}` + return `${name}: ${filePath}` +} + +/** Returns a description for search-pattern-based tools (Glob, Grep, etc.). */ +function describeSearchToolCall(name: string, pattern: string): string { + const lower = name.toLowerCase() + if (lower.includes("glob") || name === "Glob") + return `Searching files: ${pattern}` + if (lower.includes("grep") || name === "Grep") + return `Searching for: ${pattern}` + return `${name}: ${pattern}` +} function isToolBlockOpen(state: AnthropicStreamState): boolean { if (!state.contentBlockOpen) { @@ -16,6 +84,21 @@ function isToolBlockOpen(state: AnthropicStreamState): boolean { ) } +/** Whether the currently open block is a thinking block. */ +function isThinkingBlockOpen(state: AnthropicStreamState): boolean { + return state.contentBlockOpen && state.thinkingBlockOpen +} + +/** + * Claude clients persist/replay Anthropic-style tool streams more reliably + * when the assistant turn contains only the actual tool_use blocks. + * Synthetic pre-tool text is mainly needed for models like Gemini that often + * jump straight to tool calls without any visible narration. + */ +function shouldInjectSyntheticToolDescription(model: string): boolean { + return !model.toLowerCase().startsWith("claude") +} + // eslint-disable-next-line max-lines-per-function, complexity export function translateChunkToAnthropicEvents( chunk: ChatCompletionChunk, @@ -23,6 +106,14 @@ export function translateChunkToAnthropicEvents( ): Array<AnthropicStreamEventData> { const events: Array<AnthropicStreamEventData> = [] + // Always capture usage from every chunk that has it. With + // `stream_options: { include_usage: true }`, the OpenAI API sends a + // final chunk with `choices: []` and `usage` — we need to store it + // so the deferred `message_delta` can include accurate `input_tokens`. + if (chunk.usage) { + state.lastSeenUsage = chunk.usage + } + if (chunk.choices.length === 0) { return events } @@ -34,7 +125,7 @@ export function translateChunkToAnthropicEvents( events.push({ type: "message_start", message: { - id: chunk.id, + id: toAnthropicMessageId(chunk.id), type: "message", role: "assistant", content: [], @@ -57,7 +148,81 @@ export function translateChunkToAnthropicEvents( state.messageStartSent = true } + // Normalize Gemini's reasoning_text → reasoning_content so the rest of + // the logic only needs to check one field. Gemini models send reasoning + // output as `reasoning_text` while GPT models use `reasoning_content`. + if (delta.reasoning_text && !delta.reasoning_content) { + delta.reasoning_content = delta.reasoning_text + } + + // Reasoning/thinking content from models like GPT 5.4 and Gemini. + // When the client has `thinking` enabled, emit as proper Anthropic thinking + // blocks so Claude Code displays them in its dedicated thinking UI. When + // thinking is not enabled, emit as regular text blocks so the content is + // still visible to the user. + if (delta.reasoning_content) { + // If a tool block is open, close it first. + if (isToolBlockOpen(state)) { + events.push({ + type: "content_block_stop", + index: state.contentBlockIndex, + }) + state.contentBlockIndex++ + state.contentBlockOpen = false + state.thinkingBlockOpen = false + } + + if (state.thinkingEnabled) { + // Emit as a proper thinking block so Claude Code's thinking UI + // displays the reasoning content with real-time streaming. + if (!state.contentBlockOpen) { + events.push({ + type: "content_block_start", + index: state.contentBlockIndex, + content_block: { type: "thinking", thinking: "" }, + }) + state.contentBlockOpen = true + state.thinkingBlockOpen = true + } + + events.push({ + type: "content_block_delta", + index: state.contentBlockIndex, + delta: { type: "thinking_delta", thinking: delta.reasoning_content }, + }) + } else { + // Thinking not enabled — emit as regular text so content is visible. + if (!state.contentBlockOpen) { + events.push({ + type: "content_block_start", + index: state.contentBlockIndex, + content_block: { type: "text", text: "" }, + }) + state.contentBlockOpen = true + state.thinkingBlockOpen = true + } + + events.push({ + type: "content_block_delta", + index: state.contentBlockIndex, + delta: { type: "text_delta", text: delta.reasoning_content }, + }) + } + state.hasEmittedText = true + } + if (delta.content) { + // If a thinking block is open, close it before starting a text block. + if (isThinkingBlockOpen(state)) { + events.push({ + type: "content_block_stop", + index: state.contentBlockIndex, + }) + state.contentBlockIndex++ + state.contentBlockOpen = false + state.thinkingBlockOpen = false + } + if (isToolBlockOpen(state)) { // A tool block was open, so close it before starting a text block. events.push({ @@ -88,10 +253,16 @@ export function translateChunkToAnthropicEvents( text: delta.content, }, }) + state.hasEmittedText = true } if (delta.tool_calls) { for (const toolCall of delta.tool_calls) { + const anthropicToolName = + toolCall.function?.name ? + toAnthropicToolName(toolCall.function.name, state.toolNameMap) + : undefined + if (toolCall.id && toolCall.function?.name) { // New tool call starting. if (state.contentBlockOpen) { @@ -102,13 +273,52 @@ export function translateChunkToAnthropicEvents( }) state.contentBlockIndex++ state.contentBlockOpen = false + state.thinkingBlockOpen = false + } + + // Inject a descriptive text block when the model hasn't emitted any + // visible text before starting tool calls. Models like Gemini go + // straight to tool_use without any explanatory content — Claude Code + // would show a loading animation with no indication of progress. + // The description extracts key details from tool arguments (e.g. + // file paths, commands) so the user sees what's actually happening. + if ( + !state.hasEmittedText + && shouldInjectSyntheticToolDescription(chunk.model) + ) { + const description = describeToolCall( + anthropicToolName ?? toolCall.function.name, + toolCall.function.arguments, + ) + events.push( + { + type: "content_block_start", + index: state.contentBlockIndex, + content_block: { type: "text", text: "" }, + }, + { + type: "content_block_delta", + index: state.contentBlockIndex, + delta: { + type: "text_delta", + text: description, + }, + }, + { + type: "content_block_stop", + index: state.contentBlockIndex, + }, + ) + state.contentBlockIndex++ + state.hasEmittedText = true } const anthropicBlockIndex = state.contentBlockIndex state.toolCalls[toolCall.index] = { id: toolCall.id, - name: toolCall.function.name, + name: anthropicToolName ?? toolCall.function.name, anthropicBlockIndex, + accumulatedArgs: "", } events.push({ @@ -117,7 +327,7 @@ export function translateChunkToAnthropicEvents( content_block: { type: "tool_use", id: toolCall.id, - name: toolCall.function.name, + name: anthropicToolName ?? toolCall.function.name, input: {}, }, }) @@ -129,6 +339,7 @@ export function translateChunkToAnthropicEvents( // Tool call can still be empty // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (toolCallInfo) { + toolCallInfo.accumulatedArgs += toolCall.function.arguments events.push({ type: "content_block_delta", index: toolCallInfo.anthropicBlockIndex, @@ -143,6 +354,21 @@ export function translateChunkToAnthropicEvents( } if (choice.finish_reason) { + // Detect truncated tool calls: when finish_reason is "length" and tool + // calls have incomplete JSON arguments, the output hit the token limit + // mid-tool-call. Instead of passing broken JSON to Claude Code (which + // would cause it to execute a tool with invalid input), emit an + // explanatory text block and use "end_turn" so Claude Code adjusts its + // strategy (e.g., writing files in smaller chunks). + if (choice.finish_reason === "length") { + const truncated = findTruncatedToolCalls(state) + if (truncated.length > 0) { + return emitTruncationGuardEvents(state, chunk, { events, truncated }) + } + } + + const hasToolCalls = Object.keys(state.toolCalls).length > 0 + if (state.contentBlockOpen) { events.push({ type: "content_block_stop", @@ -151,40 +377,192 @@ export function translateChunkToAnthropicEvents( state.contentBlockOpen = false } - events.push( - { - type: "message_delta", - delta: { - stop_reason: mapOpenAIStopReasonToAnthropic(choice.finish_reason), - stop_sequence: null, - }, - usage: { - input_tokens: - (chunk.usage?.prompt_tokens ?? 0) - - (chunk.usage?.prompt_tokens_details?.cached_tokens ?? 0), - output_tokens: chunk.usage?.completion_tokens ?? 0, - ...(chunk.usage?.prompt_tokens_details?.cached_tokens - !== undefined && { - cache_read_input_tokens: - chunk.usage.prompt_tokens_details.cached_tokens, - }), - }, + // Some models (notably Gemini) intermittently return a non-tool_calls + // finish_reason ("stop", or a non-standard string) even when they emitted + // tool calls. Claude Code interprets stop_reason "end_turn" as "model is + // done" and skips pending tool executions, stalling the session. Coerce to + // "tool_calls" whenever tool calls are present, EXCEPT for "length" (which + // is handled by the truncation guard above and must be preserved). This + // mirrors the non-streaming correction in non-stream-translation.ts. + const correctedFinishReason = + hasToolCalls && choice.finish_reason !== "length" ? + "tool_calls" + : choice.finish_reason + + // Defer message_delta + message_stop instead of emitting immediately. + // When `stream_options: { include_usage: true }` is set, the usage chunk + // (with accurate prompt_tokens) arrives AFTER this finish_reason chunk. + // By deferring, we let `flushDeferredFinish()` emit these events with + // the real usage data once the stream ends or the usage chunk arrives. + state.deferredFinishReason = correctedFinishReason + } + + return events +} + +/** + * Emits the deferred `message_delta` + `message_stop` events after the stream + * has ended. This is called from the stream handler after all chunks have been + * processed, so the accumulated `lastSeenUsage` contains the final usage data + * (from the usage-only chunk sent by the OpenAI API with `stream_options`). + * + * If no finish was deferred (stream ended without finish_reason), returns empty. + */ +export function flushDeferredFinish( + state: AnthropicStreamState, +): Array<AnthropicStreamEventData> { + if (state.deferredFinishReason === undefined) return [] + + const usage = state.lastSeenUsage + const events: Array<AnthropicStreamEventData> = [ + { + type: "message_delta", + delta: { + // Backstop: the terminating message_delta must carry a non-null + // stop_reason. mapOpenAIStopReasonToAnthropic returns undefined for a + // non-standard finish_reason string (which JSON drops, violating the + // Anthropic streaming protocol), so coalesce — to "tool_use" when tool + // calls were emitted, else "end_turn". Mirrors the non-streaming + // backstop in non-stream-translation.ts. + stop_reason: + mapOpenAIStopReasonToAnthropic(state.deferredFinishReason) + ?? (Object.keys(state.toolCalls).length > 0 ? + "tool_use" + : "end_turn"), + stop_sequence: null, }, - { - type: "message_stop", + usage: { + input_tokens: + (usage?.prompt_tokens ?? 0) + - (usage?.prompt_tokens_details?.cached_tokens ?? 0), + output_tokens: usage?.completion_tokens ?? 0, + ...(usage?.prompt_tokens_details?.cached_tokens !== undefined && { + cache_read_input_tokens: usage.prompt_tokens_details.cached_tokens, + }), }, - ) + }, + { + type: "message_stop", + }, + ] + state.messageStopSent = true + state.deferredFinishReason = undefined + return events +} + +/** + * Empty accumulatedArgs are considered valid (no arguments to parse). + */ +export function findTruncatedToolCalls( + state: AnthropicStreamState, +): Array<{ id: string; name: string; accumulatedArgs: string }> { + return Object.values(state.toolCalls).filter((tc) => { + if (!tc.accumulatedArgs) return false + try { + JSON.parse(tc.accumulatedArgs) + return false + } catch { + return true + } + }) +} + +/** + * Detects whether a raw SSE chunk represents a complete but empty response. + * + * Some models (notably Gemini) occasionally return a single chunk with + * `finish_reason: "stop"`, `content: null`, and zero tool calls after + * completing their reasoning phase. This is effectively a "model had + * nothing to say" response. When detected before `message_start` is sent, + * the caller can retry the request transparently instead of passing an + * empty turn to Claude Code (which would cause it to silently stop). + */ +export function isEmptyStreamResponse(chunk: ChatCompletionChunk): boolean { + if (chunk.choices.length === 0) return false + const choice = chunk.choices[0] + return ( + choice.finish_reason === "stop" + && !choice.delta.content + && !choice.delta.reasoning_content + && !choice.delta.reasoning_text + && (!choice.delta.tool_calls || choice.delta.tool_calls.length === 0) + ) +} + +/** + * Emits guard events when a tool call is truncated by the output token limit. + * Closes the open tool block, emits an explanatory text block, and terminates + * with stop_reason "end_turn" so Claude Code reads the feedback instead of + * trying to execute a broken tool call. + */ +function emitTruncationGuardEvents( + state: AnthropicStreamState, + chunk: ChatCompletionChunk, + ctx: { + events: Array<AnthropicStreamEventData> + truncated: Array<{ name: string }> + }, +): Array<AnthropicStreamEventData> { + const { events, truncated } = ctx + + if (state.contentBlockOpen) { + events.push({ + type: "content_block_stop", + index: state.contentBlockIndex, + }) + state.contentBlockIndex++ + state.contentBlockOpen = false } + const toolName = truncated[0].name + events.push( + { + type: "content_block_start", + index: state.contentBlockIndex, + content_block: { type: "text", text: "" }, + }, + { + type: "content_block_delta", + index: state.contentBlockIndex, + delta: { + type: "text_delta", + text: + `[Output truncated: the response exceeded the maximum output token limit` + + ` while generating tool call "${toolName}".` + + ` Please retry with a smaller output, e.g. write the file in smaller chunks.]`, + }, + }, + { + type: "content_block_stop", + index: state.contentBlockIndex, + }, + { + type: "message_delta", + delta: { stop_reason: "end_turn", stop_sequence: null }, + usage: { + input_tokens: + ((state.lastSeenUsage ?? chunk.usage)?.prompt_tokens ?? 0) + - ((state.lastSeenUsage ?? chunk.usage)?.prompt_tokens_details + ?.cached_tokens ?? 0), + output_tokens: + (state.lastSeenUsage ?? chunk.usage)?.completion_tokens ?? 0, + }, + }, + { type: "message_stop" }, + ) + state.messageStopSent = true return events } -export function translateErrorToAnthropicErrorEvent(): AnthropicStreamEventData { +export function translateErrorToAnthropicErrorEvent( + message: string = "An unexpected error occurred during streaming.", + errorType: string = "api_error", +): AnthropicStreamEventData { return { type: "error", error: { - type: "api_error", - message: "An unexpected error occurred during streaming.", + type: errorType, + message, }, } } diff --git a/src/routes/messages/tool-name-mapping.ts b/src/routes/messages/tool-name-mapping.ts new file mode 100644 index 000000000..8acf9b0cd --- /dev/null +++ b/src/routes/messages/tool-name-mapping.ts @@ -0,0 +1,142 @@ +import { createHash } from "node:crypto" + +import { + type AnthropicAssistantMessage, + type AnthropicMessagesPayload, + isTypedTool, +} from "./anthropic-types" + +const OPENAI_TOOL_NAME_PATTERN = /^[\w-]{1,64}$/ +const FALLBACK_TOOL_NAME = "tool" +const HASH_LENGTHS = [8, 12, 16, 20, 24, 28, 32, 40] + +export interface ToolNameMap { + anthropicToOpenAI: Record<string, string> + openAIToAnthropic: Record<string, string> +} + +export function createToolNameMapFromAnthropicPayload( + payload: AnthropicMessagesPayload, +): ToolNameMap { + const names = new Set<string>() + + for (const tool of payload.tools ?? []) { + if (!isTypedTool(tool)) { + names.add(tool.name) + } + } + + for (const message of payload.messages) { + if (message.role === "assistant") { + collectAssistantToolNames(message, names) + } + } + + if (payload.tool_choice?.type === "tool" && payload.tool_choice.name) { + names.add(payload.tool_choice.name) + } + + return createToolNameMap(names) +} + +export function createToolNameMap(names: Iterable<string>): ToolNameMap { + const anthropicToOpenAI: Record<string, string> = {} + const openAIToAnthropic: Record<string, string> = {} + const usedAliases = new Set<string>() + + for (const name of new Set(names)) { + const alias = pickOpenAIToolNameAlias(name, usedAliases) + anthropicToOpenAI[name] = alias + openAIToAnthropic[alias] = name + usedAliases.add(alias) + } + + return { anthropicToOpenAI, openAIToAnthropic } +} + +export function toOpenAIToolName( + anthropicName: string, + toolNameMap: ToolNameMap | undefined, +): string { + return toolNameMap?.anthropicToOpenAI[anthropicName] ?? anthropicName +} + +export function toAnthropicToolName( + openAIName: string, + toolNameMap: ToolNameMap | undefined, +): string { + return toolNameMap?.openAIToAnthropic[openAIName] ?? openAIName +} + +function collectAssistantToolNames( + message: AnthropicAssistantMessage, + names: Set<string>, +) { + if (!Array.isArray(message.content)) { + return + } + + for (const block of message.content) { + if (block.type === "tool_use") { + names.add(block.name) + } + } +} + +function pickOpenAIToolNameAlias( + name: string, + usedAliases: Set<string>, +): string { + if (OPENAI_TOOL_NAME_PATTERN.test(name) && !usedAliases.has(name)) { + return name + } + + const sanitizedName = sanitizeToolName(name) + if ( + OPENAI_TOOL_NAME_PATTERN.test(sanitizedName) + && !usedAliases.has(sanitizedName) + ) { + return sanitizedName + } + + for (const [attempt, HASH_LENGTH] of HASH_LENGTHS.entries()) { + const hashInput = attempt === 0 ? name : `${name}:${attempt}` + const hash = hashToolName(hashInput).slice(0, HASH_LENGTH) + const alias = buildHashedAlias(sanitizedName, hash) + if (!usedAliases.has(alias)) { + return alias + } + } + + return buildHashedAlias(FALLBACK_TOOL_NAME, hashToolName(name).slice(0, 40)) +} + +function sanitizeToolName(name: string): string { + const sanitized = name.replaceAll(/[^\w-]/g, "_").replaceAll(/^_+|_+$/g, "") + return sanitized || FALLBACK_TOOL_NAME +} + +function buildHashedAlias(baseName: string, hash: string): string { + const separator = "__" + const maxBaseLength = 64 - separator.length - hash.length + if (maxBaseLength <= 0) { + return hash.slice(0, 64) + } + + const compactBase = + baseName.length <= maxBaseLength ? + baseName + : compactToolNameBase(baseName, maxBaseLength) + + return `${compactBase}${separator}${hash}` +} + +function compactToolNameBase(baseName: string, maxLength: number): string { + const prefixLength = Math.ceil(maxLength / 2) + const suffixLength = Math.floor(maxLength / 2) + return baseName.slice(0, prefixLength) + baseName.slice(-suffixLength) +} + +function hashToolName(name: string): string { + return createHash("sha1").update(name).digest("hex") +} diff --git a/src/routes/messages/utils.ts b/src/routes/messages/utils.ts index d0febfc9d..76a22db5c 100644 --- a/src/routes/messages/utils.ts +++ b/src/routes/messages/utils.ts @@ -14,3 +14,17 @@ export function mapOpenAIStopReasonToAnthropic( } as const return stopReasonMap[finishReason] } + +/** + * Converts a Copilot/OpenAI response ID (e.g. "chatcmpl-xxx" or "resp_xxx") + * into an Anthropic-style message ID with the "msg_" prefix. + * + * Claude Code uses the "msg_" prefix to identify valid message IDs when + * persisting and restoring conversation history (e.g. across VS Code + * window reloads). Without this prefix the conversation is treated as + * invalid and history is lost on reload. + */ +export function toAnthropicMessageId(upstreamId: string): string { + if (upstreamId.startsWith("msg_")) return upstreamId + return `msg_${upstreamId}` +} diff --git a/src/routes/messages/web-search-detection.ts b/src/routes/messages/web-search-detection.ts new file mode 100644 index 000000000..235405d66 --- /dev/null +++ b/src/routes/messages/web-search-detection.ts @@ -0,0 +1,142 @@ +import consola from "consola" + +import { state } from "~/lib/state" +import { + createChatCompletions, + type ChatCompletionResponse, +} from "~/services/copilot/create-chat-completions" +import { WEB_SEARCH_TOOL_NAMES } from "~/services/web-search/tool-definition" + +import { isTypedTool, type AnthropicMessagesPayload } from "./anthropic-types" + +/** + * Returns true if this request should trigger a web search. + * + * Path 1: Zero-cost — checks if any typed tool in the request has a name + * in WEB_SEARCH_TOOL_NAMES. Short-circuits to true without an API call. + * + * Path 2: Only fires when Path 1 is false AND the payload has at least one + * custom tool whose name is in WEB_SEARCH_TOOL_NAMES (i.e., the client + * declared a web-search-capable custom tool). Sends a lightweight preflight + * classification request to Copilot asking whether the last user message + * requires real-time web data. Falls back to false on any failure. + * + * Note: the preflight Copilot API call is not tracked by the outer rate + * limiter — it is an internal fan-out. See interceptor.ts for the full + * accounting of internal Copilot calls per request. + */ +export async function detectWebSearchIntent( + payload: AnthropicMessagesPayload, +): Promise<boolean> { + // Path 1: typed tool detection (free) + const hasWebSearchTypedTool = + payload.tools?.some( + (tool) => isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name), + ) ?? false + + if (hasWebSearchTypedTool) { + return true + } + + // Path 2: natural language preflight (costs one Copilot API call). + // Only fires when the client has explicitly declared a custom tool with a + // web-search name — this prevents the preflight from firing on every + // Claude Code request (which always supplies Bash/editor tools but never + // web search tools unless the user explicitly adds one). + const hasWebSearchCustomTool = + payload.tools?.some( + (tool) => !isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name), + ) ?? false + + if (!hasWebSearchCustomTool) { + return false + } + + const lastUserMessage = getLastUserMessageText(payload) + if (!lastUserMessage) { + return false + } + + try { + const preflightModel = getPreflightModel(payload.model) + const response = (await createChatCompletions({ + model: preflightModel, + stream: false, + max_tokens: 5, + messages: [ + { + role: "system", + content: + 'You are a classifier. Answer only "yes" or "no". No explanation.', + }, + { + role: "user", + // Use XML delimiters so that quote characters in the user message + // cannot break the classifier prompt (prompt injection mitigation). + content: `Does this message require searching the web for current or real-time information?\n<message>${lastUserMessage}</message>`, + }, + ], + })) as ChatCompletionResponse + + const answer = + response.choices[0]?.message.content?.trim().toLowerCase() ?? "" + return answer === "yes" + } catch (error) { + consola.warn( + "Web search preflight classification failed, treating as no-search-needed:", + error, + ) + return false + } +} + +/** + * Returns a new payload with all typed web search tools removed. + * Does not mutate the input. + */ +export function stripWebSearchTypedTools( + payload: AnthropicMessagesPayload, +): AnthropicMessagesPayload { + return { + ...payload, + tools: payload.tools?.filter( + (tool) => !isTypedTool(tool) || !WEB_SEARCH_TOOL_NAMES.has(tool.name), + ), + } +} + +function getLastUserMessageText(payload: AnthropicMessagesPayload): string { + for (let i = payload.messages.length - 1; i >= 0; i--) { + const msg = payload.messages.at(i) + if (msg?.role !== "user") continue + if (typeof msg.content === "string") return msg.content + if (Array.isArray(msg.content)) { + return msg.content + .filter( + (block): block is { type: "text"; text: string } => + block.type === "text", + ) + .map((block) => block.text) + .join(" ") + } + } + return "" +} + +/** + * Picks a small/cheap model for the single-token preflight classification. + * Prefers models whose name contains "mini", "flash", "haiku", or "small" + * as a heuristic for lower-cost models. Falls back to the request model if + * no cheaper alternative is found. + */ +function getPreflightModel(requestModel: string): string { + const models = state.models?.data ?? [] + const CHEAP_HINTS = ["mini", "flash", "haiku", "small"] + const cheap = models.find((m) => + CHEAP_HINTS.some((hint) => m.id.toLowerCase().includes(hint)), + ) + if (cheap) return cheap.id + // Fall back: any model that isn't the request model (avoids same-model round-trip) + const alternative = models.find((m) => m.id !== requestModel) + return alternative?.id ?? requestModel +} diff --git a/src/routes/models/route.ts b/src/routes/models/route.ts index 5254e2af7..8078d8b60 100644 --- a/src/routes/models/route.ts +++ b/src/routes/models/route.ts @@ -3,6 +3,10 @@ import { Hono } from "hono" import { forwardError } from "~/lib/error" import { state } from "~/lib/state" import { cacheModels } from "~/lib/utils" +import { + getModelContextWindow, + getModelMaxOutput, +} from "~/services/copilot/get-models" export const modelRoutes = new Hono() @@ -13,15 +17,36 @@ modelRoutes.get("/", async (c) => { await cacheModels() } - const models = state.models?.data.map((model) => ({ - id: model.id, - object: "model", - type: "model", - created: 0, // No date available from source - created_at: new Date(0).toISOString(), // No date available from source - owned_by: model.vendor, - display_name: model.name, - })) + // Copilot reports conservative max_prompt_tokens (e.g. 168k) but certain + // Claude models actually accept up to ~935k tokens (1M context variant). + const MODELS_WITH_1M_CONTEXT = new Set([ + "claude-opus-4.6", + "claude-opus-4.7", + "claude-opus-4.8", + "claude-sonnet-4.6", + "gemini-3.1-pro-preview", + "gemini-3.5-flash", + "gpt-5.4", + "gpt-5.5", + ]) + const EFFECTIVE_1M_INPUT = 935_000 + + const models = state.models?.data.map((model) => { + const rawInput = getModelContextWindow(model) + const effectiveInput = + MODELS_WITH_1M_CONTEXT.has(model.id) ? EFFECTIVE_1M_INPUT : rawInput + return { + id: model.id, + object: "model", + type: "model", + created: 0, + created_at: new Date(0).toISOString(), + owned_by: model.vendor, + display_name: model.name, + max_input_tokens: effectiveInput, + max_output_tokens: getModelMaxOutput(model), + } + }) return c.json({ object: "list", diff --git a/src/routes/responses/handler.ts b/src/routes/responses/handler.ts new file mode 100644 index 000000000..1118328f0 --- /dev/null +++ b/src/routes/responses/handler.ts @@ -0,0 +1,550 @@ +import type { Context } from "hono" + +import consola from "consola" +import { events } from "fetch-event-stream" +import { streamSSE } from "hono/streaming" + +import type { ResponsesPayload } from "~/services/copilot/responses-translation" + +import { copilotHeaders, copilotBaseUrl } from "~/lib/api-config" +import { awaitApproval } from "~/lib/approval" +import { + HTTPError, + buildOpenAIContextWindowErrorBody, + buildResponsesContextWindowFailedEvent, + extractUpstreamErrorMessage, + isContextWindowError, +} from "~/lib/error" +import { resolveModelId } from "~/lib/model-resolver" +import { checkBurstLimit, checkRateLimit } from "~/lib/rate-limit" +import { state } from "~/lib/state" +import { createChatCompletions } from "~/services/copilot/create-chat-completions" +import { + requiresChatCompletionsApi, + translateFromResponsesPayloadToCC, + translateFromCCToResponsesResponse, + translateFromCCStreamToResponsesEvents, + createCCToResponsesStreamState, +} from "~/services/copilot/responses-translation" + +const INACTIVITY_TIMEOUT_MS = 5 * 60 * 1000 +const MAX_IMAGE_SEARCH_DEPTH = 12 +const IMAGE_REMOVED_PLACEHOLDER = + "[Image removed by proxy after Copilot rejected the request body]" +const imageRejectedWindowKeys = new Set<string>() + +function createInactivityAbort(timeoutMs: number = INACTIVITY_TIMEOUT_MS) { + const controller = new AbortController() + let timer: ReturnType<typeof setTimeout> | undefined + + const schedule = () => { + if (timer !== undefined) clearTimeout(timer) + timer = setTimeout(() => { + const error = new Error( + `Upstream connection inactive for ${Math.round(timeoutMs / 1000)}s`, + ) + error.name = "TimeoutError" + controller.abort(error) + }, timeoutMs) + } + + schedule() + + return { + signal: controller.signal, + keepAlive: schedule, + clear: () => { + if (timer !== undefined) { + clearTimeout(timer) + timer = undefined + } + }, + } +} + +export async function handleResponses(c: Context) { + await checkRateLimit(state) + + const payload = await c.req.json<Record<string, unknown>>() + consola.debug("Responses API request:", JSON.stringify(payload).slice(-400)) + + const model = resolveAndApplyModel(payload) + + await checkBurstLimit(state, model) + + if (state.manualApprove) await awaitApproval() + + // Claude models don't support the Responses API — translate to Chat Completions + if (requiresChatCompletionsApi(model)) { + return handleViaCC(c, payload as unknown as ResponsesPayload) + } + + if (!state.copilotToken) throw new Error("Copilot token not found") + + const inputArr = payload.input as Array<Record<string, unknown>> | undefined + const isAgentCall = isAgentInitiatedCall(inputArr) + + dropEmptyToolChoice(payload) + + // Strip fields unsupported by the Copilot responses endpoint + delete payload.store + + const inactivity = createInactivityAbort() + const response = await fetchResponsesWithImageRetry( + payload, + isAgentCall, + inactivity, + ) + + if (!response.ok) { + inactivity.clear() + return handleUpstreamError(c, response, payload.stream === true) + } + + if (payload.stream === true) { + return streamResponsesPassthrough(c, response, inactivity) + } + + inactivity.clear() + const data = await response.json() + return c.json(data) +} + +/** + * Normalizes the requested model id (e.g. `claude-opus-4-8` → + * `claude-opus-4.8`) to a real Copilot model and writes it back onto the + * payload so the upstream request forwards the canonical id. Returns the + * resolved model id. + */ +function resolveAndApplyModel(payload: Record<string, unknown>): string { + const rawModel = typeof payload.model === "string" ? payload.model : "" + const model = resolveModelId(rawModel, state.models) + if (model !== rawModel) { + consola.debug(`[model-resolver] '${rawModel}' → '${model}'`) + payload.model = model + } + return model +} + +/** + * True when the input contains an assistant turn or tool call/output, marking + * this as an agent-initiated request (forwarded via the `X-Initiator` header). + */ +function isAgentInitiatedCall( + inputArr: Array<Record<string, unknown>> | undefined, +): boolean { + return ( + Array.isArray(inputArr) + && inputArr.some((item) => { + const role = typeof item.role === "string" ? item.role : "" + const type = typeof item.type === "string" ? item.type : "" + return ["assistant", "function_call", "function_call_output"].includes( + role || type, + ) + }) + ) +} + +/** Drop tool_choice when no tools are present — the API rejects this combination. */ +function dropEmptyToolChoice(payload: Record<string, unknown>): void { + if (payload.tool_choice === undefined) return + const tools = payload.tools as Array<unknown> | undefined + if (!tools || tools.length === 0) { + delete payload.tool_choice + } +} + +/** + * Posts a body to the upstream /responses endpoint, refreshing the inactivity + * timer once the response headers arrive. + * + * Forward the body VERBATIM. We deliberately do NOT trim or mutate the input: + * Codex owns its conversation state and compacts on its own (driven by the + * usage.total_tokens it reads back from each response, against the + * model_auto_compact_token_limit in .codex/config.toml). Any proxy-side + * trimming corrupts that state — it desyncs Codex's token accounting from + * what Copilot actually received and can silently drop conversation data, + * degrading the model. If the body ever exceeds Copilot's real limit, the + * upstream returns a context-window error which we translate into the signal + * Codex needs to compact (sendResponsesContextWindowError). + */ +async function postResponsesUpstream( + body: Record<string, unknown>, + inactivity: ReturnType<typeof createInactivityAbort>, + opts: { enableVision: boolean; isAgentCall: boolean }, +): Promise<Response> { + const response = await fetch(`${copilotBaseUrl(state)}/responses`, { + method: "POST", + headers: { + ...copilotHeaders(state, opts.enableVision), + "X-Initiator": opts.isAgentCall ? "agent" : "user", + }, + body: JSON.stringify(body), + signal: inactivity.signal, + // @ts-expect-error — Bun-specific option + timeout: false, + }) + inactivity.keepAlive() + return response +} + +/** + * Sends the upstream /responses request, replacing images the upstream has + * previously rejected (or rejects on this attempt with an opaque "failed to + * parse request") with text placeholders before retrying. + */ +async function fetchResponsesWithImageRetry( + payload: Record<string, unknown>, + isAgentCall: boolean, + inactivity: ReturnType<typeof createInactivityAbort>, +): Promise<Response> { + const imageWindowKey = getImageWindowKey(payload) + let upstreamPayload = payload + let enableVision = containsInputImage(payload.input) + + if ( + enableVision + && imageWindowKey + && imageRejectedWindowKeys.has(imageWindowKey) + ) { + const strippedPayload = structuredClone(payload) + const strippedImages = stripInputImages(strippedPayload.input) + if (strippedImages > 0) { + consola.debug( + `[responses] Replacing ${strippedImages} previously rejected image(s) before upstream request for ${imageWindowKey}.`, + ) + upstreamPayload = strippedPayload + enableVision = containsInputImage(strippedPayload.input) + } + } + + const response = await postResponsesUpstream(upstreamPayload, inactivity, { + enableVision, + isAgentCall, + }) + + if ( + !response.ok + && enableVision + && (await isOpaquePayloadParseFailure(response)) + ) { + const strippedPayload = structuredClone(payload) + const strippedImages = stripInputImages(strippedPayload.input) + if (strippedImages > 0) { + if (imageWindowKey) imageRejectedWindowKeys.add(imageWindowKey) + consola.warn( + `[responses] Upstream rejected an image payload as "failed to parse request"; retrying with ${strippedImages} image(s) replaced by text placeholders.`, + ) + return postResponsesUpstream(strippedPayload, inactivity, { + enableVision: containsInputImage(strippedPayload.input), + isAgentCall, + }) + } + } + + return response +} + +/** + * Handles a non-ok upstream response: translates a context-window overflow into + * the signal Codex needs to compact, otherwise throws an HTTPError. + */ +async function handleUpstreamError( + c: Context, + response: Response, + isStreaming: boolean, +): Promise<Response> { + const ctxError = await detectContextWindowError(response) + if (ctxError) { + return sendResponsesContextWindowError(c, isStreaming, ctxError) + } + throw new HTTPError("Failed to create responses completion", response) +} + +function getImageWindowKey( + payload: Record<string, unknown>, +): string | undefined { + const windowId = findStringProperty(payload, [ + "x-codex-window-id", + "window_id", + ]) + if (windowId) return `window:${windowId}` + + const sessionId = findStringProperty(payload, ["session_id", "thread_id"]) + return sessionId ? `session:${sessionId}` : undefined +} + +function findStringProperty( + value: unknown, + names: Array<string>, + depth = 0, +): string | undefined { + if (depth > MAX_IMAGE_SEARCH_DEPTH) return undefined + if (Array.isArray(value)) { + for (const item of value) { + const found = findStringProperty(item, names, depth + 1) + if (found) return found + } + return undefined + } + if (typeof value !== "object" || value === null) return undefined + + const obj = value as Record<string, unknown> + for (const name of names) { + const candidate = obj[name] + if (typeof candidate === "string" && candidate.length > 0) { + return candidate + } + } + + for (const child of Object.values(obj)) { + const found = findStringProperty(child, names, depth + 1) + if (found) return found + } + return undefined +} + +/** Streams a successful upstream /responses SSE body straight through to the client. */ +function streamResponsesPassthrough( + c: Context, + response: Response, + inactivity: ReturnType<typeof createInactivityAbort>, +) { + return streamSSE(c, async (stream) => { + try { + for await (const event of events(response)) { + inactivity.keepAlive() + if (!event.data) continue + if (event.data === "[DONE]") { + await stream.writeSSE({ data: "[DONE]" }) + break + } + await stream.writeSSE({ + event: event.event ?? undefined, + data: event.data, + }) + } + } finally { + inactivity.clear() + } + }) +} + +/** + * Reads an upstream error response and, if it indicates a context-window + * overflow (HTTP 413 or a recognizable message), returns the extracted upstream + * message. Returns null for unrelated errors. + * + * Reads from a clone so the original response body stays intact for + * `forwardError` when this is not a context-window error. + */ +async function detectContextWindowError( + response: Response, +): Promise<{ message: string } | null> { + const errorText = await response.clone().text() + let errorJson: unknown + try { + errorJson = JSON.parse(errorText) + } catch { + errorJson = null + } + const message = extractUpstreamErrorMessage( + errorJson, + errorText, + response.headers.get("content-type"), + ) + if (!isContextWindowError(message, response.status)) return null + return { message } +} + +async function isOpaquePayloadParseFailure( + response: Response, +): Promise<boolean> { + if (response.status !== 413) return false + + const errorText = await response.clone().text() + let errorJson: unknown + try { + errorJson = JSON.parse(errorText) + } catch { + errorJson = null + } + const message = extractUpstreamErrorMessage( + errorJson, + errorText, + response.headers.get("content-type"), + ) + + return message.trim().toLowerCase() === "failed to parse request" +} + +function containsInputImage(value: unknown, depth = 0): boolean { + if (depth > MAX_IMAGE_SEARCH_DEPTH) return false + if (Array.isArray(value)) { + return value.some((item) => containsInputImage(item, depth + 1)) + } + if (typeof value !== "object" || value === null) return false + + const obj = value as Record<string, unknown> + if (obj.type === "input_image" && typeof obj.image_url === "string") { + return true + } + + return Object.values(obj).some((item) => containsInputImage(item, depth + 1)) +} + +function isInputImageObject(value: unknown): boolean { + return ( + typeof value === "object" + && value !== null + && (value as Record<string, unknown>).type === "input_image" + && typeof (value as Record<string, unknown>).image_url === "string" + ) +} + +function replacementImageText() { + return { + type: "input_text", + text: IMAGE_REMOVED_PLACEHOLDER, + } +} + +function stripInputImages(value: unknown, depth = 0): number { + if (depth > MAX_IMAGE_SEARCH_DEPTH) return 0 + + if (Array.isArray(value)) { + let stripped = 0 + for (let i = 0; i < value.length; i++) { + if (isInputImageObject(value[i])) { + value[i] = replacementImageText() + stripped += 1 + } else { + stripped += stripInputImages(value[i], depth + 1) + } + } + return stripped + } + + if (typeof value !== "object" || value === null) return 0 + + let stripped = 0 + const obj = value as Record<string, unknown> + for (const [key, child] of Object.entries(obj)) { + if (isInputImageObject(child)) { + obj[key] = replacementImageText() + stripped += 1 + } else { + stripped += stripInputImages(child, depth + 1) + } + } + return stripped +} + +/** + * Emits a context-window error in the form the OpenAI Codex CLI recognizes so + * it triggers auto-compaction instead of failing. + * + * For streaming requests this MUST be a 200 OK SSE stream carrying a + * `response.failed` event with `error.code = "context_length_exceeded"` — + * Codex's transport layer discards the body of any non-2xx response before its + * SSE parser (and thus the context-window detector) ever runs. + * + * For non-streaming requests we return a standard OpenAI-shaped 400 error + * carrying the same `code`. + */ +function sendResponsesContextWindowError( + c: Context, + isStreaming: boolean, + ctxError: { message: string }, +) { + consola.warn( + `[responses] Context window exceeded — signaling Codex to compact (code=context_length_exceeded). Upstream: "${ctxError.message}"`, + ) + + if (isStreaming) { + return streamSSE(c, async (stream) => { + const event = buildResponsesContextWindowFailedEvent(ctxError.message) + // Codex's process_sse only surfaces the parsed error once the stream + // ends, so we write the single response.failed event and let the stream + // close (matching real OpenAI, which sends no [DONE] on a failed turn). + await stream.writeSSE({ + event: event.type, + data: JSON.stringify(event), + }) + }) + } + + return c.json(buildOpenAIContextWindowErrorBody(ctxError.message), 400) +} + +async function handleViaCC(c: Context, payload: ResponsesPayload) { + const ccPayload = translateFromResponsesPayloadToCC(payload) + const isStreaming = payload.stream === true + + consola.debug( + `[responses→cc] Routing ${payload.model} through /chat/completions`, + ) + + const result = await createChatCompletions(ccPayload) + + if (isStreaming && Symbol.asyncIterator in Object(result)) { + const streamState = createCCToResponsesStreamState() + const responsesId = `resp_${Date.now()}_${crypto.randomUUID().slice(0, 8)}` + + return streamSSE(c, async (stream) => { + const responseStub = { + id: responsesId, + object: "response", + model: payload.model, + status: "in_progress", + output: [], + } + await stream.writeSSE({ + event: "response.created", + data: JSON.stringify({ + type: "response.created", + response: responseStub, + }), + }) + await stream.writeSSE({ + event: "response.in_progress", + data: JSON.stringify({ + type: "response.in_progress", + response: responseStub, + }), + }) + + for await (const event of result as AsyncIterable<{ + data?: string + event?: string + }>) { + const data = event.data ?? (event as Record<string, unknown>).data + if (!data || data === "[DONE]") continue + + let parsed: Record<string, unknown> + try { + parsed = JSON.parse(data as string) as Record<string, unknown> + } catch { + continue + } + + const responsesEvents = translateFromCCStreamToResponsesEvents( + parsed, + streamState, + ) + for (const evt of responsesEvents) { + await stream.writeSSE({ event: evt.event, data: evt.data }) + } + } + await stream.writeSSE({ data: "[DONE]" }) + }) + } + + // Non-streaming + const responsesId = `resp_${Date.now()}_${crypto.randomUUID().slice(0, 8)}` + const responsesResponse = translateFromCCToResponsesResponse( + result as import("~/services/copilot/create-chat-completions").ChatCompletionResponse, + responsesId, + ) + return c.json(responsesResponse) +} diff --git a/src/routes/responses/route.ts b/src/routes/responses/route.ts new file mode 100644 index 000000000..af2423427 --- /dev/null +++ b/src/routes/responses/route.ts @@ -0,0 +1,15 @@ +import { Hono } from "hono" + +import { forwardError } from "~/lib/error" + +import { handleResponses } from "./handler" + +export const responsesRoutes = new Hono() + +responsesRoutes.post("/", async (c) => { + try { + return await handleResponses(c) + } catch (error) { + return await forwardError(c, error) + } +}) diff --git a/src/server.ts b/src/server.ts index 462a278f3..9037abba0 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,17 +1,20 @@ import { Hono } from "hono" import { cors } from "hono/cors" -import { logger } from "hono/logger" + +import "~/lib/context-vars" +import { requestLogger } from "~/lib/request-logger" import { completionRoutes } from "./routes/chat-completions/route" import { embeddingRoutes } from "./routes/embeddings/route" import { messageRoutes } from "./routes/messages/route" import { modelRoutes } from "./routes/models/route" +import { responsesRoutes } from "./routes/responses/route" import { tokenRoute } from "./routes/token/route" import { usageRoute } from "./routes/usage/route" export const server = new Hono() -server.use(logger()) +server.use(requestLogger) server.use(cors()) server.get("/", (c) => c.text("Server running")) @@ -27,5 +30,9 @@ server.route("/v1/chat/completions", completionRoutes) server.route("/v1/models", modelRoutes) server.route("/v1/embeddings", embeddingRoutes) +// OpenAI Responses API endpoint +server.route("/v1/responses", responsesRoutes) +server.route("/responses", responsesRoutes) + // Anthropic compatible endpoints server.route("/v1/messages", messageRoutes) diff --git a/src/services/copilot/create-chat-completions.ts b/src/services/copilot/create-chat-completions.ts index 8534151da..3641f79f9 100644 --- a/src/services/copilot/create-chat-completions.ts +++ b/src/services/copilot/create-chat-completions.ts @@ -5,9 +5,139 @@ import { copilotHeaders, copilotBaseUrl } from "~/lib/api-config" import { HTTPError } from "~/lib/error" import { state } from "~/lib/state" -export const createChatCompletions = async ( +import { + translateToResponsesPayload, + translateFromResponsesResponse, + translateFromResponsesStream, + createResponsesStreamState, +} from "./responses-translation" + +// Inactivity timeout for upstream fetches. Unlike AbortSignal.timeout() which +// is a hard wall-clock deadline, this resets every time data arrives — so a +// slow-but-active stream (e.g. a 6000-line Write tool call) won't be killed +// as long as chunks keep flowing. The timeout only fires when the upstream +// goes completely silent for this duration, indicating a stalled connection. +const INACTIVITY_TIMEOUT_MS = 5 * 60 * 1000 // 5 minutes of silence +const MAX_TRANSIENT_HTTP_RETRIES = 5 +const BASE_HTTP_RETRY_DELAY_MS = 750 +const RETRIABLE_UPSTREAM_STATUS_CODES = new Set([408, 429, 500, 502, 503, 504]) + +/** + * Creates an AbortController with an inactivity timer that resets on each + * call to `keepAlive()`. If no keepAlive is received within `timeoutMs`, + * the controller aborts with a descriptive TimeoutError. + * + * Call `clear()` when the operation finishes to prevent the timer from + * firing after the stream is fully consumed. + */ +function createInactivityAbort(timeoutMs: number = INACTIVITY_TIMEOUT_MS) { + const controller = new AbortController() + let timer: ReturnType<typeof setTimeout> | undefined + + const schedule = () => { + if (timer !== undefined) clearTimeout(timer) + timer = setTimeout(() => { + const error = new Error( + `Upstream connection inactive for ${Math.round(timeoutMs / 1000)}s`, + ) + error.name = "TimeoutError" + controller.abort(error) + }, timeoutMs) + } + + // Start the initial timer immediately + schedule() + + return { + signal: controller.signal, + /** Reset the inactivity timer — call on every received chunk. */ + keepAlive: schedule, + /** Cancel the timer (call when the stream ends normally). */ + clear: () => { + if (timer !== undefined) { + clearTimeout(timer) + timer = undefined + } + }, + } +} + +function isRetriableUpstreamStatus(status: number): boolean { + return RETRIABLE_UPSTREAM_STATUS_CODES.has(status) +} + +// Substrings identifying invalid_request_body errors that are DETERMINISTIC — +// caused by the shape of the request itself, so retrying the identical payload +// can never succeed. Matching these short-circuits the retry loop instead of +// burning all MAX_TRANSIENT_HTTP_RETRIES attempts (~16s) on a doomed request. +// Matched case-insensitively against the upstream error message. +const DETERMINISTIC_BODY_ERROR_SIGNATURES = [ + "assistant message prefill", + "must end with a user message", + "must be a response to a preceeding message", // upstream's spelling + "must be a response to a preceding message", + "must have a corresponding tool_use", + "unexpected tool_use_id", + "exceeds the limit", // context-window / max-prompt-tokens errors + "exceeds the maximum", +] as const + +function isDeterministicBodyErrorMessage(message: string): boolean { + const lower = message.toLowerCase() + return DETERMINISTIC_BODY_ERROR_SIGNATURES.some((sig) => lower.includes(sig)) +} + +async function isRetriableBodyError(response: Response): Promise<boolean> { + if (response.status !== 400) return false + try { + const cloned = response.clone() + const body = (await cloned.json()) as { + error?: { code?: string; message?: string } + } + if (body.error?.code !== "invalid_request_body") return false + // Deterministic request-shape errors (assistant prefill, role ordering, + // orphaned tool_result, context-window overflow) cannot be fixed by + // retrying the same payload — fail fast instead of looping. + if ( + body.error.message + && isDeterministicBodyErrorMessage(body.error.message) + ) { + return false + } + return true + } catch { + return false + } +} + +function getRetryAfterDelayMs(retryAfter: string | null): number | undefined { + if (!retryAfter) return undefined + + const seconds = Number(retryAfter) + if (Number.isFinite(seconds) && seconds >= 0) { + return Math.round(seconds * 1000) + } + + const retryAt = Date.parse(retryAfter) + if (Number.isNaN(retryAt)) return undefined + + return Math.max(0, retryAt - Date.now()) +} + +function getTransientRetryDelayMs(response: Response, attempt: number): number { + return ( + getRetryAfterDelayMs(response.headers.get("retry-after")) + ?? BASE_HTTP_RETRY_DELAY_MS * 2 ** (attempt - 1) + ) +} + +async function sleep(ms: number): Promise<void> { + await new Promise((resolve) => setTimeout(resolve, ms)) +} + +function buildRequestHeaders( payload: ChatCompletionsPayload, -) => { +): Record<string, string> { if (!state.copilotToken) throw new Error("Copilot token not found") const enableVision = payload.messages.some( @@ -16,33 +146,206 @@ export const createChatCompletions = async ( && x.content?.some((x) => x.type === "image_url"), ) - // Agent/user check for X-Initiator header - // Determine if any message is from an agent ("assistant" or "tool") const isAgentCall = payload.messages.some((msg) => ["assistant", "tool"].includes(msg.role), ) - // Build headers and add X-Initiator - const headers: Record<string, string> = { + return { ...copilotHeaders(state, enableVision), "X-Initiator": isAgentCall ? "agent" : "user", } +} + +export const createResponsesCompletion = async ( + payload: ChatCompletionsPayload, +): Promise< + ChatCompletionResponse | AsyncIterable<import("hono/streaming").SSEMessage> +> => { + const headers = buildRequestHeaders(payload) + + const responsesPayload = translateToResponsesPayload(payload) - const response = await fetch(`${copilotBaseUrl(state)}/chat/completions`, { + const inactivity = createInactivityAbort() + + const response = await fetch(`${copilotBaseUrl(state)}/responses`, { method: "POST", headers, - body: JSON.stringify(payload), + body: JSON.stringify(responsesPayload), + signal: inactivity.signal, + // @ts-expect-error — Bun-specific option + timeout: false, }) + // Headers arrived — reset the inactivity timer + inactivity.keepAlive() + if (!response.ok) { - consola.error("Failed to create chat completions", response) - throw new HTTPError("Failed to create chat completions", response) + inactivity.clear() + throw new HTTPError("Failed to create responses completion", response) } if (payload.stream) { - return events(response) + const responseId = `resp_${Date.now()}_${crypto.randomUUID().slice(0, 8)}` + const model = payload.model + const streamState = createResponsesStreamState() + + async function* streamChunks() { + try { + consola.debug("[responses-stream] Starting stream iteration") + let eventCount = 0 + let yieldCount = 0 + for await (const event of events(response)) { + inactivity.keepAlive() + eventCount++ + consola.debug( + `[responses-stream] Raw SSE event #${eventCount}:`, + JSON.stringify({ + event: event.event, + data: event.data?.slice(0, 200), + }), + ) + if (!event.data || event.data === "[DONE]") continue + let parsed: Record<string, unknown> + try { + parsed = JSON.parse(event.data) as Record<string, unknown> + } catch { + consola.debug( + "[responses-stream] Failed to parse event data as JSON", + ) + continue + } + consola.debug( + `[responses-stream] Parsed event type: ${parsed.type as string}`, + ) + const chunk = translateFromResponsesStream(parsed, { + responseId, + model, + streamState, + }) + if (chunk) { + const chunks = Array.isArray(chunk) ? chunk : [chunk] + for (const c of chunks) { + yieldCount++ + consola.debug( + `[responses-stream] Yielding chunk #${yieldCount}:`, + JSON.stringify(c).slice(0, 200), + ) + yield c + } + } else { + consola.debug( + `[responses-stream] translateFromResponsesStream returned null for type: ${parsed.type as string}`, + ) + } + } + consola.debug( + `[responses-stream] Stream ended. Total events: ${eventCount}, yielded: ${yieldCount}`, + ) + // Emit the [DONE] sentinel after all Responses API events have been + // processed. The finish chunk (with finish_reason) is emitted by + // translateFromResponsesStream on `response.completed`; this [DONE] + // tells pipeStreamToClient to stop iterating. + yield { data: "[DONE]" } + } finally { + inactivity.clear() + } + } + + return streamChunks() + } + + inactivity.clear() + const data = await response.json() + return translateFromResponsesResponse( + data as Parameters<typeof translateFromResponsesResponse>[0], + ) +} + +export const createChatCompletions = async ( + payload: ChatCompletionsPayload, +) => { + const headers = buildRequestHeaders(payload) + + const inactivity = createInactivityAbort() + + // Newer models (gpt-5.x) reject `max_tokens` and require + // `max_completion_tokens`. Claude models still use `max_tokens`. + const { max_tokens, ...rest } = payload + const usesMaxCompletionTokens = + rest.model.startsWith("gpt-5") || rest.model.startsWith("o") + let body: Record<string, unknown> = rest + if (max_tokens !== null && max_tokens !== undefined) { + const tokenKey = + usesMaxCompletionTokens ? "max_completion_tokens" : "max_tokens" + body = { ...rest, [tokenKey]: max_tokens } } + let response: Response | undefined + + for (let attempt = 1; attempt <= MAX_TRANSIENT_HTTP_RETRIES; attempt++) { + response = await fetch(`${copilotBaseUrl(state)}/chat/completions`, { + method: "POST", + headers, + body: JSON.stringify(body), + signal: inactivity.signal, + // Bun's internal fetch timer defaults to ~4 minutes and fires mid-stream + // when Copilot pauses between chunks on large (6000+ line) file edits. + // Setting timeout:false disables it; the inactivity abort above is the + // safety net — it only fires when the upstream goes completely silent. + // @ts-expect-error — Bun-specific option, not in the standard fetch types + timeout: false, + }) + + // Headers arrived — reset the inactivity timer + inactivity.keepAlive() + + if (response.ok) break + + const shouldRetry = + isRetriableUpstreamStatus(response.status) + || (await isRetriableBodyError(response)) + + if (!shouldRetry || attempt === MAX_TRANSIENT_HTTP_RETRIES) { + inactivity.clear() + throw new HTTPError("Failed to create chat completions", response) + } + + const retryDelayMs = getTransientRetryDelayMs(response, attempt) + consola.warn( + `Copilot upstream returned ${response.status} on attempt ${attempt}/${MAX_TRANSIENT_HTTP_RETRIES}; retrying in ${retryDelayMs}ms`, + ) + if (response.body) { + void response.body.cancel().catch(() => undefined) + } + await sleep(retryDelayMs) + } + + if (!response?.ok) { + inactivity.clear() + throw new Error("Copilot upstream did not produce a response") + } + + if (payload.stream) { + // Wrap the events iterator to reset the inactivity timer on each chunk + // and clean up when the stream ends. This ensures a slow-but-active + // stream (e.g. a large Write tool call) is never killed prematurely. + const upstream = events(response) + + async function* withInactivityReset() { + try { + for await (const event of upstream) { + inactivity.keepAlive() + yield event + } + } finally { + inactivity.clear() + } + } + + return withInactivityReset() + } + + inactivity.clear() return (await response.json()) as ChatCompletionResponse } @@ -71,6 +374,10 @@ export interface ChatCompletionChunk { interface Delta { content?: string | null + /** Reasoning/thinking content from models that support it (e.g. GPT 5.4). */ + reasoning_content?: string | null + /** Reasoning text from Gemini models (equivalent to reasoning_content). */ + reasoning_text?: string | null role?: "user" | "assistant" | "system" | "tool" tool_calls?: Array<{ index: number @@ -138,7 +445,17 @@ export interface ChatCompletionsPayload { presence_penalty?: number | null logit_bias?: Record<string, number> | null logprobs?: boolean | null - response_format?: { type: "json_object" } | null + response_format?: + | { type: "json_object" } + | { + type: "json_schema" + json_schema: { + name: string + schema: Record<string, unknown> + strict?: boolean + } + } + | null seed?: number | null tools?: Array<Tool> | null tool_choice?: @@ -148,6 +465,16 @@ export interface ChatCompletionsPayload { | { type: "function"; function: { name: string } } | null user?: string | null + stream_options?: { include_usage: boolean } | null + + /** + * Reasoning controls for the Responses API path (gpt-5.x and other reasoning + * models). When the incoming Anthropic request has extended thinking enabled, + * we set `summary: "auto"` so Copilot streams `reasoning_summary_text.delta` + * events in real time — without it, the model reasons silently and no + * thinking deltas are emitted (the thinking appears all at once at the end). + */ + reasoning?: { effort?: string; summary?: string } | null } export interface Tool { @@ -156,6 +483,7 @@ export interface Tool { name: string description?: string parameters: Record<string, unknown> + strict?: boolean // Structured Outputs — forwarded from Anthropic custom tool definitions } } diff --git a/src/services/copilot/get-models.ts b/src/services/copilot/get-models.ts index 3cfa30af0..890f7b2fc 100644 --- a/src/services/copilot/get-models.ts +++ b/src/services/copilot/get-models.ts @@ -48,8 +48,50 @@ export interface Model { preview: boolean vendor: string version: string + supported_endpoints?: Array<string> policy?: { state: string terms: string } } + +/** + * Safely extracts the effective input token limit from a model. + * + * Copilot's `/models` API exposes two relevant fields: + * - `max_prompt_tokens` — the actual input ceiling Copilot enforces. + * - `max_context_window_tokens` — the total window (prompt + output). + * + * We prefer `max_prompt_tokens` because that is the limit Copilot rejects + * against, and it is what Claude Code needs as `max_input_tokens` to + * trigger proactive compaction at the right time. + * + * Some models at runtime lack `capabilities` or `limits` entirely, + * despite the TypeScript types marking them as required. + */ +export function getModelContextWindow(model: Model): number | undefined { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- some models lack capabilities at runtime + const limits = model.capabilities?.limits + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- runtime guard + if (!limits) return undefined + return limits.max_prompt_tokens ?? limits.max_context_window_tokens +} + +/** + * Safely extracts the max output tokens from a model. + * Some models at runtime lack `capabilities` or `limits` entirely. + */ +export function getModelMaxOutput(model: Model): number | undefined { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- some models lack capabilities at runtime + return model.capabilities?.limits?.max_output_tokens +} + +/** + * Safely extracts the total context window (prompt + output) from a model. + * This is `max_context_window_tokens` — the full window size, NOT the + * enforced input limit. Use `getModelContextWindow()` for the input ceiling. + */ +export function getModelTotalContext(model: Model): number | undefined { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- some models lack capabilities at runtime + return model.capabilities?.limits?.max_context_window_tokens +} diff --git a/src/services/copilot/responses-translation.test.ts b/src/services/copilot/responses-translation.test.ts new file mode 100644 index 000000000..fcd090e4c --- /dev/null +++ b/src/services/copilot/responses-translation.test.ts @@ -0,0 +1,677 @@ +import { describe, expect, test } from "bun:test" + +import type { ChatCompletionsPayload } from "./create-chat-completions" +import type { Model } from "./get-models" + +import { + translateToResponsesPayload, + translateFromResponsesResponse, + translateFromResponsesStream, + createResponsesStreamState, + requiresResponsesApi, +} from "./responses-translation" + +// ─── requiresResponsesApi ────────────────────────────────────────────────── + +describe("requiresResponsesApi", () => { + test("returns true when model only supports /responses", () => { + const model = { + supported_endpoints: ["/responses"], + } as Partial<Model> as Model + expect(requiresResponsesApi(model)).toBe(true) + }) + + test("returns true when endpoints exist but /chat/completions is absent", () => { + const model = { + supported_endpoints: ["/responses", "/some-other"], + } as Partial<Model> as Model + expect(requiresResponsesApi(model)).toBe(true) + }) + + test("returns false when model supports /chat/completions", () => { + const model = { + supported_endpoints: ["/chat/completions"], + } as Partial<Model> as Model + expect(requiresResponsesApi(model)).toBe(false) + }) + + test("returns false when model has no supported_endpoints", () => { + const model = {} as Model + expect(requiresResponsesApi(model)).toBe(false) + }) + + test("returns false when model supports both endpoints", () => { + const model = { + supported_endpoints: ["/chat/completions", "/responses"], + } as Partial<Model> as Model + expect(requiresResponsesApi(model)).toBe(false) + }) +}) + +// ─── translateToResponsesPayload ────────────────────────────────────────── + +describe("translateToResponsesPayload", () => { + test("maps messages to input", () => { + const payload: ChatCompletionsPayload = { + model: "gpt-5.4", + messages: [{ role: "user", content: "Hello" }], + } + const result = translateToResponsesPayload(payload) + expect(result.input).toEqual([{ role: "user", content: "Hello" }]) + }) + + test("extracts system message as top-level instructions", () => { + const payload: ChatCompletionsPayload = { + model: "gpt-5.4", + messages: [ + { role: "system", content: "You are helpful" }, + { role: "user", content: "Hello" }, + ], + } + const result = translateToResponsesPayload(payload) + expect(result.instructions).toBe("You are helpful") + expect(result.input).toEqual([{ role: "user", content: "Hello" }]) + }) + + test("maps max_tokens to max_output_tokens", () => { + const payload: ChatCompletionsPayload = { + model: "gpt-5.4", + messages: [{ role: "user", content: "Hi" }], + max_tokens: 1000, + } + const result = translateToResponsesPayload(payload) + expect(result.max_output_tokens).toBe(1000) + expect(result).not.toHaveProperty("max_tokens") + }) + + test("maps response_format to text.format", () => { + const payload: ChatCompletionsPayload = { + model: "gpt-5.4", + messages: [{ role: "user", content: "Hi" }], + response_format: { type: "json_object" }, + } + const result = translateToResponsesPayload(payload) + expect(result.text).toEqual({ format: { type: "json_object" } }) + }) + + test("translates tools from Chat Completions format to Responses API format", () => { + const tools = [ + { + type: "function" as const, + function: { + name: "get_weather", + description: "Get weather", + parameters: { type: "object", properties: {} }, + }, + }, + ] + const payload: ChatCompletionsPayload = { + model: "gpt-5.4", + messages: [{ role: "user", content: "Hi" }], + tools, + } + const result = translateToResponsesPayload(payload) + // Responses API requires name/description/parameters at the top level, + // not nested inside a `function` object like Chat Completions format. + expect(result.tools).toEqual([ + { + type: "function", + name: "get_weather", + description: "Get weather", + parameters: { type: "object", properties: {} }, + }, + ]) + }) + + test("passes through tool_choice unchanged", () => { + const payload: ChatCompletionsPayload = { + model: "gpt-5.4", + messages: [{ role: "user", content: "Hi" }], + tool_choice: "auto", + tools: [ + { + type: "function", + function: { + name: "get_weather", + parameters: { type: "object", properties: {} }, + }, + }, + ], + } + const result = translateToResponsesPayload(payload) + expect(result.tool_choice).toBe("auto") + }) + + test("passes through stream flag", () => { + const payload: ChatCompletionsPayload = { + model: "gpt-5.4", + messages: [{ role: "user", content: "Hi" }], + stream: true, + } + const result = translateToResponsesPayload(payload) + expect(result.stream).toBe(true) + }) + + test("omits null/undefined optional fields", () => { + const payload: ChatCompletionsPayload = { + model: "gpt-5.4", + messages: [{ role: "user", content: "Hi" }], + max_tokens: null, + temperature: null, + response_format: null, + } + const result = translateToResponsesPayload(payload) + expect(result).not.toHaveProperty("max_output_tokens") + expect(result).not.toHaveProperty("temperature") + expect(result).not.toHaveProperty("text") + }) +}) + +// ─── translateToResponsesPayload (tool calls & message repair) ─────────── + +describe("translateToResponsesPayload (tool calls & repair)", () => { + test("translates assistant messages with tool_calls into function_call items", () => { + const payload: ChatCompletionsPayload = { + model: "gpt-5.4", + messages: [ + { role: "user", content: "What is the weather?" }, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_abc123", + type: "function", + function: { + name: "get_weather", + arguments: '{"city":"London"}', + }, + }, + ], + }, + { + role: "tool", + tool_call_id: "call_abc123", + content: '{"temp": 15}', + }, + { role: "user", content: "Thanks!" }, + ], + } + const result = translateToResponsesPayload(payload) + // Should produce: user msg, function_call item, function_call_output, user msg + expect(result.input).toEqual([ + { role: "user", content: "What is the weather?" }, + { + type: "function_call", + call_id: "call_abc123", + name: "get_weather", + arguments: '{"city":"London"}', + }, + { + type: "function_call_output", + call_id: "call_abc123", + output: '{"temp": 15}', + }, + { role: "user", content: "Thanks!" }, + ]) + }) + + test("converts null content on regular messages to empty string", () => { + const payload: ChatCompletionsPayload = { + model: "gpt-5.4", + messages: [ + { role: "user", content: "Hi" }, + { role: "assistant", content: null }, + { role: "user", content: "Hello again" }, + ], + } + const result = translateToResponsesPayload(payload) + expect(result.input).toEqual([ + { role: "user", content: "Hi" }, + { role: "assistant", content: "" }, + { role: "user", content: "Hello again" }, + ]) + }) + + test("translates multimodal user content to Responses input parts", () => { + const payload: ChatCompletionsPayload = { + model: "gpt-5.4", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Describe this image." }, + { + type: "image_url", + image_url: { + url: "data:image/png;base64,abc123", + detail: "high", + }, + }, + ], + }, + ], + } + const result = translateToResponsesPayload(payload) + expect(result.input).toEqual([ + { + role: "user", + content: [ + { type: "input_text", text: "Describe this image." }, + { + type: "input_image", + image_url: "data:image/png;base64,abc123", + detail: "high", + }, + ], + }, + ]) + }) +}) + +// ─── translateFromResponsesResponse ─────────────────────────────────────── + +describe("translateFromResponsesResponse", () => { + test("translates a simple text response to chat completion shape", () => { + const responsesReply = { + id: "resp_abc123", + model: "gpt-5.4", + output: [ + { + type: "message" as const, + role: "assistant" as const, + content: [{ type: "output_text", text: "Hello there!" }], + }, + ], + usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 }, + } + const result = translateFromResponsesResponse(responsesReply) + expect(result.id).toBe("resp_abc123") + expect(result.object).toBe("chat.completion") + expect(result.model).toBe("gpt-5.4") + expect(result.choices).toHaveLength(1) + expect(result.choices[0].message.role).toBe("assistant") + expect(result.choices[0].message.content).toBe("Hello there!") + expect(result.choices[0].finish_reason).toBe("stop") + expect(result.usage?.prompt_tokens).toBe(10) + expect(result.usage?.completion_tokens).toBe(5) + }) + + test("translates function_call output to tool_calls", () => { + const responsesReply = { + id: "resp_tool", + model: "gpt-5.4", + output: [ + { + type: "function_call" as const, + call_id: "call_1", + name: "get_weather", + arguments: '{"city":"London"}', + }, + ], + usage: { input_tokens: 5, output_tokens: 3, total_tokens: 8 }, + } + const result = translateFromResponsesResponse(responsesReply) + const toolCalls = result.choices[0].message.tool_calls ?? [] + expect(toolCalls).toHaveLength(1) + expect(toolCalls[0]).toEqual({ + id: "call_1", + type: "function", + function: { name: "get_weather", arguments: '{"city":"London"}' }, + }) + expect(result.choices[0].finish_reason).toBe("tool_calls") + }) + + test("handles mixed text + function_call output", () => { + const responsesReply = { + id: "resp_mix", + model: "gpt-5.4", + output: [ + { + type: "message" as const, + role: "assistant" as const, + content: [{ type: "output_text", text: "Let me check." }], + }, + { + type: "function_call" as const, + call_id: "call_2", + name: "get_weather", + arguments: "{}", + }, + ], + usage: { input_tokens: 8, output_tokens: 6, total_tokens: 14 }, + } + const result = translateFromResponsesResponse(responsesReply) + expect(result.choices[0].message.content).toBe("Let me check.") + expect(result.choices[0].message.tool_calls).toHaveLength(1) + expect(result.choices[0].finish_reason).toBe("tool_calls") + }) + + test("sets finish_reason to stop when no tool calls", () => { + const responsesReply = { + id: "resp_stop", + model: "gpt-5.4", + output: [ + { + type: "message" as const, + role: "assistant" as const, + content: [{ type: "output_text", text: "Done." }], + }, + ], + usage: { input_tokens: 2, output_tokens: 1, total_tokens: 3 }, + } + const result = translateFromResponsesResponse(responsesReply) + expect(result.choices[0].finish_reason).toBe("stop") + }) +}) + +// ─── translateFromResponsesStream ───────────────────────────────────────── + +describe("translateFromResponsesStream", () => { + test("translates output_text delta event to SSE chunk", () => { + const state = createResponsesStreamState() + const event = { + type: "response.output_text.delta", + delta: "Hello", + item_id: "item_1", + output_index: 0, + content_index: 0, + } + const chunk = translateFromResponsesStream(event, { + responseId: "resp_xyz", + model: "gpt-5.4", + streamState: state, + }) + expect(chunk).not.toBeNull() + if (!chunk || Array.isArray(chunk)) throw new Error("unexpected result") + const parsed = JSON.parse(chunk.data as string) as { + id: string + object: string + model: string + choices: Array<{ + delta: { content?: string } + finish_reason: string | null + }> + } + expect(parsed.id).toBe("resp_xyz") + expect(parsed.object).toBe("chat.completion.chunk") + expect(parsed.model).toBe("gpt-5.4") + expect(parsed.choices[0].delta.content).toBe("Hello") + expect(parsed.choices[0].finish_reason).toBeNull() + }) + + test("translates response.completed event to finish chunk (not [DONE])", () => { + const state = createResponsesStreamState() + const event = { type: "response.completed", response: { id: "resp_xyz" } } + const chunks = translateFromResponsesStream(event, { + responseId: "resp_xyz", + model: "gpt-5.4", + streamState: state, + }) + expect(chunks).not.toBeNull() + expect(Array.isArray(chunks)).toBe(true) + const arr = chunks as Array<{ data: string }> + // First chunk: finish with stop reason + const parsed = JSON.parse(arr[0].data) as { + choices: Array<{ delta: Record<string, unknown>; finish_reason: string }> + } + expect(parsed.choices[0].delta).toEqual({}) + // No tool calls seen → finish_reason is "stop" + expect(parsed.choices[0].finish_reason).toBe("stop") + }) + + test("response.completed emits tool_calls finish_reason when tool calls were present", () => { + const state = createResponsesStreamState() + + // Simulate a tool call being added + const addedEvent = { + type: "response.output_item.added", + item: { call_id: "call_123", name: "my_tool" }, + } + translateFromResponsesStream(addedEvent, { + responseId: "resp_xyz", + model: "gpt-5.4", + streamState: state, + }) + + const completedEvent = { + type: "response.completed", + response: { id: "resp_xyz" }, + } + const chunks = translateFromResponsesStream(completedEvent, { + responseId: "resp_xyz", + model: "gpt-5.4", + streamState: state, + }) + expect(chunks).not.toBeNull() + expect(Array.isArray(chunks)).toBe(true) + const arr = chunks as Array<{ data: string }> + const parsed = JSON.parse(arr[0].data) as { + choices: Array<{ delta: Record<string, unknown>; finish_reason: string }> + } + expect(parsed.choices[0].finish_reason).toBe("tool_calls") + }) + + test("response.output_text.done returns null (finish emitted on response.completed)", () => { + const state = createResponsesStreamState() + const event = { + type: "response.output_text.done", + text: "full text", + item_id: "item_1", + output_index: 0, + content_index: 0, + } + const chunk = translateFromResponsesStream(event, { + responseId: "resp_xyz", + model: "gpt-5.4", + streamState: state, + }) + // output_text.done no longer emits a finish chunk — that happens + // on response.completed so multi-output-item responses work correctly. + expect(chunk).toBeNull() + }) +}) + +// ─── translateFromResponsesStream (tool calls) ─────────────────────────── + +describe("translateFromResponsesStream (tool calls)", () => { + test("translates function_call delta event to tool_calls delta chunk (without prior output_item.added)", () => { + const state = createResponsesStreamState() + const event = { + type: "response.function_call_arguments.delta", + delta: '{"city":', + item_id: "call_1", + output_index: 0, + } + const chunk = translateFromResponsesStream(event, { + responseId: "resp_xyz", + model: "gpt-5.4", + streamState: state, + }) + expect(chunk).not.toBeNull() + if (!chunk || Array.isArray(chunk)) throw new Error("unexpected result") + const parsed = JSON.parse(chunk.data as string) as { + choices: Array<{ + delta: { tool_calls: Array<{ function: { arguments: string } }> } + finish_reason: string | null + }> + } + expect(parsed.choices[0].delta.tool_calls[0].function.arguments).toBe( + '{"city":', + ) + }) + + test("attaches call_id and name from output_item.added to first function_call delta", () => { + const state = createResponsesStreamState() + + // First, the Responses API sends the output_item.added event with function call metadata. + // Note: item.id is an opaque encrypted string that will NOT match the item_id + // on delta events — this mirrors real Copilot API behavior. + const addedEvent = { + type: "response.output_item.added", + item: { + type: "function_call", + id: "encrypted_item_id_abc", + call_id: "call_abc123", + name: "get_weather", + arguments: "", + }, + } + const addedResult = translateFromResponsesStream(addedEvent, { + responseId: "resp_xyz", + model: "gpt-5.4", + streamState: state, + }) + expect(addedResult).toBeNull() // output_item.added itself returns null + + // Then the first arguments delta should include call_id and name, + // even though its item_id differs from the output_item.added item.id. + const deltaEvent = { + type: "response.function_call_arguments.delta", + delta: '{"city":', + item_id: "different_encrypted_item_id_xyz", + output_index: 0, + } + const chunk = translateFromResponsesStream(deltaEvent, { + responseId: "resp_xyz", + model: "gpt-5.4", + streamState: state, + }) + expect(chunk).not.toBeNull() + if (!chunk || Array.isArray(chunk)) throw new Error("unexpected result") + const parsed = JSON.parse(chunk.data as string) as { + choices: Array<{ + delta: { + tool_calls: Array<{ + index: number + id?: string + type?: string + function: { name?: string; arguments: string } + }> + } + }> + } + const tc = parsed.choices[0].delta.tool_calls[0] + expect(tc.id).toBe("call_abc123") + expect(tc.type).toBe("function") + expect(tc.function.name).toBe("get_weather") + expect(tc.function.arguments).toBe('{"city":') + + // Subsequent deltas should NOT include id/name again + const delta2Event = { + type: "response.function_call_arguments.delta", + delta: '"London"}', + item_id: "different_encrypted_item_id_xyz", + output_index: 0, + } + const chunk2 = translateFromResponsesStream(delta2Event, { + responseId: "resp_xyz", + model: "gpt-5.4", + streamState: state, + }) + expect(chunk2).not.toBeNull() + if (!chunk2 || Array.isArray(chunk2)) throw new Error("unexpected result") + const parsed2 = JSON.parse(chunk2.data as string) as { + choices: Array<{ + delta: { + tool_calls: Array<{ + index: number + id?: string + type?: string + function: { name?: string; arguments: string } + }> + } + }> + } + const tc2 = parsed2.choices[0].delta.tool_calls[0] + expect(tc2.id).toBeUndefined() + expect(tc2.type).toBeUndefined() + expect(tc2.function.name).toBeUndefined() + expect(tc2.function.arguments).toBe('"London"}') + }) + + test("returns null for unrecognised event types", () => { + const state = createResponsesStreamState() + const event = { type: "response.created", response: {} } + const chunk = translateFromResponsesStream(event, { + responseId: "resp_xyz", + model: "gpt-5.4", + streamState: state, + }) + expect(chunk).toBeNull() + }) +}) + +// ─── translateFromResponsesStream (reasoning) ───────────────────────────── + +describe("translateFromResponsesStream (reasoning)", () => { + test("translates reasoning_summary_text.delta to chunk with reasoning_content", () => { + const state = createResponsesStreamState() + const event = { + type: "response.reasoning_summary_text.delta", + delta: "Let me think about this...", + item_id: "item_1", + output_index: 0, + } + const chunk = translateFromResponsesStream(event, { + responseId: "resp_xyz", + model: "gpt-5.4", + streamState: state, + }) + expect(chunk).not.toBeNull() + if (!chunk || Array.isArray(chunk)) throw new Error("unexpected result") + const parsed = JSON.parse(chunk.data as string) as { + choices: Array<{ + delta: { reasoning_content?: string; content?: string } + finish_reason: string | null + }> + } + expect(parsed.choices[0].delta.reasoning_content).toBe( + "Let me think about this...", + ) + expect(parsed.choices[0].delta.content).toBeUndefined() + expect(parsed.choices[0].finish_reason).toBeNull() + }) + + test("reasoning_summary_text.done returns null", () => { + const state = createResponsesStreamState() + const event = { + type: "response.reasoning_summary_text.done", + text: "full reasoning", + } + const chunk = translateFromResponsesStream(event, { + responseId: "resp_xyz", + model: "gpt-5.4", + streamState: state, + }) + expect(chunk).toBeNull() + }) + + test("reasoning_summary_part.added returns null", () => { + const state = createResponsesStreamState() + const event = { + type: "response.reasoning_summary_part.added", + item: {}, + } + const chunk = translateFromResponsesStream(event, { + responseId: "resp_xyz", + model: "gpt-5.4", + streamState: state, + }) + expect(chunk).toBeNull() + }) + + test("reasoning_summary_part.done returns null", () => { + const state = createResponsesStreamState() + const event = { + type: "response.reasoning_summary_part.done", + item: {}, + } + const chunk = translateFromResponsesStream(event, { + responseId: "resp_xyz", + model: "gpt-5.4", + streamState: state, + }) + expect(chunk).toBeNull() + }) +}) diff --git a/src/services/copilot/responses-translation.ts b/src/services/copilot/responses-translation.ts new file mode 100644 index 000000000..9597796ad --- /dev/null +++ b/src/services/copilot/responses-translation.ts @@ -0,0 +1,1326 @@ +/* eslint-disable max-lines */ +/** + * Translation helpers between OpenAI Chat Completions format and Responses API format. + */ + +import type { SSEMessage } from "hono/streaming" + +import { repairOrphanedToolCalls } from "~/lib/tool-call-repair" + +import type { + ContentPart, + ChatCompletionResponse, + ChatCompletionsPayload, + Tool, + ToolCall, +} from "./create-chat-completions" +import type { Model } from "./get-models" + +// ─── Routing helper ────────────────────────────────────────────────────────── + +/** + * Returns true if the model does not support /chat/completions and should + * be routed through the /responses endpoint instead. + * + * This handles models like gpt-5.4-mini that appear in the model list but + * whose `supported_endpoints` either explicitly excludes /chat/completions + * or only lists /responses. + */ +export function requiresResponsesApi(model: Model): boolean { + if (!Array.isArray(model.supported_endpoints)) return false + return !model.supported_endpoints.includes("/chat/completions") +} + +// ─── Responses API payload types ───────────────────────────────────────────── + +// Tool format for the Responses API — Codex sends various tool types: +// function, local_shell, custom, web_search, image_generation, namespace, tool_search +export interface ResponsesTool { + type: string + name?: string + description?: string + parameters?: Record<string, unknown> + strict?: boolean + [key: string]: unknown +} + +export interface ResponsesPayload { + model: string + input: Array<ResponsesInputItem> + instructions?: string + max_output_tokens?: number + temperature?: number + top_p?: number + stream?: boolean | null + tools?: Array<ResponsesTool> + tool_choice?: ChatCompletionsPayload["tool_choice"] + parallel_tool_calls?: boolean + reasoning?: { effort?: string; summary?: string } + text?: { + format: { + type: string + name?: string + schema?: Record<string, unknown> + strict?: boolean + } + } +} + +// Responses API accepts three kinds of input items: +// 1. A message (user/assistant/developer with content) +// 2. A function_call (assistant deciding to call a tool) +// 3. A function_call_output (tool result) +type ResponsesInputItem = + | { role: string; content: string | Array<ResponsesContentPart> } + | { type: "function_call"; call_id: string; name: string; arguments: string } + | { type: "function_call_output"; call_id: string; output: string } + +type ResponsesContentPart = + | { type: "input_text"; text: string } + | { type: "input_image"; image_url: string; detail?: "low" | "high" | "auto" } + +// ─── Responses API response types ──────────────────────────────────────────── + +interface ResponsesOutputMessage { + type: "message" + role: "assistant" + content: Array<{ type: string; text?: string }> +} + +interface ResponsesFunctionCall { + type: "function_call" + call_id: string + name: string + arguments: string +} + +type ResponsesOutputItem = ResponsesOutputMessage | ResponsesFunctionCall + +interface ResponsesResponse { + id: string + model: string + output: Array<ResponsesOutputItem> + usage: { + input_tokens: number + output_tokens: number + total_tokens: number + } +} + +// ─── Payload translation: Chat Completions → Responses API ─────────────────── + +export function translateToResponsesPayload( + payload: ChatCompletionsPayload, +): ResponsesPayload { + // Separate system message from the rest + const systemMsg = payload.messages.find((m) => m.role === "system") + const otherMessages = payload.messages.filter((m) => m.role !== "system") + + return { + model: payload.model, + input: translateMessagesToResponsesInput(otherMessages), + ...buildSystemInstruction(systemMsg), + ...buildOptionalScalars(payload), + ...buildTextFormat(payload.response_format), + } +} + +/** + * Translates OpenAI Chat Completions messages into the Responses API input format. + * + * Key differences: + * - Assistant messages with tool_calls → one or more `function_call` items + * - Tool result messages (role: "tool") → `function_call_output` items + * - Null content on assistant messages → empty string (Responses API rejects null) + */ +function translateMessagesToResponsesInput( + messages: Array<import("./create-chat-completions").Message>, +): Array<ResponsesInputItem> { + const items: Array<ResponsesInputItem> = [] + + for (const msg of messages) { + // Tool result messages → function_call_output + if (msg.role === "tool" && msg.tool_call_id) { + let output: string + if (typeof msg.content === "string") { + output = msg.content + } else if (msg.content !== null) { + output = JSON.stringify(msg.content) + } else { + output = "" + } + items.push({ + type: "function_call_output", + call_id: msg.tool_call_id, + output, + }) + continue + } + + // Assistant messages with tool_calls → emit function_call items + // (plus a text message if the assistant also produced content) + if ( + msg.role === "assistant" + && msg.tool_calls + && msg.tool_calls.length > 0 + ) { + // If the assistant also has text content, emit it as a message first + if (msg.content !== null && msg.content !== "") { + items.push({ + role: msg.role, + content: + typeof msg.content === "string" ? + msg.content + : JSON.stringify(msg.content), + }) + } + + // Emit each tool call as a function_call input item + for (const tc of msg.tool_calls) { + items.push({ + type: "function_call", + call_id: tc.id, + name: tc.function.name, + arguments: tc.function.arguments, + }) + } + continue + } + + // Regular messages — ensure content is never null + items.push({ + role: msg.role, + content: translateMessageContentToResponses(msg.content), + }) + } + + return items +} + +function translateMessageContentToResponses( + content: import("./create-chat-completions").Message["content"], +): string | Array<ResponsesContentPart> { + if (content === null) return "" + if (typeof content === "string") return content + return content.map((part) => translateContentPartToResponses(part)) +} + +function translateContentPartToResponses( + part: ContentPart, +): ResponsesContentPart { + if (part.type === "text") { + return { + type: "input_text", + text: part.text, + } + } + + return { + type: "input_image", + image_url: part.image_url.url, + ...(part.image_url.detail !== undefined && { + detail: part.image_url.detail, + }), + } +} + +function buildSystemInstruction( + systemMsg: ChatCompletionsPayload["messages"][number] | undefined, +): Pick<ResponsesPayload, "instructions"> | Record<string, never> { + if ( + systemMsg?.content !== null + && systemMsg?.content !== undefined + && typeof systemMsg.content === "string" + ) { + return { instructions: systemMsg.content } + } + return {} +} + +function buildResponsesTools( + tools: NonNullable<ChatCompletionsPayload["tools"]>, +): Array<ResponsesTool> { + return tools.map((tool) => ({ + type: "function" as const, + name: tool.function.name, + ...(tool.function.description !== undefined && { + description: tool.function.description, + }), + parameters: tool.function.parameters, + ...(tool.function.strict !== undefined && { + strict: tool.function.strict, + }), + })) +} + +/** True when a value is neither null nor undefined (narrows the type). */ +function isSet<T>(value: T): value is NonNullable<T> { + return value !== null && value !== undefined +} + +function buildOptionalScalars( + payload: ChatCompletionsPayload, +): Partial<ResponsesPayload> { + const out: Partial<ResponsesPayload> = {} + if (isSet(payload.max_tokens)) out.max_output_tokens = payload.max_tokens + if (isSet(payload.temperature)) out.temperature = payload.temperature + if (isSet(payload.top_p)) out.top_p = payload.top_p + if (isSet(payload.stream)) out.stream = payload.stream + if (isSet(payload.tools)) out.tools = buildResponsesTools(payload.tools) + if (isSet(payload.tool_choice) && out.tools && out.tools.length > 0) + out.tool_choice = payload.tool_choice + // Forward reasoning controls (effort + summary). `summary: "auto"` is what + // makes Copilot stream reasoning_summary_text.delta events in real time so + // the thinking block renders incrementally instead of all at once. + if (isSet(payload.reasoning)) out.reasoning = payload.reasoning + return out +} + +function buildTextFormat( + responseFormat: ChatCompletionsPayload["response_format"], +): Pick<ResponsesPayload, "text"> | Record<string, never> { + if (responseFormat !== null && responseFormat !== undefined) { + if (responseFormat.type === "json_schema") { + return { + text: { + format: { + type: "json_schema", + name: responseFormat.json_schema.name, + schema: responseFormat.json_schema.schema, + strict: responseFormat.json_schema.strict, + }, + }, + } + } + return { text: { format: { type: responseFormat.type } } } + } + return {} +} + +// ─── Routing helper: models that don't support the Responses API ───────────── +// Allow-list approach: only models known to support /responses go direct. +// Everything else is routed through /chat/completions. +const RESPONSES_API_PREFIXES = ["gpt-4.1", "gpt-5", "o1", "o3", "o4"] + +const RESPONSES_API_EXACT = new Set(["gpt-41-copilot"]) + +export function requiresChatCompletionsApi(model: string): boolean { + if (RESPONSES_API_EXACT.has(model)) return false + if (RESPONSES_API_PREFIXES.some((p) => model.startsWith(p))) return false + return true +} + +// ─── Payload translation: Responses API → Chat Completions ────────────────── +export function translateFromResponsesPayloadToCC( + payload: ResponsesPayload, +): ChatCompletionsPayload { + const messages: Array<import("./create-chat-completions").Message> = [] + + if (payload.instructions) { + messages.push({ role: "system", content: payload.instructions }) + } + + for (const item of payload.input) { + const msg = translateInputItemToMessage(item) + if (msg) messages.push(msg) + } + + repairOrphanedToolCalls(messages) + + const result: ChatCompletionsPayload = { + model: payload.model, + messages, + } + + applyOptionalPayloadFields(payload, result) + + return result +} + +function translateInputItemToMessage( + item: ResponsesInputItem, +): import("./create-chat-completions").Message | null { + const rawItem = item as Record<string, unknown> + const type = rawItem.type as string | undefined + + if (type === "function_call") { + return { + role: "assistant", + content: null, + tool_calls: [ + { + id: rawItem.call_id as string, + type: "function", + function: { + name: rawItem.name as string, + arguments: rawItem.arguments as string, + }, + }, + ], + } + } + if (type === "function_call_output") { + return { + role: "tool", + content: rawItem.output as string, + tool_call_id: rawItem.call_id as string, + } + } + + const content = translateResponsesContentToCC( + rawItem.content as string | Array<ResponsesContentPart>, + ) + if (content === null) return null + + return { + role: rawItem.role as string as + | "user" + | "assistant" + | "system" + | "developer", + content, + } +} + +function applyOptionalPayloadFields( + payload: ResponsesPayload, + result: ChatCompletionsPayload, +): void { + if (payload.max_output_tokens !== undefined) + result.max_tokens = payload.max_output_tokens + if (payload.temperature !== undefined) + result.temperature = payload.temperature + if (payload.top_p !== undefined) result.top_p = payload.top_p + if (payload.stream !== undefined) result.stream = payload.stream + if (payload.stream) { + result.stream_options = { include_usage: true } + } + + applyToolsAndFormat(payload, result) + + if ( + payload.tool_choice !== undefined + && result.tools + && result.tools.length > 0 + ) + result.tool_choice = payload.tool_choice +} + +function applyToolsAndFormat( + payload: ResponsesPayload, + result: ChatCompletionsPayload, +): void { + if (payload.tools && payload.tools.length > 0) { + const ccTools = payload.tools + .map((t) => responsesToolToCC(t)) + .filter((t): t is NonNullable<typeof t> => t !== null) + if (ccTools.length > 0) { + result.tools = ccTools + } + } + + if (payload.text?.format) { + const fmt = payload.text.format + if (fmt.type === "json_schema" && fmt.name && fmt.schema) { + result.response_format = { + type: "json_schema", + json_schema: { + name: fmt.name, + schema: fmt.schema, + strict: fmt.strict, + }, + } + } else if (fmt.type === "json_object") { + result.response_format = { type: "json_object" } + } + } +} + +function responsesToolToCC(t: ResponsesTool): Tool | null { + if (t.type === "function" && t.name) { + return { + type: "function" as const, + function: { + name: t.name, + description: t.description, + parameters: t.parameters ?? {}, + ...(t.strict !== undefined && { strict: t.strict }), + }, + } + } + if (t.type === "local_shell") { + return { + type: "function" as const, + function: { + name: "shell", + description: "Execute a shell command", + parameters: { + type: "object", + properties: { + command: { + type: "array", + items: { type: "string" }, + description: "Command and arguments to execute", + }, + }, + required: ["command"], + }, + }, + } + } + if (t.type === "custom" && t.name) { + return { + type: "function" as const, + function: { + name: t.name, + description: t.description, + parameters: t.parameters ?? {}, + }, + } + } + return null +} + +function translateResponsesContentToCC( + content: string | Array<ResponsesContentPart> | null | undefined, +): string | Array<ContentPart> | null { + if (content === null || content === undefined) return null + if (typeof content === "string") return content + + const parts: Array<ContentPart> = [] + for (const part of content) { + if (part.type === "input_text") { + parts.push({ type: "text" as const, text: part.text }) + } else if ("image_url" in part && part.image_url) { + parts.push({ + type: "image_url" as const, + image_url: { + url: part.image_url, + ...(part.detail && { detail: part.detail }), + }, + }) + } + } + + if (parts.length === 0) return null + if (parts.length === 1 && parts[0].type === "text") return parts[0].text + return parts +} + +// ─── Response translation: Chat Completions → Responses API ───────────────── +export function translateFromCCToResponsesResponse( + resp: ChatCompletionResponse, + responsesId?: string, +): Record<string, unknown> { + const id = responsesId ?? `resp_${Date.now()}` + const choice = resp.choices.at(0) + if (!choice) { + return { id, object: "response", model: resp.model, output: [], usage: {} } + } + + const output: Array<Record<string, unknown>> = [] + + if (choice.message.content) { + output.push({ + type: "message", + role: "assistant", + content: [{ type: "output_text", text: choice.message.content }], + }) + } + + if (choice.message.tool_calls) { + for (const tc of choice.message.tool_calls) { + output.push({ + type: "function_call", + call_id: tc.id, + name: tc.function.name, + arguments: tc.function.arguments, + }) + } + } + + return { + id, + object: "response", + model: resp.model, + output, + usage: { + input_tokens: resp.usage?.prompt_tokens ?? 0, + output_tokens: resp.usage?.completion_tokens ?? 0, + total_tokens: resp.usage?.total_tokens ?? 0, + }, + } +} + +// ─── Stream translation: CC SSE chunk → Responses API SSE events ──────────── +export interface CCToResponsesStreamState { + outputIndex: number + textItemAdded: boolean + reasoningSummaryAdded: boolean + pendingToolCalls: Map<number, { id: string; name: string }> + toolItemsAdded: Set<number> + usage: { input_tokens: number; output_tokens: number; total_tokens: number } + accumulatedText: string + accumulatedReasoningText: string + accumulatedToolArgs: Map<number, string> +} + +export function createCCToResponsesStreamState(): CCToResponsesStreamState { + return { + outputIndex: 0, + textItemAdded: false, + reasoningSummaryAdded: false, + pendingToolCalls: new Map(), + toolItemsAdded: new Set(), + usage: { input_tokens: 0, output_tokens: 0, total_tokens: 0 }, + accumulatedText: "", + accumulatedReasoningText: "", + accumulatedToolArgs: new Map(), + } +} + +interface ResponsesSSEEvent { + event: string + data: string +} + +export function translateFromCCStreamToResponsesEvents( + chunk: Record<string, unknown>, + streamState: CCToResponsesStreamState, +): Array<ResponsesSSEEvent> { + // Capture usage from the final chunk (sent when stream_options.include_usage is set) + const usage = chunk.usage as Record<string, number> | undefined + if (usage) { + streamState.usage = { + input_tokens: usage.prompt_tokens, + output_tokens: usage.completion_tokens, + total_tokens: usage.total_tokens, + } + } + + const choices = chunk.choices as Array<Record<string, unknown>> | undefined + if (!choices || choices.length === 0) return [] + + const choice = choices[0] + const delta = choice.delta as Record<string, unknown> | undefined + if (!delta) return [] + + const result: Array<ResponsesSSEEvent> = [] + const responseId = chunk.id as string + const model = chunk.model as string + + if (delta.content && typeof delta.content === "string") { + handleCCTextDelta(delta.content, streamState, result) + } + + const reasoning = + (delta.reasoning_content as string | undefined) + ?? (delta.reasoning_text as string | undefined) + if (reasoning) { + handleCCReasoningDelta(reasoning, streamState, result) + } + + const toolCalls = delta.tool_calls as + | Array<Record<string, unknown>> + | undefined + if (toolCalls) { + handleCCToolCallDeltas(toolCalls, streamState, result) + } + + const finishReason = choice.finish_reason as string | null + if (finishReason) { + handleCCFinishReason(finishReason, responseId, { + model, + out: result, + streamState, + }) + } + + return result +} + +function handleCCTextDelta( + content: string, + streamState: CCToResponsesStreamState, + out: Array<ResponsesSSEEvent>, +): void { + streamState.accumulatedText += content + if (!streamState.textItemAdded) { + streamState.textItemAdded = true + out.push( + { + event: "response.output_item.added", + data: JSON.stringify({ + type: "response.output_item.added", + output_index: streamState.outputIndex, + item: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: "" }], + }, + }), + }, + { + event: "response.content_part.added", + data: JSON.stringify({ + type: "response.content_part.added", + output_index: streamState.outputIndex, + content_index: 0, + part: { type: "output_text", text: "" }, + }), + }, + ) + } + out.push({ + event: "response.output_text.delta", + data: JSON.stringify({ + type: "response.output_text.delta", + output_index: 0, + content_index: 0, + delta: content, + }), + }) +} + +function handleCCReasoningDelta( + content: string, + streamState: CCToResponsesStreamState, + out: Array<ResponsesSSEEvent>, +): void { + streamState.accumulatedReasoningText += content + if (!streamState.reasoningSummaryAdded) { + streamState.reasoningSummaryAdded = true + out.push({ + event: "response.reasoning_summary_part.added", + data: JSON.stringify({ + type: "response.reasoning_summary_part.added", + output_index: streamState.outputIndex, + summary_index: 0, + part: { type: "summary_text", text: "" }, + }), + }) + } + out.push({ + event: "response.reasoning_summary_text.delta", + data: JSON.stringify({ + type: "response.reasoning_summary_text.delta", + output_index: streamState.outputIndex, + summary_index: 0, + delta: content, + }), + }) +} + +function getToolOutputIndex( + tcIndex: number, + streamState: CCToResponsesStreamState, +): number { + return streamState.textItemAdded ? + streamState.outputIndex + 1 + tcIndex + : streamState.outputIndex + tcIndex +} + +function handleCCToolCallDeltas( + toolCalls: Array<Record<string, unknown>>, + streamState: CCToResponsesStreamState, + out: Array<ResponsesSSEEvent>, +): void { + for (const tc of toolCalls) { + const index = (tc.index as number | undefined) ?? 0 + const fn = tc.function as Record<string, unknown> | undefined + + if (tc.id && fn?.name) { + streamState.pendingToolCalls.set(index, { + id: tc.id as string, + name: fn.name as string, + }) + } + + if ( + !streamState.toolItemsAdded.has(index) + && streamState.pendingToolCalls.has(index) + ) { + streamState.toolItemsAdded.add(index) + const info = streamState.pendingToolCalls.get(index) + if (info) { + out.push({ + event: "response.output_item.added", + data: JSON.stringify({ + type: "response.output_item.added", + output_index: getToolOutputIndex(index, streamState), + item: { + type: "function_call", + call_id: info.id, + name: info.name, + arguments: "", + }, + }), + }) + } + } + + if (fn?.arguments && typeof fn.arguments === "string") { + const prev = streamState.accumulatedToolArgs.get(index) ?? "" + streamState.accumulatedToolArgs.set(index, prev + fn.arguments) + out.push({ + event: "response.function_call_arguments.delta", + data: JSON.stringify({ + type: "response.function_call_arguments.delta", + output_index: getToolOutputIndex(index, streamState), + delta: fn.arguments, + }), + }) + } + } +} + +function handleCCFinishReason( + finishReason: string, + responseId: string, + { + model, + out, + streamState, + }: { + model: string + out: Array<ResponsesSSEEvent> + streamState: CCToResponsesStreamState + }, +): void { + if (finishReason === "length" && streamState.pendingToolCalls.size > 0) { + handleCCTextDelta( + "\n\n[Tool call was truncated due to output token limit. Please retry with a higher max_output_tokens.]", + streamState, + out, + ) + } + + emitDoneEvents(streamState, out) + + const status = finishReason === "length" ? "incomplete" : "completed" + out.push({ + event: "response.completed", + data: JSON.stringify({ + type: "response.completed", + response: { + id: responseId, + object: "response", + model, + status, + output: [], + usage: streamState.usage, + }, + }), + }) +} + +function emitDoneEvents( + streamState: CCToResponsesStreamState, + out: Array<ResponsesSSEEvent>, +): void { + if (streamState.reasoningSummaryAdded) { + out.push( + { + event: "response.reasoning_summary_text.done", + data: JSON.stringify({ + type: "response.reasoning_summary_text.done", + output_index: streamState.outputIndex, + summary_index: 0, + text: streamState.accumulatedReasoningText, + }), + }, + { + event: "response.reasoning_summary_part.done", + data: JSON.stringify({ + type: "response.reasoning_summary_part.done", + output_index: streamState.outputIndex, + summary_index: 0, + part: { + type: "summary_text", + text: streamState.accumulatedReasoningText, + }, + }), + }, + ) + } + + if (streamState.textItemAdded) { + out.push( + { + event: "response.content_part.done", + data: JSON.stringify({ + type: "response.content_part.done", + output_index: streamState.outputIndex, + content_index: 0, + part: { type: "output_text", text: streamState.accumulatedText }, + }), + }, + { + event: "response.output_item.done", + data: JSON.stringify({ + type: "response.output_item.done", + output_index: streamState.outputIndex, + item: { + type: "message", + role: "assistant", + content: [ + { type: "output_text", text: streamState.accumulatedText }, + ], + }, + }), + }, + ) + } + + for (const index of streamState.toolItemsAdded) { + const info = streamState.pendingToolCalls.get(index) + if (!info) continue + let args = streamState.accumulatedToolArgs.get(index) ?? "" + + // If the stream was truncated (finish_reason: "length"), tool call + // arguments may be incomplete JSON. Try to repair by closing open + // braces/brackets; if that fails, skip emitting this tool call entirely + // so the client doesn't choke on unparseable arguments. + try { + JSON.parse(args) + } catch { + const repaired = tryRepairJson(args) + if (repaired === null) continue + args = repaired + } + + out.push( + { + event: "response.function_call_arguments.done", + data: JSON.stringify({ + type: "response.function_call_arguments.done", + output_index: getToolOutputIndex(index, streamState), + call_id: info.id, + name: info.name, + arguments: args, + }), + }, + { + event: "response.output_item.done", + data: JSON.stringify({ + type: "response.output_item.done", + output_index: getToolOutputIndex(index, streamState), + item: { + type: "function_call", + call_id: info.id, + name: info.name, + arguments: args, + }, + }), + }, + ) + } +} + +export function translateFromResponsesResponse( + resp: ResponsesResponse, +): ChatCompletionResponse { + let textContent: string | null = null + const toolCalls: Array<ToolCall> = [] + + for (const item of resp.output) { + if (item.type === "message") { + const texts = item.content + .filter( + (c) => + c.type === "output_text" && c.text !== undefined && c.text !== "", + ) + .map((c) => c.text as string) + if (texts.length > 0) { + textContent = texts.join("\n\n") + } + } else { + // function_call — the union guarantees this branch + toolCalls.push({ + id: item.call_id, + type: "function", + function: { + name: item.name, + arguments: item.arguments, + }, + }) + } + } + + const finishReason = toolCalls.length > 0 ? "tool_calls" : "stop" + + return { + id: resp.id, + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model: resp.model, + choices: [ + { + index: 0, + message: { + role: "assistant", + content: textContent, + ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}), + }, + logprobs: null, + finish_reason: finishReason, + }, + ], + usage: { + prompt_tokens: resp.usage.input_tokens, + completion_tokens: resp.usage.output_tokens, + total_tokens: resp.usage.total_tokens, + }, + } +} + +// ─── Stream translation: Responses API SSE event → Chat Completion SSE chunk ─ + +/** + * Mutable state shared across a single streaming response so that + * `response.output_item.added` can hand off the tool-call identity + * (call_id + name) to the subsequent `function_call_arguments.delta` chunks. + * + * IDs from the Copilot API may be encrypted/opaque, so `item.id` from + * `output_item.added` will NOT match `item_id` from the delta events. + * We therefore use a FIFO queue: the Responses API always sends + * `output_item.added` before its corresponding `function_call_arguments.delta` + * events, so we push identity info onto the queue and shift it off when the + * first delta for a new tool call arrives. + */ +export interface ResponsesStreamState { + /** FIFO queue of tool-call identities waiting to be attached to deltas. */ + pendingToolCalls: Array<{ call_id: string; name: string }> + /** Whether the current tool call's first delta has already been sent. */ + currentToolCallSent: boolean + /** Whether any tool calls were seen during this response (for finish_reason). */ + hasToolCalls: boolean + /** Whether any text content was seen during this response. */ + hasTextContent: boolean + /** Index of the current tool call (increments per tool call for OpenAI delta format). */ + toolCallIndex: number + /** Usage data extracted from response.completed event. */ + usage?: { + prompt_tokens: number + completion_tokens: number + total_tokens: number + } +} + +export function createResponsesStreamState(): ResponsesStreamState { + return { + pendingToolCalls: [], + currentToolCallSent: false, + hasToolCalls: false, + hasTextContent: false, + toolCallIndex: -1, + } +} + +export interface TranslateStreamOptions { + responseId: string + model: string + streamState: ResponsesStreamState +} + +export function translateFromResponsesStream( + event: Record<string, unknown>, + options: TranslateStreamOptions, +): SSEMessage | Array<SSEMessage> | null { + const { responseId, model, streamState } = options + const type = event.type as string + + if (type === "response.output_text.delta") { + streamState.hasTextContent = true + return makeTextDeltaChunk(responseId, model, event.delta as string) + } + + if (type === "response.output_text.done") { + // Don't emit a finish chunk here — the response may contain more output + // items (e.g. tool calls after text). The finish chunk is emitted once + // on `response.completed` when the entire response is done. + return null + } + + // Reasoning summary text deltas — GPT 5.4 and other reasoning models emit + // these while "thinking". Translate them to reasoning_content so the + // Anthropic stream translator can emit them as thinking blocks, giving the + // user visible progress during the model's reasoning phase. + if (type === "response.reasoning_summary_text.delta") { + return makeReasoningDeltaChunk(responseId, model, event.delta as string) + } + + // Lifecycle events for reasoning summary — no content to forward. + if ( + type === "response.reasoning_summary_text.done" + || type === "response.reasoning_summary_part.added" + || type === "response.reasoning_summary_part.done" + ) { + return null + } + + if (type === "response.output_item.added") { + return handleOutputItemAdded(event, streamState) + } + + if (type === "response.function_call_arguments.delta") { + return handleFnCallArgsDelta(event, { responseId, model, streamState }) + } + + if (type === "response.function_call_arguments.done") { + // Reset so the next tool call can pick up its identity from the queue. + // Don't emit a finish chunk here — the response may contain more tool + // calls. The finish chunk is emitted once on `response.completed`. + streamState.currentToolCallSent = false + return null + } + + if (type === "response.completed") { + return handleResponseCompleted(event, { responseId, model, streamState }) + } + + return null +} + +function handleResponseCompleted( + event: Record<string, unknown>, + options: Pick<TranslateStreamOptions, "responseId" | "model" | "streamState">, +): Array<SSEMessage> { + const { responseId, model, streamState } = options + const resp = event.response as Record<string, unknown> | undefined + const usage = resp?.usage as Record<string, number> | undefined + if (usage) { + streamState.usage = { + prompt_tokens: usage.input_tokens || usage.prompt_tokens || 0, + completion_tokens: usage.output_tokens || usage.completion_tokens || 0, + total_tokens: usage.total_tokens || 0, + } + } + + const chunks: Array<SSEMessage> = [] + + chunks.push( + makeFinishChunk({ + id: responseId, + model, + finishReason: streamState.hasToolCalls ? "tool_calls" : "stop", + }), + ) + + if (streamState.usage) { + chunks.push( + makeChunk(responseId, model, { + choices: [], + usage: streamState.usage, + }), + ) + } + + return chunks +} + +/** Stash tool-call identity so argument deltas can reference it later. */ +function handleOutputItemAdded( + event: Record<string, unknown>, + streamState: ResponsesStreamState, +): null { + const item = event.item as Record<string, unknown> | undefined + // The Copilot API may encrypt/obfuscate field values, but `call_id` is + // consistently readable. Accept the item if it has a `call_id` — the `type` + // field may not always be present or may be obfuscated. + if (item && item.call_id) { + streamState.pendingToolCalls.push({ + call_id: item.call_id as string, + name: typeof item.name === "string" ? item.name : "function", + }) + streamState.hasToolCalls = true + } + return null +} + +/** Translate a function_call_arguments.delta event into a Chat Completion chunk. */ +function handleFnCallArgsDelta( + event: Record<string, unknown>, + options: Pick<TranslateStreamOptions, "responseId" | "model" | "streamState">, +): SSEMessage { + const { responseId, model, streamState } = options + + // If there's a pending tool call identity and we haven't attached it yet, + // this is the first delta for a new tool call → attach id + name. + if ( + streamState.pendingToolCalls.length > 0 + && !streamState.currentToolCallSent + ) { + // Safe to shift — length check above guarantees at least one element. + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const pending = streamState.pendingToolCalls.shift()! + streamState.currentToolCallSent = true + streamState.toolCallIndex++ + return makeToolCallChunk(responseId, model, { + index: streamState.toolCallIndex, + args: event.delta as string, + identity: { + id: pending.call_id, + type: "function", + name: pending.name, + }, + }) + } + + return makeToolCallChunk(responseId, model, { + index: streamState.toolCallIndex, + args: event.delta as string, + }) +} + +function makeTextDeltaChunk( + id: string, + model: string, + content: string, +): SSEMessage { + return makeChunk(id, model, { + choices: [ + { index: 0, delta: { content }, finish_reason: null, logprobs: null }, + ], + }) +} + +function makeReasoningDeltaChunk( + id: string, + model: string, + reasoningContent: string, +): SSEMessage { + return makeChunk(id, model, { + choices: [ + { + index: 0, + delta: { reasoning_content: reasoningContent }, + finish_reason: null, + logprobs: null, + }, + ], + }) +} + +function makeFinishChunk(opts: { + id: string + model: string + finishReason: string + usage?: { + prompt_tokens: number + completion_tokens: number + total_tokens: number + } +}): SSEMessage { + return makeChunk(opts.id, opts.model, { + choices: [ + { index: 0, delta: {}, finish_reason: opts.finishReason, logprobs: null }, + ], + ...(opts.usage && { usage: opts.usage }), + }) +} + +function makeToolCallChunk( + id: string, + model: string, + toolCallData: { + index: number + args: string + identity?: { id: string; type: string; name: string } + }, +): SSEMessage { + const { index, args, identity } = toolCallData + const toolCall: Record<string, unknown> = { + index, + ...(identity && { id: identity.id, type: identity.type }), + function: { + ...(identity && { name: identity.name }), + arguments: args, + }, + } + return makeChunk(id, model, { + choices: [ + { + index: 0, + delta: { tool_calls: [toolCall] }, + finish_reason: null, + logprobs: null, + }, + ], + }) +} + +function makeChunk( + id: string, + model: string, + extra: Record<string, unknown>, +): SSEMessage { + return { + data: JSON.stringify({ + id, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model, + ...extra, + }), + } +} + +function tryRepairJson(input: string): string | null { + const trimmed = input.trimEnd() + if (!trimmed) return null + + // Strip a trailing incomplete string (no closing quote) + let s = trimmed + // Track bracket/brace nesting to close them + let inString = false + let escape = false + const stack: Array<string> = [] + + for (const ch of s) { + if (escape) { + escape = false + continue + } + if (ch === "\\") { + escape = true + continue + } + if (ch === '"') { + inString = !inString + continue + } + if (inString) continue + switch (ch) { + case "{": { + stack.push("}") + break + } + case "[": { + stack.push("]") + break + } + case "}": + case "]": { + stack.pop() + break + } + default: { + break + } + } + } + + // If we ended inside a string, close it + if (inString) s += '"' + + // Close any open brackets/braces + while (stack.length > 0) s += stack.pop() ?? "" + + try { + JSON.parse(s) + return s + } catch { + return null + } +} diff --git a/src/services/web-search/brave.ts b/src/services/web-search/brave.ts new file mode 100644 index 000000000..59732b4fc --- /dev/null +++ b/src/services/web-search/brave.ts @@ -0,0 +1,59 @@ +import { BraveSearchError, type BraveSearchResult } from "./types" + +const BRAVE_SEARCH_URL = "https://api.search.brave.com/res/v1/web/search" +const TIMEOUT_MS = 5000 +const MAX_RESULTS = 5 + +export async function searchBrave( + query: string, + apiKey: string, +): Promise<Array<BraveSearchResult>> { + const controller = new AbortController() + const timeoutId = setTimeout(() => { + controller.abort() + }, TIMEOUT_MS) + + try { + const url = new URL(BRAVE_SEARCH_URL) + url.searchParams.set("q", query) + url.searchParams.set("count", String(MAX_RESULTS)) + + const response = await fetch(url.toString(), { + headers: { + Accept: "application/json", + "Accept-Encoding": "gzip", + "X-Subscription-Token": apiKey, + }, + signal: controller.signal, + }) + + if (!response.ok) { + throw new BraveSearchError(`HTTP ${response.status}`) + } + + const data = (await response.json()) as { + web?: { + results?: Array<{ + title: string + url: string + description?: string + }> + } + } + + return (data.web?.results ?? []).map((r) => ({ + title: r.title, + url: r.url, + description: r.description ?? "", + })) + } catch (error) { + if (error instanceof BraveSearchError) { + throw error + } + const reason = + error instanceof Error ? error.message : "unknown network error" + throw new BraveSearchError(reason) + } finally { + clearTimeout(timeoutId) + } +} diff --git a/src/services/web-search/interceptor.ts b/src/services/web-search/interceptor.ts new file mode 100644 index 000000000..e3b8eb681 --- /dev/null +++ b/src/services/web-search/interceptor.ts @@ -0,0 +1,222 @@ +import consola from "consola" + +import { state } from "~/lib/state" +import { + createChatCompletions, + type ChatCompletionsPayload, + type ChatCompletionResponse, + type Message, +} from "~/services/copilot/create-chat-completions" + +import { searchBrave } from "./brave" +import { searchTavily } from "./tavily" +import { WEB_SEARCH_FUNCTION_TOOL } from "./tool-definition" +import { WebSearchError, type WebSearchResult } from "./types" + +// The name of the tool we inject — use the constant as the single source of truth +// so interceptor and handler stay in sync if the name ever changes. +const WEB_SEARCH_TOOL_NAME = WEB_SEARCH_FUNCTION_TOOL.function.name + +interface SecondPassOptions { + payload: ChatCompletionsPayload + choice: ChatCompletionResponse["choices"][number] + webSearchCallId: string + toolResultContent: string +} + +/** + * Intercepts an OpenAI-format chat completions payload, executes any + * web_search tool call made by the model, injects the results as tool + * messages, and returns the final (second-pass) response. + * + * The caller is responsible for having already injected WEB_SEARCH_FUNCTION_TOOL + * into the payload's tools array (via prepareWebSearchPayload). + * + * Rate limiting note: this function makes up to 2 internal Copilot API calls + * (first pass + second pass) that are not tracked by the outer rate limiter. + * The rate limiter applies only to inbound client requests, not to the internal + * fan-out performed here. + */ +export async function webSearchInterceptor( + payload: ChatCompletionsPayload, +): ReturnType<typeof createChatCompletions> { + // First pass: always non-streaming so we can inspect finish_reason + const firstPassPayload: ChatCompletionsPayload = { ...payload, stream: false } + consola.debug( + "Web search first-pass payload:", + JSON.stringify(firstPassPayload), + ) + + const firstResponse = (await createChatCompletions( + firstPassPayload, + )) as ChatCompletionResponse + consola.debug( + "Web search first-pass response:", + JSON.stringify(firstResponse).slice(-400), + ) + + const choice = firstResponse.choices.at(0) + if ( + !choice + || choice.finish_reason !== "tool_calls" + || !choice.message.tool_calls + ) { + if (payload.stream) { + const streamPayload: ChatCompletionsPayload = { + ...payload, + tools: payload.tools?.filter( + (t) => t.function.name !== WEB_SEARCH_TOOL_NAME, + ), + } + return createChatCompletions(streamPayload) + } + return firstResponse + } + + const webSearchCall = choice.message.tool_calls.find( + (tc) => tc.function.name === WEB_SEARCH_TOOL_NAME, + ) + if (!webSearchCall) { + if (payload.stream) { + const streamPayload: ChatCompletionsPayload = { + ...payload, + tools: payload.tools?.filter( + (t) => t.function.name !== WEB_SEARCH_TOOL_NAME, + ), + } + return createChatCompletions(streamPayload) + } + return firstResponse + } + + // Parse query and perform search. toolResultContent is always assigned on + // every branch before buildSecondPass is called. + let toolResultContent: string + try { + const args = JSON.parse(webSearchCall.function.arguments) as { + query: string + } + const query = args.query + + try { + toolResultContent = await executeWebSearch(query) + } catch (error: unknown) { + const reason = + error instanceof WebSearchError ? error.reason : String(error) + consola.warn(`Web search failed: ${reason}`) + toolResultContent = `Web search failed: ${reason}\nPlease answer based on your training data and let the user know that web search is currently unavailable.` + } + } catch { + consola.warn("Web search: failed to parse tool call arguments") + toolResultContent = + "Web search failed: could not parse search query.\nPlease answer based on your training data and let the user know that web search is currently unavailable." + } + + return buildSecondPass({ + payload, + choice, + webSearchCallId: webSearchCall.id, + toolResultContent, + }) +} + +/** + * Prepares an OpenAI-format payload for the web search interceptor by + * injecting WEB_SEARCH_FUNCTION_TOOL into the tools array. Owning tool + * injection here keeps it co-located with the interceptor that depends on it. + */ +export function prepareWebSearchPayload( + payload: ChatCompletionsPayload, +): ChatCompletionsPayload { + return { + ...payload, + tools: [...(payload.tools ?? []), WEB_SEARCH_FUNCTION_TOOL], + } +} + +/** + * Routes a web search query to the appropriate provider based on which API + * key is configured in state. Tavily takes priority over Brave when both keys + * are set. + */ +async function executeWebSearch(query: string): Promise<string> { + if (state.tavilyApiKey) { + const results = await searchTavily(query, state.tavilyApiKey) + return formatSearchResults(query, results) + } + + if (state.braveApiKey) { + const results = await searchBrave(query, state.braveApiKey) + return formatSearchResults(query, results) + } + + throw new WebSearchError("no web search API key configured") +} + +function buildSecondPass({ + payload, + choice, + webSearchCallId, + toolResultContent, +}: SecondPassOptions): ReturnType<typeof createChatCompletions> { + const assistantMessage: Message = { + role: "assistant", + content: choice.message.content ?? null, + tool_calls: choice.message.tool_calls, + } + + // Inject a tool result for every tool_call in the assistant message. + // Non-search tool calls get an empty stub so Copilot's second pass has + // a complete result set (required — partial results cause rejection). + const toolResultMessages: Array<Message> = ( + choice.message.tool_calls ?? [] + ).map((tc) => ({ + role: "tool", + tool_call_id: tc.id, + content: tc.id === webSearchCallId ? toolResultContent : "", + })) + + const secondPassMessages: Array<Message> = [ + ...payload.messages, + assistantMessage, + ...toolResultMessages, + ] + + // Second pass: use original stream flag. + // Set tool_choice: "none" to prevent the model from invoking web_search again + // in the synthesis pass — a second tool call would produce finish_reason: + // "tool_calls" that the Anthropic client has no way to resolve (it never knew + // about the internal web_search call). + const secondPassPayload: ChatCompletionsPayload = { + ...payload, + messages: secondPassMessages, + tool_choice: "none", + } + consola.debug( + "Web search second-pass payload:", + JSON.stringify(secondPassPayload), + ) + + return createChatCompletions(secondPassPayload) +} + +function formatSearchResults( + query: string, + results: Array<WebSearchResult>, +): string { + if (results.length === 0) { + return `No results found for: "${query}"` + } + + const lines: Array<string> = [`Web search results for: "${query}"`, ""] + for (const [i, result] of results.entries()) { + lines.push( + `${i + 1}. Title: ${result.title}`, + ` URL: ${result.url}`, + ` Snippet: ${result.description}`, + "", + ) + } + + return lines.join("\n").trimEnd() +} diff --git a/src/services/web-search/system-prompt.ts b/src/services/web-search/system-prompt.ts new file mode 100644 index 000000000..53640ae80 --- /dev/null +++ b/src/services/web-search/system-prompt.ts @@ -0,0 +1,46 @@ +import type { AnthropicMessagesPayload } from "~/routes/messages/anthropic-types" + +const WEB_SEARCH_SYSTEM_INSTRUCTION = + "\n\nYou have access to a web_search tool. Use it proactively whenever a question may benefit from current information, recent events, up-to-date facts, or anything that could have changed since your training cutoff. When in doubt, search." + +/** + * Appends the web-search usage instruction to the request's system prompt so + * the model is nudged to call the injected web_search tool instead of relying + * solely on training data. + * + * Handles all three forms of the Anthropic `system` field: + * - string → append instruction + * - Array<TextBlock> → append to the last text block (or add a new one) + * - undefined → return instruction as a plain string + */ +export function appendWebSearchInstruction( + system: AnthropicMessagesPayload["system"], +): AnthropicMessagesPayload["system"] { + if (typeof system === "string") { + return system + WEB_SEARCH_SYSTEM_INSTRUCTION + } + + if (Array.isArray(system)) { + const lastTextIdx = system.findLastIndex((b) => b.type === "text") + + if (lastTextIdx !== -1) { + return system.map((b, i) => + i === lastTextIdx ? + { + ...b, + text: (b as { text: string }).text + WEB_SEARCH_SYSTEM_INSTRUCTION, + } + : b, + ) + } + + // No text block found — add a new one + return [ + ...system, + { type: "text" as const, text: WEB_SEARCH_SYSTEM_INSTRUCTION.trim() }, + ] + } + + // No system prompt at all + return WEB_SEARCH_SYSTEM_INSTRUCTION.trim() +} diff --git a/src/services/web-search/tavily.ts b/src/services/web-search/tavily.ts new file mode 100644 index 000000000..96491c601 --- /dev/null +++ b/src/services/web-search/tavily.ts @@ -0,0 +1,52 @@ +import { WebSearchError, type WebSearchResult } from "./types" + +const TAVILY_SEARCH_URL = "https://api.tavily.com/search" +const TIMEOUT_MS = 5000 +const MAX_RESULTS = 5 + +export async function searchTavily( + query: string, + apiKey: string, +): Promise<Array<WebSearchResult>> { + const controller = new AbortController() + const timeoutId = setTimeout(() => { + controller.abort() + }, TIMEOUT_MS) + + try { + const response = await fetch(TAVILY_SEARCH_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ query, max_results: MAX_RESULTS }), + signal: controller.signal, + }) + + if (!response.ok) { + throw new WebSearchError(`HTTP ${response.status}`) + } + + const data = (await response.json()) as { + results?: Array<{ title: string; url: string; content?: string }> + } + + return (data.results ?? []).map((r) => ({ + title: r.title, + url: r.url, + description: r.content ?? "", + })) + } catch (error) { + if (error instanceof WebSearchError) throw error + if (error instanceof Error && error.name === "AbortError") { + throw new WebSearchError("request timed out") + } + const reason = + error instanceof Error ? error.message : "unknown network error" + throw new WebSearchError(reason) + } finally { + clearTimeout(timeoutId) + } +} diff --git a/src/services/web-search/tool-definition.ts b/src/services/web-search/tool-definition.ts new file mode 100644 index 000000000..ceaae4b89 --- /dev/null +++ b/src/services/web-search/tool-definition.ts @@ -0,0 +1,30 @@ +import type { Tool } from "~/services/copilot/create-chat-completions" + +export const WEB_SEARCH_TOOL_NAMES = new Set([ + "web_search", + "internet_search", + "brave_search", + "bing_search", + "google_search", + "find_online", + "internet_research", +]) + +export const WEB_SEARCH_FUNCTION_TOOL: Tool = { + type: "function", + function: { + name: "web_search", + description: + "Search the web for current information. Use this when you need up-to-date facts, recent events, or information beyond your training data.", + parameters: { + type: "object", + properties: { + query: { + type: "string", + description: "The search query", + }, + }, + required: ["query"], + }, + }, +} diff --git a/src/services/web-search/types.ts b/src/services/web-search/types.ts new file mode 100644 index 000000000..d3c4ec155 --- /dev/null +++ b/src/services/web-search/types.ts @@ -0,0 +1,19 @@ +export interface WebSearchResult { + title: string + url: string + description: string +} + +export class WebSearchError extends Error { + readonly reason: string + + constructor(reason: string) { + super(`Web search failed: ${reason}`) + this.name = "WebSearchError" + this.reason = reason + } +} + +// Backward-compat aliases +export type BraveSearchResult = WebSearchResult +export const BraveSearchError = WebSearchError diff --git a/src/start.ts b/src/start.ts index 14abbbdff..472b85c70 100644 --- a/src/start.ts +++ b/src/start.ts @@ -13,6 +13,11 @@ import { state } from "./lib/state" import { setupCopilotToken, setupGitHubToken } from "./lib/token" import { cacheModels, cacheVSCodeVersion } from "./lib/utils" import { server } from "./server" +import { + getModelContextWindow, + getModelMaxOutput, + getModelTotalContext, +} from "./services/copilot/get-models" interface RunServerOptions { port: number @@ -21,12 +26,53 @@ interface RunServerOptions { manual: boolean rateLimit?: number rateLimitWait: boolean + burstCount?: number + burstWindowSeconds?: number + burstMinSpacingMs: number + burstScope: "global" | "model" githubToken?: string claudeCode: boolean showToken: boolean proxyEnv: boolean } +/** Formats a number as "Nk" if >= 1000, otherwise as-is. */ +const formatK = (v: number) => (v >= 1000 ? `${Math.round(v / 1000)}k` : `${v}`) + +interface TlsSetup { + tls: { cert: string; key: string; passphrase?: string } | undefined + scheme: "http" | "https" +} + +/** + * Resolves optional TLS config from env. HTTPS is enabled when BOTH TLS_CERT + * and TLS_KEY are set (file paths or inline PEM — srvx reads files for us). + * Exits if only one is provided. Returns plain-HTTP defaults when neither is + * set, preserving the original behavior. + */ +function resolveTlsSetup(): TlsSetup { + const cert = process.env.TLS_CERT?.trim() + const key = process.env.TLS_KEY?.trim() + + if (Boolean(cert) !== Boolean(key)) { + consola.error( + "TLS misconfiguration: set BOTH TLS_CERT and TLS_KEY, or neither.", + ) + process.exit(1) + } + + if (!cert || !key) { + return { tls: undefined, scheme: "http" } + } + + consola.info("TLS enabled — server will listen over HTTPS") + return { + tls: { cert, key, passphrase: process.env.TLS_PASSPHRASE }, + scheme: "https", + } +} + +// eslint-disable-next-line max-lines-per-function export async function runServer(options: RunServerOptions): Promise<void> { if (options.proxyEnv) { initProxyFromEnv() @@ -37,6 +83,20 @@ export async function runServer(options: RunServerOptions): Promise<void> { consola.info("Verbose logging enabled") } + const imageTrimmingEnabled = + process.env.IMAGE_CONTEXT_TRIMMING_ENABLED?.trim().toLowerCase() + if ( + imageTrimmingEnabled === "1" + || imageTrimmingEnabled === "true" + || imageTrimmingEnabled === "yes" + || imageTrimmingEnabled === "on" + ) { + const threshold = process.env.IMAGE_CONTEXT_TRIMMING_BEFORE_MESSAGES ?? "6" + consola.info( + `Processed image trimming enabled (older than ${threshold} message(s))`, + ) + } + state.accountType = options.accountType if (options.accountType !== "individual") { consola.info(`Using ${options.accountType} plan GitHub account`) @@ -45,8 +105,31 @@ export async function runServer(options: RunServerOptions): Promise<void> { state.manualApprove = options.manual state.rateLimitSeconds = options.rateLimit state.rateLimitWait = options.rateLimitWait + state.burstCount = options.burstCount + state.burstWindowSeconds = options.burstWindowSeconds + state.burstMinSpacingMs = options.burstMinSpacingMs + state.burstScope = options.burstScope state.showToken = options.showToken + const tavilyApiKey = process.env.TAVILY_API_KEY + const braveApiKey = process.env.BRAVE_API_KEY + + if (tavilyApiKey) { + state.tavilyApiKey = tavilyApiKey + consola.info("Web search enabled (Tavily)") + consola.info( + "Note: each web search request uses 2-3 internal Copilot API calls " + + "(not counted against the rate limit).", + ) + } else if (braveApiKey) { + state.braveApiKey = braveApiKey + consola.info("Web search enabled (Brave)") + consola.info( + "Note: each web search request uses 2-3 internal Copilot API calls " + + "(not counted against the rate limit).", + ) + } + await ensurePaths() await cacheVSCodeVersion() @@ -60,11 +143,26 @@ export async function runServer(options: RunServerOptions): Promise<void> { await setupCopilotToken() await cacheModels() - consola.info( - `Available models: \n${state.models?.data.map((model) => `- ${model.id}`).join("\n")}`, - ) + const modelList = state.models?.data + .map((model) => { + const totalCtx = getModelTotalContext(model) + const inputLimit = getModelContextWindow(model) + const maxOut = getModelMaxOutput(model) + const parts: Array<string> = [] + if (totalCtx) parts.push(`total: ${formatK(totalCtx)}`) + if (inputLimit) parts.push(`in: ${formatK(inputLimit)}`) + if (maxOut) parts.push(`out: ${formatK(maxOut)}`) + const suffix = parts.length > 0 ? ` (${parts.join(", ")})` : "" + return `- ${model.id}${suffix}` + }) + .join("\n") + consola.info(`Available models: \n${modelList}`) + + // Optional TLS: enable HTTPS when TLS_CERT/TLS_KEY are set (see + // resolveTlsSetup). When unset, the server stays on plain HTTP (unchanged). + const { tls, scheme } = resolveTlsSetup() - const serverUrl = `http://localhost:${options.port}` + const serverUrl = `${scheme}://localhost:${options.port}` if (options.claudeCode) { invariant(state.models, "Models should be loaded by now") @@ -114,10 +212,19 @@ export async function runServer(options: RunServerOptions): Promise<void> { `🌐 Usage Viewer: https://ericc-ch.github.io/copilot-api?endpoint=${serverUrl}/usage`, ) - serve({ + const srvxServer = serve({ fetch: server.fetch as ServerHandler, port: options.port, + // Enable HTTPS when TLS_CERT/TLS_KEY are set; otherwise stay on HTTP. + tls, + // Copilot responses can take several minutes for long generations; + // disable Bun's default 10-second idle timeout to prevent premature 500s. + // srvx forwards the `bun` object directly to Bun.serve as extra options. + bun: { idleTimeout: 0 }, }) + + // Add visual separation after srvx prints its "Listening on:" line + void srvxServer.ready().then(() => console.log()) } export const start = defineCommand({ @@ -161,6 +268,26 @@ export const start = defineCommand({ description: "Wait instead of error when rate limit is hit. Has no effect if rate limit is not set", }, + "burst-count": { + type: "string", + description: + "Max requests allowed within the burst window (positive integer). Must be used with --burst-window.", + }, + "burst-window": { + type: "string", + description: + "Burst window duration in seconds (positive number). Must be used with --burst-count.", + }, + "min-spacing": { + type: "string", + description: + "Minimum spacing between requests in milliseconds (default: 0). Prevents thundering herd.", + }, + "burst-scope": { + type: "string", + description: + 'Burst limit scope: "global" (default) or "model" (per-model burst tracking).', + }, "github-token": { alias: "g", type: "string", @@ -191,6 +318,56 @@ export const start = defineCommand({ // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition rateLimitRaw === undefined ? undefined : Number.parseInt(rateLimitRaw, 10) + const rawBurstCount = args["burst-count"] + const rawBurstWindow = args["burst-window"] + + let burstCount: number | undefined + let burstWindowSeconds: number | undefined + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (rawBurstCount !== undefined && rawBurstWindow !== undefined) { + const parsedCount = Number(rawBurstCount) + if (!Number.isInteger(parsedCount) || parsedCount < 1) { + consola.error( + `--burst-count must be a positive integer (got: ${rawBurstCount})`, + ) + process.exit(1) + } + + const parsedWindow = Number(rawBurstWindow) + if (!(parsedWindow > 0) || !Number.isFinite(parsedWindow)) { + consola.error( + `--burst-window must be a positive number greater than 0 (got: ${rawBurstWindow})`, + ) + process.exit(1) + } + + burstCount = parsedCount + burstWindowSeconds = parsedWindow + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + } else if (rawBurstCount !== undefined || rawBurstWindow !== undefined) { + const missing = + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + rawBurstCount === undefined ? "--burst-count" : "--burst-window" + consola.error( + `--burst-count and --burst-window must both be provided (missing: ${missing})`, + ) + process.exit(1) + } + + const rawMinSpacing = args["min-spacing"] + const minSpacingMs = rawMinSpacing ? Number(rawMinSpacing) : 0 + if (!Number.isFinite(minSpacingMs) || minSpacingMs < 0) { + consola.error( + `--min-spacing must be a non-negative number in milliseconds (got: ${rawMinSpacing})`, + ) + process.exit(1) + } + + const rawBurstScope = args["burst-scope"] + const burstScope: "global" | "model" = + rawBurstScope === "model" ? "model" : "global" + return runServer({ port: Number.parseInt(args.port, 10), verbose: args.verbose, @@ -202,6 +379,10 @@ export const start = defineCommand({ claudeCode: args["claude-code"], showToken: args["show-token"], proxyEnv: args["proxy-env"], + burstCount, + burstWindowSeconds, + burstMinSpacingMs: minSpacingMs, + burstScope, }) }, }) diff --git a/start-controlled.bat b/start-controlled.bat new file mode 100644 index 000000000..f4afb0a61 --- /dev/null +++ b/start-controlled.bat @@ -0,0 +1,62 @@ +@echo off +TITLE Copilot Proxy - Controlled + +echo ================================================ +echo Copilot Proxy - Controlled +echo ================================================ +echo. + +:: Load environment variables from .env if it exists +if exist .env ( + echo Loading environment from .env... + for /f "usebackq tokens=1,* delims==" %%A in (".env") do ( + if not "%%A"=="" if not "%%A:~0,1%"=="#" ( + set "%%A=%%B" + ) + ) + echo. +) + +:: --- TLS / HTTPS configuration ---------------------------------------- +:: LAN IP the other laptop uses to reach this PC (must match a cert SAN). +set "LAN_IP=192.168.0.105" +:: Point the server at the self-signed cert generated by generate-cert.bat. +set "TLS_CERT=%~dp0certs\server.crt" +set "TLS_KEY=%~dp0certs\server.key" + +if not exist "%TLS_CERT%" ( + echo TLS certificate not found at "%TLS_CERT%". + echo Run generate-cert.bat first to create it. + echo. + pause + exit /b 1 +) + +:: Build if dist/ doesn't exist +if not exist dist ( + echo Building project... + bun run build + echo. +) + +if defined TAVILY_API_KEY ( + echo Web search: enabled ^(Tavily^) +) else if defined BRAVE_API_KEY ( + echo Web search: enabled ^(Brave^) +) else ( + echo Web search: disabled ^(set TAVILY_API_KEY or BRAVE_API_KEY in .env to enable^) +) +echo. + +echo Starting server on https://localhost:3131 +echo Reachable from other machines at https://%LAN_IP%:3131 +:: echo --burst-count 10 --burst-window 30 ^(max 10 requests per 30s window^) +echo. +echo To launch Claude Code, run in a new terminal: +:: echo set ANTHROPIC_BASE_URL=https://localhost:3131 ^& set ANTHROPIC_AUTH_TOKEN=dummy ^& set ANTHROPIC_MODEL=claude-opus-4.6 ^& set ANTHROPIC_DEFAULT_SONNET_MODEL=claude-sonnet-4.6 ^& set ANTHROPIC_SMALL_FAST_MODEL=gpt-4o-mini ^& set ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-opus-4.6 ^& set DISABLE_NON_ESSENTIAL_MODEL_CALLS=1 ^& set CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 ^& claude +echo. + +:: start "" "https://ericc-ch.github.io/copilot-api?endpoint=https://%LAN_IP%:3131/usage" +bun run ./src/main.ts start --port 3131 --burst-count 300 --burst-window 30 --min-spacing 100 --burst-scope model +:: bun run ./src/main.ts start +pause diff --git a/start-https.bat b/start-https.bat new file mode 100644 index 000000000..182d26109 --- /dev/null +++ b/start-https.bat @@ -0,0 +1,64 @@ +@echo off +:: Make the script location-independent: run everything relative to this file's +:: folder so .env, certs\, and `bun run ./src/main.ts` always resolve the same +:: way whether double-clicked or called by full path. +cd /d "%~dp0" +setlocal +TITLE Copilot API Proxy (HTTPS 8443) + +echo ================================================ +echo GitHub Copilot API Proxy - HTTPS +echo Port: 8443 +echo ================================================ +echo. + +:: Load environment variables from .env if it exists +if exist .env ( + echo Loading environment from .env... + for /f "usebackq tokens=1,* delims==" %%A in (".env") do ( + if not "%%A"=="" if not "%%A:~0,1%"=="#" ( + set "%%A=%%B" + ) + ) + echo. +) + +:: --- Port + TLS / HTTPS configuration --------------------------------- +set "PORT=8443" +:: LAN IP the other laptop uses to reach this PC (must match a cert SAN). +set "LAN_IP=192.168.0.105" +:: Point the server at the self-signed cert generated by generate-cert.bat. +set "TLS_CERT=%~dp0certs\server.crt" +set "TLS_KEY=%~dp0certs\server.key" + +if not exist "%TLS_CERT%" ( + echo TLS certificate not found at "%TLS_CERT%". + echo Run generate-cert.bat first to create it. + echo. + pause + exit /b 1 +) + +:: Build if dist/ doesn't exist +if not exist dist ( + echo Building project... + bun run build + echo. +) + +if defined TAVILY_API_KEY ( + echo Web search: enabled ^(Tavily^) +) else if defined BRAVE_API_KEY ( + echo Web search: enabled ^(Brave^) +) else ( + echo Web search: disabled ^(set TAVILY_API_KEY or BRAVE_API_KEY in .env to enable^) +) +echo. + +echo Starting server on https://localhost:%PORT% +echo Reachable from other machines at https://%LAN_IP%:%PORT% +echo. + +bun run ./src/main.ts start --port %PORT% +pause +endlocal diff --git a/start-openai-https.bat b/start-openai-https.bat new file mode 100644 index 000000000..e60639c1b --- /dev/null +++ b/start-openai-https.bat @@ -0,0 +1,71 @@ +@echo off +:: Make the script location-independent: run everything relative to this file's +:: folder so .env, certs\, and `bun run ./src/main.ts` always resolve the same +:: way whether double-clicked or called by full path. +cd /d "%~dp0" +setlocal +TITLE Copilot API - OpenAI Proxy HTTPS (Codex, 9443) + +echo ================================================ +echo GitHub Copilot API - OpenAI Compatible Proxy +echo For use with Codex / OpenAI-compatible clients +echo HTTPS - Port: 9443 +echo ================================================ +echo. + +:: Load environment variables from .env if it exists +if exist .env ( + echo Loading environment from .env... + for /f "usebackq tokens=1,* delims==" %%A in (".env") do ( + if not "%%A"=="" if not "%%A:~0,1%"=="#" ( + set "%%A=%%B" + ) + ) + echo. +) + +:: --- Port + TLS / HTTPS configuration --------------------------------- +set "PORT=9443" +:: LAN IP the other machine uses to reach this PC (must match a cert SAN). +set "LAN_IP=192.168.0.105" +:: Point the server at the self-signed cert generated by generate-cert.bat. +set "TLS_CERT=%~dp0certs\server.crt" +set "TLS_KEY=%~dp0certs\server.key" + +if not exist "%TLS_CERT%" ( + echo TLS certificate not found at "%TLS_CERT%". + echo Run generate-cert.bat first to create it. + echo. + pause + exit /b 1 +) + +:: Build if dist/ doesn't exist +if not exist dist ( + echo Building project... + bun run build + echo. +) + +echo Starting OpenAI-compatible proxy on https://localhost:%PORT% +echo Reachable from other machines at https://%LAN_IP%:%PORT% +echo. +echo Codex config (codex.json or ~/.codex/config.json): +echo { +echo "model": "gpt-4o", +echo "provider": "openai", +echo "providers": { +echo "openai": { +echo "name": "openai", +echo "base_url": "https://%LAN_IP%:%PORT%/v1", +echo "env_key": "OPENAI_API_KEY" +echo } +echo } +echo } +echo. +echo Or run Codex with: +echo set OPENAI_API_KEY=dummy ^& set OPENAI_BASE_URL=https://%LAN_IP%:%PORT%/v1 ^& codex +echo. +bun run ./src/main.ts start --port %PORT% --verbose +pause +endlocal diff --git a/start-openai.bat b/start-openai.bat new file mode 100644 index 000000000..f3217ffe1 --- /dev/null +++ b/start-openai.bat @@ -0,0 +1,48 @@ +@echo off +TITLE Copilot API - OpenAI Proxy (Codex) + +echo ================================================ +echo GitHub Copilot API - OpenAI Compatible Proxy +echo For use with Codex / OpenAI-compatible clients +echo Port: 1515 +echo ================================================ +echo. + +:: Load environment variables from .env if it exists +if exist .env ( + echo Loading environment from .env... + for /f "usebackq tokens=1,* delims==" %%A in (".env") do ( + if not "%%A"=="" if not "%%A:~0,1%"=="#" ( + set "%%A=%%B" + ) + ) + echo. +) + +:: Build if dist/ doesn't exist +if not exist dist ( + echo Building project... + bun run build + echo. +) + +echo Starting OpenAI-compatible proxy on http://localhost:1515 +echo. +echo Codex config (codex.json or ~/.codex/config.json): +echo { +echo "model": "gpt-4o", +echo "provider": "openai", +echo "providers": { +echo "openai": { +echo "name": "openai", +echo "base_url": "http://localhost:1515/v1", +echo "env_key": "OPENAI_API_KEY" +echo } +echo } +echo } +echo. +echo Or run Codex with: +echo set OPENAI_API_KEY=dummy ^& set OPENAI_BASE_URL=http://localhost:1515/v1 ^& codex +echo. +bun run ./src/main.ts start --port 1515 --verbose +pause diff --git a/start-service.bat b/start-service.bat new file mode 100644 index 000000000..70fb7e297 --- /dev/null +++ b/start-service.bat @@ -0,0 +1,29 @@ +@echo off +:: Service-mode startup script for NSSM (no interactive prompts, no browser) + +:: Load environment variables from .env if it exists +cd /d "%~dp0" +if exist .env ( + for /f "usebackq tokens=1,* delims==" %%A in (".env") do ( + if not "%%A"=="" if not "%%A:~0,1%"=="#" ( + set "%%A=%%B" + ) + ) +) + +:: Use full path to bun since Local System may not have user PATH +set "BUN_EXE=C:\Users\ttbasil\.bun\bin\bun.exe" + +:: Pass the github token directly so the app doesn't try interactive OAuth +:: Read the token from the user's token file +set "TOKEN_FILE=C:\Users\ttbasil\.local\share\copilot-api\github_token" +if exist "%TOKEN_FILE%" ( + set /p GITHUB_TOKEN=<"%TOKEN_FILE%" +) else ( + echo ERROR: GitHub token not found at %TOKEN_FILE% + echo Run "bun run ./src/main.ts auth" interactively first to authenticate. + exit /b 1 +) + +:: Start the server with the token passed via CLI flag (no interactive auth needed) +"%BUN_EXE%" run ./src/main.ts start --github-token "%GITHUB_TOKEN%" diff --git a/start.bat b/start.bat index 1a0f8cb83..35ce083d5 100644 --- a/start.bat +++ b/start.bat @@ -1,20 +1,50 @@ @echo off +TITLE Copilot API Proxy + echo ================================================ -echo GitHub Copilot API Server with Usage Viewer +echo GitHub Copilot API Proxy (HTTP Mode) echo ================================================ echo. -if not exist node_modules ( - echo Installing dependencies... - bun install +:: Load environment variables from .env if it exists +if exist .env ( + echo Loading environment from .env... + for /f "usebackq tokens=1,* delims==" %%A in (".env") do ( + if not "%%A"=="" if not "%%A:~0,1%"=="#" ( + set "%%A=%%B" + ) + ) + echo. +) + +:: --- Network configuration -------------------------------------------- +:: LAN IP the other laptop uses to reach this PC. +set "LAN_IP=192.168.0.105" + +:: Build if dist/ doesn't exist +if not exist dist ( + echo Building project... + bun run build echo. ) -echo Starting server... -echo The usage viewer page will open automatically after the server starts +if defined TAVILY_API_KEY ( + echo Web search: enabled ^(Tavily^) +) else if defined BRAVE_API_KEY ( + echo Web search: enabled ^(Brave^) +) else ( + echo Web search: disabled ^(set TAVILY_API_KEY or BRAVE_API_KEY in .env to enable^) +) echo. -start "" "https://ericc-ch.github.io/copilot-api?endpoint=http://localhost:4141/usage" -bun run dev +echo Starting server on http://localhost:4141 +echo Reachable from other machines at http://%LAN_IP%:4141 +echo. +echo To launch Claude Code, run in a new terminal: +:: echo set ANTHROPIC_BASE_URL=http://localhost:4141 ^& set ANTHROPIC_AUTH_TOKEN=dummy ^& set ANTHROPIC_MODEL=claude-opus-4.6 ^& set ANTHROPIC_DEFAULT_SONNET_MODEL=claude-sonnet-4.6 ^& set ANTHROPIC_SMALL_FAST_MODEL=gpt-4o-mini ^& set ANTHROPIC_DEFAULT_HAIKU_MODEL=claude-opus-4.6 ^& set DISABLE_NON_ESSENTIAL_MODEL_CALLS=1 ^& set CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1 ^& claude +echo. -pause +start "" "https://ericc-ch.github.io/copilot-api?endpoint=http://%LAN_IP%:4141/usage" +:: bun run ./src/main.ts start --burst-count 10 --burst-window 30 +bun run ./src/main.ts start +pause \ No newline at end of file diff --git a/tests/anthropic-content-blocks.test.ts b/tests/anthropic-content-blocks.test.ts new file mode 100644 index 000000000..db6de29f2 --- /dev/null +++ b/tests/anthropic-content-blocks.test.ts @@ -0,0 +1,310 @@ +import { describe, test, expect } from "bun:test" + +import { + type AnthropicAssistantContentBlock, + type AnthropicContainerUploadBlock, + type AnthropicMessagesPayload, + type AnthropicSearchResultBlock, + type AnthropicUserContentBlock, +} from "~/routes/messages/anthropic-types" +import { translateToOpenAI } from "~/routes/messages/non-stream-translation" + +describe("New Anthropic content block types (API parity)", () => { + test("search_result block in user message produces formatted text", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What did you find?" }, + { + type: "search_result", + source: "https://example.com/article", + title: "Example Article", + content: "This is the search result content.", + } as AnthropicSearchResultBlock, + ] as Array<AnthropicUserContentBlock>, + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const userMsg = result.messages.find((m) => m.role === "user") + const content = typeof userMsg?.content === "string" ? userMsg.content : "" + expect(content).toContain("[Search: Example Article]") + expect(content).toContain("Source: https://example.com/article") + expect(content).toContain("This is the search result content.") + }) + + test("container_upload block in user message produces placeholder text", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "I uploaded a file." }, + { + type: "container_upload", + file_id: "file_abc123", + } as unknown as AnthropicContainerUploadBlock & { + type: "container_upload" + }, + ] as Array<AnthropicUserContentBlock>, + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const userMsg = result.messages.find((m) => m.role === "user") + const content = typeof userMsg?.content === "string" ? userMsg.content : "" + expect(content).toContain("[Container upload: file_abc123]") + }) + + test("web_fetch_tool_result block in user message is serialized as user text", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Fetch this page." }, + { + role: "user", + content: [ + { + type: "web_fetch_tool_result", + tool_use_id: "srv_wf_1", + content: { url: "https://example.com", text: "Page content" }, + } as unknown as AnthropicUserContentBlock, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const resultMsg = result.messages.find( + (m) => + m.role === "user" + && typeof m.content === "string" + && m.content.includes("web_fetch_tool_result"), + ) + expect(resultMsg).toBeDefined() + expect(resultMsg?.content).toContain("example.com") + }) + + test("code_execution_tool_result in assistant message with tool calls (Branch 1) appears in text", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Run some code." }, + { + role: "assistant", + content: [ + { type: "text", text: "Here are the results." }, + { + type: "code_execution_tool_result", + tool_use_id: "srv_ce_1", + content: { stdout: "Hello World", exit_code: 0 }, + } as unknown as AnthropicAssistantContentBlock, + { + type: "tool_use", + id: "call_1", + name: "Bash", + input: { command: "echo done" }, + }, + ], + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "call_1", content: "done" }, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const assistantMsg = result.messages.find( + (m) => m.role === "assistant" && typeof m.content === "string", + ) + expect(assistantMsg).toBeDefined() + expect(assistantMsg?.content).toContain("code_execution_tool_result") + expect(assistantMsg?.content).toContain("Hello World") + }) +}) + +describe("New Anthropic content block types — mixed content and image handling", () => { + test("unknown server tool result in assistant message (Branch 2, no tool_use) goes through mapContent default", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "What happened?" }, + { + role: "assistant", + content: [ + { type: "text", text: "I found something." }, + { + type: "bash_code_execution_tool_result", + tool_use_id: "srv_bash_1", + content: { stdout: "output", exit_code: 0 }, + } as unknown as AnthropicAssistantContentBlock, + ], + }, + { role: "user", content: "Continue." }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const assistantMsg = result.messages.find( + (m) => + m.role === "assistant" + && typeof m.content === "string" + && m.content.includes("bash_code_execution_tool_result"), + ) + expect(assistantMsg).toBeDefined() + expect(assistantMsg?.content).toContain("output") + }) + + test("search_result in mapContent Path B (mixed with image) produces text part", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Look at this." }, + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgo=", + }, + }, + { + type: "search_result", + source: "https://example.com", + title: "Mixed Test", + content: "Search result in image context.", + } as unknown as AnthropicUserContentBlock, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const userMsg = result.messages.find((m) => m.role === "user") + expect(Array.isArray(userMsg?.content)).toBe(true) + const parts = userMsg?.content as Array<{ type: string; text?: string }> + expect(parts.some((p) => p.type === "image_url")).toBe(true) + expect( + parts.some( + (p) => p.type === "text" && p.text?.includes("[Search: Mixed Test]"), + ), + ).toBe(true) + }) + + test("URL-based image source passes URL directly to image_url", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Look at this image." }, + { + type: "image", + source: { + type: "url", + url: "https://example.com/image.png", + }, + }, + ] as Array<AnthropicUserContentBlock>, + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const userMsg = result.messages.find((m) => m.role === "user") + expect(Array.isArray(userMsg?.content)).toBe(true) + const parts = userMsg?.content as Array<{ + type: string + image_url?: { url: string } + }> + const imagePart = parts.find((p) => p.type === "image_url") + expect(imagePart?.image_url?.url).toBe("https://example.com/image.png") + }) + + test("mapContent with image + search_result mix handles both correctly", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Look at this." }, + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgo=", + }, + }, + { + type: "search_result", + source: "https://example.com", + title: "Mixed Test", + content: "Search result in image context.", + } as unknown as AnthropicUserContentBlock, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const userMsg = result.messages.find((m) => m.role === "user") + expect(Array.isArray(userMsg?.content)).toBe(true) + const parts = userMsg?.content as Array<{ type: string; text?: string }> + expect(parts.some((p) => p.type === "image_url")).toBe(true) + expect( + parts.some( + (p) => p.type === "text" && p.text?.includes("[Search: Mixed Test]"), + ), + ).toBe(true) + }) + + test("web_search_tool_result is routed through server tool result path (not mapContent)", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { + role: "user", + content: [ + { + type: "web_search_tool_result", + tool_use_id: "srv_1", + content: [ + { + type: "web_search_result", + url: "https://example.com", + title: "Test", + }, + ], + }, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + // Should produce a user message with serialized content (not a tool message) + const userMsg = result.messages.find( + (m) => m.role === "user" && typeof m.content === "string", + ) + expect(userMsg).toBeDefined() + expect(userMsg?.content).toContain("web_search_tool_result") + // Should NOT produce a tool message (tool_result routing is separate) + const toolMsg = result.messages.find((m) => m.role === "tool") + expect(toolMsg).toBeUndefined() + }) +}) diff --git a/tests/anthropic-request.test.ts b/tests/anthropic-request.test.ts index 06c663778..790276532 100644 --- a/tests/anthropic-request.test.ts +++ b/tests/anthropic-request.test.ts @@ -1,9 +1,12 @@ import { describe, test, expect } from "bun:test" import { z } from "zod" -import type { AnthropicMessagesPayload } from "~/routes/messages/anthropic-types" - -import { translateToOpenAI } from "../src/routes/messages/non-stream-translation" +import { + isTypedTool, + type AnthropicMessagesPayload, + type AnthropicTool, +} from "~/routes/messages/anthropic-types" +import { translateToOpenAI } from "~/routes/messages/non-stream-translation" // Zod schema for a single message in the chat completion request. const messageSchema = z.object({ @@ -146,11 +149,12 @@ describe("Anthropic to OpenAI translation logic", () => { const openAIPayload = translateToOpenAI(anthropicPayload) expect(isValidChatCompletionRequest(openAIPayload)).toBe(true) - // Check that thinking content is combined with text content + // Thinking blocks should be stripped — Copilot doesn't understand them + // and they inflate the prompt token count const assistantMessage = openAIPayload.messages.find( (m) => m.role === "assistant", ) - expect(assistantMessage?.content).toContain( + expect(assistantMessage?.content).not.toContain( "Let me think about this simple math problem...", ) expect(assistantMessage?.content).toContain("2+2 equals 4.") @@ -184,11 +188,11 @@ describe("Anthropic to OpenAI translation logic", () => { const openAIPayload = translateToOpenAI(anthropicPayload) expect(isValidChatCompletionRequest(openAIPayload)).toBe(true) - // Check that thinking content is included in the message content + // Thinking blocks should be stripped — only text content forwarded const assistantMessage = openAIPayload.messages.find( (m) => m.role === "assistant", ) - expect(assistantMessage?.content).toContain( + expect(assistantMessage?.content).not.toContain( "I need to call the weather API", ) expect(assistantMessage?.content).toContain( @@ -197,6 +201,477 @@ describe("Anthropic to OpenAI translation logic", () => { expect(assistantMessage?.tool_calls).toHaveLength(1) expect(assistantMessage?.tool_calls?.[0].function.name).toBe("get_weather") }) + + test("should filter out Anthropic typed tools (no input_schema) from tools array", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [{ role: "user", content: "Hello" }], + max_tokens: 100, + tools: [ + // Custom tool — should be kept + { + name: "Bash", + description: "Run shell commands", + input_schema: { + type: "object", + properties: {}, + additionalProperties: false, + }, + }, + // Anthropic-typed tool — should be filtered + { type: "bash_20250124", name: "bash" } as unknown as AnthropicTool, + ], + } + const result = translateToOpenAI(anthropicPayload) + // Only the custom "Bash" tool survives + expect(result.tools).toHaveLength(1) + expect(result.tools?.[0].function.name).toBe("Bash") + }) +}) + +describe("Anthropic new content block types (Task 6)", () => { + test("document block in user message produces placeholder text", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "Summarize this PDF." }, + { + type: "document", + title: "My Report", + source: { + type: "base64", + media_type: "application/pdf", + data: "JVBERi0x", + }, + }, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const userMsg = result.messages.find((m) => m.role === "user") + expect(typeof userMsg?.content).toBe("string") + const content = userMsg?.content as string + expect(content).toContain("Summarize this PDF.") + expect(content).toContain("[Document: PDF content not displayable]") + }) + + test("server_tool_use block in assistant message is serialized to text", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Search for something." }, + { + role: "assistant", + content: [ + { + type: "server_tool_use", + id: "srv_1", + name: "web_search", + input: { query: "test" }, + }, + { type: "text", text: "I searched for you." }, + ], + }, + { role: "user", content: "Thanks." }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const assistantMsg = result.messages.find((m) => m.role === "assistant") + const content = assistantMsg?.content as string + expect(content).toContain("[Server tool use:") + expect(content).toContain("web_search") + expect(content).toContain("I searched for you.") + }) +}) + +describe("Anthropic new content block types (Task 2)", () => { + test("should handle document blocks in user messages", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "What does this PDF say?" }, + { + type: "document", + source: { + type: "base64", + media_type: "application/pdf", + data: "JVBERi0x", + }, + }, + ], + }, + ], + max_tokens: 100, + } + // Should not throw; document block is converted to placeholder text + expect(() => translateToOpenAI(anthropicPayload)).not.toThrow() + const result = translateToOpenAI(anthropicPayload) + expect(isValidChatCompletionRequest(result)).toBe(true) + // Placeholder text must appear in the message content + const userMsg = result.messages.find((m) => m.role === "user") + expect(typeof userMsg?.content).toBe("string") + expect(userMsg?.content as string).toContain( + "[Document: PDF content not displayable]", + ) + }) + + test("should handle redacted_thinking blocks in assistant messages", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Think hard about this." }, + { + role: "assistant", + content: [ + { type: "redacted_thinking", data: "EncryptedThinkingData==" }, + { type: "text", text: "Here is my answer." }, + ], + }, + { role: "user", content: "Follow up." }, + ], + max_tokens: 100, + } + expect(() => translateToOpenAI(anthropicPayload)).not.toThrow() + const result = translateToOpenAI(anthropicPayload) + // The redacted_thinking block is stripped; only the text block survives + const assistantMsg = result.messages.find((m) => m.role === "assistant") + expect(assistantMsg?.content).toContain("Here is my answer.") + // redacted_thinking data must NOT appear as raw base64 + expect(assistantMsg?.content as string).not.toContain( + "EncryptedThinkingData==", + ) + }) +}) + +describe("handleUserMessage array tool_result content and web_search_tool_result (Task 8)", () => { + test("tool_result with array content containing image is translated to vision ContentPart", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Take a screenshot." }, + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "call_sc", + name: "computer", + input: { action: "screenshot" }, + }, + ], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "call_sc", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgo=", + }, + }, + ], + }, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const toolMsg = result.messages.find((m) => m.role === "tool") + expect(typeof toolMsg?.content).toBe("string") + expect(toolMsg?.content).toContain( + "Non-text tool result forwarded in the following user message", + ) + + const userMsgs = result.messages.filter((m) => m.role === "user") + const followUpMsg = userMsgs.at(-1) + expect(Array.isArray(followUpMsg?.content)).toBe(true) + const parts = followUpMsg?.content as Array<{ type: string }> + expect(parts[0].type).toBe("image_url") + }) + + test("tool_result with array content containing text is translated to string", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Run a command." }, + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "call_b", + name: "Bash", + input: { command: "ls" }, + }, + ], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "call_b", + content: [{ type: "text", text: "file1.txt\nfile2.txt" }], + }, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const toolMsg = result.messages.find((m) => m.role === "tool") + expect(typeof toolMsg?.content).toBe("string") + expect(toolMsg?.content).toContain("file1.txt") + }) + + test("oversized tool_result text is condensed with head, middle, tail, and compaction guidance", () => { + const largeText = + "A".repeat(9000) + + "MIDDLE" + + "B".repeat(9000) + + "C".repeat(4500) + + "TAILMARK" + + "D".repeat(4500) + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Run a verbose command." }, + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "call_large", + name: "Bash", + input: { command: "verbose" }, + }, + ], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "call_large", + content: [{ type: "text", text: largeText }], + }, + ], + }, + ], + max_tokens: 100, + } + + const result = translateToOpenAI(anthropicPayload) + const toolMsg = result.messages.find((m) => m.role === "tool") + expect(typeof toolMsg?.content).toBe("string") + expect(toolMsg?.content).toContain("[Tool result condensed by proxy:") + expect(toolMsg?.content).toContain( + "compact older conversation state before discarding this fresh tool result", + ) + expect(toolMsg?.content).toContain("=== BEGIN TOOL RESULT HEAD ===") + expect(toolMsg?.content).toContain( + "=== BEGIN TOOL RESULT MIDDLE SAMPLE ===", + ) + expect(toolMsg?.content).toContain("=== BEGIN TOOL RESULT TAIL ===") + expect(toolMsg?.content).toContain("AAAA") + expect(toolMsg?.content).toContain("MIDDLE") + expect(toolMsg?.content).toContain("TAILMARK") + }) + + test("web_search_tool_result block is serialized as user message", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Search the web." }, + { + role: "user", + content: [ + { + type: "web_search_tool_result", + tool_use_id: "srv_ws_1", + content: [ + { + type: "web_search_result", + url: "https://example.com", + title: "Example", + encrypted_content: "abc", + }, + ], + }, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const webResultMsg = result.messages.find( + (m) => + m.role === "user" + && typeof m.content === "string" + && m.content.includes("web_search_tool_result"), + ) + expect(webResultMsg).toBeDefined() + expect(webResultMsg?.content).toContain("example.com") + }) +}) + +describe("handleAssistantMessage redacted_thinking and server_tool_use (Task 7)", () => { + test("redacted_thinking block is stripped from assistant message (Branch 2, no tool calls)", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Think hard." }, + { + role: "assistant", + content: [ + { type: "redacted_thinking", data: "EncryptedBinaryData==" }, + { type: "text", text: "My considered answer." }, + ], + }, + { role: "user", content: "Follow up." }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const assistantMsg = result.messages.find((m) => m.role === "assistant") + // Content must contain the text but NOT the redacted data + expect(assistantMsg?.content).toContain("My considered answer.") + expect(assistantMsg?.content).not.toContain("EncryptedBinaryData==") + }) + + test("server_tool_use block is serialized in assistant message with tool calls (Branch 1)", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [ + { role: "user", content: "Do something." }, + { + role: "assistant", + content: [ + { + type: "server_tool_use", + id: "srv_1", + name: "web_search", + input: { query: "test" }, + }, + { type: "text", text: "Let me also call a tool." }, + { + type: "tool_use", + id: "call_1", + name: "Bash", + input: { command: "ls" }, + }, + ], + }, + ], + max_tokens: 100, + } + const result = translateToOpenAI(anthropicPayload) + const assistantMsg = result.messages.find((m) => m.role === "assistant") + // Branch 1: has tool_use, so content is the text + server_tool_use serialized + expect(assistantMsg?.content).toContain("Let me also call a tool.") + expect(assistantMsg?.content).toContain("[Server tool use:") + expect(assistantMsg?.tool_calls).toHaveLength(1) + expect(assistantMsg?.tool_calls?.[0].function.name).toBe("Bash") + }) +}) + +describe("strict field forwarding (Task 3)", () => { + test("should forward strict:true from custom tool definitions to OpenAI", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [{ role: "user", content: "Hello" }], + max_tokens: 100, + tools: [ + { + name: "getWeather", + description: "Get weather", + input_schema: { + type: "object", + properties: { location: { type: "string" } }, + required: ["location"], + }, + strict: true, + }, + ], + } + const result = translateToOpenAI(anthropicPayload) + expect(result.tools?.[0].function.strict).toBe(true) + }) + + test("should not add strict field when not provided", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4", + messages: [{ role: "user", content: "Hello" }], + max_tokens: 100, + tools: [ + { + name: "getWeather", + description: "Get weather", + input_schema: { type: "object", properties: {} }, + }, + ], + } + const result = translateToOpenAI(anthropicPayload) + expect(result.tools?.[0].function.strict).toBeUndefined() + }) +}) + +function getTranslatedModel(model: string): string { + const result = translateToOpenAI({ + model, + messages: [{ role: "user", content: "Hi" }], + max_tokens: 10, + }) + return result.model +} + +describe("translateModelName normalization", () => { + test("normalizes claude-sonnet-4-6 to claude-sonnet-4", () => { + expect(getTranslatedModel("claude-sonnet-4-6")).toBe("claude-sonnet-4") + }) + + test("normalizes claude-haiku-4-5 to claude-haiku-4 (was missing before)", () => { + expect(getTranslatedModel("claude-haiku-4-5")).toBe("claude-haiku-4") + }) + + test("normalizes claude-opus-4-6 to claude-opus-4", () => { + expect(getTranslatedModel("claude-opus-4-6")).toBe("claude-opus-4") + }) + + test("does NOT change claude-sonnet-3-5 (stable 3.x name)", () => { + expect(getTranslatedModel("claude-sonnet-3-5")).toBe("claude-sonnet-3-5") + }) + + test("does NOT change claude-haiku-3-5 (stable 3.x name)", () => { + expect(getTranslatedModel("claude-haiku-3-5")).toBe("claude-haiku-3-5") + }) + + test("normalizes long versioned names like claude-sonnet-4-6-20251231", () => { + expect(getTranslatedModel("claude-sonnet-4-6-20251231")).toBe( + "claude-sonnet-4", + ) + }) + + test("does NOT change non-claude models", () => { + expect(getTranslatedModel("gpt-4o")).toBe("gpt-4o") + expect(getTranslatedModel("grok-2")).toBe("grok-2") + }) }) describe("OpenAI Chat Completion v1 Request Payload Validation with Zod", () => { @@ -311,3 +786,82 @@ describe("OpenAI Chat Completion v1 Request Payload Validation with Zod", () => expect(isValidChatCompletionRequest(123)).toBe(false) }) }) + +describe("isTypedTool discriminator", () => { + test("returns true for a typed tool (no input_schema)", () => { + const typedTool = { type: "bash_20250124", name: "bash" } + expect(isTypedTool(typedTool)).toBe(true) + }) + + test("returns false for a custom tool (has input_schema)", () => { + const customTool = { + name: "Bash", + description: "Run shell commands", + input_schema: {}, + } + expect(isTypedTool(customTool)).toBe(false) + }) + + test("returns false for custom tool even if it has extra fields", () => { + const customTool = { + name: "Bash", + input_schema: {}, + strict: true, + cache_control: { type: "ephemeral" }, + } + expect(isTypedTool(customTool as AnthropicTool)).toBe(false) + }) +}) + +describe("output_config.format → response_format translation", () => { + const titlePayload = { + model: "claude-haiku-4.5", + messages: [{ role: "user" as const, content: "Hello" }], + max_tokens: 1000, + system: "Generate a title.", + output_config: { + format: { + type: "json_schema" as const, + schema: { + type: "object", + properties: { title: { type: "string" } }, + required: ["title"], + additionalProperties: false, + }, + }, + }, + } + + test("does NOT send response_format for Claude models (Copilot 422s on it) but still enforces JSON via prompt", () => { + const result = translateToOpenAI(titlePayload) + // Copilot's Claude backend rejects response_format with HTTP 422 + // Unprocessable Entity. The structured-output handler enforces JSON via the + // system prompt and repairs free-text → JSON afterward, so response_format + // is redundant for Claude and must be omitted. + expect(result.response_format).toBeUndefined() + const sys = result.messages.find((m) => m.role === "system") + expect(sys).toBeDefined() + const content = sys?.content as string + expect(content).toContain("Generate a title.") + expect(content).toContain( + "IMPORTANT: You MUST respond with valid JSON only", + ) + }) + + test("still sends response_format json_object for non-Claude models", () => { + const result = translateToOpenAI({ ...titlePayload, model: "gpt-4o" }) + expect(result.response_format).toEqual({ type: "json_object" }) + }) + + test("does not enforce JSON when output_config.format is absent", () => { + const result = translateToOpenAI({ + model: "claude-haiku-4.5", + messages: [{ role: "user", content: "Hello" }], + max_tokens: 1000, + system: "You are helpful.", + }) + expect(result.response_format).toBeUndefined() + const content = result.messages[0]?.content as string + expect(content).not.toContain("IMPORTANT: You MUST respond") + }) +}) diff --git a/tests/anthropic-response.test.ts b/tests/anthropic-response.test.ts index ecd71aacc..db23eb816 100644 --- a/tests/anthropic-response.test.ts +++ b/tests/anthropic-response.test.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-lines */ import { describe, test, expect } from "bun:test" import { z } from "zod" @@ -6,9 +7,23 @@ import type { ChatCompletionResponse, } from "~/services/copilot/create-chat-completions" -import { type AnthropicStreamState } from "~/routes/messages/anthropic-types" +import { + type AnthropicMessagesPayload, + type AnthropicStreamState, +} from "~/routes/messages/anthropic-types" +import { isEmptyNonStreamingResponse } from "~/routes/messages/handler" import { translateToAnthropic } from "~/routes/messages/non-stream-translation" -import { translateChunkToAnthropicEvents } from "~/routes/messages/stream-translation" +import { + flushDeferredFinish, + translateChunkToAnthropicEvents, +} from "~/routes/messages/stream-translation" +import { + createToolNameMapFromAnthropicPayload, + toOpenAIToolName, +} from "~/routes/messages/tool-name-mapping" + +const LONG_MCP_TOOL_NAME = + "mcp__plugin_chrome-devtools-mcp_chrome-devtools__get_console_message" const anthropicUsageSchema = z.object({ input_tokens: z.number().int(), @@ -67,6 +82,7 @@ function isValidAnthropicStreamEvent(payload: unknown): boolean { return anthropicStreamEventSchema.safeParse(payload).success } +// eslint-disable-next-line max-lines-per-function describe("OpenAI to Anthropic Non-Streaming Response Translation", () => { test("should translate a simple text response correctly", () => { const openAIResponse: ChatCompletionResponse = { @@ -96,7 +112,7 @@ describe("OpenAI to Anthropic Non-Streaming Response Translation", () => { expect(isValidAnthropicResponse(anthropicResponse)).toBe(true) - expect(anthropicResponse.id).toBe("chatcmpl-123") + expect(anthropicResponse.id).toBe("msg_chatcmpl-123") expect(anthropicResponse.stop_reason).toBe("end_turn") expect(anthropicResponse.usage.input_tokens).toBe(9) expect(anthropicResponse.content[0].type).toBe("text") @@ -160,6 +176,74 @@ describe("OpenAI to Anthropic Non-Streaming Response Translation", () => { } }) + test("should restore long MCP tool names from aliased OpenAI tool calls", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4.6", + messages: [{ role: "user", content: "Inspect the console output." }], + max_tokens: 1000, + tools: [ + { + name: LONG_MCP_TOOL_NAME, + description: "Fetch a console message from the browser session.", + input_schema: { + type: "object", + properties: { + request_id: { type: "string" }, + }, + required: ["request_id"], + additionalProperties: false, + }, + }, + ], + } + const toolNameMap = createToolNameMapFromAnthropicPayload(anthropicPayload) + const aliasedToolName = toOpenAIToolName(LONG_MCP_TOOL_NAME, toolNameMap) + + const openAIResponse: ChatCompletionResponse = { + id: "chatcmpl-tool-alias", + object: "chat.completion", + created: 1677652288, + model: "claude-sonnet-4", + choices: [ + { + index: 0, + message: { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_alias", + type: "function", + function: { + name: aliasedToolName, + arguments: '{"request_id":"req_1"}', + }, + }, + ], + }, + finish_reason: "tool_calls", + logprobs: null, + }, + ], + usage: { + prompt_tokens: 30, + completion_tokens: 20, + total_tokens: 50, + }, + } + + const anthropicResponse = translateToAnthropic(openAIResponse, toolNameMap) + expect(anthropicResponse.content[0]?.type).toBe("tool_use") + if (anthropicResponse.content[0]?.type === "tool_use") { + expect(anthropicResponse.content[0].name).toBe(LONG_MCP_TOOL_NAME) + expect(anthropicResponse.content[0].input).toEqual({ + request_id: "req_1", + }) + } else { + throw new Error("Expected tool_use block") + } + }) + test("should translate a response stopped due to length", () => { const openAIResponse: ChatCompletionResponse = { id: "chatcmpl-789", @@ -189,8 +273,210 @@ describe("OpenAI to Anthropic Non-Streaming Response Translation", () => { expect(isValidAnthropicResponse(anthropicResponse)).toBe(true) expect(anthropicResponse.stop_reason).toBe("max_tokens") }) + + // Regression: a completed non-streaming message must NEVER carry + // stop_reason: null. Anthropic clients (Claude Code, and strict SDK callers) + // treat a null stop_reason on a non-stream response as a protocol violation. + // Some upstream models (notably Gemini via Copilot) return a degenerate + // payload — finish_reason null, or an empty choices array — which previously + // mapped straight through to stop_reason: null with empty content. + test("coerces a null upstream finish_reason to a non-null stop_reason", () => { + const openAIResponse = { + id: "chatcmpl-null-finish", + object: "chat.completion", + created: 1677652288, + model: "claude-sonnet-4", + choices: [ + { + index: 0, + message: { role: "assistant", content: "Partial answer." }, + // Runtime reality: Copilot can send null here even though the + // ChoiceNonStreaming type declares finish_reason as non-null. + finish_reason: null, + logprobs: null, + }, + ], + usage: { prompt_tokens: 10, completion_tokens: 3, total_tokens: 13 }, + } as unknown as ChatCompletionResponse + + const anthropicResponse = translateToAnthropic(openAIResponse) + + expect(anthropicResponse.stop_reason).not.toBeNull() + expect(anthropicResponse.stop_reason).toBe("end_turn") + expect(isValidAnthropicResponse(anthropicResponse)).toBe(true) + }) + + test("never returns stop_reason null for an empty choices array", () => { + const openAIResponse = { + id: "chatcmpl-no-choices", + object: "chat.completion", + created: 1677652288, + model: "claude-sonnet-4", + choices: [], + usage: { prompt_tokens: 10, completion_tokens: 0, total_tokens: 10 }, + } as unknown as ChatCompletionResponse + + const anthropicResponse = translateToAnthropic(openAIResponse) + + expect(anthropicResponse.stop_reason).not.toBeNull() + expect(anthropicResponse.stop_reason).toBe("end_turn") + }) + + // Regression (HIGH): tool calls present but a degenerate null finish_reason. + // The backstop must NOT label this "end_turn" — that would tell the client + // the turn is done and the pending tool calls would never execute. It must + // resolve to "tool_use" so the client runs the tools. + test("tool calls with a null finish_reason resolve to tool_use, not end_turn", () => { + const openAIResponse = { + id: "chatcmpl-tool-null-finish", + object: "chat.completion", + created: 1677652288, + model: "claude-sonnet-4", + choices: [ + { + index: 0, + message: { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_x", + type: "function", + function: { + name: "get_current_weather", + arguments: '{"location":"Boston, MA"}', + }, + }, + ], + }, + finish_reason: null, + logprobs: null, + }, + ], + usage: { prompt_tokens: 30, completion_tokens: 20, total_tokens: 50 }, + } as unknown as ChatCompletionResponse + + const anthropicResponse = translateToAnthropic(openAIResponse) + + expect(anthropicResponse.stop_reason).toBe("tool_use") + expect(anthropicResponse.content[0]?.type).toBe("tool_use") + }) + + // A length-truncated tool call must stay "max_tokens" (the truncation guard + // depends on correctedStopReason === "length"); it must NOT be masked as + // tool_use by the coercion. + test("tool calls with finish_reason length are not masked as tool_use", () => { + const openAIResponse: ChatCompletionResponse = { + id: "chatcmpl-tool-length", + object: "chat.completion", + created: 1677652288, + model: "claude-sonnet-4", + choices: [ + { + index: 0, + message: { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_y", + type: "function", + function: { + name: "get_current_weather", + // Complete JSON → not truncated → passes through as a tool call + arguments: '{"location":"Boston, MA"}', + }, + }, + ], + }, + finish_reason: "length", + logprobs: null, + }, + ], + usage: { prompt_tokens: 30, completion_tokens: 2048, total_tokens: 2078 }, + } + + const anthropicResponse = translateToAnthropic(openAIResponse) + expect(anthropicResponse.stop_reason).toBe("max_tokens") + }) + + test("whitespace-only content with finish_reason stop is treated as empty", () => { + const response = { + id: "chatcmpl-whitespace", + object: "chat.completion", + created: 1677652288, + model: "claude-sonnet-4", + choices: [ + { + index: 0, + message: { role: "assistant", content: " \n " }, + finish_reason: "stop", + logprobs: null, + }, + ], + usage: { prompt_tokens: 10, completion_tokens: 1, total_tokens: 11 }, + } as unknown as ChatCompletionResponse + + expect(isEmptyNonStreamingResponse(response)).toBe(true) + }) +}) + +describe("isEmptyNonStreamingResponse — degenerate upstream detection", () => { + test("treats a null finish_reason with empty content as empty", () => { + const response = { + id: "chatcmpl-null-empty", + object: "chat.completion", + created: 1677652288, + model: "claude-sonnet-4", + choices: [ + { + index: 0, + message: { role: "assistant", content: "" }, + finish_reason: null, + logprobs: null, + }, + ], + usage: { prompt_tokens: 10, completion_tokens: 0, total_tokens: 10 }, + } as unknown as ChatCompletionResponse + + expect(isEmptyNonStreamingResponse(response)).toBe(true) + }) + + test("treats an empty choices array as empty", () => { + const response = { + id: "chatcmpl-empty-choices", + object: "chat.completion", + created: 1677652288, + model: "claude-sonnet-4", + choices: [], + usage: { prompt_tokens: 10, completion_tokens: 0, total_tokens: 10 }, + } as unknown as ChatCompletionResponse + + expect(isEmptyNonStreamingResponse(response)).toBe(true) + }) + + test("does not flag a normal completed response as empty", () => { + const response: ChatCompletionResponse = { + id: "chatcmpl-ok", + object: "chat.completion", + created: 1677652288, + model: "claude-sonnet-4", + choices: [ + { + index: 0, + message: { role: "assistant", content: "Real answer." }, + finish_reason: "stop", + logprobs: null, + }, + ], + usage: { prompt_tokens: 10, completion_tokens: 4, total_tokens: 14 }, + } + + expect(isEmptyNonStreamingResponse(response)).toBe(false) + }) }) +// eslint-disable-next-line max-lines-per-function describe("OpenAI to Anthropic Streaming Response Translation", () => { test("should translate a simple text stream correctly", () => { const openAIStream: Array<ChatCompletionChunk> = [ @@ -249,9 +535,13 @@ describe("OpenAI to Anthropic Streaming Response Translation", () => { const streamState: AnthropicStreamState = { messageStartSent: false, + messageStopSent: false, contentBlockIndex: 0, contentBlockOpen: false, + thinkingBlockOpen: false, + hasEmittedText: false, toolCalls: {}, + thinkingEnabled: false, } const translatedStream = openAIStream.flatMap((chunk) => translateChunkToAnthropicEvents(chunk, streamState), @@ -261,7 +551,6 @@ describe("OpenAI to Anthropic Streaming Response Translation", () => { expect(isValidAnthropicStreamEvent(event)).toBe(true) } }) - test("should translate a stream with tool calls", () => { const openAIStream: Array<ChatCompletionChunk> = [ { @@ -349,9 +638,13 @@ describe("OpenAI to Anthropic Streaming Response Translation", () => { // Streaming translation requires state const streamState: AnthropicStreamState = { messageStartSent: false, + messageStopSent: false, contentBlockIndex: 0, contentBlockOpen: false, + thinkingBlockOpen: false, + hasEmittedText: false, toolCalls: {}, + thinkingEnabled: false, } const translatedStream = openAIStream.flatMap((chunk) => translateChunkToAnthropicEvents(chunk, streamState), @@ -362,4 +655,500 @@ describe("OpenAI to Anthropic Streaming Response Translation", () => { expect(isValidAnthropicStreamEvent(event)).toBe(true) } }) + + test("restores long MCP tool names in streaming tool events", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4.6", + messages: [{ role: "user", content: "Inspect the console output." }], + max_tokens: 1000, + tools: [ + { + name: LONG_MCP_TOOL_NAME, + description: "Fetch a console message from the browser session.", + input_schema: { + type: "object", + properties: { + request_id: { type: "string" }, + }, + required: ["request_id"], + additionalProperties: false, + }, + }, + ], + } + const toolNameMap = createToolNameMapFromAnthropicPayload(anthropicPayload) + const aliasedToolName = toOpenAIToolName(LONG_MCP_TOOL_NAME, toolNameMap) + + const openAIStream: Array<ChatCompletionChunk> = [ + { + id: "cmpl-tool-alias", + object: "chat.completion.chunk", + created: 1677652288, + model: "claude-sonnet-4", + choices: [ + { + index: 0, + delta: { role: "assistant" }, + finish_reason: null, + logprobs: null, + }, + ], + }, + { + id: "cmpl-tool-alias", + object: "chat.completion.chunk", + created: 1677652288, + model: "claude-sonnet-4", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + id: "call_alias", + type: "function", + function: { name: aliasedToolName, arguments: "" }, + }, + ], + }, + finish_reason: null, + logprobs: null, + }, + ], + }, + { + id: "cmpl-tool-alias", + object: "chat.completion.chunk", + created: 1677652288, + model: "claude-sonnet-4", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { index: 0, function: { arguments: '{"request_id":"req_1"}' } }, + ], + }, + finish_reason: null, + logprobs: null, + }, + ], + }, + { + id: "cmpl-tool-alias", + object: "chat.completion.chunk", + created: 1677652288, + model: "claude-sonnet-4", + choices: [ + { index: 0, delta: {}, finish_reason: "tool_calls", logprobs: null }, + ], + }, + ] + + const streamState: AnthropicStreamState = { + messageStartSent: false, + messageStopSent: false, + contentBlockIndex: 0, + contentBlockOpen: false, + thinkingBlockOpen: false, + hasEmittedText: false, + toolCalls: {}, + toolNameMap, + thinkingEnabled: false, + } + + const translatedStream = openAIStream.flatMap((chunk) => + translateChunkToAnthropicEvents(chunk, streamState), + ) + const toolUseStart = translatedStream.find( + (event) => + event.type === "content_block_start" + && event.content_block.type === "tool_use", + ) + + expect(toolUseStart).toBeDefined() + if ( + toolUseStart?.type === "content_block_start" + && toolUseStart.content_block.type === "tool_use" + ) { + expect(toolUseStart.content_block.name).toBe(LONG_MCP_TOOL_NAME) + } else { + throw new Error("Expected tool_use content_block_start event") + } + }) + + test("does not inject synthetic tool narration for Claude tool-call streams", () => { + const openAIStream: Array<ChatCompletionChunk> = [ + { + id: "cmpl-claude-tool", + object: "chat.completion.chunk", + created: 1677652288, + model: "claude-opus-4", + choices: [ + { + index: 0, + delta: { role: "assistant" }, + finish_reason: null, + logprobs: null, + }, + ], + }, + { + id: "cmpl-claude-tool", + object: "chat.completion.chunk", + created: 1677652288, + model: "claude-opus-4", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + id: "call_claude", + type: "function", + function: { + name: "Read", + arguments: '{"file_path":"a.txt"}', + }, + }, + ], + }, + finish_reason: null, + logprobs: null, + }, + ], + }, + { + id: "cmpl-claude-tool", + object: "chat.completion.chunk", + created: 1677652288, + model: "claude-opus-4", + choices: [ + { index: 0, delta: {}, finish_reason: "tool_calls", logprobs: null }, + ], + }, + ] + + const streamState: AnthropicStreamState = { + messageStartSent: false, + messageStopSent: false, + contentBlockIndex: 0, + contentBlockOpen: false, + thinkingBlockOpen: false, + hasEmittedText: false, + toolCalls: {}, + thinkingEnabled: false, + } + + const translatedStream = openAIStream.flatMap((chunk) => + translateChunkToAnthropicEvents(chunk, streamState), + ) + + const textDeltas = translatedStream.filter( + (event) => + event.type === "content_block_delta" + && event.delta.type === "text_delta", + ) + expect(textDeltas).toHaveLength(0) + + const toolStarts = translatedStream.filter( + (event) => + event.type === "content_block_start" + && event.content_block.type === "tool_use", + ) + expect(toolStarts).toHaveLength(1) + }) + + test("still injects synthetic tool narration for non-Claude tool-call streams", () => { + const openAIStream: Array<ChatCompletionChunk> = [ + { + id: "cmpl-gemini-tool", + object: "chat.completion.chunk", + created: 1677652288, + model: "gemini-2.5-pro", + choices: [ + { + index: 0, + delta: { role: "assistant" }, + finish_reason: null, + logprobs: null, + }, + ], + }, + { + id: "cmpl-gemini-tool", + object: "chat.completion.chunk", + created: 1677652288, + model: "gemini-2.5-pro", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + id: "call_gemini", + type: "function", + function: { + name: "Read", + arguments: '{"file_path":"a.txt"}', + }, + }, + ], + }, + finish_reason: null, + logprobs: null, + }, + ], + }, + { + id: "cmpl-gemini-tool", + object: "chat.completion.chunk", + created: 1677652288, + model: "gemini-2.5-pro", + choices: [ + { index: 0, delta: {}, finish_reason: "tool_calls", logprobs: null }, + ], + }, + ] + + const streamState: AnthropicStreamState = { + messageStartSent: false, + messageStopSent: false, + contentBlockIndex: 0, + contentBlockOpen: false, + thinkingBlockOpen: false, + hasEmittedText: false, + toolCalls: {}, + thinkingEnabled: false, + } + + const translatedStream = openAIStream.flatMap((chunk) => + translateChunkToAnthropicEvents(chunk, streamState), + ) + + const textDeltas = translatedStream.filter( + (event) => + event.type === "content_block_delta" + && event.delta.type === "text_delta", + ) + expect(textDeltas.length).toBeGreaterThan(0) + }) +}) + +function freshStreamState(): AnthropicStreamState { + return { + messageStartSent: false, + messageStopSent: false, + contentBlockIndex: 0, + contentBlockOpen: false, + thinkingBlockOpen: false, + hasEmittedText: false, + toolCalls: {}, + thinkingEnabled: false, + } +} + +function finalStreamDelta(state: AnthropicStreamState) { + const delta = flushDeferredFinish(state).find( + (e) => e.type === "message_delta", + ) + if (delta?.type !== "message_delta") { + throw new Error("Expected a terminating message_delta") + } + return delta +} + +describe("Streaming terminating stop_reason — never null/omitted", () => { + // Regression: a streamed tool call whose finish chunk reports a non-standard + // finish_reason (Copilot runtime can violate its own type) must still defer a + // "tool_calls" reason and terminate as "tool_use" — not omit stop_reason, + // which would make Claude Code skip executing the tool. + test("tool-call stream with a non-standard finish_reason terminates as tool_use", () => { + const state = freshStreamState() + const stream: Array<ChatCompletionChunk> = [ + { + id: "c1", + object: "chat.completion.chunk", + created: 1, + model: "gemini-2.5-pro", + choices: [ + { + index: 0, + delta: { role: "assistant" }, + finish_reason: null, + logprobs: null, + }, + ], + }, + { + id: "c1", + object: "chat.completion.chunk", + created: 1, + model: "gemini-2.5-pro", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + id: "call_a", + type: "function", + function: { name: "get_weather", arguments: '{"loc":"NYC"}' }, + }, + ], + }, + finish_reason: null, + logprobs: null, + }, + ], + }, + { + id: "c1", + object: "chat.completion.chunk", + created: 1, + model: "gemini-2.5-pro", + // Non-standard finish_reason string — runtime type violation. + choices: [ + { + index: 0, + delta: {}, + finish_reason: "STOP" as never, + logprobs: null, + }, + ], + }, + ] + for (const chunk of stream) translateChunkToAnthropicEvents(chunk, state) + + const delta = finalStreamDelta(state) + expect(delta.delta.stop_reason).toBe("tool_use") + }) + + // A tool-call stream that finishes with the normal "stop" mismatch must also + // terminate as tool_use (existing Gemini-correction behavior, now broadened). + test("tool-call stream finishing with stop terminates as tool_use", () => { + const state = freshStreamState() + translateChunkToAnthropicEvents( + { + id: "c2", + object: "chat.completion.chunk", + created: 1, + model: "gemini-2.5-pro", + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + id: "call_b", + type: "function", + function: { name: "get_weather", arguments: '{"loc":"LA"}' }, + }, + ], + }, + finish_reason: null, + logprobs: null, + }, + ], + }, + state, + ) + translateChunkToAnthropicEvents( + { + id: "c2", + object: "chat.completion.chunk", + created: 1, + model: "gemini-2.5-pro", + choices: [ + { index: 0, delta: {}, finish_reason: "stop", logprobs: null }, + ], + }, + state, + ) + + expect(finalStreamDelta(state).delta.stop_reason).toBe("tool_use") + }) + + // A plain text stream finishing with a non-standard finish_reason and no tool + // calls must terminate as end_turn (not an omitted stop_reason). + test("text stream with a non-standard finish_reason terminates as end_turn", () => { + const state = freshStreamState() + translateChunkToAnthropicEvents( + { + id: "c3", + object: "chat.completion.chunk", + created: 1, + model: "gemini-2.5-pro", + choices: [ + { + index: 0, + delta: { content: "Hello" }, + finish_reason: null, + logprobs: null, + }, + ], + }, + state, + ) + translateChunkToAnthropicEvents( + { + id: "c3", + object: "chat.completion.chunk", + created: 1, + model: "gemini-2.5-pro", + choices: [ + { + index: 0, + delta: {}, + finish_reason: "DONE" as never, + logprobs: null, + }, + ], + }, + state, + ) + + expect(finalStreamDelta(state).delta.stop_reason).toBe("end_turn") + }) + + // A normal text stream still terminates correctly as end_turn. + test("normal text stream terminates as end_turn", () => { + const state = freshStreamState() + translateChunkToAnthropicEvents( + { + id: "c4", + object: "chat.completion.chunk", + created: 1, + model: "gpt-4o", + choices: [ + { + index: 0, + delta: { content: "Hi" }, + finish_reason: null, + logprobs: null, + }, + ], + }, + state, + ) + translateChunkToAnthropicEvents( + { + id: "c4", + object: "chat.completion.chunk", + created: 1, + model: "gpt-4o", + choices: [ + { index: 0, delta: {}, finish_reason: "stop", logprobs: null }, + ], + }, + state, + ) + + expect(finalStreamDelta(state).delta.stop_reason).toBe("end_turn") + }) }) diff --git a/tests/anthropic-tool-name-request.test.ts b/tests/anthropic-tool-name-request.test.ts new file mode 100644 index 000000000..fa851c55e --- /dev/null +++ b/tests/anthropic-tool-name-request.test.ts @@ -0,0 +1,67 @@ +import { describe, test, expect } from "bun:test" + +import type { AnthropicMessagesPayload } from "~/routes/messages/anthropic-types" + +import { translateToOpenAI } from "~/routes/messages/non-stream-translation" + +const LONG_MCP_TOOL_NAME = + "mcp__plugin_chrome-devtools-mcp_chrome-devtools__get_console_message" + +describe("Anthropic tool name aliasing", () => { + test("aliases long MCP tool names consistently across request fields", () => { + const anthropicPayload: AnthropicMessagesPayload = { + model: "claude-sonnet-4.6", + messages: [ + { role: "user", content: "Inspect the browser console." }, + { + role: "assistant", + content: [ + { + type: "text", + text: "Checking the browser console now.", + }, + { + type: "tool_use", + id: "toolu_1", + name: LONG_MCP_TOOL_NAME, + input: { request_id: "req_1" }, + }, + ], + }, + ], + max_tokens: 1000, + tools: [ + { + name: LONG_MCP_TOOL_NAME, + description: "Fetch a console message from the browser session.", + input_schema: { + type: "object", + properties: { + request_id: { type: "string" }, + }, + required: ["request_id"], + additionalProperties: false, + }, + }, + ], + tool_choice: { type: "tool", name: LONG_MCP_TOOL_NAME }, + } + + const openAIPayload = translateToOpenAI(anthropicPayload) + + const aliasedName = openAIPayload.tools?.[0]?.function.name + expect(aliasedName).toBeDefined() + expect(aliasedName).not.toBe(LONG_MCP_TOOL_NAME) + expect(aliasedName).toMatch(/^[\w-]{1,64}$/) + + expect(openAIPayload.tool_choice).toEqual({ + type: "function", + function: { name: aliasedName as string }, + }) + + const assistantMessage = openAIPayload.messages.find( + (message) => message.role === "assistant", + ) + expect(assistantMessage?.tool_calls?.[0]?.function.name).toBe(aliasedName) + }) +}) diff --git a/tests/attachment-overhead.test.ts b/tests/attachment-overhead.test.ts new file mode 100644 index 000000000..dd257527e --- /dev/null +++ b/tests/attachment-overhead.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, test } from "bun:test" + +import type { AnthropicMessagesPayload } from "~/routes/messages/anthropic-types" + +import { estimateAdditionalAttachmentTokens } from "~/routes/messages/attachment-overhead" + +describe("estimateAdditionalAttachmentTokens", () => { + test("adds substantial overhead for direct PDF attachments", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.6", + max_tokens: 1024, + messages: [ + { + role: "user", + content: [ + { + type: "document", + source: { + type: "base64", + media_type: "application/pdf", + data: "A".repeat(240_000), + }, + }, + { type: "text", text: "Summarize this." }, + ], + }, + ], + } + + expect(estimateAdditionalAttachmentTokens(payload)).toBeGreaterThan(2_500) + }) + + test("counts nested document attachments inside tool_result blocks", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.6", + max_tokens: 1024, + messages: [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "toolu_1", + content: [ + { + type: "document", + source: { + type: "text", + data: "x".repeat(20_000), + }, + }, + ], + }, + ], + }, + ], + } + + expect(estimateAdditionalAttachmentTokens(payload)).toBe(5_000) + }) + + test("counts nested image attachments inside tool_result blocks", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-sonnet-4.6", + max_tokens: 1024, + messages: [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "toolu_img", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "A".repeat(240_000), + }, + }, + ], + }, + ], + }, + ], + } + + expect(estimateAdditionalAttachmentTokens(payload)).toBeGreaterThan(1_600) + }) + + test("ignores plain text conversations without attachments", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.6", + max_tokens: 1024, + messages: [{ role: "user", content: "Hello" }], + } + + expect(estimateAdditionalAttachmentTokens(payload)).toBe(0) + }) +}) diff --git a/tests/burst-rate-limit.test.ts b/tests/burst-rate-limit.test.ts new file mode 100644 index 000000000..4b1375d67 --- /dev/null +++ b/tests/burst-rate-limit.test.ts @@ -0,0 +1,79 @@ +import { describe, test, expect } from "bun:test" + +import type { State } from "~/lib/state" + +import { checkBurstLimit } from "~/lib/rate-limit" + +function makeState(overrides: Partial<State> = {}): State { + return { + accountType: "individual", + manualApprove: false, + rateLimitWait: false, + showToken: false, + burstRequestTimestamps: [], + burstMinSpacingMs: 0, + burstScope: "global", + burstPerModelTimestamps: new Map(), + ...overrides, + } +} + +describe("checkBurstLimit", () => { + test("returns immediately when burst limiting is not configured", async () => { + const state = makeState() + const start = Date.now() + await checkBurstLimit(state) + expect(Date.now() - start).toBeLessThan(50) + expect(state.burstRequestTimestamps).toHaveLength(0) + }) + + test("returns immediately when only burstCount is set (not configured)", async () => { + const state = makeState({ burstCount: 3 }) + const start = Date.now() + await checkBurstLimit(state) + expect(Date.now() - start).toBeLessThan(50) + }) + + test("returns immediately when only burstWindowSeconds is set (not configured)", async () => { + const state = makeState({ burstWindowSeconds: 10 }) + const start = Date.now() + await checkBurstLimit(state) + expect(Date.now() - start).toBeLessThan(50) + }) + + test("records timestamp and proceeds when under the burst limit", async () => { + const state = makeState({ burstCount: 3, burstWindowSeconds: 10 }) + await checkBurstLimit(state) + expect(state.burstRequestTimestamps).toHaveLength(1) + await checkBurstLimit(state) + expect(state.burstRequestTimestamps).toHaveLength(2) + await checkBurstLimit(state) + expect(state.burstRequestTimestamps).toHaveLength(3) + }) + + test("prunes expired timestamps before checking the limit", async () => { + const state = makeState({ burstCount: 2, burstWindowSeconds: 1 }) + // Inject 2 timestamps that are already expired (2 seconds ago) + const expired = Date.now() - 2000 + state.burstRequestTimestamps = [expired, expired] + // Should proceed immediately (expired entries pruned → window is empty) + const start = Date.now() + await checkBurstLimit(state) + expect(Date.now() - start).toBeLessThan(50) + expect(state.burstRequestTimestamps).toHaveLength(1) + }) + + test("waits when the window is full and proceeds once a slot opens", async () => { + // Use a very short window so the test doesn't take long + const state = makeState({ burstCount: 1, burstWindowSeconds: 0.1 }) + // Fill the window with a timestamp right now + state.burstRequestTimestamps = [Date.now()] + const start = Date.now() + // This call should wait ~100ms for the slot to expire + await checkBurstLimit(state) + const elapsed = Date.now() - start + expect(elapsed).toBeGreaterThanOrEqual(50) // at least 50ms wait (window is 100ms) + expect(elapsed).toBeLessThan(500) // but not excessively long + expect(state.burstRequestTimestamps).toHaveLength(1) + }) +}) diff --git a/tests/create-chat-completions-retry.test.ts b/tests/create-chat-completions-retry.test.ts new file mode 100644 index 000000000..b80cc8002 --- /dev/null +++ b/tests/create-chat-completions-retry.test.ts @@ -0,0 +1,68 @@ +import { describe, test, expect, mock, beforeEach } from "bun:test" + +import type { ChatCompletionsPayload } from "~/services/copilot/create-chat-completions" + +import { state } from "~/lib/state" +import { createChatCompletions } from "~/services/copilot/create-chat-completions" + +state.copilotToken = "test-token" +state.vsCodeVersion = "1.0.0" +state.accountType = "individual" + +const fetchMock = mock( + ( + _url: string, + opts: { headers: Record<string, string> }, + ): Promise<unknown> => { + return Promise.resolve({ + ok: true, + json: () => ({ id: "123", object: "chat.completion", choices: [] }), + headers: opts.headers, + }) + }, +) + +// @ts-expect-error - Mock fetch doesn't implement all fetch properties +;(globalThis as unknown as { fetch: typeof fetch }).fetch = fetchMock + +beforeEach(() => { + fetchMock.mockClear() +}) + +describe("createChatCompletions transient upstream retries", () => { + test("retries GitHub/Copilot 502 HTML errors and returns the later success", async () => { + const payload: ChatCompletionsPayload = { + messages: [{ role: "user", content: "hi" }], + model: "gpt-test", + } + + fetchMock + .mockResolvedValueOnce( + new Response( + "<!DOCTYPE html><html><head><title>Unicorn! · GitHub

We had issues producing the response to your request.

", + { + status: 502, + headers: { "content-type": "text/html; charset=utf-8" }, + }, + ), + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + id: "retry-ok", + object: "chat.completion", + choices: [], + }), + { + status: 200, + headers: { "content-type": "application/json" }, + }, + ), + ) + + const response = await createChatCompletions(payload) + + expect(fetchMock).toHaveBeenCalledTimes(2) + expect((response as { id: string }).id).toBe("retry-ok") + }) +}) diff --git a/tests/create-chat-completions.test.ts b/tests/create-chat-completions.test.ts index d18e741aa..4969999a9 100644 --- a/tests/create-chat-completions.test.ts +++ b/tests/create-chat-completions.test.ts @@ -54,3 +54,13 @@ test("sets X-Initiator to user if only user present", async () => { ).headers expect(headers["X-Initiator"]).toBe("user") }) + +test("disables Bun internal body-read timeout to prevent mid-stream timeouts on large files", async () => { + const payload: ChatCompletionsPayload = { + messages: [{ role: "user", content: "hi" }], + model: "gpt-test", + } + await createChatCompletions(payload) + const fetchOpts = fetchMock.mock.calls[2][1] as Record + expect(fetchOpts.timeout).toBe(false) +}) diff --git a/tests/deterministic-body-error.test.ts b/tests/deterministic-body-error.test.ts new file mode 100644 index 000000000..962dd44bc --- /dev/null +++ b/tests/deterministic-body-error.test.ts @@ -0,0 +1,97 @@ +import { describe, test, expect, mock, beforeEach } from "bun:test" + +import type { ChatCompletionsPayload } from "~/services/copilot/create-chat-completions" + +import { state } from "~/lib/state" +import { createChatCompletions } from "~/services/copilot/create-chat-completions" + +state.copilotToken = "test-token" +state.vsCodeVersion = "1.0.0" +state.accountType = "individual" + +const fetchMock = mock((): Promise => Promise.resolve()) + +// @ts-expect-error - Mock fetch doesn't implement all fetch properties +;(globalThis as unknown as { fetch: typeof fetch }).fetch = fetchMock + +beforeEach(() => { + fetchMock.mockReset() +}) + +function bodyError(message: string): Response { + return new Response( + JSON.stringify({ error: { message, code: "invalid_request_body" } }), + { status: 400, headers: { "content-type": "application/json" } }, + ) +} + +describe("deterministic invalid_request_body errors are NOT retried", () => { + test("assistant prefill error fails immediately without burning retries", async () => { + fetchMock.mockResolvedValue( + bodyError( + "This model does not support assistant message prefill. The conversation must end with a user message.", + ), + ) + + const payload: ChatCompletionsPayload = { + messages: [{ role: "assistant", content: "primed" }], + model: "claude-opus-4.8", + } + + let threw = false + try { + await createChatCompletions(payload) + } catch { + threw = true + } + expect(threw).toBe(true) + // Deterministic shape error → exactly one upstream call, no exponential backoff. + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + test("orphaned tool_result error fails immediately", async () => { + fetchMock.mockResolvedValue( + bodyError( + "messages with role 'tool' must be a response to a preceeding message with 'tool_calls'.", + ), + ) + + const payload: ChatCompletionsPayload = { + messages: [{ role: "user", content: "hi" }], + model: "claude-opus-4.8", + } + + let threw = false + try { + await createChatCompletions(payload) + } catch { + threw = true + } + expect(threw).toBe(true) + expect(fetchMock).toHaveBeenCalledTimes(1) + }) +}) + +describe("genuinely transient invalid_request_body errors are still retried", () => { + test("a non-deterministic body error retries and can recover", async () => { + fetchMock + .mockResolvedValueOnce( + bodyError("temporary backend validation hiccup, please retry"), + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ id: "ok", object: "chat.completion", choices: [] }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ) + + const payload: ChatCompletionsPayload = { + messages: [{ role: "user", content: "hi" }], + model: "gpt-test", + } + + const response = await createChatCompletions(payload) + expect(fetchMock).toHaveBeenCalledTimes(2) + expect((response as { id: string }).id).toBe("ok") + }) +}) diff --git a/tests/error.test.ts b/tests/error.test.ts new file mode 100644 index 000000000..4e92f1380 --- /dev/null +++ b/tests/error.test.ts @@ -0,0 +1,284 @@ +import type { Context } from "hono" + +import { describe, test, expect, mock } from "bun:test" + +import { + HTTPError, + buildAnthropicContextWindowErrorResponse, + buildOpenAIContextWindowErrorBody, + buildResponsesContextWindowFailedEvent, + contextWindowErrorMessage, + CONTEXT_LENGTH_EXCEEDED_CODE, + forwardError, + isContextWindowError, + formatAnthropicContextWindowError, + sendAnthropicInvalidRequestError, +} from "~/lib/error" + +function makeContext(jsonFn = mock(), headerFn = mock()) { + return { json: jsonFn, header: headerFn } as unknown as Context +} + +function makeHTTPError(body: string, status: number): HTTPError { + const response = new Response(body, { status }) + return new HTTPError("Failed", response) +} + +describe("forwardError — HTTPError with JSON body", () => { + test("converts context-window Copilot error to Anthropic format with token numbers", async () => { + const jsonFn = mock() + const headerFn = mock() + const c = makeContext(jsonFn, headerFn) + const err = makeHTTPError( + JSON.stringify({ + error: { + message: "prompt token count of 55059 exceeds the limit of 12288", + code: "model_max_prompt_tokens_exceeded", + }, + }), + 400, + ) + await forwardError(c, err) + expect(jsonFn).toHaveBeenCalledTimes(1) + const [body, status] = jsonFn.mock.calls[0] as [unknown, number] + expect(status).toBe(400) + const typed = body as { + type: string + request_id: string + error: { type: string; message: string } + } + expect(typed.type).toBe("error") + expect(typed.request_id).toMatch(/^req_/) + expect(headerFn).toHaveBeenCalledWith("request-id", typed.request_id) + expect(typed.error.type).toBe("invalid_request_error") + expect(typed.error.message).toBe( + "prompt is too long: 55059 tokens > 12288 maximum", + ) + }) + + test("wraps plain-text error in envelope", async () => { + const jsonFn = mock() + const c = makeContext(jsonFn) + const err = makeHTTPError("Bad Gateway", 502) + await forwardError(c, err) + const [body, status] = jsonFn.mock.calls[0] as [unknown, number] + expect(status).toBe(502) + expect( + (body as { error: { message: string; type: string } }).error.type, + ).toBe("error") + expect((body as { error: { message: string } }).error.message).toBe( + "Bad Gateway", + ) + }) + + test("summarizes HTML upstream error pages instead of returning raw markup", async () => { + const jsonFn = mock() + const c = makeContext(jsonFn) + const err = makeHTTPError( + "Unicorn! · GitHub

We had issues producing the response to your request.

", + 502, + ) + await forwardError(c, err) + const [body, status] = jsonFn.mock.calls[0] as [unknown, number] + expect(status).toBe(502) + expect((body as { error: { message: string } }).error.message).toBe( + "Upstream returned an HTML error page: Unicorn! · GitHub - We had issues producing the response to your request.", + ) + }) + + test("preserves status code from upstream", async () => { + const jsonFn = mock() + const c = makeContext(jsonFn) + const err = makeHTTPError( + JSON.stringify({ error: { message: "not found" } }), + 404, + ) + await forwardError(c, err) + const [, status] = jsonFn.mock.calls[0] as [unknown, number] + expect(status).toBe(404) + }) +}) + +describe("forwardError — non-HTTPError", () => { + test("returns 500 with error message", async () => { + const jsonFn = mock() + const c = makeContext(jsonFn) + await forwardError(c, new Error("something broke")) + const [body, status] = jsonFn.mock.calls[0] as [unknown, number] + expect(status).toBe(500) + expect((body as { error: { message: string } }).error.message).toBe( + "something broke", + ) + }) +}) + +describe("isContextWindowError", () => { + test("detects Copilot 'exceeds the limit' format", () => { + expect( + isContextWindowError( + "prompt token count of 622303 exceeds the limit of 168000", + ), + ).toBe(true) + }) + + test("detects model_max_prompt_tokens_exceeded code", () => { + expect( + isContextWindowError( + '{"error":{"message":"prompt too big","code":"model_max_prompt_tokens_exceeded"}}', + ), + ).toBe(true) + }) + + test("detects 'exceeds the context window'", () => { + expect( + isContextWindowError("This request exceeds the context window"), + ).toBe(true) + }) + + test("detects context_length_exceeded", () => { + expect(isContextWindowError("context_length_exceeded")).toBe(true) + }) + + test("detects 'maximum context length'", () => { + expect( + isContextWindowError( + "This model's maximum context length is 128000 tokens", + ), + ).toBe(true) + }) + + test("detects 'input exceeds'", () => { + expect(isContextWindowError("input exceeds model limit")).toBe(true) + }) + + test("does not treat opaque parser 413 as context-window", () => { + expect(isContextWindowError("failed to parse request", 413)).toBe(false) + }) + + test("returns false for unrelated errors", () => { + expect(isContextWindowError("rate limit exceeded")).toBe(false) + expect(isContextWindowError("internal server error")).toBe(false) + expect(isContextWindowError("model not found")).toBe(false) + }) +}) + +describe("formatAnthropicContextWindowError", () => { + test("extracts token numbers from Copilot error format", () => { + expect( + formatAnthropicContextWindowError( + "prompt token count of 622303 exceeds the limit of 168000", + ), + ).toBe("prompt is too long: 622303 tokens > 168000 maximum") + }) + + test("handles comma-separated numbers", () => { + expect( + formatAnthropicContextWindowError( + "prompt token count of 622,303 exceeds the limit of 168,000", + ), + ).toBe("prompt is too long: 622303 tokens > 168000 maximum") + }) + + test("falls back to defaults when no numbers found", () => { + const result = formatAnthropicContextWindowError("context_length_exceeded") + expect(result).toMatch(/^prompt is too long: \d+ tokens > \d+ maximum$/) + }) + + test("falls back to defaults for empty string", () => { + const result = formatAnthropicContextWindowError("") + expect(result).toMatch(/^prompt is too long: \d+ tokens > \d+ maximum$/) + }) +}) + +describe("buildAnthropicContextWindowErrorResponse", () => { + test("uses the same request id in the header payload and body", () => { + const result = buildAnthropicContextWindowErrorResponse( + "prompt token count of 622303 exceeds the limit of 168000", + ) + expect(result.requestId).toMatch(/^req_/) + expect(result.body.request_id).toBe(result.requestId) + expect(result.body.error.message).toBe( + "prompt is too long: 622303 tokens > 168000 maximum", + ) + }) +}) + +describe("sendAnthropicInvalidRequestError", () => { + test("returns Anthropic-shaped invalid_request_error with request-id header", () => { + const jsonFn = mock() + const headerFn = mock() + const c = makeContext(jsonFn, headerFn) + + sendAnthropicInvalidRequestError(c, "Embedded image is too small.") + + const [body, status] = jsonFn.mock.calls[0] as [unknown, number] + const typed = body as { + type: string + request_id: string + error: { type: string; message: string } + } + expect(status).toBe(400) + expect(typed.type).toBe("error") + expect(typed.request_id).toMatch(/^req_/) + expect(typed.error.type).toBe("invalid_request_error") + expect(typed.error.message).toBe("Embedded image is too small.") + expect(headerFn).toHaveBeenCalledWith("request-id", typed.request_id) + }) +}) + +describe("Responses API (Codex) context-window signaling", () => { + // Codex's SSE detector matches strictly on this code; do not change it + // without re-verifying against codex-rs/codex-api/src/sse/responses.rs. + test("exposes the exact code Codex pattern-matches", () => { + expect(CONTEXT_LENGTH_EXCEEDED_CODE).toBe("context_length_exceeded") + }) + + test("buildResponsesContextWindowFailedEvent has the shape Codex parses", () => { + const event = buildResponsesContextWindowFailedEvent( + "prompt token count of 1402500 exceeds the limit of 935000", + ) + expect(event.type).toBe("response.failed") + const response = event.response as { + status: string + error: { code: string; message: string } + } + expect(response.status).toBe("failed") + // The decisive field: Codex reads response.error.code. + expect(response.error.code).toBe("context_length_exceeded") + // A real, informative upstream message is preserved (no fabrication). + expect(response.error.message).toContain("exceeds the limit of 935000") + }) + + test("does NOT fabricate token counts when upstream message is generic", () => { + const event = buildResponsesContextWindowFailedEvent( + "failed to parse request", + ) + const response = event.response as { error: { message: string } } + // The misleading "1402500 tokens > 935000" fabrication must be gone. + expect(response.error.message).not.toMatch(/\d+ tokens > \d+/) + expect(response.error.message).not.toContain("1402500") + }) + + test("contextWindowErrorMessage keeps genuine token-count messages", () => { + expect( + contextWindowErrorMessage( + "prompt token count of 55059 exceeds the limit of 12288", + ), + ).toBe("prompt token count of 55059 exceeds the limit of 12288") + }) + + test("contextWindowErrorMessage falls back for unhelpful upstream text", () => { + expect(contextWindowErrorMessage("failed to parse request")).not.toContain( + "failed to parse request", + ) + expect(contextWindowErrorMessage(undefined)).toMatch(/context window/i) + }) + + test("buildOpenAIContextWindowErrorBody is OpenAI-shaped with the code", () => { + const body = buildOpenAIContextWindowErrorBody("input exceeds model limit") + expect(body.error.type).toBe("invalid_request_error") + expect(body.error.code).toBe("context_length_exceeded") + expect(body.error.param).toBeNull() + expect(body.error.message).toBe("input exceeds model limit") + }) +}) diff --git a/tests/gemini-compaction.test.ts b/tests/gemini-compaction.test.ts new file mode 100644 index 000000000..dda557dc5 --- /dev/null +++ b/tests/gemini-compaction.test.ts @@ -0,0 +1,367 @@ +/* eslint-disable max-lines-per-function */ +import { describe, expect, test } from "bun:test" + +import type { AnthropicMessagesPayload } from "~/routes/messages/anthropic-types" +import type { ChatCompletionResponse } from "~/services/copilot/create-chat-completions" +import type { Model } from "~/services/copilot/get-models" + +import { scaleTokensForModel } from "~/routes/messages/count-tokens-handler" +import { + buildSyntheticCompactionResponse, + looksLikeCompactionRequest, + shouldUsePlainTextCompactionFallback, +} from "~/routes/messages/handler" + +function makeModel( + id: string, + maxPromptTokens: number, + maxContextWindowTokens: number, +): Model { + return { + id, + object: "model", + name: id, + vendor: "google", + version: "preview", + model_picker_enabled: true, + preview: true, + capabilities: { + family: "gemini", + object: "capabilities", + tokenizer: "o200k_base", + type: "chat", + limits: { + max_prompt_tokens: maxPromptTokens, + max_context_window_tokens: maxContextWindowTokens, + max_output_tokens: 64_000, + }, + supports: {}, + }, + } +} + +function makeResponse( + content: string | null, + finishReason: "stop" | "tool_calls" | "length", + hasToolCalls: boolean = false, +): ChatCompletionResponse { + return { + id: "chatcmpl_test", + object: "chat.completion", + created: 0, + model: "gemini-3.1-pro-preview", + choices: [ + { + index: 0, + message: { + role: "assistant", + content, + ...(hasToolCalls && { + tool_calls: [ + { + id: "call_1", + type: "function", + function: { + name: "TodoWrite", + arguments: "{}", + }, + }, + ], + }), + }, + logprobs: null, + finish_reason: finishReason, + }, + ], + usage: { + prompt_tokens: 100, + completion_tokens: 20, + total_tokens: 120, + }, + } +} + +describe("Gemini compaction safeguards", () => { + test("detects likely compaction requests from summary prompts", () => { + const payload: AnthropicMessagesPayload = { + model: "gemini-3.1-pro-preview", + max_tokens: 2048, + tools: [ + { + name: "TodoWrite", + description: "todo tool", + input_schema: { type: "object", properties: {} }, + }, + ], + messages: [ + { + role: "user", + content: + "Please summarize the conversation into a compact summary for continuation.", + }, + ], + } + + expect(looksLikeCompactionRequest(payload)).toBe(true) + }) + + test("detects explicit requests to generate a continuation summary", () => { + const payload: AnthropicMessagesPayload = { + model: "gemini-3.1-pro-preview", + max_tokens: 2048, + messages: [ + { + role: "user", + content: + "Create a conversation continuation summary so this session can be resumed later.", + }, + ], + } + + expect(looksLikeCompactionRequest(payload)).toBe(true) + }) + + test("detects vscode compact command payloads", () => { + const payload: AnthropicMessagesPayload = { + model: "gemini-3.1-pro-preview", + max_tokens: 2048, + messages: [ + { + role: "user", + content: + "/compact\ncompact\n", + }, + ], + } + + expect(looksLikeCompactionRequest(payload)).toBe(true) + }) + + test("does not treat resumed post-compaction prompts as compaction requests", () => { + const payload: AnthropicMessagesPayload = { + model: "gemini-3.1-pro-preview", + max_tokens: 2048, + messages: [ + { + role: "user", + content: + "This session is being continued from a previous conversation that ran out of context. " + + "Resume directly from the latest state above and continue the task.", + }, + ], + } + + expect(looksLikeCompactionRequest(payload)).toBe(false) + }) + + test("ignores older summary text in history when the latest user turn is continue", () => { + const payload: AnthropicMessagesPayload = { + model: "gemini-3.1-pro-preview", + max_tokens: 2048, + messages: [ + { + role: "assistant", + content: + "Conversation continuation summary:\n[user] Prior work summary\n\nContinue from the latest state above.", + }, + { + role: "user", + content: "continue", + }, + ], + } + + expect(looksLikeCompactionRequest(payload)).toBe(false) + }) + + test("does not treat pasted continuation scaffold as a compaction request", () => { + const payload: AnthropicMessagesPayload = { + model: "gemini-3.1-pro-preview", + max_tokens: 2048, + messages: [ + { + role: "user", + content: + "This session is being continued from a previous conversation that ran out of context. " + + "The summary below covers the earlier portion of the conversation. " + + "Conversation continuation summary: [assistant] prior work summary. " + + "Resume directly from the latest state above and continue the task.", + }, + ], + } + + expect(looksLikeCompactionRequest(payload)).toBe(false) + }) + + test("does not treat claude auto-compact continuation payloads as compaction requests", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-sonnet-4.6", + max_tokens: 2048, + messages: [ + { + role: "user", + content: + "This session is being continued from a previous conversation that ran out of context. " + + "The summary below covers the earlier portion of the conversation. " + + "Conversation continuation summary: [assistant] prior work summary. " + + "Continue from the latest state above. Older context was compacted to fit the model context window. " + + "Continue the conversation from where it left off without asking the user any further questions. " + + "Resume directly — do not acknowledge the summary, do not recap what was happening.", + }, + ], + } + + expect(looksLikeCompactionRequest(payload)).toBe(false) + }) + + test("uses plain-text fallback when compaction response tries to call tools", () => { + const payload: AnthropicMessagesPayload = { + model: "gemini-3.1-pro-preview", + max_tokens: 2048, + tools: [ + { + name: "TodoWrite", + description: "todo tool", + input_schema: { type: "object", properties: {} }, + }, + ], + messages: [ + { + role: "user", + content: + "Create a conversation summary so this session can be continued from a previous conversation.", + }, + ], + } + + expect( + shouldUsePlainTextCompactionFallback( + payload, + makeResponse(null, "tool_calls", true), + ), + ).toBe(true) + }) + + test("does not use the fallback for ordinary non-compaction requests", () => { + const payload: AnthropicMessagesPayload = { + model: "gemini-3.1-pro-preview", + max_tokens: 2048, + tools: [ + { + name: "TodoWrite", + description: "todo tool", + input_schema: { type: "object", properties: {} }, + }, + ], + messages: [{ role: "user", content: "Continue editing the toolbar." }], + } + + expect( + shouldUsePlainTextCompactionFallback( + payload, + makeResponse(null, "tool_calls", true), + ), + ).toBe(false) + }) + + test("scales Gemini token counts with extra safety margin", () => { + const gemini = makeModel("gemini-3.1-pro-preview", 136_000, 200_000) + expect(scaleTokensForModel(136_000, gemini)).toBeGreaterThan(200_000) + }) + + test("builds a synthetic plain-text summary when Gemini compaction still fails", () => { + const payload: AnthropicMessagesPayload = { + model: "gemini-3.1-pro-preview", + max_tokens: 2048, + messages: [ + { + role: "user", + content: + "Create a conversation summary so this session can be continued from a previous conversation.", + }, + { + role: "assistant", + content: + "We investigated the failing seed script and found the API healthcheck was flaky.", + }, + ], + } + + const response = buildSyntheticCompactionResponse(payload) + expect(response.choices[0]?.message.content).toContain( + "Conversation continuation summary:", + ) + expect(response.choices[0]?.message.content).toContain( + "failing seed script", + ) + expect(response.choices[0]?.finish_reason).toBe("stop") + }) + + test("uses head and tail preservation when compaction fragments are long", () => { + const payload: AnthropicMessagesPayload = { + model: "gemini-3.1-pro-preview", + max_tokens: 2048, + messages: [ + { + role: "user", + content: + "Create a conversation summary so this session can be continued from a previous conversation.", + }, + { + role: "assistant", + content: `${"A".repeat(500)}MIDDLE${"Z".repeat(500)}`, + }, + ], + } + + const response = buildSyntheticCompactionResponse(payload) + const content = response.choices[0]?.message.content ?? "" + expect(content).toContain("AAAA") + expect(content).toContain("ZZZZ") + expect(content).toContain(" ... ") + }) + + test("applies extra proactive buffer for Gemini windows", () => { + const gemini = makeModel("gemini-3.1-pro-preview", 136_000, 200_000) + expect(scaleTokensForModel(128_000, gemini)).toBeGreaterThan(210_000) + }) + + test("does not false-positive on system-reminder content containing compaction keywords", () => { + // Claude Code injects tags into user messages with + // skill descriptions that contain words like "compact", "conversation", + // "session". These should not trigger compaction detection. + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.6", + max_tokens: 32000, + tools: [ + { + name: "Read", + description: "read tool", + input_schema: { type: "object", properties: {} }, + }, + ], + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: + "\nThe following skills are available:\n" + + "- claude-api: caching, thinking, compaction, tool use\n" + + "- insights: Generate a report analyzing your Claude Code sessions\n" + + "- auto-memory, persists across conversations\n" + + "", + }, + { + type: "text", + text: "open the below page in playwright mcp\n\nhttps://example.com/api-details", + }, + ], + }, + ], + } + + expect(looksLikeCompactionRequest(payload)).toBe(false) + }) +}) diff --git a/tests/image-trimming.test.ts b/tests/image-trimming.test.ts new file mode 100644 index 000000000..27d382528 --- /dev/null +++ b/tests/image-trimming.test.ts @@ -0,0 +1,179 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test" + +import type { AnthropicMessagesPayload } from "~/routes/messages/anthropic-types" + +import { + fetchWithImageStripping, + type ImageStrippingResult, +} from "~/routes/messages/image-stripping" + +const originalEnabled = process.env.IMAGE_CONTEXT_TRIMMING_ENABLED +const originalThreshold = process.env.IMAGE_CONTEXT_TRIMMING_BEFORE_MESSAGES + +beforeEach(() => { + delete process.env.IMAGE_CONTEXT_TRIMMING_ENABLED + delete process.env.IMAGE_CONTEXT_TRIMMING_BEFORE_MESSAGES +}) + +afterEach(() => { + if (originalEnabled === undefined) { + delete process.env.IMAGE_CONTEXT_TRIMMING_ENABLED + } else { + process.env.IMAGE_CONTEXT_TRIMMING_ENABLED = originalEnabled + } + + if (originalThreshold === undefined) { + delete process.env.IMAGE_CONTEXT_TRIMMING_BEFORE_MESSAGES + } else { + process.env.IMAGE_CONTEXT_TRIMMING_BEFORE_MESSAGES = originalThreshold + } +}) + +async function capturePayload( + payload: AnthropicMessagesPayload, +): Promise> { + return fetchWithImageStripping( + (receivedPayload) => Promise.resolve(receivedPayload), + payload, + ) +} + +function buildImage(data = "A".repeat(4_000)) { + return { + type: "image" as const, + source: { + type: "base64" as const, + media_type: "image/png" as const, + data, + }, + } +} + +describe("processed image trimming", () => { + test("does not trim anything when feature is disabled", async () => { + const payload: AnthropicMessagesPayload = { + model: "claude-sonnet-4.6", + max_tokens: 1024, + messages: [ + { + role: "user", + content: [{ type: "text", text: "look" }, buildImage()], + }, + { role: "assistant", content: "I inspected it." }, + { role: "user", content: "continue" }, + ], + } + + const result = await capturePayload(payload) + expect(result.response).toEqual(payload) + }) + + test("trims processed images once they are older than the configured message threshold", async () => { + process.env.IMAGE_CONTEXT_TRIMMING_ENABLED = "true" + process.env.IMAGE_CONTEXT_TRIMMING_BEFORE_MESSAGES = "2" + + const payload: AnthropicMessagesPayload = { + model: "claude-sonnet-4.6", + max_tokens: 1024, + messages: [ + { + role: "user", + content: [{ type: "text", text: "look" }, buildImage()], + }, + { role: "assistant", content: "I inspected the screenshot." }, + { role: "user", content: "make a change" }, + { role: "assistant", content: "Done." }, + ], + } + + const result = await capturePayload(payload) + const firstMessage = result.response.messages[0] + expect(firstMessage.role).toBe("user") + if (typeof firstMessage.content === "string") { + throw new TypeError("Expected block content") + } + expect(firstMessage.content[1]).toEqual({ + type: "text", + text: "[Processed image trimmed to reduce request size]", + }) + }) + + test("keeps back-to-back screenshots when the assistant has not processed them yet", async () => { + process.env.IMAGE_CONTEXT_TRIMMING_ENABLED = "true" + process.env.IMAGE_CONTEXT_TRIMMING_BEFORE_MESSAGES = "0" + + const payload: AnthropicMessagesPayload = { + model: "claude-sonnet-4.6", + max_tokens: 1024, + messages: [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "toolu_1", + content: [buildImage("A".repeat(5_000))], + }, + ], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "toolu_2", + content: [buildImage("B".repeat(5_000))], + }, + ], + }, + ], + } + + const result = await capturePayload(payload) + expect(result.response).toEqual(payload) + }) + + test("trims nested tool_result images after a later assistant explanation", async () => { + process.env.IMAGE_CONTEXT_TRIMMING_ENABLED = "true" + process.env.IMAGE_CONTEXT_TRIMMING_BEFORE_MESSAGES = "1" + + const payload: AnthropicMessagesPayload = { + model: "claude-sonnet-4.6", + max_tokens: 1024, + messages: [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "toolu_1", + content: [buildImage("A".repeat(5_000))], + }, + ], + }, + { + role: "assistant", + content: "I compared both screenshots and found the layout issue.", + }, + { role: "user", content: "continue" }, + ], + } + + const result = await capturePayload(payload) + const firstMessage = result.response.messages[0] + if (typeof firstMessage.content === "string") { + throw new TypeError("Expected block content") + } + const toolResult = firstMessage.content[0] + if ( + toolResult.type !== "tool_result" + || typeof toolResult.content === "string" + ) { + throw new TypeError("Expected nested tool result content") + } + expect(toolResult.content[0]).toEqual({ + type: "text", + text: "[Processed image trimmed to reduce request size]", + }) + }) +}) diff --git a/tests/image-validation.test.ts b/tests/image-validation.test.ts new file mode 100644 index 000000000..4c8deaf62 --- /dev/null +++ b/tests/image-validation.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, test } from "bun:test" + +import type { AnthropicMessagesPayload } from "~/routes/messages/anthropic-types" + +import { findInvalidEmbeddedImage } from "~/routes/messages/image-validation" + +describe("findInvalidEmbeddedImage", () => { + test("flags degenerate 1x1 PNG user images", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.6", + max_tokens: 16, + messages: [ + { + role: "user", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9WnSUswAAAAASUVORK5CYII=", + }, + }, + ], + }, + ], + } + + expect(findInvalidEmbeddedImage(payload)).toEqual({ + mediaType: "image/png", + width: 1, + height: 1, + }) + }) + + test("allows normal PNG images", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.6", + max_tokens: 16, + messages: [ + { + role: "user", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgoAAAANSUhEUgAAAGQAAABkCAYAAABw4pVUAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAAHVSURBVHhe7dGxaYMBFMTgf/9dvE8myBpOLzgwBORXqPiaK0/P6+f3nTseDvmughxTkGMKckxBjinIMQU5piDHFOSYj4O8nyf/wD+Xgkj451IQCf9cCiLhn0tBJPxzKYiEfy4FkfDPpSAS/rkURMI/l4JI+OdSEAn/XAoi4Z9LQST8cymIhH8uBZHwz6UgEv65FETCP5eCSPjnUhAJ/1wKIuGfS0Ek/HMpiIR/LgWR8M+lIBL+uRREwj+Xgkj451IQCf9cCiLhn0tBJPxzKYiEfy4FkfDPpSAS/rkURMI/l4JI+OdSEAn/XAoi4Z9LQST8cymIhH8uBZHwz6UgEv65FETCP5eCSPjnUhAJ/1wKIuGfS0Ek/HMpiIR/LgWR8M+lIBL+uRREwj+Xgkj451IQCf9cCiLhn0tBJPxzKYiEfy4FkfDPpSAS/rkURMI/l4JI+OdSEAn/XAoi4Z9LQST8cymIhH8uBZHwz6UgEv65FETCP5eCSPjnUhAJ/1wKIuGfS0Ek/HMpiIR/LgWR8M+lIBL+uRREwj+Xgkj451IQCf9cCiLhn0tBJPxzKYiEfy4FkfDPpSAS/rl8HCSOghxTkGMKckxBjinIMQU5piDHFOSYghzzBwck1lCKw6QmAAAAAElFTkSuQmCC", + }, + }, + ], + }, + ], + } + + expect(findInvalidEmbeddedImage(payload)).toBeUndefined() + }) + + test("checks images nested inside tool results", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.6", + max_tokens: 16, + messages: [ + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "tool_1", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: "image/png", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAusB9WnSUswAAAAASUVORK5CYII=", + }, + }, + ], + }, + ], + }, + ], + } + + expect(findInvalidEmbeddedImage(payload)?.width).toBe(1) + }) +}) diff --git a/tests/large-edit-guidance.test.ts b/tests/large-edit-guidance.test.ts new file mode 100644 index 000000000..e37aca8fe --- /dev/null +++ b/tests/large-edit-guidance.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, test } from "bun:test" + +import type { ChatCompletionsPayload } from "~/services/copilot/create-chat-completions" + +import { applyLargeEditGuidance } from "~/routes/messages/large-edit-guidance" + +function buildPayload(toolNames: Array): ChatCompletionsPayload { + return { + model: "claude-opus-4.6", + max_tokens: 64_000, + messages: [{ role: "user", content: "Make the requested code change." }], + tools: toolNames.map((name) => ({ + type: "function", + function: { + name, + description: `${name} tool`, + parameters: { + type: "object", + properties: {}, + additionalProperties: false, + }, + }, + })), + } +} + +describe("applyLargeEditGuidance", () => { + test("injects a system hint for low-output models with file-edit tools", () => { + const payload = buildPayload(["Read", "Write", "Edit"]) + + applyLargeEditGuidance(payload, 32_000) + + expect(payload.messages[0]).toMatchObject({ + role: "system", + }) + expect(payload.messages[0]?.content).toContain("File-editing budget:") + expect(payload.messages[0]?.content).toContain("32,000") + }) + + test("upgrades guidance for obviously risky one-shot large rewrite requests", () => { + const payload = buildPayload(["Write"]) + payload.messages = [ + { + role: "user", + content: + "Write the complete file in exactly one Write call with 9000 lines. Do not split it.", + }, + ] + + applyLargeEditGuidance(payload, 32_000) + + expect(payload.messages[0]?.content).toContain( + "High-risk large edit detected:", + ) + expect(payload.messages[0]?.content).toContain( + "Do not satisfy this request with one massive Write/Edit/MultiEdit call", + ) + }) + + test("does not inject guidance when no file-edit tools are present", () => { + const payload = buildPayload(["Read", "Bash"]) + + applyLargeEditGuidance(payload, 32_000) + + expect(payload.messages).toHaveLength(1) + expect(payload.messages[0]?.role).toBe("user") + }) + + test("does not inject guidance for higher-output models", () => { + const payload = buildPayload(["Write"]) + + applyLargeEditGuidance(payload, 64_000) + + expect(payload.messages).toHaveLength(1) + expect(payload.messages[0]?.role).toBe("user") + }) + + test("does not duplicate guidance when it already exists", () => { + const payload = buildPayload(["Write"]) + payload.messages.unshift({ + role: "system", + content: + "File-editing budget: this model can emit about 32,000 output tokens in one turn.", + }) + + applyLargeEditGuidance(payload, 32_000) + + expect( + payload.messages.filter( + (message) => + message.role === "system" + && typeof message.content === "string" + && message.content.includes("File-editing budget:"), + ), + ).toHaveLength(1) + }) +}) diff --git a/tests/message-invariants.test.ts b/tests/message-invariants.test.ts new file mode 100644 index 000000000..bde0f7b53 --- /dev/null +++ b/tests/message-invariants.test.ts @@ -0,0 +1,225 @@ +import { describe, test, expect } from "bun:test" + +import type { AnthropicMessagesPayload } from "~/routes/messages/anthropic-types" +import type { Message } from "~/services/copilot/create-chat-completions" + +import { translateToOpenAI } from "~/routes/messages/non-stream-translation" + +/** + * Regression tests for two message-array invariant violations that broke + * Claude Code's multi-agent / workflow ("ultracode") mode against the Copilot + * Claude backend. Both manifest as `invalid_request_body` 400s. + * + * Bug A — assistant message prefill: + * A request whose final Anthropic message has role:"assistant" (a "prefill" + * priming the reply) is translated into an OpenAI array that also ends with + * an assistant message. Copilot's Claude backend rejects this: + * "This model does not support assistant message prefill. The conversation + * must end with a user message." + * + * Bug B — merge drops tool_calls: + * mergeConsecutiveSameRoleMessages merges two consecutive assistant messages + * by copying only `.content`, silently dropping the second message's + * `tool_calls`. A following tool_result then has no owning assistant turn: + * "messages with role 'tool' must be a response to a preceeding message + * with 'tool_calls'." + */ + +function toolCallIds(messages: Array): Array { + return messages + .filter((m) => m.role === "assistant" && m.tool_calls) + .flatMap((m) => m.tool_calls ?? []) + .map((tc) => tc.id) +} + +function toolResultIds(messages: Array): Array { + return messages + .filter((m) => m.role === "tool") + .map((m) => m.tool_call_id) + .filter((id): id is string => id !== undefined) +} + +describe("Bug A: assistant prefill normalization", () => { + test("never emits a trailing assistant message — conversation must end with user", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.8", + max_tokens: 1024, + messages: [ + { role: "user", content: "Write a haiku about the sea." }, + { role: "assistant", content: [{ type: "text", text: "Here it is:" }] }, + ], + } + + const result = translateToOpenAI(payload) + + expect(result.messages.at(-1)?.role).not.toBe("assistant") + }) + + test("preserves the prefill text when normalizing a trailing assistant", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.8", + max_tokens: 1024, + messages: [ + { role: "user", content: "Continue the story." }, + { + role: "assistant", + content: [{ type: "text", text: "Once upon a time" }], + }, + ], + } + + const result = translateToOpenAI(payload) + + // The prefill content must survive somewhere in the array (not dropped). + expect(JSON.stringify(result.messages)).toContain("Once upon a time") + // And it must still end with a user turn. + expect(result.messages.at(-1)?.role).toBe("user") + }) + + test("does NOT append a continuation when the conversation already ends with user", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.8", + max_tokens: 1024, + messages: [ + { role: "user", content: "First." }, + { role: "assistant", content: [{ type: "text", text: "Reply." }] }, + { role: "user", content: "Second." }, + ], + } + + const result = translateToOpenAI(payload) + + // Last message is the genuine user turn, unchanged. + expect(result.messages.at(-1)?.role).toBe("user") + expect(result.messages.at(-1)?.content).toContain("Second.") + // Exactly system?+user+assistant+user — no spurious extra message. + expect(result.messages.filter((m) => m.role === "user")).toHaveLength(2) + }) + + test("does NOT strand an assistant turn that has pending tool_calls", () => { + // An assistant message whose last block is a tool_use is NOT a prefill — + // appending a user message here would orphan the tool_calls. This shape is + // degenerate as a final message, but the normalization must never create an + // assistant(tool_calls) → user adjacency. + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.8", + max_tokens: 1024, + messages: [ + { role: "user", content: "Run it." }, + { + role: "assistant", + content: [ + { type: "tool_use", id: "toolu_pending", name: "run", input: {} }, + ], + }, + ], + } + + const result = translateToOpenAI(payload) + + const last = result.messages.at(-1) + // If we appended a user message, it must not directly follow an assistant + // that still has unanswered tool_calls. + if (last?.role === "user") { + const prev = result.messages.at(-2) + expect(prev?.tool_calls?.length ?? 0).toBe(0) + } + }) +}) + +describe("Bug B: merge must not drop tool_calls", () => { + test("merging consecutive assistant messages keeps tool_calls so the tool_result stays anchored", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.8", + max_tokens: 1024, + messages: [ + { role: "user", content: "Do it." }, + { role: "assistant", content: [{ type: "text", text: "Thinking..." }] }, + { + role: "assistant", + content: [ + { type: "tool_use", id: "toolu_1", name: "run", input: {} }, + ], + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "toolu_1", content: "done" }, + ], + }, + ], + } + + const result = translateToOpenAI(payload) + + // The tool_call id must still be produced by some assistant turn. + const callIds = new Set(toolCallIds(result.messages)) + expect(callIds.has("toolu_1")).toBe(true) + + // Every tool message must reference a known assistant tool_call. + for (const id of toolResultIds(result.messages)) { + expect(callIds.has(id)).toBe(true) + } + }) + + test("the tool message immediately follows an assistant bearing matching tool_calls", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.8", + max_tokens: 1024, + messages: [ + { role: "user", content: "Go." }, + { role: "assistant", content: [{ type: "text", text: "Step one." }] }, + { + role: "assistant", + content: [{ type: "tool_use", id: "toolu_x", name: "do", input: {} }], + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "toolu_x", content: "ok" }, + ], + }, + ], + } + + const result = translateToOpenAI(payload) + + const toolIndex = result.messages.findIndex((m) => m.role === "tool") + expect(toolIndex).toBeGreaterThan(0) + const owner = result.messages[toolIndex - 1] + expect(owner.role).toBe("assistant") + expect(owner.tool_calls?.map((tc) => tc.id)).toContain("toolu_x") + }) + + test("collapses consecutive assistants into one turn — no adjacent same-role messages, text preserved", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.8", + max_tokens: 1024, + messages: [ + { role: "user", content: "Do it." }, + { role: "assistant", content: [{ type: "text", text: "Thinking..." }] }, + { + role: "assistant", + content: [ + { type: "tool_use", id: "toolu_1", name: "run", input: {} }, + ], + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "toolu_1", content: "done" }, + ], + }, + ], + } + + const result = translateToOpenAI(payload) + + // No two adjacent messages share a role (Copilot expects alternation). + for (let i = 1; i < result.messages.length; i++) { + expect(result.messages[i].role).not.toBe(result.messages[i - 1].role) + } + // The preceding assistant text must not be lost in the collapse. + expect(JSON.stringify(result.messages)).toContain("Thinking...") + }) +}) diff --git a/tests/model-resolver.test.ts b/tests/model-resolver.test.ts new file mode 100644 index 000000000..1eaefd82a --- /dev/null +++ b/tests/model-resolver.test.ts @@ -0,0 +1,154 @@ +import { describe, test, expect } from "bun:test" + +import type { ModelsResponse } from "~/services/copilot/get-models" + +import { resolveModelId } from "~/lib/model-resolver" + +function makeModels(ids: Array): ModelsResponse { + return { + object: "list", + data: ids.map((id) => ({ + id, + name: id, + object: "model", + vendor: "copilot", + version: "1", + model_picker_enabled: true, + preview: false, + capabilities: { + family: "gpt", + tokenizer: "o200k_base", + type: "chat", + object: "model_capabilities", + supports: {}, + limits: { + max_context_window_tokens: 128000, + max_prompt_tokens: 128000, + max_output_tokens: 4096, + }, + }, + })), + } +} + +const COPILOT_MODELS = makeModels([ + "claude-opus-4.6", + "claude-opus-4.7", + "claude-opus-4.8", + "claude-sonnet-4.6", + "claude-sonnet-4.5", + "claude-opus-4.5", + "claude-haiku-4.5", + "gemini-3.1-pro-preview", + "gemini-2.5-pro", + "gpt-5.4", + "gpt-4.1", + "gpt-4.1-2025-04-14", + "gpt-4-0125-preview", + "gpt-4o-2024-11-20", + "gpt-4", +]) + +describe("resolveModelId — exact match", () => { + test("returns the same id when it exactly matches an available model", () => { + expect(resolveModelId("claude-opus-4.8", COPILOT_MODELS)).toBe( + "claude-opus-4.8", + ) + }) + + test("prefers an exact (hyphenated) match over canonical rewriting", () => { + // gpt-4-0125-preview legitimately uses hyphens — must not be altered + expect(resolveModelId("gpt-4-0125-preview", COPILOT_MODELS)).toBe( + "gpt-4-0125-preview", + ) + }) +}) + +describe("resolveModelId — hyphen/dot normalization", () => { + test("resolves claude-opus-4-8 to claude-opus-4.8", () => { + expect(resolveModelId("claude-opus-4-8", COPILOT_MODELS)).toBe( + "claude-opus-4.8", + ) + }) + + test("resolves claude-sonnet-4-6 to claude-sonnet-4.6", () => { + expect(resolveModelId("claude-sonnet-4-6", COPILOT_MODELS)).toBe( + "claude-sonnet-4.6", + ) + }) + + test("resolves gpt-5-4 to gpt-5.4", () => { + expect(resolveModelId("gpt-5-4", COPILOT_MODELS)).toBe("gpt-5.4") + }) + + test("resolves gemini-3-1-pro-preview to gemini-3.1-pro-preview", () => { + expect(resolveModelId("gemini-3-1-pro-preview", COPILOT_MODELS)).toBe( + "gemini-3.1-pro-preview", + ) + }) + + test("resolves gpt-4-1 to gpt-4.1 (without colliding with gpt-4.1-2025-04-14)", () => { + expect(resolveModelId("gpt-4-1", COPILOT_MODELS)).toBe("gpt-4.1") + }) + + test("is case-insensitive when matching", () => { + expect(resolveModelId("Claude-Opus-4-8", COPILOT_MODELS)).toBe( + "claude-opus-4.8", + ) + }) +}) + +describe("resolveModelId — Anthropic date-stamp stripping", () => { + test("resolves claude-haiku-4-5-20251001 to claude-haiku-4.5", () => { + expect(resolveModelId("claude-haiku-4-5-20251001", COPILOT_MODELS)).toBe( + "claude-haiku-4.5", + ) + }) + + test("resolves claude-sonnet-4-5-20250929 to claude-sonnet-4.5", () => { + expect(resolveModelId("claude-sonnet-4-5-20250929", COPILOT_MODELS)).toBe( + "claude-sonnet-4.5", + ) + }) + + test("resolves a dotted+dated id (claude-haiku-4.5-20251001)", () => { + expect(resolveModelId("claude-haiku-4.5-20251001", COPILOT_MODELS)).toBe( + "claude-haiku-4.5", + ) + }) + + test("is case-insensitive with a date stamp", () => { + expect(resolveModelId("Claude-Haiku-4-5-20251001", COPILOT_MODELS)).toBe( + "claude-haiku-4.5", + ) + }) + + test("does not strip Copilot's own hyphen-dated ids (gpt-4.1-2025-04-14)", () => { + // Ends in two digits, not eight — exact match must win, untouched. + expect(resolveModelId("gpt-4.1-2025-04-14", COPILOT_MODELS)).toBe( + "gpt-4.1-2025-04-14", + ) + }) + + test("returns original when stripping still finds no match", () => { + expect(resolveModelId("claude-haiku-9-9-20251001", COPILOT_MODELS)).toBe( + "claude-haiku-9-9-20251001", + ) + }) +}) + +describe("resolveModelId — no match", () => { + test("returns the original id when no model matches", () => { + expect(resolveModelId("claude-opus-9-9", COPILOT_MODELS)).toBe( + "claude-opus-9-9", + ) + }) + + test("returns the original id when models is undefined", () => { + expect(resolveModelId("claude-opus-4-8", undefined)).toBe("claude-opus-4-8") + }) + + test("returns the original id for empty/whitespace input", () => { + expect(resolveModelId("", COPILOT_MODELS)).toBe("") + }) +}) diff --git a/tests/model-selector.test.ts b/tests/model-selector.test.ts new file mode 100644 index 000000000..b04cb1aed --- /dev/null +++ b/tests/model-selector.test.ts @@ -0,0 +1,118 @@ +import { describe, test, expect } from "bun:test" + +import type { ModelsResponse } from "~/services/copilot/get-models" + +import { selectModelForTokenCount } from "~/lib/model-selector" + +function makeModels( + list: Array<{ + id: string + max_context_window_tokens?: number + max_prompt_tokens?: number + }>, +): ModelsResponse { + return { + object: "list", + data: list.map((m) => ({ + id: m.id, + name: m.id, + object: "model", + vendor: "copilot", + version: "1", + model_picker_enabled: true, + preview: false, + capabilities: { + family: "gpt", + tokenizer: "o200k_base", + type: "chat", + object: "model_capabilities", + supports: {}, + limits: { + max_context_window_tokens: m.max_context_window_tokens, + max_prompt_tokens: m.max_prompt_tokens, + max_output_tokens: 4096, + }, + }, + })), + } +} + +describe("selectModelForTokenCount — no overflow", () => { + test("returns switched: false when tokens within limit", () => { + const models = makeModels([ + { id: "gpt-4o", max_context_window_tokens: 128000 }, + ]) + const result = selectModelForTokenCount("gpt-4o", models, 1000) + expect(result.switched).toBe(false) + expect(result.model).toBe("gpt-4o") + }) + + test("returns switched: false when tokens exactly equal limit", () => { + const models = makeModels([ + { id: "gpt-4o", max_context_window_tokens: 128000 }, + ]) + const result = selectModelForTokenCount("gpt-4o", models, 128000) + expect(result.switched).toBe(false) + }) +}) + +describe("selectModelForTokenCount — overflow detected", () => { + test("switches to largest context model when overflow", () => { + const models = makeModels([ + { id: "small-model", max_context_window_tokens: 12288 }, + { id: "large-model", max_context_window_tokens: 128000 }, + ]) + const result = selectModelForTokenCount("small-model", models, 55000) + expect(result.switched).toBe(true) + expect(result.model).toBe("large-model") + }) + + test("reason string contains prompt size, requested model, new model", () => { + const models = makeModels([ + { id: "small-model", max_context_window_tokens: 12288 }, + { id: "large-model", max_context_window_tokens: 128000 }, + ]) + const result = selectModelForTokenCount("small-model", models, 55000) + expect(result.reason).toContain("55000") + expect(result.reason).toContain("small-model") + expect(result.reason).toContain("large-model") + }) + + test("returns switched: false when already on largest context model", () => { + const models = makeModels([ + { id: "small-model", max_context_window_tokens: 12288 }, + { id: "large-model", max_context_window_tokens: 128000 }, + ]) + const result = selectModelForTokenCount("large-model", models, 130000) + expect(result.switched).toBe(false) + expect(result.model).toBe("large-model") + expect(result.reason).toBeUndefined() + }) +}) + +describe("selectModelForTokenCount — missing data", () => { + test("returns switched: false when requested model not in list", () => { + const models = makeModels([ + { id: "other-model", max_context_window_tokens: 128000 }, + ]) + const result = selectModelForTokenCount("unknown-model", models, 999999) + expect(result.switched).toBe(false) + expect(result.model).toBe("unknown-model") + }) + + test("returns switched: false when model has no context window data", () => { + const models = makeModels([{ id: "mystery-model" }]) + const result = selectModelForTokenCount("mystery-model", models, 999999) + expect(result.switched).toBe(false) + }) + + test("falls back to max_prompt_tokens when max_context_window_tokens absent", () => { + const models = makeModels([ + { id: "small-model", max_prompt_tokens: 4096 }, + { id: "large-model", max_context_window_tokens: 128000 }, + ]) + const result = selectModelForTokenCount("small-model", models, 10000) + expect(result.switched).toBe(true) + expect(result.model).toBe("large-model") + }) +}) diff --git a/tests/orphaned-tool-result.test.ts b/tests/orphaned-tool-result.test.ts new file mode 100644 index 000000000..6096b4cc8 --- /dev/null +++ b/tests/orphaned-tool-result.test.ts @@ -0,0 +1,193 @@ +import { describe, test, expect } from "bun:test" + +import type { AnthropicMessagesPayload } from "~/routes/messages/anthropic-types" +import type { Message } from "~/services/copilot/create-chat-completions" + +import { translateToOpenAI } from "~/routes/messages/non-stream-translation" + +/** + * Regression tests for orphaned tool_result blocks on the Anthropic → + * Chat Completions path (used by Claude models). + * + * Copilot's backend re-converts our OpenAI payload back to Anthropic for + * Claude models and strictly validates that every tool_result references a + * tool_use in the immediately-preceding assistant message: + * + * "messages.N.content.0: unexpected tool_use_id found in tool_result blocks: + * toolu_xxx. Each tool_result block must have a corresponding tool_use + * block in the previous message." (code: invalid_request_body) + * + * Orphaned tool_result blocks arise when conversation history is edited — + * context compaction drops an assistant tool_use turn, a parallel/interrupted + * tool call is cancelled, etc. The translation must not forward a `tool` + * message whose tool_call_id has no matching assistant tool_calls entry. + */ + +function toolCallIds(messages: Array): Array { + return messages + .filter((m) => m.role === "assistant" && m.tool_calls) + .flatMap((m) => m.tool_calls ?? []) + .map((tc) => tc.id) +} + +function toolResultIds(messages: Array): Array { + return messages + .filter((m) => m.role === "tool") + .map((m) => m.tool_call_id) + .filter((id): id is string => id !== undefined) +} + +describe("orphaned tool_result repair (Chat Completions path)", () => { + test("drops a tool_result whose tool_use was removed by compaction", () => { + // messages[0] user, messages[1] assistant (plain text — NO tool_use), + // messages[2] user with an orphaned tool_result. This is exactly the + // shape Copilot rejected: messages.2.content.0 references a tool_use_id + // that has no corresponding tool_use in messages[1]. + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.8", + max_tokens: 1024, + messages: [ + { role: "user", content: "Run the build." }, + { + role: "assistant", + content: [{ type: "text", text: "Okay, here is the summary." }], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "toolu_01XqGuK9chsSnyoedsgjS4Xf", + content: "build output", + }, + { type: "text", text: "Continue please." }, + ], + }, + ], + } + + const result = translateToOpenAI(payload) + + // No tool message may reference an id that no assistant turn produced. + const callIds = new Set(toolCallIds(result.messages)) + for (const resultId of toolResultIds(result.messages)) { + expect(callIds.has(resultId)).toBe(true) + } + // The orphaned id specifically must be gone. + expect(toolResultIds(result.messages)).not.toContain( + "toolu_01XqGuK9chsSnyoedsgjS4Xf", + ) + // The accompanying user text must survive (not silently dropped). + const serialized = JSON.stringify(result.messages) + expect(serialized).toContain("Continue please.") + }) + + test("keeps a properly paired tool_use → tool_result intact", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.8", + max_tokens: 1024, + messages: [ + { role: "user", content: "What is the weather?" }, + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "toolu_weather_1", + name: "get_weather", + input: { city: "London" }, + }, + ], + }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "toolu_weather_1", + content: "15C", + }, + ], + }, + ], + } + + const result = translateToOpenAI(payload) + + const callIds = new Set(toolCallIds(result.messages)) + expect(callIds.has("toolu_weather_1")).toBe(true) + expect(toolResultIds(result.messages)).toContain("toolu_weather_1") + }) + + test("drops only the orphan when paired and orphaned results are mixed", () => { + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.8", + max_tokens: 1024, + messages: [ + { role: "user", content: "Do two things." }, + { + role: "assistant", + content: [ + { + type: "tool_use", + id: "toolu_keep", + name: "do_thing", + input: {}, + }, + ], + }, + { + role: "user", + content: [ + { type: "tool_result", tool_use_id: "toolu_keep", content: "ok" }, + { + type: "tool_result", + tool_use_id: "toolu_orphan", + content: "stale", + }, + ], + }, + ], + } + + const result = translateToOpenAI(payload) + + const ids = toolResultIds(result.messages) + expect(ids).toContain("toolu_keep") + expect(ids).not.toContain("toolu_orphan") + }) + + test("produces no consecutive same-role messages after converting an orphan", () => { + // An orphan converted to a `user` role must not leave two adjacent user + // messages — Copilot expects strictly alternating roles. Repair runs + // before the consecutive-role merge so the converted message coalesces. + const payload: AnthropicMessagesPayload = { + model: "claude-opus-4.8", + max_tokens: 1024, + messages: [ + { role: "user", content: "Start." }, + { role: "assistant", content: [{ type: "text", text: "Summary." }] }, + { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "toolu_gone", + content: "leftover", + }, + ], + }, + { role: "user", content: "Next instruction." }, + ], + } + + const result = translateToOpenAI(payload) + + // No two consecutive messages share a role. + for (let i = 1; i < result.messages.length; i++) { + expect(result.messages[i].role).not.toBe(result.messages[i - 1].role) + } + // No tool message survives without a matching tool_use. + expect(toolResultIds(result.messages)).toHaveLength(0) + }) +}) diff --git a/tests/responses-context-window.test.ts b/tests/responses-context-window.test.ts new file mode 100644 index 000000000..5f14ccf08 --- /dev/null +++ b/tests/responses-context-window.test.ts @@ -0,0 +1,283 @@ +import { describe, test, expect, mock, beforeEach } from "bun:test" + +import { state } from "~/lib/state" +import { responsesRoutes } from "~/routes/responses/route" + +// Minimal state so handleResponses reaches the upstream fetch. +// `gpt-5.5` routes to the native /responses path via its `gpt-5` prefix, so no +// model catalog is needed (resolveModelId returns the id unchanged). +state.copilotToken = "test-token" +state.vsCodeVersion = "1.0.0" +state.accountType = "individual" +state.models = undefined +state.manualApprove = false +state.rateLimitSeconds = undefined +state.rateLimitWait = false + +const fetchMock = mock((): Promise => Promise.resolve()) + +// @ts-expect-error - Mock fetch doesn't implement all fetch properties +;(globalThis as unknown as { fetch: typeof fetch }).fetch = fetchMock + +beforeEach(() => { + fetchMock.mockReset() +}) + +/** A real context-window rejection with token-limit language. */ +function upstreamContext413(): Response { + return new Response( + JSON.stringify({ + error: { + message: "prompt token count of 940000 exceeds the limit of 922000", + }, + }), + { + status: 413, + headers: { "content-type": "application/json" }, + }, + ) +} + +/** Copilot's opaque parser/body-size rejection. */ +function upstreamOpaque413(): Response { + return new Response( + JSON.stringify({ error: { message: "failed to parse request" } }), + { + status: 413, + headers: { "content-type": "application/json" }, + }, + ) +} + +async function postResponses(body: unknown): Promise { + return responsesRoutes.request("/", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }) +} + +describe("Responses route — context-window 413 → Codex compaction signal", () => { + test("streaming: returns 200 SSE with a response.failed / context_length_exceeded event", async () => { + fetchMock.mockResolvedValue(upstreamContext413()) + + const res = await postResponses({ + model: "gpt-5.5", + stream: true, + input: [{ role: "user", content: "hello" }], + }) + + // MUST be 200 — Codex's transport discards the body of any non-2xx + // response before its SSE parser runs, so a 4xx can never compact. + expect(res.status).toBe(200) + expect(res.headers.get("content-type")).toContain("text/event-stream") + + const text = await res.text() + expect(text).toContain("event: response.failed") + expect(text).toContain('"code":"context_length_exceeded"') + expect(text).toContain("940000 exceeds the limit of 922000") + // The old fabricated token figures must not appear. + expect(text).not.toContain("1402500") + }) + + test("non-streaming: returns OpenAI-shaped 400 with the context_length_exceeded code", async () => { + fetchMock.mockResolvedValue(upstreamContext413()) + + const res = await postResponses({ + model: "gpt-5.5", + stream: false, + input: [{ role: "user", content: "hello" }], + }) + + expect(res.status).toBe(400) + const body = (await res.json()) as { + error: { code: string; type: string } + } + expect(body.error.code).toBe("context_length_exceeded") + expect(body.error.type).toBe("invalid_request_error") + }) + + test("opaque Copilot 413 parser failures are not misclassified as context-window", async () => { + fetchMock.mockResolvedValue(upstreamOpaque413()) + + const res = await postResponses({ + model: "gpt-5.5", + stream: true, + input: [{ role: "user", content: "hello" }], + }) + + expect(res.status).toBe(413) + const text = await res.text() + expect(text).toContain("failed to parse request") + expect(text).not.toContain("context_length_exceeded") + }) + + test("function_call_output images enable Copilot vision headers", async () => { + fetchMock.mockResolvedValue( + new Response( + JSON.stringify({ id: "resp_1", object: "response", output: [] }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ) + + await postResponses({ + model: "gpt-5.5", + stream: false, + input: [ + { + type: "function_call_output", + call_id: "call_1", + output: [ + { type: "input_text", text: "Screenshot:" }, + { + type: "input_image", + image_url: "data:image/png;base64,AAAA", + }, + ], + }, + ], + }) + + const [, init] = fetchMock.mock.calls[0] as unknown as [ + string, + { headers: Record }, + ] + expect(init.headers["copilot-vision-request"]).toBe("true") + }) + + test("opaque Copilot 413 with images retries once with image placeholders", async () => { + fetchMock + .mockResolvedValueOnce(upstreamOpaque413()) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ id: "resp_1", object: "response", output: [] }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ) + + const res = await postResponses({ + model: "gpt-5.5", + stream: false, + input: [ + { + type: "function_call_output", + call_id: "call_1", + output: [ + { type: "input_text", text: "Screenshot:" }, + { + type: "input_image", + image_url: "data:image/png;base64,AAAA", + }, + ], + }, + ], + }) + + expect(res.status).toBe(200) + expect(fetchMock).toHaveBeenCalledTimes(2) + + const firstCall = fetchMock.mock.calls[0] as unknown as [ + string, + { body: string; headers: Record }, + ] + const retryCall = fetchMock.mock.calls[1] as unknown as [ + string, + { body: string; headers: Record }, + ] + expect(firstCall[1].headers["copilot-vision-request"]).toBe("true") + expect(firstCall[1].body).toContain('"type":"input_image"') + expect(retryCall[1].body).not.toContain('"type":"input_image"') + expect(retryCall[1].body).not.toContain("data:image/png") + expect(retryCall[1].body).toContain( + "Image removed by proxy after Copilot rejected the request body", + ) + }) + + test("after one image parser rejection, later requests in the same window strip before upstream", async () => { + const input = [ + { + type: "function_call_output", + call_id: "call_1", + output: [ + { type: "input_text", text: "Screenshot:" }, + { + type: "input_image", + image_url: "data:image/png;base64,BBBB", + }, + ], + }, + ] + const metadata = { + session_id: "session-image-cache-test", + "x-codex-window-id": "session-image-cache-test:0", + } + + fetchMock + .mockResolvedValueOnce(upstreamOpaque413()) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ id: "resp_1", object: "response", output: [] }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ) + + await postResponses({ + model: "gpt-5.5", + stream: false, + metadata, + input, + }) + expect(fetchMock).toHaveBeenCalledTimes(2) + + fetchMock.mockReset() + fetchMock.mockResolvedValueOnce( + new Response( + JSON.stringify({ id: "resp_2", object: "response", output: [] }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ) + + const res = await postResponses({ + model: "gpt-5.5", + stream: false, + metadata, + input, + }) + + expect(res.status).toBe(200) + expect(fetchMock).toHaveBeenCalledTimes(1) + const [, init] = fetchMock.mock.calls[0] as unknown as [ + string, + { body: string; headers: Record }, + ] + expect(init.body).not.toContain('"type":"input_image"') + expect(init.body).not.toContain("data:image/png") + expect(init.headers["copilot-vision-request"]).toBeUndefined() + }) + + test("unrelated upstream errors are NOT misclassified as context-window", async () => { + fetchMock.mockResolvedValue( + new Response( + JSON.stringify({ error: { message: "internal server error" } }), + { + status: 500, + headers: { "content-type": "application/json" }, + }, + ), + ) + + const res = await postResponses({ + model: "gpt-5.5", + stream: true, + input: [{ role: "user", content: "hello" }], + }) + + // Falls through to normal error forwarding (500), no compaction signal. + expect(res.status).toBe(500) + const text = await res.text() + expect(text).not.toContain("context_length_exceeded") + // The original upstream body must survive detection (clone, not consume), + // so forwardError can still read and forward it. + expect(text).toContain("internal server error") + }) +}) diff --git a/tests/responses-usage-passthrough.test.ts b/tests/responses-usage-passthrough.test.ts new file mode 100644 index 000000000..876c515da --- /dev/null +++ b/tests/responses-usage-passthrough.test.ts @@ -0,0 +1,90 @@ +import { describe, test, expect, mock, beforeEach } from "bun:test" + +import { state } from "~/lib/state" +import { responsesRoutes } from "~/routes/responses/route" + +// Minimal state so handleResponses reaches the upstream fetch. +// `gpt-5.5` routes to the native /responses path via its `gpt-5` prefix. +state.copilotToken = "test-token" +state.vsCodeVersion = "1.0.0" +state.accountType = "individual" +state.models = undefined +state.manualApprove = false +state.rateLimitSeconds = undefined +state.rateLimitWait = false + +const fetchMock = mock((): Promise => Promise.resolve()) + +// @ts-expect-error - Mock fetch doesn't implement all fetch properties +;(globalThis as unknown as { fetch: typeof fetch }).fetch = fetchMock + +beforeEach(() => { + fetchMock.mockReset() +}) + +/** A minimal successful non-streaming /responses body. */ +function upstreamOk(): Response { + return new Response( + JSON.stringify({ id: "resp_1", object: "response", output: [] }), + { status: 200, headers: { "content-type": "application/json" } }, + ) +} + +/** Parses the JSON body the handler forwarded to the upstream fetch. */ +function forwardedBody(): Record { + const call = fetchMock.mock.calls[0] as unknown as [string, { body: string }] + return JSON.parse(call[1].body) as Record +} + +async function postResponses(body: unknown): Promise { + return responsesRoutes.request("/", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }) +} + +describe("Responses route — verbatim passthrough (Codex owns compaction)", () => { + test("a large item is forwarded byte-for-byte (no per-item trim)", async () => { + fetchMock.mockResolvedValue(upstreamOk()) + + // ~300KB single item: far above the old 40KB per-item cap. It MUST be + // forwarded untouched so Copilot's reported token usage reflects the true + // context size and Codex can decide to auto-compact on its own. + const big = "word ".repeat(60_000) // ~300KB + await postResponses({ + model: "gpt-5.5", + stream: false, + input: [{ role: "user", content: big }], + }) + + const sent = forwardedBody() + const input = sent.input as Array<{ content: string }> + expect(input[0].content).toBe(big) + expect(input[0].content).not.toContain("[...truncated...]") + expect(input[0].content).not.toContain("[removed]") + }) + + test("even a multi-megabyte body is forwarded verbatim — never trimmed", async () => { + fetchMock.mockResolvedValue(upstreamOk()) + + // Two ~3MB items → ~6MB total. The proxy must NOT mutate the body: Codex + // owns its conversation state and compacts itself once usage crosses the + // configured limit. Trimming here would desync Codex's token accounting + // from what Copilot received and could silently drop conversation data. + const huge = "x".repeat(3_000_000) + const sentInput = [ + { role: "user", content: huge }, + { role: "user", content: huge }, + ] + await postResponses({ model: "gpt-5.5", stream: false, input: sentInput }) + + const sent = forwardedBody() + const input = sent.input as Array<{ content: string }> + // Both items survive intact — nothing trimmed, nothing removed. + expect(input).toHaveLength(2) + expect(input[0].content).toBe(huge) + expect(input[1].content).toBe(huge) + expect(JSON.stringify(sent).length).toBeGreaterThan(6_000_000) + }) +}) diff --git a/tests/retriable-fetch-error.test.ts b/tests/retriable-fetch-error.test.ts new file mode 100644 index 000000000..59d5c5aeb --- /dev/null +++ b/tests/retriable-fetch-error.test.ts @@ -0,0 +1,33 @@ +import { describe, test, expect } from "bun:test" + +import { isRetriableFetchError } from "~/routes/messages/handler" + +describe("isRetriableFetchError", () => { + test("treats a socket reset as retriable when ECONNRESET is on error.code", () => { + // Bun/undici raise ECONNRESET with name "Error" and the marker on `.code`, + // not `.name`. This is the real-world shape that previously fell through + // to a hard 500 instead of being retried. + const error = Object.assign(new Error("The socket connection was closed"), { + code: "ECONNRESET", + }) + + expect(isRetriableFetchError(error)).toBe(true) + }) + + test("treats an inactivity TimeoutError (marker on error.name) as retriable", () => { + const error = new Error("Upstream connection inactive for 300s") + error.name = "TimeoutError" + + expect(isRetriableFetchError(error)).toBe(true) + }) + + test("does not retry an unrelated error", () => { + const error = Object.assign(new Error("boom"), { code: "ERR_UNKNOWN" }) + + expect(isRetriableFetchError(error)).toBe(false) + }) + + test("does not retry a non-Error value", () => { + expect(isRetriableFetchError("nope")).toBe(false) + }) +}) diff --git a/tests/web-search-extra.test.ts b/tests/web-search-extra.test.ts new file mode 100644 index 000000000..8ff3d5080 --- /dev/null +++ b/tests/web-search-extra.test.ts @@ -0,0 +1,207 @@ +import { describe, test, expect, spyOn, afterEach, mock } from "bun:test" + +import type { + ChatCompletionsPayload, + ChatCompletionResponse, +} from "~/services/copilot/create-chat-completions" + +import * as stateModule from "~/lib/state" +import * as createChatCompletionsModule from "~/services/copilot/create-chat-completions" +import { webSearchInterceptor } from "~/services/web-search/interceptor" +import { appendWebSearchInstruction } from "~/services/web-search/system-prompt" + +function makeCopilotResponse( + finishReason: "stop" | "tool_calls", + toolCalls?: Array<{ id: string; name: string; arguments: string }>, +): ChatCompletionResponse { + return { + id: "resp-1", + object: "chat.completion", + created: 0, + model: "gpt-4o", + choices: [ + { + index: 0, + logprobs: null, + finish_reason: finishReason, + message: { + role: "assistant", + content: finishReason === "stop" ? "Here is my answer." : null, + tool_calls: toolCalls?.map((tc) => ({ + id: tc.id, + type: "function" as const, + function: { name: tc.name, arguments: tc.arguments }, + })), + }, + }, + ], + } +} + +function makePayload(stream = false): ChatCompletionsPayload { + return { + model: "gpt-4o", + stream, + messages: [{ role: "user", content: "What is the weather today?" }], + tools: [ + { + type: "function", + function: { + name: "web_search", + description: "Search the web", + parameters: { + type: "object", + properties: { query: { type: "string" } }, + required: ["query"], + }, + }, + }, + ], + } +} + +describe("webSearchInterceptor — streaming preservation", () => { + afterEach(() => { + mock.restore() + }) + + test("re-issues with stream:true when Copilot returns stop (no tool call)", async () => { + const stopResponse = makeCopilotResponse("stop") + const streamingPayload: ChatCompletionsPayload = makePayload(true) + const createSpy = spyOn( + createChatCompletionsModule, + "createChatCompletions", + ) + .mockResolvedValueOnce(stopResponse) // first pass: non-streaming inspection + .mockResolvedValueOnce(stopResponse) // streaming re-issue: no tool call, returned to caller + + await webSearchInterceptor(streamingPayload) + + // Interceptor must have made exactly 2 calls + expect(createSpy).toHaveBeenCalledTimes(2) + + // Second call must NOT include the web_search tool + const secondCallArg = createSpy.mock.calls[1][0] + expect(secondCallArg.stream).toBe(true) + expect( + secondCallArg.tools?.some((t) => t.function.name === "web_search"), + ).toBe(false) + }) + + test("does NOT re-issue when stream is false and Copilot returns stop", async () => { + const stopResponse = makeCopilotResponse("stop") + const nonStreamingPayload: ChatCompletionsPayload = makePayload(false) + const createSpy = spyOn( + createChatCompletionsModule, + "createChatCompletions", + ).mockResolvedValue(stopResponse) + + await webSearchInterceptor(nonStreamingPayload) + + // Only 1 call — non-streaming first pass returned directly, no re-issue + expect(createSpy).toHaveBeenCalledTimes(1) + }) + + test("re-issues with stream:true when Copilot calls a different tool (not web_search)", async () => { + const otherToolResponse = makeCopilotResponse("tool_calls", [ + { id: "tc-1", name: "bash", arguments: '{"command":"ls"}' }, + ]) + const streamingPayload: ChatCompletionsPayload = makePayload(true) + const createSpy = spyOn( + createChatCompletionsModule, + "createChatCompletions", + ) + .mockResolvedValueOnce(otherToolResponse) + .mockResolvedValueOnce(otherToolResponse) + + await webSearchInterceptor(streamingPayload) + + expect(createSpy).toHaveBeenCalledTimes(2) + const secondCallArg = createSpy.mock.calls[1][0] + expect(secondCallArg.stream).toBe(true) + expect( + secondCallArg.tools?.some((t) => t.function.name === "web_search"), + ).toBe(false) + }) +}) + +describe("isWebSearchEnabled — Tavily", () => { + test("returns false when neither key is set", () => { + const originalBrave = stateModule.state.braveApiKey + const originalTavily = stateModule.state.tavilyApiKey + stateModule.state.braveApiKey = undefined + stateModule.state.tavilyApiKey = undefined + try { + expect(stateModule.isWebSearchEnabled()).toBe(false) + } finally { + stateModule.state.braveApiKey = originalBrave + stateModule.state.tavilyApiKey = originalTavily + } + }) + + test("returns true when tavilyApiKey is set", () => { + const originalBrave = stateModule.state.braveApiKey + const originalTavily = stateModule.state.tavilyApiKey + stateModule.state.braveApiKey = undefined + stateModule.state.tavilyApiKey = "test-key" + try { + expect(stateModule.isWebSearchEnabled()).toBe(true) + } finally { + stateModule.state.braveApiKey = originalBrave + stateModule.state.tavilyApiKey = originalTavily + } + }) + + test("returns true when both keys are set", () => { + const originalBrave = stateModule.state.braveApiKey + const originalTavily = stateModule.state.tavilyApiKey + stateModule.state.braveApiKey = "brave-key" + stateModule.state.tavilyApiKey = "tavily-key" + try { + expect(stateModule.isWebSearchEnabled()).toBe(true) + } finally { + stateModule.state.braveApiKey = originalBrave + stateModule.state.tavilyApiKey = originalTavily + } + }) +}) + +describe("appendWebSearchInstruction", () => { + test("appends instruction to string system prompt", () => { + const result = appendWebSearchInstruction("You are a helpful assistant.") + expect(typeof result).toBe("string") + expect(result as string).toContain("You are a helpful assistant.") + expect(result as string).toContain("web_search") + }) + + test("appends instruction to last text block in array system prompt", () => { + const system = [ + { type: "text" as const, text: "First block." }, + { type: "text" as const, text: "Second block." }, + ] + const result = appendWebSearchInstruction(system) + expect(Array.isArray(result)).toBe(true) + const blocks = result as typeof system + expect(blocks[0].text).toBe("First block.") + expect(blocks[1].text).toContain("Second block.") + expect(blocks[1].text).toContain("web_search") + }) + + test("adds new text block when array has no text blocks", () => { + const system = [{ type: "tool_result" as const, text: "some tool result" }] + const result = appendWebSearchInstruction( + system as unknown as Parameters[0], + ) + expect(Array.isArray(result)).toBe(true) + const blocks = result as Array<{ type: string; text: string }> + expect(blocks).toHaveLength(2) + expect(blocks[1].type).toBe("text") + expect(blocks[1].text).toContain("web_search") + }) + + test("returns instruction string when system is undefined", () => { + const result = appendWebSearchInstruction(undefined) + expect(typeof result).toBe("string") + expect(result as string).toContain("web_search") + }) +}) diff --git a/tests/web-search.test.ts b/tests/web-search.test.ts new file mode 100644 index 000000000..68888d3c6 --- /dev/null +++ b/tests/web-search.test.ts @@ -0,0 +1,787 @@ +import { + describe, + test, + expect, + spyOn, + beforeEach, + afterEach, + mock, +} from "bun:test" + +import type { + ChatCompletionsPayload, + ChatCompletionResponse, +} from "~/services/copilot/create-chat-completions" + +import * as stateModule from "~/lib/state" +import { state } from "~/lib/state" +import { + isTypedTool, + type AnthropicTool, +} from "~/routes/messages/anthropic-types" +import * as createChatCompletionsModule from "~/services/copilot/create-chat-completions" +import * as braveModule from "~/services/web-search/brave" +import { + webSearchInterceptor, + prepareWebSearchPayload, +} from "~/services/web-search/interceptor" +import * as tavilyModule from "~/services/web-search/tavily" +import { + WEB_SEARCH_TOOL_NAMES, + WEB_SEARCH_FUNCTION_TOOL, +} from "~/services/web-search/tool-definition" +import { BraveSearchError, WebSearchError } from "~/services/web-search/types" + +describe("WEB_SEARCH_TOOL_NAMES", () => { + test("contains web_search", () => { + expect(WEB_SEARCH_TOOL_NAMES.has("web_search")).toBe(true) + }) + + test("contains internet_research", () => { + expect(WEB_SEARCH_TOOL_NAMES.has("internet_research")).toBe(true) + }) + + test("does NOT contain search (too generic)", () => { + expect(WEB_SEARCH_TOOL_NAMES.has("search")).toBe(false) + }) +}) + +describe("Typed tool detection guard", () => { + test("typed tool named web_search — no input_schema — matches", () => { + const tool: AnthropicTool = { + type: "web_search_20260101", + name: "web_search", + } + expect(isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name)).toBe(true) + }) + + test("typed tool named internet_research matches", () => { + const tool: AnthropicTool = { + type: "internet_research_20260101", + name: "internet_research", + } + expect(isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name)).toBe(true) + }) + + test("future versioned type — detected by name, not type string (different version)", () => { + const tool: AnthropicTool = { + type: "web_search_20260601", + name: "web_search", + } + expect(isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name)).toBe(true) + }) + + test("custom tool named web_search WITH input_schema — NOT matched", () => { + const tool: AnthropicTool = { + name: "web_search", + input_schema: { type: "object", properties: {} }, + } + expect(isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name)).toBe( + false, + ) + }) + + test("custom tool named search WITH input_schema — NOT matched", () => { + const tool: AnthropicTool = { + name: "search", + input_schema: { type: "object", properties: {} }, + } + expect(isTypedTool(tool) && WEB_SEARCH_TOOL_NAMES.has(tool.name)).toBe( + false, + ) + }) +}) + +describe("WEB_SEARCH_FUNCTION_TOOL", () => { + test("type is function", () => { + expect(WEB_SEARCH_FUNCTION_TOOL.type).toBe("function") + }) + + test("function name is web_search", () => { + expect(WEB_SEARCH_FUNCTION_TOOL.function.name).toBe("web_search") + }) + + test("has parameters with query property", () => { + const params = WEB_SEARCH_FUNCTION_TOOL.function.parameters as { + properties: { query: unknown } + required: Array + } + expect(params.properties.query).toBeDefined() + expect(params.required).toContain("query") + }) +}) + +describe("searchBrave — result formatting", () => { + afterEach(() => { + mock.restore() + }) + + test("formats top 5 results as Array", async () => { + const mockResponse = { + web: { + results: [ + { + title: "Result 1", + url: "https://example.com/1", + description: "Desc 1", + }, + { + title: "Result 2", + url: "https://example.com/2", + description: "Desc 2", + }, + { + title: "Result 3", + url: "https://example.com/3", + description: "Desc 3", + }, + { + title: "Result 4", + url: "https://example.com/4", + description: "Desc 4", + }, + { + title: "Result 5", + url: "https://example.com/5", + description: "Desc 5", + }, + ], + }, + } + spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify(mockResponse), { status: 200 }), + ) + + const results = await braveModule.searchBrave("test query", "fake-api-key") + expect(results).toHaveLength(5) + expect(results[0]).toEqual({ + title: "Result 1", + url: "https://example.com/1", + description: "Desc 1", + }) + expect(results[4]?.url).toBe("https://example.com/5") + }) + + test("returns empty array when web.results is empty", async () => { + const mockResponse = { web: { results: [] } } + spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify(mockResponse), { status: 200 }), + ) + + const results = await braveModule.searchBrave( + "nothing here", + "fake-api-key", + ) + expect(results).toHaveLength(0) + }) + + test("returns empty array when web key is absent", async () => { + const mockResponse = {} + spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify(mockResponse), { status: 200 }), + ) + + const results = await braveModule.searchBrave("nothing", "fake-api-key") + expect(results).toHaveLength(0) + }) + + test("uses empty string for missing description field", async () => { + const mockResponse = { + web: { results: [{ title: "T", url: "https://u.com" }] }, + } + spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify(mockResponse), { status: 200 }), + ) + + const results = await braveModule.searchBrave("q", "fake-api-key") + expect(results[0]?.description).toBe("") + }) +}) + +describe("searchBrave — error handling", () => { + afterEach(() => { + mock.restore() + }) + + test("throws BraveSearchError on non-200 response", async () => { + spyOn(globalThis, "fetch").mockResolvedValue( + new Response("Forbidden", { status: 403 }), + ) + + let threw: unknown + try { + await braveModule.searchBrave("query", "bad-key") + } catch (e) { + threw = e + } + expect(threw).toBeInstanceOf(BraveSearchError) + }) + + test("BraveSearchError reason includes status code on non-200", async () => { + spyOn(globalThis, "fetch").mockResolvedValue( + new Response("Forbidden", { status: 403 }), + ) + + let threw: unknown + try { + await braveModule.searchBrave("query", "bad-key") + } catch (e) { + threw = e + } + expect(threw).toBeInstanceOf(BraveSearchError) + expect((threw as InstanceType).reason).toContain( + "403", + ) + }) + + test("throws BraveSearchError on network failure", async () => { + spyOn(globalThis, "fetch").mockRejectedValue(new Error("network error")) + + let threw: unknown + try { + await braveModule.searchBrave("query", "key") + } catch (e) { + threw = e + } + expect(threw).toBeInstanceOf(BraveSearchError) + }) +}) + +// Helper: build a minimal non-streaming ChatCompletionResponse +function makeCopilotResponse( + finishReason: "stop" | "tool_calls", + toolCalls?: Array<{ id: string; name: string; arguments: string }>, +): ChatCompletionResponse { + return { + id: "resp-1", + object: "chat.completion", + created: 0, + model: "gpt-4o", + choices: [ + { + index: 0, + logprobs: null, + finish_reason: finishReason, + message: { + role: "assistant", + content: finishReason === "stop" ? "Here is my answer." : null, + tool_calls: toolCalls?.map((tc) => ({ + id: tc.id, + type: "function" as const, + function: { name: tc.name, arguments: tc.arguments }, + })), + }, + }, + ], + } +} + +function makePayload(stream = false): ChatCompletionsPayload { + return { + model: "gpt-4o", + stream, + messages: [{ role: "user", content: "What is the weather today?" }], + tools: [ + { + type: "function", + function: { + name: "web_search", + description: "Search the web", + parameters: { + type: "object", + properties: { query: { type: "string" } }, + required: ["query"], + }, + }, + }, + ], + } +} + +describe("webSearchInterceptor — no search path", () => { + afterEach(() => { + mock.restore() + }) + + test("returns response as-is when finish_reason is stop", async () => { + const stopResponse = makeCopilotResponse("stop") + const createSpy = spyOn( + createChatCompletionsModule, + "createChatCompletions", + ).mockResolvedValue(stopResponse) + + const result = await webSearchInterceptor(makePayload()) + + expect(result).toEqual(stopResponse) + expect(createSpy).toHaveBeenCalledTimes(1) + }) + + test("returns response as-is when tool_calls is for a different tool", async () => { + const otherToolResponse = makeCopilotResponse("tool_calls", [ + { id: "tc-1", name: "bash", arguments: '{"command":"ls"}' }, + ]) + const createSpy = spyOn( + createChatCompletionsModule, + "createChatCompletions", + ).mockResolvedValue(otherToolResponse) + + const result = await webSearchInterceptor(makePayload()) + + expect(result).toEqual(otherToolResponse) + expect(createSpy).toHaveBeenCalledTimes(1) + }) +}) + +describe("webSearchInterceptor — search path", () => { + beforeEach(() => { + // The interceptor guards on state.braveApiKey before calling searchBrave. + // Set a fake key so the guard passes and the spy is reached. + state.braveApiKey = "test-key" + }) + + afterEach(() => { + state.braveApiKey = undefined + mock.restore() + }) + + test("calls Brave and makes a second Copilot call when web_search is triggered", async () => { + const firstResponse = makeCopilotResponse("tool_calls", [ + { + id: "tc-ws", + name: "web_search", + arguments: '{"query":"latest AI news"}', + }, + ]) + const finalResponse = makeCopilotResponse("stop") + const braveResults = [ + { + title: "AI News", + url: "https://ainews.com", + description: "Latest AI developments", + }, + ] + + const createSpy = spyOn( + createChatCompletionsModule, + "createChatCompletions", + ) + .mockResolvedValueOnce(firstResponse) + .mockResolvedValueOnce(finalResponse) + spyOn(braveModule, "searchBrave").mockResolvedValue(braveResults) + + const result = await webSearchInterceptor(makePayload()) + + expect(createSpy).toHaveBeenCalledTimes(2) + expect(result).toEqual(finalResponse) + expect(createSpy.mock.calls[0]?.[0]?.stream).toBe(false) + }) + + test("second pass uses original stream flag", async () => { + const firstResponse = makeCopilotResponse("tool_calls", [ + { id: "tc-ws", name: "web_search", arguments: '{"query":"news"}' }, + ]) + const finalResponse = makeCopilotResponse("stop") + + const createSpy = spyOn( + createChatCompletionsModule, + "createChatCompletions", + ) + .mockResolvedValueOnce(firstResponse) + .mockResolvedValueOnce(finalResponse) + spyOn(braveModule, "searchBrave").mockResolvedValue([]) + + await webSearchInterceptor(makePayload(true)) + + expect(createSpy).toHaveBeenCalledTimes(2) + expect(createSpy.mock.calls[0]?.[0]?.stream).toBe(false) + expect(createSpy.mock.calls[1]?.[0]?.stream).toBe(true) + }) + + test("injects stub tool results for non-search tool_calls alongside web_search", async () => { + const firstResponse = makeCopilotResponse("tool_calls", [ + { id: "tc-ws", name: "web_search", arguments: '{"query":"q"}' }, + { id: "tc-bash", name: "bash", arguments: '{"command":"ls"}' }, + ]) + const finalResponse = makeCopilotResponse("stop") + + const createSpy = spyOn( + createChatCompletionsModule, + "createChatCompletions", + ) + .mockResolvedValueOnce(firstResponse) + .mockResolvedValueOnce(finalResponse) + spyOn(braveModule, "searchBrave").mockResolvedValue([]) + + await webSearchInterceptor(makePayload()) + + const secondCallMessages = createSpy.mock.calls[1]?.[0]?.messages + const toolMessages = secondCallMessages.filter((m) => m.role === "tool") + expect(toolMessages).toHaveLength(2) + const toolIds = toolMessages.map((m) => m.tool_call_id) + expect(toolIds).toContain("tc-ws") + expect(toolIds).toContain("tc-bash") + }) + + test("injects failure message when Brave throws BraveSearchError", async () => { + const firstResponse = makeCopilotResponse("tool_calls", [ + { id: "tc-ws", name: "web_search", arguments: '{"query":"q"}' }, + ]) + const finalResponse = makeCopilotResponse("stop") + + const createSpy = spyOn( + createChatCompletionsModule, + "createChatCompletions", + ) + .mockResolvedValueOnce(firstResponse) + .mockResolvedValueOnce(finalResponse) + spyOn(braveModule, "searchBrave").mockRejectedValue( + new BraveSearchError("HTTP 429"), + ) + + await webSearchInterceptor(makePayload()) + + expect(createSpy).toHaveBeenCalledTimes(2) + const secondCallMessages = createSpy.mock.calls[1]?.[0]?.messages + const toolMsg = secondCallMessages.find((m) => m.role === "tool") + expect(toolMsg?.content).toContain("Web search failed") + expect(toolMsg?.content).toContain("training data") + }) + + test("injects failure message when query JSON.parse fails", async () => { + const firstResponse = makeCopilotResponse("tool_calls", [ + { id: "tc-ws", name: "web_search", arguments: "INVALID_JSON" }, + ]) + const finalResponse = makeCopilotResponse("stop") + + const createSpy = spyOn( + createChatCompletionsModule, + "createChatCompletions", + ) + .mockResolvedValueOnce(firstResponse) + .mockResolvedValueOnce(finalResponse) + spyOn(braveModule, "searchBrave").mockResolvedValue([]) + + await webSearchInterceptor(makePayload()) + + expect(createSpy).toHaveBeenCalledTimes(2) + const secondCallMessages = createSpy.mock.calls[1]?.[0]?.messages + const toolMsg = secondCallMessages.find((m) => m.role === "tool") + expect(toolMsg?.content).toContain("Web search failed") + }) + + test("second pass sets tool_choice: 'none' to prevent re-invoking web search", async () => { + const firstResponse = makeCopilotResponse("tool_calls", [ + { id: "tc-ws", name: "web_search", arguments: '{"query":"q"}' }, + ]) + const finalResponse = makeCopilotResponse("stop") + + const createSpy = spyOn( + createChatCompletionsModule, + "createChatCompletions", + ) + .mockResolvedValueOnce(firstResponse) + .mockResolvedValueOnce(finalResponse) + spyOn(braveModule, "searchBrave").mockResolvedValue([]) + + // Even if the original payload had a different tool_choice, second pass + // must override with "none" to prevent the model from re-triggering search. + const payloadWithChoice: ChatCompletionsPayload = { + ...makePayload(), + tool_choice: { type: "function", function: { name: "web_search" } }, + } + + await webSearchInterceptor(payloadWithChoice) + + expect(createSpy.mock.calls[1]?.[0]?.tool_choice).toBe("none") + }) +}) + +describe("prepareWebSearchPayload", () => { + test("appends WEB_SEARCH_FUNCTION_TOOL to tools array", () => { + const payload = makePayload() + const prepared = prepareWebSearchPayload(payload) + const names = prepared.tools?.map((t) => t.function.name) ?? [] + expect(names).toContain(WEB_SEARCH_FUNCTION_TOOL.function.name) + }) + + test("does not mutate original payload", () => { + const payload = makePayload() + const originalLength = payload.tools?.length ?? 0 + prepareWebSearchPayload(payload) + expect(payload.tools?.length).toBe(originalLength) + }) + + test("works when payload has no tools", () => { + const payload: ChatCompletionsPayload = { + ...makePayload(), + tools: undefined, + } + const prepared = prepareWebSearchPayload(payload) + expect(prepared.tools).toHaveLength(1) + expect(prepared.tools?.[0]?.function.name).toBe( + WEB_SEARCH_FUNCTION_TOOL.function.name, + ) + }) +}) + +describe("prepareWebSearchPayload — always-on injection", () => { + test("preserves existing custom tools alongside WEB_SEARCH_FUNCTION_TOOL", () => { + const base: ChatCompletionsPayload = { + model: "gpt-4o", + messages: [{ role: "user", content: "hello" }], + tools: [ + { type: "function", function: { name: "my_tool", parameters: {} } }, + ], + } + const prepared = prepareWebSearchPayload(base) + expect(prepared.tools?.some((t) => t.function.name === "my_tool")).toBe( + true, + ) + expect(prepared.tools?.some((t) => t.function.name === "web_search")).toBe( + true, + ) + }) +}) + +describe("isWebSearchEnabled", () => { + test("returns false when braveApiKey is not set", () => { + const originalKey = stateModule.state.braveApiKey + stateModule.state.braveApiKey = undefined + try { + expect(stateModule.isWebSearchEnabled()).toBe(false) + } finally { + stateModule.state.braveApiKey = originalKey + } + }) + + test("returns true when braveApiKey is set", () => { + const originalKey = stateModule.state.braveApiKey + stateModule.state.braveApiKey = "test-key" + try { + expect(stateModule.isWebSearchEnabled()).toBe(true) + } finally { + stateModule.state.braveApiKey = originalKey + } + }) +}) + +describe("searchTavily — result formatting", () => { + afterEach(() => { + mock.restore() + }) + + test("formats results mapping content to description", async () => { + const mockResponse = { + results: [ + { title: "T1", url: "https://t.com/1", content: "C1" }, + { title: "T2", url: "https://t.com/2", content: "C2" }, + ], + } + spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify(mockResponse), { status: 200 }), + ) + + const results = await tavilyModule.searchTavily("test query", "fake-key") + expect(results).toHaveLength(2) + expect(results[0]).toEqual({ + title: "T1", + url: "https://t.com/1", + description: "C1", + }) + expect(results[1]?.url).toBe("https://t.com/2") + }) + + test("returns empty array when results is empty", async () => { + const mockResponse = { results: [] } + spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify(mockResponse), { status: 200 }), + ) + + const results = await tavilyModule.searchTavily("nothing here", "fake-key") + expect(results).toHaveLength(0) + }) + + test("returns empty array when results key is absent", async () => { + const mockResponse = {} + spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify(mockResponse), { status: 200 }), + ) + + const results = await tavilyModule.searchTavily("nothing", "fake-key") + expect(results).toHaveLength(0) + }) + + test("sends Authorization: Bearer header", async () => { + const fetchSpy = spyOn(globalThis, "fetch").mockResolvedValue( + new Response(JSON.stringify({ results: [] }), { status: 200 }), + ) + + await tavilyModule.searchTavily("q", "my-secret-key") + + expect(fetchSpy).toHaveBeenCalled() + // Take the most recent call — it's the one our searchTavily just made + const lastCall = fetchSpy.mock.calls.at(-1) + expect(lastCall).toBeDefined() + const headers = lastCall?.[1]?.headers as Record + expect(headers["Authorization"]).toBe("Bearer my-secret-key") + }) +}) + +describe("searchTavily — error handling", () => { + afterEach(() => { + mock.restore() + }) + + test("throws WebSearchError on non-200 response", async () => { + spyOn(globalThis, "fetch").mockResolvedValue( + new Response("Unauthorized", { status: 401 }), + ) + + let threw: unknown + try { + await tavilyModule.searchTavily("q", "bad-key") + } catch (e) { + threw = e + } + expect(threw).toBeInstanceOf(WebSearchError) + }) + + test("WebSearchError reason includes status code on non-200", async () => { + spyOn(globalThis, "fetch").mockResolvedValue( + new Response("Unauthorized", { status: 401 }), + ) + + let threw: unknown + try { + await tavilyModule.searchTavily("q", "bad-key") + } catch (e) { + threw = e + } + expect(threw).toBeInstanceOf(WebSearchError) + expect((threw as WebSearchError).reason).toContain("401") + }) + + test("throws WebSearchError on network failure", async () => { + spyOn(globalThis, "fetch").mockRejectedValue(new Error("network error")) + + let threw: unknown + try { + await tavilyModule.searchTavily("q", "key") + } catch (e) { + threw = e + } + expect(threw).toBeInstanceOf(WebSearchError) + }) + + test("throws WebSearchError with 'request timed out' reason on AbortError", async () => { + spyOn(globalThis, "fetch").mockRejectedValue( + Object.assign(new Error("aborted"), { name: "AbortError" }), + ) + + let threw: unknown + try { + await tavilyModule.searchTavily("q", "key") + } catch (e) { + threw = e + } + expect(threw).toBeInstanceOf(WebSearchError) + expect((threw as WebSearchError).reason).toBe("request timed out") + }) +}) + +describe("webSearchInterceptor — Tavily search path", () => { + beforeEach(() => { + state.tavilyApiKey = "tavily-test-key" + state.braveApiKey = undefined + }) + + afterEach(() => { + state.tavilyApiKey = undefined + state.braveApiKey = undefined + mock.restore() + }) + + test("calls Tavily and makes a second Copilot call when web_search is triggered", async () => { + const firstResponse = makeCopilotResponse("tool_calls", [ + { + id: "tc-ws", + name: "web_search", + arguments: '{"query":"latest AI news"}', + }, + ]) + const finalResponse = makeCopilotResponse("stop") + const tavilyResults = [ + { + title: "AI News", + url: "https://ainews.com", + description: "Latest AI developments", + }, + ] + + const createSpy = spyOn( + createChatCompletionsModule, + "createChatCompletions", + ) + .mockResolvedValueOnce(firstResponse) + .mockResolvedValueOnce(finalResponse) + spyOn(tavilyModule, "searchTavily").mockResolvedValue(tavilyResults) + + const result = await webSearchInterceptor(makePayload()) + + expect(createSpy).toHaveBeenCalledTimes(2) + expect(result).toEqual(finalResponse) + expect(createSpy.mock.calls[0]?.[0]?.stream).toBe(false) + }) + + test("prefers Tavily over Brave when both keys are set", async () => { + state.tavilyApiKey = "tavily-key" + state.braveApiKey = "brave-key" + + const firstResponse = makeCopilotResponse("tool_calls", [ + { + id: "tc-ws", + name: "web_search", + arguments: '{"query":"latest news"}', + }, + ]) + const finalResponse = makeCopilotResponse("stop") + + spyOn(createChatCompletionsModule, "createChatCompletions") + .mockResolvedValueOnce(firstResponse) + .mockResolvedValueOnce(finalResponse) + const tavilySpy = spyOn(tavilyModule, "searchTavily").mockResolvedValue([]) + const braveSpy = spyOn(braveModule, "searchBrave").mockResolvedValue([]) + + await webSearchInterceptor(makePayload()) + + expect(tavilySpy).toHaveBeenCalled() + expect(braveSpy).not.toHaveBeenCalled() + + state.braveApiKey = undefined + }) + + test("injects failure message when Tavily throws WebSearchError", async () => { + const firstResponse = makeCopilotResponse("tool_calls", [ + { id: "tc-ws", name: "web_search", arguments: '{"query":"q"}' }, + ]) + const finalResponse = makeCopilotResponse("stop") + + const createSpy = spyOn( + createChatCompletionsModule, + "createChatCompletions", + ) + .mockResolvedValueOnce(firstResponse) + .mockResolvedValueOnce(finalResponse) + spyOn(tavilyModule, "searchTavily").mockRejectedValue( + new WebSearchError("HTTP 429"), + ) + + await webSearchInterceptor(makePayload()) + + expect(createSpy).toHaveBeenCalledTimes(2) + const secondCallMessages = createSpy.mock.calls[1]?.[0]?.messages + const toolMsg = secondCallMessages.find((m) => m.role === "tool") + expect(toolMsg?.content).toContain("Web search failed") + expect(toolMsg?.content).toContain("training data") + }) +})