From 3045552d316590138117c2cff6afc266a1b0b7e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Cnghyane=E2=80=9D?= <“hoangvananhnghia99@gmail.com”> Date: Tue, 10 Jun 2025 19:48:57 +0700 Subject: [PATCH 1/9] feat: enhance tokenizer and add tool support - Improve tokenizer to handle complex message types (tool calls, content parts) - Add comprehensive tool support to chat completions - Add better error handling for unsupported features - Add null safety checks for message processing --- src/lib/api-config.ts | 2 +- src/lib/tokenizer.ts | 82 +++++++++++++++++-- src/routes/chat-completions/handler.ts | 5 +- .../copilot/create-chat-completions.ts | 79 ++++++++++++++++-- 4 files changed, 155 insertions(+), 13 deletions(-) diff --git a/src/lib/api-config.ts b/src/lib/api-config.ts index 8075145c9..13e13051d 100644 --- a/src/lib/api-config.ts +++ b/src/lib/api-config.ts @@ -15,7 +15,7 @@ const API_VERSION = "2025-04-01" export const copilotBaseUrl = (state: State) => `https://api.${state.accountType}.githubcopilot.com` -export const copilotHeaders = (state: State, vision: boolean = false) => { +export const copilotHeaders = (state: State, vision = false) => { const headers: Record = { Authorization: `Bearer ${state.copilotToken}`, "content-type": standardHeaders()["content-type"], diff --git a/src/lib/tokenizer.ts b/src/lib/tokenizer.ts index 98797c6b6..b6a4ec80b 100644 --- a/src/lib/tokenizer.ts +++ b/src/lib/tokenizer.ts @@ -1,12 +1,84 @@ import { countTokens } from "gpt-tokenizer/model/gpt-4o" -import type { Message } from "~/services/copilot/create-chat-completions" +import type { Message, ContentPart, ToolCall } from "~/services/copilot/create-chat-completions" + +// Convert Message to gpt-tokenizer compatible format +interface ChatMessage { + role: "user" | "assistant" | "system" + content: string +} + +const convertToTokenizerFormat = (message: Message): ChatMessage | null => { + // Handle tool role messages - convert to assistant for token counting + const role = message.role === "tool" ? "assistant" : message.role + + // Handle string content + if (typeof message.content === "string") { + return { + role: role as "user" | "assistant" | "system", + content: message.content, + } + } + + // Handle null content (can happen with tool calls) + if (message.content === null) { + // If there are tool calls, convert them to text for token counting + if (message.tool_calls && message.tool_calls.length > 0) { + const toolCallsText = message.tool_calls + .map((toolCall: ToolCall) => { + return `Function call: ${toolCall.function.name}(${toolCall.function.arguments})` + }) + .join(" ") + + return { + role: role as "user" | "assistant" | "system", + content: toolCallsText, + } + } + + // If it's a tool response, use the tool_call_id and name for context + if (message.role === "tool" && message.name) { + return { + role: "assistant", + content: `Tool response from ${message.name}`, + } + } + + return null + } + + // Handle ContentPart array - extract text content + const textContent = message.content + .map((part: ContentPart) => { + if (part.type === "input_text" && part.text) { + return part.text + } + // For image parts, we can't count tokens meaningfully, so we'll skip them + // or provide a placeholder. For now, we'll skip them. + return "" + }) + .filter(Boolean) + .join(" ") + + // Only return a message if we have actual text content + if (textContent.trim()) { + return { + role: role as "user" | "assistant" | "system", + content: textContent, + } + } + + return null +} export const getTokenCount = (messages: Array) => { - const input = messages.filter( - (m) => m.role !== "assistant" && typeof m.content === "string", - ) - const output = messages.filter((m) => m.role === "assistant") + // Convert messages to tokenizer-compatible format + const convertedMessages = messages + .map(convertToTokenizerFormat) + .filter((m): m is ChatMessage => m !== null) + + const input = convertedMessages.filter((m) => m.role !== "assistant") + const output = convertedMessages.filter((m) => m.role === "assistant") const inputTokens = countTokens(input) const outputTokens = countTokens(output) diff --git a/src/routes/chat-completions/handler.ts b/src/routes/chat-completions/handler.ts index 9755ecd29..ce113c723 100644 --- a/src/routes/chat-completions/handler.ts +++ b/src/routes/chat-completions/handler.ts @@ -19,7 +19,10 @@ export async function handleCompletion(c: Context) { let payload = await c.req.json() - consola.info("Current token count:", getTokenCount(payload.messages)) + if(payload.messages) { + consola.info("Current token count:", getTokenCount(payload.messages)) + } + if (state.manualApprove) await awaitApproval() diff --git a/src/services/copilot/create-chat-completions.ts b/src/services/copilot/create-chat-completions.ts index 7d54d11f0..983a7ae53 100644 --- a/src/services/copilot/create-chat-completions.ts +++ b/src/services/copilot/create-chat-completions.ts @@ -15,18 +15,32 @@ export const createChatCompletions = async ( const visionEnable = payload.messages.some( (x) => - typeof x.content !== "string" + (x.content && typeof x.content !== "string") && x.content.some((x) => x.type === "image_url"), ) + // Check if tools are being used + const toolsEnable = Boolean(payload.tools && payload.tools.length > 0) + const response = await fetch(`${copilotBaseUrl(state)}/chat/completions`, { method: "POST", headers: copilotHeaders(state, visionEnable), body: JSON.stringify(payload), }) - if (!response.ok) + if (!response.ok) { + const errorText = await response.text() + + // If tools are not supported, provide a helpful error message + if (toolsEnable && response.status === 400) { + throw new HTTPError( + `Failed to create chat completions. GitHub Copilot may not support tool calls. Error: ${errorText}`, + response + ) + } + throw new HTTPError("Failed to create chat completions", response) + } if (payload.stream) { return events(response) @@ -36,8 +50,19 @@ export const createChatCompletions = async ( } const intoCopilotMessage = (message: Message) => { + // Skip processing for assistant messages (they may have tool_calls) + if (message.role === "assistant") return false + + // Skip processing for tool messages (they have specific format) + if (message.role === "tool") return false + + // Skip processing for string content if (typeof message.content === "string") return false + // Skip processing for null content + if (message.content === null) return false + + // Transform content parts for vision support for (const part of message.content) { if (part.type === "input_image") part.type = "image_url" } @@ -56,12 +81,23 @@ export interface ChatCompletionChunk { interface Delta { content?: string role?: string + tool_calls?: Array +} + +interface DeltaToolCall { + index: number + id?: string + type?: "function" + function?: { + name?: string + arguments?: string + } } interface Choice { index: number delta: Delta - finish_reason: "stop" | null + finish_reason: "stop" | "tool_calls" | "length" | "content_filter" | null logprobs: null } @@ -79,7 +115,7 @@ interface ChoiceNonStreaming { index: number message: Message logprobs: null - finish_reason: "stop" + finish_reason: "stop" | "tool_calls" | "length" | "content_filter" } // Payload types @@ -93,11 +129,32 @@ export interface ChatCompletionsPayload { stop?: Array n?: number stream?: boolean + tools?: Array + tool_choice?: "none" | "auto" | ToolChoice +} + +export interface Tool { + type: "function" + function: { + name: string + description?: string + parameters?: Record + } +} + +export interface ToolChoice { + type: "function" + function: { + name: string + } } export interface Message { - role: "user" | "assistant" | "system" - content: string | Array + role: "user" | "assistant" | "system" | "tool" + content: string | Array | null + tool_calls?: Array + tool_call_id?: string + name?: string } // https://platform.openai.com/docs/api-reference @@ -107,5 +164,15 @@ export interface ContentPart { text?: string image_url?: string } + +export interface ToolCall { + id: string + type: "function" + function: { + name: string + arguments: string + } +} + // https://platform.openai.com/docs/guides/images-vision#giving-a-model-images-as-input // Note: copilot use "image_url", but openai use "input_image" From c367a9397dd4600c34c2e927460fca0ad97d7f2c Mon Sep 17 00:00:00 2001 From: "Tang, Tom" Date: Sun, 15 Jun 2025 17:07:04 +1000 Subject: [PATCH 2/9] fix: resolve Anthropic streaming response parsing issue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixed streaming response handler in TypeScript implementation to properly parse chunks from fetch-event-stream library. The issue was that the library returns objects with a .data property containing the actual SSE data, but the original parser only handled string or direct object formats. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .env.example | 15 + main.py | 1844 +++++++++++++++++ src/routes/anthropic/handler.ts | 241 +++ src/routes/anthropic/route.ts | 10 + src/server.ts | 5 + src/services/anthropic/converters.ts | 356 ++++ src/services/anthropic/streaming.ts | 269 +++ .../copilot/create-chat-completions.ts | 2 +- src/types/anthropic.ts | 184 ++ 9 files changed, 2925 insertions(+), 1 deletion(-) create mode 100644 .env.example create mode 100644 main.py create mode 100644 src/routes/anthropic/handler.ts create mode 100644 src/routes/anthropic/route.ts create mode 100644 src/services/anthropic/converters.ts create mode 100644 src/services/anthropic/streaming.ts create mode 100644 src/types/anthropic.ts diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..847c23a20 --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# Required API Keys +ANTHROPIC_API_KEY="your-anthropic-api-key" # Needed if proxying *to* Anthropic +OPENAI_API_KEY="dummy-key" # Can be any value for local OpenAI-compatible APIs + +# OpenAI-Compatible API Configuration +OPENAI_API_BASE="http://localhost:4142" # Your local OpenAI-compatible endpoint + +# Optional: Provider Preference and Model Mapping +# Controls which provider (google or openai) is preferred for mapping haiku/sonnet. +# Defaults to openai if not set. +PREFERRED_PROVIDER="openai" + +# Use models available on your local API (adjust these to match your available models) +BIG_MODEL="claude-sonnet-4" +SMALL_MODEL="claude-sonnet-4" \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 000000000..7cf191ca9 --- /dev/null +++ b/main.py @@ -0,0 +1,1844 @@ +""" +Single-file FastAPI application to proxy Anthropic API requests to an OpenAI-compatible API (e.g., OpenRouter). +Handles request/response conversion, streaming, and dynamic model selection. +""" + +import dataclasses +import enum +import json +import logging +import os +import sys +import time +import traceback +import uuid +from datetime import datetime, timezone +from logging.config import dictConfig +from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, List, + Literal, Optional, Tuple, Union, cast) + +import fastapi +import openai +import tiktoken +import uvicorn +from dotenv import load_dotenv +from fastapi import Request +from fastapi.responses import JSONResponse, Response, StreamingResponse +from openai.types.chat import (ChatCompletionMessageParam, + ChatCompletionToolParam) +from pydantic import BaseModel, Field, ValidationError, field_validator +from pydantic_settings import BaseSettings, SettingsConfigDict +from rich.console import Console +from rich.panel import Panel +from rich.rule import Rule +from rich.text import Text + +load_dotenv() + + +class Settings(BaseSettings): + """Application settings loaded from environment variables.""" + + model_config = SettingsConfigDict(env_file=".env", extra="ignore") + + openai_api_key: str = Field(default="dummy-key", alias="OPENAI_API_KEY") + big_model_name: str = Field(default="claude-sonnet-4", alias="BIG_MODEL") + small_model_name: str = Field(default="claude-sonnet-4", alias="SMALL_MODEL") + base_url: str = Field(default="http://localhost:4142", alias="OPENAI_API_BASE") + referrer_url: str = "http://localhost:8080/claude_proxy" + preferred_provider: str = Field(default="openai", alias="PREFERRED_PROVIDER") + + app_name: str = "AnthropicProxy" + app_version: str = "0.2.0" + log_level: str = "INFO" + log_file_path: Optional[str] = "log.jsonl" + host: str = "127.0.0.1" + port: int = 8080 + reload: bool = True + + +settings = Settings() + + +_console = Console() +_error_console = Console(stderr=True, style="bold red") + + +class JSONFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + header = { + "timestamp": datetime.fromtimestamp( + record.created, timezone.utc + ).isoformat(), + "level": record.levelname, + "logger": record.name, + } + log_payload = getattr(record, "log_record", None) + if isinstance(log_payload, LogRecord): + header["detail"] = dataclasses.asdict(log_payload) + else: + header["message"] = record.getMessage() + if record.exc_info: + exc_type, exc_value, exc_tb = record.exc_info + header["error"] = { + "name": exc_type.__name__ if exc_type else "UnknownError", + "message": str(exc_value), + "stack_trace": "".join( + traceback.format_exception(exc_type, exc_value, exc_tb) + ), + "args": exc_value.args if hasattr(exc_value, "args") else [], + } + return json.dumps(header, ensure_ascii=False) + + +class ConsoleJSONFormatter(JSONFormatter): + def format(self, record: logging.LogRecord) -> str: + log_dict = json.loads(super().format(record)) + if ( + "detail" in log_dict + and "error" in log_dict["detail"] + and log_dict["detail"]["error"] + ): + if "stack_trace" in log_dict["detail"]["error"]: + del log_dict["detail"]["error"]["stack_trace"] + elif "error" in log_dict and log_dict["error"]: + if "stack_trace" in log_dict["error"]: + del log_dict["error"]["stack_trace"] + return json.dumps(log_dict) + + +dictConfig( + { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "json": {"()": JSONFormatter}, + "console_json": {"()": ConsoleJSONFormatter}, + }, + "handlers": { + "default": { + "class": "logging.StreamHandler", + "formatter": "console_json", + "stream": "ext://sys.stdout", + }, + }, + "loggers": { + "": {"handlers": ["default"], "level": "WARNING"}, + settings.app_name: { + "handlers": ["default"], + "level": settings.log_level.upper(), + "propagate": False, + }, + "uvicorn": {"handlers": ["default"], "level": "INFO", "propagate": False}, + "uvicorn.error": { + "handlers": ["default"], + "level": "INFO", + "propagate": False, + }, + "uvicorn.access": { + "handlers": ["default"], + "level": "INFO", + "propagate": False, + }, + }, + } +) + + +class LogEvent(enum.Enum): + MODEL_SELECTION = "model_selection" + REQUEST_START = "request_start" + REQUEST_COMPLETED = "request_completed" + REQUEST_FAILURE = "request_failure" + ANTHROPIC_REQUEST = "anthropic_body" + OPENAI_REQUEST = "openai_request" + OPENAI_RESPONSE = "openai_response" + ANTHROPIC_RESPONSE = "anthropic_response" + STREAMING_REQUEST = "streaming_request" + STREAM_INTERRUPTED = "stream_interrupted" + TOKEN_COUNT = "token_count" + TOKEN_ENCODER_LOAD_FAILED = "token_encoder_load_failed" + SYSTEM_PROMPT_ADJUSTED = "system_prompt_adjusted" + TOOL_INPUT_SERIALIZATION_FAILURE = "tool_input_serialization_failure" + IMAGE_FORMAT_UNSUPPORTED = "image_format_unsupported" + MESSAGE_FORMAT_NORMALIZED = "message_format_normalized" + TOOL_RESULT_SERIALIZATION_FAILURE = "tool_result_serialization_failure" + TOOL_RESULT_PROCESSING = "tool_result_processing" + TOOL_CHOICE_UNSUPPORTED = "tool_choice_unsupported" + TOOL_ARGS_TYPE_MISMATCH = "tool_args_type_mismatch" + TOOL_ARGS_PARSE_FAILURE = "tool_args_parse_failure" + TOOL_ARGS_UNEXPECTED = "tool_args_unexpected" + TOOL_ID_PLACEHOLDER = "tool_id_placeholder" + TOOL_ID_UPDATED = "tool_id_updated" + PARAMETER_UNSUPPORTED = "parameter_unsupported" + HEALTH_CHECK = "health_check" + PROVIDER_ERROR_DETAILS = "provider_error_details" + + +@dataclasses.dataclass +class LogError: + name: str + message: str + stack_trace: Optional[str] = None + args: Optional[Tuple[Any, ...]] = None + + +@dataclasses.dataclass +class LogRecord: + event: str + message: str + request_id: Optional[str] = None + data: Optional[Dict[str, Any]] = None + error: Optional[LogError] = None + + +_logger = logging.getLogger(settings.app_name) + +if settings.log_file_path: + try: + log_dir = os.path.dirname(settings.log_file_path) + if log_dir: + os.makedirs(log_dir, exist_ok=True) + file_handler = logging.FileHandler(settings.log_file_path, mode="a") + file_handler.setFormatter(JSONFormatter()) + _logger.addHandler(file_handler) + except Exception as e: + _error_console.print( + f"Failed to configure file logging to {settings.log_file_path}: {e}" + ) + + +def _log(level: int, record: LogRecord, exc: Optional[Exception] = None) -> None: + if exc: + record.error = LogError( + name=type(exc).__name__, + message=str(exc), + stack_trace="".join( + traceback.format_exception(type(exc), exc, exc.__traceback__) + ), + args=exc.args if hasattr(exc, "args") else tuple(), + ) + if not record.message and str(exc): + record.message = str(exc) + elif not record.message: + record.message = "An unspecified error occurred" + + _logger.log(level=level, msg=record.message, extra={"log_record": record}) + + +def debug(record: LogRecord): + _log(logging.DEBUG, record) + + +def info(record: LogRecord): + _log(logging.INFO, record) + + +def warning(record: LogRecord, exc: Optional[Exception] = None): + _log(logging.WARNING, record, exc=exc) + + +def error(record: LogRecord, exc: Optional[Exception] = None): + if exc: + _error_console.print_exception(show_locals=False, width=120) + _log(logging.ERROR, record, exc=exc) + + +def critical(record: LogRecord, exc: Optional[Exception] = None): + _log(logging.CRITICAL, record, exc=exc) + + +class ContentBlockText(BaseModel): + type: Literal["text"] + text: str + + +class ContentBlockImageSource(BaseModel): + type: str + media_type: str + data: str + + +class ContentBlockImage(BaseModel): + type: Literal["image"] + source: ContentBlockImageSource + + +class ContentBlockToolUse(BaseModel): + type: Literal["tool_use"] + id: str + name: str + input: Dict[str, Any] + + +class ContentBlockToolResult(BaseModel): + type: Literal["tool_result"] + tool_use_id: str + content: Union[str, List[Dict[str, Any]], List[Any]] + is_error: Optional[bool] = None + + +ContentBlock = Union[ + ContentBlockText, ContentBlockImage, ContentBlockToolUse, ContentBlockToolResult +] + + +class SystemContent(BaseModel): + type: Literal["text"] + text: str + + +class Message(BaseModel): + role: Literal["user", "assistant"] + content: Union[str, List[ContentBlock]] + + +class Tool(BaseModel): + name: str + description: Optional[str] = None + input_schema: Dict[str, Any] = Field(..., alias="input_schema") + + +class ToolChoice(BaseModel): + type: Literal["auto", "any", "tool"] + name: Optional[str] = None + + +class MessagesRequest(BaseModel): + model: str + max_tokens: int + messages: List[Message] + system: Optional[Union[str, List[SystemContent]]] = None + stop_sequences: Optional[List[str]] = None + stream: Optional[bool] = False + temperature: Optional[float] = None + top_p: Optional[float] = None + top_k: Optional[int] = None + metadata: Optional[Dict[str, Any]] = None + tools: Optional[List[Tool]] = None + tool_choice: Optional[ToolChoice] = None + + @field_validator("top_k") + def check_top_k(cls, v: Optional[int]) -> Optional[int]: + if v is not None: + req_id = info.context.get("request_id") if info.context else None + warning( + LogRecord( + event=LogEvent.PARAMETER_UNSUPPORTED.value, + message="Parameter 'top_k' provided by client but is not directly supported by the OpenAI Chat Completions API and will be ignored.", + request_id=req_id, + data={"parameter": "top_k", "value": v}, + ) + ) + return v + + +class TokenCountRequest(BaseModel): + model: str + messages: List[Message] + system: Optional[Union[str, List[SystemContent]]] = None + tools: Optional[List[Tool]] = None + + +class TokenCountResponse(BaseModel): + input_tokens: int + + +class Usage(BaseModel): + input_tokens: int + output_tokens: int + + +class ProviderErrorMetadata(BaseModel): + provider_name: str + raw_error: Optional[Dict[str, Any]] = None + + +class AnthropicErrorType(str, enum.Enum): + INVALID_REQUEST = "invalid_request_error" + AUTHENTICATION = "authentication_error" + PERMISSION = "permission_error" + NOT_FOUND = "not_found_error" + RATE_LIMIT = "rate_limit_error" + API_ERROR = "api_error" + OVERLOADED = "overloaded_error" + REQUEST_TOO_LARGE = "request_too_large_error" + + +class AnthropicErrorDetail(BaseModel): + type: AnthropicErrorType + message: str + provider: Optional[str] = None + provider_message: Optional[str] = None + provider_code: Optional[Union[str, int]] = None + + +class AnthropicErrorResponse(BaseModel): + type: Literal["error"] = "error" + error: AnthropicErrorDetail + + +class MessagesResponse(BaseModel): + id: str + type: Literal["message"] = "message" + role: Literal["assistant"] = "assistant" + model: str + content: List[ContentBlock] + stop_reason: Optional[ + Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "error"] + ] = None + stop_sequence: Optional[str] = None + usage: Usage + + +STATUS_CODE_ERROR_MAP: Dict[int, AnthropicErrorType] = { + 400: AnthropicErrorType.INVALID_REQUEST, + 401: AnthropicErrorType.AUTHENTICATION, + 403: AnthropicErrorType.PERMISSION, + 404: AnthropicErrorType.NOT_FOUND, + 413: AnthropicErrorType.REQUEST_TOO_LARGE, + 422: AnthropicErrorType.INVALID_REQUEST, + 429: AnthropicErrorType.RATE_LIMIT, + 500: AnthropicErrorType.API_ERROR, + 502: AnthropicErrorType.API_ERROR, + 503: AnthropicErrorType.OVERLOADED, + 504: AnthropicErrorType.API_ERROR, +} + + +def extract_provider_error_details( + error_details_dict: Optional[Dict[str, Any]], +) -> Optional[ProviderErrorMetadata]: + if not isinstance(error_details_dict, dict): + return None + metadata = error_details_dict.get("metadata") + if not isinstance(metadata, dict): + return None + provider_name = metadata.get("provider_name") + raw_error_str = metadata.get("raw") + + if not provider_name or not isinstance(provider_name, str): + return None + + parsed_raw_error: Optional[Dict[str, Any]] = None + if isinstance(raw_error_str, str): + try: + parsed_raw_error = json.loads(raw_error_str) + except json.JSONDecodeError: + warning( + LogRecord( + event=LogEvent.PROVIDER_ERROR_DETAILS.value, + message=f"Failed to parse raw provider error string for {provider_name}.", + ) + ) + parsed_raw_error = {"raw_string_parse_failed": raw_error_str} + elif isinstance(raw_error_str, dict): + parsed_raw_error = raw_error_str + + return ProviderErrorMetadata( + provider_name=provider_name, raw_error=parsed_raw_error + ) + + +try: + openai_client = openai.AsyncClient( + api_key=settings.openai_api_key, + base_url=settings.base_url, + default_headers={ + "HTTP-Referer": settings.referrer_url, + "X-Title": settings.app_name, + }, + timeout=180.0, + ) +except Exception as e: + critical( + LogRecord( + event="openai_client_init_failed", + message="Failed to initialize OpenAI client", + ), + exc=e, + ) + sys.exit(1) + + +_token_encoder_cache: Dict[str, tiktoken.Encoding] = {} + + +def get_token_encoder( + model_name: str = "gpt-4", request_id: Optional[str] = None +) -> tiktoken.Encoding: + """Gets a tiktoken encoder, caching it for performance.""" + + cache_key = "gpt-4" + if cache_key not in _token_encoder_cache: + try: + _token_encoder_cache[cache_key] = tiktoken.encoding_for_model(cache_key) + except Exception: + try: + _token_encoder_cache[cache_key] = tiktoken.get_encoding("cl100k_base") + warning( + LogRecord( + event=LogEvent.TOKEN_ENCODER_LOAD_FAILED.value, + message=f"Could not load tiktoken encoder for '{cache_key}', using 'cl100k_base'. Token counts may be approximate.", + request_id=request_id, + data={"model_tried": cache_key}, + ) + ) + except Exception as e_cl: + critical( + LogRecord( + event=LogEvent.TOKEN_ENCODER_LOAD_FAILED.value, + message="Failed to load any tiktoken encoder (gpt-4, cl100k_base). Token counting will be inaccurate.", + request_id=request_id, + ), + exc=e_cl, + ) + + class DummyEncoder: + def encode(self, text: str) -> List[int]: + return list(range(len(text))) + + _token_encoder_cache[cache_key] = DummyEncoder() + return _token_encoder_cache[cache_key] + + +def count_tokens_for_anthropic_request( + messages: List[Message], + system: Optional[Union[str, List[SystemContent]]], + model_name: str, + tools: Optional[List[Tool]] = None, + request_id: Optional[str] = None, +) -> int: + enc = get_token_encoder(model_name, request_id) + total_tokens = 0 + + if isinstance(system, str): + total_tokens += len(enc.encode(system)) + elif isinstance(system, list): + for block in system: + if isinstance(block, SystemContent) and block.type == "text": + total_tokens += len(enc.encode(block.text)) + + for msg in messages: + total_tokens += 4 + if msg.role: + total_tokens += len(enc.encode(msg.role)) + + if isinstance(msg.content, str): + total_tokens += len(enc.encode(msg.content)) + elif isinstance(msg.content, list): + for block in msg.content: + if isinstance(block, ContentBlockText): + total_tokens += len(enc.encode(block.text)) + elif isinstance(block, ContentBlockImage): + total_tokens += 768 + elif isinstance(block, ContentBlockToolUse): + total_tokens += len(enc.encode(block.name)) + try: + input_str = json.dumps(block.input) + total_tokens += len(enc.encode(input_str)) + except Exception: + warning( + LogRecord( + event=LogEvent.TOOL_INPUT_SERIALIZATION_FAILURE.value, + message="Failed to serialize tool input for token counting.", + data={"tool_name": block.name}, + request_id=request_id, + ) + ) + elif isinstance(block, ContentBlockToolResult): + try: + content_str = "" + if isinstance(block.content, str): + content_str = block.content + elif isinstance(block.content, list): + for item in block.content: + if ( + isinstance(item, dict) + and item.get("type") == "text" + ): + content_str += item.get("text", "") + else: + content_str += json.dumps(item) + else: + content_str = json.dumps(block.content) + total_tokens += len(enc.encode(content_str)) + except Exception: + warning( + LogRecord( + event=LogEvent.TOOL_RESULT_SERIALIZATION_FAILURE.value, + message="Failed to serialize tool result for token counting.", + request_id=request_id, + ) + ) + + if tools: + total_tokens += 2 + for tool in tools: + total_tokens += len(enc.encode(tool.name)) + if tool.description: + total_tokens += len(enc.encode(tool.description)) + try: + schema_str = json.dumps(tool.input_schema) + total_tokens += len(enc.encode(schema_str)) + except Exception: + warning( + LogRecord( + event=LogEvent.TOOL_INPUT_SERIALIZATION_FAILURE.value, + message="Failed to serialize tool schema for token counting.", + data={"tool_name": tool.name}, + request_id=request_id, + ) + ) + debug( + LogRecord( + event=LogEvent.TOKEN_COUNT.value, + message=f"Estimated {total_tokens} input tokens for model {model_name}", + data={"model": model_name, "token_count": total_tokens}, + request_id=request_id, + ) + ) + return total_tokens + + +StopReasonType = Optional[ + Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "error"] +] + + +def _serialize_tool_result_content_for_openai( + anthropic_tool_result_content: Union[str, List[Dict[str, Any]], List[Any]], + request_id: Optional[str], + log_context: Dict, +) -> str: + """ + Serializes Anthropic tool result content (which can be complex) into a single string + as expected by OpenAI for the 'content' field of a 'tool' role message. + """ + if isinstance(anthropic_tool_result_content, str): + return anthropic_tool_result_content + + if isinstance(anthropic_tool_result_content, list): + processed_parts = [] + contains_non_text_block = False + for item in anthropic_tool_result_content: + if isinstance(item, dict) and item.get("type") == "text" and "text" in item: + processed_parts.append(str(item["text"])) + else: + try: + processed_parts.append(json.dumps(item)) + contains_non_text_block = True + except TypeError: + processed_parts.append( + f"" + ) + contains_non_text_block = True + + result_str = "\n".join(processed_parts) + if contains_non_text_block: + warning( + LogRecord( + event=LogEvent.TOOL_RESULT_PROCESSING.value, + message="Tool result content list contained non-text or complex items; parts were JSON stringified.", + request_id=request_id, + data={**log_context, "result_str_preview": result_str[:100]}, + ) + ) + return result_str + + try: + return json.dumps(anthropic_tool_result_content) + except TypeError as e: + warning( + LogRecord( + event=LogEvent.TOOL_RESULT_SERIALIZATION_FAILURE.value, + message=f"Failed to serialize tool result content to JSON: {e}. Returning error JSON.", + request_id=request_id, + data=log_context, + ) + ) + return json.dumps( + { + "error": "Serialization failed", + "original_type": str(type(anthropic_tool_result_content)), + } + ) + + +def convert_anthropic_to_openai_messages( + anthropic_messages: List[Message], + anthropic_system: Optional[Union[str, List[SystemContent]]] = None, + request_id: Optional[str] = None, +) -> List[Dict[str, Any]]: + openai_messages: List[Dict[str, Any]] = [] + + system_text_content = "" + if isinstance(anthropic_system, str): + system_text_content = anthropic_system + elif isinstance(anthropic_system, list): + system_texts = [ + block.text + for block in anthropic_system + if isinstance(block, SystemContent) and block.type == "text" + ] + if len(system_texts) < len(anthropic_system): + warning( + LogRecord( + event=LogEvent.SYSTEM_PROMPT_ADJUSTED.value, + message="Non-text content blocks in Anthropic system prompt were ignored.", + request_id=request_id, + ) + ) + system_text_content = "\n".join(system_texts) + + if system_text_content: + openai_messages.append({"role": "system", "content": system_text_content}) + + for i, msg in enumerate(anthropic_messages): + role = msg.role + content = msg.content + + if isinstance(content, str): + openai_messages.append({"role": role, "content": content}) + continue + + if isinstance(content, list): + openai_parts_for_user_message = [] + assistant_tool_calls = [] + text_content_for_assistant = [] + + if not content and role == "user": + openai_messages.append({"role": "user", "content": ""}) + continue + if not content and role == "assistant": + openai_messages.append({"role": "assistant", "content": ""}) + continue + + for block_idx, block in enumerate(content): + block_log_ctx = { + "anthropic_message_index": i, + "block_index": block_idx, + "block_type": block.type, + } + + if isinstance(block, ContentBlockText): + if role == "user": + openai_parts_for_user_message.append( + {"type": "text", "text": block.text} + ) + elif role == "assistant": + text_content_for_assistant.append(block.text) + + elif isinstance(block, ContentBlockImage) and role == "user": + if block.source.type == "base64": + openai_parts_for_user_message.append( + { + "type": "image_url", + "image_url": { + "url": f"data:{block.source.media_type};base64,{block.source.data}" + }, + } + ) + else: + warning( + LogRecord( + event=LogEvent.IMAGE_FORMAT_UNSUPPORTED.value, + message=f"Image block with source type '{block.source.type}' (expected 'base64') ignored in user message {i}.", + request_id=request_id, + data=block_log_ctx, + ) + ) + + elif isinstance(block, ContentBlockToolUse) and role == "assistant": + try: + args_str = json.dumps(block.input) + except Exception as e: + error( + LogRecord( + event=LogEvent.TOOL_INPUT_SERIALIZATION_FAILURE.value, + message=f"Failed to serialize tool input for tool '{block.name}'. Using empty JSON.", + request_id=request_id, + data={ + **block_log_ctx, + "tool_id": block.id, + "tool_name": block.name, + }, + ), + exc=e, + ) + args_str = "{}" + + assistant_tool_calls.append( + { + "id": block.id, + "type": "function", + "function": {"name": block.name, "arguments": args_str}, + } + ) + + elif isinstance(block, ContentBlockToolResult) and role == "user": + serialized_content = _serialize_tool_result_content_for_openai( + block.content, request_id, block_log_ctx + ) + openai_messages.append( + { + "role": "tool", + "tool_call_id": block.tool_use_id, + "content": serialized_content, + } + ) + + if role == "user" and openai_parts_for_user_message: + is_multimodal = any( + part["type"] == "image_url" + for part in openai_parts_for_user_message + ) + if is_multimodal or len(openai_parts_for_user_message) > 1: + openai_messages.append( + {"role": "user", "content": openai_parts_for_user_message} + ) + elif ( + len(openai_parts_for_user_message) == 1 + and openai_parts_for_user_message[0]["type"] == "text" + ): + openai_messages.append( + { + "role": "user", + "content": openai_parts_for_user_message[0]["text"], + } + ) + elif not openai_parts_for_user_message: + openai_messages.append({"role": "user", "content": ""}) + + if role == "assistant": + assistant_text = "\n".join(filter(None, text_content_for_assistant)) + if assistant_text: + openai_messages.append( + {"role": "assistant", "content": assistant_text} + ) + + if assistant_tool_calls: + if ( + openai_messages + and openai_messages[-1]["role"] == "assistant" + and openai_messages[-1].get("content") + ): + openai_messages.append( + { + "role": "assistant", + "content": None, + "tool_calls": assistant_tool_calls, + } + ) + + elif ( + openai_messages + and openai_messages[-1]["role"] == "assistant" + and not openai_messages[-1].get("tool_calls") + ): + openai_messages[-1]["tool_calls"] = assistant_tool_calls + openai_messages[-1]["content"] = None + else: + openai_messages.append( + { + "role": "assistant", + "content": None, + "tool_calls": assistant_tool_calls, + } + ) + + final_openai_messages = [] + for msg_dict in openai_messages: + if ( + msg_dict.get("role") == "assistant" + and msg_dict.get("tool_calls") + and msg_dict.get("content") is not None + ): + warning( + LogRecord( + event=LogEvent.MESSAGE_FORMAT_NORMALIZED.value, + message="Corrected assistant message with tool_calls to have content: None.", + request_id=request_id, + data={"original_content": msg_dict["content"]}, + ) + ) + msg_dict["content"] = None + final_openai_messages.append(msg_dict) + + return final_openai_messages + + +def convert_anthropic_tools_to_openai( + anthropic_tools: Optional[List[Tool]], +) -> Optional[List[Dict[str, Any]]]: + if not anthropic_tools: + return None + return [ + { + "type": "function", + "function": { + "name": t.name, + "description": t.description or "", + "parameters": t.input_schema, + }, + } + for t in anthropic_tools + ] + + +def convert_anthropic_tool_choice_to_openai( + anthropic_choice: Optional[ToolChoice], + request_id: Optional[str] = None, +) -> Optional[Union[str, Dict[str, Any]]]: + if not anthropic_choice: + return None + if anthropic_choice.type == "auto": + return "auto" + if anthropic_choice.type == "any": + warning( + LogRecord( + event=LogEvent.TOOL_CHOICE_UNSUPPORTED.value, + message="Anthropic tool_choice type 'any' mapped to OpenAI 'auto'. Exact behavior might differ (OpenAI 'auto' allows no tool use).", + request_id=request_id, + data={"anthropic_tool_choice": anthropic_choice.model_dump()}, + ) + ) + return "auto" + if anthropic_choice.type == "tool" and anthropic_choice.name: + return {"type": "function", "function": {"name": anthropic_choice.name}} + + warning( + LogRecord( + event=LogEvent.TOOL_CHOICE_UNSUPPORTED.value, + message=f"Unsupported Anthropic tool_choice: {anthropic_choice.model_dump()}. Defaulting to 'auto'.", + request_id=request_id, + data={"anthropic_tool_choice": anthropic_choice.model_dump()}, + ) + ) + return "auto" + + +def convert_openai_to_anthropic_response( + openai_response: openai.types.chat.ChatCompletion, + original_anthropic_model_name: str, + request_id: Optional[str] = None, +) -> MessagesResponse: + anthropic_content: List[ContentBlock] = [] + anthropic_stop_reason: StopReasonType = None + + stop_reason_map: Dict[Optional[str], StopReasonType] = { + "stop": "end_turn", + "length": "max_tokens", + "tool_calls": "tool_use", + "function_call": "tool_use", + "content_filter": "stop_sequence", + None: "end_turn", + } + + if openai_response.choices: + choice = openai_response.choices[0] + message = choice.message + finish_reason = choice.finish_reason + + anthropic_stop_reason = stop_reason_map.get(finish_reason, "end_turn") + + if message.content: + anthropic_content.append( + ContentBlockText(type="text", text=message.content) + ) + + if message.tool_calls: + for call in message.tool_calls: + if call.type == "function": + tool_input_dict: Dict[str, Any] = {} + try: + parsed_input = json.loads(call.function.arguments) + if isinstance(parsed_input, dict): + tool_input_dict = parsed_input + else: + tool_input_dict = {"value": parsed_input} + warning( + LogRecord( + event=LogEvent.TOOL_ARGS_TYPE_MISMATCH.value, + message=f"OpenAI tool arguments for '{call.function.name}' parsed to non-dict type '{type(parsed_input).__name__}'. Wrapped in 'value'.", + request_id=request_id, + data={ + "tool_name": call.function.name, + "tool_id": call.id, + }, + ) + ) + except json.JSONDecodeError as e: + error( + LogRecord( + event=LogEvent.TOOL_ARGS_PARSE_FAILURE.value, + message=f"Failed to parse JSON arguments for tool '{call.function.name}'. Storing raw string.", + request_id=request_id, + data={ + "tool_name": call.function.name, + "tool_id": call.id, + "raw_args": call.function.arguments, + }, + ), + exc=e, + ) + tool_input_dict = { + "error_parsing_arguments": call.function.arguments + } + + anthropic_content.append( + ContentBlockToolUse( + type="tool_use", + id=call.id, + name=call.function.name, + input=tool_input_dict, + ) + ) + if finish_reason == "tool_calls": + anthropic_stop_reason = "tool_use" + + if not anthropic_content: + anthropic_content.append(ContentBlockText(type="text", text="")) + + usage = openai_response.usage + anthropic_usage = Usage( + input_tokens=usage.prompt_tokens if usage else 0, + output_tokens=usage.completion_tokens if usage else 0, + ) + + response_id = ( + f"msg_{openai_response.id}" + if openai_response.id + else f"msg_{request_id}_completed" + ) + + return MessagesResponse( + id=response_id, + type="message", + role="assistant", + model=original_anthropic_model_name, + content=anthropic_content, + stop_reason=anthropic_stop_reason, + usage=anthropic_usage, + ) + + +def _get_anthropic_error_details_from_exc( + exc: Exception, +) -> Tuple[AnthropicErrorType, str, int, Optional[ProviderErrorMetadata]]: + """Maps caught exceptions to Anthropic error type, message, status code, and provider details.""" + error_type = AnthropicErrorType.API_ERROR + error_message = str(exc) + status_code = 500 + provider_details: Optional[ProviderErrorMetadata] = None + + if isinstance(exc, openai.APIError): + error_message = exc.message or str(exc) + status_code = exc.status_code or 500 + error_type = STATUS_CODE_ERROR_MAP.get( + status_code, AnthropicErrorType.API_ERROR + ) + + if hasattr(exc, "body") and isinstance(exc.body, dict): + actual_error_details = exc.body.get("error", exc.body) + provider_details = extract_provider_error_details(actual_error_details) + + if isinstance(exc, openai.AuthenticationError): + error_type = AnthropicErrorType.AUTHENTICATION + elif isinstance(exc, openai.RateLimitError): + error_type = AnthropicErrorType.RATE_LIMIT + elif isinstance(exc, (openai.BadRequestError, openai.UnprocessableEntityError)): + error_type = AnthropicErrorType.INVALID_REQUEST + elif isinstance(exc, openai.PermissionDeniedError): + error_type = AnthropicErrorType.PERMISSION + elif isinstance(exc, openai.NotFoundError): + error_type = AnthropicErrorType.NOT_FOUND + + return error_type, error_message, status_code, provider_details + + +def _format_anthropic_error_sse_event( + error_type: AnthropicErrorType, + message: str, + provider_details: Optional[ProviderErrorMetadata] = None, +) -> str: + """Formats an error into the Anthropic SSE 'error' event structure.""" + anthropic_err_detail = AnthropicErrorDetail(type=error_type, message=message) + if provider_details: + anthropic_err_detail.provider = provider_details.provider_name + if provider_details.raw_error and isinstance( + provider_details.raw_error.get("error"), dict + ): + prov_err_obj = provider_details.raw_error["error"] + anthropic_err_detail.provider_message = prov_err_obj.get("message") + anthropic_err_detail.provider_code = prov_err_obj.get("code") + elif provider_details.raw_error and isinstance( + provider_details.raw_error.get("message"), str + ): + anthropic_err_detail.provider_message = provider_details.raw_error.get( + "message" + ) + anthropic_err_detail.provider_code = provider_details.raw_error.get("code") + + error_response = AnthropicErrorResponse(error=anthropic_err_detail) + return f"event: error\ndata: {error_response.model_dump_json()}\n\n" + + +async def handle_anthropic_streaming_response_from_openai_stream( + openai_stream: openai.AsyncStream[openai.types.chat.ChatCompletionChunk], + original_anthropic_model_name: str, + estimated_input_tokens: int, + request_id: str, + start_time_mono: float, +) -> AsyncGenerator[str, None]: + """ + Consumes an OpenAI stream and yields Anthropic-compatible SSE events. + BUGFIX: Correctly handles content block indexing for mixed text/tool_use. + """ + + anthropic_message_id = f"msg_stream_{request_id}_{uuid.uuid4().hex[:8]}" + + next_anthropic_block_idx = 0 + text_block_anthropic_idx: Optional[int] = None + + openai_tool_idx_to_anthropic_block_idx: Dict[int, int] = {} + + tool_states: Dict[int, Dict[str, Any]] = {} + + sent_tool_block_starts: set[int] = set() + + output_token_count = 0 + final_anthropic_stop_reason: StopReasonType = None + + enc = get_token_encoder(original_anthropic_model_name, request_id) + + openai_to_anthropic_stop_reason_map: Dict[Optional[str], StopReasonType] = { + "stop": "end_turn", + "length": "max_tokens", + "tool_calls": "tool_use", + "function_call": "tool_use", + "content_filter": "stop_sequence", + None: None, + } + + stream_status_code = 200 + stream_final_message = "Streaming request completed successfully." + stream_log_event = LogEvent.REQUEST_COMPLETED.value + + try: + message_start_event_data = { + "type": "message_start", + "message": { + "id": anthropic_message_id, + "type": "message", + "role": "assistant", + "model": original_anthropic_model_name, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": estimated_input_tokens, "output_tokens": 0}, + }, + } + yield f"event: message_start\ndata: {json.dumps(message_start_event_data)}\n\n" + yield f"event: ping\ndata: {json.dumps({'type': 'ping'})}\n\n" + + async for chunk in openai_stream: + if not chunk.choices: + continue + + delta = chunk.choices[0].delta + openai_finish_reason = chunk.choices[0].finish_reason + + if delta.content: + output_token_count += len(enc.encode(delta.content)) + if text_block_anthropic_idx is None: + text_block_anthropic_idx = next_anthropic_block_idx + next_anthropic_block_idx += 1 + start_text_event = { + "type": "content_block_start", + "index": text_block_anthropic_idx, + "content_block": {"type": "text", "text": ""}, + } + yield f"event: content_block_start\ndata: {json.dumps(start_text_event)}\n\n" + + text_delta_event = { + "type": "content_block_delta", + "index": text_block_anthropic_idx, + "delta": {"type": "text_delta", "text": delta.content}, + } + yield f"event: content_block_delta\ndata: {json.dumps(text_delta_event)}\n\n" + + if delta.tool_calls: + for tool_delta in delta.tool_calls: + openai_tc_idx = tool_delta.index + + if openai_tc_idx not in openai_tool_idx_to_anthropic_block_idx: + current_anthropic_tool_block_idx = next_anthropic_block_idx + next_anthropic_block_idx += 1 + openai_tool_idx_to_anthropic_block_idx[openai_tc_idx] = ( + current_anthropic_tool_block_idx + ) + + tool_states[current_anthropic_tool_block_idx] = { + "id": tool_delta.id + or f"tool_ph_{request_id}_{current_anthropic_tool_block_idx}", + "name": "", + "arguments_buffer": "", + } + if not tool_delta.id: + warning( + LogRecord( + LogEvent.TOOL_ID_PLACEHOLDER.value, + f"Generated placeholder Tool ID for OpenAI tool index {openai_tc_idx} -> Anthropic block {current_anthropic_tool_block_idx}", + request_id, + ) + ) + else: + current_anthropic_tool_block_idx = ( + openai_tool_idx_to_anthropic_block_idx[openai_tc_idx] + ) + + tool_state = tool_states[current_anthropic_tool_block_idx] + + if tool_delta.id and tool_state["id"].startswith("tool_ph_"): + debug( + LogRecord( + LogEvent.TOOL_ID_UPDATED.value, + f"Updated placeholder Tool ID for Anthropic block {current_anthropic_tool_block_idx} to {tool_delta.id}", + request_id, + ) + ) + tool_state["id"] = tool_delta.id + + if tool_delta.function: + if tool_delta.function.name: + tool_state["name"] = tool_delta.function.name + if tool_delta.function.arguments: + tool_state["arguments_buffer"] += ( + tool_delta.function.arguments + ) + output_token_count += len( + enc.encode(tool_delta.function.arguments) + ) + + if ( + current_anthropic_tool_block_idx not in sent_tool_block_starts + and tool_state["id"] + and not tool_state["id"].startswith("tool_ph_") + and tool_state["name"] + ): + start_tool_event = { + "type": "content_block_start", + "index": current_anthropic_tool_block_idx, + "content_block": { + "type": "tool_use", + "id": tool_state["id"], + "name": tool_state["name"], + "input": {}, + }, + } + yield f"event: content_block_start\ndata: {json.dumps(start_tool_event)}\n\n" + sent_tool_block_starts.add(current_anthropic_tool_block_idx) + + if ( + tool_delta.function + and tool_delta.function.arguments + and current_anthropic_tool_block_idx in sent_tool_block_starts + ): + args_delta_event = { + "type": "content_block_delta", + "index": current_anthropic_tool_block_idx, + "delta": { + "type": "input_json_delta", + "partial_json": tool_delta.function.arguments, + }, + } + yield f"event: content_block_delta\ndata: {json.dumps(args_delta_event)}\n\n" + + if openai_finish_reason: + final_anthropic_stop_reason = openai_to_anthropic_stop_reason_map.get( + openai_finish_reason, "end_turn" + ) + if openai_finish_reason == "tool_calls": + final_anthropic_stop_reason = "tool_use" + break + + if text_block_anthropic_idx is not None: + yield f"event: content_block_stop\ndata: {json.dumps({'type': 'content_block_stop', 'index': text_block_anthropic_idx})}\n\n" + + for anthropic_tool_idx in sent_tool_block_starts: + tool_state_to_finalize = tool_states.get(anthropic_tool_idx) + if tool_state_to_finalize: + try: + json.loads(tool_state_to_finalize["arguments_buffer"]) + except json.JSONDecodeError: + warning( + LogRecord( + event=LogEvent.TOOL_ARGS_PARSE_FAILURE.value, + message=f"Buffered arguments for tool '{tool_state_to_finalize.get('name')}' (Anthropic block {anthropic_tool_idx}) did not form valid JSON.", + request_id=request_id, + data={ + "buffered_args": tool_state_to_finalize[ + "arguments_buffer" + ][:100] + }, + ) + ) + yield f"event: content_block_stop\ndata: {json.dumps({'type': 'content_block_stop', 'index': anthropic_tool_idx})}\n\n" + + if final_anthropic_stop_reason is None: + final_anthropic_stop_reason = "end_turn" + + message_delta_event = { + "type": "message_delta", + "delta": { + "stop_reason": final_anthropic_stop_reason, + "stop_sequence": None, + }, + "usage": {"output_tokens": output_token_count}, + } + yield f"event: message_delta\ndata: {json.dumps(message_delta_event)}\n\n" + yield f"event: message_stop\ndata: {json.dumps({'type': 'message_stop'})}\n\n" + + except Exception as e: + stream_status_code = 500 + stream_log_event = LogEvent.REQUEST_FAILURE.value + error_type, error_msg_str, _, provider_err_details = ( + _get_anthropic_error_details_from_exc(e) + ) + stream_final_message = f"Error during OpenAI stream conversion: {error_msg_str}" + final_anthropic_stop_reason = "error" + + error( + LogRecord( + event=LogEvent.STREAM_INTERRUPTED.value, + message=stream_final_message, + request_id=request_id, + data={ + "error_type": error_type.value, + "provider_details": provider_err_details.model_dump() + if provider_err_details + else None, + }, + ), + exc=e, + ) + yield _format_anthropic_error_sse_event( + error_type, error_msg_str, provider_err_details + ) + + finally: + duration_ms = (time.monotonic() - start_time_mono) * 1000 + log_data = { + "status_code": stream_status_code, + "duration_ms": duration_ms, + "input_tokens": estimated_input_tokens, + "output_tokens": output_token_count, + "stop_reason": final_anthropic_stop_reason, + } + if stream_log_event == LogEvent.REQUEST_COMPLETED.value: + info( + LogRecord( + event=stream_log_event, + message=stream_final_message, + request_id=request_id, + data=log_data, + ) + ) + else: + error( + LogRecord( + event=stream_log_event, + message=stream_final_message, + request_id=request_id, + data=log_data, + ) + ) + + +app = fastapi.FastAPI( + title=settings.app_name, + description="Routes Anthropic API requests to an OpenAI-compatible API, selecting models dynamically.", + version=settings.app_version, + docs_url=None, + redoc_url=None, +) + + +def select_target_model(client_model_name: str, request_id: str) -> str: + """Selects the target OpenRouter model based on the client's request.""" + client_model_lower = client_model_name.lower() + target_model: str + + if "opus" in client_model_lower or "sonnet" in client_model_lower: + target_model = settings.big_model_name + elif "haiku" in client_model_lower: + target_model = settings.small_model_name + else: + target_model = settings.small_model_name + warning( + LogRecord( + event=LogEvent.MODEL_SELECTION.value, + message=f"Unknown client model '{client_model_name}', defaulting to SMALL model '{target_model}'.", + request_id=request_id, + data={ + "client_model": client_model_name, + "default_target_model": target_model, + }, + ) + ) + + debug( + LogRecord( + event=LogEvent.MODEL_SELECTION.value, + message=f"Client model '{client_model_name}' mapped to target model '{target_model}'.", + request_id=request_id, + data={"client_model": client_model_name, "target_model": target_model}, + ) + ) + return target_model + + +def _build_anthropic_error_response( + error_type: AnthropicErrorType, + message: str, + status_code: int, + provider_details: Optional[ProviderErrorMetadata] = None, +) -> JSONResponse: + """Creates a JSONResponse with Anthropic-formatted error.""" + err_detail = AnthropicErrorDetail(type=error_type, message=message) + if provider_details: + err_detail.provider = provider_details.provider_name + if provider_details.raw_error: + if isinstance(provider_details.raw_error, dict): + prov_err_obj = provider_details.raw_error.get("error") + if isinstance(prov_err_obj, dict): + err_detail.provider_message = prov_err_obj.get("message") + err_detail.provider_code = prov_err_obj.get("code") + elif isinstance(provider_details.raw_error.get("message"), str): + err_detail.provider_message = provider_details.raw_error.get( + "message" + ) + err_detail.provider_code = provider_details.raw_error.get("code") + + error_resp_model = AnthropicErrorResponse(error=err_detail) + return JSONResponse( + status_code=status_code, content=error_resp_model.model_dump(exclude_unset=True) + ) + + +async def _log_and_return_error_response( + request: Request, + status_code: int, + anthropic_error_type: AnthropicErrorType, + error_message: str, + provider_details: Optional[ProviderErrorMetadata] = None, + caught_exception: Optional[Exception] = None, +) -> JSONResponse: + request_id = getattr(request.state, "request_id", "unknown") + start_time_mono = getattr(request.state, "start_time_monotonic", time.monotonic()) + duration_ms = (time.monotonic() - start_time_mono) * 1000 + + log_data = { + "status_code": status_code, + "duration_ms": duration_ms, + "error_type": anthropic_error_type.value, + "client_ip": request.client.host if request.client else "unknown", + } + if provider_details: + log_data["provider_name"] = provider_details.provider_name + log_data["provider_raw_error"] = provider_details.raw_error + + error( + LogRecord( + event=LogEvent.REQUEST_FAILURE.value, + message=f"Request failed: {error_message}", + request_id=request_id, + data=log_data, + ), + exc=caught_exception, + ) + return _build_anthropic_error_response( + anthropic_error_type, error_message, status_code, provider_details + ) + + +@app.post("/v1/messages", response_model=None, tags=["API"], status_code=200) +async def create_message_proxy( + request: Request, +) -> Union[JSONResponse, StreamingResponse]: + """ + Main endpoint for Anthropic message completions, proxied to an OpenAI-compatible API. + Handles request/response conversions, streaming, and dynamic model selection. + """ + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + request.state.start_time_monotonic = time.monotonic() + + try: + raw_body = await request.json() + debug( + LogRecord( + LogEvent.ANTHROPIC_REQUEST.value, + "Received Anthropic request body", + request_id, + {"body": raw_body}, + ) + ) + + anthropic_request = MessagesRequest.model_validate( + raw_body, context={"request_id": request_id} + ) + except json.JSONDecodeError as e: + return await _log_and_return_error_response( + request, + 400, + AnthropicErrorType.INVALID_REQUEST, + "Invalid JSON body.", + caught_exception=e, + ) + except ValidationError as e: + return await _log_and_return_error_response( + request, + 422, + AnthropicErrorType.INVALID_REQUEST, + f"Invalid request body: {e.errors()}", + caught_exception=e, + ) + + is_stream = anthropic_request.stream or False + target_model_name = select_target_model(anthropic_request.model, request_id) + + estimated_input_tokens = count_tokens_for_anthropic_request( + messages=anthropic_request.messages, + system=anthropic_request.system, + model_name=anthropic_request.model, + tools=anthropic_request.tools, + request_id=request_id, + ) + + info( + LogRecord( + event=LogEvent.REQUEST_START.value, + message="Processing new message request", + request_id=request_id, + data={ + "client_model": anthropic_request.model, + "target_model": target_model_name, + "stream": is_stream, + "estimated_input_tokens": estimated_input_tokens, + "client_ip": request.client.host if request.client else "unknown", + "user_agent": request.headers.get("user-agent", "unknown"), + }, + ) + ) + + try: + openai_messages = convert_anthropic_to_openai_messages( + anthropic_request.messages, anthropic_request.system, request_id=request_id + ) + openai_tools = convert_anthropic_tools_to_openai(anthropic_request.tools) + openai_tool_choice = convert_anthropic_tool_choice_to_openai( + anthropic_request.tool_choice, request_id + ) + except Exception as e: + return await _log_and_return_error_response( + request, + 500, + AnthropicErrorType.API_ERROR, + "Error during request conversion.", + caught_exception=e, + ) + + openai_params: Dict[str, Any] = { + "model": target_model_name, + "messages": cast(List[ChatCompletionMessageParam], openai_messages), + "max_tokens": anthropic_request.max_tokens, + "stream": is_stream, + } + if anthropic_request.temperature is not None: + openai_params["temperature"] = anthropic_request.temperature + if anthropic_request.top_p is not None: + openai_params["top_p"] = anthropic_request.top_p + if anthropic_request.stop_sequences: + openai_params["stop"] = anthropic_request.stop_sequences + if openai_tools: + openai_params["tools"] = cast( + Optional[List[ChatCompletionToolParam]], openai_tools + ) + if openai_tool_choice: + openai_params["tool_choice"] = openai_tool_choice + if anthropic_request.metadata and anthropic_request.metadata.get("user_id"): + openai_params["user"] = str(anthropic_request.metadata.get("user_id")) + + debug( + LogRecord( + LogEvent.OPENAI_REQUEST.value, + "Prepared OpenAI request parameters", + request_id, + {"params": openai_params}, + ) + ) + + try: + if is_stream: + debug( + LogRecord( + LogEvent.STREAMING_REQUEST.value, + "Initiating streaming request to OpenAI-compatible API", + request_id, + ) + ) + openai_stream_response = await openai_client.chat.completions.create( + **openai_params + ) + return StreamingResponse( + handle_anthropic_streaming_response_from_openai_stream( + openai_stream_response, + anthropic_request.model, + estimated_input_tokens, + request_id, + request.state.start_time_monotonic, + ), + media_type="text/event-stream", + ) + else: + debug( + LogRecord( + LogEvent.OPENAI_REQUEST.value, + "Sending non-streaming request to OpenAI-compatible API", + request_id, + ) + ) + openai_response_obj = await openai_client.chat.completions.create( + **openai_params + ) + + debug( + LogRecord( + LogEvent.OPENAI_RESPONSE.value, + "Received OpenAI response", + request_id, + {"response": openai_response_obj.model_dump()}, + ) + ) + + anthropic_response_obj = convert_openai_to_anthropic_response( + openai_response_obj, anthropic_request.model, request_id=request_id + ) + duration_ms = (time.monotonic() - request.state.start_time_monotonic) * 1000 + info( + LogRecord( + event=LogEvent.REQUEST_COMPLETED.value, + message="Non-streaming request completed successfully", + request_id=request_id, + data={ + "status_code": 200, + "duration_ms": duration_ms, + "input_tokens": anthropic_response_obj.usage.input_tokens, + "output_tokens": anthropic_response_obj.usage.output_tokens, + "stop_reason": anthropic_response_obj.stop_reason, + }, + ) + ) + debug( + LogRecord( + LogEvent.ANTHROPIC_RESPONSE.value, + "Prepared Anthropic response", + request_id, + {"response": anthropic_response_obj.model_dump(exclude_unset=True)}, + ) + ) + return JSONResponse( + content=anthropic_response_obj.model_dump(exclude_unset=True) + ) + + except openai.APIError as e: + err_type, err_msg, err_status, prov_details = ( + _get_anthropic_error_details_from_exc(e) + ) + return await _log_and_return_error_response( + request, err_status, err_type, err_msg, prov_details, e + ) + except Exception as e: + return await _log_and_return_error_response( + request, + 500, + AnthropicErrorType.API_ERROR, + "An unexpected error occurred while processing the request.", + caught_exception=e, + ) + + +@app.post( + "/v1/messages/count_tokens", response_model=TokenCountResponse, tags=["Utility"] +) +async def count_tokens_endpoint(request: Request) -> TokenCountResponse: + """Estimates token count for given Anthropic messages and system prompt.""" + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + start_time_mono = time.monotonic() + + try: + body = await request.json() + count_request = TokenCountRequest.model_validate(body) + except json.JSONDecodeError as e: + raise fastapi.HTTPException(status_code=400, detail="Invalid JSON body.") from e + except ValidationError as e: + raise fastapi.HTTPException( + status_code=422, detail=f"Invalid request body: {e.errors()}" + ) from e + + token_count = count_tokens_for_anthropic_request( + messages=count_request.messages, + system=count_request.system, + model_name=count_request.model, + tools=count_request.tools, + request_id=request_id, + ) + duration_ms = (time.monotonic() - start_time_mono) * 1000 + info( + LogRecord( + event=LogEvent.TOKEN_COUNT.value, + message=f"Counted {token_count} tokens", + request_id=request_id, + data={ + "duration_ms": duration_ms, + "token_count": token_count, + "model": count_request.model, + }, + ) + ) + return TokenCountResponse(input_tokens=token_count) + + +@app.get("/", include_in_schema=False, tags=["Health"]) +async def root_health_check() -> JSONResponse: + """Basic health check and information endpoint.""" + debug( + LogRecord( + event=LogEvent.HEALTH_CHECK.value, message="Root health check accessed" + ) + ) + return JSONResponse( + { + "proxy_name": settings.app_name, + "version": settings.app_version, + "status": "ok", + "timestamp": datetime.now(timezone.utc).isoformat(), + } + ) + + +@app.exception_handler(openai.APIError) +async def openai_api_error_handler(request: Request, exc: openai.APIError): + err_type, err_msg, err_status, prov_details = _get_anthropic_error_details_from_exc( + exc + ) + return await _log_and_return_error_response( + request, err_status, err_type, err_msg, prov_details, exc + ) + + +@app.exception_handler(ValidationError) +async def pydantic_validation_error_handler(request: Request, exc: ValidationError): + return await _log_and_return_error_response( + request, + 422, + AnthropicErrorType.INVALID_REQUEST, + f"Validation error: {exc.errors()}", + caught_exception=exc, + ) + + +@app.exception_handler(json.JSONDecodeError) +async def json_decode_error_handler(request: Request, exc: json.JSONDecodeError): + return await _log_and_return_error_response( + request, + 400, + AnthropicErrorType.INVALID_REQUEST, + "Invalid JSON format.", + caught_exception=exc, + ) + + +@app.exception_handler(Exception) +async def generic_exception_handler(request: Request, exc: Exception): + return await _log_and_return_error_response( + request, + 500, + AnthropicErrorType.API_ERROR, + "An unexpected internal server error occurred.", + caught_exception=exc, + ) + + +@app.middleware("http") +async def logging_middleware( + request: Request, call_next: Callable[[Request], Awaitable[Response]] +) -> Response: + if not hasattr(request.state, "request_id"): + request.state.request_id = str(uuid.uuid4()) + if not hasattr(request.state, "start_time_monotonic"): + request.state.start_time_monotonic = time.monotonic() + + response = await call_next(request) + + response.headers["X-Request-ID"] = request.state.request_id + duration_ms = (time.monotonic() - request.state.start_time_monotonic) * 1000 + response.headers["X-Response-Time-ms"] = str(duration_ms) + + return response + + +if __name__ == "__main__": + _console.print( + r"""[bold blue] + /$$ /$$ + | $$ | $$ + /$$$$$$$| $$ /$$$$$$ /$$ /$$ /$$$$$$$ /$$$$$$ /$$$$$$ /$$$$$$ /$$$$$$ /$$ /$$ /$$ /$$ + /$$_____/| $$ |____ $$| $$ | $$ /$$__ $$ /$$__ $$ /$$__ $$ /$$__ $$ /$$__ $$| $$ /$$/| $$ | $$ +| $$ | $$ /$$$$$$$| $$ | $$| $$ | $$| $$$$$$$$ | $$ \ $$| $$ \__/| $$ \ $$ \ $$$$/ | $$ | $$ +| $$ | $$ /$$__ $$| $$ | $$| $$ | $$| $$_____/ | $$ | $$| $$ | $$ | $$ >$$ $$ | $$ | $$ +| $$$$$$$| $$| $$$$$$$| $$$$$$/| $$$$$$$| $$$$$$$ | $$$$$$$/| $$ | $$$$$$/ /$$/\ $$| $$$$$$$ + \_______/|__/ \_______/ \______/ \_______/ \_______/ | $$____/ |__/ \______/ |__/ \__/ \____ $$ + | $$ /$$ | $$ + | $$ | $$$$$$/ + |__/ \______/ + [/]""", + justify="left", + ) + config_details_text = Text.assemble( + (" Version : ", "default"), + (f"v{settings.app_version}", "bold cyan"), + ("\n Big Model : ", "default"), + (settings.big_model_name, "magenta"), + ("\n Small Model : ", "default"), + (settings.small_model_name, "green"), + ("\n Log Level : ", "default"), + (settings.log_level.upper(), "yellow"), + ("\n Log File : ", "default"), + (settings.log_file_path or "Disabled", "dim"), + ("\n Listening on : ", "default"), + (f"http://{settings.host}:{settings.port}", "bold white"), + ("\n Reload : ", "default"), + ("Enabled", "bold orange1") if settings.reload else ("Disabled", "dim"), + ) + _console.print( + Panel( + config_details_text, + title="Anthropic Proxy Configuration", + border_style="blue", + expand=False, + ) + ) + _console.print(Rule("Starting Uvicorn server...", style="dim blue")) + + uvicorn.run( + "__main__:app", + host=settings.host, + port=settings.port, + reload=settings.reload, + log_config=None, + access_log=False, + ) diff --git a/src/routes/anthropic/handler.ts b/src/routes/anthropic/handler.ts new file mode 100644 index 000000000..22c03cbbd --- /dev/null +++ b/src/routes/anthropic/handler.ts @@ -0,0 +1,241 @@ +import type { Context } from "hono" +import { randomUUID } from "node:crypto" +import { streamSSE } from "hono/streaming" +import consola from "consola" + +import { awaitApproval } from "~/lib/approval" +import { checkRateLimit } from "~/lib/rate-limit" +import { state } from "~/lib/state" +import { getTokenCount } from "~/lib/tokenizer" +import { isNullish } from "~/lib/is-nullish" +import { HTTPError } from "~/lib/http-error" + +import type { + AnthropicMessagesRequest, + AnthropicMessagesResponse, + AnthropicTokenCountRequest, + AnthropicTokenCountResponse, + AnthropicErrorResponse +} from "~/types/anthropic" + +import { + convertAnthropicToOpenAIMessages, + convertAnthropicToolsToOpenAI, + convertAnthropicToolChoiceToOpenAI, + convertOpenAIToAnthropicResponse +} from "~/services/anthropic/converters" + +import { convertOpenAIStreamToAnthropic } from "~/services/anthropic/streaming" + +import { + createChatCompletions, + type ChatCompletionsPayload +} from "~/services/copilot/create-chat-completions" + +export async function handleAnthropicMessages(c: Context) { + await checkRateLimit(state) + + const requestId = randomUUID() + + try { + const anthropicRequest = await c.req.json() + + consola.info("Received Anthropic messages request", anthropicRequest) + + if (anthropicRequest.messages) { + const tokenCount = getTokenCount( + convertAnthropicToOpenAIMessages(anthropicRequest.messages, anthropicRequest.system) + ) + consola.info("Estimated token count:", tokenCount) + } + + if (state.manualApprove) { + await awaitApproval() + } + + // Convert Anthropic request to OpenAI format + const openaiMessages = convertAnthropicToOpenAIMessages( + anthropicRequest.messages, + anthropicRequest.system + ) + + const openaiTools = convertAnthropicToolsToOpenAI(anthropicRequest.tools) + const openaiToolChoice = convertAnthropicToolChoiceToOpenAI(anthropicRequest.tool_choice) + + // Build OpenAI payload + let openaiPayload: ChatCompletionsPayload = { + model: selectCopilotModel(anthropicRequest.model), + messages: openaiMessages, + stream: anthropicRequest.stream || false + } + + // Set max_tokens + if (isNullish(anthropicRequest.max_tokens)) { + const selectedModel = state.models?.data.find( + (model) => model.id === openaiPayload.model + ) + openaiPayload.max_tokens = selectedModel?.capabilities.limits.max_output_tokens + } else { + openaiPayload.max_tokens = anthropicRequest.max_tokens + } + + // Add optional parameters + if (anthropicRequest.temperature !== undefined) { + openaiPayload.temperature = anthropicRequest.temperature + } + if (anthropicRequest.top_p !== undefined) { + openaiPayload.top_p = anthropicRequest.top_p + } + if (anthropicRequest.stop_sequences) { + openaiPayload.stop = anthropicRequest.stop_sequences + } + if (openaiTools) { + openaiPayload.tools = openaiTools + } + if (openaiToolChoice) { + openaiPayload.tool_choice = openaiToolChoice + } + + consola.debug("Converted to OpenAI payload", { + model: openaiPayload.model, + messageCount: openaiPayload.messages.length, + hasTools: Boolean(openaiPayload.tools), + requestId + }) + + const response = await createChatCompletions(openaiPayload) + + // Handle streaming response + if (anthropicRequest.stream && isAsyncIterable(response)) { + return streamSSE(c, async (stream) => { + const estimatedInputTokens = getTokenCount(openaiMessages) + + for await (const sseEvent of convertOpenAIStreamToAnthropic( + response, + anthropicRequest.model, + estimatedInputTokens, + requestId + )) { + await stream.write(sseEvent) + } + }) + } + + // Handle non-streaming response + if (isNonStreamingResponse(response)) { + const anthropicResponse = convertOpenAIToAnthropicResponse( + response, + anthropicRequest.model, + requestId + ) + + consola.info("Anthropic messages request completed", { + model: anthropicResponse.model, + stopReason: anthropicResponse.stop_reason, + inputTokens: anthropicResponse.usage.input_tokens, + outputTokens: anthropicResponse.usage.output_tokens, + requestId + }) + + return c.json(anthropicResponse) + } + + throw new Error("Unexpected response type from OpenAI") + + } catch (error) { + consola.error("Error handling Anthropic messages request:", error) + + if (error instanceof HTTPError) { + const errorResponse: AnthropicErrorResponse = { + type: "error", + error: { + type: error.response.status >= 400 && error.response.status < 500 + ? "invalid_request_error" + : "api_error", + message: error.message + } + } + return c.json(errorResponse, error.response.status) + } + + const errorResponse: AnthropicErrorResponse = { + type: "error", + error: { + type: "api_error", + message: error instanceof Error ? error.message : "An unexpected error occurred" + } + } + return c.json(errorResponse, 500) + } +} + +export async function handleAnthropicTokenCount(c: Context) { + try { + const request = await c.req.json() + + consola.info("Received Anthropic token count request", { + model: request.model, + messageCount: request.messages.length + }) + + const openaiMessages = convertAnthropicToOpenAIMessages( + request.messages, + request.system + ) + + const tokenCount = getTokenCount(openaiMessages) + + const response: AnthropicTokenCountResponse = { + input_tokens: tokenCount + } + + consola.info("Token count completed", { + tokens: tokenCount, + model: request.model + }) + + return c.json(response) + + } catch (error) { + consola.error("Error counting tokens:", error) + + const errorResponse: AnthropicErrorResponse = { + type: "error", + error: { + type: "invalid_request_error", + message: error instanceof Error ? error.message : "Failed to count tokens" + } + } + return c.json(errorResponse, 400) + } +} + +function selectCopilotModel(anthropicModel: string): string { + // Map Anthropic model names to available Copilot models + const modelName = anthropicModel.toLowerCase() + + if (!state.models?.data) { + // Fallback to a default model name if models aren't cached + return "claude-3-5-sonnet-20241022" + } + + // Try to find a Claude model first + const claudeModel = state.models.data.find(model => + model.id.toLowerCase().includes("claude") + ) + + if (claudeModel) { + return claudeModel.id + } + + // Fallback to first available model + return state.models.data[0]?.id || "claude-3-5-sonnet-20241022" +} + +function isAsyncIterable(obj: any): obj is AsyncIterable { + return obj != null && typeof obj[Symbol.asyncIterator] === "function" +} + +function isNonStreamingResponse(response: any): response is import("~/services/copilot/create-chat-completions").ChatCompletionResponse { + return response && typeof response === "object" && "choices" in response +} \ No newline at end of file diff --git a/src/routes/anthropic/route.ts b/src/routes/anthropic/route.ts new file mode 100644 index 000000000..5789a3ed3 --- /dev/null +++ b/src/routes/anthropic/route.ts @@ -0,0 +1,10 @@ +import { Hono } from "hono" +import { handleAnthropicMessages, handleAnthropicTokenCount } from "./handler" + +export const anthropicRoutes = new Hono() + +// Main Anthropic messages endpoint +anthropicRoutes.post("/", handleAnthropicMessages) + +// Token counting endpoint +anthropicRoutes.post("/count_tokens", handleAnthropicTokenCount) \ No newline at end of file diff --git a/src/server.ts b/src/server.ts index eb65371bf..ff50e71b2 100644 --- a/src/server.ts +++ b/src/server.ts @@ -5,6 +5,7 @@ import { logger } from "hono/logger" import { completionRoutes } from "./routes/chat-completions/route" import { embeddingRoutes } from "./routes/embeddings/route" import { modelRoutes } from "./routes/models/route" +import { anthropicRoutes } from "./routes/anthropic/route" export const server = new Hono() @@ -13,6 +14,7 @@ server.use(cors()) server.get("/", (c) => c.text("Server running")) +// OpenAI-compatible endpoints server.route("/chat/completions", completionRoutes) server.route("/models", modelRoutes) server.route("/embeddings", embeddingRoutes) @@ -21,3 +23,6 @@ server.route("/embeddings", embeddingRoutes) server.route("/v1/chat/completions", completionRoutes) server.route("/v1/models", modelRoutes) server.route("/v1/embeddings", embeddingRoutes) + +// Anthropic-compatible endpoints +server.route("/v1/messages", anthropicRoutes) diff --git a/src/services/anthropic/converters.ts b/src/services/anthropic/converters.ts new file mode 100644 index 000000000..ce23ee322 --- /dev/null +++ b/src/services/anthropic/converters.ts @@ -0,0 +1,356 @@ +import { randomUUID } from "node:crypto" +import type { + AnthropicMessage, + AnthropicTool, + AnthropicToolChoice, + ContentBlock, + ContentBlockText, + ContentBlockImage, + ContentBlockToolUse, + ContentBlockToolResult, + SystemContent, + AnthropicMessagesResponse, + AnthropicUsage +} from "~/types/anthropic" +import type { + Message, + Tool, + ToolChoice, + ChatCompletionResponse, + ContentPart, + ToolCall +} from "~/services/copilot/create-chat-completions" + +export function convertAnthropicToOpenAIMessages( + anthropicMessages: Array, + anthropicSystem?: string | Array +): Array { + const openaiMessages: Array = [] + + // Handle system message + let systemTextContent = "" + if (typeof anthropicSystem === "string") { + systemTextContent = anthropicSystem + } else if (Array.isArray(anthropicSystem)) { + const systemTexts = anthropicSystem + .filter((block): block is SystemContent => block.type === "text") + .map(block => block.text) + systemTextContent = systemTexts.join("\n") + } + + if (systemTextContent) { + openaiMessages.push({ + role: "system", + content: systemTextContent + }) + } + + // Convert messages + for (const msg of anthropicMessages) { + const role = msg.role + const content = msg.content + + if (typeof content === "string") { + openaiMessages.push({ + role, + content + }) + continue + } + + if (Array.isArray(content)) { + const openaiPartsForUserMessage: Array = [] + const assistantToolCalls: Array = [] + const textContentForAssistant: Array = [] + + if (content.length === 0) { + openaiMessages.push({ role, content: "" }) + continue + } + + for (const block of content) { + if (isContentBlockText(block)) { + if (role === "user") { + openaiPartsForUserMessage.push({ + type: "text", + text: block.text + }) + } else if (role === "assistant") { + textContentForAssistant.push(block.text) + } + } else if (isContentBlockImage(block) && role === "user") { + if (block.source.type === "base64") { + openaiPartsForUserMessage.push({ + type: "image_url", + image_url: `data:${block.source.media_type};base64,${block.source.data}` + }) + } + } else if (isContentBlockToolUse(block) && role === "assistant") { + try { + const argsStr = JSON.stringify(block.input) + assistantToolCalls.push({ + id: block.id, + type: "function", + function: { + name: block.name, + arguments: argsStr + } + }) + } catch (e) { + console.warn(`Failed to serialize tool input for ${block.name}:`, e) + assistantToolCalls.push({ + id: block.id, + type: "function", + function: { + name: block.name, + arguments: "{}" + } + }) + } + } else if (isContentBlockToolResult(block) && role === "user") { + const serializedContent = serializeToolResultContent(block.content) + openaiMessages.push({ + role: "tool", + content: serializedContent, + tool_call_id: block.tool_use_id + }) + } + } + + // Handle user message with multimodal content + if (role === "user" && openaiPartsForUserMessage.length > 0) { + const isMultimodal = openaiPartsForUserMessage.some(part => part.type === "image_url") + if (isMultimodal || openaiPartsForUserMessage.length > 1) { + openaiMessages.push({ + role: "user", + content: openaiPartsForUserMessage + }) + } else if (openaiPartsForUserMessage.length === 1 && openaiPartsForUserMessage[0].type === "text") { + openaiMessages.push({ + role: "user", + content: openaiPartsForUserMessage[0].text || "" + }) + } + } + + // Handle assistant message with text and/or tool calls + if (role === "assistant") { + const assistantText = textContentForAssistant.filter(Boolean).join("\n") + + if (assistantText && assistantToolCalls.length > 0) { + // Text and tool calls - need separate messages + openaiMessages.push({ + role: "assistant", + content: assistantText + }) + openaiMessages.push({ + role: "assistant", + content: null, + tool_calls: assistantToolCalls + }) + } else if (assistantText) { + // Just text + openaiMessages.push({ + role: "assistant", + content: assistantText + }) + } else if (assistantToolCalls.length > 0) { + // Just tool calls + openaiMessages.push({ + role: "assistant", + content: null, + tool_calls: assistantToolCalls + }) + } else { + // Empty message + openaiMessages.push({ + role: "assistant", + content: "" + }) + } + } + } + } + + return openaiMessages +} + +export function convertAnthropicToolsToOpenAI( + anthropicTools?: Array +): Array | undefined { + if (!anthropicTools || anthropicTools.length === 0) { + return undefined + } + + return anthropicTools.map(tool => ({ + type: "function" as const, + function: { + name: tool.name, + description: tool.description || "", + parameters: tool.input_schema + } + })) +} + +export function convertAnthropicToolChoiceToOpenAI( + anthropicChoice?: AnthropicToolChoice +): "auto" | "none" | ToolChoice | undefined { + if (!anthropicChoice) { + return undefined + } + + if (anthropicChoice.type === "auto") { + return "auto" + } + if (anthropicChoice.type === "any") { + // Map 'any' to 'auto' as closest equivalent + return "auto" + } + if (anthropicChoice.type === "tool" && anthropicChoice.name) { + return { + type: "function", + function: { + name: anthropicChoice.name + } + } + } + + return "auto" +} + +export function convertOpenAIToAnthropicResponse( + openaiResponse: ChatCompletionResponse, + originalAnthropicModel: string, + requestId?: string +): AnthropicMessagesResponse { + const anthropicContent: Array = [] + let anthropicStopReason: AnthropicMessagesResponse["stop_reason"] = "end_turn" + + const stopReasonMap: Record = { + "stop": "end_turn", + "length": "max_tokens", + "tool_calls": "tool_use", + "function_call": "tool_use", + "content_filter": "stop_sequence" + } + + if (openaiResponse.choices && openaiResponse.choices.length > 0) { + const choice = openaiResponse.choices[0] + const message = choice.message + const finishReason = choice.finish_reason + + anthropicStopReason = stopReasonMap[finishReason] || "end_turn" + + if (message.content) { + anthropicContent.push({ + type: "text", + text: message.content + }) + } + + if (message.tool_calls) { + for (const call of message.tool_calls) { + if (call.type === "function") { + let toolInputDict: Record = {} + try { + const parsedInput = JSON.parse(call.function.arguments) + if (typeof parsedInput === "object" && parsedInput !== null) { + toolInputDict = parsedInput + } else { + toolInputDict = { value: parsedInput } + } + } catch (e) { + console.warn(`Failed to parse tool arguments for ${call.function.name}:`, e) + toolInputDict = { error_parsing_arguments: call.function.arguments } + } + + anthropicContent.push({ + type: "tool_use", + id: call.id, + name: call.function.name, + input: toolInputDict + }) + } + } + if (finishReason === "tool_calls") { + anthropicStopReason = "tool_use" + } + } + } + + if (anthropicContent.length === 0) { + anthropicContent.push({ + type: "text", + text: "" + }) + } + + const usage: AnthropicUsage = { + input_tokens: 0, + output_tokens: 0 + } + + const responseId = openaiResponse.id ? + `msg_${openaiResponse.id}` : + `msg_${requestId || randomUUID()}_completed` + + return { + id: responseId, + type: "message", + role: "assistant", + model: originalAnthropicModel, + content: anthropicContent, + stop_reason: anthropicStopReason, + usage + } +} + +function serializeToolResultContent( + content: string | Array> | Array +): string { + if (typeof content === "string") { + return content + } + + if (Array.isArray(content)) { + const processedParts: Array = [] + for (const item of content) { + if (typeof item === "object" && item !== null && item.type === "text" && "text" in item) { + processedParts.push(String(item.text)) + } else { + try { + processedParts.push(JSON.stringify(item)) + } catch { + processedParts.push(``) + } + } + } + return processedParts.join("\n") + } + + try { + return JSON.stringify(content) + } catch { + return JSON.stringify({ + error: "Serialization failed", + original_type: typeof content + }) + } +} + +// Type guards +function isContentBlockText(block: ContentBlock): block is ContentBlockText { + return block.type === "text" +} + +function isContentBlockImage(block: ContentBlock): block is ContentBlockImage { + return block.type === "image" +} + +function isContentBlockToolUse(block: ContentBlock): block is ContentBlockToolUse { + return block.type === "tool_use" +} + +function isContentBlockToolResult(block: ContentBlock): block is ContentBlockToolResult { + return block.type === "tool_result" +} \ No newline at end of file diff --git a/src/services/anthropic/streaming.ts b/src/services/anthropic/streaming.ts new file mode 100644 index 000000000..2109e0da6 --- /dev/null +++ b/src/services/anthropic/streaming.ts @@ -0,0 +1,269 @@ +import { randomUUID } from "node:crypto" +import type { + AnthropicStreamEvent, + AnthropicMessageStartEvent, + AnthropicContentBlockStartEvent, + AnthropicContentBlockDeltaEvent, + AnthropicContentBlockStopEvent, + AnthropicMessageDeltaEvent, + AnthropicMessageStopEvent, + AnthropicPingEvent, + AnthropicErrorEvent +} from "~/types/anthropic" +import type { ChatCompletionChunk } from "~/services/copilot/create-chat-completions" +import { getTokenCount } from "~/lib/tokenizer" + +export async function* convertOpenAIStreamToAnthropic( + openaiStream: AsyncIterable, + originalAnthropicModel: string, + estimatedInputTokens: number, + requestId: string +): AsyncGenerator { + const anthropicMessageId = `msg_stream_${requestId}_${randomUUID().slice(0, 8)}` + + let nextAnthropicBlockIdx = 0 + let textBlockAnthropicIdx: number | null = null + const openaiToolIdxToAnthropicBlockIdx: Map = new Map() + const toolStates: Map = new Map() + const sentToolBlockStarts = new Set() + + let outputTokenCount = 0 + let finalAnthropicStopReason: "end_turn" | "max_tokens" | "stop_sequence" | "tool_use" | "error" = "end_turn" + + const openaiToAnthropicStopReasonMap: Record = { + "stop": "end_turn", + "length": "max_tokens", + "tool_calls": "tool_use", + "function_call": "tool_use", + "content_filter": "stop_sequence" + } + + try { + // Send message_start event + const messageStartEvent: AnthropicMessageStartEvent = { + type: "message_start", + message: { + id: anthropicMessageId, + type: "message", + role: "assistant", + model: originalAnthropicModel, + content: [], + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: estimatedInputTokens, output_tokens: 0 } + } + } + yield `event: message_start\ndata: ${JSON.stringify(messageStartEvent)}\n\n` + + // Send ping event + const pingEvent: AnthropicPingEvent = { type: "ping" } + yield `event: ping\ndata: ${JSON.stringify(pingEvent)}\n\n` + + for await (const chunk of openaiStream) { + // Parse the chunk - events() from fetch-event-stream returns parsed objects + let parsedChunk: ChatCompletionChunk + if (typeof chunk === "string") { + try { + // Handle SSE format: "data: {...}" + const dataMatch = chunk.match(/^data: (.+)$/) + if (dataMatch) { + parsedChunk = JSON.parse(dataMatch[1]) + } else { + parsedChunk = JSON.parse(chunk) + } + } catch (e) { + console.warn("Failed to parse chunk:", chunk) + continue + } + } else if (chunk && typeof chunk === "object" && chunk.data) { + // Handle fetch-event-stream format where chunk.data contains the actual data + try { + parsedChunk = typeof chunk.data === "string" ? JSON.parse(chunk.data) : chunk.data + } catch (e) { + console.warn("Failed to parse chunk.data:", chunk.data) + continue + } + } else { + parsedChunk = chunk as ChatCompletionChunk + } + + if (!parsedChunk.choices || parsedChunk.choices.length === 0) { + continue + } + + const delta = parsedChunk.choices[0].delta + const openaiFinishReason = parsedChunk.choices[0].finish_reason + + // Handle text content + if (delta.content) { + outputTokenCount += estimateTokenCount(delta.content) + + if (textBlockAnthropicIdx === null) { + textBlockAnthropicIdx = nextAnthropicBlockIdx + nextAnthropicBlockIdx += 1 + + const startTextEvent: AnthropicContentBlockStartEvent = { + type: "content_block_start", + index: textBlockAnthropicIdx, + content_block: { type: "text", text: "" } + } + yield `event: content_block_start\ndata: ${JSON.stringify(startTextEvent)}\n\n` + } + + const textDeltaEvent: AnthropicContentBlockDeltaEvent = { + type: "content_block_delta", + index: textBlockAnthropicIdx, + delta: { type: "text_delta", text: delta.content } + } + yield `event: content_block_delta\ndata: ${JSON.stringify(textDeltaEvent)}\n\n` + } + + // Handle tool calls + if (delta.tool_calls) { + for (const toolDelta of delta.tool_calls) { + const openaiTcIdx = toolDelta.index + + if (!openaiToolIdxToAnthropicBlockIdx.has(openaiTcIdx)) { + const currentAnthropicToolBlockIdx = nextAnthropicBlockIdx + nextAnthropicBlockIdx += 1 + openaiToolIdxToAnthropicBlockIdx.set(openaiTcIdx, currentAnthropicToolBlockIdx) + + toolStates.set(currentAnthropicToolBlockIdx, { + id: toolDelta.id || `tool_ph_${requestId}_${currentAnthropicToolBlockIdx}`, + name: "", + argumentsBuffer: "" + }) + } + + const currentAnthropicToolBlockIdx = openaiToolIdxToAnthropicBlockIdx.get(openaiTcIdx)! + const toolState = toolStates.get(currentAnthropicToolBlockIdx)! + + // Update tool state + if (toolDelta.id && toolState.id.startsWith("tool_ph_")) { + toolState.id = toolDelta.id + } + + if (toolDelta.function) { + if (toolDelta.function.name) { + toolState.name = toolDelta.function.name + } + if (toolDelta.function.arguments) { + toolState.argumentsBuffer += toolDelta.function.arguments + outputTokenCount += estimateTokenCount(toolDelta.function.arguments) + } + } + + // Send content_block_start if we have enough info and haven't sent it yet + if ( + !sentToolBlockStarts.has(currentAnthropicToolBlockIdx) && + toolState.id && + !toolState.id.startsWith("tool_ph_") && + toolState.name + ) { + const startToolEvent: AnthropicContentBlockStartEvent = { + type: "content_block_start", + index: currentAnthropicToolBlockIdx, + content_block: { + type: "tool_use", + id: toolState.id, + name: toolState.name, + input: {} + } + } + yield `event: content_block_start\ndata: ${JSON.stringify(startToolEvent)}\n\n` + sentToolBlockStarts.add(currentAnthropicToolBlockIdx) + } + + // Send delta if we have arguments and have started the block + if ( + toolDelta.function?.arguments && + sentToolBlockStarts.has(currentAnthropicToolBlockIdx) + ) { + const argsDeltaEvent: AnthropicContentBlockDeltaEvent = { + type: "content_block_delta", + index: currentAnthropicToolBlockIdx, + delta: { + type: "input_json_delta", + partial_json: toolDelta.function.arguments + } + } + yield `event: content_block_delta\ndata: ${JSON.stringify(argsDeltaEvent)}\n\n` + } + } + } + + // Handle finish reason + if (openaiFinishReason) { + finalAnthropicStopReason = openaiToAnthropicStopReasonMap[openaiFinishReason] || "end_turn" + if (openaiFinishReason === "tool_calls") { + finalAnthropicStopReason = "tool_use" + } + break + } + } + + // Send content_block_stop events + if (textBlockAnthropicIdx !== null) { + const stopEvent: AnthropicContentBlockStopEvent = { + type: "content_block_stop", + index: textBlockAnthropicIdx + } + yield `event: content_block_stop\ndata: ${JSON.stringify(stopEvent)}\n\n` + } + + for (const anthropicToolIdx of sentToolBlockStarts) { + const toolState = toolStates.get(anthropicToolIdx) + if (toolState) { + try { + JSON.parse(toolState.argumentsBuffer) + } catch { + console.warn(`Invalid JSON in tool arguments for ${toolState.name}`) + } + } + + const stopEvent: AnthropicContentBlockStopEvent = { + type: "content_block_stop", + index: anthropicToolIdx + } + yield `event: content_block_stop\ndata: ${JSON.stringify(stopEvent)}\n\n` + } + + // Send message_delta event + const messageDeltaEvent: AnthropicMessageDeltaEvent = { + type: "message_delta", + delta: { + stop_reason: finalAnthropicStopReason, + stop_sequence: undefined + }, + usage: { output_tokens: outputTokenCount } + } + yield `event: message_delta\ndata: ${JSON.stringify(messageDeltaEvent)}\n\n` + + // Send message_stop event + const messageStopEvent: AnthropicMessageStopEvent = { + type: "message_stop" + } + yield `event: message_stop\ndata: ${JSON.stringify(messageStopEvent)}\n\n` + + } catch (error) { + console.error("Error in stream conversion:", error) + + const errorEvent: AnthropicErrorEvent = { + type: "error", + error: { + type: "api_error", + message: error instanceof Error ? error.message : "An unexpected error occurred" + } + } + yield `event: error\ndata: ${JSON.stringify(errorEvent)}\n\n` + } +} + +function estimateTokenCount(text: string): number { + // Simple token estimation - roughly 4 characters per token + return Math.ceil(text.length / 4) +} \ No newline at end of file diff --git a/src/services/copilot/create-chat-completions.ts b/src/services/copilot/create-chat-completions.ts index 983a7ae53..a6b8a4d24 100644 --- a/src/services/copilot/create-chat-completions.ts +++ b/src/services/copilot/create-chat-completions.ts @@ -160,7 +160,7 @@ export interface Message { // https://platform.openai.com/docs/api-reference export interface ContentPart { - type: "input_image" | "input_text" | "image_url" + type: "input_image" | "text" | "image_url" text?: string image_url?: string } diff --git a/src/types/anthropic.ts b/src/types/anthropic.ts new file mode 100644 index 000000000..9485e5c43 --- /dev/null +++ b/src/types/anthropic.ts @@ -0,0 +1,184 @@ +// Anthropic API types based on the Python implementation + +export interface ContentBlockText { + type: "text" + text: string +} + +export interface ContentBlockImageSource { + type: string + media_type: string + data: string +} + +export interface ContentBlockImage { + type: "image" + source: ContentBlockImageSource +} + +export interface ContentBlockToolUse { + type: "tool_use" + id: string + name: string + input: Record +} + +export interface ContentBlockToolResult { + type: "tool_result" + tool_use_id: string + content: string | Array> | Array + is_error?: boolean +} + +export type ContentBlock = + | ContentBlockText + | ContentBlockImage + | ContentBlockToolUse + | ContentBlockToolResult + +export interface SystemContent { + type: "text" + text: string +} + +export interface AnthropicMessage { + role: "user" | "assistant" + content: string | Array +} + +export interface AnthropicTool { + name: string + description?: string + input_schema: Record +} + +export interface AnthropicToolChoice { + type: "auto" | "any" | "tool" + name?: string +} + +export interface AnthropicMessagesRequest { + model: string + max_tokens: number + messages: Array + system?: string | Array + stop_sequences?: Array + stream?: boolean + temperature?: number + top_p?: number + top_k?: number + metadata?: Record + tools?: Array + tool_choice?: AnthropicToolChoice +} + +export interface AnthropicUsage { + input_tokens: number + output_tokens: number +} + +export interface AnthropicMessagesResponse { + id: string + type: "message" + role: "assistant" + model: string + content: Array + stop_reason?: "end_turn" | "max_tokens" | "stop_sequence" | "tool_use" | "error" + stop_sequence?: string + usage: AnthropicUsage +} + +export interface AnthropicTokenCountRequest { + model: string + messages: Array + system?: string | Array + tools?: Array +} + +export interface AnthropicTokenCountResponse { + input_tokens: number +} + +export interface AnthropicErrorDetail { + type: "invalid_request_error" | "authentication_error" | "permission_error" | + "not_found_error" | "rate_limit_error" | "api_error" | "overloaded_error" | + "request_too_large_error" + message: string + provider?: string + provider_message?: string + provider_code?: string | number +} + +export interface AnthropicErrorResponse { + type: "error" + error: AnthropicErrorDetail +} + +// Streaming event types +export interface AnthropicStreamEvent { + type: string + [key: string]: any +} + +export interface AnthropicMessageStartEvent extends AnthropicStreamEvent { + type: "message_start" + message: { + id: string + type: "message" + role: "assistant" + model: string + content: Array + stop_reason: null + stop_sequence: null + usage: { input_tokens: number; output_tokens: number } + } +} + +export interface AnthropicContentBlockStartEvent extends AnthropicStreamEvent { + type: "content_block_start" + index: number + content_block: { + type: "text" | "tool_use" + text?: string + id?: string + name?: string + input?: Record + } +} + +export interface AnthropicContentBlockDeltaEvent extends AnthropicStreamEvent { + type: "content_block_delta" + index: number + delta: { + type: "text_delta" | "input_json_delta" + text?: string + partial_json?: string + } +} + +export interface AnthropicContentBlockStopEvent extends AnthropicStreamEvent { + type: "content_block_stop" + index: number +} + +export interface AnthropicMessageDeltaEvent extends AnthropicStreamEvent { + type: "message_delta" + delta: { + stop_reason: "end_turn" | "max_tokens" | "stop_sequence" | "tool_use" | "error" + stop_sequence?: string + } + usage: { output_tokens: number } +} + +export interface AnthropicMessageStopEvent extends AnthropicStreamEvent { + type: "message_stop" +} + +export interface AnthropicPingEvent extends AnthropicStreamEvent { + type: "ping" +} + +export interface AnthropicErrorEvent extends AnthropicStreamEvent { + type: "error" + error: AnthropicErrorDetail +} \ No newline at end of file From 84a44880be786a5f425c7277c00f489d15b9d514 Mon Sep 17 00:00:00 2001 From: "Tang, Tom" Date: Sun, 15 Jun 2025 17:10:07 +1000 Subject: [PATCH 3/9] docs: update README with Claude Code integration info MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove ko-fi support section - Add dedicated Claude Code integration section with setup instructions - Document how to configure Claude Code to use this proxy with environment variables 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 27729ef03..2c9122ea4 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,24 @@ ⚠️ **EDUCATIONAL PURPOSE ONLY** ⚠️ This project is a reverse-engineered implementation of the GitHub Copilot API created for educational purposes only. It is not officially supported by GitHub and should not be used in production environments. -[![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/E1E519XS7W) +## Claude Code Integration + +This fork includes Anthropic-compatible endpoints that make it work seamlessly with [Claude Code](https://claude.ai/code), Anthropic's official CLI for Claude. The server provides `/v1/messages` endpoints that translate between Anthropic's API format and GitHub Copilot's OpenAI-compatible interface. + +### Using with Claude Code + +1. Start the server: + ```sh + bun run start + ``` + +2. Configure Claude Code to use this proxy: + ```sh + export ANTHROPIC_API_KEY="dummy-key" + export ANTHROPIC_BASE_URL="http://localhost:4143" + ``` + +3. Use Claude Code normally - it will route through GitHub Copilot while maintaining full compatibility with Anthropic's API format. ## Project Overview From 054ae8a29f90618a62a3b29d5409cee88ea8533f Mon Sep 17 00:00:00 2001 From: "Tang, Tom" Date: Sun, 15 Jun 2025 17:12:10 +1000 Subject: [PATCH 4/9] docs: remove npx usage and update start command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove all npx-related sections since the package is no longer available - Update Claude Code start command to use bun run dev start --port 4143 --business --verbose - Update all example commands to use bun run dev instead of npx - Keep Docker section for alternative deployment option 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- README.md | 44 +++++++++++--------------------------------- 1 file changed, 11 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 2c9122ea4..cabc363b4 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ This fork includes Anthropic-compatible endpoints that make it work seamlessly w 1. Start the server: ```sh - bun run start + bun run dev start --port 4143 --business --verbose ``` 2. Configure Claude Code to use this proxy: @@ -57,26 +57,6 @@ Run the container docker run -p 4141:4141 copilot-api ``` -## Using with npx - -You can run the project directly using npx: - -```sh -npx copilot-api@latest start -``` - -With options: - -```sh -npx copilot-api@latest start --port 8080 -``` - -For authentication only: - -```sh -npx copilot-api@latest auth -``` - ## Command Structure Copilot API now uses a subcommand structure with two main commands: @@ -109,38 +89,36 @@ The following command line options are available for the `start` command: ## Example Usage -Using with npx: - ```sh # Basic usage with start command -npx copilot-api@latest start +bun run dev start # Run on custom port with verbose logging -npx copilot-api@latest start --port 8080 --verbose +bun run dev start --port 4143 --verbose # Use with a business plan GitHub account -npx copilot-api@latest start --business +bun run dev start --business # Use with an enterprise plan GitHub account -npx copilot-api@latest start --enterprise +bun run dev start --enterprise # Enable manual approval for each request -npx copilot-api@latest start --manual +bun run dev start --manual # Set rate limit to 30 seconds between requests -npx copilot-api@latest start --rate-limit 30 +bun run dev start --rate-limit 30 # Wait instead of error when rate limit is hit -npx copilot-api@latest start --rate-limit 30 --wait +bun run dev start --rate-limit 30 --wait # Provide GitHub token directly -npx copilot-api@latest start --github-token ghp_YOUR_TOKEN_HERE +bun run dev start --github-token ghp_YOUR_TOKEN_HERE # Run only the auth flow -npx copilot-api@latest auth +bun run dev auth # Run auth flow with verbose logging -npx copilot-api@latest auth --verbose +bun run dev auth --verbose ``` ## Running from Source From 143709e7359953a6848d0fc5965c90d196daaeb5 Mon Sep 17 00:00:00 2001 From: "Tang, Tom" Date: Sun, 15 Jun 2025 17:13:56 +1000 Subject: [PATCH 5/9] docs: add complete Claude Code environment configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add ANTHROPIC_MODEL=claude-sonnet-4 environment variable - Add set -e CLAUDE_CODE_USE_BEDROCK to disable Bedrock usage - Update configuration instructions for complete Claude Code setup 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cabc363b4..bda56fa13 100644 --- a/README.md +++ b/README.md @@ -14,10 +14,12 @@ This fork includes Anthropic-compatible endpoints that make it work seamlessly w bun run dev start --port 4143 --business --verbose ``` -2. Configure Claude Code to use this proxy: +2. Configure Claude Code environment variables: ```sh export ANTHROPIC_API_KEY="dummy-key" export ANTHROPIC_BASE_URL="http://localhost:4143" + export ANTHROPIC_MODEL="claude-sonnet-4" + set -e CLAUDE_CODE_USE_BEDROCK ``` 3. Use Claude Code normally - it will route through GitHub Copilot while maintaining full compatibility with Anthropic's API format. From 014fb977b250b621578e1a5a8bce1357f6c932d2 Mon Sep 17 00:00:00 2001 From: "Tang, Tom" Date: Sun, 15 Jun 2025 17:37:23 +1000 Subject: [PATCH 6/9] feat: improve model selection and remove lint hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Enhance selectCopilotModel to prioritize exact matches before falling back to claude-3.7-sonnet - Remove pre-commit lint hooks to streamline development workflow 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- package.json | 6 ------ src/routes/anthropic/handler.ts | 29 ++++++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/package.json b/package.json index 3e1a4952c..2899992ae 100644 --- a/package.json +++ b/package.json @@ -31,12 +31,6 @@ "release": "bumpp && bun publish --access public", "start": "NODE_ENV=production bun run ./src/main.ts" }, - "simple-git-hooks": { - "pre-commit": "bunx lint-staged" - }, - "lint-staged": { - "*": "bunx eslint --fix" - }, "dependencies": { "citty": "^0.1.6", "consola": "^3.4.2", diff --git a/src/routes/anthropic/handler.ts b/src/routes/anthropic/handler.ts index 22c03cbbd..9894c4ce3 100644 --- a/src/routes/anthropic/handler.ts +++ b/src/routes/anthropic/handler.ts @@ -40,7 +40,7 @@ export async function handleAnthropicMessages(c: Context) { try { const anthropicRequest = await c.req.json() - consola.info("Received Anthropic messages request", anthropicRequest) + consola.info("Received Anthropic messages request, requestModel:", anthropicRequest.model) if (anthropicRequest.messages) { const tokenCount = getTokenCount( @@ -219,17 +219,40 @@ function selectCopilotModel(anthropicModel: string): string { return "claude-3-5-sonnet-20241022" } - // Try to find a Claude model first + // First try exact match (case-insensitive) + const exactMatch = state.models.data.find(model => + model.id.toLowerCase() === modelName + ) + + if (exactMatch) { + consola.debug(`Found exact model match: ${exactMatch.id}`) + return exactMatch.id + } + + // Then try to find claude-3.7-sonnet specifically + const preferredModel = state.models.data.find(model => + model.id.toLowerCase() === "claude-3.7-sonnet" + ) + + if (preferredModel) { + consola.debug(`Using preferred model: ${preferredModel.id}`) + return preferredModel.id + } + + // Then try to find any Claude model const claudeModel = state.models.data.find(model => model.id.toLowerCase().includes("claude") ) if (claudeModel) { + consola.debug(`Using claude model: ${claudeModel.id}`) return claudeModel.id } // Fallback to first available model - return state.models.data[0]?.id || "claude-3-5-sonnet-20241022" + const fallbackModel = state.models.data[0]?.id || "claude-3-5-sonnet-20241022" + consola.debug(`Using fallback model: ${fallbackModel}`) + return fallbackModel } function isAsyncIterable(obj: any): obj is AsyncIterable { From 65581c191f6594a578e9e1950d0609db8d284932 Mon Sep 17 00:00:00 2001 From: "Tang, Tom" Date: Sun, 15 Jun 2025 17:39:53 +1000 Subject: [PATCH 7/9] chore: remove unused files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove .env.example and main.py as they are no longer needed. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .env.example | 15 - main.py | 1844 -------------------------------------------------- 2 files changed, 1859 deletions(-) delete mode 100644 .env.example delete mode 100644 main.py diff --git a/.env.example b/.env.example deleted file mode 100644 index 847c23a20..000000000 --- a/.env.example +++ /dev/null @@ -1,15 +0,0 @@ -# Required API Keys -ANTHROPIC_API_KEY="your-anthropic-api-key" # Needed if proxying *to* Anthropic -OPENAI_API_KEY="dummy-key" # Can be any value for local OpenAI-compatible APIs - -# OpenAI-Compatible API Configuration -OPENAI_API_BASE="http://localhost:4142" # Your local OpenAI-compatible endpoint - -# Optional: Provider Preference and Model Mapping -# Controls which provider (google or openai) is preferred for mapping haiku/sonnet. -# Defaults to openai if not set. -PREFERRED_PROVIDER="openai" - -# Use models available on your local API (adjust these to match your available models) -BIG_MODEL="claude-sonnet-4" -SMALL_MODEL="claude-sonnet-4" \ No newline at end of file diff --git a/main.py b/main.py deleted file mode 100644 index 7cf191ca9..000000000 --- a/main.py +++ /dev/null @@ -1,1844 +0,0 @@ -""" -Single-file FastAPI application to proxy Anthropic API requests to an OpenAI-compatible API (e.g., OpenRouter). -Handles request/response conversion, streaming, and dynamic model selection. -""" - -import dataclasses -import enum -import json -import logging -import os -import sys -import time -import traceback -import uuid -from datetime import datetime, timezone -from logging.config import dictConfig -from typing import (Any, AsyncGenerator, Awaitable, Callable, Dict, List, - Literal, Optional, Tuple, Union, cast) - -import fastapi -import openai -import tiktoken -import uvicorn -from dotenv import load_dotenv -from fastapi import Request -from fastapi.responses import JSONResponse, Response, StreamingResponse -from openai.types.chat import (ChatCompletionMessageParam, - ChatCompletionToolParam) -from pydantic import BaseModel, Field, ValidationError, field_validator -from pydantic_settings import BaseSettings, SettingsConfigDict -from rich.console import Console -from rich.panel import Panel -from rich.rule import Rule -from rich.text import Text - -load_dotenv() - - -class Settings(BaseSettings): - """Application settings loaded from environment variables.""" - - model_config = SettingsConfigDict(env_file=".env", extra="ignore") - - openai_api_key: str = Field(default="dummy-key", alias="OPENAI_API_KEY") - big_model_name: str = Field(default="claude-sonnet-4", alias="BIG_MODEL") - small_model_name: str = Field(default="claude-sonnet-4", alias="SMALL_MODEL") - base_url: str = Field(default="http://localhost:4142", alias="OPENAI_API_BASE") - referrer_url: str = "http://localhost:8080/claude_proxy" - preferred_provider: str = Field(default="openai", alias="PREFERRED_PROVIDER") - - app_name: str = "AnthropicProxy" - app_version: str = "0.2.0" - log_level: str = "INFO" - log_file_path: Optional[str] = "log.jsonl" - host: str = "127.0.0.1" - port: int = 8080 - reload: bool = True - - -settings = Settings() - - -_console = Console() -_error_console = Console(stderr=True, style="bold red") - - -class JSONFormatter(logging.Formatter): - def format(self, record: logging.LogRecord) -> str: - header = { - "timestamp": datetime.fromtimestamp( - record.created, timezone.utc - ).isoformat(), - "level": record.levelname, - "logger": record.name, - } - log_payload = getattr(record, "log_record", None) - if isinstance(log_payload, LogRecord): - header["detail"] = dataclasses.asdict(log_payload) - else: - header["message"] = record.getMessage() - if record.exc_info: - exc_type, exc_value, exc_tb = record.exc_info - header["error"] = { - "name": exc_type.__name__ if exc_type else "UnknownError", - "message": str(exc_value), - "stack_trace": "".join( - traceback.format_exception(exc_type, exc_value, exc_tb) - ), - "args": exc_value.args if hasattr(exc_value, "args") else [], - } - return json.dumps(header, ensure_ascii=False) - - -class ConsoleJSONFormatter(JSONFormatter): - def format(self, record: logging.LogRecord) -> str: - log_dict = json.loads(super().format(record)) - if ( - "detail" in log_dict - and "error" in log_dict["detail"] - and log_dict["detail"]["error"] - ): - if "stack_trace" in log_dict["detail"]["error"]: - del log_dict["detail"]["error"]["stack_trace"] - elif "error" in log_dict and log_dict["error"]: - if "stack_trace" in log_dict["error"]: - del log_dict["error"]["stack_trace"] - return json.dumps(log_dict) - - -dictConfig( - { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - "json": {"()": JSONFormatter}, - "console_json": {"()": ConsoleJSONFormatter}, - }, - "handlers": { - "default": { - "class": "logging.StreamHandler", - "formatter": "console_json", - "stream": "ext://sys.stdout", - }, - }, - "loggers": { - "": {"handlers": ["default"], "level": "WARNING"}, - settings.app_name: { - "handlers": ["default"], - "level": settings.log_level.upper(), - "propagate": False, - }, - "uvicorn": {"handlers": ["default"], "level": "INFO", "propagate": False}, - "uvicorn.error": { - "handlers": ["default"], - "level": "INFO", - "propagate": False, - }, - "uvicorn.access": { - "handlers": ["default"], - "level": "INFO", - "propagate": False, - }, - }, - } -) - - -class LogEvent(enum.Enum): - MODEL_SELECTION = "model_selection" - REQUEST_START = "request_start" - REQUEST_COMPLETED = "request_completed" - REQUEST_FAILURE = "request_failure" - ANTHROPIC_REQUEST = "anthropic_body" - OPENAI_REQUEST = "openai_request" - OPENAI_RESPONSE = "openai_response" - ANTHROPIC_RESPONSE = "anthropic_response" - STREAMING_REQUEST = "streaming_request" - STREAM_INTERRUPTED = "stream_interrupted" - TOKEN_COUNT = "token_count" - TOKEN_ENCODER_LOAD_FAILED = "token_encoder_load_failed" - SYSTEM_PROMPT_ADJUSTED = "system_prompt_adjusted" - TOOL_INPUT_SERIALIZATION_FAILURE = "tool_input_serialization_failure" - IMAGE_FORMAT_UNSUPPORTED = "image_format_unsupported" - MESSAGE_FORMAT_NORMALIZED = "message_format_normalized" - TOOL_RESULT_SERIALIZATION_FAILURE = "tool_result_serialization_failure" - TOOL_RESULT_PROCESSING = "tool_result_processing" - TOOL_CHOICE_UNSUPPORTED = "tool_choice_unsupported" - TOOL_ARGS_TYPE_MISMATCH = "tool_args_type_mismatch" - TOOL_ARGS_PARSE_FAILURE = "tool_args_parse_failure" - TOOL_ARGS_UNEXPECTED = "tool_args_unexpected" - TOOL_ID_PLACEHOLDER = "tool_id_placeholder" - TOOL_ID_UPDATED = "tool_id_updated" - PARAMETER_UNSUPPORTED = "parameter_unsupported" - HEALTH_CHECK = "health_check" - PROVIDER_ERROR_DETAILS = "provider_error_details" - - -@dataclasses.dataclass -class LogError: - name: str - message: str - stack_trace: Optional[str] = None - args: Optional[Tuple[Any, ...]] = None - - -@dataclasses.dataclass -class LogRecord: - event: str - message: str - request_id: Optional[str] = None - data: Optional[Dict[str, Any]] = None - error: Optional[LogError] = None - - -_logger = logging.getLogger(settings.app_name) - -if settings.log_file_path: - try: - log_dir = os.path.dirname(settings.log_file_path) - if log_dir: - os.makedirs(log_dir, exist_ok=True) - file_handler = logging.FileHandler(settings.log_file_path, mode="a") - file_handler.setFormatter(JSONFormatter()) - _logger.addHandler(file_handler) - except Exception as e: - _error_console.print( - f"Failed to configure file logging to {settings.log_file_path}: {e}" - ) - - -def _log(level: int, record: LogRecord, exc: Optional[Exception] = None) -> None: - if exc: - record.error = LogError( - name=type(exc).__name__, - message=str(exc), - stack_trace="".join( - traceback.format_exception(type(exc), exc, exc.__traceback__) - ), - args=exc.args if hasattr(exc, "args") else tuple(), - ) - if not record.message and str(exc): - record.message = str(exc) - elif not record.message: - record.message = "An unspecified error occurred" - - _logger.log(level=level, msg=record.message, extra={"log_record": record}) - - -def debug(record: LogRecord): - _log(logging.DEBUG, record) - - -def info(record: LogRecord): - _log(logging.INFO, record) - - -def warning(record: LogRecord, exc: Optional[Exception] = None): - _log(logging.WARNING, record, exc=exc) - - -def error(record: LogRecord, exc: Optional[Exception] = None): - if exc: - _error_console.print_exception(show_locals=False, width=120) - _log(logging.ERROR, record, exc=exc) - - -def critical(record: LogRecord, exc: Optional[Exception] = None): - _log(logging.CRITICAL, record, exc=exc) - - -class ContentBlockText(BaseModel): - type: Literal["text"] - text: str - - -class ContentBlockImageSource(BaseModel): - type: str - media_type: str - data: str - - -class ContentBlockImage(BaseModel): - type: Literal["image"] - source: ContentBlockImageSource - - -class ContentBlockToolUse(BaseModel): - type: Literal["tool_use"] - id: str - name: str - input: Dict[str, Any] - - -class ContentBlockToolResult(BaseModel): - type: Literal["tool_result"] - tool_use_id: str - content: Union[str, List[Dict[str, Any]], List[Any]] - is_error: Optional[bool] = None - - -ContentBlock = Union[ - ContentBlockText, ContentBlockImage, ContentBlockToolUse, ContentBlockToolResult -] - - -class SystemContent(BaseModel): - type: Literal["text"] - text: str - - -class Message(BaseModel): - role: Literal["user", "assistant"] - content: Union[str, List[ContentBlock]] - - -class Tool(BaseModel): - name: str - description: Optional[str] = None - input_schema: Dict[str, Any] = Field(..., alias="input_schema") - - -class ToolChoice(BaseModel): - type: Literal["auto", "any", "tool"] - name: Optional[str] = None - - -class MessagesRequest(BaseModel): - model: str - max_tokens: int - messages: List[Message] - system: Optional[Union[str, List[SystemContent]]] = None - stop_sequences: Optional[List[str]] = None - stream: Optional[bool] = False - temperature: Optional[float] = None - top_p: Optional[float] = None - top_k: Optional[int] = None - metadata: Optional[Dict[str, Any]] = None - tools: Optional[List[Tool]] = None - tool_choice: Optional[ToolChoice] = None - - @field_validator("top_k") - def check_top_k(cls, v: Optional[int]) -> Optional[int]: - if v is not None: - req_id = info.context.get("request_id") if info.context else None - warning( - LogRecord( - event=LogEvent.PARAMETER_UNSUPPORTED.value, - message="Parameter 'top_k' provided by client but is not directly supported by the OpenAI Chat Completions API and will be ignored.", - request_id=req_id, - data={"parameter": "top_k", "value": v}, - ) - ) - return v - - -class TokenCountRequest(BaseModel): - model: str - messages: List[Message] - system: Optional[Union[str, List[SystemContent]]] = None - tools: Optional[List[Tool]] = None - - -class TokenCountResponse(BaseModel): - input_tokens: int - - -class Usage(BaseModel): - input_tokens: int - output_tokens: int - - -class ProviderErrorMetadata(BaseModel): - provider_name: str - raw_error: Optional[Dict[str, Any]] = None - - -class AnthropicErrorType(str, enum.Enum): - INVALID_REQUEST = "invalid_request_error" - AUTHENTICATION = "authentication_error" - PERMISSION = "permission_error" - NOT_FOUND = "not_found_error" - RATE_LIMIT = "rate_limit_error" - API_ERROR = "api_error" - OVERLOADED = "overloaded_error" - REQUEST_TOO_LARGE = "request_too_large_error" - - -class AnthropicErrorDetail(BaseModel): - type: AnthropicErrorType - message: str - provider: Optional[str] = None - provider_message: Optional[str] = None - provider_code: Optional[Union[str, int]] = None - - -class AnthropicErrorResponse(BaseModel): - type: Literal["error"] = "error" - error: AnthropicErrorDetail - - -class MessagesResponse(BaseModel): - id: str - type: Literal["message"] = "message" - role: Literal["assistant"] = "assistant" - model: str - content: List[ContentBlock] - stop_reason: Optional[ - Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "error"] - ] = None - stop_sequence: Optional[str] = None - usage: Usage - - -STATUS_CODE_ERROR_MAP: Dict[int, AnthropicErrorType] = { - 400: AnthropicErrorType.INVALID_REQUEST, - 401: AnthropicErrorType.AUTHENTICATION, - 403: AnthropicErrorType.PERMISSION, - 404: AnthropicErrorType.NOT_FOUND, - 413: AnthropicErrorType.REQUEST_TOO_LARGE, - 422: AnthropicErrorType.INVALID_REQUEST, - 429: AnthropicErrorType.RATE_LIMIT, - 500: AnthropicErrorType.API_ERROR, - 502: AnthropicErrorType.API_ERROR, - 503: AnthropicErrorType.OVERLOADED, - 504: AnthropicErrorType.API_ERROR, -} - - -def extract_provider_error_details( - error_details_dict: Optional[Dict[str, Any]], -) -> Optional[ProviderErrorMetadata]: - if not isinstance(error_details_dict, dict): - return None - metadata = error_details_dict.get("metadata") - if not isinstance(metadata, dict): - return None - provider_name = metadata.get("provider_name") - raw_error_str = metadata.get("raw") - - if not provider_name or not isinstance(provider_name, str): - return None - - parsed_raw_error: Optional[Dict[str, Any]] = None - if isinstance(raw_error_str, str): - try: - parsed_raw_error = json.loads(raw_error_str) - except json.JSONDecodeError: - warning( - LogRecord( - event=LogEvent.PROVIDER_ERROR_DETAILS.value, - message=f"Failed to parse raw provider error string for {provider_name}.", - ) - ) - parsed_raw_error = {"raw_string_parse_failed": raw_error_str} - elif isinstance(raw_error_str, dict): - parsed_raw_error = raw_error_str - - return ProviderErrorMetadata( - provider_name=provider_name, raw_error=parsed_raw_error - ) - - -try: - openai_client = openai.AsyncClient( - api_key=settings.openai_api_key, - base_url=settings.base_url, - default_headers={ - "HTTP-Referer": settings.referrer_url, - "X-Title": settings.app_name, - }, - timeout=180.0, - ) -except Exception as e: - critical( - LogRecord( - event="openai_client_init_failed", - message="Failed to initialize OpenAI client", - ), - exc=e, - ) - sys.exit(1) - - -_token_encoder_cache: Dict[str, tiktoken.Encoding] = {} - - -def get_token_encoder( - model_name: str = "gpt-4", request_id: Optional[str] = None -) -> tiktoken.Encoding: - """Gets a tiktoken encoder, caching it for performance.""" - - cache_key = "gpt-4" - if cache_key not in _token_encoder_cache: - try: - _token_encoder_cache[cache_key] = tiktoken.encoding_for_model(cache_key) - except Exception: - try: - _token_encoder_cache[cache_key] = tiktoken.get_encoding("cl100k_base") - warning( - LogRecord( - event=LogEvent.TOKEN_ENCODER_LOAD_FAILED.value, - message=f"Could not load tiktoken encoder for '{cache_key}', using 'cl100k_base'. Token counts may be approximate.", - request_id=request_id, - data={"model_tried": cache_key}, - ) - ) - except Exception as e_cl: - critical( - LogRecord( - event=LogEvent.TOKEN_ENCODER_LOAD_FAILED.value, - message="Failed to load any tiktoken encoder (gpt-4, cl100k_base). Token counting will be inaccurate.", - request_id=request_id, - ), - exc=e_cl, - ) - - class DummyEncoder: - def encode(self, text: str) -> List[int]: - return list(range(len(text))) - - _token_encoder_cache[cache_key] = DummyEncoder() - return _token_encoder_cache[cache_key] - - -def count_tokens_for_anthropic_request( - messages: List[Message], - system: Optional[Union[str, List[SystemContent]]], - model_name: str, - tools: Optional[List[Tool]] = None, - request_id: Optional[str] = None, -) -> int: - enc = get_token_encoder(model_name, request_id) - total_tokens = 0 - - if isinstance(system, str): - total_tokens += len(enc.encode(system)) - elif isinstance(system, list): - for block in system: - if isinstance(block, SystemContent) and block.type == "text": - total_tokens += len(enc.encode(block.text)) - - for msg in messages: - total_tokens += 4 - if msg.role: - total_tokens += len(enc.encode(msg.role)) - - if isinstance(msg.content, str): - total_tokens += len(enc.encode(msg.content)) - elif isinstance(msg.content, list): - for block in msg.content: - if isinstance(block, ContentBlockText): - total_tokens += len(enc.encode(block.text)) - elif isinstance(block, ContentBlockImage): - total_tokens += 768 - elif isinstance(block, ContentBlockToolUse): - total_tokens += len(enc.encode(block.name)) - try: - input_str = json.dumps(block.input) - total_tokens += len(enc.encode(input_str)) - except Exception: - warning( - LogRecord( - event=LogEvent.TOOL_INPUT_SERIALIZATION_FAILURE.value, - message="Failed to serialize tool input for token counting.", - data={"tool_name": block.name}, - request_id=request_id, - ) - ) - elif isinstance(block, ContentBlockToolResult): - try: - content_str = "" - if isinstance(block.content, str): - content_str = block.content - elif isinstance(block.content, list): - for item in block.content: - if ( - isinstance(item, dict) - and item.get("type") == "text" - ): - content_str += item.get("text", "") - else: - content_str += json.dumps(item) - else: - content_str = json.dumps(block.content) - total_tokens += len(enc.encode(content_str)) - except Exception: - warning( - LogRecord( - event=LogEvent.TOOL_RESULT_SERIALIZATION_FAILURE.value, - message="Failed to serialize tool result for token counting.", - request_id=request_id, - ) - ) - - if tools: - total_tokens += 2 - for tool in tools: - total_tokens += len(enc.encode(tool.name)) - if tool.description: - total_tokens += len(enc.encode(tool.description)) - try: - schema_str = json.dumps(tool.input_schema) - total_tokens += len(enc.encode(schema_str)) - except Exception: - warning( - LogRecord( - event=LogEvent.TOOL_INPUT_SERIALIZATION_FAILURE.value, - message="Failed to serialize tool schema for token counting.", - data={"tool_name": tool.name}, - request_id=request_id, - ) - ) - debug( - LogRecord( - event=LogEvent.TOKEN_COUNT.value, - message=f"Estimated {total_tokens} input tokens for model {model_name}", - data={"model": model_name, "token_count": total_tokens}, - request_id=request_id, - ) - ) - return total_tokens - - -StopReasonType = Optional[ - Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "error"] -] - - -def _serialize_tool_result_content_for_openai( - anthropic_tool_result_content: Union[str, List[Dict[str, Any]], List[Any]], - request_id: Optional[str], - log_context: Dict, -) -> str: - """ - Serializes Anthropic tool result content (which can be complex) into a single string - as expected by OpenAI for the 'content' field of a 'tool' role message. - """ - if isinstance(anthropic_tool_result_content, str): - return anthropic_tool_result_content - - if isinstance(anthropic_tool_result_content, list): - processed_parts = [] - contains_non_text_block = False - for item in anthropic_tool_result_content: - if isinstance(item, dict) and item.get("type") == "text" and "text" in item: - processed_parts.append(str(item["text"])) - else: - try: - processed_parts.append(json.dumps(item)) - contains_non_text_block = True - except TypeError: - processed_parts.append( - f"" - ) - contains_non_text_block = True - - result_str = "\n".join(processed_parts) - if contains_non_text_block: - warning( - LogRecord( - event=LogEvent.TOOL_RESULT_PROCESSING.value, - message="Tool result content list contained non-text or complex items; parts were JSON stringified.", - request_id=request_id, - data={**log_context, "result_str_preview": result_str[:100]}, - ) - ) - return result_str - - try: - return json.dumps(anthropic_tool_result_content) - except TypeError as e: - warning( - LogRecord( - event=LogEvent.TOOL_RESULT_SERIALIZATION_FAILURE.value, - message=f"Failed to serialize tool result content to JSON: {e}. Returning error JSON.", - request_id=request_id, - data=log_context, - ) - ) - return json.dumps( - { - "error": "Serialization failed", - "original_type": str(type(anthropic_tool_result_content)), - } - ) - - -def convert_anthropic_to_openai_messages( - anthropic_messages: List[Message], - anthropic_system: Optional[Union[str, List[SystemContent]]] = None, - request_id: Optional[str] = None, -) -> List[Dict[str, Any]]: - openai_messages: List[Dict[str, Any]] = [] - - system_text_content = "" - if isinstance(anthropic_system, str): - system_text_content = anthropic_system - elif isinstance(anthropic_system, list): - system_texts = [ - block.text - for block in anthropic_system - if isinstance(block, SystemContent) and block.type == "text" - ] - if len(system_texts) < len(anthropic_system): - warning( - LogRecord( - event=LogEvent.SYSTEM_PROMPT_ADJUSTED.value, - message="Non-text content blocks in Anthropic system prompt were ignored.", - request_id=request_id, - ) - ) - system_text_content = "\n".join(system_texts) - - if system_text_content: - openai_messages.append({"role": "system", "content": system_text_content}) - - for i, msg in enumerate(anthropic_messages): - role = msg.role - content = msg.content - - if isinstance(content, str): - openai_messages.append({"role": role, "content": content}) - continue - - if isinstance(content, list): - openai_parts_for_user_message = [] - assistant_tool_calls = [] - text_content_for_assistant = [] - - if not content and role == "user": - openai_messages.append({"role": "user", "content": ""}) - continue - if not content and role == "assistant": - openai_messages.append({"role": "assistant", "content": ""}) - continue - - for block_idx, block in enumerate(content): - block_log_ctx = { - "anthropic_message_index": i, - "block_index": block_idx, - "block_type": block.type, - } - - if isinstance(block, ContentBlockText): - if role == "user": - openai_parts_for_user_message.append( - {"type": "text", "text": block.text} - ) - elif role == "assistant": - text_content_for_assistant.append(block.text) - - elif isinstance(block, ContentBlockImage) and role == "user": - if block.source.type == "base64": - openai_parts_for_user_message.append( - { - "type": "image_url", - "image_url": { - "url": f"data:{block.source.media_type};base64,{block.source.data}" - }, - } - ) - else: - warning( - LogRecord( - event=LogEvent.IMAGE_FORMAT_UNSUPPORTED.value, - message=f"Image block with source type '{block.source.type}' (expected 'base64') ignored in user message {i}.", - request_id=request_id, - data=block_log_ctx, - ) - ) - - elif isinstance(block, ContentBlockToolUse) and role == "assistant": - try: - args_str = json.dumps(block.input) - except Exception as e: - error( - LogRecord( - event=LogEvent.TOOL_INPUT_SERIALIZATION_FAILURE.value, - message=f"Failed to serialize tool input for tool '{block.name}'. Using empty JSON.", - request_id=request_id, - data={ - **block_log_ctx, - "tool_id": block.id, - "tool_name": block.name, - }, - ), - exc=e, - ) - args_str = "{}" - - assistant_tool_calls.append( - { - "id": block.id, - "type": "function", - "function": {"name": block.name, "arguments": args_str}, - } - ) - - elif isinstance(block, ContentBlockToolResult) and role == "user": - serialized_content = _serialize_tool_result_content_for_openai( - block.content, request_id, block_log_ctx - ) - openai_messages.append( - { - "role": "tool", - "tool_call_id": block.tool_use_id, - "content": serialized_content, - } - ) - - if role == "user" and openai_parts_for_user_message: - is_multimodal = any( - part["type"] == "image_url" - for part in openai_parts_for_user_message - ) - if is_multimodal or len(openai_parts_for_user_message) > 1: - openai_messages.append( - {"role": "user", "content": openai_parts_for_user_message} - ) - elif ( - len(openai_parts_for_user_message) == 1 - and openai_parts_for_user_message[0]["type"] == "text" - ): - openai_messages.append( - { - "role": "user", - "content": openai_parts_for_user_message[0]["text"], - } - ) - elif not openai_parts_for_user_message: - openai_messages.append({"role": "user", "content": ""}) - - if role == "assistant": - assistant_text = "\n".join(filter(None, text_content_for_assistant)) - if assistant_text: - openai_messages.append( - {"role": "assistant", "content": assistant_text} - ) - - if assistant_tool_calls: - if ( - openai_messages - and openai_messages[-1]["role"] == "assistant" - and openai_messages[-1].get("content") - ): - openai_messages.append( - { - "role": "assistant", - "content": None, - "tool_calls": assistant_tool_calls, - } - ) - - elif ( - openai_messages - and openai_messages[-1]["role"] == "assistant" - and not openai_messages[-1].get("tool_calls") - ): - openai_messages[-1]["tool_calls"] = assistant_tool_calls - openai_messages[-1]["content"] = None - else: - openai_messages.append( - { - "role": "assistant", - "content": None, - "tool_calls": assistant_tool_calls, - } - ) - - final_openai_messages = [] - for msg_dict in openai_messages: - if ( - msg_dict.get("role") == "assistant" - and msg_dict.get("tool_calls") - and msg_dict.get("content") is not None - ): - warning( - LogRecord( - event=LogEvent.MESSAGE_FORMAT_NORMALIZED.value, - message="Corrected assistant message with tool_calls to have content: None.", - request_id=request_id, - data={"original_content": msg_dict["content"]}, - ) - ) - msg_dict["content"] = None - final_openai_messages.append(msg_dict) - - return final_openai_messages - - -def convert_anthropic_tools_to_openai( - anthropic_tools: Optional[List[Tool]], -) -> Optional[List[Dict[str, Any]]]: - if not anthropic_tools: - return None - return [ - { - "type": "function", - "function": { - "name": t.name, - "description": t.description or "", - "parameters": t.input_schema, - }, - } - for t in anthropic_tools - ] - - -def convert_anthropic_tool_choice_to_openai( - anthropic_choice: Optional[ToolChoice], - request_id: Optional[str] = None, -) -> Optional[Union[str, Dict[str, Any]]]: - if not anthropic_choice: - return None - if anthropic_choice.type == "auto": - return "auto" - if anthropic_choice.type == "any": - warning( - LogRecord( - event=LogEvent.TOOL_CHOICE_UNSUPPORTED.value, - message="Anthropic tool_choice type 'any' mapped to OpenAI 'auto'. Exact behavior might differ (OpenAI 'auto' allows no tool use).", - request_id=request_id, - data={"anthropic_tool_choice": anthropic_choice.model_dump()}, - ) - ) - return "auto" - if anthropic_choice.type == "tool" and anthropic_choice.name: - return {"type": "function", "function": {"name": anthropic_choice.name}} - - warning( - LogRecord( - event=LogEvent.TOOL_CHOICE_UNSUPPORTED.value, - message=f"Unsupported Anthropic tool_choice: {anthropic_choice.model_dump()}. Defaulting to 'auto'.", - request_id=request_id, - data={"anthropic_tool_choice": anthropic_choice.model_dump()}, - ) - ) - return "auto" - - -def convert_openai_to_anthropic_response( - openai_response: openai.types.chat.ChatCompletion, - original_anthropic_model_name: str, - request_id: Optional[str] = None, -) -> MessagesResponse: - anthropic_content: List[ContentBlock] = [] - anthropic_stop_reason: StopReasonType = None - - stop_reason_map: Dict[Optional[str], StopReasonType] = { - "stop": "end_turn", - "length": "max_tokens", - "tool_calls": "tool_use", - "function_call": "tool_use", - "content_filter": "stop_sequence", - None: "end_turn", - } - - if openai_response.choices: - choice = openai_response.choices[0] - message = choice.message - finish_reason = choice.finish_reason - - anthropic_stop_reason = stop_reason_map.get(finish_reason, "end_turn") - - if message.content: - anthropic_content.append( - ContentBlockText(type="text", text=message.content) - ) - - if message.tool_calls: - for call in message.tool_calls: - if call.type == "function": - tool_input_dict: Dict[str, Any] = {} - try: - parsed_input = json.loads(call.function.arguments) - if isinstance(parsed_input, dict): - tool_input_dict = parsed_input - else: - tool_input_dict = {"value": parsed_input} - warning( - LogRecord( - event=LogEvent.TOOL_ARGS_TYPE_MISMATCH.value, - message=f"OpenAI tool arguments for '{call.function.name}' parsed to non-dict type '{type(parsed_input).__name__}'. Wrapped in 'value'.", - request_id=request_id, - data={ - "tool_name": call.function.name, - "tool_id": call.id, - }, - ) - ) - except json.JSONDecodeError as e: - error( - LogRecord( - event=LogEvent.TOOL_ARGS_PARSE_FAILURE.value, - message=f"Failed to parse JSON arguments for tool '{call.function.name}'. Storing raw string.", - request_id=request_id, - data={ - "tool_name": call.function.name, - "tool_id": call.id, - "raw_args": call.function.arguments, - }, - ), - exc=e, - ) - tool_input_dict = { - "error_parsing_arguments": call.function.arguments - } - - anthropic_content.append( - ContentBlockToolUse( - type="tool_use", - id=call.id, - name=call.function.name, - input=tool_input_dict, - ) - ) - if finish_reason == "tool_calls": - anthropic_stop_reason = "tool_use" - - if not anthropic_content: - anthropic_content.append(ContentBlockText(type="text", text="")) - - usage = openai_response.usage - anthropic_usage = Usage( - input_tokens=usage.prompt_tokens if usage else 0, - output_tokens=usage.completion_tokens if usage else 0, - ) - - response_id = ( - f"msg_{openai_response.id}" - if openai_response.id - else f"msg_{request_id}_completed" - ) - - return MessagesResponse( - id=response_id, - type="message", - role="assistant", - model=original_anthropic_model_name, - content=anthropic_content, - stop_reason=anthropic_stop_reason, - usage=anthropic_usage, - ) - - -def _get_anthropic_error_details_from_exc( - exc: Exception, -) -> Tuple[AnthropicErrorType, str, int, Optional[ProviderErrorMetadata]]: - """Maps caught exceptions to Anthropic error type, message, status code, and provider details.""" - error_type = AnthropicErrorType.API_ERROR - error_message = str(exc) - status_code = 500 - provider_details: Optional[ProviderErrorMetadata] = None - - if isinstance(exc, openai.APIError): - error_message = exc.message or str(exc) - status_code = exc.status_code or 500 - error_type = STATUS_CODE_ERROR_MAP.get( - status_code, AnthropicErrorType.API_ERROR - ) - - if hasattr(exc, "body") and isinstance(exc.body, dict): - actual_error_details = exc.body.get("error", exc.body) - provider_details = extract_provider_error_details(actual_error_details) - - if isinstance(exc, openai.AuthenticationError): - error_type = AnthropicErrorType.AUTHENTICATION - elif isinstance(exc, openai.RateLimitError): - error_type = AnthropicErrorType.RATE_LIMIT - elif isinstance(exc, (openai.BadRequestError, openai.UnprocessableEntityError)): - error_type = AnthropicErrorType.INVALID_REQUEST - elif isinstance(exc, openai.PermissionDeniedError): - error_type = AnthropicErrorType.PERMISSION - elif isinstance(exc, openai.NotFoundError): - error_type = AnthropicErrorType.NOT_FOUND - - return error_type, error_message, status_code, provider_details - - -def _format_anthropic_error_sse_event( - error_type: AnthropicErrorType, - message: str, - provider_details: Optional[ProviderErrorMetadata] = None, -) -> str: - """Formats an error into the Anthropic SSE 'error' event structure.""" - anthropic_err_detail = AnthropicErrorDetail(type=error_type, message=message) - if provider_details: - anthropic_err_detail.provider = provider_details.provider_name - if provider_details.raw_error and isinstance( - provider_details.raw_error.get("error"), dict - ): - prov_err_obj = provider_details.raw_error["error"] - anthropic_err_detail.provider_message = prov_err_obj.get("message") - anthropic_err_detail.provider_code = prov_err_obj.get("code") - elif provider_details.raw_error and isinstance( - provider_details.raw_error.get("message"), str - ): - anthropic_err_detail.provider_message = provider_details.raw_error.get( - "message" - ) - anthropic_err_detail.provider_code = provider_details.raw_error.get("code") - - error_response = AnthropicErrorResponse(error=anthropic_err_detail) - return f"event: error\ndata: {error_response.model_dump_json()}\n\n" - - -async def handle_anthropic_streaming_response_from_openai_stream( - openai_stream: openai.AsyncStream[openai.types.chat.ChatCompletionChunk], - original_anthropic_model_name: str, - estimated_input_tokens: int, - request_id: str, - start_time_mono: float, -) -> AsyncGenerator[str, None]: - """ - Consumes an OpenAI stream and yields Anthropic-compatible SSE events. - BUGFIX: Correctly handles content block indexing for mixed text/tool_use. - """ - - anthropic_message_id = f"msg_stream_{request_id}_{uuid.uuid4().hex[:8]}" - - next_anthropic_block_idx = 0 - text_block_anthropic_idx: Optional[int] = None - - openai_tool_idx_to_anthropic_block_idx: Dict[int, int] = {} - - tool_states: Dict[int, Dict[str, Any]] = {} - - sent_tool_block_starts: set[int] = set() - - output_token_count = 0 - final_anthropic_stop_reason: StopReasonType = None - - enc = get_token_encoder(original_anthropic_model_name, request_id) - - openai_to_anthropic_stop_reason_map: Dict[Optional[str], StopReasonType] = { - "stop": "end_turn", - "length": "max_tokens", - "tool_calls": "tool_use", - "function_call": "tool_use", - "content_filter": "stop_sequence", - None: None, - } - - stream_status_code = 200 - stream_final_message = "Streaming request completed successfully." - stream_log_event = LogEvent.REQUEST_COMPLETED.value - - try: - message_start_event_data = { - "type": "message_start", - "message": { - "id": anthropic_message_id, - "type": "message", - "role": "assistant", - "model": original_anthropic_model_name, - "content": [], - "stop_reason": None, - "stop_sequence": None, - "usage": {"input_tokens": estimated_input_tokens, "output_tokens": 0}, - }, - } - yield f"event: message_start\ndata: {json.dumps(message_start_event_data)}\n\n" - yield f"event: ping\ndata: {json.dumps({'type': 'ping'})}\n\n" - - async for chunk in openai_stream: - if not chunk.choices: - continue - - delta = chunk.choices[0].delta - openai_finish_reason = chunk.choices[0].finish_reason - - if delta.content: - output_token_count += len(enc.encode(delta.content)) - if text_block_anthropic_idx is None: - text_block_anthropic_idx = next_anthropic_block_idx - next_anthropic_block_idx += 1 - start_text_event = { - "type": "content_block_start", - "index": text_block_anthropic_idx, - "content_block": {"type": "text", "text": ""}, - } - yield f"event: content_block_start\ndata: {json.dumps(start_text_event)}\n\n" - - text_delta_event = { - "type": "content_block_delta", - "index": text_block_anthropic_idx, - "delta": {"type": "text_delta", "text": delta.content}, - } - yield f"event: content_block_delta\ndata: {json.dumps(text_delta_event)}\n\n" - - if delta.tool_calls: - for tool_delta in delta.tool_calls: - openai_tc_idx = tool_delta.index - - if openai_tc_idx not in openai_tool_idx_to_anthropic_block_idx: - current_anthropic_tool_block_idx = next_anthropic_block_idx - next_anthropic_block_idx += 1 - openai_tool_idx_to_anthropic_block_idx[openai_tc_idx] = ( - current_anthropic_tool_block_idx - ) - - tool_states[current_anthropic_tool_block_idx] = { - "id": tool_delta.id - or f"tool_ph_{request_id}_{current_anthropic_tool_block_idx}", - "name": "", - "arguments_buffer": "", - } - if not tool_delta.id: - warning( - LogRecord( - LogEvent.TOOL_ID_PLACEHOLDER.value, - f"Generated placeholder Tool ID for OpenAI tool index {openai_tc_idx} -> Anthropic block {current_anthropic_tool_block_idx}", - request_id, - ) - ) - else: - current_anthropic_tool_block_idx = ( - openai_tool_idx_to_anthropic_block_idx[openai_tc_idx] - ) - - tool_state = tool_states[current_anthropic_tool_block_idx] - - if tool_delta.id and tool_state["id"].startswith("tool_ph_"): - debug( - LogRecord( - LogEvent.TOOL_ID_UPDATED.value, - f"Updated placeholder Tool ID for Anthropic block {current_anthropic_tool_block_idx} to {tool_delta.id}", - request_id, - ) - ) - tool_state["id"] = tool_delta.id - - if tool_delta.function: - if tool_delta.function.name: - tool_state["name"] = tool_delta.function.name - if tool_delta.function.arguments: - tool_state["arguments_buffer"] += ( - tool_delta.function.arguments - ) - output_token_count += len( - enc.encode(tool_delta.function.arguments) - ) - - if ( - current_anthropic_tool_block_idx not in sent_tool_block_starts - and tool_state["id"] - and not tool_state["id"].startswith("tool_ph_") - and tool_state["name"] - ): - start_tool_event = { - "type": "content_block_start", - "index": current_anthropic_tool_block_idx, - "content_block": { - "type": "tool_use", - "id": tool_state["id"], - "name": tool_state["name"], - "input": {}, - }, - } - yield f"event: content_block_start\ndata: {json.dumps(start_tool_event)}\n\n" - sent_tool_block_starts.add(current_anthropic_tool_block_idx) - - if ( - tool_delta.function - and tool_delta.function.arguments - and current_anthropic_tool_block_idx in sent_tool_block_starts - ): - args_delta_event = { - "type": "content_block_delta", - "index": current_anthropic_tool_block_idx, - "delta": { - "type": "input_json_delta", - "partial_json": tool_delta.function.arguments, - }, - } - yield f"event: content_block_delta\ndata: {json.dumps(args_delta_event)}\n\n" - - if openai_finish_reason: - final_anthropic_stop_reason = openai_to_anthropic_stop_reason_map.get( - openai_finish_reason, "end_turn" - ) - if openai_finish_reason == "tool_calls": - final_anthropic_stop_reason = "tool_use" - break - - if text_block_anthropic_idx is not None: - yield f"event: content_block_stop\ndata: {json.dumps({'type': 'content_block_stop', 'index': text_block_anthropic_idx})}\n\n" - - for anthropic_tool_idx in sent_tool_block_starts: - tool_state_to_finalize = tool_states.get(anthropic_tool_idx) - if tool_state_to_finalize: - try: - json.loads(tool_state_to_finalize["arguments_buffer"]) - except json.JSONDecodeError: - warning( - LogRecord( - event=LogEvent.TOOL_ARGS_PARSE_FAILURE.value, - message=f"Buffered arguments for tool '{tool_state_to_finalize.get('name')}' (Anthropic block {anthropic_tool_idx}) did not form valid JSON.", - request_id=request_id, - data={ - "buffered_args": tool_state_to_finalize[ - "arguments_buffer" - ][:100] - }, - ) - ) - yield f"event: content_block_stop\ndata: {json.dumps({'type': 'content_block_stop', 'index': anthropic_tool_idx})}\n\n" - - if final_anthropic_stop_reason is None: - final_anthropic_stop_reason = "end_turn" - - message_delta_event = { - "type": "message_delta", - "delta": { - "stop_reason": final_anthropic_stop_reason, - "stop_sequence": None, - }, - "usage": {"output_tokens": output_token_count}, - } - yield f"event: message_delta\ndata: {json.dumps(message_delta_event)}\n\n" - yield f"event: message_stop\ndata: {json.dumps({'type': 'message_stop'})}\n\n" - - except Exception as e: - stream_status_code = 500 - stream_log_event = LogEvent.REQUEST_FAILURE.value - error_type, error_msg_str, _, provider_err_details = ( - _get_anthropic_error_details_from_exc(e) - ) - stream_final_message = f"Error during OpenAI stream conversion: {error_msg_str}" - final_anthropic_stop_reason = "error" - - error( - LogRecord( - event=LogEvent.STREAM_INTERRUPTED.value, - message=stream_final_message, - request_id=request_id, - data={ - "error_type": error_type.value, - "provider_details": provider_err_details.model_dump() - if provider_err_details - else None, - }, - ), - exc=e, - ) - yield _format_anthropic_error_sse_event( - error_type, error_msg_str, provider_err_details - ) - - finally: - duration_ms = (time.monotonic() - start_time_mono) * 1000 - log_data = { - "status_code": stream_status_code, - "duration_ms": duration_ms, - "input_tokens": estimated_input_tokens, - "output_tokens": output_token_count, - "stop_reason": final_anthropic_stop_reason, - } - if stream_log_event == LogEvent.REQUEST_COMPLETED.value: - info( - LogRecord( - event=stream_log_event, - message=stream_final_message, - request_id=request_id, - data=log_data, - ) - ) - else: - error( - LogRecord( - event=stream_log_event, - message=stream_final_message, - request_id=request_id, - data=log_data, - ) - ) - - -app = fastapi.FastAPI( - title=settings.app_name, - description="Routes Anthropic API requests to an OpenAI-compatible API, selecting models dynamically.", - version=settings.app_version, - docs_url=None, - redoc_url=None, -) - - -def select_target_model(client_model_name: str, request_id: str) -> str: - """Selects the target OpenRouter model based on the client's request.""" - client_model_lower = client_model_name.lower() - target_model: str - - if "opus" in client_model_lower or "sonnet" in client_model_lower: - target_model = settings.big_model_name - elif "haiku" in client_model_lower: - target_model = settings.small_model_name - else: - target_model = settings.small_model_name - warning( - LogRecord( - event=LogEvent.MODEL_SELECTION.value, - message=f"Unknown client model '{client_model_name}', defaulting to SMALL model '{target_model}'.", - request_id=request_id, - data={ - "client_model": client_model_name, - "default_target_model": target_model, - }, - ) - ) - - debug( - LogRecord( - event=LogEvent.MODEL_SELECTION.value, - message=f"Client model '{client_model_name}' mapped to target model '{target_model}'.", - request_id=request_id, - data={"client_model": client_model_name, "target_model": target_model}, - ) - ) - return target_model - - -def _build_anthropic_error_response( - error_type: AnthropicErrorType, - message: str, - status_code: int, - provider_details: Optional[ProviderErrorMetadata] = None, -) -> JSONResponse: - """Creates a JSONResponse with Anthropic-formatted error.""" - err_detail = AnthropicErrorDetail(type=error_type, message=message) - if provider_details: - err_detail.provider = provider_details.provider_name - if provider_details.raw_error: - if isinstance(provider_details.raw_error, dict): - prov_err_obj = provider_details.raw_error.get("error") - if isinstance(prov_err_obj, dict): - err_detail.provider_message = prov_err_obj.get("message") - err_detail.provider_code = prov_err_obj.get("code") - elif isinstance(provider_details.raw_error.get("message"), str): - err_detail.provider_message = provider_details.raw_error.get( - "message" - ) - err_detail.provider_code = provider_details.raw_error.get("code") - - error_resp_model = AnthropicErrorResponse(error=err_detail) - return JSONResponse( - status_code=status_code, content=error_resp_model.model_dump(exclude_unset=True) - ) - - -async def _log_and_return_error_response( - request: Request, - status_code: int, - anthropic_error_type: AnthropicErrorType, - error_message: str, - provider_details: Optional[ProviderErrorMetadata] = None, - caught_exception: Optional[Exception] = None, -) -> JSONResponse: - request_id = getattr(request.state, "request_id", "unknown") - start_time_mono = getattr(request.state, "start_time_monotonic", time.monotonic()) - duration_ms = (time.monotonic() - start_time_mono) * 1000 - - log_data = { - "status_code": status_code, - "duration_ms": duration_ms, - "error_type": anthropic_error_type.value, - "client_ip": request.client.host if request.client else "unknown", - } - if provider_details: - log_data["provider_name"] = provider_details.provider_name - log_data["provider_raw_error"] = provider_details.raw_error - - error( - LogRecord( - event=LogEvent.REQUEST_FAILURE.value, - message=f"Request failed: {error_message}", - request_id=request_id, - data=log_data, - ), - exc=caught_exception, - ) - return _build_anthropic_error_response( - anthropic_error_type, error_message, status_code, provider_details - ) - - -@app.post("/v1/messages", response_model=None, tags=["API"], status_code=200) -async def create_message_proxy( - request: Request, -) -> Union[JSONResponse, StreamingResponse]: - """ - Main endpoint for Anthropic message completions, proxied to an OpenAI-compatible API. - Handles request/response conversions, streaming, and dynamic model selection. - """ - request_id = str(uuid.uuid4()) - request.state.request_id = request_id - request.state.start_time_monotonic = time.monotonic() - - try: - raw_body = await request.json() - debug( - LogRecord( - LogEvent.ANTHROPIC_REQUEST.value, - "Received Anthropic request body", - request_id, - {"body": raw_body}, - ) - ) - - anthropic_request = MessagesRequest.model_validate( - raw_body, context={"request_id": request_id} - ) - except json.JSONDecodeError as e: - return await _log_and_return_error_response( - request, - 400, - AnthropicErrorType.INVALID_REQUEST, - "Invalid JSON body.", - caught_exception=e, - ) - except ValidationError as e: - return await _log_and_return_error_response( - request, - 422, - AnthropicErrorType.INVALID_REQUEST, - f"Invalid request body: {e.errors()}", - caught_exception=e, - ) - - is_stream = anthropic_request.stream or False - target_model_name = select_target_model(anthropic_request.model, request_id) - - estimated_input_tokens = count_tokens_for_anthropic_request( - messages=anthropic_request.messages, - system=anthropic_request.system, - model_name=anthropic_request.model, - tools=anthropic_request.tools, - request_id=request_id, - ) - - info( - LogRecord( - event=LogEvent.REQUEST_START.value, - message="Processing new message request", - request_id=request_id, - data={ - "client_model": anthropic_request.model, - "target_model": target_model_name, - "stream": is_stream, - "estimated_input_tokens": estimated_input_tokens, - "client_ip": request.client.host if request.client else "unknown", - "user_agent": request.headers.get("user-agent", "unknown"), - }, - ) - ) - - try: - openai_messages = convert_anthropic_to_openai_messages( - anthropic_request.messages, anthropic_request.system, request_id=request_id - ) - openai_tools = convert_anthropic_tools_to_openai(anthropic_request.tools) - openai_tool_choice = convert_anthropic_tool_choice_to_openai( - anthropic_request.tool_choice, request_id - ) - except Exception as e: - return await _log_and_return_error_response( - request, - 500, - AnthropicErrorType.API_ERROR, - "Error during request conversion.", - caught_exception=e, - ) - - openai_params: Dict[str, Any] = { - "model": target_model_name, - "messages": cast(List[ChatCompletionMessageParam], openai_messages), - "max_tokens": anthropic_request.max_tokens, - "stream": is_stream, - } - if anthropic_request.temperature is not None: - openai_params["temperature"] = anthropic_request.temperature - if anthropic_request.top_p is not None: - openai_params["top_p"] = anthropic_request.top_p - if anthropic_request.stop_sequences: - openai_params["stop"] = anthropic_request.stop_sequences - if openai_tools: - openai_params["tools"] = cast( - Optional[List[ChatCompletionToolParam]], openai_tools - ) - if openai_tool_choice: - openai_params["tool_choice"] = openai_tool_choice - if anthropic_request.metadata and anthropic_request.metadata.get("user_id"): - openai_params["user"] = str(anthropic_request.metadata.get("user_id")) - - debug( - LogRecord( - LogEvent.OPENAI_REQUEST.value, - "Prepared OpenAI request parameters", - request_id, - {"params": openai_params}, - ) - ) - - try: - if is_stream: - debug( - LogRecord( - LogEvent.STREAMING_REQUEST.value, - "Initiating streaming request to OpenAI-compatible API", - request_id, - ) - ) - openai_stream_response = await openai_client.chat.completions.create( - **openai_params - ) - return StreamingResponse( - handle_anthropic_streaming_response_from_openai_stream( - openai_stream_response, - anthropic_request.model, - estimated_input_tokens, - request_id, - request.state.start_time_monotonic, - ), - media_type="text/event-stream", - ) - else: - debug( - LogRecord( - LogEvent.OPENAI_REQUEST.value, - "Sending non-streaming request to OpenAI-compatible API", - request_id, - ) - ) - openai_response_obj = await openai_client.chat.completions.create( - **openai_params - ) - - debug( - LogRecord( - LogEvent.OPENAI_RESPONSE.value, - "Received OpenAI response", - request_id, - {"response": openai_response_obj.model_dump()}, - ) - ) - - anthropic_response_obj = convert_openai_to_anthropic_response( - openai_response_obj, anthropic_request.model, request_id=request_id - ) - duration_ms = (time.monotonic() - request.state.start_time_monotonic) * 1000 - info( - LogRecord( - event=LogEvent.REQUEST_COMPLETED.value, - message="Non-streaming request completed successfully", - request_id=request_id, - data={ - "status_code": 200, - "duration_ms": duration_ms, - "input_tokens": anthropic_response_obj.usage.input_tokens, - "output_tokens": anthropic_response_obj.usage.output_tokens, - "stop_reason": anthropic_response_obj.stop_reason, - }, - ) - ) - debug( - LogRecord( - LogEvent.ANTHROPIC_RESPONSE.value, - "Prepared Anthropic response", - request_id, - {"response": anthropic_response_obj.model_dump(exclude_unset=True)}, - ) - ) - return JSONResponse( - content=anthropic_response_obj.model_dump(exclude_unset=True) - ) - - except openai.APIError as e: - err_type, err_msg, err_status, prov_details = ( - _get_anthropic_error_details_from_exc(e) - ) - return await _log_and_return_error_response( - request, err_status, err_type, err_msg, prov_details, e - ) - except Exception as e: - return await _log_and_return_error_response( - request, - 500, - AnthropicErrorType.API_ERROR, - "An unexpected error occurred while processing the request.", - caught_exception=e, - ) - - -@app.post( - "/v1/messages/count_tokens", response_model=TokenCountResponse, tags=["Utility"] -) -async def count_tokens_endpoint(request: Request) -> TokenCountResponse: - """Estimates token count for given Anthropic messages and system prompt.""" - request_id = str(uuid.uuid4()) - request.state.request_id = request_id - start_time_mono = time.monotonic() - - try: - body = await request.json() - count_request = TokenCountRequest.model_validate(body) - except json.JSONDecodeError as e: - raise fastapi.HTTPException(status_code=400, detail="Invalid JSON body.") from e - except ValidationError as e: - raise fastapi.HTTPException( - status_code=422, detail=f"Invalid request body: {e.errors()}" - ) from e - - token_count = count_tokens_for_anthropic_request( - messages=count_request.messages, - system=count_request.system, - model_name=count_request.model, - tools=count_request.tools, - request_id=request_id, - ) - duration_ms = (time.monotonic() - start_time_mono) * 1000 - info( - LogRecord( - event=LogEvent.TOKEN_COUNT.value, - message=f"Counted {token_count} tokens", - request_id=request_id, - data={ - "duration_ms": duration_ms, - "token_count": token_count, - "model": count_request.model, - }, - ) - ) - return TokenCountResponse(input_tokens=token_count) - - -@app.get("/", include_in_schema=False, tags=["Health"]) -async def root_health_check() -> JSONResponse: - """Basic health check and information endpoint.""" - debug( - LogRecord( - event=LogEvent.HEALTH_CHECK.value, message="Root health check accessed" - ) - ) - return JSONResponse( - { - "proxy_name": settings.app_name, - "version": settings.app_version, - "status": "ok", - "timestamp": datetime.now(timezone.utc).isoformat(), - } - ) - - -@app.exception_handler(openai.APIError) -async def openai_api_error_handler(request: Request, exc: openai.APIError): - err_type, err_msg, err_status, prov_details = _get_anthropic_error_details_from_exc( - exc - ) - return await _log_and_return_error_response( - request, err_status, err_type, err_msg, prov_details, exc - ) - - -@app.exception_handler(ValidationError) -async def pydantic_validation_error_handler(request: Request, exc: ValidationError): - return await _log_and_return_error_response( - request, - 422, - AnthropicErrorType.INVALID_REQUEST, - f"Validation error: {exc.errors()}", - caught_exception=exc, - ) - - -@app.exception_handler(json.JSONDecodeError) -async def json_decode_error_handler(request: Request, exc: json.JSONDecodeError): - return await _log_and_return_error_response( - request, - 400, - AnthropicErrorType.INVALID_REQUEST, - "Invalid JSON format.", - caught_exception=exc, - ) - - -@app.exception_handler(Exception) -async def generic_exception_handler(request: Request, exc: Exception): - return await _log_and_return_error_response( - request, - 500, - AnthropicErrorType.API_ERROR, - "An unexpected internal server error occurred.", - caught_exception=exc, - ) - - -@app.middleware("http") -async def logging_middleware( - request: Request, call_next: Callable[[Request], Awaitable[Response]] -) -> Response: - if not hasattr(request.state, "request_id"): - request.state.request_id = str(uuid.uuid4()) - if not hasattr(request.state, "start_time_monotonic"): - request.state.start_time_monotonic = time.monotonic() - - response = await call_next(request) - - response.headers["X-Request-ID"] = request.state.request_id - duration_ms = (time.monotonic() - request.state.start_time_monotonic) * 1000 - response.headers["X-Response-Time-ms"] = str(duration_ms) - - return response - - -if __name__ == "__main__": - _console.print( - r"""[bold blue] - /$$ /$$ - | $$ | $$ - /$$$$$$$| $$ /$$$$$$ /$$ /$$ /$$$$$$$ /$$$$$$ /$$$$$$ /$$$$$$ /$$$$$$ /$$ /$$ /$$ /$$ - /$$_____/| $$ |____ $$| $$ | $$ /$$__ $$ /$$__ $$ /$$__ $$ /$$__ $$ /$$__ $$| $$ /$$/| $$ | $$ -| $$ | $$ /$$$$$$$| $$ | $$| $$ | $$| $$$$$$$$ | $$ \ $$| $$ \__/| $$ \ $$ \ $$$$/ | $$ | $$ -| $$ | $$ /$$__ $$| $$ | $$| $$ | $$| $$_____/ | $$ | $$| $$ | $$ | $$ >$$ $$ | $$ | $$ -| $$$$$$$| $$| $$$$$$$| $$$$$$/| $$$$$$$| $$$$$$$ | $$$$$$$/| $$ | $$$$$$/ /$$/\ $$| $$$$$$$ - \_______/|__/ \_______/ \______/ \_______/ \_______/ | $$____/ |__/ \______/ |__/ \__/ \____ $$ - | $$ /$$ | $$ - | $$ | $$$$$$/ - |__/ \______/ - [/]""", - justify="left", - ) - config_details_text = Text.assemble( - (" Version : ", "default"), - (f"v{settings.app_version}", "bold cyan"), - ("\n Big Model : ", "default"), - (settings.big_model_name, "magenta"), - ("\n Small Model : ", "default"), - (settings.small_model_name, "green"), - ("\n Log Level : ", "default"), - (settings.log_level.upper(), "yellow"), - ("\n Log File : ", "default"), - (settings.log_file_path or "Disabled", "dim"), - ("\n Listening on : ", "default"), - (f"http://{settings.host}:{settings.port}", "bold white"), - ("\n Reload : ", "default"), - ("Enabled", "bold orange1") if settings.reload else ("Disabled", "dim"), - ) - _console.print( - Panel( - config_details_text, - title="Anthropic Proxy Configuration", - border_style="blue", - expand=False, - ) - ) - _console.print(Rule("Starting Uvicorn server...", style="dim blue")) - - uvicorn.run( - "__main__:app", - host=settings.host, - port=settings.port, - reload=settings.reload, - log_config=None, - access_log=False, - ) From 57a8c0dedde77d6a23655205656d663cc85d7021 Mon Sep 17 00:00:00 2001 From: "Tang, Tom" Date: Sun, 15 Jun 2025 17:43:11 +1000 Subject: [PATCH 8/9] docs: add comprehensive API endpoints documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document all implemented endpoints including OpenAI-compatible and Anthropic-compatible routes with descriptions and HTTP methods. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/README.md b/README.md index bda56fa13..3474d0b42 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,34 @@ ⚠️ **EDUCATIONAL PURPOSE ONLY** ⚠️ This project is a reverse-engineered implementation of the GitHub Copilot API created for educational purposes only. It is not officially supported by GitHub and should not be used in production environments. +## API Endpoints + +This project implements multiple API endpoints to provide compatibility with different AI tools and frameworks: + +### OpenAI-Compatible Endpoints + +These endpoints follow the OpenAI API specification and are compatible with tools that expect OpenAI's API format: + +- **`POST /chat/completions`** - Chat completion endpoint for conversational AI +- **`POST /v1/chat/completions`** - Same as above with v1 prefix for tool compatibility +- **`GET /models`** - List available models from GitHub Copilot +- **`GET /v1/models`** - Same as above with v1 prefix +- **`POST /embeddings`** - Generate text embeddings +- **`POST /v1/embeddings`** - Same as above with v1 prefix + +### Anthropic-Compatible Endpoints + +These endpoints follow the Anthropic API specification and are compatible with tools that expect Claude's API format: + +- **`POST /v1/messages`** - Main Anthropic messages endpoint for chat completion +- **`POST /v1/messages/count_tokens`** - Token counting endpoint for input estimation + +### Server Status + +- **`GET /`** - Server health check endpoint + +All endpoints proxy requests to GitHub Copilot's API while maintaining compatibility with the respective API formats. The server automatically handles authentication, request/response translation, and model selection. + ## Claude Code Integration This fork includes Anthropic-compatible endpoints that make it work seamlessly with [Claude Code](https://claude.ai/code), Anthropic's official CLI for Claude. The server provides `/v1/messages` endpoints that translate between Anthropic's API format and GitHub Copilot's OpenAI-compatible interface. From a9fe534509989fae906561669418a7efc4beca14 Mon Sep 17 00:00:00 2001 From: "Tang, Tom" Date: Wed, 18 Jun 2025 21:36:07 +1000 Subject: [PATCH 9/9] feat: enhance Copilot token management with improved logging and error handling - Introduced a utility function to format timestamps for better log readability. - Implemented a mechanism to track refresh failures and log detailed error messages. - Added checks for multiple refresh failures to prompt users about potential GitHub token expiration. This update improves the reliability and transparency of the Copilot token refresh process. --- src/lib/token.ts | 49 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/src/lib/token.ts b/src/lib/token.ts index aa669676d..3d13665a3 100644 --- a/src/lib/token.ts +++ b/src/lib/token.ts @@ -15,20 +15,59 @@ const readGithubToken = () => fs.readFile(PATHS.GITHUB_TOKEN_PATH, "utf8") const writeGithubToken = (token: string) => fs.writeFile(PATHS.GITHUB_TOKEN_PATH, token) +// Utility function to format timestamp as "2001-01-01 13:00:00" +const formatTimestamp = () => { + const now = new Date() + const year = now.getFullYear() + const month = String(now.getMonth() + 1).padStart(2, '0') + const day = String(now.getDate()).padStart(2, '0') + const hours = String(now.getHours()).padStart(2, '0') + const minutes = String(now.getMinutes()).padStart(2, '0') + const seconds = String(now.getSeconds()).padStart(2, '0') + return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}` +} + +// Track the refresh interval to prevent multiple intervals +let refreshIntervalId: NodeJS.Timeout | null = null +let refreshFailureCount = 0 +const MAX_REFRESH_FAILURES = 3 + export const setupCopilotToken = async () => { + // Clear any existing interval + if (refreshIntervalId) { + clearInterval(refreshIntervalId) + refreshIntervalId = null + } + const { token, refresh_in } = await getCopilotToken() state.copilotToken = token + refreshFailureCount = 0 // Reset failure count on successful setup const refreshInterval = (refresh_in - 60) * 1000 + consola.info(`[${formatTimestamp()}] Copilot token will refresh in ${Math.round(refreshInterval / 1000)} seconds`) - setInterval(async () => { - consola.start("Refreshing Copilot token") + refreshIntervalId = setInterval(async () => { + consola.start(`[${formatTimestamp()}] Refreshing Copilot token`) try { - const { token } = await getCopilotToken() + const { token, refresh_in: newRefreshIn } = await getCopilotToken() state.copilotToken = token + refreshFailureCount = 0 // Reset failure count on success + consola.success(`[${formatTimestamp()}] Copilot token refreshed successfully`) + consola.debug(`[${formatTimestamp()}] Next refresh in ${newRefreshIn - 60} seconds`) } catch (error) { - consola.error("Failed to refresh Copilot token:", error) - throw error + refreshFailureCount++ + consola.error(`[${formatTimestamp()}] Failed to refresh Copilot token (attempt ${refreshFailureCount}/${MAX_REFRESH_FAILURES}):`, error) + + // If we've failed too many times, check if GitHub token is still valid + if (refreshFailureCount >= MAX_REFRESH_FAILURES) { + consola.error(`[${formatTimestamp()}] Multiple refresh failures detected. This might indicate an expired GitHub token.`) + consola.info(`[${formatTimestamp()}] Consider running the 'auth' command to refresh your GitHub token`) + refreshFailureCount = 0 // Reset to prevent spam + } + + // Don't throw the error - this would stop the interval + // Instead, log the error and let the interval continue + // The next refresh attempt will try again } }, refreshInterval) }