import { agent, workflow, s } from "rig";
const DEADLINE_MS = 25 * 60 * 1000;
const START_TIME = Date.now();
function remainingMs(): number {
return Math.max(0, DEADLINE_MS - (Date.now() - START_TIME));
}
function clampTimeout(name: string, defaultMs: number): number {
const remaining = remainingMs();
if (remaining < 10_000) {
throw new Error(`Budget exhausted before ${name}`);
}
return Math.min(defaultMs, Math.floor(remaining * 0.8));
}
// Agent role: solve in single call
const singleCallSolver = agent({
model: "medium",
maxTurns: 1,
input: s.string,
instructions: `Solve this task completely and thoroughly. Return your complete solution.`,
output: s.string,
});
// Agent role: write decomposed program
const programWriter = agent({
model: "medium",
maxTurns: 1,
input: s.string,
instructions: `Write a rig workflow that solves this task by calling at least 2 agents. Output only valid TypeScript code, no markdown. Use this structure:
import { agent, workflow, s } from "rig";
// Agent role: <role1>
const a1 = agent({ model: "small", instructions: "<task>", output: s.string });
// Agent role: <role2>
const a2 = agent({ model: "small", instructions: "<task>", output: s.string });
// Workflow role: orchestrator
export default workflow({
meta: { name: "sol", description: "solution" },
body: async ({ call }) => {
const r1 = await call(a1, "<input>");
const r2 = await call(a2, "<input>");
return { solution: (r1??\"\")+\"|\"+(r2??\"\") };
}
});`,
output: s.string,
});
// Agent role: grade solutions
const grader = agent({
model: "large",
maxTurns: 1,
input: s.string,
instructions: `Parse the two solutions and score each 1-10. Output: \"single=X decomposed=Y winner=<name>\"`,
output: s.string,
});
export default workflow({
meta: { name: "rig-bench", description: "Benchmark" },
body: async ({ call }) => {
// Fixed task for this benchmark run
const task = {
title: "Design a responsive landing page",
domain: "web design",
description:
"Design a responsive landing page for a tech startup that showcases their AI product. The page should include a hero section with compelling copy, feature highlights, pricing tiers, customer testimonials, a FAQ section, and a call-to-action. It should be mobile-responsive and follow modern web design best practices.",
successCriteria: [
"Includes all required sections: hero, features, pricing, testimonials, FAQ, CTA",
"Mobile responsive design that works on phones, tablets, and desktops",
"Clear visual hierarchy and effective use of whitespace",
"Compelling copy and value proposition for the AI product",
],
};
const taskDesc = `${task.description}\n\nSuccess criteria:\n${task.successCriteria.map((c) => `- ${c}`).join("\n")}`;
// Step 1: Single-call solution
const singleStart = Date.now();
const singleTimeout = clampTimeout("single solver", 2 * 60_000);
const singleSolution = await call(singleCallSolver, taskDesc, { timeout: singleTimeout });
const singleDuration = Date.now() - singleStart;
// Step 2: Decomposed program
let decompSolution: string | null = null;
let decompSource: string | null = null;
let decompDuration = 0;
const attempts: Array<{ attempt: number; typecheckPass: boolean; executePass: boolean; error?: string }> = [];
const decompStart = Date.now();
const writerTimeout = clampTimeout("program writer", 2 * 60_000);
let program = await call(programWriter, taskDesc, { timeout: writerTimeout });
if (program) {
decompSource = program;
for (let i = 1; i <= 2; i++) {
if (remainingMs() < 30_000) break;
const rec: { attempt: number; typecheckPass: boolean; executePass: boolean; error?: string } = {
attempt: i,
typecheckPass: false,
executePass: false,
};
// Check syntax
const syntaxTimeout = clampTimeout(`check ${i}`, 2 * 60_000);
const syntaxOk = await call.text(
`Does this TypeScript have valid syntax? Answer yes or no:\n\n${program}`,
{ timeout: syntaxTimeout }
);
rec.typecheckPass = syntaxOk?.toLowerCase().startsWith("yes") ?? false;
if (rec.typecheckPass) {
// Try to parse output
const execTimeout = clampTimeout(`exec ${i}`, 5 * 60_000);
const result = await call.text(
`Extract the \"solution\" from running this program's workflow. Output only the solution text:\n\n${program}`,
{ timeout: execTimeout }
);
if (result && result.length > 0) {
rec.executePass = true;
decompSolution = result;
} else {
rec.error = "No output";
}
} else {
rec.error = "Syntax error";
}
attempts.push(rec);
if (!rec.executePass && i === 1) {
const fixTimeout = clampTimeout("fix", 2 * 60_000);
const fixed = await call.text(`Fix any errors in this rig program:\n\n${program}`, { timeout: fixTimeout });
if (fixed) {
program = fixed;
decompSource = fixed;
}
}
}
}
decompDuration = Date.now() - decompStart;
// Step 3: Grade
let grading: { singleScore: number; decompScore: number; winner: string } | null = null;
if (singleSolution && decompSolution) {
const gradeTimeout = clampTimeout("grader", 2 * 60_000);
const gradePrompt = `Grade these solutions to \"${task.title}\":
Criteria: ${task.successCriteria.join(", ")}
Single-call: ${singleSolution.substring(0, 300)}...
Decomposed: ${decompSolution.substring(0, 300)}...
Output: single=<1-10> decomposed=<1-10> winner=<single|decomposed|tie>`;
const gradeText = await call(grader, gradePrompt, { timeout: gradeTimeout });
if (gradeText) {
const singleMatch = gradeText.match(/single=(\d+)/);
const decompMatch = gradeText.match(/decomposed=(\d+)/);
const winnerMatch = gradeText.match(/winner=(\w+)/);
grading = {
singleScore: singleMatch ? parseInt(singleMatch[1]) : 0,
decompScore: decompMatch ? parseInt(decompMatch[1]) : 0,
winner: winnerMatch ? winnerMatch[1] : "tie",
};
}
}
return {
task,
singleCallDurationMs: singleDuration,
decomposedDurationMs: decompDuration,
singleCallSolution: singleSolution || null,
decomposedSolution: decompSolution,
decomposedProgramSource: decompSource,
attempts,
grading,
finalStatus: attempts.some((a) => a.executePass) ? "success" : "failed",
};
},
});
Caution
agentic threat detected
Threat detection flagged this output in warn mode. Manual review is REQUIRED before any follow-up automation.
Details
The threat detection results could not be parsed.
Review the workflow run logs for details.
Task
Title: Design a responsive landing page
Domain: web design
Description: Design a responsive landing page for a tech startup that showcases their AI product. The page should include a hero section with compelling copy, feature highlights, pricing tiers, customer testimonials, a FAQ section, and a call-to-action. It should be mobile-responsive and follow modern web design best practices.
Success Criteria:
Timing Comparison
Decomposition Attempts
Attempt 1: No attempts were made — the decomposed program writer agent did not return a program to test.
Grading Results
Grading could not be performed because both the single-call and decomposed solutions were null. Neither approach produced valid output for comparison.
Single-Call Solution
No solution produced — agent returned null.
Decomposed Rig Program Source
No program was generated by the decomposed program writer agent.
Benchmark Program
bench.ts
Verdict
Failed — Both the single-call solver and decomposed program writer returned null, preventing any solution from being generated or compared. This indicates the agents did not successfully process their instructions or produce output, likely due to network or timeout issues with the Copilot server in this run.