diff --git a/evals/azure-app-onboard-prereq/eval.yaml b/evals/azure-app-onboard-prereq/eval.yaml index 425e2b2e6..e259159d1 100644 --- a/evals/azure-app-onboard-prereq/eval.yaml +++ b/evals/azure-app-onboard-prereq/eval.yaml @@ -39,17 +39,17 @@ stimuli: # ── integration-diversity.test.ts ────────────────────────────────── # test: e2e — postgresql-event-sourcing (Java/Spring Boot + Kafka) - name: "Dependency Compat - Java Spring Boot" - prompt: "Can you check if my dependencies are compatible with Azure?" + turns: + - "Can you check if my dependencies are compatible with Azure?" + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: output skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -114,17 +114,17 @@ stimuli: # test: e2e — yamtrack-django (Python/Django + PostgreSQL + Redis) - name: "Dependency Compat - Django Local Database" - prompt: "My app uses a local database — check what I need to change before moving to Azure" + turns: + - "My app uses a local database — check what I need to change before moving to Azure" + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: output skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -188,17 +188,17 @@ stimuli: # test: e2e — fullstack-starter (4-component monorepo + GCP migration) - name: "Diversity - Monorepo Multi-Component Blockers" - prompt: "Before I deploy to Azure, scan my repo and tell me what prerequisites or blockers I need to resolve." + turns: + - "Before I deploy to Azure, scan my repo and tell me what prerequisites or blockers I need to resolve." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: output skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -256,17 +256,17 @@ stimuli: # ── integration-functional.test.ts ───────────────────────────────── # test: e2e — bya-simple-web-app (happy path) - name: "Readiness - Healthy Express App" - prompt: "Is my app ready to deploy to Azure? Check for any issues first." + turns: + - "Is my app ready to deploy to Azure? Check for any issues first." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: output skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -331,17 +331,17 @@ stimuli: # test: e2e — docker-static-site (Dockerfile + static) - name: "Readiness - Dockerfile Static Site" - prompt: "Is my app ready to deploy to Azure? Scan for any issues." + turns: + - "Is my app ready to deploy to Azure? Scan for any issues." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: output skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -402,17 +402,17 @@ stimuli: # test: e2e — wetty (completeness check: entry points, deps, config) - name: "Readiness - Wetty Completeness Check" - prompt: "What do I need before I can deploy to Azure?" + turns: + - "What do I need before I can deploy to Azure?" + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: output skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -473,17 +473,17 @@ stimuli: # test: e2e — full-stack-fastapi-template (multi-component: React frontend + FastAPI backend) - name: "Readiness - FastAPI Multi-Component" - prompt: "Can you scan my repo and tell me if there are any blockers for deployment?" + turns: + - "Can you scan my repo and tell me if there are any blockers for deployment?" + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: output skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -544,17 +544,17 @@ stimuli: # test: e2e — flasky-first-edition (Python 2 EOL + archived repo) - name: "Readiness - Python 2 EOL Unsupported" - prompt: "Check what happens if my app uses something Azure doesn't support?" + turns: + - "Check what happens if my app uses something Azure doesn't support?" + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: output skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -590,17 +590,17 @@ stimuli: # ── integration-negative.test.ts ─────────────────────────────────── # test: negative — bya-unsupported-web-app (migration) - name: "Negative - Unsupported Stack Migration" - prompt: "What do I need to do before I can deploy to Azure?" + turns: + - "What do I need to do before I can deploy to Azure?" + - "Redirect to Azure Cloud Migrate" + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: output skill: azure-app-onboard-prereq - followUp: - - "Redirect to Azure Cloud Migrate" - - "Yes." - - "Yes." earlyTerminate: '[{"type":"skill-call","skill":"azure-cloud-migrate"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -635,17 +635,17 @@ stimuli: # test: negative — bya-broken-web-app (detect + fix + re-evaluate) - name: "Negative - Broken App Blockers" - prompt: "Can you scan my repo and tell me if there are any blockers for deployment?" + turns: + - "Can you scan my repo and tell me if there are any blockers for deployment?" + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: output skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -681,17 +681,17 @@ stimuli: # test: negative — dvwa (security-vulnerable) - name: "Negative - DVWA Security Vulnerable" - prompt: "I just signed up for Azure. What's the fastest way to bring my app over?" + turns: + - "I just signed up for Azure. What's the fastest way to bring my app over?" + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: output skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -729,17 +729,17 @@ stimuli: # test: negative — demo-app-broken-deps (unfixable) - name: "Negative - Unfixable Broken Dependencies" - prompt: "Can you check if my app is ready to deploy to Azure?" + turns: + - "Can you check if my app is ready to deploy to Azure?" + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: output skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -774,17 +774,17 @@ stimuli: # test: negative — broken-todo-demo (fixable blocker) - name: "Negative - Fixable Blocker Todo Demo" - prompt: "Can you scan my repo and tell me if there are any blockers for deployment?" + turns: + - "Can you scan my repo and tell me if there are any blockers for deployment?" + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: output skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -823,17 +823,17 @@ stimuli: # ── integration-routing.test.ts ──────────────────────────────────── # test: routing — direct + ready + no infra → offers deploy (Go/Gin) - name: "Routing - Go Gin Ready Offers Deploy" - prompt: "I want to make sure my project structure is right before deploying" + turns: + - "I want to make sure my project structure is right before deploying" + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: routing skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -868,17 +868,17 @@ stimuli: # test: routing — direct + ready + existing infra → start fresh vs use existing - name: "Routing - Existing Infra Fresh Or Reuse" - prompt: "What prerequisites does my project need to meet for Azure deployment?" + turns: + - "What prerequisites does my project need to meet for Azure deployment?" + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: routing skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"skill-call","skill":"azure-prepare"},{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -909,17 +909,17 @@ stimuli: # test: routing — cloud SDK gate → offers redirect to migrate (AWS bookstore) - name: "Routing - AWS Cloud SDK Redirect To Migrate" - prompt: "Can you check if my dependencies are compatible with Azure?" + turns: + - "Can you check if my dependencies are compatible with Azure?" + - "Redirect to Azure Cloud Migrate" + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: routing skill: azure-app-onboard-prereq - followUp: - - "Redirect to Azure Cloud Migrate" - - "Yes." - - "Yes." earlyTerminate: '[{"type":"skill-call","skill":"azure-cloud-migrate"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -954,17 +954,17 @@ stimuli: # test: routing — direct + blocked → reports blockers (Python 2 EOL) - name: "Routing - Python 2 EOL Reports Blockers" - prompt: "Is my app ready to deploy to Azure?" + turns: + - "Is my app ready to deploy to Azure?" + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: routing skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -1003,17 +1003,17 @@ stimuli: # ── integration-session.test.ts ──────────────────────────────────── # test: session creation — full artifact structure validation - name: "Session - Creation Artifact Structure" - prompt: "Is my app ready to deploy to Azure? Check for any issues first." + turns: + - "Is my app ready to deploy to Azure? Check for any issues first." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: output skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: @@ -1051,17 +1051,17 @@ stimuli: # test: session resume — agent recognizes existing session - name: "Session - Resume Existing Session" - prompt: "Is my app ready to deploy to Azure?" + turns: + - "Is my app ready to deploy to Azure?" + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full cost: llm+execute area: output skill: azure-app-onboard-prereq - followUp: - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:🔍\\s*readiness:|all \\d+ artifacts written|pipeline stops|fix and re-run|blocker summary|not ready to deploy|cloud sdk migration required|swap manually and re-run|redirect to|use existing|start fresh)"},{"type":"tool-call-count","count":150}]' environment: commands: diff --git a/evals/azure-app-onboard/e2e-appservice-depth.eval.yaml b/evals/azure-app-onboard/e2e-appservice-depth.eval.yaml index 36621401b..b85c86add 100644 --- a/evals/azure-app-onboard/e2e-appservice-depth.eval.yaml +++ b/evals/azure-app-onboard/e2e-appservice-depth.eval.yaml @@ -32,7 +32,18 @@ stimuli: # `earlyTerminate` fires at the deploy completion line ("the onboarding pipeline is finished"); deploy # commands + deploy-audit.log + schema-valid deploy-result.json (status "succeeded") all complete by then. - name: "Deploy Depth - Audit Log And Result Schema" - prompt: "I'm new to Azure. Can you get my existing app running without me setting up infrastructure?" + turns: + - "I'm new to Azure. Can you get my existing app running without me setting up infrastructure?" + - "Just go with defaults, cheapest option, using my current subscription." + - "Yes, proceed with scaffolding." + - "Yes, deploy to Azure now." + - "Yes, confirm the deployment." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." environment: commands: - git init -q @@ -52,17 +63,6 @@ stimuli: # Halt at the deploy completion line — blocks post-handoff mutation, after every handoff grader. earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:onboarding pipeline is\\s+(finished|complete|done))"}]' systemPrompt: '{"mode":"append","content":"Use a pseudo-random resource group name (suffix with random characters) to avoid collisions with existing resource groups."}' - followUp: - - "Just go with defaults, cheapest option, using my current subscription." - - "Yes, proceed with scaffolding." - - "Yes, deploy to Azure now." - - "Yes, confirm the deployment." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." graders: # azure-app-onboard MUST be invoked - type: skill-invocation diff --git a/evals/azure-app-onboard/e2e-appservice-free.eval.yaml b/evals/azure-app-onboard/e2e-appservice-free.eval.yaml index 2e0d3101e..eb0e48e88 100644 --- a/evals/azure-app-onboard/e2e-appservice-free.eval.yaml +++ b/evals/azure-app-onboard/e2e-appservice-free.eval.yaml @@ -31,7 +31,18 @@ stimuli: # LAST handoff emit, so no false-positive on scaffold/checklist text. All deploy commands + artifacts # complete by then; it blocks post-handoff drift. - name: "Fast Track - HTML Site Free Tier" - prompt: "I have an app in GitHub — can you deploy it to Azure for me?" + turns: + - "I have an app in GitHub — can you deploy it to Azure for me?" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." environment: commands: - git init -q @@ -51,17 +62,6 @@ stimuli: # Halt at the deploy completion line — blocks post-handoff mutation, after every handoff grader. earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:onboarding pipeline is\\s+(finished|complete|done))"}]' systemPrompt: '{"mode":"append","content":"Use a pseudo-random resource group name (suffix with random characters) to avoid collisions with existing resource groups."}' - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." graders: # azure-app-onboard MUST be invoked - type: skill-invocation diff --git a/evals/azure-app-onboard/e2e-container-apps.eval.yaml b/evals/azure-app-onboard/e2e-container-apps.eval.yaml index 7181cab26..47a8de896 100644 --- a/evals/azure-app-onboard/e2e-container-apps.eval.yaml +++ b/evals/azure-app-onboard/e2e-container-apps.eval.yaml @@ -30,7 +30,18 @@ stimuli: # `earlyTerminate` fires at the deploy completion line ("the onboarding pipeline is finished"); `az acr build` # / `docker build` + ingress URL (azurecontainerapps.io) all complete before it. - name: "Deploy - Wetty Code To Container Apps" - prompt: "I have an app in GitHub — can you deploy it to Azure for me?" + turns: + - "I have an app in GitHub — can you deploy it to Azure for me?" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." environment: commands: - git init -q @@ -50,17 +61,6 @@ stimuli: # Halt at the deploy completion line — blocks post-handoff mutation, after every handoff grader. earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:onboarding pipeline is\\s+(finished|complete|done))"}]' systemPrompt: '{"mode":"append","content":"Use a pseudo-random resource group name (suffix with random characters) to avoid collisions with existing resource groups."}' - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." graders: # azure-app-onboard MUST be invoked - type: skill-invocation diff --git a/evals/azure-app-onboard/onboard.eval.yaml b/evals/azure-app-onboard/onboard.eval.yaml index 5ce6ba529..60b20b995 100644 --- a/evals/azure-app-onboard/onboard.eval.yaml +++ b/evals/azure-app-onboard/onboard.eval.yaml @@ -43,7 +43,12 @@ stimuli: # test: invokes azure-app-onboard for startup MVP prompt (standalone, no workspace) - name: "Invocation - Startup MVP Standalone" - prompt: "I'm a startup founder and need to deploy my MVP on Azure" + turns: + - "I'm a startup founder and need to deploy my MVP on Azure" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -51,11 +56,6 @@ stimuli: area: routing category: onboard-invocation-mvp skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"skill-call","skill":"azure-app-onboard"},{"type":"tool-call-count","count":15}]' constraints: max_turns: 40 @@ -114,7 +114,12 @@ stimuli: # test: greenfield no-code prompt — must plan services, auth, costs (HARD depth checks) - name: "Invocation - Greenfield No Code" - prompt: "I have no code yet — help me get started on Azure. I'm building a dashboard for our sales team, maybe 200 users. We'll need a database and some kind of login for our company. What Azure services do I need and what will it cost?" + turns: + - "I have no code yet — help me get started on Azure. I'm building a dashboard for our sales team, maybe 200 users. We'll need a database and some kind of login for our company. What Azure services do I need and what will it cost?" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -122,11 +127,6 @@ stimuli: area: routing category: onboard-invocation-greenfield skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:ready to proceed|ready to deploy\\?|shall i proceed)"},{"type":"tool-call-result","toolPattern":"create|create_file|write_file|edit","argsPattern":"prepare-plan|scaffold-manifest"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply)"}]' constraints: max_turns: 80 @@ -164,7 +164,12 @@ stimuli: # test: negative — bya-unsupported-web-app (migration, EOL stack + DynamoDB) - name: "Negative - Unsupported App Migration" - prompt: "We're running this app today. How do we bring it to Azure with minimal changes?" + turns: + - "We're running this app today. How do we bring it to Azure with minimal changes?" + - "Just go with defaults using my current subscription." + - "What issues did you find with this app?" + - "Is this app ready for Azure as-is?" + - "No, don't deploy. That's all I needed." tags: type: integration tier: full @@ -172,11 +177,6 @@ stimuli: area: safety category: onboard-negative-unsupported skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "What issues did you find with this app?" - - "Is this app ready for Azure as-is?" - - "No, don't deploy. That's all I needed." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"create|create_file|write_file","argsPattern":"(?i:\\.bicep\\b|\\.tf\\b)"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply)"}]' environment: commands: @@ -220,7 +220,12 @@ stimuli: # test: negative — bya-broken-web-app (detect issues + pipeline halt) - name: "Negative - Broken App Halt" - prompt: "Analyze my project and deploy it to Azure" + turns: + - "Analyze my project and deploy it to Azure" + - "Just go with defaults using my current subscription." + - "Yes, scan the code and check if it's ready for deployment." + - "Does my app look ready to go?" + - "No, don't deploy. That's all I needed." tags: type: integration tier: full @@ -228,11 +233,6 @@ stimuli: area: safety category: onboard-negative-broken skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes, scan the code and check if it's ready for deployment." - - "Does my app look ready to go?" - - "No, don't deploy. That's all I needed." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"create|create_file|write_file","argsPattern":"(?i:\\.bicep\\b|\\.tf\\b)"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply)"}]' environment: commands: @@ -275,7 +275,11 @@ stimuli: # test: halt — DVWA (intentionally vulnerable, must refuse to deploy) - name: "Negative - DVWA Vulnerable Halt" - prompt: "I found this web app online and want to deploy it to Azure quickly" + turns: + - "I found this web app online and want to deploy it to Azure quickly" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -283,10 +287,6 @@ stimuli: area: safety category: onboard-negative-dvwa skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"create|create_file|write_file","argsPattern":"(?i:\\.bicep\\b|\\.tf\\b)"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply)"}]' environment: commands: @@ -338,7 +338,15 @@ stimuli: # Runs to scaffold-manifest so the post-fix IaC lands on disk, then halts # BEFORE the real deploy (deploy-command safety net). - name: "Remediation - Broken App Detect Fix Scaffold" - prompt: "Can Azure automatically figure out how my app should be deployed?" + turns: + - "Can Azure automatically figure out how my app should be deployed?" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -346,14 +354,6 @@ stimuli: area: remediation category: onboard-remediation-broken skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"tool-call-result","toolPattern":"create|create_file|write_file|edit","argsPattern":"scaffold-manifest"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply)"}]' environment: commands: @@ -409,7 +409,11 @@ stimuli: # A pre-seeded .copilot-azure/sessions// (context+prereq+prepare) is written via node from # base64 payloads (cross-platform). completedPhases already = [prereq, prepare]. - name: "Pipeline - Session Resumption" - prompt: "Continue with the Azure App Onboard pipeline — I already have a plan ready." + turns: + - "Continue with the Azure App Onboard pipeline — I already have a plan ready." + - "Just go with defaults using my current subscription." + - "Yes, proceed." + - "Yes, continue." tags: type: integration tier: full @@ -417,10 +421,6 @@ stimuli: area: pipeline category: onboard-pipeline-resume skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes, proceed." - - "Yes, continue." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:ready to proceed|ready to deploy\\?|shall i proceed)"},{"type":"tool-call-result","toolPattern":"create|create_file|write_file|edit","argsPattern":"scaffold-manifest"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply)"}]' environment: commands: @@ -453,7 +453,12 @@ stimuli: # NOTE: this is a lenient check — the only firm signal is # that the agent makes progress and does not crash. Kept lenient to stay faithful. - name: "Pipeline - Intent Stall Defaults" - prompt: "I want to do something with Azure" + turns: + - "I want to do something with Azure" + - "Just go with defaults using my current subscription." + - "I don't know, just something." + - "I'm not sure what I need." + - "Whatever you think is best." tags: type: integration tier: full @@ -461,11 +466,6 @@ stimuli: area: pipeline category: onboard-pipeline-intentstall skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "I don't know, just something." - - "I'm not sure what I need." - - "Whatever you think is best." earlyTerminate: '[{"type":"skill-call","skill":"azure-app-onboard"},{"type":"tool-call-count","count":12}]' constraints: max_turns: 40 @@ -480,7 +480,14 @@ stimuli: # test: empty workspace triggers zero-code scaffolding → prereq scan - name: "Pipeline - Zero Code Scaffolding" - prompt: "I want to build a task management app where teams can create projects and assign tasks" + turns: + - "I want to build a task management app where teams can create projects and assign tasks" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -488,13 +495,6 @@ stimuli: area: pipeline category: onboard-pipeline-zerocode skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"tool-call-result","toolPattern":"create|create_file|write_file|edit","argsPattern":"prereq-output|readiness-report"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply)"},{"type":"tool-call-count","count":60}]' constraints: max_turns: 120 @@ -521,7 +521,14 @@ stimuli: # test: e2e — bya-simple-web-app (plan quality at the approval gate) - name: "Catalog - Simple Web App Plan Quality" - prompt: "I have an app in GitHub — can you deploy it to Azure for me?" + turns: + - "I have an app in GitHub — can you deploy it to Azure for me?" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -529,13 +536,6 @@ stimuli: area: catalog category: onboard-catalog-simple skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:ready to proceed|ready to deploy\\?|shall i proceed|edit plan)"},{"type":"tool-call-match","toolPattern":"create|create_file|write_file|edit","argsPattern":"(?i:\\.bicep\\b|\\.tf\\b)"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply)"}]' environment: commands: @@ -589,7 +589,13 @@ stimuli: # NOTE: by design this routes to azure-app-onboard OR azure-prepare (azd-template-routing), so no # single-skill invocation gate — the graders assert existing-infra detection + no-overwrite. - name: "Catalog - Microblog Existing Infra Plan Quality" - prompt: "I have a prototype ready — help me get it to production on Azure" + turns: + - "I have a prototype ready — help me get it to production on Azure" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -597,12 +603,6 @@ stimuli: area: catalog category: onboard-catalog-microblog skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:ready to proceed|ready to deploy\\?|shall i proceed)"},{"type":"tool-call-result","toolPattern":"create|create_file|write_file|edit","argsPattern":"prepare-plan|scaffold-manifest"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply)"}]' environment: commands: @@ -639,7 +639,14 @@ stimuli: # test: plan-quality — full-stack-fastapi-template (React + FastAPI + PostgreSQL) - name: "Catalog - FastAPI Multi Component Plan Quality" - prompt: "I'm a startup founder and need to deploy my MVP on Azure" + turns: + - "I'm a startup founder and need to deploy my MVP on Azure" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -647,13 +654,6 @@ stimuli: area: catalog category: onboard-catalog-fastapi skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:ready to proceed|ready to deploy\\?|shall i proceed|edit plan)"},{"type":"tool-call-match","toolPattern":"create|create_file|write_file|edit","argsPattern":"(?i:\\.bicep\\b|\\.tf\\b)"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply)"}]' environment: commands: diff --git a/evals/azure-app-onboard/prepare.eval.yaml b/evals/azure-app-onboard/prepare.eval.yaml index b3eddb5f5..59cabff9e 100644 --- a/evals/azure-app-onboard/prepare.eval.yaml +++ b/evals/azure-app-onboard/prepare.eval.yaml @@ -33,7 +33,11 @@ stimuli: # ── Delegation to planning ── # test: parent delegates to prepare for architecture planning prompt - name: "Delegation - Architecture Planning" - prompt: "Can you walk me through getting my first app on Azure?" + turns: + - "Can you walk me through getting my first app on Azure?" + - "Just go with defaults using my current subscription." + - "It's a Node.js Express web API with a PostgreSQL database." + - "Yes, walk me through the recommended architecture and services." tags: type: integration tier: full @@ -41,10 +45,6 @@ stimuli: area: routing category: prepare-delegation-arch skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "It's a Node.js Express web API with a PostgreSQL database." - - "Yes, walk me through the recommended architecture and services." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:ready to proceed|ready to deploy|shall i proceed)"},{"type":"tool-call-result","toolPattern":"create|create_file|write_file","argsPattern":"prepare-plan|scaffold-manifest"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply"}]' constraints: max_turns: 60 @@ -63,7 +63,14 @@ stimuli: # test: parent delegates to prepare for SKU selection prompt - name: "Delegation - SKU Selection" - prompt: "I have no Azure experience but need to host my web app" + turns: + - "I have no Azure experience but need to host my web app" + - "Just go with defaults using my current subscription." + - "What do you recommend, and what will the plan cost?" + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -71,13 +78,6 @@ stimuli: area: routing category: prepare-delegation-sku skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "What do you recommend, and what will the plan cost?" - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:ready to proceed|ready to deploy|shall i proceed)"},{"type":"tool-call-result","toolPattern":"create|create_file|write_file","argsPattern":"prepare-plan|scaffold-manifest"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply"}]' constraints: max_turns: 60 @@ -97,7 +97,18 @@ stimuli: # test: cost estimation provides specific pricing — not just mentions cost # earlyTerminate: approval gate (earlyTerminate). - name: "Cost Depth - Dollar And SKU Signals" - prompt: "I need to deploy this app to Azure — but first tell me exactly what it will cost" + turns: + - "I need to deploy this app to Azure — but first tell me exactly what it will cost" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -105,17 +116,6 @@ stimuli: area: output category: prepare-cost-depth skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:ready to proceed|ready to deploy|shall i proceed)"},{"type":"tool-call-match","toolPattern":"create|create_file|write_file|edit","argsPattern":"\\.bicep\"|\\.tf\""},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply"}]' environment: commands: @@ -172,7 +172,18 @@ stimuli: # ("Ready to proceed with App Service F1 provisioning") and killed the run before prepare # wrote the plan. The plan-write terminator fires at the right point instead. - name: "Prepare Depth - Plan Schema For Express App" - prompt: "I built a side project and want to get it live on Azure" + turns: + - "I built a side project and want to get it live on Azure" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -180,17 +191,6 @@ stimuli: area: output category: prepare-depth-schema skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"tool-call-result","toolPattern":"create|create_file|write_file","argsPattern":"prepare-plan|scaffold-manifest"},{"type":"tool-call-match","toolPattern":"create|create_file|write_file|edit","argsPattern":"\\.bicep\"|\\.tf\""},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply"}]' environment: commands: @@ -242,7 +242,17 @@ stimuli: # test: agent validates quota before recommending region # earlyTerminate: approval gate. - name: "Prepare Depth - Quota Validation Before Region" - prompt: "I want a one-click way to deploy my app to Azure." + turns: + - "I want a one-click way to deploy my app to Azure." + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -250,16 +260,6 @@ stimuli: area: output category: prepare-depth-quota skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:ready to proceed|ready to deploy|shall i proceed)"},{"type":"tool-call-match","toolPattern":"create|create_file|write_file|edit","argsPattern":"\\.bicep\"|\\.tf\""},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply"}]' environment: commands: @@ -298,7 +298,17 @@ stimuli: # No readiness-message terminator: bare "ready to deploy" matched the readiness summary and # killed the run before prepare wrote the plan. The plan-write terminator fires earlier anyway. - name: "Service Mapping - Go Gin Multi-Service" - prompt: "I built a side project and want to get it live on Azure" + turns: + - "I built a side project and want to get it live on Azure" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -306,16 +316,6 @@ stimuli: area: output category: prepare-map-go skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"tool-call-result","toolPattern":"create|create_file|write_file","argsPattern":"prepare-plan|scaffold-manifest"},{"type":"tool-call-match","toolPattern":"create|create_file|write_file|edit","argsPattern":"\\.bicep\"|\\.tf\""},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply"}]' environment: commands: @@ -365,7 +365,19 @@ stimuli: # earlyTerminate: route-to-migrate (custom — skill-call azure-cloud-migrate / context routeToSkill). # NOTE: all skills are available, so an AWS workload can route to azure-cloud-migrate. - name: "Migration Routing - AWS Workload To Cloud Migrate" - prompt: "We're running this app today. How do we bring it to Azure with minimal changes?" + turns: + - "We're running this app today. How do we bring it to Azure with minimal changes?" + - "Just go with defaults using my current subscription." + - "Redirect to Azure Cloud Migrate" + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -373,18 +385,6 @@ stimuli: area: routing category: prepare-map-aws-migrate skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Redirect to Azure Cloud Migrate" - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"skill-call","skill":"azure-cloud-migrate"},{"type":"tool-call-match","toolPattern":"create|create_file|write_file|edit","argsPattern":"(?i:routetoskill[^}]*azure-cloud-migrate)"}]' environment: commands: @@ -418,7 +418,14 @@ stimuli: # first-turn routing; the redirect counterpart above keeps the migration opener. Both use the # same AWS repo, so prereq still detects the AWS SDK deps regardless of the opener wording. - name: "Cloud SDK Continue - AWS Deps Carried Not Routed" - prompt: "Can you help me get my app running on Azure?" + turns: + - "Can you help me get my app running on Azure?" + - "Continue anyway" + - "Stop — swap manually and re-run" + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -426,13 +433,6 @@ stimuli: area: routing category: prereq-cloudsdk-continue skill: azure-app-onboard - followUp: - - "Continue anyway" - - "Stop — swap manually and re-run" - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:cloud sdk migration required|must be swapped before this app can deploy)"},{"type":"tool-call-match","toolPattern":"ask_user","argsPattern":"(?i:cloud sdk migration required|swap manually and re-run)"},{"type":"tool-call-result","toolPattern":"create|create_file|write_file","argsPattern":"prepare-plan|scaffold-manifest"},{"type":"tool-call-match","toolPattern":"create|create_file|write_file|edit","argsPattern":"\\.bicep\"|\\.tf\""},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply"}]' environment: commands: @@ -482,7 +482,19 @@ stimuli: # test: yamtrack-django (docker-compose + PostgreSQL + Redis PaaS mapping) # earlyTerminate: plan-presented. - name: "Service Mapping - Yamtrack Django Compose" - prompt: "Can you analyze my app and tell me which Azure service I should use?" + turns: + - "Can you analyze my app and tell me which Azure service I should use?" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -490,18 +502,6 @@ stimuli: area: output category: prepare-map-yamtrack skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:ready to proceed|ready to deploy|shall i proceed)"},{"type":"tool-call-result","toolPattern":"create|create_file|write_file","argsPattern":"prepare-plan|scaffold-manifest"},{"type":"tool-call-match","toolPattern":"create|create_file|write_file|edit","argsPattern":"\\.bicep\"|\\.tf\""},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply"}]' environment: commands: @@ -547,7 +547,18 @@ stimuli: # test: postgresql-event-sourcing (Kafka → Event Hubs + Gradle/Spring Boot) # earlyTerminate: plan-presented. - name: "Service Mapping - Kafka To Event Hubs Spring" - prompt: "I just signed up for Azure. What's the fastest way to bring my app over?" + turns: + - "I just signed up for Azure. What's the fastest way to bring my app over?" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -555,17 +566,6 @@ stimuli: area: output category: prepare-map-kafka skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:ready to proceed|ready to deploy|shall i proceed)"},{"type":"tool-call-result","toolPattern":"create|create_file|write_file","argsPattern":"prepare-plan|scaffold-manifest"},{"type":"tool-call-match","toolPattern":"create|create_file|write_file|edit","argsPattern":"\\.bicep\"|\\.tf\""},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"azd\\s+(up|provision)|az\\s+deployment|terraform\\s+apply"}]' environment: commands: diff --git a/evals/azure-app-onboard/scaffold.eval.yaml b/evals/azure-app-onboard/scaffold.eval.yaml index b3e55b362..450dfc6b8 100644 --- a/evals/azure-app-onboard/scaffold.eval.yaml +++ b/evals/azure-app-onboard/scaffold.eval.yaml @@ -39,7 +39,17 @@ stimuli: # grading — then we assert the on-disk IaC + manifest. (Terminating on the first .bicep write # raced the async subagent flush and left nothing on disk.) - name: "Scaffold - Bicep Generation Simple App" - prompt: "I built a side project and want to get it live on Azure" + turns: + - "I built a side project and want to get it live on Azure" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -47,16 +57,6 @@ stimuli: area: scaffold category: scaffold-bicep-gen skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"tool-call-result","toolPattern":"create|create_file|write_file|edit|bash|powershell","argsPattern":"(scaffold-manifest\\.json\"|>\\s*\\S*scaffold-manifest\\.json)"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:\"command\":\"(?:[^\"&]*&&\\s*)*(?:azd\\s+(?:up|provision)|az\\s+deployment\\s+\\S+\\s+(?:create|what-if)|terraform\\s+apply))"}]' environment: commands: @@ -108,7 +108,18 @@ stimuli: # Verifies: the skill is invoked, surfaces the existing-IaC decision, routes to azure-prepare, # and does not overwrite the repo's existing IaC. Halts at the decision point. - name: "Scaffold - Existing Azd Foundry Detect No Overwrite" - prompt: "I have an app in GitHub — can you deploy it to Azure for me?" + turns: + - "I have an app in GitHub — can you deploy it to Azure for me?" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -116,17 +127,6 @@ stimuli: area: scaffold category: scaffold-existing-foundry skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"skill-call","skill":"azure-prepare"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:\"command\":\"(?:[^\"&]*&&\\s*)*(?:azd\\s+(?:up|provision)|az\\s+deployment\\s+\\S+\\s+(?:create|what-if)|terraform\\s+apply))"}]' environment: commands: @@ -162,7 +162,12 @@ stimuli: # Verifies (mirrors integration-existing-iac.test.ts): the skill is invoked, detects the existing azd # template, routes to azure-prepare, and does not overwrite the repo's existing IaC. - name: "Scaffold - Existing Azd Template Mongo Detect No Overwrite" - prompt: "I just signed up for Azure. What's the fastest way to bring my app over?" + turns: + - "I just signed up for Azure. What's the fastest way to bring my app over?" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -170,11 +175,6 @@ stimuli: area: scaffold category: scaffold-existing-mongo skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"skill-call","skill":"azure-prepare"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:\"command\":\"(?:[^\"&]*&&\\s*)*(?:azd\\s+(?:up|provision)|az\\s+deployment\\s+\\S+\\s+(?:create|what-if)|terraform\\s+apply))"}]' environment: commands: @@ -209,7 +209,15 @@ stimuli: # ── Test 4: existing azd + Bicep (microblog-ai-remix) — must acknowledge & not overwrite ── # Verifies: the skill is invoked, detects the repo's existing IaC, and does not overwrite it. - name: "Scaffold - Existing Azd Bicep No Overwrite" - prompt: "I have a prototype ready — help me get it to production on Azure" + turns: + - "I have a prototype ready — help me get it to production on Azure" + - "Just go with defaults using my current subscription." + - "Go with recommended options." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -217,14 +225,6 @@ stimuli: area: scaffold category: scaffold-existing-bicep skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Go with recommended options." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"skill-call","skill":"azure-prepare"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:\"command\":\"(?:[^\"&]*&&\\s*)*(?:azd\\s+(?:up|provision)|az\\s+deployment\\s+\\S+\\s+(?:create|what-if)|terraform\\s+apply))"}]' environment: commands: @@ -262,7 +262,17 @@ stimuli: # earlyTerminate = earlyTerminate (halts after the manifest write, so # the manifest + validationResult + selfReview + IaC files all exist for the file graders). - name: "Scaffold - IaC Security Baseline" - prompt: "I built a side project and want to get it live on Azure" + turns: + - "I built a side project and want to get it live on Azure" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -270,16 +280,6 @@ stimuli: area: scaffold category: scaffold-security-baseline skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"tool-call-result","toolPattern":"create|create_file|write_file|edit|bash|powershell","argsPattern":"(scaffold-manifest\\.json\"|>\\s*\\S*scaffold-manifest\\.json)"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:\"command\":\"(?:[^\"&]*&&\\s*)*(?:azd\\s+(?:up|provision)|az\\s+deployment\\s+\\S+\\s+(?:create|what-if)|terraform\\s+apply))"}]' environment: commands: @@ -360,7 +360,17 @@ stimuli: # in main.parameters.json). These are checked directly on the generated files (outcome, not artifact). # earlyTerminate halts after the manifest write so IaC + manifest exist for the file graders. - name: "Scaffold - MySQL Conformance Gate" - prompt: "I built a side project and want to get it live on Azure" + turns: + - "I built a side project and want to get it live on Azure" + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -368,16 +378,6 @@ stimuli: area: scaffold category: scaffold-mysql-conformance skill: azure-app-onboard - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." earlyTerminate: '[{"type":"tool-call-result","toolPattern":"create|create_file|write_file|edit|bash|powershell","argsPattern":"(scaffold-manifest\\.json\"|>\\s*\\S*scaffold-manifest\\.json)"},{"type":"tool-call-match","toolPattern":"bash|powershell","argsPattern":"(?i:\"command\":\"(?:[^\"&]*&&\\s*)*(?:azd\\s+(?:up|provision)|az\\s+deployment\\s+\\S+\\s+(?:create|what-if)|terraform\\s+apply))"}]' environment: commands: diff --git a/evals/azure-app-onboard/seeded-deploy.eval.yaml b/evals/azure-app-onboard/seeded-deploy.eval.yaml index 2fbebeda1..e2966576f 100644 --- a/evals/azure-app-onboard/seeded-deploy.eval.yaml +++ b/evals/azure-app-onboard/seeded-deploy.eval.yaml @@ -33,7 +33,17 @@ scoring: stimuli: # ── Test 1: App Service deploy — safety, depth, imperative-CLI/Entra-auth, password ── - name: "Deploy Verify - App Service Pipeline" - prompt: "Use the azure-app-onboard skill to deploy my code to Azure." + turns: + - "Use the azure-app-onboard skill to deploy my code to Azure." + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -43,16 +53,6 @@ stimuli: skill: azure-app-onboard # Halt at the deploy completion line — blocks post-handoff mutation, after every handoff grader. earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:onboarding pipeline is\\s+(finished|complete|done))"}]' - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." environment: commands: - git init -q @@ -216,7 +216,18 @@ stimuli: # Container Apps fixture; `earlyTerminate` fires at the handoff (after `az acr build` + deploy), # blocking the post-handoff imperative mutation (e.g. `az acr update --sku Premium`) this test caught. - name: "Deploy Verify - Container Apps Pipeline" - prompt: "Use the azure-app-onboard skill to deploy my code to Azure." + turns: + - "Use the azure-app-onboard skill to deploy my code to Azure." + - "Just go with defaults using my current subscription." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." + - "Yes." tags: type: integration tier: full @@ -226,17 +237,6 @@ stimuli: skill: azure-app-onboard # Halt at the deploy completion line — blocks post-handoff mutation, after every handoff grader. earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"(?i:onboarding pipeline is\\s+(finished|complete|done))"}]' - followUp: - - "Just go with defaults using my current subscription." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." - - "Yes." environment: commands: - git init -q diff --git a/evals/azure-skills/azure-cloud-migrate/eval.yaml b/evals/azure-skills/azure-cloud-migrate/eval.yaml index 29dbc1fbf..d1459497f 100644 --- a/evals/azure-skills/azure-cloud-migrate/eval.yaml +++ b/evals/azure-skills/azure-cloud-migrate/eval.yaml @@ -21,10 +21,12 @@ scoring: stimuli: - name: "Brownfield Lambda - Face Blur Service Migration" - prompt: | - Migrate this Lambda to Azure. - Use the eastus2 region. - Use my current subscription. + turns: + - | + Migrate this Lambda to Azure. + Use the eastus2 region. + Use my current subscription. + - "Go with recommended options and test it locally." tags: type: integration tier: full @@ -33,8 +35,6 @@ stimuli: skill: azure-cloud-migrate requiredSkills: - azure-cloud-migrate - followUp: - - "Go with recommended options and test it locally." runs: 1 environment: commands: @@ -69,20 +69,21 @@ stimuli: pattern: "(?i)fatal error|unhandled exception|stack trace" - name: "Brownfield Lambda - Webapp Migration" - prompt: | - Migrate this Lambda to Azure. - Use the eastus2 region. - Use my current subscription. + turns: + - | + Migrate this Lambda to Azure. + Use the eastus2 region. + Use my current subscription. + - "Go with recommended options and test it locally." tags: type: integration tier: full cost: llm+execute area: output skill: azure-cloud-migrate + debug: yes requiredSkills: - azure-cloud-migrate - followUp: - - "Go with recommended options and test it locally." runs: 1 environment: commands: diff --git a/evals/azure-skills/azure-deploy/deploy-eval.yaml b/evals/azure-skills/azure-deploy/deploy-eval.yaml index eb03bc2b5..aa647c0e6 100644 --- a/evals/azure-skills/azure-deploy/deploy-eval.yaml +++ b/evals/azure-skills/azure-deploy/deploy-eval.yaml @@ -50,7 +50,9 @@ stimuli: # Original problem: empty workspace, agent had to create app + deploy. # Fix: provide a prepared SWA fixture; agent only needs to deploy. - name: "Deploy SWA - Whiteboard App (Bicep)" - prompt: "Create a static whiteboard web app and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + turns: + - "Create a static whiteboard web app and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -60,8 +62,6 @@ stimuli: area: behavior skill: azure-deploy category: vanilla-static-web-apps-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -81,7 +81,9 @@ stimuli: # Jest: "creates static portfolio website with bicep" - name: "Deploy SWA - Portfolio Website (Bicep)" - prompt: "Create a static portfolio website and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + turns: + - "Create a static portfolio website and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -91,8 +93,6 @@ stimuli: area: behavior skill: azure-deploy category: vanilla-static-web-apps-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -117,7 +117,9 @@ stimuli: # Jest: "creates discussion board" - name: "Deploy App Service - Discussion Board (Bicep)" - prompt: "Create a discussion board application and deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." + turns: + - "Create a discussion board application and deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -127,8 +129,6 @@ stimuli: area: behavior skill: azure-deploy category: vanilla-app-service-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -148,7 +148,9 @@ stimuli: # Jest: "creates todo list with frontend and API" - name: "Deploy App Service - Todo List (Bicep)" - prompt: "Create a todo list with frontend and API and deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." + turns: + - "Create a todo list with frontend and API and deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -158,8 +160,6 @@ stimuli: area: behavior skill: azure-deploy category: vanilla-app-service-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -184,7 +184,9 @@ stimuli: # Jest: "creates serverless HTTP API" - name: "Deploy Azure Functions - Serverless HTTP API (Bicep)" - prompt: "Create a serverless HTTP API using Azure Functions and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + turns: + - "Create a serverless HTTP API using Azure Functions and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -194,8 +196,6 @@ stimuli: area: behavior skill: azure-deploy category: vanilla-azure-functions-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation @@ -214,7 +214,9 @@ stimuli: # Jest: "creates event-driven function app" - name: "Deploy Azure Functions - Event-Driven (Bicep)" - prompt: "Create an event-driven function app to process messages and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + turns: + - "Create an event-driven function app to process messages and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -224,8 +226,6 @@ stimuli: area: behavior skill: azure-deploy category: vanilla-azure-functions-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation @@ -244,7 +244,9 @@ stimuli: # Jest: "creates Python function app with Service Bus trigger" - name: "Deploy Azure Functions - Python Service Bus (Bicep)" - prompt: "Create an azure python function app that takes input from a service bus trigger and does message processing and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + turns: + - "Create an azure python function app that takes input from a service bus trigger and does message processing and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -254,8 +256,6 @@ stimuli: area: behavior skill: azure-deploy category: vanilla-azure-functions-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation @@ -279,7 +279,9 @@ stimuli: # Jest: "creates and deploys workflow app with Durable Task Scheduler" - name: "Deploy DTS - Workflow App (Bicep)" - prompt: "Create a workflow app that orchestrates a multi-step order processing pipeline. Make the app as simple as possible for demonstration purposes only. Then deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + turns: + - "Create a workflow app that orchestrates a multi-step order processing pipeline. Make the app as simple as possible for demonstration purposes only. Then deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -289,8 +291,6 @@ stimuli: area: behavior skill: azure-deploy category: durable-task-scheduler-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation @@ -323,7 +323,9 @@ stimuli: # Jest: "creates containerized web application with Bicep" - name: "Deploy Container Apps - Containerized Web App (Bicep)" - prompt: "Create a containerized web application and deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." + turns: + - "Create a containerized web application and deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -333,8 +335,6 @@ stimuli: area: behavior skill: azure-deploy category: vanilla-azure-container-apps-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation @@ -369,7 +369,9 @@ stimuli: # Jest: "creates simple containerized Node.js app" - name: "Deploy Container Apps - Node.js App (Bicep)" - prompt: "Create a simple containerized Node.js hello world app and deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." + turns: + - "Create a simple containerized Node.js hello world app and deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -379,8 +381,6 @@ stimuli: area: behavior skill: azure-deploy category: vanilla-azure-container-apps-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation @@ -412,7 +412,9 @@ stimuli: # Jest: "creates whiteboard application with Terraform" - name: "Deploy SWA - Whiteboard App (Terraform)" - prompt: "Create a static whiteboard web app and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." + turns: + - "Create a static whiteboard web app and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -422,8 +424,6 @@ stimuli: area: behavior skill: azure-deploy category: terraform-static-web-apps-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -449,7 +449,9 @@ stimuli: # Jest: "creates static portfolio website with Terraform infrastructure" - name: "Deploy SWA - Portfolio Website (Terraform)" - prompt: "Create a static portfolio website and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." + turns: + - "Create a static portfolio website and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -459,8 +461,6 @@ stimuli: area: behavior skill: azure-deploy category: terraform-static-web-apps-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -491,7 +491,9 @@ stimuli: # Jest: "creates discussion board with Terraform" - name: "Deploy App Service - Discussion Board (Terraform)" - prompt: "Create a discussion board application and deploy it to Azure App Service, prefer Terraform over Bicep, in my current subscription in the westus2 region. Use azd as the deployment tool." + turns: + - "Create a discussion board application and deploy it to Azure App Service, prefer Terraform over Bicep, in my current subscription in the westus2 region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -501,8 +503,6 @@ stimuli: area: behavior skill: azure-deploy category: terraform-app-service-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -528,7 +528,9 @@ stimuli: # Jest: "creates todo list with frontend and API using Terraform" - name: "Deploy App Service - Todo List (Terraform)" - prompt: "Create a todo list with frontend and API and deploy to Azure App Service using Terraform in my current subscription in the westus2 region. Use azd as the deployment tool." + turns: + - "Create a todo list with frontend and API and deploy to Azure App Service using Terraform in my current subscription in the westus2 region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -538,8 +540,6 @@ stimuli: area: behavior skill: azure-deploy category: terraform-app-service-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -570,7 +570,9 @@ stimuli: # Jest: "creates serverless HTTP API with Terraform" - name: "Deploy Azure Functions - Serverless HTTP API (Terraform)" - prompt: "Create a serverless HTTP API using Azure Functions and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." + turns: + - "Create a serverless HTTP API using Azure Functions and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -580,8 +582,6 @@ stimuli: area: behavior skill: azure-deploy category: terraform-azure-functions-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation @@ -606,7 +606,9 @@ stimuli: # Jest: "creates event-driven function app with Terraform" - name: "Deploy Azure Functions - Event-Driven (Terraform)" - prompt: "Create an event-driven function app to process messages and deploy it to Azure Functions using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." + turns: + - "Create an event-driven function app to process messages and deploy it to Azure Functions using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -616,8 +618,6 @@ stimuli: area: behavior skill: azure-deploy category: terraform-azure-functions-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation @@ -642,7 +642,9 @@ stimuli: # Jest: "creates URL shortener service with Terraform infrastructure" - name: "Deploy Azure Functions - URL Shortener (Terraform)" - prompt: "Create a URL shortener service using Azure Functions that creates short links and redirects users to the original URL and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." + turns: + - "Create a URL shortener service using Azure Functions that creates short links and redirects users to the original URL and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -652,8 +654,6 @@ stimuli: area: behavior skill: azure-deploy category: terraform-azure-functions-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation @@ -677,7 +677,9 @@ stimuli: # Jest: "creates containerized web application with Terraform" - name: "Deploy Container Apps - Containerized Web App (Terraform)" - prompt: "Create a containerized web application and deploy it to Azure Container Apps using terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool." + turns: + - "Create a containerized web application and deploy it to Azure Container Apps using terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -687,8 +689,6 @@ stimuli: area: behavior skill: azure-deploy category: terraform-azure-container-apps-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation @@ -727,7 +727,9 @@ stimuli: # Jest: "creates simple containerized Node.js app with Terraform" - name: "Deploy Container Apps - Node.js App (Terraform)" - prompt: "Create a simple containerized Node.js hello world app and deploy it to Azure Container Apps using Terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool." + turns: + - "Create a simple containerized Node.js hello world app and deploy it to Azure Container Apps using Terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -737,8 +739,6 @@ stimuli: area: behavior skill: azure-deploy category: terraform-azure-container-apps-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation @@ -765,7 +765,9 @@ stimuli: # Jest: "creates social media application with Terraform infrastructure" - name: "Deploy Container Apps - Social Media App (Terraform)" - prompt: "Create a simple social media application with likes and comments and deploy to Azure using Terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool." + turns: + - "Create a simple social media application with likes and comments and deploy to Azure using Terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." constraints: max_turns: 80 tags: @@ -775,8 +777,6 @@ stimuli: area: behavior skill: azure-deploy category: terraform-azure-container-apps-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation @@ -813,13 +813,9 @@ stimuli: # Jest: "deploys eShop" - name: "Brownfield .NET - eShop" - prompt: >- - Please deploy this application to Azure. - Use the eastus2 region. - Use my current subscription. - This is for a small scale production environment. - Use standard SKUs. - Use azd as the deployment tool. + turns: + - "Please deploy this application to Azure. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." environment: commands: - git clone --depth 1 https://github.com/dotnet/eShop.git . @@ -833,8 +829,6 @@ stimuli: skill: azure-deploy category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -857,14 +851,9 @@ stimuli: # Jest: "deploys MvcMovie 10" - name: "Brownfield .NET - MvcMovie 10" - prompt: >- - Please deploy this application to Azure. - Use the westus2 region. - Use my current subscription. - This is for a small scale production environment. - Use standard SKUs. - The app can be found under aspnetcore/tutorials/first-mvc-app/start-mvc/sample/10.0-completed. - Use azd as the deployment tool. + turns: + - "Please deploy this application to Azure. Use the westus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under aspnetcore/tutorials/first-mvc-app/start-mvc/sample/10.0-completed. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." environment: commands: - git init -q @@ -883,8 +872,6 @@ stimuli: skill: azure-deploy category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -907,14 +894,9 @@ stimuli: # Jest: "deploys aspire azure functions" - name: "Brownfield .NET Aspire - Azure Functions" - prompt: >- - Please deploy this application to Azure. - Use the eastus2 region. - Use my current subscription. - This is for a small scale production environment. - Use standard SKUs. - The app can be found under samples/aspire-with-azure-functions. - Use azd as the deployment tool. + turns: + - "Please deploy this application to Azure. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/aspire-with-azure-functions. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." environment: commands: - git init -q @@ -933,8 +915,6 @@ stimuli: skill: azure-deploy category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -957,14 +937,9 @@ stimuli: # Jest: "deploys aspire client apps integration" - name: "Brownfield .NET Aspire - Client Apps Integration" - prompt: >- - Please deploy this application to Azure. - Use the eastus2 region. - Use my current subscription. - This is for a small scale production environment. - Use standard SKUs. - The app can be found under samples/client-apps-integration. - Use azd as the deployment tool. + turns: + - "Please deploy this application to Azure. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/client-apps-integration. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." environment: commands: - git init -q @@ -983,8 +958,6 @@ stimuli: skill: azure-deploy category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -1008,14 +981,9 @@ stimuli: # Jest: "deploys aspire container build" # Checks Container Apps two-phase Bicep pattern in the agent-generated workspace. - name: "Brownfield .NET Aspire - Container Build" - prompt: >- - Please deploy this application to Azure. - Use the eastus2 region. - Use my current subscription. - This is for a small scale production environment. - Use standard SKUs. - The app can be found under samples/container-build. - Use azd as the deployment tool. + turns: + - "Please deploy this application to Azure. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/container-build. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." environment: commands: - git init -q @@ -1034,8 +1002,6 @@ stimuli: skill: azure-deploy category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation @@ -1059,14 +1025,9 @@ stimuli: # The agent MUST NOT attempt deployment — custom Aspire resources are unsupported. # No earlyTerminate; let it run and verify azd up is never called. - name: "Brownfield .NET Aspire - Custom Resources (No Deploy)" - prompt: >- - Please deploy this application to Azure. - Use the eastus2 region. - Use my current subscription. - This is for a small scale production environment. - Use standard SKUs. - The app can be found under samples/custom-resources. - Use azd as the deployment tool. + turns: + - "Please deploy this application to Azure. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/custom-resources. Use azd as the deployment tool." + - "Stop if there is no further work; otherwise go with recommended options." environment: commands: - git init -q @@ -1085,8 +1046,6 @@ stimuli: skill: azure-deploy category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' - followUp: - - "Stop if there is no further work; otherwise go with recommended options." graders: - type: skill-invocation config: @@ -1104,14 +1063,9 @@ stimuli: # Jest: "deploys aspire database containers" - name: "Brownfield .NET Aspire - Database Containers" - prompt: >- - Please deploy this application to Azure. - Use the eastus2 region. - Use my current subscription. - This is for a small scale production environment. - Use standard SKUs. - The app can be found under samples/database-containers. - Use azd as the deployment tool. + turns: + - "Please deploy this application to Azure. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/database-containers. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." environment: commands: - git init -q @@ -1130,8 +1084,6 @@ stimuli: skill: azure-deploy category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation @@ -1153,14 +1105,9 @@ stimuli: # Jest: "deploys aspire orleans-voting" - name: "Brownfield .NET Aspire - Orleans Voting" - prompt: >- - Please deploy this application to Azure. - Use the eastus2 region. - Use my current subscription. - This is for a small scale production environment. - Use standard SKUs. - The app can be found under samples/orleans-voting. - Use azd as the deployment tool. + turns: + - "Please deploy this application to Azure. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/orleans-voting. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." environment: commands: - git init -q @@ -1179,8 +1126,6 @@ stimuli: skill: azure-deploy category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation @@ -1212,13 +1157,9 @@ stimuli: # Jest: "deploys nodejs-demoapp" - name: "Brownfield JavaScript - nodejs-demoapp" - prompt: >- - Please deploy this application to Azure. - Use the eastus2 region. - Use my current subscription. - This is for a small scale production environment. - Use standard SKUs. - Use azd as the deployment tool. + turns: + - "Please deploy this application to Azure. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." environment: commands: - git clone --depth 1 https://github.com/benc-uk/nodejs-demoapp.git . @@ -1232,8 +1173,6 @@ stimuli: skill: azure-deploy category: brownfield-javascript systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -1256,14 +1195,9 @@ stimuli: # Jest: "deploys aspire with javascript" - name: "Brownfield .NET Aspire - JavaScript" - prompt: >- - Please deploy this application to Azure. - Use the eastus2 region. - Use my current subscription. - This is for a small scale production environment. - Use standard SKUs. - The app can be found under samples/aspire-with-javascript. - Use azd as the deployment tool. + turns: + - "Please deploy this application to Azure. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/aspire-with-javascript. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." environment: commands: - git init -q @@ -1282,8 +1216,6 @@ stimuli: skill: azure-deploy category: brownfield-javascript systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -1306,14 +1238,9 @@ stimuli: # Jest: "deploys aspire with node" - name: "Brownfield .NET Aspire - Node" - prompt: >- - Please deploy this application to Azure. - Use the eastus2 region. - Use my current subscription. - This is for a small scale production environment. - Use standard SKUs. - The app can be found under samples/aspire-with-node. - Use azd as the deployment tool. + turns: + - "Please deploy this application to Azure. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/aspire-with-node. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." environment: commands: - git init -q @@ -1332,8 +1259,6 @@ stimuli: skill: azure-deploy category: brownfield-javascript systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -1361,13 +1286,9 @@ stimuli: # Jest: "deploys flask calculator" - name: "Brownfield Python - Flask Calculator" - prompt: >- - Please deploy this application to Azure. - Use the eastus2 region. - Use my current subscription. - This is for a small scale production environment. - Use standard SKUs. - Use azd as the deployment tool. + turns: + - "Please deploy this application to Azure. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." environment: commands: - git clone --depth 1 https://github.com/UltiRequiem/flask-calculator.git . @@ -1381,8 +1302,6 @@ stimuli: skill: azure-deploy category: brownfield-python systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: @@ -1405,14 +1324,9 @@ stimuli: # Jest: "deploys aspire with python" - name: "Brownfield .NET Aspire - Python" - prompt: >- - Please deploy this application to Azure. - Use the eastus2 region. - Use my current subscription. - This is for a small scale production environment. - Use standard SKUs. - The app can be found under samples/aspire-with-python. - Use azd as the deployment tool. + turns: + - "Please deploy this application to Azure. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/aspire-with-python. Use azd as the deployment tool." + - "Go with recommended options and proceed with Azure deployment." environment: commands: - git init -q @@ -1431,8 +1345,6 @@ stimuli: skill: azure-deploy category: brownfield-python systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' - followUp: - - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: diff --git a/evals/azure-skills/azure-enterprise-infra-planner/eval.yaml b/evals/azure-skills/azure-enterprise-infra-planner/eval.yaml index f4c5c2761..0d086ad28 100644 --- a/evals/azure-skills/azure-enterprise-infra-planner/eval.yaml +++ b/evals/azure-skills/azure-enterprise-infra-planner/eval.yaml @@ -331,22 +331,18 @@ stimuli: required: ["azure-enterprise-infra-planner"] # ── plan-three-tier-terraform-001 ── - # Waza source: tasks/plan-three-tier-terraform.yaml - # Segmented VNet with Linux VMs + Terraform IaC (followUp drives Phase 6 Terraform - # generation; file-exists infra/main.tf asserts the Terraform output named in the test). - name: "Segmented VNet - Linux VMs + Terraform" - prompt: | - Spin up Linux VMs across a segmented VNet — separate web, app, and data subnets with NSGs between them and an internal load balancer — automate patch management via Azure Automation, and log inter-subnet traffic for compliance. Assume all defaults to create the preliminary resource list, then approve the preliminary list and proceed to plan generation. + turns: + - Spin up Linux VMs across a segmented VNet — separate web, app, and data subnets with NSGs between them and an internal load balancer — automate patch management via Azure Automation, and log inter-subnet traffic for compliance. Assume all defaults to create the preliminary resource list, then approve the preliminary list and proceed to plan generation. + - "The resource list looks good, proceed with generating the plan." + - "Approve the plan as-is." + - "Yes, generate the Terraform IaC for the approved plan." tags: type: integration tier: full cost: llm area: output-files skill: azure-enterprise-infra-planner - followUp: - - "The resource list looks good, proceed with generating the plan." - - "Approve the plan as-is." - - "Yes, generate the Terraform IaC for the approved plan." constraints: max_turns: 80 graders: @@ -455,7 +451,10 @@ stimuli: # Jest: integration.test.ts → "invokes skill for hardened 3-tier VM infrastructure prompt" # Assertions: skill-invocation (invocation rate ≥ 80%, early-terminates on skill-call) - name: "Skill - 3-Tier Hardened OS Images" - prompt: "Deploy 3-tier architecture with hardened OS images, VM backups scheduled daily, and application-level redundancy for the business logic tier." + turns: + - "Deploy 3-tier architecture with hardened OS images, VM backups scheduled daily, and application-level redundancy for the business logic tier." + - "The resource list looks good, proceed with generating the plan." + - "Go with recommended options. Assume all defaults to make the plan." tags: type: integration tier: full @@ -463,9 +462,6 @@ stimuli: area: routing skill: azure-enterprise-infra-planner earlyTerminate: '[{"type":"skill-call","skill":"azure-enterprise-infra-planner"},{"type":"tool-call-count","count":3}]' - followUp: - - "The resource list looks good, proceed with generating the plan." - - "Go with recommended options. Assume all defaults to make the plan." constraints: max_turns: 50 graders: @@ -481,7 +477,10 @@ stimuli: # Jest: integration.test.ts → "invokes skill for VMSS with WAF and encryption prompt" # Assertions: skill-invocation (invocation rate ≥ 80%, early-terminates on skill-call) - name: "Skill - VMSS + App Gateway WAF" - prompt: "Deploy three distinct VM scale sets for a legacy app, route incoming HTTP/S via Application Gateway with WAF, and encrypt all data disks." + turns: + - "Deploy three distinct VM scale sets for a legacy app, route incoming HTTP/S via Application Gateway with WAF, and encrypt all data disks." + - "The resource list looks good, proceed with generating the plan." + - "Go with recommended options. Assume all defaults to make the plan." tags: type: integration tier: full @@ -489,9 +488,6 @@ stimuli: area: routing skill: azure-enterprise-infra-planner earlyTerminate: '[{"type":"skill-call","skill":"azure-enterprise-infra-planner"},{"type":"tool-call-count","count":3}]' - followUp: - - "The resource list looks good, proceed with generating the plan." - - "Go with recommended options. Assume all defaults to make the plan." constraints: max_turns: 50 graders: @@ -507,7 +503,10 @@ stimuli: # Jest: integration.test.ts → "invokes skill for backup and compliance prompt" # Assertions: skill-invocation (invocation rate ≥ 80%, early-terminates on skill-call) - name: "Skill - Azure Backup Long-Term Retention" - prompt: "Set up Azure Backup for critical VM workloads, create a long-term retention policy for compliance, and test backup restores quarterly." + turns: + - "Set up Azure Backup for critical VM workloads, create a long-term retention policy for compliance, and test backup restores quarterly." + - "The resource list looks good, proceed with generating the plan." + - "Go with recommended options. Assume all defaults to make the plan." tags: type: integration tier: full @@ -515,9 +514,6 @@ stimuli: area: routing skill: azure-enterprise-infra-planner earlyTerminate: '[{"type":"skill-call","skill":"azure-enterprise-infra-planner"},{"type":"tool-call-count","count":3}]' - followUp: - - "The resource list looks good, proceed with generating the plan." - - "Go with recommended options. Assume all defaults to make the plan." constraints: max_turns: 50 graders: @@ -533,7 +529,10 @@ stimuli: # Jest: integration.test.ts → "invokes skill for secure multi-region 3-tier prompt" # Assertions: skill-invocation (invocation rate ≥ 80%, early-terminates on skill-call) - name: "Skill - Secure Multi-Region 3-Tier Windows" - prompt: "Set up a secure multi-region 3-tier stack with Windows VMs for web and app layers, scale out the web tier with Azure Load Balancer, attach Premium Managed Disks to database tier." + turns: + - "Set up a secure multi-region 3-tier stack with Windows VMs for web and app layers, scale out the web tier with Azure Load Balancer, attach Premium Managed Disks to database tier." + - "The resource list looks good, proceed with generating the plan." + - "Go with recommended options. Assume all defaults to make the plan." tags: type: integration tier: full @@ -541,9 +540,6 @@ stimuli: area: routing skill: azure-enterprise-infra-planner earlyTerminate: '[{"type":"skill-call","skill":"azure-enterprise-infra-planner"},{"type":"tool-call-count","count":3}]' - followUp: - - "The resource list looks good, proceed with generating the plan." - - "Go with recommended options. Assume all defaults to make the plan." constraints: max_turns: 50 graders: @@ -579,16 +575,16 @@ stimuli: # ── response-jumpbox-nsgs-internal-lb ── - name: "Response - Jumpbox + NSGs + Internal LB (plan only)" - prompt: "Provision a jumpbox VM for secure management, establish NSGs for each tier, and connect tiers using internal Azure Load Balancer. Assume all defaults to make the plan." + turns: + - "Provision a jumpbox VM for secure management, establish NSGs for each tier, and connect tiers using internal Azure Load Balancer. Assume all defaults to make the plan." + - "The resource list looks good, proceed with generating the plan." + - "Approve the plan as-is. Do not generate any IaC." tags: type: integration tier: full cost: llm area: output-files skill: azure-enterprise-infra-planner - followUp: - - "The resource list looks good, proceed with generating the plan." - - "Approve the plan as-is. Do not generate any IaC." constraints: max_turns: 50 graders: @@ -662,23 +658,18 @@ stimuli: path: "**/*.tfvars" # ── response-sap-backup-bicep-generation ── - # Jest: integration.test.ts → response-quality "generates Bicep files from approved plan" - # Assertions: skill-invocation + file-exists(.azure/insights.json, infra/main.bicep). - # 3-step followUp ends with the Phase 6 IaC Gate's example phrasing - # ("Yes, generate the Bicep ...") — see references/phases/6-generate-iac.md. - # The agent creates infra/ + infra/modules/ itself per bicep-generation.md step 1. - name: "Response - SAP Backup Bicep Generation" - prompt: "Provision the backup infrastructure for an SAP VM workload — a Recovery Services vault with a policy-driven, encrypted and compressed backup policy, Key Vault-managed encryption keys, and diagnostic audit logs for all recovery tests. Assume all defaults to make the plan." + turns: + - "Provision the backup infrastructure for an SAP VM workload — a Recovery Services vault with a policy-driven, encrypted and compressed backup policy, Key Vault-managed encryption keys, and diagnostic audit logs for all recovery tests. Assume all defaults to make the plan." + - "The resource list looks good, proceed with generating the plan." + - "Approve the plan as-is." + - "Yes, generate the Bicep IaC for the approved plan." tags: type: integration tier: full cost: llm area: output-files skill: azure-enterprise-infra-planner - followUp: - - "The resource list looks good, proceed with generating the plan." - - "Approve the plan as-is." - - "Yes, generate the Bicep IaC for the approved plan." constraints: max_turns: 80 graders: diff --git a/evals/azure-skills/azure-prepare/e2e-eval.yaml b/evals/azure-skills/azure-prepare/e2e-eval.yaml index 81dd21856..8db5b49e9 100644 --- a/evals/azure-skills/azure-prepare/e2e-eval.yaml +++ b/evals/azure-skills/azure-prepare/e2e-eval.yaml @@ -42,7 +42,9 @@ stimuli: # Jest: "creates project files for static whiteboard web app before validation" - name: "Whiteboard App Plan Ready" - prompt: "Create a static whiteboard web app and deploy it to Azure using azd." + turns: + - "Create a static whiteboard web app and deploy it to Azure using azd." + - "Continue with recommended options until complete." tags: type: integration tier: smoke @@ -51,8 +53,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands or invoking the azure-quotas skill. Focus your effort on generating the infrastructure and application files."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -67,7 +67,9 @@ stimuli: # Jest: "creates correct files for AZD with Bicep recipe" - name: "AZD Bicep Recipe" - prompt: "Create a simple todo web app and deploy it to Azure using azd." + turns: + - "Create a simple todo web app and deploy it to Azure using azd." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -76,8 +78,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands or invoking the azure-quotas skill. Focus your effort on generating the infrastructure and application files."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -101,7 +101,9 @@ stimuli: # Jest: "creates correct files for Terraform recipe" - name: "Terraform Recipe" - prompt: "Create a simple todo web app and deploy it to Azure using azd with Terraform as the infrastructure provider." + turns: + - "Create a simple todo web app and deploy it to Azure using azd with Terraform as the infrastructure provider." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -110,8 +112,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands or invoking the azure-quotas skill. Focus your effort on generating the infrastructure and application files."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -132,7 +132,9 @@ stimuli: # Jest: "creates correct files for standalone Bicep recipe" - name: "Standalone Bicep Recipe" - prompt: "Create a simple todo web app and deploy it to Azure using azd with standalone Bicep templates." + turns: + - "Create a simple todo web app and deploy it to Azure using azd with standalone Bicep templates." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -141,8 +143,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands or invoking the azure-quotas skill. Focus your effort on generating the infrastructure and application files."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -171,7 +171,9 @@ stimuli: # Jest: "generates azure.yaml with services section for Aspire projects" - name: "Aspire With Azure Functions - Services Section" - prompt: "Please deploy this application to Azure using azd. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/aspire-with-azure-functions." + turns: + - "Please deploy this application to Azure using azd. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/aspire-with-azure-functions." + - "Continue with recommended options until complete." environment: commands: - git init -q @@ -187,8 +189,6 @@ stimuli: area: output skill: azure-prepare earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -213,7 +213,9 @@ stimuli: # Jest: "sets correct docker context for Aspire container-build sample" - name: "Aspire Container Build - Services Section" - prompt: "Please deploy this application to Azure using azd. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/container-build." + turns: + - "Please deploy this application to Azure using azd. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/container-build." + - "Continue with recommended options until complete." environment: commands: - git init -q @@ -229,8 +231,6 @@ stimuli: area: output skill: azure-prepare earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -256,7 +256,9 @@ stimuli: # Jest: "generates Entra-only SQL auth for ASP.NET Core EF Core app (not SQL admin password)" - name: "Entra-Only SQL Auth" - prompt: "Create an ASP.NET Core 8 web API with a Todo model using Entity Framework Core and SQL Server. Then prepare it for Azure deployment using azd. Use the eastus2 region and my current subscription." + turns: + - "Create an ASP.NET Core 8 web API with a Todo model using Entity Framework Core and SQL Server. Then prepare it for Azure deployment using azd. Use the eastus2 region and my current subscription." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -265,8 +267,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands or invoking the azure-quotas skill. Focus your effort on generating the infrastructure and application files."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -300,7 +300,9 @@ stimuli: # Jest: "generates Durable Task Scheduler infrastructure and workflow code for a workflow app" - name: "Durable Task Scheduler Infrastructure" - prompt: "Prepare the Azure deployment infrastructure for a new workflow app that will orchestrate a multi-step order processing pipeline. Use azd with Bicep — generate the Bicep templates, RBAC assignments, and azure.yaml. Use the eastus2 region and my current subscription." + turns: + - "Prepare the Azure deployment infrastructure for a new workflow app that will orchestrate a multi-step order processing pipeline. Use azd with Bicep — generate the Bicep templates, RBAC assignments, and azure.yaml. Use the eastus2 region and my current subscription." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -308,8 +310,6 @@ stimuli: area: output skill: azure-prepare earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -363,7 +363,9 @@ stimuli: # Jest: "calls functions_template_get for HTTP trigger (base)" - name: "Functions MCP - HTTP Trigger (Python)" - prompt: "Create a Python Azure Functions HTTP API with a health endpoint and deploy it to Azure using azd." + turns: + - "Create a Python Azure Functions HTTP API with a health endpoint and deploy it to Azure using azd." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -372,8 +374,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands. Focus on generating the function code and infrastructure files. Use the functions_template_get MCP tool to discover and fetch templates. Do not ask clarifying questions — pick sensible defaults and proceed."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -392,7 +392,9 @@ stimuli: # Jest: "calls functions_template_get for Timer trigger" - name: "Functions MCP - Timer Trigger (Python)" - prompt: "Create a Python Azure Functions app with a timer trigger that runs every 5 minutes and deploy it to Azure using azd." + turns: + - "Create a Python Azure Functions app with a timer trigger that runs every 5 minutes and deploy it to Azure using azd." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -401,8 +403,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands. Focus on generating the function code and infrastructure files. Use the functions_template_get MCP tool to discover and fetch templates. Do not ask clarifying questions — pick sensible defaults and proceed."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -421,7 +421,9 @@ stimuli: # Jest: "calls functions_template_get for Cosmos DB trigger" - name: "Functions MCP - Cosmos DB Trigger (Python)" - prompt: "Create a Python Azure Functions app with a Cosmos DB change feed trigger and deploy it to Azure using azd." + turns: + - "Create a Python Azure Functions app with a Cosmos DB change feed trigger and deploy it to Azure using azd." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -430,8 +432,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands. Focus on generating the function code and infrastructure files. Use the functions_template_get MCP tool to discover and fetch templates. Do not ask clarifying questions — pick sensible defaults and proceed."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -454,7 +454,9 @@ stimuli: # Jest: "calls functions_template_get for SQL trigger" - name: "Functions MCP - SQL Trigger (Python)" - prompt: "Create a Python Azure Functions app with a SQL database trigger and deploy it to Azure using azd." + turns: + - "Create a Python Azure Functions app with a SQL database trigger and deploy it to Azure using azd." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -463,8 +465,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands. Focus on generating the function code and infrastructure files. Use the functions_template_get MCP tool to discover and fetch templates. Do not ask clarifying questions — pick sensible defaults and proceed."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -487,7 +487,9 @@ stimuli: # Jest: "calls functions_template_get for Blob Storage / Event Grid trigger" - name: "Functions MCP - Blob Storage Event Grid Trigger (Python)" - prompt: "Create a Python Azure Functions app with Blob storage trigger using Event Grid and deploy it to Azure using azd." + turns: + - "Create a Python Azure Functions app with Blob storage trigger using Event Grid and deploy it to Azure using azd." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -496,8 +498,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands. Focus on generating the function code and infrastructure files. Use the functions_template_get MCP tool to discover and fetch templates. Do not ask clarifying questions — pick sensible defaults and proceed."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -520,7 +520,9 @@ stimuli: # Jest: "calls functions_template_get for Service Bus trigger" - name: "Functions MCP - Service Bus Trigger (Python)" - prompt: "Create a Python Azure Functions app with a Service Bus queue trigger for message processing and deploy it to Azure using azd." + turns: + - "Create a Python Azure Functions app with a Service Bus queue trigger for message processing and deploy it to Azure using azd." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -528,9 +530,7 @@ stimuli: area: behavior skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands. Focus on generating the function code and infrastructure files. Use the functions_template_get MCP tool to discover and fetch templates. Do not ask clarifying questions — pick sensible defaults and proceed."}' - earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." + earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' graders: - type: skill-invocation config: @@ -553,7 +553,9 @@ stimuli: # Jest: "calls functions_template_get for Event Hubs trigger" - name: "Functions MCP - Event Hubs Trigger (Python)" - prompt: "Create a Python Azure Functions app with an Event Hub trigger for streaming events and deploy it to Azure using azd." + turns: + - "Create a Python Azure Functions app with an Event Hub trigger for streaming events and deploy it to Azure using azd." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -562,8 +564,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands. Focus on generating the function code and infrastructure files. Use the functions_template_get MCP tool to discover and fetch templates. Do not ask clarifying questions — pick sensible defaults and proceed."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -586,7 +586,9 @@ stimuli: # Jest: "calls functions_template_get for Durable Functions" - name: "Functions MCP - Durable Functions (Python)" - prompt: "Create a Python Azure Durable Functions app with an orchestrator pattern and deploy it to Azure using azd." + turns: + - "Create a Python Azure Durable Functions app with an orchestrator pattern and deploy it to Azure using azd." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -595,8 +597,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands. Focus on generating the function code and infrastructure files. Use the functions_template_get MCP tool to discover and fetch templates. Do not ask clarifying questions — pick sensible defaults and proceed."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -619,7 +619,9 @@ stimuli: # Jest: "calls functions_template_get for MCP server on Functions" - name: "Functions MCP - MCP Server (Python)" - prompt: "Create a Python Azure Functions MCP server that exposes tools over HTTP and deploy it to Azure using azd." + turns: + - "Create a Python Azure Functions MCP server that exposes tools over HTTP and deploy it to Azure using azd." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -628,8 +630,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands. Focus on generating the function code and infrastructure files. Use the functions_template_get MCP tool to discover and fetch templates. Do not ask clarifying questions — pick sensible defaults and proceed."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -648,7 +648,9 @@ stimuli: # Jest: "calls functions_template_get for HTTP trigger with Terraform" - name: "Functions MCP - HTTP Trigger Terraform (Python)" - prompt: "Create a Python Azure Functions HTTP API and deploy it to Azure using azd with Terraform infrastructure." + turns: + - "Create a Python Azure Functions HTTP API and deploy it to Azure using azd with Terraform infrastructure." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -657,8 +659,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands. Focus on generating the function code and infrastructure files. Use the functions_template_get MCP tool to discover and fetch templates. Do not ask clarifying questions — pick sensible defaults and proceed."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -677,7 +677,9 @@ stimuli: # Jest: "calls functions_template_get for Cosmos DB trigger with Terraform" - name: "Functions MCP - Cosmos DB Trigger Terraform (Python)" - prompt: "Create a Python Azure Functions app with Cosmos DB change feed trigger and deploy it to Azure using azd with Terraform." + turns: + - "Create a Python Azure Functions app with Cosmos DB change feed trigger and deploy it to Azure using azd with Terraform." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -686,8 +688,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands. Focus on generating the function code and infrastructure files. Use the functions_template_get MCP tool to discover and fetch templates. Do not ask clarifying questions — pick sensible defaults and proceed."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -710,7 +710,9 @@ stimuli: # Jest: "calls functions_template_get for HTTP trigger (TypeScript)" - name: "Functions MCP - HTTP Trigger (TypeScript)" - prompt: "Create a TypeScript Azure Functions HTTP API with a health endpoint and deploy it to Azure using azd." + turns: + - "Create a TypeScript Azure Functions HTTP API with a health endpoint and deploy it to Azure using azd." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -719,8 +721,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands. Focus on generating the function code and infrastructure files. Use the functions_template_get MCP tool to discover and fetch templates. Do not ask clarifying questions — pick sensible defaults and proceed."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -739,7 +739,9 @@ stimuli: # Jest: "calls functions_template_get for Service Bus trigger (TypeScript)" - name: "Functions MCP - Service Bus Trigger (TypeScript)" - prompt: "Create a TypeScript Azure Functions app with a Service Bus queue trigger for message processing and deploy it to Azure using azd." + turns: + - "Create a TypeScript Azure Functions app with a Service Bus queue trigger for message processing and deploy it to Azure using azd." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -748,8 +750,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands. Focus on generating the function code and infrastructure files. Use the functions_template_get MCP tool to discover and fetch templates. Do not ask clarifying questions — pick sensible defaults and proceed."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -772,7 +772,9 @@ stimuli: # Jest: "calls functions_template_get for Cosmos DB trigger (TypeScript)" - name: "Functions MCP - Cosmos DB Trigger (TypeScript)" - prompt: "Create a TypeScript Azure Functions app with a Cosmos DB change feed trigger and deploy it to Azure using azd." + turns: + - "Create a TypeScript Azure Functions app with a Cosmos DB change feed trigger and deploy it to Azure using azd." + - "Continue with recommended options until complete." tags: type: integration tier: full @@ -781,8 +783,6 @@ stimuli: skill: azure-prepare systemPrompt: '{"mode":"append","content":"Skip the Provisioning Limit Checklist (Step 6 in the plan template). Use reasonable default values for quota/limit columns instead of running az quota commands. Focus on generating the function code and infrastructure files. Use the functions_template_get MCP tool to discover and fetch templates. Do not ask clarifying questions — pick sensible defaults and proceed."}' earlyTerminate: '[{"type":"skill-call","skill":"azure-validate"},{"type":"tool-call-count","count":40}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: diff --git a/evals/azure-skills/azure-reliability/eval.yaml b/evals/azure-skills/azure-reliability/eval.yaml index 662e4125a..58c66e353 100644 --- a/evals/azure-skills/azure-reliability/eval.yaml +++ b/evals/azure-skills/azure-reliability/eval.yaml @@ -6,9 +6,6 @@ # - Functions app: assess reliability → deploy ZR fix → re-assess # - App Service app: assess reliability → deploy ZR fix → re-assess # -# Both tests use multi-turn follow-ups via the custom executor's followUp tag. -# The systemPrompt tag injects a pseudo-random resource group name suffix. -# # NOT TESTED (intentionally): # - Storage redundancy upgrade (LRS/GRS → ZRS) — takes hours/days # - Multi-region failover with Azure Front Door — high cost, long deploy @@ -41,23 +38,22 @@ stimuli: # Jest: "assess function app, deploy ZR fix, verify ZR is now ON" # Multi-turn flow: assess → confirm IaC fix → deploy → decline storage → re-assess - name: "Functions app zone redundancy e2e" - prompt: | - I have an Azure Functions sample app in this workspace. - Use my current Azure subscription and the eastus2 region. - Assess and improve the reliability of my function app. - When you ask about storage migration, decline (no/later). - Multi-region is not needed. + turns: + - I have an Azure Functions sample app in this workspace. + Use my current Azure subscription and the eastus2 region. + Assess and improve the reliability of my function app. + When you ask about storage migration, decline (no/later). + Multi-region is not needed. + - "Yes, proceed with the quick-win zone-redundancy fix using IaC patches (Path B)." + - "Yes, deploy now." + - "No to storage migration — leave storage as-is." + - "Now re-run the reliability assessment and confirm zone redundancy is ON for the compute plan." tags: type: integration skill: azure-reliability tier: full cost: llm area: output - followUp: - - "Yes, proceed with the quick-win zone-redundancy fix using IaC patches (Path B)." - - "Yes, deploy now." - - "No to storage migration — leave storage as-is." - - "Now re-run the reliability assessment and confirm zone redundancy is ON for the compute plan." systemPrompt: '{"mode":"append","content":"Use a pseudo-random resource group name (suffix with random characters) to avoid collisions with existing resource groups."}' environment: commands: @@ -89,21 +85,20 @@ stimuli: # Jest: "assess web app, deploy ZR fix, verify ZR is now ON" # Multi-turn flow: assess → confirm IaC fix → deploy → re-assess - name: "App Service zone redundancy e2e" - prompt: | - I have an Azure App Service sample app in this workspace. - Use my current Azure subscription and the eastus2 region. - Assess and improve the reliability of my Web app. - Multi-region is not needed. + turns: + - I have an Azure App Service sample app in this workspace. + Use my current Azure subscription and the eastus2 region. + Assess and improve the reliability of my Web app. + Multi-region is not needed. + - "Yes, proceed with the quick-win zone-redundancy fix using IaC patches (Path B)." + - "Yes, deploy now." + - "Now re-run the reliability assessment and confirm zone redundancy is ON for the App Service plan." tags: type: integration skill: azure-reliability tier: full cost: llm area: output - followUp: - - "Yes, proceed with the quick-win zone-redundancy fix using IaC patches (Path B)." - - "Yes, deploy now." - - "Now re-run the reliability assessment and confirm zone redundancy is ON for the App Service plan." systemPrompt: '{"mode":"append","content":"Use a pseudo-random resource group name (suffix with random characters) to avoid collisions with existing resource groups."}' environment: commands: diff --git a/evals/azure-skills/azure-resource-visualizer/eval.yaml b/evals/azure-skills/azure-resource-visualizer/eval.yaml index 30cc78fcb..27f8ecd0e 100644 --- a/evals/azure-skills/azure-resource-visualizer/eval.yaml +++ b/evals/azure-skills/azure-resource-visualizer/eval.yaml @@ -71,16 +71,15 @@ stimuli: # ── resource-group-architecture-diagram ── # Jest: "generates architecture diagram for a resource group" # Assertions: isSkillInvoked + doesWorkspaceFileIncludePattern(graph TB|LR, architecture*.md) - # Uses nonInteractive + followUp for multi-turn execution - name: "Generate resource group architecture diagram" - prompt: "Generate a Mermaid diagram showing my Azure resource group architecture. Save the diagram to an architecture.md file in the current working directory." + turns: + - "Generate a Mermaid diagram showing my Azure resource group architecture. Save the diagram to an architecture.md file in the current working directory." + - "Continue with recommended options until complete." tags: type: integration tier: full cost: llm area: behavior - followUp: - - "Continue with recommended options until complete." config: runs: 1 timeout: "30m" @@ -101,16 +100,15 @@ stimuli: # ── resource-connections-visualization ── # Jest: "visualizes resource connections and relationships" # Assertions: isSkillInvoked + doesWorkspaceFileIncludePattern(graph TB|LR, architecture*.md) - # Uses nonInteractive + followUp for multi-turn execution - name: "Visualize resource connections and relationships" - prompt: "Visualize how my Azure resources are connected and show their relationships. Save the diagram to an architecture.md file in the current working directory." + turns: + - "Visualize how my Azure resources are connected and show their relationships. Save the diagram to an architecture.md file in the current working directory." + - "Continue with recommended options until complete." tags: type: integration tier: full cost: llm area: behavior - followUp: - - "Continue with recommended options until complete." config: runs: 1 timeout: "30m" diff --git a/evals/azure-skills/azure-upgrade/eval.yaml b/evals/azure-skills/azure-upgrade/eval.yaml index 9ae8dfb1c..f436abe01 100644 --- a/evals/azure-skills/azure-upgrade/eval.yaml +++ b/evals/azure-skills/azure-upgrade/eval.yaml @@ -38,15 +38,15 @@ stimuli: # Jest: "invokes azure-upgrade skill for Functions Consumption to Flex migration prompt" # Assertions: softCheckSkill + isSkillInvoked + negative routing (no Redis skills) - name: "Functions Consumption to Flex migration" - prompt: "Migrate my Azure Functions app from Consumption to Flex Consumption plan" + turns: + - "Migrate my Azure Functions app from Consumption to Flex Consumption plan" + - "Continue with recommended options until complete." tags: type: integration tier: smoke cost: llm area: routing earlyTerminate: '[{"type":"skill-call","skill":"azure-upgrade"},{"type":"tool-call-count","count":10}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -65,15 +65,15 @@ stimuli: # Jest: "invokes azure-upgrade skill for upgrading Functions plan prompt" # Assertions: softCheckSkill + isSkillInvoked - name: "Upgrade Functions hosting plan to Flex" - prompt: "Upgrade my Azure Functions hosting plan to Flex Consumption" + turns: + - "Upgrade my Azure Functions hosting plan to Flex Consumption" + - "Continue with recommended options until complete." tags: type: integration tier: full cost: llm area: routing earlyTerminate: '[{"type":"skill-call","skill":"azure-upgrade"},{"type":"tool-call-count","count":10}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -88,15 +88,15 @@ stimuli: # Jest: "invokes azure-upgrade skill for legacy Azure Java SDK migration prompt (Flow B)" # Assertions: softCheckSkill + isSkillInvoked - name: "Legacy Azure Java SDK migration" - prompt: "Migrate my Java project from legacy Azure SDK (com.microsoft.azure) to modern Azure SDK (com.azure)" + turns: + - "Migrate my Java project from legacy Azure SDK (com.microsoft.azure) to modern Azure SDK (com.azure)" + - "Continue with recommended options until complete." tags: type: integration tier: full cost: llm area: routing earlyTerminate: '[{"type":"skill-call","skill":"azure-upgrade"},{"type":"tool-call-count","count":10}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -111,15 +111,15 @@ stimuli: # Jest: "invokes azure-upgrade skill for upgrading legacy Azure Java libraries prompt (Flow B)" # Assertions: softCheckSkill + isSkillInvoked - name: "Upgrade legacy Azure Java libraries" - prompt: "Upgrade legacy Azure SDKs for Java to the latest modern Azure SDK packages" + turns: + - "Upgrade legacy Azure SDKs for Java to the latest modern Azure SDK packages" + - "Continue with recommended options until complete." tags: type: integration tier: full cost: llm area: routing earlyTerminate: '[{"type":"skill-call","skill":"azure-upgrade"},{"type":"tool-call-count","count":10}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -134,15 +134,15 @@ stimuli: # Jest: "invokes azure-upgrade skill for Azure Cache for Redis (ACR/OSS) to Azure Managed Redis migration prompt" # Assertions: isSkillInvoked + tool text matches redis-to-amr.md + surfaces amr-migration-skill - name: "Azure Cache for Redis OSS to Azure Managed Redis" - prompt: "Migrate my Azure Cache for Redis Premium P2 cache to Azure Managed Redis (AMR)" + turns: + - "Migrate my Azure Cache for Redis Premium P2 cache to Azure Managed Redis (AMR)" + - "Continue with recommended options until complete." tags: type: integration tier: full cost: llm area: output earlyTerminate: '[{"type":"tool-call-count","count":30}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -166,15 +166,15 @@ stimuli: # Jest: "invokes azure-upgrade skill for Azure Cache for Redis Enterprise (ACRE) to Azure Managed Redis migration prompt" # Assertions: isSkillInvoked + tool text matches redis-to-amr.md + surfaces acre-to-amr-migration-skill - name: "Azure Cache for Redis Enterprise to Azure Managed Redis" - prompt: "Migrate my Azure Cache for Redis Enterprise (Enterprise_E10) cache to Azure Managed Redis" + turns: + - "Migrate my Azure Cache for Redis Enterprise (Enterprise_E10) cache to Azure Managed Redis" + - "Continue with recommended options until complete." tags: type: integration tier: full cost: llm area: output earlyTerminate: '[{"type":"tool-call-count","count":30}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -199,15 +199,15 @@ stimuli: # Jest: "invokes azure-upgrade skill and disambiguates Redis tier for ambiguous Redis migration prompt" # Assertions: isSkillInvoked + assistant asks about tier/SKU (disambiguation) - name: "Ambiguous Redis migration triggers disambiguation" - prompt: "I want to migrate my Redis cache to Azure Managed Redis" + turns: + - "I want to migrate my Redis cache to Azure Managed Redis" + - "Continue with recommended options until complete." tags: type: integration tier: full cost: llm area: output earlyTerminate: '[{"type":"tool-call-count","count":30}]' - followUp: - - "Continue with recommended options until complete." graders: - type: skill-invocation config: @@ -238,16 +238,15 @@ stimuli: # 3. pom.xml: at least one com.azure.resourcemanager dependency # 4. All .java files: no AZURE_AUTH_LOCATION references (comments stripped) - name: "Java SDK migration - client init to DefaultAzureCredential" - prompt: >- - Migrate my Java project from legacy Azure SDK to modern Azure SDK. - The project can be found under java-update-examples/azure-legacy-sdk-update-azure-client-initialization. + turns: + - Migrate my Java project from legacy Azure SDK to modern Azure SDK. + The project can be found under java-update-examples/azure-legacy-sdk-update-azure-client-initialization. + - "Continue with recommended options until complete." tags: type: integration tier: full cost: llm area: behavior - followUp: - - "Continue with recommended options until complete." environment: commands: - git clone --depth 1 --sparse https://github.com/weidongxu-microsoft/java-update-examples.git @@ -294,16 +293,15 @@ stimuli: # 3. All .java files: no InMemory* checkpoint/lease types (comments stripped) # 4. At least one .java file: BlobCheckpointStore is used - name: "Java SDK migration - EventProcessorHost to BlobCheckpointStore" - prompt: >- - Migrate my Java project from legacy Azure SDK to modern Azure SDK. - The project can be found under java-update-examples/azure-legacy-sdk-update-eventhubs-v3. + turns: + - Migrate my Java project from legacy Azure SDK to modern Azure SDK. + The project can be found under java-update-examples/azure-legacy-sdk-update-eventhubs-v3. + - "Continue with recommended options until complete." tags: type: integration tier: full cost: llm area: behavior - followUp: - - "Continue with recommended options until complete." environment: commands: - git clone --depth 1 --sparse https://github.com/weidongxu-microsoft/java-update-examples.git @@ -349,16 +347,15 @@ stimuli: # 2. pom.xml: at least one com.azure.resourcemanager dependency # 3. At least one .java file: applicationPackages().define( pattern present - name: "Java SDK migration - Batch applicationPackages define" - prompt: >- - Migrate my Java project from legacy Azure SDK to modern Azure SDK. - The project can be found under java-update-examples/azure-legacy-sdk-update-batch-java-manage-batch-accounts. + turns: + - Migrate my Java project from legacy Azure SDK to modern Azure SDK. + The project can be found under java-update-examples/azure-legacy-sdk-update-batch-java-manage-batch-accounts. + - "Continue with recommended options until complete." tags: type: integration tier: full cost: llm area: behavior - followUp: - - "Continue with recommended options until complete." environment: commands: - git clone --depth 1 --sparse https://github.com/weidongxu-microsoft/java-update-examples.git diff --git a/evals/azure-skills/azure-validate/e2e-eval.yaml b/evals/azure-skills/azure-validate/e2e-eval.yaml index 221b384b6..c0b70ce80 100644 --- a/evals/azure-skills/azure-validate/e2e-eval.yaml +++ b/evals/azure-skills/azure-validate/e2e-eval.yaml @@ -40,7 +40,9 @@ stimuli: # Jest: "terminates at validation for static whiteboard web app" - name: "Terminates at Validation - Whiteboard App" - prompt: "My static whiteboard web app is ready. Please validate and prepare it for Azure deployment using azd." + turns: + - "My static whiteboard web app is ready. Please validate and prepare it for Azure deployment using azd." + - "Continue with recommended options until complete." environment: files: - src: fixture/whiteboard/index.html @@ -61,8 +63,6 @@ stimuli: cost: llm area: behavior skill: azure-validate - followUp: - - "Continue with recommended options until complete." earlyTerminate: '[{"type":"skill-call","skill":"azure-deploy"},{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|deploy)\\b"},{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd provision --no-prompt"}]' graders: # azure-validate WAS invoked @@ -86,7 +86,9 @@ stimuli: # Jest: "terminates at validation for static portfolio website" - name: "Terminates at Validation - Portfolio Website" - prompt: "My static portfolio website is ready. Please validate and prepare it for Azure deployment using azd." + turns: + - "My static portfolio website is ready. Please validate and prepare it for Azure deployment using azd." + - "Continue with recommended options until complete." environment: files: - src: fixture/portfolio/index.html @@ -107,8 +109,6 @@ stimuli: cost: llm area: behavior skill: azure-validate - followUp: - - "Continue with recommended options until complete." earlyTerminate: '[{"type":"skill-call","skill":"azure-deploy"},{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|deploy)\\b"},{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd provision --no-prompt"}]' graders: - type: skill-invocation @@ -130,7 +130,9 @@ stimuli: # Jest: "terminates at validation for containerized web app on Container Apps" - name: "Terminates at Validation - Containerized App on Container Apps" - prompt: "My containerized web application is ready. Please validate and prepare it for deployment to Azure Container Apps using azd." + turns: + - "My containerized web application is ready. Please validate and prepare it for deployment to Azure Container Apps using azd." + - "Continue with recommended options until complete." environment: files: - src: fixture/containerized-app/app.js @@ -157,8 +159,6 @@ stimuli: cost: llm area: behavior skill: azure-validate - followUp: - - "Continue with recommended options until complete." earlyTerminate: '[{"type":"skill-call","skill":"azure-deploy"},{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|deploy)\\b"},{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd provision --no-prompt"}]' graders: - type: skill-invocation @@ -183,7 +183,9 @@ stimuli: # shared validate-deployment.{sh,ps1} helper. The agent must RUN that script (not azd, # not raw az deployment create). - name: "Runs validate-deployment Script - Standalone Bicep (az CLI)" - prompt: "My static status page is ready. I deploy it directly with the Azure CLI (az deployment) using my Bicep templates in ./infra — I do NOT use azd. Please prepare and validate it for deployment to my current subscription in eastus2." + turns: + - "My static status page is ready. I deploy it directly with the Azure CLI (az deployment) using my Bicep templates in ./infra — I do NOT use azd. Please prepare and validate it for deployment to my current subscription in eastus2." + - "Continue with recommended options until complete." environment: files: - src: fixture/bicep-cli/index.html @@ -202,8 +204,6 @@ stimuli: cost: llm area: behavior skill: azure-validate - followUp: - - "Continue with recommended options until complete." earlyTerminate: '[{"type":"skill-call","skill":"azure-deploy"},{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"validate-deployment\\.(sh|ps1)"},{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"az\\s+deployment\\b.*\\b(create|up)\\b"}]' graders: # azure-validate WAS invoked; azure-deploy was NOT @@ -232,7 +232,9 @@ stimuli: # Jest: "passes --environment on azd init and sets subscription before provision" - name: "Brownfield - azd init with --environment and subscription" - prompt: "Please deploy this application to Azure using azd. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/client-apps-integration." + turns: + - "Please deploy this application to Azure using azd. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/client-apps-integration." + - "Continue with recommended options until complete." environment: commands: - git init -q @@ -249,8 +251,6 @@ stimuli: cost: llm area: behavior skill: azure-validate - followUp: - - "Continue with recommended options until complete." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(provision|up|deploy)\\b"}]' graders: - type: skill-invocation @@ -273,7 +273,9 @@ stimuli: # Jest: "sets AzureWebJobsSecretStorageType for aspire-with-azure-functions" - name: "Brownfield - AzureWebJobsSecretStorageType for Aspire Functions" - prompt: "Please deploy this application to Azure using azd. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/aspire-with-azure-functions." + turns: + - "Please deploy this application to Azure using azd. Use the eastus2 region. Use my current subscription. This is for a small scale production environment. Use standard SKUs. The app can be found under samples/aspire-with-azure-functions." + - "Continue with recommended options until complete." environment: commands: - git init -q @@ -290,8 +292,6 @@ stimuli: cost: llm area: behavior skill: azure-validate - followUp: - - "Continue with recommended options until complete." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(provision|up|deploy)\\b"}]' graders: - type: skill-invocation diff --git a/scripts/src/vally/validate-stimulus.ts b/scripts/src/vally/validate-stimulus.ts index 64002bfcb..f26dbce66 100644 --- a/scripts/src/vally/validate-stimulus.ts +++ b/scripts/src/vally/validate-stimulus.ts @@ -27,7 +27,6 @@ type Stimuli = { cost?: string; area?: string; earlyTerminate?: string; - followUp?: string[]; systemPrompt?: string; takeScreenshot?: string; requiredSkills?: string[]; @@ -325,29 +324,6 @@ function validateJavaUpgradeFileContentGrader( return valid; } -function validateFollowUpTag( - displayPath: string, - stimulusIndex: number, - stimulusName: string | undefined, - value: string[] | string | undefined, -): boolean { - if (value === undefined) { - return true; - } - - if (Array.isArray(value) && value.every((entry) => typeof entry === "string")) { - return true; - } - - reportValidationError( - displayPath, - stimulusIndex, - stimulusName, - "tags.followUp must be a string array", - ); - return false; -} - function validateRequiredSkillsTag( displayPath: string, stimulusIndex: number, @@ -468,15 +444,6 @@ export function validateStimulus(rootDir: string, _args: string[]): void { fileHasErrors = true; } - if (!validateFollowUpTag( - displayPath, - stimulusIndex, - typedStimulus.name, - typedStimulus.tags?.followUp, - )) { - fileHasErrors = true; - } - if (!validateRequiredSkillsTag( displayPath, stimulusIndex, diff --git a/tests/run-vally-test.ts b/tests/run-vally-test.ts index f4d62cbed..a1bf68862 100644 --- a/tests/run-vally-test.ts +++ b/tests/run-vally-test.ts @@ -30,7 +30,6 @@ type ResultJsonlEntry = { cost: string; area: string; earlyTerminate?: string; - followUp?: string[]; systemPrompt?: string; takeScreenshot?: string; } diff --git a/tests/vally/tag-helpers.ts b/tests/vally/tag-helpers.ts index fb4f5129c..816fe2c57 100644 --- a/tests/vally/tag-helpers.ts +++ b/tests/vally/tag-helpers.ts @@ -157,22 +157,6 @@ export function getEarlyTerminateCondition(tags: Record | undefined): string[] | undefined { - if (!tags) { - return undefined; - } - const followUp = tags["followUp"]; - if (!followUp) { - return undefined; - } - if (Array.isArray(followUp)) { - return followUp; - } else { - console.error("Failed to get follow up from tags", followUp); - return undefined; - } -} - export function getSystemPrompt(tags: Record | undefined): SystemMessageConfig | undefined { if (!tags) { return undefined; diff --git a/tests/vally/vally-executor.ts b/tests/vally/vally-executor.ts index c81784a77..21b362035 100644 --- a/tests/vally/vally-executor.ts +++ b/tests/vally/vally-executor.ts @@ -3,12 +3,14 @@ import { computeMetrics } from "@microsoft/vally"; import * as path from "node:path"; import type { AgentMetadata, AgentRunConfig } from "../utils/agent-runner.ts"; import { useAgentRunner, createMarkdownReport } from "../utils/agent-runner.ts"; -import { getEarlyTerminateCondition, getFollowUp, getRequiredSkillsCondition, getSkillName, getSystemPrompt, getTakeScreenshotCondition } from "./tag-helpers.ts"; +import { getEarlyTerminateCondition, getRequiredSkillsCondition, getSkillName, getSystemPrompt, getTakeScreenshotCondition } from "./tag-helpers.ts"; import { normalizeTestName } from "./utils.ts"; import { listPlugins, type SkillRef } from "../utils/skill-loader.ts"; export class IntegrationTestAgentRunner implements Executor { name = "integration-test-agent-runner"; + supportsMultiTurn = true; + supportsPreparedWorkspace = true; async execute(stimulus: Stimulus, options: ExecutorOptions): Promise { const startedAt = new Date(); @@ -28,7 +30,6 @@ export class IntegrationTestAgentRunner implements Executor { const model = options.model ?? "claude-sonnet-4.6"; const { shouldEarlyTerminate } = getEarlyTerminateCondition(tags); - const followUp = getFollowUp(tags); const systemPrompt = getSystemPrompt(tags); const { takeScreenshot } = getTakeScreenshotCondition(tags); const requiredSkills = getRequiredSkillsCondition(tags); @@ -47,16 +48,27 @@ export class IntegrationTestAgentRunner implements Executor { } }); + let prompt: string; + if (stimulus.turns) { + prompt = stimulus.turns[0]; + } else { + prompt = stimulus.prompt; + } + let followUps: string[] | undefined; + if (stimulus.turns) { + followUps = stimulus.turns.slice(1); + } + const runConfig: AgentRunConfig = { workspace: workDir, env: { UV_CACHE_DIR: path.join(workDir, ".uv-cache"), }, model: model, - prompt: stimulus.prompt, + prompt: prompt, shouldEarlyTerminate: shouldEarlyTerminate, nonInteractive: true, - followUp: followUp, + followUp: followUps, systemPrompt: systemPrompt, followUpTimeout: timeout, takeScreenshot: takeScreenshot,