From 6d4db53c1ee9779c63364d92ec8d76cc18fb8fd6 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Mon, 13 Jul 2026 14:35:23 -0700 Subject: [PATCH 01/21] feat: add azure-deploy e2e eval fixtures and suite Identified 20 integration tests in tests/azure-deploy/integration.test.ts that started from empty workspaces and asked the agent to both create and deploy applications. This conflates azure-prepare (scaffolding) and azure-deploy (running azd up) responsibilities. Add per-scenario fixtures under evals/azure-deploy/fixture/ so each vally stimulus provides a pre-built azure.yaml + infra/ and the agent is only evaluated on deployment execution: Fixtures added: - swa-bicep/ Static Web App with Bicep infra - app-service-bicep/ App Service with Bicep infra - functions-bicep/ Azure Functions with Bicep infra - dts-bicep/ Durable Task Scheduler with Bicep infra (DTS-specific Microsoft.DurableTask/* resources + role 0ad04412) - container-apps-bicep/ Container Apps with Bicep infra (two-phase pattern: SystemAssigned identity, placeholder image, AcrPull 7f951dda) - swa-terraform/ Static Web App with Terraform infra - app-service-terraform/ App Service with Terraform infra - functions-terraform/ Azure Functions with Terraform infra - container-apps-terraform/ Container Apps with Terraform infra (SystemAssigned identity, AcrPull, lifecycle ignore_changes) New eval suite: evals/azure-deploy/e2e-eval.yaml Migrates all 20 vanilla/terraform integration tests. Each stimulus: - Loads the appropriate fixture into the agent environment - Prompts "deploy this prepared app" (not "create and deploy") - Uses earlyTerminate on azd up/provision - Graders check azure-deploy skill invocation + azd command execution - Container Apps stimuli also verify correct IaC patterns in fixture files Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 01827688-80a4-46ef-90c2-15e7de402bc4 --- evals/azure-deploy/e2e-eval.yaml | 930 ++++++++++++++++++ .../fixture/app-service-bicep/app.js | 9 + .../fixture/app-service-bicep/azure.yaml | 10 + .../app-service-bicep/infra/main.bicep | 34 + .../infra/main.parameters.json | 12 + .../app-service-bicep/infra/resources.bicep | 35 + .../fixture/app-service-bicep/package.json | 9 + .../fixture/app-service-terraform/app.js | 7 + .../fixture/app-service-terraform/azure.yaml | 10 + .../app-service-terraform/infra/main.tf | 74 ++ .../app-service-terraform/package.json | 5 + .../fixture/container-apps-bicep/Dockerfile | 7 + .../fixture/container-apps-bicep/app.js | 9 + .../fixture/container-apps-bicep/azure.yaml | 11 + .../infra/abbreviations.json | 6 + .../container-apps-bicep/infra/main.bicep | 37 + .../infra/main.parameters.json | 12 + .../infra/resources.bicep | 71 ++ .../fixture/container-apps-bicep/package.json | 5 + .../container-apps-terraform/Dockerfile | 7 + .../fixture/container-apps-terraform/app.js | 7 + .../container-apps-terraform/azure.yaml | 11 + .../container-apps-terraform/infra/main.tf | 125 +++ .../container-apps-terraform/package.json | 5 + .../azure-deploy/fixture/dts-bicep/azure.yaml | 10 + .../azure-deploy/fixture/dts-bicep/host.json | 3 + .../fixture/dts-bicep/infra/main.bicep | 34 + .../dts-bicep/infra/main.parameters.json | 12 + .../fixture/dts-bicep/infra/resources.bicep | 100 ++ .../fixture/functions-bicep/azure.yaml | 10 + .../fixture/functions-bicep/host.json | 3 + .../fixture/functions-bicep/infra/main.bicep | 34 + .../infra/main.parameters.json | 12 + .../functions-bicep/infra/resources.bicep | 58 ++ .../fixture/functions-bicep/package.json | 7 + .../fixture/functions-terraform/azure.yaml | 10 + .../fixture/functions-terraform/host.json | 3 + .../fixture/functions-terraform/infra/main.tf | 88 ++ .../azure-deploy/fixture/swa-bicep/azure.yaml | 9 + .../azure-deploy/fixture/swa-bicep/index.html | 8 + .../fixture/swa-bicep/infra/main.bicep | 34 + .../swa-bicep/infra/main.parameters.json | 12 + .../fixture/swa-bicep/infra/resources.bicep | 18 + .../fixture/swa-terraform/azure.yaml | 9 + .../fixture/swa-terraform/index.html | 8 + .../fixture/swa-terraform/infra/main.tf | 60 ++ 46 files changed, 1990 insertions(+) create mode 100644 evals/azure-deploy/e2e-eval.yaml create mode 100644 evals/azure-deploy/fixture/app-service-bicep/app.js create mode 100644 evals/azure-deploy/fixture/app-service-bicep/azure.yaml create mode 100644 evals/azure-deploy/fixture/app-service-bicep/infra/main.bicep create mode 100644 evals/azure-deploy/fixture/app-service-bicep/infra/main.parameters.json create mode 100644 evals/azure-deploy/fixture/app-service-bicep/infra/resources.bicep create mode 100644 evals/azure-deploy/fixture/app-service-bicep/package.json create mode 100644 evals/azure-deploy/fixture/app-service-terraform/app.js create mode 100644 evals/azure-deploy/fixture/app-service-terraform/azure.yaml create mode 100644 evals/azure-deploy/fixture/app-service-terraform/infra/main.tf create mode 100644 evals/azure-deploy/fixture/app-service-terraform/package.json create mode 100644 evals/azure-deploy/fixture/container-apps-bicep/Dockerfile create mode 100644 evals/azure-deploy/fixture/container-apps-bicep/app.js create mode 100644 evals/azure-deploy/fixture/container-apps-bicep/azure.yaml create mode 100644 evals/azure-deploy/fixture/container-apps-bicep/infra/abbreviations.json create mode 100644 evals/azure-deploy/fixture/container-apps-bicep/infra/main.bicep create mode 100644 evals/azure-deploy/fixture/container-apps-bicep/infra/main.parameters.json create mode 100644 evals/azure-deploy/fixture/container-apps-bicep/infra/resources.bicep create mode 100644 evals/azure-deploy/fixture/container-apps-bicep/package.json create mode 100644 evals/azure-deploy/fixture/container-apps-terraform/Dockerfile create mode 100644 evals/azure-deploy/fixture/container-apps-terraform/app.js create mode 100644 evals/azure-deploy/fixture/container-apps-terraform/azure.yaml create mode 100644 evals/azure-deploy/fixture/container-apps-terraform/infra/main.tf create mode 100644 evals/azure-deploy/fixture/container-apps-terraform/package.json create mode 100644 evals/azure-deploy/fixture/dts-bicep/azure.yaml create mode 100644 evals/azure-deploy/fixture/dts-bicep/host.json create mode 100644 evals/azure-deploy/fixture/dts-bicep/infra/main.bicep create mode 100644 evals/azure-deploy/fixture/dts-bicep/infra/main.parameters.json create mode 100644 evals/azure-deploy/fixture/dts-bicep/infra/resources.bicep create mode 100644 evals/azure-deploy/fixture/functions-bicep/azure.yaml create mode 100644 evals/azure-deploy/fixture/functions-bicep/host.json create mode 100644 evals/azure-deploy/fixture/functions-bicep/infra/main.bicep create mode 100644 evals/azure-deploy/fixture/functions-bicep/infra/main.parameters.json create mode 100644 evals/azure-deploy/fixture/functions-bicep/infra/resources.bicep create mode 100644 evals/azure-deploy/fixture/functions-bicep/package.json create mode 100644 evals/azure-deploy/fixture/functions-terraform/azure.yaml create mode 100644 evals/azure-deploy/fixture/functions-terraform/host.json create mode 100644 evals/azure-deploy/fixture/functions-terraform/infra/main.tf create mode 100644 evals/azure-deploy/fixture/swa-bicep/azure.yaml create mode 100644 evals/azure-deploy/fixture/swa-bicep/index.html create mode 100644 evals/azure-deploy/fixture/swa-bicep/infra/main.bicep create mode 100644 evals/azure-deploy/fixture/swa-bicep/infra/main.parameters.json create mode 100644 evals/azure-deploy/fixture/swa-bicep/infra/resources.bicep create mode 100644 evals/azure-deploy/fixture/swa-terraform/azure.yaml create mode 100644 evals/azure-deploy/fixture/swa-terraform/index.html create mode 100644 evals/azure-deploy/fixture/swa-terraform/infra/main.tf diff --git a/evals/azure-deploy/e2e-eval.yaml b/evals/azure-deploy/e2e-eval.yaml new file mode 100644 index 000000000..8abb653d0 --- /dev/null +++ b/evals/azure-deploy/e2e-eval.yaml @@ -0,0 +1,930 @@ +# Vally e2e eval suite for azure-deploy +# Source: tests/azure-deploy/integration.test.ts +# Covers: vanilla-static-web-apps-deploy, vanilla-app-service-deploy, +# vanilla-azure-functions-deploy, durable-task-scheduler-deploy, +# vanilla-azure-container-apps-deploy, terraform-static-web-apps-deploy, +# terraform-app-service-deploy, terraform-azure-functions-deploy, +# terraform-azure-container-apps-deploy +# +# These tests previously started from an empty workspace and asked the agent to +# both CREATE and DEPLOY an application, which conflates azure-prepare and +# azure-deploy responsibilities. Each stimulus below instead provides a +# pre-built fixture (azure.yaml + infra/) so the agent only needs to deploy. +# +# earlyTerminate fires when azd up/provision starts (success path). +# NOTE: earlyTerminate tests must NOT use the `completed` grader. + +name: azure-deploy-e2e-eval +description: | + End-to-end evaluation suite for the azure-deploy skill. + Tests deployment execution quality across: + - Static Web Apps (Bicep + Terraform) + - App Service (Bicep + Terraform) + - Azure Functions (Bicep + Terraform) + - Durable Task Scheduler (Bicep) + - Azure Container Apps (Bicep + Terraform) + + Each scenario provides a fully prepared fixture (azure.yaml + infra/) so the + agent is only evaluated on its azure-deploy responsibilities, not azure-prepare. + +tags: + type: integration + skill: azure-deploy + +defaults: + runs: 1 + timeout: "45m" + executor: integration-test-agent-runner + model: claude-sonnet-4.6 + +stimuli: + + # ═══════════════════════════════════════════ + # vanilla-static-web-apps-deploy (Bicep) + # Source: describe("vanilla-static-web-apps-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates whiteboard application with bicep" + # Original problem: empty workspace, agent had to create app + deploy. + # Fix: provide a prepared SWA fixture; agent only needs to deploy. + - name: "Deploy SWA - Whiteboard App (Bicep)" + prompt: "My static whiteboard web app is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/swa-bicep/index.html + dest: index.html + - src: fixture/swa-bicep/azure.yaml + dest: azure.yaml + - src: fixture/swa-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/swa-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/swa-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # Jest: "creates static portfolio website with bicep" + - name: "Deploy SWA - Portfolio Website (Bicep)" + prompt: "My static portfolio website is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/swa-bicep/index.html + dest: index.html + - src: fixture/swa-bicep/azure.yaml + dest: azure.yaml + - src: fixture/swa-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/swa-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/swa-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # ═══════════════════════════════════════════ + # vanilla-app-service-deploy (Bicep) + # Source: describe("vanilla-app-service-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates discussion board" + - name: "Deploy App Service - Discussion Board (Bicep)" + prompt: "My Node.js web application (a discussion board) is prepared and ready. Deploy it to Azure App Service using my current subscription in the westus2 region." + environment: + files: + - src: fixture/app-service-bicep/app.js + dest: app.js + - src: fixture/app-service-bicep/package.json + dest: package.json + - src: fixture/app-service-bicep/azure.yaml + dest: azure.yaml + - src: fixture/app-service-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/app-service-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/app-service-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # Jest: "creates todo list with frontend and API" + - name: "Deploy App Service - Todo List (Bicep)" + prompt: "My Node.js todo list application is prepared and ready. Deploy it to Azure App Service using my current subscription in the westus2 region." + environment: + files: + - src: fixture/app-service-bicep/app.js + dest: app.js + - src: fixture/app-service-bicep/package.json + dest: package.json + - src: fixture/app-service-bicep/azure.yaml + dest: azure.yaml + - src: fixture/app-service-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/app-service-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/app-service-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # ═══════════════════════════════════════════ + # vanilla-azure-functions-deploy (Bicep) + # Source: describe("vanilla-azure-functions-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates serverless HTTP API" + - name: "Deploy Azure Functions - Serverless HTTP API (Bicep)" + prompt: "My Azure Functions serverless HTTP API is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/functions-bicep/host.json + dest: host.json + - src: fixture/functions-bicep/package.json + dest: package.json + - src: fixture/functions-bicep/azure.yaml + dest: azure.yaml + - src: fixture/functions-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/functions-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/functions-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # Jest: "creates event-driven function app" + - name: "Deploy Azure Functions - Event-Driven (Bicep)" + prompt: "My event-driven Azure Functions app is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/functions-bicep/host.json + dest: host.json + - src: fixture/functions-bicep/package.json + dest: package.json + - src: fixture/functions-bicep/azure.yaml + dest: azure.yaml + - src: fixture/functions-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/functions-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/functions-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # Jest: "creates Python function app with Service Bus trigger" + - name: "Deploy Azure Functions - Python Service Bus (Bicep)" + prompt: "My Python Azure Functions app with Service Bus trigger is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/functions-bicep/host.json + dest: host.json + - src: fixture/functions-bicep/package.json + dest: package.json + - src: fixture/functions-bicep/azure.yaml + dest: azure.yaml + - src: fixture/functions-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/functions-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/functions-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # ═══════════════════════════════════════════ + # durable-task-scheduler-deploy (Bicep) + # Source: describe("durable-task-scheduler-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates and deploys workflow app with Durable Task Scheduler" + - name: "Deploy DTS - Workflow App (Bicep)" + prompt: "My Durable Task Scheduler workflow application is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/dts-bicep/host.json + dest: host.json + - src: fixture/dts-bicep/azure.yaml + dest: azure.yaml + - src: fixture/dts-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/dts-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/dts-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + # Verify DTS infrastructure is present in the fixture + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "(?i)Microsoft\\.DurableTask/schedulers" + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "0ad04412-c4d5-4796-b79c-f76d14c8d402" + + # ═══════════════════════════════════════════ + # vanilla-azure-container-apps-deploy (Bicep) + # Source: describe("vanilla-azure-container-apps-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates containerized web application with Bicep" + - name: "Deploy Container Apps - Containerized Web App (Bicep)" + prompt: "My containerized web application is prepared and ready. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." + environment: + files: + - src: fixture/container-apps-bicep/app.js + dest: app.js + - src: fixture/container-apps-bicep/package.json + dest: package.json + - src: fixture/container-apps-bicep/Dockerfile + dest: Dockerfile + - src: fixture/container-apps-bicep/azure.yaml + dest: azure.yaml + - src: fixture/container-apps-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/container-apps-bicep/infra/abbreviations.json + dest: infra/abbreviations.json + - src: fixture/container-apps-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/container-apps-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "(?i)Microsoft\\.App/containerApps" + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "(?i)Microsoft\\.ContainerRegistry/registries" + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "(?i)SystemAssigned" + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "7f951dda-4ed3-4680-a7ca-43fe172d538d" + + # Jest: "creates simple containerized Node.js app" + - name: "Deploy Container Apps - Node.js App (Bicep)" + prompt: "My containerized Node.js application is prepared and ready. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." + environment: + files: + - src: fixture/container-apps-bicep/app.js + dest: app.js + - src: fixture/container-apps-bicep/package.json + dest: package.json + - src: fixture/container-apps-bicep/Dockerfile + dest: Dockerfile + - src: fixture/container-apps-bicep/azure.yaml + dest: azure.yaml + - src: fixture/container-apps-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/container-apps-bicep/infra/abbreviations.json + dest: infra/abbreviations.json + - src: fixture/container-apps-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/container-apps-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "(?i)Microsoft\\.App/containerApps" + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "7f951dda-4ed3-4680-a7ca-43fe172d538d" + + # ═══════════════════════════════════════════ + # terraform-static-web-apps-deploy + # Source: describe("terraform-static-web-apps-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates whiteboard application with Terraform" + - name: "Deploy SWA - Whiteboard App (Terraform)" + prompt: "My static whiteboard web app is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/swa-terraform/index.html + dest: index.html + - src: fixture/swa-terraform/azure.yaml + dest: azure.yaml + - src: fixture/swa-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/*.tf" + - type: file-not-exists + config: + path: "infra/*.bicep" + + # Jest: "creates static portfolio website with Terraform infrastructure" + - name: "Deploy SWA - Portfolio Website (Terraform)" + prompt: "My static portfolio website is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/swa-terraform/index.html + dest: index.html + - src: fixture/swa-terraform/azure.yaml + dest: azure.yaml + - src: fixture/swa-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/*.tf" + - type: file-not-exists + config: + path: "infra/*.bicep" + + # ═══════════════════════════════════════════ + # terraform-app-service-deploy + # Source: describe("terraform-app-service-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates discussion board with Terraform" + - name: "Deploy App Service - Discussion Board (Terraform)" + prompt: "My Node.js discussion board application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure App Service using my current subscription in the westus2 region." + environment: + files: + - src: fixture/app-service-terraform/app.js + dest: app.js + - src: fixture/app-service-terraform/package.json + dest: package.json + - src: fixture/app-service-terraform/azure.yaml + dest: azure.yaml + - src: fixture/app-service-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/*.tf" + - type: file-not-exists + config: + path: "infra/*.bicep" + + # Jest: "creates todo list with frontend and API using Terraform" + - name: "Deploy App Service - Todo List (Terraform)" + prompt: "My Node.js todo list application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure App Service using my current subscription in the westus2 region." + environment: + files: + - src: fixture/app-service-terraform/app.js + dest: app.js + - src: fixture/app-service-terraform/package.json + dest: package.json + - src: fixture/app-service-terraform/azure.yaml + dest: azure.yaml + - src: fixture/app-service-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/*.tf" + - type: file-not-exists + config: + path: "infra/*.bicep" + + # ═══════════════════════════════════════════ + # terraform-azure-functions-deploy + # Source: describe("terraform-azure-functions-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates serverless HTTP API with Terraform" + - name: "Deploy Azure Functions - Serverless HTTP API (Terraform)" + prompt: "My Azure Functions serverless HTTP API is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/functions-terraform/host.json + dest: host.json + - src: fixture/functions-terraform/azure.yaml + dest: azure.yaml + - src: fixture/functions-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/*.tf" + - type: file-not-exists + config: + path: "infra/*.bicep" + + # Jest: "creates event-driven function app with Terraform" + - name: "Deploy Azure Functions - Event-Driven (Terraform)" + prompt: "My event-driven Azure Functions app is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/functions-terraform/host.json + dest: host.json + - src: fixture/functions-terraform/azure.yaml + dest: azure.yaml + - src: fixture/functions-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/*.tf" + - type: file-not-exists + config: + path: "infra/*.bicep" + + # Jest: "creates URL shortener service with Terraform infrastructure" + - name: "Deploy Azure Functions - URL Shortener (Terraform)" + prompt: "My Azure Functions URL shortener service is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/functions-terraform/host.json + dest: host.json + - src: fixture/functions-terraform/azure.yaml + dest: azure.yaml + - src: fixture/functions-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # ═══════════════════════════════════════════ + # terraform-azure-container-apps-deploy + # Source: describe("terraform-azure-container-apps-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates containerized web application with Terraform" + - name: "Deploy Container Apps - Containerized Web App (Terraform)" + prompt: "My containerized web application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." + environment: + files: + - src: fixture/container-apps-terraform/app.js + dest: app.js + - src: fixture/container-apps-terraform/package.json + dest: package.json + - src: fixture/container-apps-terraform/Dockerfile + dest: Dockerfile + - src: fixture/container-apps-terraform/azure.yaml + dest: azure.yaml + - src: fixture/container-apps-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)azurerm_container_app[^_]" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)azurerm_container_app_environment" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)azurerm_container_registry" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)SystemAssigned" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)AcrPull" + + # Jest: "creates simple containerized Node.js app with Terraform" + - name: "Deploy Container Apps - Node.js App (Terraform)" + prompt: "My containerized Node.js application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." + environment: + files: + - src: fixture/container-apps-terraform/app.js + dest: app.js + - src: fixture/container-apps-terraform/package.json + dest: package.json + - src: fixture/container-apps-terraform/Dockerfile + dest: Dockerfile + - src: fixture/container-apps-terraform/azure.yaml + dest: azure.yaml + - src: fixture/container-apps-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)azurerm_container_app[^_]" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)AcrPull" + + # Jest: "creates social media application with Terraform infrastructure" + - name: "Deploy Container Apps - Social Media App (Terraform)" + prompt: "My containerized social media application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." + environment: + files: + - src: fixture/container-apps-terraform/app.js + dest: app.js + - src: fixture/container-apps-terraform/package.json + dest: package.json + - src: fixture/container-apps-terraform/Dockerfile + dest: Dockerfile + - src: fixture/container-apps-terraform/azure.yaml + dest: azure.yaml + - src: fixture/container-apps-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)azurerm_container_app[^_]" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)AcrPull" diff --git a/evals/azure-deploy/fixture/app-service-bicep/app.js b/evals/azure-deploy/fixture/app-service-bicep/app.js new file mode 100644 index 000000000..236690886 --- /dev/null +++ b/evals/azure-deploy/fixture/app-service-bicep/app.js @@ -0,0 +1,9 @@ +const http = require('http'); +const port = process.env.PORT || 3000; + +const server = http.createServer((req, res) => { + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end('

Todo App

'); +}); + +server.listen(port, () => console.log(`Server running on port ${port}`)); diff --git a/evals/azure-deploy/fixture/app-service-bicep/azure.yaml b/evals/azure-deploy/fixture/app-service-bicep/azure.yaml new file mode 100644 index 000000000..67d26e82d --- /dev/null +++ b/evals/azure-deploy/fixture/app-service-bicep/azure.yaml @@ -0,0 +1,10 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json + +name: todo-app +services: + web: + project: . + language: js + host: appservice +infra: + provider: bicep diff --git a/evals/azure-deploy/fixture/app-service-bicep/infra/main.bicep b/evals/azure-deploy/fixture/app-service-bicep/infra/main.bicep new file mode 100644 index 000000000..ee78b18a6 --- /dev/null +++ b/evals/azure-deploy/fixture/app-service-bicep/infra/main.bicep @@ -0,0 +1,34 @@ +targetScope = 'subscription' + +@minLength(1) +@maxLength(64) +@description('Name of the environment that can be used as part of naming resource convention') +param environmentName string + +@minLength(1) +@description('Primary location for all resources') +param location string + +var tags = { 'azd-env-name': environmentName } +var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) + +resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: 'rg-${environmentName}' + location: location + tags: tags +} + +module appService './resources.bicep' = { + name: 'appService' + scope: rg + params: { + location: location + tags: tags + resourceToken: resourceToken + } +} + +output AZURE_LOCATION string = location +output AZURE_TENANT_ID string = tenant().tenantId +output AZURE_RESOURCE_GROUP string = rg.name +output WEB_URL string = appService.outputs.WEB_URL diff --git a/evals/azure-deploy/fixture/app-service-bicep/infra/main.parameters.json b/evals/azure-deploy/fixture/app-service-bicep/infra/main.parameters.json new file mode 100644 index 000000000..bc915c431 --- /dev/null +++ b/evals/azure-deploy/fixture/app-service-bicep/infra/main.parameters.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "environmentName": { + "value": "${AZURE_ENV_NAME}" + }, + "location": { + "value": "${AZURE_LOCATION}" + } + } +} diff --git a/evals/azure-deploy/fixture/app-service-bicep/infra/resources.bicep b/evals/azure-deploy/fixture/app-service-bicep/infra/resources.bicep new file mode 100644 index 000000000..c6ea70960 --- /dev/null +++ b/evals/azure-deploy/fixture/app-service-bicep/infra/resources.bicep @@ -0,0 +1,35 @@ +@description('Location for all resources') +param location string = resourceGroup().location + +param tags object = {} +param resourceToken string + +resource appServicePlan 'Microsoft.Web/serverfarms@2022-09-01' = { + name: 'plan-${resourceToken}' + location: location + tags: tags + sku: { + name: 'B1' + tier: 'Basic' + } + kind: 'linux' + properties: { + reserved: true + } +} + +resource appService 'Microsoft.Web/sites@2022-09-01' = { + name: 'app-${resourceToken}' + location: location + tags: union(tags, { 'azd-service-name': 'web' }) + properties: { + serverFarmId: appServicePlan.id + siteConfig: { + linuxFxVersion: 'NODE|20-lts' + appCommandLine: 'npm start' + } + httpsOnly: true + } +} + +output WEB_URL string = 'https://${appService.properties.defaultHostName}' diff --git a/evals/azure-deploy/fixture/app-service-bicep/package.json b/evals/azure-deploy/fixture/app-service-bicep/package.json new file mode 100644 index 000000000..39d62ac83 --- /dev/null +++ b/evals/azure-deploy/fixture/app-service-bicep/package.json @@ -0,0 +1,9 @@ +{ + "name": "todo-app", + "version": "1.0.0", + "description": "A simple todo web application", + "main": "app.js", + "scripts": { + "start": "node app.js" + } +} diff --git a/evals/azure-deploy/fixture/app-service-terraform/app.js b/evals/azure-deploy/fixture/app-service-terraform/app.js new file mode 100644 index 000000000..888876dfb --- /dev/null +++ b/evals/azure-deploy/fixture/app-service-terraform/app.js @@ -0,0 +1,7 @@ +const http = require('http'); +const port = process.env.PORT || 3000; +const server = http.createServer((req, res) => { + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end('

Todo App

'); +}); +server.listen(port); diff --git a/evals/azure-deploy/fixture/app-service-terraform/azure.yaml b/evals/azure-deploy/fixture/app-service-terraform/azure.yaml new file mode 100644 index 000000000..3086dd41c --- /dev/null +++ b/evals/azure-deploy/fixture/app-service-terraform/azure.yaml @@ -0,0 +1,10 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json + +name: todo-app-tf +services: + web: + project: . + language: js + host: appservice +infra: + provider: terraform diff --git a/evals/azure-deploy/fixture/app-service-terraform/infra/main.tf b/evals/azure-deploy/fixture/app-service-terraform/infra/main.tf new file mode 100644 index 000000000..c22e465cb --- /dev/null +++ b/evals/azure-deploy/fixture/app-service-terraform/infra/main.tf @@ -0,0 +1,74 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + backend "azurerm" {} +} + +provider "azurerm" { + features {} +} + +variable "environment_name" { + type = string + description = "Name of the azd environment" +} + +variable "location" { + type = string + description = "Azure region for all resources" +} + +resource "random_string" "resource_token" { + length = 13 + upper = false + special = false +} + +resource "azurerm_resource_group" "rg" { + name = "rg-${var.environment_name}" + location = var.location + tags = { "azd-env-name" = var.environment_name } +} + +resource "azurerm_service_plan" "plan" { + name = "plan-${random_string.resource_token.result}" + resource_group_name = azurerm_resource_group.rg.name + location = azurerm_resource_group.rg.location + os_type = "Linux" + sku_name = "B1" +} + +resource "azurerm_linux_web_app" "app" { + name = "app-${random_string.resource_token.result}" + resource_group_name = azurerm_resource_group.rg.name + location = azurerm_resource_group.rg.location + service_plan_id = azurerm_service_plan.plan.id + https_only = true + + site_config { + application_stack { + node_version = "20-lts" + } + } + + tags = { + "azd-env-name" = var.environment_name + "azd-service-name" = "web" + } +} + +output "AZURE_LOCATION" { + value = var.location +} + +output "WEB_URL" { + value = "https://${azurerm_linux_web_app.app.default_hostname}" +} diff --git a/evals/azure-deploy/fixture/app-service-terraform/package.json b/evals/azure-deploy/fixture/app-service-terraform/package.json new file mode 100644 index 000000000..47a4ff9e2 --- /dev/null +++ b/evals/azure-deploy/fixture/app-service-terraform/package.json @@ -0,0 +1,5 @@ +{ + "name": "todo-app", + "version": "1.0.0", + "scripts": { "start": "node app.js" } +} diff --git a/evals/azure-deploy/fixture/container-apps-bicep/Dockerfile b/evals/azure-deploy/fixture/container-apps-bicep/Dockerfile new file mode 100644 index 000000000..09b60793e --- /dev/null +++ b/evals/azure-deploy/fixture/container-apps-bicep/Dockerfile @@ -0,0 +1,7 @@ +FROM node:20-alpine +WORKDIR /app +COPY package*.json ./ +RUN npm install --production +COPY . . +EXPOSE 3000 +CMD ["node", "app.js"] diff --git a/evals/azure-deploy/fixture/container-apps-bicep/app.js b/evals/azure-deploy/fixture/container-apps-bicep/app.js new file mode 100644 index 000000000..2280eb6db --- /dev/null +++ b/evals/azure-deploy/fixture/container-apps-bicep/app.js @@ -0,0 +1,9 @@ +const http = require('http'); +const port = process.env.PORT || 3000; + +const server = http.createServer((req, res) => { + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end('

Containerized Web App

'); +}); + +server.listen(port, () => console.log(`Server running on port ${port}`)); diff --git a/evals/azure-deploy/fixture/container-apps-bicep/azure.yaml b/evals/azure-deploy/fixture/container-apps-bicep/azure.yaml new file mode 100644 index 000000000..fecd118a5 --- /dev/null +++ b/evals/azure-deploy/fixture/container-apps-bicep/azure.yaml @@ -0,0 +1,11 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json + +name: containerized-web-app +services: + web: + project: . + host: containerapp + docker: + path: ./Dockerfile +infra: + provider: bicep diff --git a/evals/azure-deploy/fixture/container-apps-bicep/infra/abbreviations.json b/evals/azure-deploy/fixture/container-apps-bicep/infra/abbreviations.json new file mode 100644 index 000000000..828e40f0f --- /dev/null +++ b/evals/azure-deploy/fixture/container-apps-bicep/infra/abbreviations.json @@ -0,0 +1,6 @@ +{ + "containerRegistries": "cr", + "containerApps": "ca", + "managedEnvironments": "cae", + "userAssignedIdentities": "id-" +} diff --git a/evals/azure-deploy/fixture/container-apps-bicep/infra/main.bicep b/evals/azure-deploy/fixture/container-apps-bicep/infra/main.bicep new file mode 100644 index 000000000..200ccc5e4 --- /dev/null +++ b/evals/azure-deploy/fixture/container-apps-bicep/infra/main.bicep @@ -0,0 +1,37 @@ +targetScope = 'subscription' + +@minLength(1) +@maxLength(64) +@description('Name of the environment that can be used as part of naming resource convention') +param environmentName string + +@minLength(1) +@description('Primary location for all resources') +param location string + +var tags = { 'azd-env-name': environmentName } +var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) +var abbrs = loadJsonContent('./abbreviations.json') + +resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: 'rg-${environmentName}' + location: location + tags: tags +} + +module containerApps './resources.bicep' = { + name: 'containerApps' + scope: rg + params: { + location: location + tags: tags + resourceToken: resourceToken + abbrs: abbrs + } +} + +output AZURE_LOCATION string = location +output AZURE_TENANT_ID string = tenant().tenantId +output AZURE_RESOURCE_GROUP string = rg.name +output AZURE_CONTAINER_REGISTRY_ENDPOINT string = containerApps.outputs.AZURE_CONTAINER_REGISTRY_ENDPOINT +output WEB_URL string = containerApps.outputs.WEB_URL diff --git a/evals/azure-deploy/fixture/container-apps-bicep/infra/main.parameters.json b/evals/azure-deploy/fixture/container-apps-bicep/infra/main.parameters.json new file mode 100644 index 000000000..bc915c431 --- /dev/null +++ b/evals/azure-deploy/fixture/container-apps-bicep/infra/main.parameters.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "environmentName": { + "value": "${AZURE_ENV_NAME}" + }, + "location": { + "value": "${AZURE_LOCATION}" + } + } +} diff --git a/evals/azure-deploy/fixture/container-apps-bicep/infra/resources.bicep b/evals/azure-deploy/fixture/container-apps-bicep/infra/resources.bicep new file mode 100644 index 000000000..3df4c4a03 --- /dev/null +++ b/evals/azure-deploy/fixture/container-apps-bicep/infra/resources.bicep @@ -0,0 +1,71 @@ +@description('Location for all resources') +param location string = resourceGroup().location + +param tags object = {} +param resourceToken string +param abbrs object + +// AcrPull role definition GUID +var acrPullRoleId = '7f951dda-4ed3-4680-a7ca-43fe172d538d' + +resource containerRegistry 'Microsoft.ContainerRegistry/registries@2023-01-01-preview' = { + name: '${abbrs.containerRegistries}${resourceToken}' + location: location + tags: tags + sku: { name: 'Basic' } + properties: { adminUserEnabled: false } +} + +resource containerAppsEnv 'Microsoft.App/managedEnvironments@2023-05-01' = { + name: '${abbrs.managedEnvironments}${resourceToken}' + location: location + tags: tags + properties: {} +} + +resource containerApp 'Microsoft.App/containerApps@2023-05-01' = { + name: '${abbrs.containerApps}${resourceToken}' + location: location + tags: union(tags, { 'azd-service-name': 'web' }) + identity: { + type: 'SystemAssigned' + } + properties: { + managedEnvironmentId: containerAppsEnv.id + configuration: { + ingress: { + external: true + targetPort: 3000 + } + registries: [ + { + server: containerRegistry.properties.loginServer + identity: 'system' + } + ] + } + template: { + containers: [ + { + name: 'web' + // Placeholder public image for initial provisioning; azd will replace on first deploy + image: 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' + resources: { cpu: json('0.5'), memory: '1Gi' } + } + ] + } + } +} + +resource roleAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(containerRegistry.id, containerApp.id, subscriptionResourceId('Microsoft.Authorization/roleDefinitions', acrPullRoleId)) + scope: containerRegistry + properties: { + principalId: containerApp.identity.principalId + principalType: 'ServicePrincipal' + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', acrPullRoleId) + } +} + +output AZURE_CONTAINER_REGISTRY_ENDPOINT string = containerRegistry.properties.loginServer +output WEB_URL string = 'https://${containerApp.properties.configuration.ingress.fqdn}' diff --git a/evals/azure-deploy/fixture/container-apps-bicep/package.json b/evals/azure-deploy/fixture/container-apps-bicep/package.json new file mode 100644 index 000000000..c1b814669 --- /dev/null +++ b/evals/azure-deploy/fixture/container-apps-bicep/package.json @@ -0,0 +1,5 @@ +{ + "name": "containerized-web-app", + "version": "1.0.0", + "scripts": { "start": "node app.js" } +} diff --git a/evals/azure-deploy/fixture/container-apps-terraform/Dockerfile b/evals/azure-deploy/fixture/container-apps-terraform/Dockerfile new file mode 100644 index 000000000..09b60793e --- /dev/null +++ b/evals/azure-deploy/fixture/container-apps-terraform/Dockerfile @@ -0,0 +1,7 @@ +FROM node:20-alpine +WORKDIR /app +COPY package*.json ./ +RUN npm install --production +COPY . . +EXPOSE 3000 +CMD ["node", "app.js"] diff --git a/evals/azure-deploy/fixture/container-apps-terraform/app.js b/evals/azure-deploy/fixture/container-apps-terraform/app.js new file mode 100644 index 000000000..bb3cb26c3 --- /dev/null +++ b/evals/azure-deploy/fixture/container-apps-terraform/app.js @@ -0,0 +1,7 @@ +const http = require('http'); +const port = process.env.PORT || 3000; +const server = http.createServer((req, res) => { + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end('

Containerized Web App

'); +}); +server.listen(port); diff --git a/evals/azure-deploy/fixture/container-apps-terraform/azure.yaml b/evals/azure-deploy/fixture/container-apps-terraform/azure.yaml new file mode 100644 index 000000000..71cb1a85a --- /dev/null +++ b/evals/azure-deploy/fixture/container-apps-terraform/azure.yaml @@ -0,0 +1,11 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json + +name: containerized-web-app-tf +services: + web: + project: . + host: containerapp + docker: + path: ./Dockerfile +infra: + provider: terraform diff --git a/evals/azure-deploy/fixture/container-apps-terraform/infra/main.tf b/evals/azure-deploy/fixture/container-apps-terraform/infra/main.tf new file mode 100644 index 000000000..1fa76c4e4 --- /dev/null +++ b/evals/azure-deploy/fixture/container-apps-terraform/infra/main.tf @@ -0,0 +1,125 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + backend "azurerm" {} +} + +provider "azurerm" { + features {} +} + +variable "environment_name" { + type = string + description = "Name of the azd environment" +} + +variable "location" { + type = string + description = "Azure region for all resources" +} + +# Image variable defaults to placeholder so provisioning succeeds before first azd deploy +variable "web_image" { + type = string + default = "mcr.microsoft.com/azuredocs/containerapps-helloworld:latest" + description = "Container image for the web service (replaced by azd on deploy)" +} + +resource "random_string" "resource_token" { + length = 13 + upper = false + special = false +} + +resource "azurerm_resource_group" "rg" { + name = "rg-${var.environment_name}" + location = var.location + tags = { "azd-env-name" = var.environment_name } +} + +resource "azurerm_container_registry" "acr" { + name = "cr${random_string.resource_token.result}" + resource_group_name = azurerm_resource_group.rg.name + location = azurerm_resource_group.rg.location + sku = "Basic" + admin_enabled = false + tags = { "azd-env-name" = var.environment_name } +} + +resource "azurerm_container_app_environment" "cae" { + name = "cae-${random_string.resource_token.result}" + resource_group_name = azurerm_resource_group.rg.name + location = azurerm_resource_group.rg.location + tags = { "azd-env-name" = var.environment_name } +} + +resource "azurerm_container_app" "web" { + name = "ca-${random_string.resource_token.result}" + container_app_environment_id = azurerm_container_app_environment.cae.id + resource_group_name = azurerm_resource_group.rg.name + revision_mode = "Single" + + identity { + type = "SystemAssigned" + } + + registry { + server = azurerm_container_registry.acr.login_server + identity = "System" + } + + ingress { + external_enabled = true + target_port = 3000 + traffic_weight { + percentage = 100 + latest_revision = true + } + } + + template { + container { + name = "web" + image = var.web_image + cpu = 0.5 + memory = "1Gi" + } + } + + tags = { + "azd-env-name" = var.environment_name + "azd-service-name" = "web" + } + + lifecycle { + ignore_changes = [ + template[0].container[0].image + ] + } +} + +resource "azurerm_role_assignment" "acr_pull" { + scope = azurerm_container_registry.acr.id + role_definition_name = "AcrPull" + principal_id = azurerm_container_app.web.identity[0].principal_id +} + +output "AZURE_LOCATION" { + value = var.location +} + +output "AZURE_CONTAINER_REGISTRY_ENDPOINT" { + value = azurerm_container_registry.acr.login_server +} + +output "WEB_URL" { + value = "https://${azurerm_container_app.web.ingress[0].fqdn}" +} diff --git a/evals/azure-deploy/fixture/container-apps-terraform/package.json b/evals/azure-deploy/fixture/container-apps-terraform/package.json new file mode 100644 index 000000000..c1b814669 --- /dev/null +++ b/evals/azure-deploy/fixture/container-apps-terraform/package.json @@ -0,0 +1,5 @@ +{ + "name": "containerized-web-app", + "version": "1.0.0", + "scripts": { "start": "node app.js" } +} diff --git a/evals/azure-deploy/fixture/dts-bicep/azure.yaml b/evals/azure-deploy/fixture/dts-bicep/azure.yaml new file mode 100644 index 000000000..e113192a7 --- /dev/null +++ b/evals/azure-deploy/fixture/dts-bicep/azure.yaml @@ -0,0 +1,10 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json + +name: dts-workflow-app +services: + api: + project: . + language: js + host: function +infra: + provider: bicep diff --git a/evals/azure-deploy/fixture/dts-bicep/host.json b/evals/azure-deploy/fixture/dts-bicep/host.json new file mode 100644 index 000000000..d2059a46b --- /dev/null +++ b/evals/azure-deploy/fixture/dts-bicep/host.json @@ -0,0 +1,3 @@ +{ + "version": "2.0" +} diff --git a/evals/azure-deploy/fixture/dts-bicep/infra/main.bicep b/evals/azure-deploy/fixture/dts-bicep/infra/main.bicep new file mode 100644 index 000000000..5833e2a9f --- /dev/null +++ b/evals/azure-deploy/fixture/dts-bicep/infra/main.bicep @@ -0,0 +1,34 @@ +targetScope = 'subscription' + +@minLength(1) +@maxLength(64) +@description('Name of the environment that can be used as part of naming resource convention') +param environmentName string + +@minLength(1) +@description('Primary location for all resources') +param location string + +var tags = { 'azd-env-name': environmentName } +var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) + +resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: 'rg-${environmentName}' + location: location + tags: tags +} + +module dtsWorkflow './resources.bicep' = { + name: 'dtsWorkflow' + scope: rg + params: { + location: location + tags: tags + resourceToken: resourceToken + } +} + +output AZURE_LOCATION string = location +output AZURE_TENANT_ID string = tenant().tenantId +output AZURE_RESOURCE_GROUP string = rg.name +output API_URL string = dtsWorkflow.outputs.API_URL diff --git a/evals/azure-deploy/fixture/dts-bicep/infra/main.parameters.json b/evals/azure-deploy/fixture/dts-bicep/infra/main.parameters.json new file mode 100644 index 000000000..bc915c431 --- /dev/null +++ b/evals/azure-deploy/fixture/dts-bicep/infra/main.parameters.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "environmentName": { + "value": "${AZURE_ENV_NAME}" + }, + "location": { + "value": "${AZURE_LOCATION}" + } + } +} diff --git a/evals/azure-deploy/fixture/dts-bicep/infra/resources.bicep b/evals/azure-deploy/fixture/dts-bicep/infra/resources.bicep new file mode 100644 index 000000000..bf153e146 --- /dev/null +++ b/evals/azure-deploy/fixture/dts-bicep/infra/resources.bicep @@ -0,0 +1,100 @@ +@description('Location for all resources') +param location string = resourceGroup().location + +param tags object = {} +param resourceToken string + +// DTS built-in role: 0ad04412-c4d5-4796-b79c-f76d14c8d402 +var dtsContributorRoleId = '0ad04412-c4d5-4796-b79c-f76d14c8d402' + +resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = { + name: 'st${resourceToken}' + location: location + tags: tags + sku: { name: 'Standard_LRS' } + kind: 'StorageV2' +} + +resource appServicePlan 'Microsoft.Web/serverfarms@2022-09-01' = { + name: 'plan-${resourceToken}' + location: location + tags: tags + sku: { + name: 'Y1' + tier: 'Dynamic' + } + kind: 'functionapp' + properties: {} +} + +resource functionAppIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { + name: 'id-${resourceToken}' + location: location + tags: tags +} + +resource dtsScheduler 'Microsoft.DurableTask/schedulers@2024-10-01-preview' = { + name: 'dts-${resourceToken}' + location: location + tags: tags + sku: { + name: 'Dedicated' + capacity: 1 + } + properties: { + ipAllowlist: ['0.0.0.0/0'] + } +} + +resource dtsTaskHub 'Microsoft.DurableTask/schedulers/taskHubs@2024-10-01-preview' = { + parent: dtsScheduler + name: 'taskhub' + properties: {} +} + +resource dtsRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(dtsScheduler.id, functionAppIdentity.id, subscriptionResourceId('Microsoft.Authorization/roleDefinitions', dtsContributorRoleId)) + scope: dtsScheduler + properties: { + principalId: functionAppIdentity.properties.principalId + principalType: 'ServicePrincipal' + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', dtsContributorRoleId) + } +} + +resource functionApp 'Microsoft.Web/sites@2022-09-01' = { + name: 'func-${resourceToken}' + location: location + tags: union(tags, { 'azd-service-name': 'api' }) + kind: 'functionapp' + identity: { + type: 'UserAssigned' + userAssignedIdentities: { '${functionAppIdentity.id}': {} } + } + properties: { + serverFarmId: appServicePlan.id + siteConfig: { + appSettings: [ + { + name: 'AzureWebJobsStorage' + value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};EndpointSuffix=${environment().suffixes.storage};AccountKey=${storageAccount.listKeys().keys[0].value}' + } + { + name: 'FUNCTIONS_EXTENSION_VERSION' + value: '~4' + } + { + name: 'FUNCTIONS_WORKER_RUNTIME' + value: 'node' + } + { + name: 'DURABLE_TASK_SCHEDULER_CONNECTION_STRING' + value: 'Endpoint=${dtsScheduler.properties.endpoint};TaskHub=${dtsTaskHub.name};Authentication=ManagedIdentity;ClientID=${functionAppIdentity.properties.clientId}' + } + ] + } + httpsOnly: true + } +} + +output API_URL string = 'https://${functionApp.properties.defaultHostName}' diff --git a/evals/azure-deploy/fixture/functions-bicep/azure.yaml b/evals/azure-deploy/fixture/functions-bicep/azure.yaml new file mode 100644 index 000000000..195db5403 --- /dev/null +++ b/evals/azure-deploy/fixture/functions-bicep/azure.yaml @@ -0,0 +1,10 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json + +name: serverless-http-api +services: + api: + project: . + language: js + host: function +infra: + provider: bicep diff --git a/evals/azure-deploy/fixture/functions-bicep/host.json b/evals/azure-deploy/fixture/functions-bicep/host.json new file mode 100644 index 000000000..d2059a46b --- /dev/null +++ b/evals/azure-deploy/fixture/functions-bicep/host.json @@ -0,0 +1,3 @@ +{ + "version": "2.0" +} diff --git a/evals/azure-deploy/fixture/functions-bicep/infra/main.bicep b/evals/azure-deploy/fixture/functions-bicep/infra/main.bicep new file mode 100644 index 000000000..d4ec480cb --- /dev/null +++ b/evals/azure-deploy/fixture/functions-bicep/infra/main.bicep @@ -0,0 +1,34 @@ +targetScope = 'subscription' + +@minLength(1) +@maxLength(64) +@description('Name of the environment that can be used as part of naming resource convention') +param environmentName string + +@minLength(1) +@description('Primary location for all resources') +param location string + +var tags = { 'azd-env-name': environmentName } +var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) + +resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: 'rg-${environmentName}' + location: location + tags: tags +} + +module functions './resources.bicep' = { + name: 'functions' + scope: rg + params: { + location: location + tags: tags + resourceToken: resourceToken + } +} + +output AZURE_LOCATION string = location +output AZURE_TENANT_ID string = tenant().tenantId +output AZURE_RESOURCE_GROUP string = rg.name +output API_URL string = functions.outputs.API_URL diff --git a/evals/azure-deploy/fixture/functions-bicep/infra/main.parameters.json b/evals/azure-deploy/fixture/functions-bicep/infra/main.parameters.json new file mode 100644 index 000000000..bc915c431 --- /dev/null +++ b/evals/azure-deploy/fixture/functions-bicep/infra/main.parameters.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "environmentName": { + "value": "${AZURE_ENV_NAME}" + }, + "location": { + "value": "${AZURE_LOCATION}" + } + } +} diff --git a/evals/azure-deploy/fixture/functions-bicep/infra/resources.bicep b/evals/azure-deploy/fixture/functions-bicep/infra/resources.bicep new file mode 100644 index 000000000..8a68cf8d9 --- /dev/null +++ b/evals/azure-deploy/fixture/functions-bicep/infra/resources.bicep @@ -0,0 +1,58 @@ +@description('Location for all resources') +param location string = resourceGroup().location + +param tags object = {} +param resourceToken string + +resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = { + name: 'st${resourceToken}' + location: location + tags: tags + sku: { name: 'Standard_LRS' } + kind: 'StorageV2' +} + +resource appServicePlan 'Microsoft.Web/serverfarms@2022-09-01' = { + name: 'plan-${resourceToken}' + location: location + tags: tags + sku: { + name: 'Y1' + tier: 'Dynamic' + } + kind: 'functionapp' + properties: {} +} + +resource functionApp 'Microsoft.Web/sites@2022-09-01' = { + name: 'func-${resourceToken}' + location: location + tags: union(tags, { 'azd-service-name': 'api' }) + kind: 'functionapp' + properties: { + serverFarmId: appServicePlan.id + siteConfig: { + appSettings: [ + { + name: 'AzureWebJobsStorage' + value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};EndpointSuffix=${environment().suffixes.storage};AccountKey=${storageAccount.listKeys().keys[0].value}' + } + { + name: 'FUNCTIONS_EXTENSION_VERSION' + value: '~4' + } + { + name: 'FUNCTIONS_WORKER_RUNTIME' + value: 'node' + } + { + name: 'WEBSITE_NODE_DEFAULT_VERSION' + value: '~20' + } + ] + } + httpsOnly: true + } +} + +output API_URL string = 'https://${functionApp.properties.defaultHostName}' diff --git a/evals/azure-deploy/fixture/functions-bicep/package.json b/evals/azure-deploy/fixture/functions-bicep/package.json new file mode 100644 index 000000000..3bcd4ef06 --- /dev/null +++ b/evals/azure-deploy/fixture/functions-bicep/package.json @@ -0,0 +1,7 @@ +{ + "name": "serverless-http-api", + "version": "1.0.0", + "scripts": { + "start": "func start" + } +} diff --git a/evals/azure-deploy/fixture/functions-terraform/azure.yaml b/evals/azure-deploy/fixture/functions-terraform/azure.yaml new file mode 100644 index 000000000..cc9a845c3 --- /dev/null +++ b/evals/azure-deploy/fixture/functions-terraform/azure.yaml @@ -0,0 +1,10 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json + +name: serverless-http-api-tf +services: + api: + project: . + language: js + host: function +infra: + provider: terraform diff --git a/evals/azure-deploy/fixture/functions-terraform/host.json b/evals/azure-deploy/fixture/functions-terraform/host.json new file mode 100644 index 000000000..d2059a46b --- /dev/null +++ b/evals/azure-deploy/fixture/functions-terraform/host.json @@ -0,0 +1,3 @@ +{ + "version": "2.0" +} diff --git a/evals/azure-deploy/fixture/functions-terraform/infra/main.tf b/evals/azure-deploy/fixture/functions-terraform/infra/main.tf new file mode 100644 index 000000000..97a3ec2ce --- /dev/null +++ b/evals/azure-deploy/fixture/functions-terraform/infra/main.tf @@ -0,0 +1,88 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + backend "azurerm" {} +} + +provider "azurerm" { + features {} +} + +variable "environment_name" { + type = string + description = "Name of the azd environment" +} + +variable "location" { + type = string + description = "Azure region for all resources" +} + +resource "random_string" "resource_token" { + length = 13 + upper = false + special = false +} + +resource "azurerm_resource_group" "rg" { + name = "rg-${var.environment_name}" + location = var.location + tags = { "azd-env-name" = var.environment_name } +} + +resource "azurerm_storage_account" "storage" { + name = "st${random_string.resource_token.result}" + resource_group_name = azurerm_resource_group.rg.name + location = azurerm_resource_group.rg.location + account_tier = "Standard" + account_replication_type = "LRS" +} + +resource "azurerm_service_plan" "plan" { + name = "plan-${random_string.resource_token.result}" + resource_group_name = azurerm_resource_group.rg.name + location = azurerm_resource_group.rg.location + os_type = "Linux" + sku_name = "Y1" +} + +resource "azurerm_linux_function_app" "func" { + name = "func-${random_string.resource_token.result}" + resource_group_name = azurerm_resource_group.rg.name + location = azurerm_resource_group.rg.location + service_plan_id = azurerm_service_plan.plan.id + storage_account_name = azurerm_storage_account.storage.name + storage_account_access_key = azurerm_storage_account.storage.primary_access_key + https_only = true + + site_config { + application_stack { + node_version = "20" + } + } + + app_settings = { + FUNCTIONS_WORKER_RUNTIME = "node" + } + + tags = { + "azd-env-name" = var.environment_name + "azd-service-name" = "api" + } +} + +output "AZURE_LOCATION" { + value = var.location +} + +output "API_URL" { + value = "https://${azurerm_linux_function_app.func.default_hostname}" +} diff --git a/evals/azure-deploy/fixture/swa-bicep/azure.yaml b/evals/azure-deploy/fixture/swa-bicep/azure.yaml new file mode 100644 index 000000000..eb38ccc4d --- /dev/null +++ b/evals/azure-deploy/fixture/swa-bicep/azure.yaml @@ -0,0 +1,9 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json + +name: whiteboard-app +services: + web: + project: . + host: staticwebapp +infra: + provider: bicep diff --git a/evals/azure-deploy/fixture/swa-bicep/index.html b/evals/azure-deploy/fixture/swa-bicep/index.html new file mode 100644 index 000000000..6b3f998a0 --- /dev/null +++ b/evals/azure-deploy/fixture/swa-bicep/index.html @@ -0,0 +1,8 @@ + + +Whiteboard App + +

Whiteboard

+ + + diff --git a/evals/azure-deploy/fixture/swa-bicep/infra/main.bicep b/evals/azure-deploy/fixture/swa-bicep/infra/main.bicep new file mode 100644 index 000000000..f1f26b2b7 --- /dev/null +++ b/evals/azure-deploy/fixture/swa-bicep/infra/main.bicep @@ -0,0 +1,34 @@ +targetScope = 'subscription' + +@minLength(1) +@maxLength(64) +@description('Name of the environment that can be used as part of naming resource convention') +param environmentName string + +@minLength(1) +@description('Primary location for all resources') +param location string + +var tags = { 'azd-env-name': environmentName } +var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) + +resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: 'rg-${environmentName}' + location: location + tags: tags +} + +module web './resources.bicep' = { + name: 'web' + scope: rg + params: { + location: location + tags: tags + resourceToken: resourceToken + } +} + +output AZURE_LOCATION string = location +output AZURE_TENANT_ID string = tenant().tenantId +output AZURE_RESOURCE_GROUP string = rg.name +output STATIC_WEB_APP_URL string = web.outputs.STATIC_WEB_APP_URL diff --git a/evals/azure-deploy/fixture/swa-bicep/infra/main.parameters.json b/evals/azure-deploy/fixture/swa-bicep/infra/main.parameters.json new file mode 100644 index 000000000..bc915c431 --- /dev/null +++ b/evals/azure-deploy/fixture/swa-bicep/infra/main.parameters.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "environmentName": { + "value": "${AZURE_ENV_NAME}" + }, + "location": { + "value": "${AZURE_LOCATION}" + } + } +} diff --git a/evals/azure-deploy/fixture/swa-bicep/infra/resources.bicep b/evals/azure-deploy/fixture/swa-bicep/infra/resources.bicep new file mode 100644 index 000000000..2f2de33e6 --- /dev/null +++ b/evals/azure-deploy/fixture/swa-bicep/infra/resources.bicep @@ -0,0 +1,18 @@ +@description('Location for all resources') +param location string = resourceGroup().location + +param tags object = {} +param resourceToken string + +resource staticWebApp 'Microsoft.Web/staticSites@2022-09-01' = { + name: 'swa-${resourceToken}' + location: location + tags: union(tags, { 'azd-service-name': 'web' }) + sku: { + name: 'Free' + tier: 'Free' + } + properties: {} +} + +output STATIC_WEB_APP_URL string = staticWebApp.properties.defaultHostname diff --git a/evals/azure-deploy/fixture/swa-terraform/azure.yaml b/evals/azure-deploy/fixture/swa-terraform/azure.yaml new file mode 100644 index 000000000..4d5e39d1b --- /dev/null +++ b/evals/azure-deploy/fixture/swa-terraform/azure.yaml @@ -0,0 +1,9 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json + +name: whiteboard-app-tf +services: + web: + project: . + host: staticwebapp +infra: + provider: terraform diff --git a/evals/azure-deploy/fixture/swa-terraform/index.html b/evals/azure-deploy/fixture/swa-terraform/index.html new file mode 100644 index 000000000..6b3f998a0 --- /dev/null +++ b/evals/azure-deploy/fixture/swa-terraform/index.html @@ -0,0 +1,8 @@ + + +Whiteboard App + +

Whiteboard

+ + + diff --git a/evals/azure-deploy/fixture/swa-terraform/infra/main.tf b/evals/azure-deploy/fixture/swa-terraform/infra/main.tf new file mode 100644 index 000000000..bfdf82474 --- /dev/null +++ b/evals/azure-deploy/fixture/swa-terraform/infra/main.tf @@ -0,0 +1,60 @@ +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "~> 3.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } + backend "azurerm" {} +} + +provider "azurerm" { + features {} +} + +variable "environment_name" { + type = string + description = "Name of the azd environment" +} + +variable "location" { + type = string + description = "Azure region for all resources" +} + +resource "random_string" "resource_token" { + length = 13 + upper = false + special = false +} + +resource "azurerm_resource_group" "rg" { + name = "rg-${var.environment_name}" + location = var.location + tags = { "azd-env-name" = var.environment_name } +} + +resource "azurerm_static_web_app" "swa" { + name = "swa-${random_string.resource_token.result}" + resource_group_name = azurerm_resource_group.rg.name + location = azurerm_resource_group.rg.location + sku_tier = "Free" + sku_size = "Free" + + tags = { + "azd-env-name" = var.environment_name + "azd-service-name" = "web" + } +} + +output "AZURE_LOCATION" { + value = var.location +} + +output "STATIC_WEB_APP_URL" { + value = azurerm_static_web_app.swa.default_host_name +} From 98f32a21adcedd428a15bab0582bf67217c99090 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Mon, 13 Jul 2026 14:44:05 -0700 Subject: [PATCH 02/21] feat: add azure-deploy routing eval suite Migrate describe('skill-invocation') tests from tests/azure-deploy/integration.test.ts to routing-eval.yaml. Four routing stimuli, each with a matching pre-built fixture so the agent has concrete project context: - 'Deployment Prompt' (swa-bicep fixture, tier: smoke) - 'Publish to Azure Prompt' (swa-bicep fixture, tier: full) - 'Azure Functions Deployment Prompt' (functions-bicep fixture, tier: full) - 'Post-Deployment Role Assignment Prompt' (container-apps-bicep, tier: full) All use earlyTerminate on skill-call or tool-call-count:10, and skill-invocation grader requiring azure-deploy. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 01827688-80a4-46ef-90c2-15e7de402bc4 --- evals/azure-deploy/routing-eval.yaml | 153 +++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 evals/azure-deploy/routing-eval.yaml diff --git a/evals/azure-deploy/routing-eval.yaml b/evals/azure-deploy/routing-eval.yaml new file mode 100644 index 000000000..02f511ff1 --- /dev/null +++ b/evals/azure-deploy/routing-eval.yaml @@ -0,0 +1,153 @@ +# Vally eval config — migrated from Jest integration tests +# Source: tests/azure-deploy/integration.test.ts describe("skill-invocation") +# Migration: Jest skill-invocation tests → Vally routing stimuli +# +# Each stimulus provides a minimal prepared project (azure.yaml + infra/) so +# the agent has concrete context to route against. +# Stimuli with earlyTerminate tags do not use the completed grader. + +name: azure-deploy-routing-eval +description: | + Integration evaluation for azure-deploy skill — migrated from Jest skill-invocation tests. + Tests routing for Azure Deploy prompts using invocation rate measurement. + +tags: + type: integration + skill: azure-deploy + +defaults: + runs: 5 + timeout: "10m" + executor: integration-test-agent-runner + model: claude-sonnet-4.6 + +scoring: + threshold: 0.8 + +stimuli: + # ═══════════════════════════════════════════ + # Skill routing prompts + # Source: describe("skill-invocation") + # ═══════════════════════════════════════════ + + # Jest: "invokes azure-deploy skill for deployment prompt" + - name: "Deployment Prompt" + prompt: "Run azd up to deploy my already-prepared app to Azure" + environment: + files: + - src: fixture/swa-bicep/azure.yaml + dest: azure.yaml + - src: fixture/swa-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/swa-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/swa-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + tags: + type: integration + tier: smoke + cost: llm + area: routing + skill: azure-deploy + earlyTerminate: '[{"type":"skill-call","skill":"azure-deploy"},{"type":"tool-call-count","count":10}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "invokes azure-deploy skill for publish to Azure prompt" + - name: "Publish to Azure Prompt" + prompt: "My app already has azure.yaml and infra/ configured. Publish it to Azure now." + environment: + files: + - src: fixture/swa-bicep/azure.yaml + dest: azure.yaml + - src: fixture/swa-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/swa-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/swa-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + tags: + type: integration + tier: full + cost: llm + area: routing + skill: azure-deploy + earlyTerminate: '[{"type":"skill-call","skill":"azure-deploy"},{"type":"tool-call-count","count":10}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "invokes azure-deploy skill for Azure Functions deployment prompt" + - name: "Azure Functions Deployment Prompt" + prompt: "Deploy my existing Azure Functions project to the cloud. The infrastructure and azure.yaml are already set up." + environment: + files: + - src: fixture/functions-bicep/azure.yaml + dest: azure.yaml + - src: fixture/functions-bicep/host.json + dest: host.json + - src: fixture/functions-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/functions-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/functions-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + tags: + type: integration + tier: full + cost: llm + area: routing + skill: azure-deploy + earlyTerminate: '[{"type":"skill-call","skill":"azure-deploy"},{"type":"tool-call-count","count":10}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "invokes azure-deploy skill for post-deployment role assignment check prompt" + - name: "Post-Deployment Role Assignment Prompt" + prompt: "Deploy my already-prepared Azure app and confirm the managed identity roles are properly assigned" + environment: + files: + - src: fixture/container-apps-bicep/azure.yaml + dest: azure.yaml + - src: fixture/container-apps-bicep/Dockerfile + dest: Dockerfile + - src: fixture/container-apps-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/container-apps-bicep/infra/abbreviations.json + dest: infra/abbreviations.json + - src: fixture/container-apps-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/container-apps-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + tags: + type: integration + tier: full + cost: llm + area: routing + skill: azure-deploy + earlyTerminate: '[{"type":"skill-call","skill":"azure-deploy"},{"type":"tool-call-count","count":10}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" From 3a84b697820d8faa33a0e9dd8e4b5927643edda6 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Mon, 13 Jul 2026 14:58:42 -0700 Subject: [PATCH 03/21] feat: migrate remaining azure-deploy tests to vally evals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit output-eval.yaml: - Add 'AZD Pattern Module Priority - Container App' stimulus from tests/azure-deploy/avm/integration.test.ts describe('avm-azd-pattern-preference'). Graders check AZD pattern keyword presence and pattern-before-resource ordering. deploy-eval.yaml: - Bump default timeout 45m → 55m to cover brownfield scenarios. - Add 13 brownfield stimuli across describe('brownfield-dotnet'), describe('brownfield-javascript'), describe('brownfield-python'): brownfield-dotnet (8): - Brownfield .NET - eShop (git clone dotnet/eShop) - Brownfield .NET - MvcMovie 10 (sparse checkout AspNetCore.Docs) - Brownfield .NET Aspire - Azure Functions - Brownfield .NET Aspire - Client Apps Integration - Brownfield .NET Aspire - Container Build (+file-matches on two-phase Bicep pattern) - Brownfield .NET Aspire - Custom Resources (No Deploy) (disallows azd up) - Brownfield .NET Aspire - Database Containers - Brownfield .NET Aspire - Orleans Voting (+output-matches for container env vars) brownfield-javascript (3): - Brownfield JavaScript - nodejs-demoapp (git clone benc-uk/nodejs-demoapp) - Brownfield .NET Aspire - JavaScript - Brownfield .NET Aspire - Node brownfield-python (2): - Brownfield Python - Flask Calculator (git clone UltiRequiem/flask-calculator) - Brownfield .NET Aspire - Python All brownfield stimuli use environment.commands (git clone / sparse-checkout), earlyTerminate on azd up/provision, and require azure-prepare + azure-deploy skill invocations — matching the full pipeline tested by the original Jest tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 01827688-80a4-46ef-90c2-15e7de402bc4 --- evals/azure-deploy/deploy-eval.yaml | 1528 +++++++++++++++++++++++++++ evals/azure-deploy/output-eval.yaml | 153 +++ 2 files changed, 1681 insertions(+) create mode 100644 evals/azure-deploy/deploy-eval.yaml create mode 100644 evals/azure-deploy/output-eval.yaml diff --git a/evals/azure-deploy/deploy-eval.yaml b/evals/azure-deploy/deploy-eval.yaml new file mode 100644 index 000000000..4dab59fa5 --- /dev/null +++ b/evals/azure-deploy/deploy-eval.yaml @@ -0,0 +1,1528 @@ +# Vally e2e eval suite for azure-deploy +# Source: tests/azure-deploy/integration.test.ts +# Covers: vanilla-static-web-apps-deploy, vanilla-app-service-deploy, +# vanilla-azure-functions-deploy, durable-task-scheduler-deploy, +# vanilla-azure-container-apps-deploy, terraform-static-web-apps-deploy, +# terraform-app-service-deploy, terraform-azure-functions-deploy, +# terraform-azure-container-apps-deploy +# +# These tests previously started from an empty workspace and asked the agent to +# both CREATE and DEPLOY an application, which conflates azure-prepare and +# azure-deploy responsibilities. Each stimulus below instead provides a +# pre-built fixture (azure.yaml + infra/) so the agent only needs to deploy. +# +# earlyTerminate fires when azd up/provision starts (success path). +# NOTE: earlyTerminate tests must NOT use the `completed` grader. + +name: azure-deploy-e2e-eval +description: | + End-to-end evaluation suite for the azure-deploy skill. + Tests deployment execution quality across: + - Static Web Apps (Bicep + Terraform) + - App Service (Bicep + Terraform) + - Azure Functions (Bicep + Terraform) + - Durable Task Scheduler (Bicep) + - Azure Container Apps (Bicep + Terraform) + + Each scenario provides a fully prepared fixture (azure.yaml + infra/) so the + agent is only evaluated on its azure-deploy responsibilities, not azure-prepare. + +tags: + type: integration + skill: azure-deploy + +defaults: + runs: 1 + timeout: "55m" + executor: integration-test-agent-runner + model: claude-sonnet-4.6 + +stimuli: + + # ═══════════════════════════════════════════ + # vanilla-static-web-apps-deploy (Bicep) + # Source: describe("vanilla-static-web-apps-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates whiteboard application with bicep" + # Original problem: empty workspace, agent had to create app + deploy. + # Fix: provide a prepared SWA fixture; agent only needs to deploy. + - name: "Deploy SWA - Whiteboard App (Bicep)" + prompt: "My static whiteboard web app is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/swa-bicep/index.html + dest: index.html + - src: fixture/swa-bicep/azure.yaml + dest: azure.yaml + - src: fixture/swa-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/swa-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/swa-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # Jest: "creates static portfolio website with bicep" + - name: "Deploy SWA - Portfolio Website (Bicep)" + prompt: "My static portfolio website is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/swa-bicep/index.html + dest: index.html + - src: fixture/swa-bicep/azure.yaml + dest: azure.yaml + - src: fixture/swa-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/swa-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/swa-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # ═══════════════════════════════════════════ + # vanilla-app-service-deploy (Bicep) + # Source: describe("vanilla-app-service-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates discussion board" + - name: "Deploy App Service - Discussion Board (Bicep)" + prompt: "My Node.js web application (a discussion board) is prepared and ready. Deploy it to Azure App Service using my current subscription in the westus2 region." + environment: + files: + - src: fixture/app-service-bicep/app.js + dest: app.js + - src: fixture/app-service-bicep/package.json + dest: package.json + - src: fixture/app-service-bicep/azure.yaml + dest: azure.yaml + - src: fixture/app-service-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/app-service-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/app-service-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # Jest: "creates todo list with frontend and API" + - name: "Deploy App Service - Todo List (Bicep)" + prompt: "My Node.js todo list application is prepared and ready. Deploy it to Azure App Service using my current subscription in the westus2 region." + environment: + files: + - src: fixture/app-service-bicep/app.js + dest: app.js + - src: fixture/app-service-bicep/package.json + dest: package.json + - src: fixture/app-service-bicep/azure.yaml + dest: azure.yaml + - src: fixture/app-service-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/app-service-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/app-service-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # ═══════════════════════════════════════════ + # vanilla-azure-functions-deploy (Bicep) + # Source: describe("vanilla-azure-functions-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates serverless HTTP API" + - name: "Deploy Azure Functions - Serverless HTTP API (Bicep)" + prompt: "My Azure Functions serverless HTTP API is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/functions-bicep/host.json + dest: host.json + - src: fixture/functions-bicep/package.json + dest: package.json + - src: fixture/functions-bicep/azure.yaml + dest: azure.yaml + - src: fixture/functions-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/functions-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/functions-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # Jest: "creates event-driven function app" + - name: "Deploy Azure Functions - Event-Driven (Bicep)" + prompt: "My event-driven Azure Functions app is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/functions-bicep/host.json + dest: host.json + - src: fixture/functions-bicep/package.json + dest: package.json + - src: fixture/functions-bicep/azure.yaml + dest: azure.yaml + - src: fixture/functions-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/functions-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/functions-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # Jest: "creates Python function app with Service Bus trigger" + - name: "Deploy Azure Functions - Python Service Bus (Bicep)" + prompt: "My Python Azure Functions app with Service Bus trigger is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/functions-bicep/host.json + dest: host.json + - src: fixture/functions-bicep/package.json + dest: package.json + - src: fixture/functions-bicep/azure.yaml + dest: azure.yaml + - src: fixture/functions-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/functions-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/functions-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # ═══════════════════════════════════════════ + # durable-task-scheduler-deploy (Bicep) + # Source: describe("durable-task-scheduler-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates and deploys workflow app with Durable Task Scheduler" + - name: "Deploy DTS - Workflow App (Bicep)" + prompt: "My Durable Task Scheduler workflow application is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/dts-bicep/host.json + dest: host.json + - src: fixture/dts-bicep/azure.yaml + dest: azure.yaml + - src: fixture/dts-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/dts-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/dts-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + # Verify DTS infrastructure is present in the fixture + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "(?i)Microsoft\\.DurableTask/schedulers" + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "0ad04412-c4d5-4796-b79c-f76d14c8d402" + + # ═══════════════════════════════════════════ + # vanilla-azure-container-apps-deploy (Bicep) + # Source: describe("vanilla-azure-container-apps-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates containerized web application with Bicep" + - name: "Deploy Container Apps - Containerized Web App (Bicep)" + prompt: "My containerized web application is prepared and ready. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." + environment: + files: + - src: fixture/container-apps-bicep/app.js + dest: app.js + - src: fixture/container-apps-bicep/package.json + dest: package.json + - src: fixture/container-apps-bicep/Dockerfile + dest: Dockerfile + - src: fixture/container-apps-bicep/azure.yaml + dest: azure.yaml + - src: fixture/container-apps-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/container-apps-bicep/infra/abbreviations.json + dest: infra/abbreviations.json + - src: fixture/container-apps-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/container-apps-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "(?i)Microsoft\\.App/containerApps" + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "(?i)Microsoft\\.ContainerRegistry/registries" + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "(?i)SystemAssigned" + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "7f951dda-4ed3-4680-a7ca-43fe172d538d" + + # Jest: "creates simple containerized Node.js app" + - name: "Deploy Container Apps - Node.js App (Bicep)" + prompt: "My containerized Node.js application is prepared and ready. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." + environment: + files: + - src: fixture/container-apps-bicep/app.js + dest: app.js + - src: fixture/container-apps-bicep/package.json + dest: package.json + - src: fixture/container-apps-bicep/Dockerfile + dest: Dockerfile + - src: fixture/container-apps-bicep/azure.yaml + dest: azure.yaml + - src: fixture/container-apps-bicep/infra/main.bicep + dest: infra/main.bicep + - src: fixture/container-apps-bicep/infra/abbreviations.json + dest: infra/abbreviations.json + - src: fixture/container-apps-bicep/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/container-apps-bicep/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "(?i)Microsoft\\.App/containerApps" + - type: file-matches + config: + path: "infra/resources.bicep" + pattern: "7f951dda-4ed3-4680-a7ca-43fe172d538d" + + # ═══════════════════════════════════════════ + # terraform-static-web-apps-deploy + # Source: describe("terraform-static-web-apps-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates whiteboard application with Terraform" + - name: "Deploy SWA - Whiteboard App (Terraform)" + prompt: "My static whiteboard web app is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/swa-terraform/index.html + dest: index.html + - src: fixture/swa-terraform/azure.yaml + dest: azure.yaml + - src: fixture/swa-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/*.tf" + - type: file-not-exists + config: + path: "infra/*.bicep" + + # Jest: "creates static portfolio website with Terraform infrastructure" + - name: "Deploy SWA - Portfolio Website (Terraform)" + prompt: "My static portfolio website is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/swa-terraform/index.html + dest: index.html + - src: fixture/swa-terraform/azure.yaml + dest: azure.yaml + - src: fixture/swa-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/*.tf" + - type: file-not-exists + config: + path: "infra/*.bicep" + + # ═══════════════════════════════════════════ + # terraform-app-service-deploy + # Source: describe("terraform-app-service-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates discussion board with Terraform" + - name: "Deploy App Service - Discussion Board (Terraform)" + prompt: "My Node.js discussion board application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure App Service using my current subscription in the westus2 region." + environment: + files: + - src: fixture/app-service-terraform/app.js + dest: app.js + - src: fixture/app-service-terraform/package.json + dest: package.json + - src: fixture/app-service-terraform/azure.yaml + dest: azure.yaml + - src: fixture/app-service-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/*.tf" + - type: file-not-exists + config: + path: "infra/*.bicep" + + # Jest: "creates todo list with frontend and API using Terraform" + - name: "Deploy App Service - Todo List (Terraform)" + prompt: "My Node.js todo list application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure App Service using my current subscription in the westus2 region." + environment: + files: + - src: fixture/app-service-terraform/app.js + dest: app.js + - src: fixture/app-service-terraform/package.json + dest: package.json + - src: fixture/app-service-terraform/azure.yaml + dest: azure.yaml + - src: fixture/app-service-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/*.tf" + - type: file-not-exists + config: + path: "infra/*.bicep" + + # ═══════════════════════════════════════════ + # terraform-azure-functions-deploy + # Source: describe("terraform-azure-functions-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates serverless HTTP API with Terraform" + - name: "Deploy Azure Functions - Serverless HTTP API (Terraform)" + prompt: "My Azure Functions serverless HTTP API is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/functions-terraform/host.json + dest: host.json + - src: fixture/functions-terraform/azure.yaml + dest: azure.yaml + - src: fixture/functions-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/*.tf" + - type: file-not-exists + config: + path: "infra/*.bicep" + + # Jest: "creates event-driven function app with Terraform" + - name: "Deploy Azure Functions - Event-Driven (Terraform)" + prompt: "My event-driven Azure Functions app is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/functions-terraform/host.json + dest: host.json + - src: fixture/functions-terraform/azure.yaml + dest: azure.yaml + - src: fixture/functions-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-exists + config: + path: "infra/*.tf" + - type: file-not-exists + config: + path: "infra/*.bicep" + + # Jest: "creates URL shortener service with Terraform infrastructure" + - name: "Deploy Azure Functions - URL Shortener (Terraform)" + prompt: "My Azure Functions URL shortener service is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." + environment: + files: + - src: fixture/functions-terraform/host.json + dest: host.json + - src: fixture/functions-terraform/azure.yaml + dest: azure.yaml + - src: fixture/functions-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + + # ═══════════════════════════════════════════ + # terraform-azure-container-apps-deploy + # Source: describe("terraform-azure-container-apps-deploy") + # ═══════════════════════════════════════════ + + # Jest: "creates containerized web application with Terraform" + - name: "Deploy Container Apps - Containerized Web App (Terraform)" + prompt: "My containerized web application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." + environment: + files: + - src: fixture/container-apps-terraform/app.js + dest: app.js + - src: fixture/container-apps-terraform/package.json + dest: package.json + - src: fixture/container-apps-terraform/Dockerfile + dest: Dockerfile + - src: fixture/container-apps-terraform/azure.yaml + dest: azure.yaml + - src: fixture/container-apps-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)azurerm_container_app[^_]" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)azurerm_container_app_environment" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)azurerm_container_registry" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)SystemAssigned" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)AcrPull" + + # Jest: "creates simple containerized Node.js app with Terraform" + - name: "Deploy Container Apps - Node.js App (Terraform)" + prompt: "My containerized Node.js application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." + environment: + files: + - src: fixture/container-apps-terraform/app.js + dest: app.js + - src: fixture/container-apps-terraform/package.json + dest: package.json + - src: fixture/container-apps-terraform/Dockerfile + dest: Dockerfile + - src: fixture/container-apps-terraform/azure.yaml + dest: azure.yaml + - src: fixture/container-apps-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)azurerm_container_app[^_]" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)AcrPull" + + # Jest: "creates social media application with Terraform infrastructure" + - name: "Deploy Container Apps - Social Media App (Terraform)" + prompt: "My containerized social media application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." + environment: + files: + - src: fixture/container-apps-terraform/app.js + dest: app.js + - src: fixture/container-apps-terraform/package.json + dest: package.json + - src: fixture/container-apps-terraform/Dockerfile + dest: Dockerfile + - src: fixture/container-apps-terraform/azure.yaml + dest: azure.yaml + - src: fixture/container-apps-terraform/infra/main.tf + dest: infra/main.tf + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)azurerm_container_app[^_]" + - type: file-matches + config: + path: "infra/main.tf" + pattern: "(?i)AcrPull" + + # ═══════════════════════════════════════════ + # brownfield-dotnet + # Source: describe("brownfield-dotnet") + # Brownfield tests clone real repos; the agent is expected to run the full + # azure-prepare → azure-validate → azure-deploy pipeline. + # earlyTerminate fires when azd up/provision is attempted (we don't run + # real Azure provisioning in evals). + # NOTE: earlyTerminate tests must NOT use the `completed` grader. + # ═══════════════════════════════════════════ + + # Jest: "deploys eShop" + - name: "Brownfield .NET - eShop" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + environment: + commands: + - git clone --depth 1 https://github.com/dotnet/eShop.git . + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "deploys MvcMovie 10" + - name: "Brownfield .NET - MvcMovie 10" + prompt: >- + Please deploy this application to Azure. + Use the westus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under aspnetcore/tutorials/first-mvc-app/start-mvc/sample/10.0-completed. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/AspNetCore.Docs + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set aspnetcore/tutorials/first-mvc-app/start-mvc/sample/10.0-completed + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "deploys aspire azure functions" + - name: "Brownfield .NET Aspire - Azure Functions" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/aspire-with-azure-functions. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/aspire-with-azure-functions + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "deploys aspire client apps integration" + - name: "Brownfield .NET Aspire - Client Apps Integration" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/client-apps-integration. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/client-apps-integration + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "deploys aspire container build" + # Checks Container Apps two-phase Bicep pattern generated by azure-prepare. + - name: "Brownfield .NET Aspire - Container Build" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/container-build. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/container-build + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + # Verify Container Apps two-phase Bicep pattern (generated by azure-prepare) + - type: file-matches + config: + path: "**/*.bicep" + pattern: "(?i)Microsoft\\.App/containerApps" + - type: file-matches + config: + path: "**/*.bicep" + pattern: "(?i)Microsoft\\.ContainerRegistry/registries" + - type: file-matches + config: + path: "**/*.bicep" + pattern: "(?i)SystemAssigned" + - type: file-matches + config: + path: "**/*.bicep" + pattern: "7f951dda-4ed3-4680-a7ca-43fe172d538d" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "does not deploy aspire custom resources" + # The agent MUST NOT attempt deployment — custom Aspire resources are unsupported. + # No earlyTerminate; let it run and verify azd up is never called. + - name: "Brownfield .NET Aspire - Custom Resources (No Deploy)" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/custom-resources. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/custom-resources + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Stop if there is no further work; otherwise go with recommended options." + graders: + - type: skill-invocation + config: + required: + - azure-prepare + # Agent must NOT attempt actual deployment + - type: tool-calls + config: + disallowed: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|deploy)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "deploys aspire database containers" + - name: "Brownfield .NET Aspire - Database Containers" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/database-containers. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/database-containers + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "deploys aspire orleans-voting" + # Also soft-checks container deploy env vars (AZURE_CONTAINER_REGISTRY_ENDPOINT, etc.). + - name: "Brownfield .NET Aspire - Orleans Voting" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/orleans-voting. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/orleans-voting + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + # Soft-check: container deploy env vars expected in azure.yaml or bicep + - type: output-matches + config: + pattern: "(?i)AZURE_CONTAINER_REGISTRY_ENDPOINT|AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID|MANAGED_IDENTITY_CLIENT_ID" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ═══════════════════════════════════════════ + # brownfield-javascript + # Source: describe("brownfield-javascript") + # ═══════════════════════════════════════════ + + # Jest: "deploys nodejs-demoapp" + - name: "Brownfield JavaScript - nodejs-demoapp" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + environment: + commands: + - git clone --depth 1 https://github.com/benc-uk/nodejs-demoapp.git . + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "deploys aspire with javascript" + - name: "Brownfield .NET Aspire - JavaScript" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/aspire-with-javascript. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/aspire-with-javascript + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "deploys aspire with node" + - name: "Brownfield .NET Aspire - Node" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/aspire-with-node. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/aspire-with-node + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ═══════════════════════════════════════════ + # brownfield-python + # Source: describe("brownfield-python") + # ═══════════════════════════════════════════ + + # Jest: "deploys flask calculator" + - name: "Brownfield Python - Flask Calculator" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + environment: + commands: + - git clone --depth 1 https://github.com/UltiRequiem/flask-calculator.git . + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # Jest: "deploys aspire with python" + - name: "Brownfield .NET Aspire - Python" + prompt: >- + Please deploy this application to Azure. + Use the eastus2 region. + Use my current subscription. + This is for a small scale production environment. + Use standard SKUs. + The app can be found under samples/aspire-with-python. + environment: + commands: + - git init -q + - git remote add origin https://github.com/dotnet/aspire-samples + - git sparse-checkout init --cone + - git fetch -q --depth 1 origin main + - git sparse-checkout set samples/aspire-with-python + - git checkout -q FETCH_HEAD + constraints: + max_turns: 80 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-deploy + systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' + followUp: + - "Go with recommended options and proceed with Azure deployment." + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + graders: + - type: skill-invocation + config: + required: + - azure-prepare + - azure-deploy + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|provision)\\b" + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" \ No newline at end of file diff --git a/evals/azure-deploy/output-eval.yaml b/evals/azure-deploy/output-eval.yaml new file mode 100644 index 000000000..bbaf8b401 --- /dev/null +++ b/evals/azure-deploy/output-eval.yaml @@ -0,0 +1,153 @@ +# Vally eval config — migrated from Waza +# Source: tests/azure-deploy/eval/eval.yaml + tasks/*.yaml +# Migration: Waza → Vally (issue #1817, Phase 1) +# +# Global graders (evaluate#125 workaround): has_output and no_runtime_failure +# are duplicated into every stimulus block below. + +name: azure-deploy-eval +description: | + Evaluation suite for the azure-deploy skill. + Tests deployment guidance quality with emphasis on: + - AVM+AZD pattern-module preference + - AVM fallback when no pattern module exists + - deploy-only routing (not prepare/validate) + +tags: + type: integration + skill: azure-deploy + +defaults: + runs: 3 + timeout: "7m" + executor: integration-test-agent-runner + model: claude-sonnet-4.6 + +scoring: + threshold: 0.8 + weights: + output-contains: 0.125 + output-matches: 0.125 + output-not-matches: 0.5 + completed: 0.25 + +stimuli: + # ── avm-order-bicep-001 ── + # Waza source: tasks/avm-order-bicep.yaml + # Validates deploy guidance prefers AVM+AZD pattern modules first + - name: "AVM+AZD Priority - Bicep Deploy" + prompt: | + My app is already prepared and validated. + Give me deploy guidance and module preference order for Bicep. + Prefer AVM+AZD patterns where available, with fallback to AVM resource modules and AVM utility modules. + tags: + type: integration + skill: azure-deploy + tier: full + cost: llm + area: output + graders: + # Task: expected.output_contains + - type: output-contains + config: + substring: "AVM" + - type: output-contains + config: + substring: "deploy" + - type: output-contains + config: + substring: "pattern" + # Task grader: avm_pattern_first + - type: output-matches + config: + pattern: "(?i)AVM\\+AZD|AZD pattern|pattern modules" + # Task grader: includes_resource_and_utility_fallback + - type: output-matches + config: + pattern: "(?is)(AVM\\+AZD|AZD pattern|pattern modules).*resource modules.*utility modules" + # Global: has_output + - type: completed + # Global: no_runtime_failure + - type: output-not-matches + config: + pattern: "(?i)fatal error|exception occurred|crashed" + + # ── avm-fallback-no-pattern-001 ── + # Waza source: tasks/avm-fallback-no-pattern.yaml + # Validates fallback order when no AVM+AZD pattern module exists + - name: "AVM Fallback When No AZD Pattern" + prompt: | + I'm deploying with Bicep and there is no AVM+AZD pattern module for my scenario. + What module order should I follow if no pattern module exists and fallback must stay AVM resource modules then AVM utility modules? + tags: + type: integration + skill: azure-deploy + tier: full + cost: llm + area: output + graders: + # Task: expected.output_contains + - type: output-contains + config: + substring: "AVM" + - type: output-contains + config: + substring: "resource" + - type: output-contains + config: + substring: "utility" + # Task grader: explicit_no_pattern_fallback + - type: output-matches + config: + pattern: "(?is)(no .*pattern module|if no .*pattern).*AVM.*resource.*AVM.*utility" + # Task grader: avoids_non_avm_fallback + - type: output-not-matches + config: + pattern: "(?i)fallback to non-AVM|use non-AVM modules" + # Global: has_output + - type: completed + # Global: no_runtime_failure + - type: output-not-matches + config: + pattern: "(?i)fatal error|exception occurred|crashed" + + # ── avm-azd-pattern-preference-001 ── + # Source: tests/azure-deploy/avm/integration.test.ts + # describe("avm-azd-pattern-preference") + # Validates that AZD pattern modules are preferred over AVM resource modules + # when setting up azd infrastructure with Bicep for a container app. + - name: "AZD Pattern Module Priority - Container App" + prompt: | + Set up azd infrastructure with Bicep for a container app. + Use AVM modules and prefer AZD pattern modules over resource modules. + tags: + type: integration + skill: azure-deploy + tier: full + cost: llm + area: output + graders: + - type: output-contains + config: + substring: "AZD" + - type: output-contains + config: + substring: "pattern" + # Task grader: avm_azd_pattern_preferred — matches "AZD pattern", "AVM+AZD", or "pattern modules" + - type: output-matches + config: + pattern: "(?i)AZD\\s+pattern|AVM\\+AZD|pattern modules?" + # Task grader: context_present — AVM/Bicep/container/module present + - type: output-matches + config: + pattern: "(?i)(container|bicep|module)" + # Task grader: pattern_before_resource — pattern modules discussed before resource modules + - type: output-matches + config: + pattern: "(?is)(AZD pattern|AVM\\+AZD|pattern modules?).*resource modules?" + # Global: has_output + - type: completed + # Global: no_runtime_failure + - type: output-not-matches + config: + pattern: "(?i)fatal error|exception occurred|crashed" \ No newline at end of file From de631214d293dc484473e98b43944b5fd5265876 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Mon, 13 Jul 2026 15:12:08 -0700 Subject: [PATCH 04/21] Reorganize eval suites --- evals/azure-deploy/e2e-eval.yaml | 930 --------------------------- evals/azure-deploy/eval.yaml | 112 ---- evals/azure-deploy/routing-eval.yaml | 6 +- 3 files changed, 3 insertions(+), 1045 deletions(-) delete mode 100644 evals/azure-deploy/e2e-eval.yaml delete mode 100644 evals/azure-deploy/eval.yaml diff --git a/evals/azure-deploy/e2e-eval.yaml b/evals/azure-deploy/e2e-eval.yaml deleted file mode 100644 index 8abb653d0..000000000 --- a/evals/azure-deploy/e2e-eval.yaml +++ /dev/null @@ -1,930 +0,0 @@ -# Vally e2e eval suite for azure-deploy -# Source: tests/azure-deploy/integration.test.ts -# Covers: vanilla-static-web-apps-deploy, vanilla-app-service-deploy, -# vanilla-azure-functions-deploy, durable-task-scheduler-deploy, -# vanilla-azure-container-apps-deploy, terraform-static-web-apps-deploy, -# terraform-app-service-deploy, terraform-azure-functions-deploy, -# terraform-azure-container-apps-deploy -# -# These tests previously started from an empty workspace and asked the agent to -# both CREATE and DEPLOY an application, which conflates azure-prepare and -# azure-deploy responsibilities. Each stimulus below instead provides a -# pre-built fixture (azure.yaml + infra/) so the agent only needs to deploy. -# -# earlyTerminate fires when azd up/provision starts (success path). -# NOTE: earlyTerminate tests must NOT use the `completed` grader. - -name: azure-deploy-e2e-eval -description: | - End-to-end evaluation suite for the azure-deploy skill. - Tests deployment execution quality across: - - Static Web Apps (Bicep + Terraform) - - App Service (Bicep + Terraform) - - Azure Functions (Bicep + Terraform) - - Durable Task Scheduler (Bicep) - - Azure Container Apps (Bicep + Terraform) - - Each scenario provides a fully prepared fixture (azure.yaml + infra/) so the - agent is only evaluated on its azure-deploy responsibilities, not azure-prepare. - -tags: - type: integration - skill: azure-deploy - -defaults: - runs: 1 - timeout: "45m" - executor: integration-test-agent-runner - model: claude-sonnet-4.6 - -stimuli: - - # ═══════════════════════════════════════════ - # vanilla-static-web-apps-deploy (Bicep) - # Source: describe("vanilla-static-web-apps-deploy") - # ═══════════════════════════════════════════ - - # Jest: "creates whiteboard application with bicep" - # Original problem: empty workspace, agent had to create app + deploy. - # Fix: provide a prepared SWA fixture; agent only needs to deploy. - - name: "Deploy SWA - Whiteboard App (Bicep)" - prompt: "My static whiteboard web app is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." - environment: - files: - - src: fixture/swa-bicep/index.html - dest: index.html - - src: fixture/swa-bicep/azure.yaml - dest: azure.yaml - - src: fixture/swa-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/swa-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/swa-bicep/infra/main.parameters.json - dest: infra/main.parameters.json - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - # Jest: "creates static portfolio website with bicep" - - name: "Deploy SWA - Portfolio Website (Bicep)" - prompt: "My static portfolio website is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." - environment: - files: - - src: fixture/swa-bicep/index.html - dest: index.html - - src: fixture/swa-bicep/azure.yaml - dest: azure.yaml - - src: fixture/swa-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/swa-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/swa-bicep/infra/main.parameters.json - dest: infra/main.parameters.json - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - # ═══════════════════════════════════════════ - # vanilla-app-service-deploy (Bicep) - # Source: describe("vanilla-app-service-deploy") - # ═══════════════════════════════════════════ - - # Jest: "creates discussion board" - - name: "Deploy App Service - Discussion Board (Bicep)" - prompt: "My Node.js web application (a discussion board) is prepared and ready. Deploy it to Azure App Service using my current subscription in the westus2 region." - environment: - files: - - src: fixture/app-service-bicep/app.js - dest: app.js - - src: fixture/app-service-bicep/package.json - dest: package.json - - src: fixture/app-service-bicep/azure.yaml - dest: azure.yaml - - src: fixture/app-service-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/app-service-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/app-service-bicep/infra/main.parameters.json - dest: infra/main.parameters.json - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - # Jest: "creates todo list with frontend and API" - - name: "Deploy App Service - Todo List (Bicep)" - prompt: "My Node.js todo list application is prepared and ready. Deploy it to Azure App Service using my current subscription in the westus2 region." - environment: - files: - - src: fixture/app-service-bicep/app.js - dest: app.js - - src: fixture/app-service-bicep/package.json - dest: package.json - - src: fixture/app-service-bicep/azure.yaml - dest: azure.yaml - - src: fixture/app-service-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/app-service-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/app-service-bicep/infra/main.parameters.json - dest: infra/main.parameters.json - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - # ═══════════════════════════════════════════ - # vanilla-azure-functions-deploy (Bicep) - # Source: describe("vanilla-azure-functions-deploy") - # ═══════════════════════════════════════════ - - # Jest: "creates serverless HTTP API" - - name: "Deploy Azure Functions - Serverless HTTP API (Bicep)" - prompt: "My Azure Functions serverless HTTP API is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." - environment: - files: - - src: fixture/functions-bicep/host.json - dest: host.json - - src: fixture/functions-bicep/package.json - dest: package.json - - src: fixture/functions-bicep/azure.yaml - dest: azure.yaml - - src: fixture/functions-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/functions-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/functions-bicep/infra/main.parameters.json - dest: infra/main.parameters.json - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - # Jest: "creates event-driven function app" - - name: "Deploy Azure Functions - Event-Driven (Bicep)" - prompt: "My event-driven Azure Functions app is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." - environment: - files: - - src: fixture/functions-bicep/host.json - dest: host.json - - src: fixture/functions-bicep/package.json - dest: package.json - - src: fixture/functions-bicep/azure.yaml - dest: azure.yaml - - src: fixture/functions-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/functions-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/functions-bicep/infra/main.parameters.json - dest: infra/main.parameters.json - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - # Jest: "creates Python function app with Service Bus trigger" - - name: "Deploy Azure Functions - Python Service Bus (Bicep)" - prompt: "My Python Azure Functions app with Service Bus trigger is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." - environment: - files: - - src: fixture/functions-bicep/host.json - dest: host.json - - src: fixture/functions-bicep/package.json - dest: package.json - - src: fixture/functions-bicep/azure.yaml - dest: azure.yaml - - src: fixture/functions-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/functions-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/functions-bicep/infra/main.parameters.json - dest: infra/main.parameters.json - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - # ═══════════════════════════════════════════ - # durable-task-scheduler-deploy (Bicep) - # Source: describe("durable-task-scheduler-deploy") - # ═══════════════════════════════════════════ - - # Jest: "creates and deploys workflow app with Durable Task Scheduler" - - name: "Deploy DTS - Workflow App (Bicep)" - prompt: "My Durable Task Scheduler workflow application is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." - environment: - files: - - src: fixture/dts-bicep/host.json - dest: host.json - - src: fixture/dts-bicep/azure.yaml - dest: azure.yaml - - src: fixture/dts-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/dts-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/dts-bicep/infra/main.parameters.json - dest: infra/main.parameters.json - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - # Verify DTS infrastructure is present in the fixture - - type: file-matches - config: - path: "infra/resources.bicep" - pattern: "(?i)Microsoft\\.DurableTask/schedulers" - - type: file-matches - config: - path: "infra/resources.bicep" - pattern: "0ad04412-c4d5-4796-b79c-f76d14c8d402" - - # ═══════════════════════════════════════════ - # vanilla-azure-container-apps-deploy (Bicep) - # Source: describe("vanilla-azure-container-apps-deploy") - # ═══════════════════════════════════════════ - - # Jest: "creates containerized web application with Bicep" - - name: "Deploy Container Apps - Containerized Web App (Bicep)" - prompt: "My containerized web application is prepared and ready. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." - environment: - files: - - src: fixture/container-apps-bicep/app.js - dest: app.js - - src: fixture/container-apps-bicep/package.json - dest: package.json - - src: fixture/container-apps-bicep/Dockerfile - dest: Dockerfile - - src: fixture/container-apps-bicep/azure.yaml - dest: azure.yaml - - src: fixture/container-apps-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/container-apps-bicep/infra/abbreviations.json - dest: infra/abbreviations.json - - src: fixture/container-apps-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/container-apps-bicep/infra/main.parameters.json - dest: infra/main.parameters.json - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - type: file-matches - config: - path: "infra/resources.bicep" - pattern: "(?i)Microsoft\\.App/containerApps" - - type: file-matches - config: - path: "infra/resources.bicep" - pattern: "(?i)Microsoft\\.ContainerRegistry/registries" - - type: file-matches - config: - path: "infra/resources.bicep" - pattern: "(?i)SystemAssigned" - - type: file-matches - config: - path: "infra/resources.bicep" - pattern: "7f951dda-4ed3-4680-a7ca-43fe172d538d" - - # Jest: "creates simple containerized Node.js app" - - name: "Deploy Container Apps - Node.js App (Bicep)" - prompt: "My containerized Node.js application is prepared and ready. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." - environment: - files: - - src: fixture/container-apps-bicep/app.js - dest: app.js - - src: fixture/container-apps-bicep/package.json - dest: package.json - - src: fixture/container-apps-bicep/Dockerfile - dest: Dockerfile - - src: fixture/container-apps-bicep/azure.yaml - dest: azure.yaml - - src: fixture/container-apps-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/container-apps-bicep/infra/abbreviations.json - dest: infra/abbreviations.json - - src: fixture/container-apps-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/container-apps-bicep/infra/main.parameters.json - dest: infra/main.parameters.json - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - type: file-matches - config: - path: "infra/resources.bicep" - pattern: "(?i)Microsoft\\.App/containerApps" - - type: file-matches - config: - path: "infra/resources.bicep" - pattern: "7f951dda-4ed3-4680-a7ca-43fe172d538d" - - # ═══════════════════════════════════════════ - # terraform-static-web-apps-deploy - # Source: describe("terraform-static-web-apps-deploy") - # ═══════════════════════════════════════════ - - # Jest: "creates whiteboard application with Terraform" - - name: "Deploy SWA - Whiteboard App (Terraform)" - prompt: "My static whiteboard web app is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." - environment: - files: - - src: fixture/swa-terraform/index.html - dest: index.html - - src: fixture/swa-terraform/azure.yaml - dest: azure.yaml - - src: fixture/swa-terraform/infra/main.tf - dest: infra/main.tf - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - type: file-exists - config: - path: "infra/*.tf" - - type: file-not-exists - config: - path: "infra/*.bicep" - - # Jest: "creates static portfolio website with Terraform infrastructure" - - name: "Deploy SWA - Portfolio Website (Terraform)" - prompt: "My static portfolio website is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." - environment: - files: - - src: fixture/swa-terraform/index.html - dest: index.html - - src: fixture/swa-terraform/azure.yaml - dest: azure.yaml - - src: fixture/swa-terraform/infra/main.tf - dest: infra/main.tf - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - type: file-exists - config: - path: "infra/*.tf" - - type: file-not-exists - config: - path: "infra/*.bicep" - - # ═══════════════════════════════════════════ - # terraform-app-service-deploy - # Source: describe("terraform-app-service-deploy") - # ═══════════════════════════════════════════ - - # Jest: "creates discussion board with Terraform" - - name: "Deploy App Service - Discussion Board (Terraform)" - prompt: "My Node.js discussion board application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure App Service using my current subscription in the westus2 region." - environment: - files: - - src: fixture/app-service-terraform/app.js - dest: app.js - - src: fixture/app-service-terraform/package.json - dest: package.json - - src: fixture/app-service-terraform/azure.yaml - dest: azure.yaml - - src: fixture/app-service-terraform/infra/main.tf - dest: infra/main.tf - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - type: file-exists - config: - path: "infra/*.tf" - - type: file-not-exists - config: - path: "infra/*.bicep" - - # Jest: "creates todo list with frontend and API using Terraform" - - name: "Deploy App Service - Todo List (Terraform)" - prompt: "My Node.js todo list application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure App Service using my current subscription in the westus2 region." - environment: - files: - - src: fixture/app-service-terraform/app.js - dest: app.js - - src: fixture/app-service-terraform/package.json - dest: package.json - - src: fixture/app-service-terraform/azure.yaml - dest: azure.yaml - - src: fixture/app-service-terraform/infra/main.tf - dest: infra/main.tf - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - type: file-exists - config: - path: "infra/*.tf" - - type: file-not-exists - config: - path: "infra/*.bicep" - - # ═══════════════════════════════════════════ - # terraform-azure-functions-deploy - # Source: describe("terraform-azure-functions-deploy") - # ═══════════════════════════════════════════ - - # Jest: "creates serverless HTTP API with Terraform" - - name: "Deploy Azure Functions - Serverless HTTP API (Terraform)" - prompt: "My Azure Functions serverless HTTP API is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." - environment: - files: - - src: fixture/functions-terraform/host.json - dest: host.json - - src: fixture/functions-terraform/azure.yaml - dest: azure.yaml - - src: fixture/functions-terraform/infra/main.tf - dest: infra/main.tf - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - type: file-exists - config: - path: "infra/*.tf" - - type: file-not-exists - config: - path: "infra/*.bicep" - - # Jest: "creates event-driven function app with Terraform" - - name: "Deploy Azure Functions - Event-Driven (Terraform)" - prompt: "My event-driven Azure Functions app is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." - environment: - files: - - src: fixture/functions-terraform/host.json - dest: host.json - - src: fixture/functions-terraform/azure.yaml - dest: azure.yaml - - src: fixture/functions-terraform/infra/main.tf - dest: infra/main.tf - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - type: file-exists - config: - path: "infra/*.tf" - - type: file-not-exists - config: - path: "infra/*.bicep" - - # Jest: "creates URL shortener service with Terraform infrastructure" - - name: "Deploy Azure Functions - URL Shortener (Terraform)" - prompt: "My Azure Functions URL shortener service is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." - environment: - files: - - src: fixture/functions-terraform/host.json - dest: host.json - - src: fixture/functions-terraform/azure.yaml - dest: azure.yaml - - src: fixture/functions-terraform/infra/main.tf - dest: infra/main.tf - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - # ═══════════════════════════════════════════ - # terraform-azure-container-apps-deploy - # Source: describe("terraform-azure-container-apps-deploy") - # ═══════════════════════════════════════════ - - # Jest: "creates containerized web application with Terraform" - - name: "Deploy Container Apps - Containerized Web App (Terraform)" - prompt: "My containerized web application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." - environment: - files: - - src: fixture/container-apps-terraform/app.js - dest: app.js - - src: fixture/container-apps-terraform/package.json - dest: package.json - - src: fixture/container-apps-terraform/Dockerfile - dest: Dockerfile - - src: fixture/container-apps-terraform/azure.yaml - dest: azure.yaml - - src: fixture/container-apps-terraform/infra/main.tf - dest: infra/main.tf - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - type: file-matches - config: - path: "infra/main.tf" - pattern: "(?i)azurerm_container_app[^_]" - - type: file-matches - config: - path: "infra/main.tf" - pattern: "(?i)azurerm_container_app_environment" - - type: file-matches - config: - path: "infra/main.tf" - pattern: "(?i)azurerm_container_registry" - - type: file-matches - config: - path: "infra/main.tf" - pattern: "(?i)SystemAssigned" - - type: file-matches - config: - path: "infra/main.tf" - pattern: "(?i)AcrPull" - - # Jest: "creates simple containerized Node.js app with Terraform" - - name: "Deploy Container Apps - Node.js App (Terraform)" - prompt: "My containerized Node.js application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." - environment: - files: - - src: fixture/container-apps-terraform/app.js - dest: app.js - - src: fixture/container-apps-terraform/package.json - dest: package.json - - src: fixture/container-apps-terraform/Dockerfile - dest: Dockerfile - - src: fixture/container-apps-terraform/azure.yaml - dest: azure.yaml - - src: fixture/container-apps-terraform/infra/main.tf - dest: infra/main.tf - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - type: file-matches - config: - path: "infra/main.tf" - pattern: "(?i)azurerm_container_app[^_]" - - type: file-matches - config: - path: "infra/main.tf" - pattern: "(?i)AcrPull" - - # Jest: "creates social media application with Terraform infrastructure" - - name: "Deploy Container Apps - Social Media App (Terraform)" - prompt: "My containerized social media application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." - environment: - files: - - src: fixture/container-apps-terraform/app.js - dest: app.js - - src: fixture/container-apps-terraform/package.json - dest: package.json - - src: fixture/container-apps-terraform/Dockerfile - dest: Dockerfile - - src: fixture/container-apps-terraform/azure.yaml - dest: azure.yaml - - src: fixture/container-apps-terraform/infra/main.tf - dest: infra/main.tf - constraints: - max_turns: 50 - tags: - type: integration - tier: full - cost: llm - area: behavior - skill: azure-deploy - followUp: - - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' - graders: - - type: skill-invocation - config: - required: - - azure-deploy - - type: tool-calls - config: - required: - - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|provision)\\b" - - type: file-matches - config: - path: "infra/main.tf" - pattern: "(?i)azurerm_container_app[^_]" - - type: file-matches - config: - path: "infra/main.tf" - pattern: "(?i)AcrPull" diff --git a/evals/azure-deploy/eval.yaml b/evals/azure-deploy/eval.yaml deleted file mode 100644 index 5e2b97dae..000000000 --- a/evals/azure-deploy/eval.yaml +++ /dev/null @@ -1,112 +0,0 @@ -# Vally eval config — migrated from Waza -# Source: tests/azure-deploy/eval/eval.yaml + tasks/*.yaml -# Migration: Waza → Vally (issue #1817, Phase 1) -# -# Global graders (evaluate#125 workaround): has_output and no_runtime_failure -# are duplicated into every stimulus block below. - -name: azure-deploy-eval -description: | - Evaluation suite for the azure-deploy skill. - Tests deployment guidance quality with emphasis on: - - AVM+AZD pattern-module preference - - AVM fallback when no pattern module exists - - deploy-only routing (not prepare/validate) - -tags: - type: integration - skill: azure-deploy - -defaults: - runs: 3 - timeout: "7m" - executor: integration-test-agent-runner - model: claude-sonnet-4.6 - -scoring: - threshold: 0.8 - weights: - output-contains: 0.125 - output-matches: 0.125 - output-not-matches: 0.5 - completed: 0.25 - -stimuli: - # ── avm-order-bicep-001 ── - # Waza source: tasks/avm-order-bicep.yaml - # Validates deploy guidance prefers AVM+AZD pattern modules first - - name: "AVM+AZD Priority - Bicep Deploy" - prompt: | - My app is already prepared and validated. - Give me deploy guidance and module preference order for Bicep. - Prefer AVM+AZD patterns where available, with fallback to AVM resource modules and AVM utility modules. - tags: - type: integration - skill: azure-deploy - tier: full - cost: llm - area: output - graders: - # Task: expected.output_contains - - type: output-contains - config: - substring: "AVM" - - type: output-contains - config: - substring: "deploy" - - type: output-contains - config: - substring: "pattern" - # Task grader: avm_pattern_first - - type: output-matches - config: - pattern: "(?i)AVM\\+AZD|AZD pattern|pattern modules" - # Task grader: includes_resource_and_utility_fallback - - type: output-matches - config: - pattern: "(?is)(AVM\\+AZD|AZD pattern|pattern modules).*resource modules.*utility modules" - # Global: has_output - - type: completed - # Global: no_runtime_failure - - type: output-not-matches - config: - pattern: "(?i)fatal error|exception occurred|crashed" - - # ── avm-fallback-no-pattern-001 ── - # Waza source: tasks/avm-fallback-no-pattern.yaml - # Validates fallback order when no AVM+AZD pattern module exists - - name: "AVM Fallback When No AZD Pattern" - prompt: | - I'm deploying with Bicep and there is no AVM+AZD pattern module for my scenario. - What module order should I follow if no pattern module exists and fallback must stay AVM resource modules then AVM utility modules? - tags: - type: integration - skill: azure-deploy - tier: full - cost: llm - area: output - graders: - # Task: expected.output_contains - - type: output-contains - config: - substring: "AVM" - - type: output-contains - config: - substring: "resource" - - type: output-contains - config: - substring: "utility" - # Task grader: explicit_no_pattern_fallback - - type: output-matches - config: - pattern: "(?is)(no .*pattern module|if no .*pattern).*AVM.*resource.*AVM.*utility" - # Task grader: avoids_non_avm_fallback - - type: output-not-matches - config: - pattern: "(?i)fallback to non-AVM|use non-AVM modules" - # Global: has_output - - type: completed - # Global: no_runtime_failure - - type: output-not-matches - config: - pattern: "(?i)fatal error|exception occurred|crashed" \ No newline at end of file diff --git a/evals/azure-deploy/routing-eval.yaml b/evals/azure-deploy/routing-eval.yaml index 02f511ff1..b60294125 100644 --- a/evals/azure-deploy/routing-eval.yaml +++ b/evals/azure-deploy/routing-eval.yaml @@ -74,7 +74,7 @@ stimuli: dest: infra/main.parameters.json tags: type: integration - tier: full + tier: smoke cost: llm area: routing skill: azure-deploy @@ -105,7 +105,7 @@ stimuli: dest: infra/main.parameters.json tags: type: integration - tier: full + tier: smoke cost: llm area: routing skill: azure-deploy @@ -138,7 +138,7 @@ stimuli: dest: infra/main.parameters.json tags: type: integration - tier: full + tier: smoke cost: llm area: routing skill: azure-deploy From efa6446f238b5f5e70bdf0f277c37b4c45a16b36 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Tue, 14 Jul 2026 15:26:22 -0700 Subject: [PATCH 05/21] eval: add azd deployment tool hint to all azure-deploy eval prompts Add 'Use azd as the deployment tool.' to all 40 stimuli across: - deploy-eval.yaml (33 prompts) - routing-eval.yaml (4 prompts) - output-eval.yaml (3 prompts) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 01827688-80a4-46ef-90c2-15e7de402bc4 --- evals/azure-deploy/deploy-eval.yaml | 53 +++-- evals/azure-deploy/output-eval.yaml | 307 ++++++++++++++------------- evals/azure-deploy/routing-eval.yaml | 8 +- 3 files changed, 192 insertions(+), 176 deletions(-) diff --git a/evals/azure-deploy/deploy-eval.yaml b/evals/azure-deploy/deploy-eval.yaml index 4dab59fa5..5e5e197db 100644 --- a/evals/azure-deploy/deploy-eval.yaml +++ b/evals/azure-deploy/deploy-eval.yaml @@ -48,7 +48,7 @@ stimuli: # Original problem: empty workspace, agent had to create app + deploy. # Fix: provide a prepared SWA fixture; agent only needs to deploy. - name: "Deploy SWA - Whiteboard App (Bicep)" - prompt: "My static whiteboard web app is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + prompt: "My static whiteboard web app is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." environment: files: - src: fixture/swa-bicep/index.html @@ -85,7 +85,7 @@ stimuli: # Jest: "creates static portfolio website with bicep" - name: "Deploy SWA - Portfolio Website (Bicep)" - prompt: "My static portfolio website is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + prompt: "My static portfolio website is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." environment: files: - src: fixture/swa-bicep/index.html @@ -127,7 +127,7 @@ stimuli: # Jest: "creates discussion board" - name: "Deploy App Service - Discussion Board (Bicep)" - prompt: "My Node.js web application (a discussion board) is prepared and ready. Deploy it to Azure App Service using my current subscription in the westus2 region." + prompt: "My Node.js web application (a discussion board) is prepared and ready. Deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." environment: files: - src: fixture/app-service-bicep/app.js @@ -166,7 +166,7 @@ stimuli: # Jest: "creates todo list with frontend and API" - name: "Deploy App Service - Todo List (Bicep)" - prompt: "My Node.js todo list application is prepared and ready. Deploy it to Azure App Service using my current subscription in the westus2 region." + prompt: "My Node.js todo list application is prepared and ready. Deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." environment: files: - src: fixture/app-service-bicep/app.js @@ -210,7 +210,7 @@ stimuli: # Jest: "creates serverless HTTP API" - name: "Deploy Azure Functions - Serverless HTTP API (Bicep)" - prompt: "My Azure Functions serverless HTTP API is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + prompt: "My Azure Functions serverless HTTP API is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." environment: files: - src: fixture/functions-bicep/host.json @@ -249,7 +249,7 @@ stimuli: # Jest: "creates event-driven function app" - name: "Deploy Azure Functions - Event-Driven (Bicep)" - prompt: "My event-driven Azure Functions app is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + prompt: "My event-driven Azure Functions app is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." environment: files: - src: fixture/functions-bicep/host.json @@ -288,7 +288,7 @@ stimuli: # Jest: "creates Python function app with Service Bus trigger" - name: "Deploy Azure Functions - Python Service Bus (Bicep)" - prompt: "My Python Azure Functions app with Service Bus trigger is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + prompt: "My Python Azure Functions app with Service Bus trigger is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." environment: files: - src: fixture/functions-bicep/host.json @@ -332,7 +332,7 @@ stimuli: # Jest: "creates and deploys workflow app with Durable Task Scheduler" - name: "Deploy DTS - Workflow App (Bicep)" - prompt: "My Durable Task Scheduler workflow application is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region." + prompt: "My Durable Task Scheduler workflow application is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." environment: files: - src: fixture/dts-bicep/host.json @@ -383,7 +383,7 @@ stimuli: # Jest: "creates containerized web application with Bicep" - name: "Deploy Container Apps - Containerized Web App (Bicep)" - prompt: "My containerized web application is prepared and ready. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." + prompt: "My containerized web application is prepared and ready. Deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." environment: files: - src: fixture/container-apps-bicep/app.js @@ -442,7 +442,7 @@ stimuli: # Jest: "creates simple containerized Node.js app" - name: "Deploy Container Apps - Node.js App (Bicep)" - prompt: "My containerized Node.js application is prepared and ready. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." + prompt: "My containerized Node.js application is prepared and ready. Deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." environment: files: - src: fixture/container-apps-bicep/app.js @@ -498,7 +498,7 @@ stimuli: # Jest: "creates whiteboard application with Terraform" - name: "Deploy SWA - Whiteboard App (Terraform)" - prompt: "My static whiteboard web app is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." + prompt: "My static whiteboard web app is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." environment: files: - src: fixture/swa-terraform/index.html @@ -537,7 +537,7 @@ stimuli: # Jest: "creates static portfolio website with Terraform infrastructure" - name: "Deploy SWA - Portfolio Website (Terraform)" - prompt: "My static portfolio website is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." + prompt: "My static portfolio website is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." environment: files: - src: fixture/swa-terraform/index.html @@ -581,7 +581,7 @@ stimuli: # Jest: "creates discussion board with Terraform" - name: "Deploy App Service - Discussion Board (Terraform)" - prompt: "My Node.js discussion board application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure App Service using my current subscription in the westus2 region." + prompt: "My Node.js discussion board application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." environment: files: - src: fixture/app-service-terraform/app.js @@ -622,7 +622,7 @@ stimuli: # Jest: "creates todo list with frontend and API using Terraform" - name: "Deploy App Service - Todo List (Terraform)" - prompt: "My Node.js todo list application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure App Service using my current subscription in the westus2 region." + prompt: "My Node.js todo list application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." environment: files: - src: fixture/app-service-terraform/app.js @@ -668,7 +668,7 @@ stimuli: # Jest: "creates serverless HTTP API with Terraform" - name: "Deploy Azure Functions - Serverless HTTP API (Terraform)" - prompt: "My Azure Functions serverless HTTP API is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." + prompt: "My Azure Functions serverless HTTP API is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." environment: files: - src: fixture/functions-terraform/host.json @@ -707,7 +707,7 @@ stimuli: # Jest: "creates event-driven function app with Terraform" - name: "Deploy Azure Functions - Event-Driven (Terraform)" - prompt: "My event-driven Azure Functions app is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." + prompt: "My event-driven Azure Functions app is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." environment: files: - src: fixture/functions-terraform/host.json @@ -746,7 +746,7 @@ stimuli: # Jest: "creates URL shortener service with Terraform infrastructure" - name: "Deploy Azure Functions - URL Shortener (Terraform)" - prompt: "My Azure Functions URL shortener service is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region." + prompt: "My Azure Functions URL shortener service is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." environment: files: - src: fixture/functions-terraform/host.json @@ -784,7 +784,7 @@ stimuli: # Jest: "creates containerized web application with Terraform" - name: "Deploy Container Apps - Containerized Web App (Terraform)" - prompt: "My containerized web application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." + prompt: "My containerized web application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." environment: files: - src: fixture/container-apps-terraform/app.js @@ -841,7 +841,7 @@ stimuli: # Jest: "creates simple containerized Node.js app with Terraform" - name: "Deploy Container Apps - Node.js App (Terraform)" - prompt: "My containerized Node.js application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." + prompt: "My containerized Node.js application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." environment: files: - src: fixture/container-apps-terraform/app.js @@ -886,7 +886,7 @@ stimuli: # Jest: "creates social media application with Terraform infrastructure" - name: "Deploy Container Apps - Social Media App (Terraform)" - prompt: "My containerized social media application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region." + prompt: "My containerized social media application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." environment: files: - src: fixture/container-apps-terraform/app.js @@ -947,6 +947,7 @@ stimuli: Use my current subscription. This is for a small scale production environment. Use standard SKUs. + Use azd as the deployment tool. environment: commands: - git clone --depth 1 https://github.com/dotnet/eShop.git . @@ -986,6 +987,7 @@ stimuli: This is for a small scale production environment. Use standard SKUs. The app can be found under aspnetcore/tutorials/first-mvc-app/start-mvc/sample/10.0-completed. + Use azd as the deployment tool. environment: commands: - git init -q @@ -1030,6 +1032,7 @@ stimuli: This is for a small scale production environment. Use standard SKUs. The app can be found under samples/aspire-with-azure-functions. + Use azd as the deployment tool. environment: commands: - git init -q @@ -1074,6 +1077,7 @@ stimuli: This is for a small scale production environment. Use standard SKUs. The app can be found under samples/client-apps-integration. + Use azd as the deployment tool. environment: commands: - git init -q @@ -1119,6 +1123,7 @@ stimuli: This is for a small scale production environment. Use standard SKUs. The app can be found under samples/container-build. + Use azd as the deployment tool. environment: commands: - git init -q @@ -1182,6 +1187,7 @@ stimuli: This is for a small scale production environment. Use standard SKUs. The app can be found under samples/custom-resources. + Use azd as the deployment tool. environment: commands: - git init -q @@ -1225,6 +1231,7 @@ stimuli: This is for a small scale production environment. Use standard SKUs. The app can be found under samples/database-containers. + Use azd as the deployment tool. environment: commands: - git init -q @@ -1270,6 +1277,7 @@ stimuli: This is for a small scale production environment. Use standard SKUs. The app can be found under samples/orleans-voting. + Use azd as the deployment tool. environment: commands: - git init -q @@ -1322,6 +1330,7 @@ stimuli: Use my current subscription. This is for a small scale production environment. Use standard SKUs. + Use azd as the deployment tool. environment: commands: - git clone --depth 1 https://github.com/benc-uk/nodejs-demoapp.git . @@ -1361,6 +1370,7 @@ stimuli: This is for a small scale production environment. Use standard SKUs. The app can be found under samples/aspire-with-javascript. + Use azd as the deployment tool. environment: commands: - git init -q @@ -1405,6 +1415,7 @@ stimuli: This is for a small scale production environment. Use standard SKUs. The app can be found under samples/aspire-with-node. + Use azd as the deployment tool. environment: commands: - git init -q @@ -1453,6 +1464,7 @@ stimuli: Use my current subscription. This is for a small scale production environment. Use standard SKUs. + Use azd as the deployment tool. environment: commands: - git clone --depth 1 https://github.com/UltiRequiem/flask-calculator.git . @@ -1492,6 +1504,7 @@ stimuli: This is for a small scale production environment. Use standard SKUs. The app can be found under samples/aspire-with-python. + Use azd as the deployment tool. environment: commands: - git init -q diff --git a/evals/azure-deploy/output-eval.yaml b/evals/azure-deploy/output-eval.yaml index bbaf8b401..c553060f7 100644 --- a/evals/azure-deploy/output-eval.yaml +++ b/evals/azure-deploy/output-eval.yaml @@ -1,153 +1,156 @@ -# Vally eval config — migrated from Waza -# Source: tests/azure-deploy/eval/eval.yaml + tasks/*.yaml -# Migration: Waza → Vally (issue #1817, Phase 1) -# -# Global graders (evaluate#125 workaround): has_output and no_runtime_failure -# are duplicated into every stimulus block below. - -name: azure-deploy-eval -description: | - Evaluation suite for the azure-deploy skill. - Tests deployment guidance quality with emphasis on: - - AVM+AZD pattern-module preference - - AVM fallback when no pattern module exists - - deploy-only routing (not prepare/validate) - -tags: - type: integration - skill: azure-deploy - -defaults: - runs: 3 - timeout: "7m" - executor: integration-test-agent-runner - model: claude-sonnet-4.6 - -scoring: - threshold: 0.8 - weights: - output-contains: 0.125 - output-matches: 0.125 - output-not-matches: 0.5 - completed: 0.25 - -stimuli: - # ── avm-order-bicep-001 ── - # Waza source: tasks/avm-order-bicep.yaml - # Validates deploy guidance prefers AVM+AZD pattern modules first - - name: "AVM+AZD Priority - Bicep Deploy" - prompt: | - My app is already prepared and validated. - Give me deploy guidance and module preference order for Bicep. - Prefer AVM+AZD patterns where available, with fallback to AVM resource modules and AVM utility modules. - tags: - type: integration - skill: azure-deploy - tier: full - cost: llm - area: output - graders: - # Task: expected.output_contains - - type: output-contains - config: - substring: "AVM" - - type: output-contains - config: - substring: "deploy" - - type: output-contains - config: - substring: "pattern" - # Task grader: avm_pattern_first - - type: output-matches - config: - pattern: "(?i)AVM\\+AZD|AZD pattern|pattern modules" - # Task grader: includes_resource_and_utility_fallback - - type: output-matches - config: - pattern: "(?is)(AVM\\+AZD|AZD pattern|pattern modules).*resource modules.*utility modules" - # Global: has_output - - type: completed - # Global: no_runtime_failure - - type: output-not-matches - config: - pattern: "(?i)fatal error|exception occurred|crashed" - - # ── avm-fallback-no-pattern-001 ── - # Waza source: tasks/avm-fallback-no-pattern.yaml - # Validates fallback order when no AVM+AZD pattern module exists - - name: "AVM Fallback When No AZD Pattern" - prompt: | - I'm deploying with Bicep and there is no AVM+AZD pattern module for my scenario. - What module order should I follow if no pattern module exists and fallback must stay AVM resource modules then AVM utility modules? - tags: - type: integration - skill: azure-deploy - tier: full - cost: llm - area: output - graders: - # Task: expected.output_contains - - type: output-contains - config: - substring: "AVM" - - type: output-contains - config: - substring: "resource" - - type: output-contains - config: - substring: "utility" - # Task grader: explicit_no_pattern_fallback - - type: output-matches - config: - pattern: "(?is)(no .*pattern module|if no .*pattern).*AVM.*resource.*AVM.*utility" - # Task grader: avoids_non_avm_fallback - - type: output-not-matches - config: - pattern: "(?i)fallback to non-AVM|use non-AVM modules" - # Global: has_output - - type: completed - # Global: no_runtime_failure - - type: output-not-matches - config: - pattern: "(?i)fatal error|exception occurred|crashed" - - # ── avm-azd-pattern-preference-001 ── - # Source: tests/azure-deploy/avm/integration.test.ts - # describe("avm-azd-pattern-preference") - # Validates that AZD pattern modules are preferred over AVM resource modules - # when setting up azd infrastructure with Bicep for a container app. - - name: "AZD Pattern Module Priority - Container App" - prompt: | - Set up azd infrastructure with Bicep for a container app. - Use AVM modules and prefer AZD pattern modules over resource modules. - tags: - type: integration - skill: azure-deploy - tier: full - cost: llm - area: output - graders: - - type: output-contains - config: - substring: "AZD" - - type: output-contains - config: - substring: "pattern" - # Task grader: avm_azd_pattern_preferred — matches "AZD pattern", "AVM+AZD", or "pattern modules" - - type: output-matches - config: - pattern: "(?i)AZD\\s+pattern|AVM\\+AZD|pattern modules?" - # Task grader: context_present — AVM/Bicep/container/module present - - type: output-matches - config: - pattern: "(?i)(container|bicep|module)" - # Task grader: pattern_before_resource — pattern modules discussed before resource modules - - type: output-matches - config: - pattern: "(?is)(AZD pattern|AVM\\+AZD|pattern modules?).*resource modules?" - # Global: has_output - - type: completed - # Global: no_runtime_failure - - type: output-not-matches - config: +# Vally eval config — migrated from Waza +# Source: tests/azure-deploy/eval/eval.yaml + tasks/*.yaml +# Migration: Waza → Vally (issue #1817, Phase 1) +# +# Global graders (evaluate#125 workaround): has_output and no_runtime_failure +# are duplicated into every stimulus block below. + +name: azure-deploy-eval +description: | + Evaluation suite for the azure-deploy skill. + Tests deployment guidance quality with emphasis on: + - AVM+AZD pattern-module preference + - AVM fallback when no pattern module exists + - deploy-only routing (not prepare/validate) + +tags: + type: integration + skill: azure-deploy + +defaults: + runs: 3 + timeout: "7m" + executor: integration-test-agent-runner + model: claude-sonnet-4.6 + +scoring: + threshold: 0.8 + weights: + output-contains: 0.125 + output-matches: 0.125 + output-not-matches: 0.5 + completed: 0.25 + +stimuli: + # ── avm-order-bicep-001 ── + # Waza source: tasks/avm-order-bicep.yaml + # Validates deploy guidance prefers AVM+AZD pattern modules first + - name: "AVM+AZD Priority - Bicep Deploy" + prompt: | + My app is already prepared and validated. + Give me deploy guidance and module preference order for Bicep. + Prefer AVM+AZD patterns where available, with fallback to AVM resource modules and AVM utility modules. + Use azd as the deployment tool. + tags: + type: integration + skill: azure-deploy + tier: full + cost: llm + area: output + graders: + # Task: expected.output_contains + - type: output-contains + config: + substring: "AVM" + - type: output-contains + config: + substring: "deploy" + - type: output-contains + config: + substring: "pattern" + # Task grader: avm_pattern_first + - type: output-matches + config: + pattern: "(?i)AVM\\+AZD|AZD pattern|pattern modules" + # Task grader: includes_resource_and_utility_fallback + - type: output-matches + config: + pattern: "(?is)(AVM\\+AZD|AZD pattern|pattern modules).*resource modules.*utility modules" + # Global: has_output + - type: completed + # Global: no_runtime_failure + - type: output-not-matches + config: + pattern: "(?i)fatal error|exception occurred|crashed" + + # ── avm-fallback-no-pattern-001 ── + # Waza source: tasks/avm-fallback-no-pattern.yaml + # Validates fallback order when no AVM+AZD pattern module exists + - name: "AVM Fallback When No AZD Pattern" + prompt: | + I'm deploying with Bicep and there is no AVM+AZD pattern module for my scenario. + What module order should I follow if no pattern module exists and fallback must stay AVM resource modules then AVM utility modules? + Use azd as the deployment tool. + tags: + type: integration + skill: azure-deploy + tier: full + cost: llm + area: output + graders: + # Task: expected.output_contains + - type: output-contains + config: + substring: "AVM" + - type: output-contains + config: + substring: "resource" + - type: output-contains + config: + substring: "utility" + # Task grader: explicit_no_pattern_fallback + - type: output-matches + config: + pattern: "(?is)(no .*pattern module|if no .*pattern).*AVM.*resource.*AVM.*utility" + # Task grader: avoids_non_avm_fallback + - type: output-not-matches + config: + pattern: "(?i)fallback to non-AVM|use non-AVM modules" + # Global: has_output + - type: completed + # Global: no_runtime_failure + - type: output-not-matches + config: + pattern: "(?i)fatal error|exception occurred|crashed" + + # ── avm-azd-pattern-preference-001 ── + # Source: tests/azure-deploy/avm/integration.test.ts + # describe("avm-azd-pattern-preference") + # Validates that AZD pattern modules are preferred over AVM resource modules + # when setting up azd infrastructure with Bicep for a container app. + - name: "AZD Pattern Module Priority - Container App" + prompt: | + Set up azd infrastructure with Bicep for a container app. + Use AVM modules and prefer AZD pattern modules over resource modules. + Use azd as the deployment tool. + tags: + type: integration + skill: azure-deploy + tier: full + cost: llm + area: output + graders: + - type: output-contains + config: + substring: "AZD" + - type: output-contains + config: + substring: "pattern" + # Task grader: avm_azd_pattern_preferred — matches "AZD pattern", "AVM+AZD", or "pattern modules" + - type: output-matches + config: + pattern: "(?i)AZD\\s+pattern|AVM\\+AZD|pattern modules?" + # Task grader: context_present — AVM/Bicep/container/module present + - type: output-matches + config: + pattern: "(?i)(container|bicep|module)" + # Task grader: pattern_before_resource — pattern modules discussed before resource modules + - type: output-matches + config: + pattern: "(?is)(AZD pattern|AVM\\+AZD|pattern modules?).*resource modules?" + # Global: has_output + - type: completed + # Global: no_runtime_failure + - type: output-not-matches + config: pattern: "(?i)fatal error|exception occurred|crashed" \ No newline at end of file diff --git a/evals/azure-deploy/routing-eval.yaml b/evals/azure-deploy/routing-eval.yaml index b60294125..7014e74d6 100644 --- a/evals/azure-deploy/routing-eval.yaml +++ b/evals/azure-deploy/routing-eval.yaml @@ -32,7 +32,7 @@ stimuli: # Jest: "invokes azure-deploy skill for deployment prompt" - name: "Deployment Prompt" - prompt: "Run azd up to deploy my already-prepared app to Azure" + prompt: "Run azd up to deploy my already-prepared app to Azure. Use azd as the deployment tool." environment: files: - src: fixture/swa-bicep/azure.yaml @@ -61,7 +61,7 @@ stimuli: # Jest: "invokes azure-deploy skill for publish to Azure prompt" - name: "Publish to Azure Prompt" - prompt: "My app already has azure.yaml and infra/ configured. Publish it to Azure now." + prompt: "My app already has azure.yaml and infra/ configured. Publish it to Azure now. Use azd as the deployment tool." environment: files: - src: fixture/swa-bicep/azure.yaml @@ -90,7 +90,7 @@ stimuli: # Jest: "invokes azure-deploy skill for Azure Functions deployment prompt" - name: "Azure Functions Deployment Prompt" - prompt: "Deploy my existing Azure Functions project to the cloud. The infrastructure and azure.yaml are already set up." + prompt: "Deploy my existing Azure Functions project to the cloud. The infrastructure and azure.yaml are already set up. Use azd as the deployment tool." environment: files: - src: fixture/functions-bicep/azure.yaml @@ -121,7 +121,7 @@ stimuli: # Jest: "invokes azure-deploy skill for post-deployment role assignment check prompt" - name: "Post-Deployment Role Assignment Prompt" - prompt: "Deploy my already-prepared Azure app and confirm the managed identity roles are properly assigned" + prompt: "Deploy my already-prepared Azure app and confirm the managed identity roles are properly assigned Use azd as the deployment tool." environment: files: - src: fixture/container-apps-bicep/azure.yaml From c9f2da8a5ddf2fc23d1f4a6ee5356514556c9bfd Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Tue, 14 Jul 2026 15:31:16 -0700 Subject: [PATCH 06/21] limit to 1 output test per run --- evals/azure-deploy/output-eval.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/azure-deploy/output-eval.yaml b/evals/azure-deploy/output-eval.yaml index c553060f7..65eedba00 100644 --- a/evals/azure-deploy/output-eval.yaml +++ b/evals/azure-deploy/output-eval.yaml @@ -18,7 +18,7 @@ tags: skill: azure-deploy defaults: - runs: 3 + runs: 1 timeout: "7m" executor: integration-test-agent-runner model: claude-sonnet-4.6 From d97fc6bdb16f3b7d3e60eb4f4922d882b3b62b77 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Wed, 15 Jul 2026 14:07:36 -0700 Subject: [PATCH 07/21] eval: add category tags to azure-deploy deploy-eval stimuli Map each stimulus back to its original Jest describe group: - vanilla-static-web-apps-deploy (2) - vanilla-app-service-deploy (2) - vanilla-azure-functions-deploy (3) - durable-task-scheduler-deploy (1) - vanilla-azure-container-apps-deploy (2) - terraform-static-web-apps-deploy (2) - terraform-app-service-deploy (2) - terraform-azure-functions-deploy (3) - terraform-azure-container-apps-deploy (3) - brownfield-dotnet (8) - brownfield-javascript (3) - brownfield-python (2) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 01827688-80a4-46ef-90c2-15e7de402bc4 --- evals/azure-deploy/deploy-eval.yaml | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/evals/azure-deploy/deploy-eval.yaml b/evals/azure-deploy/deploy-eval.yaml index 5e5e197db..c7865436d 100644 --- a/evals/azure-deploy/deploy-eval.yaml +++ b/evals/azure-deploy/deploy-eval.yaml @@ -69,6 +69,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: vanilla-static-web-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -106,6 +107,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: vanilla-static-web-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -150,6 +152,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: vanilla-app-service-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -189,6 +192,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: vanilla-app-service-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -233,6 +237,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: vanilla-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -272,6 +277,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: vanilla-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -311,6 +317,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: vanilla-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -353,6 +360,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: durable-task-scheduler-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -410,6 +418,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: vanilla-azure-container-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -469,6 +478,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: vanilla-azure-container-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -515,6 +525,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: terraform-static-web-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -554,6 +565,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: terraform-static-web-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -600,6 +612,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: terraform-app-service-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -641,6 +654,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: terraform-app-service-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -685,6 +699,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: terraform-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -724,6 +739,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: terraform-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -763,6 +779,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: terraform-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -805,6 +822,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: terraform-azure-container-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -862,6 +880,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: terraform-azure-container-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -907,6 +926,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: terraform-azure-container-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' @@ -959,6 +979,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." @@ -1004,6 +1025,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." @@ -1049,6 +1071,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." @@ -1094,6 +1117,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." @@ -1140,6 +1164,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." @@ -1204,6 +1229,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Stop if there is no further work; otherwise go with recommended options." @@ -1248,6 +1274,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." @@ -1294,6 +1321,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: brownfield-dotnet systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." @@ -1342,6 +1370,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: brownfield-javascript systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." @@ -1387,6 +1416,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: brownfield-javascript systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." @@ -1432,6 +1462,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: brownfield-javascript systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." @@ -1476,6 +1507,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: brownfield-python systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." @@ -1521,6 +1553,7 @@ stimuli: cost: llm area: behavior skill: azure-deploy + category: brownfield-python systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." From 46d70282296ac1afcc1c6c668c5316a127d36ac1 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Wed, 15 Jul 2026 14:24:18 -0700 Subject: [PATCH 08/21] eval: switch earlyTerminate to tool-call-result in azure-deploy deploy-eval 32 stimuli had both a tool-calls grader checking for azd up/provision and an earlyTerminate using tool-call-match on the same command. tool-call-match fires when the call starts (before execution), so the grader may not see a completed call in the trajectory. Switching to tool-call-result ensures early termination happens after the tool call completes, allowing the tool-calls grader to evaluate the result. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 01827688-80a4-46ef-90c2-15e7de402bc4 --- evals/azure-deploy/deploy-eval.yaml | 66 ++++++++++++++--------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/evals/azure-deploy/deploy-eval.yaml b/evals/azure-deploy/deploy-eval.yaml index c7865436d..9dfac603b 100644 --- a/evals/azure-deploy/deploy-eval.yaml +++ b/evals/azure-deploy/deploy-eval.yaml @@ -11,7 +11,7 @@ # azure-deploy responsibilities. Each stimulus below instead provides a # pre-built fixture (azure.yaml + infra/) so the agent only needs to deploy. # -# earlyTerminate fires when azd up/provision starts (success path). +# earlyTerminate fires when azd up/provision completes (tool-call-result, success path). # NOTE: earlyTerminate tests must NOT use the `completed` grader. name: azure-deploy-e2e-eval @@ -72,7 +72,7 @@ stimuli: category: vanilla-static-web-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -110,7 +110,7 @@ stimuli: category: vanilla-static-web-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -155,7 +155,7 @@ stimuli: category: vanilla-app-service-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -195,7 +195,7 @@ stimuli: category: vanilla-app-service-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -240,7 +240,7 @@ stimuli: category: vanilla-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -280,7 +280,7 @@ stimuli: category: vanilla-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -320,7 +320,7 @@ stimuli: category: vanilla-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -363,7 +363,7 @@ stimuli: category: durable-task-scheduler-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -421,7 +421,7 @@ stimuli: category: vanilla-azure-container-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -481,7 +481,7 @@ stimuli: category: vanilla-azure-container-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -528,7 +528,7 @@ stimuli: category: terraform-static-web-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -568,7 +568,7 @@ stimuli: category: terraform-static-web-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -615,7 +615,7 @@ stimuli: category: terraform-app-service-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -657,7 +657,7 @@ stimuli: category: terraform-app-service-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -702,7 +702,7 @@ stimuli: category: terraform-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -742,7 +742,7 @@ stimuli: category: terraform-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -782,7 +782,7 @@ stimuli: category: terraform-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -825,7 +825,7 @@ stimuli: category: terraform-azure-container-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -883,7 +883,7 @@ stimuli: category: terraform-azure-container-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -929,7 +929,7 @@ stimuli: category: terraform-azure-container-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -983,7 +983,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -1029,7 +1029,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -1075,7 +1075,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -1121,7 +1121,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -1168,7 +1168,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -1278,7 +1278,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -1325,7 +1325,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -1374,7 +1374,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -1420,7 +1420,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -1466,7 +1466,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -1511,7 +1511,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: @@ -1557,7 +1557,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' graders: - type: skill-invocation config: From 7d14b8d700720c918802c3a37b65044a725c1cfa Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Wed, 15 Jul 2026 14:27:53 -0700 Subject: [PATCH 09/21] eval: change earlyTerminate to detect deploy URL in assistant output Replace azd up/provision tool-call-result earlyTerminate with assistant-message-match on Azure deployment link patterns: - *.azurewebsites.net (App Service) - *.azurestaticapps.net (Static Web Apps) - *.azurecontainerapps.io (Container Apps) - *.web.core.windows.net (Storage static website) This matches the original shouldEarlyTerminateForCompletedDeployment logic in tests/azure-deploy/utils.ts. Tests now run through actual deployment and terminate when the agent surfaces a live app URL. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 01827688-80a4-46ef-90c2-15e7de402bc4 --- evals/azure-deploy/deploy-eval.yaml | 68 +++++++++++++++-------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/evals/azure-deploy/deploy-eval.yaml b/evals/azure-deploy/deploy-eval.yaml index 9dfac603b..43e823ea3 100644 --- a/evals/azure-deploy/deploy-eval.yaml +++ b/evals/azure-deploy/deploy-eval.yaml @@ -11,7 +11,9 @@ # azure-deploy responsibilities. Each stimulus below instead provides a # pre-built fixture (azure.yaml + infra/) so the agent only needs to deploy. # -# earlyTerminate fires when azd up/provision completes (tool-call-result, success path). +# earlyTerminate fires when the agent outputs a deploy link (azurewebsites.net, +# azurestaticapps.net, azurecontainerapps.io, web.core.windows.net) — indicating +# the deployment completed and the app is live. # NOTE: earlyTerminate tests must NOT use the `completed` grader. name: azure-deploy-e2e-eval @@ -72,7 +74,7 @@ stimuli: category: vanilla-static-web-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -110,7 +112,7 @@ stimuli: category: vanilla-static-web-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -155,7 +157,7 @@ stimuli: category: vanilla-app-service-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -195,7 +197,7 @@ stimuli: category: vanilla-app-service-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -240,7 +242,7 @@ stimuli: category: vanilla-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -280,7 +282,7 @@ stimuli: category: vanilla-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -320,7 +322,7 @@ stimuli: category: vanilla-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -363,7 +365,7 @@ stimuli: category: durable-task-scheduler-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -421,7 +423,7 @@ stimuli: category: vanilla-azure-container-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -481,7 +483,7 @@ stimuli: category: vanilla-azure-container-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -528,7 +530,7 @@ stimuli: category: terraform-static-web-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -568,7 +570,7 @@ stimuli: category: terraform-static-web-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -615,7 +617,7 @@ stimuli: category: terraform-app-service-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -657,7 +659,7 @@ stimuli: category: terraform-app-service-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -702,7 +704,7 @@ stimuli: category: terraform-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -742,7 +744,7 @@ stimuli: category: terraform-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -782,7 +784,7 @@ stimuli: category: terraform-azure-functions-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -825,7 +827,7 @@ stimuli: category: terraform-azure-container-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -883,7 +885,7 @@ stimuli: category: terraform-azure-container-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -929,7 +931,7 @@ stimuli: category: terraform-azure-container-apps-deploy followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -983,7 +985,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1029,7 +1031,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1075,7 +1077,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1121,7 +1123,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1168,7 +1170,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1278,7 +1280,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1325,7 +1327,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1374,7 +1376,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1420,7 +1422,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1466,7 +1468,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1511,7 +1513,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1557,7 +1559,7 @@ stimuli: systemPrompt: '{"mode":"append","content":"Use pseudo random name resource group name such that it is less likely to have collision with existing ones."}' followUp: - "Go with recommended options and proceed with Azure deployment." - earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"azd\\s+(up|provision)\\b"}]' + earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: From 9518bb74819fd575cbd95adc6f2c30b46e35729b Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Wed, 15 Jul 2026 14:31:59 -0700 Subject: [PATCH 10/21] eval: add takeScreenshot tag to azure-deploy deploy-eval stimuli Add has-deployment-url takeScreenshot condition to the 17 stimuli that had takeScreenshot configured in the original Jest tests. Screenshot triggers when the agent outputs any Azure deployment link: - *.azurewebsites.net (App Service) - *.azurestaticapps.net (Static Web Apps) - *.azurecontainerapps.io (Container Apps) - *.web.core.windows.net (Storage static website) Stimuli with takeScreenshot (17): - SWA Bicep (2), App Service Bicep (2) - SWA Terraform (2), App Service Terraform (2) - Brownfield .NET: eShop, MvcMovie 10, Aspire Azure Functions, Aspire Client Apps Integration - Brownfield JavaScript: nodejs-demoapp, Aspire JS, Aspire Node - Brownfield Python: Flask Calculator, Aspire Python Stimuli without takeScreenshot (16): - Azure Functions (Bicep/Terraform), DTS, Container Apps (Bicep/Terraform), Brownfield .NET: Container Build, Custom Resources, DB Containers, Orleans Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 01827688-80a4-46ef-90c2-15e7de402bc4 --- evals/azure-deploy/deploy-eval.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/evals/azure-deploy/deploy-eval.yaml b/evals/azure-deploy/deploy-eval.yaml index 43e823ea3..d1a046146 100644 --- a/evals/azure-deploy/deploy-eval.yaml +++ b/evals/azure-deploy/deploy-eval.yaml @@ -75,6 +75,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -113,6 +114,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -158,6 +160,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -198,6 +201,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -531,6 +535,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -571,6 +576,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -618,6 +624,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -660,6 +667,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -986,6 +994,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1032,6 +1041,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1078,6 +1088,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1124,6 +1135,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1377,6 +1389,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1423,6 +1436,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1469,6 +1483,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1514,6 +1529,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: @@ -1560,6 +1576,7 @@ stimuli: followUp: - "Go with recommended options and proceed with Azure deployment." earlyTerminate: '[{"type":"assistant-message-match","contentPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' + takeScreenshot: '[{"type":"has-deployment-url","urlPattern":"https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)"}]' graders: - type: skill-invocation config: From a6d0dac5b1f0952fb189a4e5a375a63c2f10e7ab Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Wed, 15 Jul 2026 14:34:23 -0700 Subject: [PATCH 11/21] update CI test command --- .github/workflows/test-azure-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-azure-deploy.yml b/.github/workflows/test-azure-deploy.yml index 6ee83d3f6..6b204b497 100644 --- a/.github/workflows/test-azure-deploy.yml +++ b/.github/workflows/test-azure-deploy.yml @@ -188,7 +188,7 @@ jobs: TEST_GROUP: ${{ matrix.test-group }} run: | echo test with $MODEL_OVERRIDE - npm run test:integration -- azure-deploy "$TEST_GROUP" + npm run test:vally -- --skill azure-deploy --tag "category=$TEST_GROUP" continue-on-error: true - name: Generate report From ba6d31bb269afda24c3bdc2b94c061fa70183fb2 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Wed, 15 Jul 2026 14:48:37 -0700 Subject: [PATCH 12/21] eval: add application code to DTS fixture for order processing workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dts-bicep fixture was missing Node.js app code. The azure.yaml declares language:js host:function, so azd expects deployable source. Added (Azure Functions v4 model): - package.json — @azure/functions v4, @azure/durabletask deps - src/functions/orderOrchestrator.js — orchestrator coordinating 3 sequential steps: validateOrder → processPayment → sendConfirmation - src/functions/orderActivities.js — the three activity functions - src/functions/startOrder.js — HTTP POST trigger to start the pipeline Updated deploy-eval.yaml to include all new files in the stimulus environment.files list. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 01827688-80a4-46ef-90c2-15e7de402bc4 --- evals/azure-deploy/deploy-eval.yaml | 8 +++++ .../fixture/dts-bicep/package.json | 20 +++++++++++++ .../src/functions/orderActivities.js | 29 ++++++++++++++++++ .../src/functions/orderOrchestrator.js | 30 +++++++++++++++++++ .../dts-bicep/src/functions/startOrder.js | 27 +++++++++++++++++ 5 files changed, 114 insertions(+) create mode 100644 evals/azure-deploy/fixture/dts-bicep/package.json create mode 100644 evals/azure-deploy/fixture/dts-bicep/src/functions/orderActivities.js create mode 100644 evals/azure-deploy/fixture/dts-bicep/src/functions/orderOrchestrator.js create mode 100644 evals/azure-deploy/fixture/dts-bicep/src/functions/startOrder.js diff --git a/evals/azure-deploy/deploy-eval.yaml b/evals/azure-deploy/deploy-eval.yaml index d1a046146..a2c26df39 100644 --- a/evals/azure-deploy/deploy-eval.yaml +++ b/evals/azure-deploy/deploy-eval.yaml @@ -352,6 +352,14 @@ stimuli: dest: host.json - src: fixture/dts-bicep/azure.yaml dest: azure.yaml + - src: fixture/dts-bicep/package.json + dest: package.json + - src: fixture/dts-bicep/src/functions/orderOrchestrator.js + dest: src/functions/orderOrchestrator.js + - src: fixture/dts-bicep/src/functions/orderActivities.js + dest: src/functions/orderActivities.js + - src: fixture/dts-bicep/src/functions/startOrder.js + dest: src/functions/startOrder.js - src: fixture/dts-bicep/infra/main.bicep dest: infra/main.bicep - src: fixture/dts-bicep/infra/resources.bicep diff --git a/evals/azure-deploy/fixture/dts-bicep/package.json b/evals/azure-deploy/fixture/dts-bicep/package.json new file mode 100644 index 000000000..77b2f7bdc --- /dev/null +++ b/evals/azure-deploy/fixture/dts-bicep/package.json @@ -0,0 +1,20 @@ +{ + "name": "order-processing-workflow", + "version": "1.0.0", + "description": "Multi-step order processing pipeline using Azure Durable Task Scheduler", + "main": "src/index.js", + "scripts": { + "start": "func start", + "build": "echo 'No build step required'" + }, + "dependencies": { + "@azure/durabletask": "^0.1.0", + "@azure/functions": "^4.0.0" + }, + "devDependencies": { + "azure-functions-core-tools": "^4.x" + }, + "engines": { + "node": ">=18.0.0" + } +} diff --git a/evals/azure-deploy/fixture/dts-bicep/src/functions/orderActivities.js b/evals/azure-deploy/fixture/dts-bicep/src/functions/orderActivities.js new file mode 100644 index 000000000..1c6ce69db --- /dev/null +++ b/evals/azure-deploy/fixture/dts-bicep/src/functions/orderActivities.js @@ -0,0 +1,29 @@ +const { app } = require("@azure/functions"); + +app.generic("validateOrder", { + trigger: app.trigger.generic({ type: "activityTrigger" }), + handler: async (order) => { + if (!order.orderId || !order.amount || order.amount <= 0) { + return { success: false, reason: "Invalid order: missing orderId or invalid amount" }; + } + if (!order.customerEmail) { + return { success: false, reason: "Invalid order: missing customer email" }; + } + return { success: true }; + }, +}); + +app.generic("processPayment", { + trigger: app.trigger.generic({ type: "activityTrigger" }), + handler: async ({ orderId, amount }) => { + const paymentId = `pay-${orderId}-${Date.now()}`; + return { paymentId, orderId, amount, processedAt: new Date().toISOString() }; + }, +}); + +app.generic("sendConfirmation", { + trigger: app.trigger.generic({ type: "activityTrigger" }), + handler: async ({ orderId, email, paymentId }) => { + return { sent: true, orderId, email, paymentId }; + }, +}); diff --git a/evals/azure-deploy/fixture/dts-bicep/src/functions/orderOrchestrator.js b/evals/azure-deploy/fixture/dts-bicep/src/functions/orderOrchestrator.js new file mode 100644 index 000000000..2af10a39b --- /dev/null +++ b/evals/azure-deploy/fixture/dts-bicep/src/functions/orderOrchestrator.js @@ -0,0 +1,30 @@ +const { app } = require("@azure/functions"); + +app.generic("orderProcessingOrchestrator", { + trigger: app.trigger.generic({ type: "orchestrationTrigger" }), + handler: async (ctx) => { + const order = ctx.getInput(); + + const validationResult = await ctx.callActivity("validateOrder", order); + if (!validationResult.success) { + return { status: "rejected", reason: validationResult.reason }; + } + + const paymentResult = await ctx.callActivity("processPayment", { + orderId: order.orderId, + amount: order.amount, + }); + + await ctx.callActivity("sendConfirmation", { + orderId: order.orderId, + email: order.customerEmail, + paymentId: paymentResult.paymentId, + }); + + return { + status: "completed", + orderId: order.orderId, + paymentId: paymentResult.paymentId, + }; + }, +}); diff --git a/evals/azure-deploy/fixture/dts-bicep/src/functions/startOrder.js b/evals/azure-deploy/fixture/dts-bicep/src/functions/startOrder.js new file mode 100644 index 000000000..317c65c8e --- /dev/null +++ b/evals/azure-deploy/fixture/dts-bicep/src/functions/startOrder.js @@ -0,0 +1,27 @@ +const { app } = require("@azure/functions"); + +app.http("startOrder", { + methods: ["POST"], + authLevel: "function", + handler: async (request, context) => { + const order = await request.json(); + if (!order.orderId || !order.amount || !order.customerEmail) { + return { + status: 400, + jsonBody: { error: "Request must include orderId, amount, and customerEmail" }, + }; + } + + // In production, use DurableTaskSchedulerClient with the connection string. + // For local dev, use DurableClient from @azure/durable-functions. + context.log(`Starting order processing for orderId: ${order.orderId}`); + + return { + status: 202, + jsonBody: { + orderId: order.orderId, + message: "Order processing started", + }, + }; + }, +}); From 8bcc661a8525ab4484808e45584428d9d07fabbf Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Wed, 15 Jul 2026 15:39:21 -0700 Subject: [PATCH 13/21] eval: rename whiteboard to test-sample-app in SWA fixture HTML Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 01827688-80a4-46ef-90c2-15e7de402bc4 --- evals/azure-deploy/fixture/swa-bicep/index.html | 4 ++-- evals/azure-deploy/fixture/swa-terraform/index.html | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/evals/azure-deploy/fixture/swa-bicep/index.html b/evals/azure-deploy/fixture/swa-bicep/index.html index 6b3f998a0..1f0bc04a7 100644 --- a/evals/azure-deploy/fixture/swa-bicep/index.html +++ b/evals/azure-deploy/fixture/swa-bicep/index.html @@ -1,8 +1,8 @@ -Whiteboard App +test-sample-app -

Whiteboard

+

test-sample-app

diff --git a/evals/azure-deploy/fixture/swa-terraform/index.html b/evals/azure-deploy/fixture/swa-terraform/index.html index 6b3f998a0..1f0bc04a7 100644 --- a/evals/azure-deploy/fixture/swa-terraform/index.html +++ b/evals/azure-deploy/fixture/swa-terraform/index.html @@ -1,8 +1,8 @@ -Whiteboard App +test-sample-app -

Whiteboard

+

test-sample-app

From 1dcf20ff010bc0c961e00795dd83311de0b46c67 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Tue, 21 Jul 2026 09:59:01 -0700 Subject: [PATCH 14/21] restore test prompts --- evals/azure-deploy/deploy-eval.yaml | 312 +++--------------- .../fixture/app-service-bicep/app.js | 9 - .../fixture/app-service-bicep/azure.yaml | 10 - .../app-service-bicep/infra/main.bicep | 34 -- .../infra/main.parameters.json | 12 - .../app-service-bicep/infra/resources.bicep | 35 -- .../fixture/app-service-bicep/package.json | 9 - .../fixture/app-service-terraform/app.js | 7 - .../fixture/app-service-terraform/azure.yaml | 10 - .../app-service-terraform/infra/main.tf | 74 ----- .../app-service-terraform/package.json | 5 - .../fixture/container-apps-bicep/Dockerfile | 7 - .../fixture/container-apps-bicep/app.js | 9 - .../fixture/container-apps-bicep/azure.yaml | 11 - .../infra/abbreviations.json | 6 - .../container-apps-bicep/infra/main.bicep | 37 --- .../infra/main.parameters.json | 12 - .../infra/resources.bicep | 71 ---- .../fixture/container-apps-bicep/package.json | 5 - .../container-apps-terraform/Dockerfile | 7 - .../fixture/container-apps-terraform/app.js | 7 - .../container-apps-terraform/azure.yaml | 11 - .../container-apps-terraform/infra/main.tf | 125 ------- .../container-apps-terraform/package.json | 5 - .../azure-deploy/fixture/dts-bicep/azure.yaml | 10 - .../azure-deploy/fixture/dts-bicep/host.json | 3 - .../fixture/dts-bicep/infra/main.bicep | 34 -- .../dts-bicep/infra/main.parameters.json | 12 - .../fixture/dts-bicep/infra/resources.bicep | 100 ------ .../fixture/dts-bicep/package.json | 20 -- .../src/functions/orderActivities.js | 29 -- .../src/functions/orderOrchestrator.js | 30 -- .../dts-bicep/src/functions/startOrder.js | 27 -- .../fixture/functions-bicep/azure.yaml | 10 - .../fixture/functions-bicep/host.json | 3 - .../fixture/functions-bicep/infra/main.bicep | 34 -- .../infra/main.parameters.json | 12 - .../functions-bicep/infra/resources.bicep | 58 ---- .../fixture/functions-bicep/package.json | 7 - .../fixture/functions-terraform/azure.yaml | 10 - .../fixture/functions-terraform/host.json | 3 - .../fixture/functions-terraform/infra/main.tf | 88 ----- .../azure-deploy/fixture/swa-bicep/azure.yaml | 9 - .../azure-deploy/fixture/swa-bicep/index.html | 8 - .../fixture/swa-bicep/infra/main.bicep | 34 -- .../swa-bicep/infra/main.parameters.json | 12 - .../fixture/swa-bicep/infra/resources.bicep | 18 - .../fixture/swa-terraform/azure.yaml | 9 - .../fixture/swa-terraform/index.html | 8 - .../fixture/swa-terraform/infra/main.tf | 60 ---- 50 files changed, 43 insertions(+), 1435 deletions(-) delete mode 100644 evals/azure-deploy/fixture/app-service-bicep/app.js delete mode 100644 evals/azure-deploy/fixture/app-service-bicep/azure.yaml delete mode 100644 evals/azure-deploy/fixture/app-service-bicep/infra/main.bicep delete mode 100644 evals/azure-deploy/fixture/app-service-bicep/infra/main.parameters.json delete mode 100644 evals/azure-deploy/fixture/app-service-bicep/infra/resources.bicep delete mode 100644 evals/azure-deploy/fixture/app-service-bicep/package.json delete mode 100644 evals/azure-deploy/fixture/app-service-terraform/app.js delete mode 100644 evals/azure-deploy/fixture/app-service-terraform/azure.yaml delete mode 100644 evals/azure-deploy/fixture/app-service-terraform/infra/main.tf delete mode 100644 evals/azure-deploy/fixture/app-service-terraform/package.json delete mode 100644 evals/azure-deploy/fixture/container-apps-bicep/Dockerfile delete mode 100644 evals/azure-deploy/fixture/container-apps-bicep/app.js delete mode 100644 evals/azure-deploy/fixture/container-apps-bicep/azure.yaml delete mode 100644 evals/azure-deploy/fixture/container-apps-bicep/infra/abbreviations.json delete mode 100644 evals/azure-deploy/fixture/container-apps-bicep/infra/main.bicep delete mode 100644 evals/azure-deploy/fixture/container-apps-bicep/infra/main.parameters.json delete mode 100644 evals/azure-deploy/fixture/container-apps-bicep/infra/resources.bicep delete mode 100644 evals/azure-deploy/fixture/container-apps-bicep/package.json delete mode 100644 evals/azure-deploy/fixture/container-apps-terraform/Dockerfile delete mode 100644 evals/azure-deploy/fixture/container-apps-terraform/app.js delete mode 100644 evals/azure-deploy/fixture/container-apps-terraform/azure.yaml delete mode 100644 evals/azure-deploy/fixture/container-apps-terraform/infra/main.tf delete mode 100644 evals/azure-deploy/fixture/container-apps-terraform/package.json delete mode 100644 evals/azure-deploy/fixture/dts-bicep/azure.yaml delete mode 100644 evals/azure-deploy/fixture/dts-bicep/host.json delete mode 100644 evals/azure-deploy/fixture/dts-bicep/infra/main.bicep delete mode 100644 evals/azure-deploy/fixture/dts-bicep/infra/main.parameters.json delete mode 100644 evals/azure-deploy/fixture/dts-bicep/infra/resources.bicep delete mode 100644 evals/azure-deploy/fixture/dts-bicep/package.json delete mode 100644 evals/azure-deploy/fixture/dts-bicep/src/functions/orderActivities.js delete mode 100644 evals/azure-deploy/fixture/dts-bicep/src/functions/orderOrchestrator.js delete mode 100644 evals/azure-deploy/fixture/dts-bicep/src/functions/startOrder.js delete mode 100644 evals/azure-deploy/fixture/functions-bicep/azure.yaml delete mode 100644 evals/azure-deploy/fixture/functions-bicep/host.json delete mode 100644 evals/azure-deploy/fixture/functions-bicep/infra/main.bicep delete mode 100644 evals/azure-deploy/fixture/functions-bicep/infra/main.parameters.json delete mode 100644 evals/azure-deploy/fixture/functions-bicep/infra/resources.bicep delete mode 100644 evals/azure-deploy/fixture/functions-bicep/package.json delete mode 100644 evals/azure-deploy/fixture/functions-terraform/azure.yaml delete mode 100644 evals/azure-deploy/fixture/functions-terraform/host.json delete mode 100644 evals/azure-deploy/fixture/functions-terraform/infra/main.tf delete mode 100644 evals/azure-deploy/fixture/swa-bicep/azure.yaml delete mode 100644 evals/azure-deploy/fixture/swa-bicep/index.html delete mode 100644 evals/azure-deploy/fixture/swa-bicep/infra/main.bicep delete mode 100644 evals/azure-deploy/fixture/swa-bicep/infra/main.parameters.json delete mode 100644 evals/azure-deploy/fixture/swa-bicep/infra/resources.bicep delete mode 100644 evals/azure-deploy/fixture/swa-terraform/azure.yaml delete mode 100644 evals/azure-deploy/fixture/swa-terraform/index.html delete mode 100644 evals/azure-deploy/fixture/swa-terraform/infra/main.tf diff --git a/evals/azure-deploy/deploy-eval.yaml b/evals/azure-deploy/deploy-eval.yaml index a2c26df39..3b317c9a4 100644 --- a/evals/azure-deploy/deploy-eval.yaml +++ b/evals/azure-deploy/deploy-eval.yaml @@ -50,19 +50,7 @@ stimuli: # Original problem: empty workspace, agent had to create app + deploy. # Fix: provide a prepared SWA fixture; agent only needs to deploy. - name: "Deploy SWA - Whiteboard App (Bicep)" - prompt: "My static whiteboard web app is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." - environment: - files: - - src: fixture/swa-bicep/index.html - dest: index.html - - src: fixture/swa-bicep/azure.yaml - dest: azure.yaml - - src: fixture/swa-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/swa-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/swa-bicep/infra/main.parameters.json - dest: infra/main.parameters.json + prompt: "Create a static whiteboard web app and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -80,6 +68,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -89,19 +78,7 @@ stimuli: # Jest: "creates static portfolio website with bicep" - name: "Deploy SWA - Portfolio Website (Bicep)" - prompt: "My static portfolio website is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." - environment: - files: - - src: fixture/swa-bicep/index.html - dest: index.html - - src: fixture/swa-bicep/azure.yaml - dest: azure.yaml - - src: fixture/swa-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/swa-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/swa-bicep/infra/main.parameters.json - dest: infra/main.parameters.json + prompt: "Create a static portfolio website and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -119,6 +96,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -133,21 +111,7 @@ stimuli: # Jest: "creates discussion board" - name: "Deploy App Service - Discussion Board (Bicep)" - prompt: "My Node.js web application (a discussion board) is prepared and ready. Deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." - environment: - files: - - src: fixture/app-service-bicep/app.js - dest: app.js - - src: fixture/app-service-bicep/package.json - dest: package.json - - src: fixture/app-service-bicep/azure.yaml - dest: azure.yaml - - src: fixture/app-service-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/app-service-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/app-service-bicep/infra/main.parameters.json - dest: infra/main.parameters.json + prompt: "Create a discussion board application and deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -165,6 +129,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -174,21 +139,7 @@ stimuli: # Jest: "creates todo list with frontend and API" - name: "Deploy App Service - Todo List (Bicep)" - prompt: "My Node.js todo list application is prepared and ready. Deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." - environment: - files: - - src: fixture/app-service-bicep/app.js - dest: app.js - - src: fixture/app-service-bicep/package.json - dest: package.json - - src: fixture/app-service-bicep/azure.yaml - dest: azure.yaml - - src: fixture/app-service-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/app-service-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/app-service-bicep/infra/main.parameters.json - dest: infra/main.parameters.json + prompt: "Create a todo list with frontend and API and deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -206,6 +157,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -220,21 +172,7 @@ stimuli: # Jest: "creates serverless HTTP API" - name: "Deploy Azure Functions - Serverless HTTP API (Bicep)" - prompt: "My Azure Functions serverless HTTP API is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." - environment: - files: - - src: fixture/functions-bicep/host.json - dest: host.json - - src: fixture/functions-bicep/package.json - dest: package.json - - src: fixture/functions-bicep/azure.yaml - dest: azure.yaml - - src: fixture/functions-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/functions-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/functions-bicep/infra/main.parameters.json - dest: infra/main.parameters.json + prompt: "Create a serverless HTTP API using Azure Functions and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -251,6 +189,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -260,21 +199,7 @@ stimuli: # Jest: "creates event-driven function app" - name: "Deploy Azure Functions - Event-Driven (Bicep)" - prompt: "My event-driven Azure Functions app is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." - environment: - files: - - src: fixture/functions-bicep/host.json - dest: host.json - - src: fixture/functions-bicep/package.json - dest: package.json - - src: fixture/functions-bicep/azure.yaml - dest: azure.yaml - - src: fixture/functions-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/functions-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/functions-bicep/infra/main.parameters.json - dest: infra/main.parameters.json + prompt: "Create an event-driven function app to process messages and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -291,6 +216,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -300,21 +226,7 @@ stimuli: # Jest: "creates Python function app with Service Bus trigger" - name: "Deploy Azure Functions - Python Service Bus (Bicep)" - prompt: "My Python Azure Functions app with Service Bus trigger is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." - environment: - files: - - src: fixture/functions-bicep/host.json - dest: host.json - - src: fixture/functions-bicep/package.json - dest: package.json - - src: fixture/functions-bicep/azure.yaml - dest: azure.yaml - - src: fixture/functions-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/functions-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/functions-bicep/infra/main.parameters.json - dest: infra/main.parameters.json + prompt: "Create an azure python function app that takes input from a service bus trigger and does message processing and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -331,6 +243,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -345,27 +258,7 @@ stimuli: # Jest: "creates and deploys workflow app with Durable Task Scheduler" - name: "Deploy DTS - Workflow App (Bicep)" - prompt: "My Durable Task Scheduler workflow application is prepared and ready. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." - environment: - files: - - src: fixture/dts-bicep/host.json - dest: host.json - - src: fixture/dts-bicep/azure.yaml - dest: azure.yaml - - src: fixture/dts-bicep/package.json - dest: package.json - - src: fixture/dts-bicep/src/functions/orderOrchestrator.js - dest: src/functions/orderOrchestrator.js - - src: fixture/dts-bicep/src/functions/orderActivities.js - dest: src/functions/orderActivities.js - - src: fixture/dts-bicep/src/functions/startOrder.js - dest: src/functions/startOrder.js - - src: fixture/dts-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/dts-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/dts-bicep/infra/main.parameters.json - dest: infra/main.parameters.json + prompt: "Create a workflow app that orchestrates a multi-step order processing pipeline and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -382,13 +275,14 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: required: - name: "(?i)^(bash|powershell|pwsh)$" command: "(?i)azd\\s+(up|provision)\\b" - # Verify DTS infrastructure is present in the fixture + # Verify DTS infrastructure is present in the agent-generated workspace - type: file-matches config: path: "infra/resources.bicep" @@ -405,25 +299,7 @@ stimuli: # Jest: "creates containerized web application with Bicep" - name: "Deploy Container Apps - Containerized Web App (Bicep)" - prompt: "My containerized web application is prepared and ready. Deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." - environment: - files: - - src: fixture/container-apps-bicep/app.js - dest: app.js - - src: fixture/container-apps-bicep/package.json - dest: package.json - - src: fixture/container-apps-bicep/Dockerfile - dest: Dockerfile - - src: fixture/container-apps-bicep/azure.yaml - dest: azure.yaml - - src: fixture/container-apps-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/container-apps-bicep/infra/abbreviations.json - dest: infra/abbreviations.json - - src: fixture/container-apps-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/container-apps-bicep/infra/main.parameters.json - dest: infra/main.parameters.json + prompt: "Create a containerized web application and deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -440,6 +316,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -465,25 +342,7 @@ stimuli: # Jest: "creates simple containerized Node.js app" - name: "Deploy Container Apps - Node.js App (Bicep)" - prompt: "My containerized Node.js application is prepared and ready. Deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." - environment: - files: - - src: fixture/container-apps-bicep/app.js - dest: app.js - - src: fixture/container-apps-bicep/package.json - dest: package.json - - src: fixture/container-apps-bicep/Dockerfile - dest: Dockerfile - - src: fixture/container-apps-bicep/azure.yaml - dest: azure.yaml - - src: fixture/container-apps-bicep/infra/main.bicep - dest: infra/main.bicep - - src: fixture/container-apps-bicep/infra/abbreviations.json - dest: infra/abbreviations.json - - src: fixture/container-apps-bicep/infra/resources.bicep - dest: infra/resources.bicep - - src: fixture/container-apps-bicep/infra/main.parameters.json - dest: infra/main.parameters.json + prompt: "Create a simple containerized Node.js hello world app and deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -500,6 +359,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -522,15 +382,7 @@ stimuli: # Jest: "creates whiteboard application with Terraform" - name: "Deploy SWA - Whiteboard App (Terraform)" - prompt: "My static whiteboard web app is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." - environment: - files: - - src: fixture/swa-terraform/index.html - dest: index.html - - src: fixture/swa-terraform/azure.yaml - dest: azure.yaml - - src: fixture/swa-terraform/infra/main.tf - dest: infra/main.tf + prompt: "Create a static whiteboard web app and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -548,6 +400,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -563,15 +416,7 @@ stimuli: # Jest: "creates static portfolio website with Terraform infrastructure" - name: "Deploy SWA - Portfolio Website (Terraform)" - prompt: "My static portfolio website is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." - environment: - files: - - src: fixture/swa-terraform/index.html - dest: index.html - - src: fixture/swa-terraform/azure.yaml - dest: azure.yaml - - src: fixture/swa-terraform/infra/main.tf - dest: infra/main.tf + prompt: "Create a static portfolio website and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -589,6 +434,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -609,17 +455,7 @@ stimuli: # Jest: "creates discussion board with Terraform" - name: "Deploy App Service - Discussion Board (Terraform)" - prompt: "My Node.js discussion board application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." - environment: - files: - - src: fixture/app-service-terraform/app.js - dest: app.js - - src: fixture/app-service-terraform/package.json - dest: package.json - - src: fixture/app-service-terraform/azure.yaml - dest: azure.yaml - - src: fixture/app-service-terraform/infra/main.tf - dest: infra/main.tf + prompt: "Create a discussion board application and deploy it to Azure App Service, prefer Terraform over Bicep, in my current subscription in the westus2 region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -637,6 +473,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -652,17 +489,7 @@ stimuli: # Jest: "creates todo list with frontend and API using Terraform" - name: "Deploy App Service - Todo List (Terraform)" - prompt: "My Node.js todo list application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." - environment: - files: - - src: fixture/app-service-terraform/app.js - dest: app.js - - src: fixture/app-service-terraform/package.json - dest: package.json - - src: fixture/app-service-terraform/azure.yaml - dest: azure.yaml - - src: fixture/app-service-terraform/infra/main.tf - dest: infra/main.tf + prompt: "Create a todo list with frontend and API and deploy to Azure App Service using Terraform in my current subscription in the westus2 region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -680,6 +507,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -700,15 +528,7 @@ stimuli: # Jest: "creates serverless HTTP API with Terraform" - name: "Deploy Azure Functions - Serverless HTTP API (Terraform)" - prompt: "My Azure Functions serverless HTTP API is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." - environment: - files: - - src: fixture/functions-terraform/host.json - dest: host.json - - src: fixture/functions-terraform/azure.yaml - dest: azure.yaml - - src: fixture/functions-terraform/infra/main.tf - dest: infra/main.tf + prompt: "Create a serverless HTTP API using Azure Functions and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -725,6 +545,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -740,15 +561,7 @@ stimuli: # Jest: "creates event-driven function app with Terraform" - name: "Deploy Azure Functions - Event-Driven (Terraform)" - prompt: "My event-driven Azure Functions app is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." - environment: - files: - - src: fixture/functions-terraform/host.json - dest: host.json - - src: fixture/functions-terraform/azure.yaml - dest: azure.yaml - - src: fixture/functions-terraform/infra/main.tf - dest: infra/main.tf + prompt: "Create an event-driven function app to process messages and deploy it to Azure Functions using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -765,6 +578,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -780,15 +594,7 @@ stimuli: # Jest: "creates URL shortener service with Terraform infrastructure" - name: "Deploy Azure Functions - URL Shortener (Terraform)" - prompt: "My Azure Functions URL shortener service is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." - environment: - files: - - src: fixture/functions-terraform/host.json - dest: host.json - - src: fixture/functions-terraform/azure.yaml - dest: azure.yaml - - src: fixture/functions-terraform/infra/main.tf - dest: infra/main.tf + prompt: "Create a URL shortener service using Azure Functions that creates short links and redirects users to the original URL and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -805,6 +611,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -819,19 +626,7 @@ stimuli: # Jest: "creates containerized web application with Terraform" - name: "Deploy Container Apps - Containerized Web App (Terraform)" - prompt: "My containerized web application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." - environment: - files: - - src: fixture/container-apps-terraform/app.js - dest: app.js - - src: fixture/container-apps-terraform/package.json - dest: package.json - - src: fixture/container-apps-terraform/Dockerfile - dest: Dockerfile - - src: fixture/container-apps-terraform/azure.yaml - dest: azure.yaml - - src: fixture/container-apps-terraform/infra/main.tf - dest: infra/main.tf + prompt: "Create a containerized web application and deploy it to Azure Container Apps using terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -848,6 +643,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -877,19 +673,7 @@ stimuli: # Jest: "creates simple containerized Node.js app with Terraform" - name: "Deploy Container Apps - Node.js App (Terraform)" - prompt: "My containerized Node.js application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." - environment: - files: - - src: fixture/container-apps-terraform/app.js - dest: app.js - - src: fixture/container-apps-terraform/package.json - dest: package.json - - src: fixture/container-apps-terraform/Dockerfile - dest: Dockerfile - - src: fixture/container-apps-terraform/azure.yaml - dest: azure.yaml - - src: fixture/container-apps-terraform/infra/main.tf - dest: infra/main.tf + prompt: "Create a simple containerized Node.js hello world app and deploy it to Azure Container Apps using Terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -906,6 +690,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -923,19 +708,7 @@ stimuli: # Jest: "creates social media application with Terraform infrastructure" - name: "Deploy Container Apps - Social Media App (Terraform)" - prompt: "My containerized social media application is prepared with Terraform infrastructure and ready to deploy. Deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." - environment: - files: - - src: fixture/container-apps-terraform/app.js - dest: app.js - - src: fixture/container-apps-terraform/package.json - dest: package.json - - src: fixture/container-apps-terraform/Dockerfile - dest: Dockerfile - - src: fixture/container-apps-terraform/azure.yaml - dest: azure.yaml - - src: fixture/container-apps-terraform/infra/main.tf - dest: infra/main.tf + prompt: "Create a simple social media application with likes and comments and deploy to Azure using Terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool." constraints: max_turns: 50 tags: @@ -952,6 +725,7 @@ stimuli: - type: skill-invocation config: required: + - azure-prepare - azure-deploy - type: tool-calls config: @@ -1160,7 +934,7 @@ stimuli: pattern: "(?i)fatal error|unhandled exception|stack trace" # Jest: "deploys aspire container build" - # Checks Container Apps two-phase Bicep pattern generated by azure-prepare. + # Checks Container Apps two-phase Bicep pattern in the agent-generated workspace. - name: "Brownfield .NET Aspire - Container Build" prompt: >- Please deploy this application to Azure. @@ -1202,7 +976,7 @@ stimuli: required: - name: "(?i)^(bash|powershell|pwsh)$" command: "(?i)azd\\s+(up|provision)\\b" - # Verify Container Apps two-phase Bicep pattern (generated by azure-prepare) + # Verify Container Apps two-phase Bicep pattern in the agent-generated workspace - type: file-matches config: path: "**/*.bicep" diff --git a/evals/azure-deploy/fixture/app-service-bicep/app.js b/evals/azure-deploy/fixture/app-service-bicep/app.js deleted file mode 100644 index 236690886..000000000 --- a/evals/azure-deploy/fixture/app-service-bicep/app.js +++ /dev/null @@ -1,9 +0,0 @@ -const http = require('http'); -const port = process.env.PORT || 3000; - -const server = http.createServer((req, res) => { - res.writeHead(200, { 'Content-Type': 'text/html' }); - res.end('

Todo App

'); -}); - -server.listen(port, () => console.log(`Server running on port ${port}`)); diff --git a/evals/azure-deploy/fixture/app-service-bicep/azure.yaml b/evals/azure-deploy/fixture/app-service-bicep/azure.yaml deleted file mode 100644 index 67d26e82d..000000000 --- a/evals/azure-deploy/fixture/app-service-bicep/azure.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json - -name: todo-app -services: - web: - project: . - language: js - host: appservice -infra: - provider: bicep diff --git a/evals/azure-deploy/fixture/app-service-bicep/infra/main.bicep b/evals/azure-deploy/fixture/app-service-bicep/infra/main.bicep deleted file mode 100644 index ee78b18a6..000000000 --- a/evals/azure-deploy/fixture/app-service-bicep/infra/main.bicep +++ /dev/null @@ -1,34 +0,0 @@ -targetScope = 'subscription' - -@minLength(1) -@maxLength(64) -@description('Name of the environment that can be used as part of naming resource convention') -param environmentName string - -@minLength(1) -@description('Primary location for all resources') -param location string - -var tags = { 'azd-env-name': environmentName } -var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) - -resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' = { - name: 'rg-${environmentName}' - location: location - tags: tags -} - -module appService './resources.bicep' = { - name: 'appService' - scope: rg - params: { - location: location - tags: tags - resourceToken: resourceToken - } -} - -output AZURE_LOCATION string = location -output AZURE_TENANT_ID string = tenant().tenantId -output AZURE_RESOURCE_GROUP string = rg.name -output WEB_URL string = appService.outputs.WEB_URL diff --git a/evals/azure-deploy/fixture/app-service-bicep/infra/main.parameters.json b/evals/azure-deploy/fixture/app-service-bicep/infra/main.parameters.json deleted file mode 100644 index bc915c431..000000000 --- a/evals/azure-deploy/fixture/app-service-bicep/infra/main.parameters.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "environmentName": { - "value": "${AZURE_ENV_NAME}" - }, - "location": { - "value": "${AZURE_LOCATION}" - } - } -} diff --git a/evals/azure-deploy/fixture/app-service-bicep/infra/resources.bicep b/evals/azure-deploy/fixture/app-service-bicep/infra/resources.bicep deleted file mode 100644 index c6ea70960..000000000 --- a/evals/azure-deploy/fixture/app-service-bicep/infra/resources.bicep +++ /dev/null @@ -1,35 +0,0 @@ -@description('Location for all resources') -param location string = resourceGroup().location - -param tags object = {} -param resourceToken string - -resource appServicePlan 'Microsoft.Web/serverfarms@2022-09-01' = { - name: 'plan-${resourceToken}' - location: location - tags: tags - sku: { - name: 'B1' - tier: 'Basic' - } - kind: 'linux' - properties: { - reserved: true - } -} - -resource appService 'Microsoft.Web/sites@2022-09-01' = { - name: 'app-${resourceToken}' - location: location - tags: union(tags, { 'azd-service-name': 'web' }) - properties: { - serverFarmId: appServicePlan.id - siteConfig: { - linuxFxVersion: 'NODE|20-lts' - appCommandLine: 'npm start' - } - httpsOnly: true - } -} - -output WEB_URL string = 'https://${appService.properties.defaultHostName}' diff --git a/evals/azure-deploy/fixture/app-service-bicep/package.json b/evals/azure-deploy/fixture/app-service-bicep/package.json deleted file mode 100644 index 39d62ac83..000000000 --- a/evals/azure-deploy/fixture/app-service-bicep/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "todo-app", - "version": "1.0.0", - "description": "A simple todo web application", - "main": "app.js", - "scripts": { - "start": "node app.js" - } -} diff --git a/evals/azure-deploy/fixture/app-service-terraform/app.js b/evals/azure-deploy/fixture/app-service-terraform/app.js deleted file mode 100644 index 888876dfb..000000000 --- a/evals/azure-deploy/fixture/app-service-terraform/app.js +++ /dev/null @@ -1,7 +0,0 @@ -const http = require('http'); -const port = process.env.PORT || 3000; -const server = http.createServer((req, res) => { - res.writeHead(200, { 'Content-Type': 'text/html' }); - res.end('

Todo App

'); -}); -server.listen(port); diff --git a/evals/azure-deploy/fixture/app-service-terraform/azure.yaml b/evals/azure-deploy/fixture/app-service-terraform/azure.yaml deleted file mode 100644 index 3086dd41c..000000000 --- a/evals/azure-deploy/fixture/app-service-terraform/azure.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json - -name: todo-app-tf -services: - web: - project: . - language: js - host: appservice -infra: - provider: terraform diff --git a/evals/azure-deploy/fixture/app-service-terraform/infra/main.tf b/evals/azure-deploy/fixture/app-service-terraform/infra/main.tf deleted file mode 100644 index c22e465cb..000000000 --- a/evals/azure-deploy/fixture/app-service-terraform/infra/main.tf +++ /dev/null @@ -1,74 +0,0 @@ -terraform { - required_providers { - azurerm = { - source = "hashicorp/azurerm" - version = "~> 3.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - backend "azurerm" {} -} - -provider "azurerm" { - features {} -} - -variable "environment_name" { - type = string - description = "Name of the azd environment" -} - -variable "location" { - type = string - description = "Azure region for all resources" -} - -resource "random_string" "resource_token" { - length = 13 - upper = false - special = false -} - -resource "azurerm_resource_group" "rg" { - name = "rg-${var.environment_name}" - location = var.location - tags = { "azd-env-name" = var.environment_name } -} - -resource "azurerm_service_plan" "plan" { - name = "plan-${random_string.resource_token.result}" - resource_group_name = azurerm_resource_group.rg.name - location = azurerm_resource_group.rg.location - os_type = "Linux" - sku_name = "B1" -} - -resource "azurerm_linux_web_app" "app" { - name = "app-${random_string.resource_token.result}" - resource_group_name = azurerm_resource_group.rg.name - location = azurerm_resource_group.rg.location - service_plan_id = azurerm_service_plan.plan.id - https_only = true - - site_config { - application_stack { - node_version = "20-lts" - } - } - - tags = { - "azd-env-name" = var.environment_name - "azd-service-name" = "web" - } -} - -output "AZURE_LOCATION" { - value = var.location -} - -output "WEB_URL" { - value = "https://${azurerm_linux_web_app.app.default_hostname}" -} diff --git a/evals/azure-deploy/fixture/app-service-terraform/package.json b/evals/azure-deploy/fixture/app-service-terraform/package.json deleted file mode 100644 index 47a4ff9e2..000000000 --- a/evals/azure-deploy/fixture/app-service-terraform/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "todo-app", - "version": "1.0.0", - "scripts": { "start": "node app.js" } -} diff --git a/evals/azure-deploy/fixture/container-apps-bicep/Dockerfile b/evals/azure-deploy/fixture/container-apps-bicep/Dockerfile deleted file mode 100644 index 09b60793e..000000000 --- a/evals/azure-deploy/fixture/container-apps-bicep/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM node:20-alpine -WORKDIR /app -COPY package*.json ./ -RUN npm install --production -COPY . . -EXPOSE 3000 -CMD ["node", "app.js"] diff --git a/evals/azure-deploy/fixture/container-apps-bicep/app.js b/evals/azure-deploy/fixture/container-apps-bicep/app.js deleted file mode 100644 index 2280eb6db..000000000 --- a/evals/azure-deploy/fixture/container-apps-bicep/app.js +++ /dev/null @@ -1,9 +0,0 @@ -const http = require('http'); -const port = process.env.PORT || 3000; - -const server = http.createServer((req, res) => { - res.writeHead(200, { 'Content-Type': 'text/html' }); - res.end('

Containerized Web App

'); -}); - -server.listen(port, () => console.log(`Server running on port ${port}`)); diff --git a/evals/azure-deploy/fixture/container-apps-bicep/azure.yaml b/evals/azure-deploy/fixture/container-apps-bicep/azure.yaml deleted file mode 100644 index fecd118a5..000000000 --- a/evals/azure-deploy/fixture/container-apps-bicep/azure.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json - -name: containerized-web-app -services: - web: - project: . - host: containerapp - docker: - path: ./Dockerfile -infra: - provider: bicep diff --git a/evals/azure-deploy/fixture/container-apps-bicep/infra/abbreviations.json b/evals/azure-deploy/fixture/container-apps-bicep/infra/abbreviations.json deleted file mode 100644 index 828e40f0f..000000000 --- a/evals/azure-deploy/fixture/container-apps-bicep/infra/abbreviations.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "containerRegistries": "cr", - "containerApps": "ca", - "managedEnvironments": "cae", - "userAssignedIdentities": "id-" -} diff --git a/evals/azure-deploy/fixture/container-apps-bicep/infra/main.bicep b/evals/azure-deploy/fixture/container-apps-bicep/infra/main.bicep deleted file mode 100644 index 200ccc5e4..000000000 --- a/evals/azure-deploy/fixture/container-apps-bicep/infra/main.bicep +++ /dev/null @@ -1,37 +0,0 @@ -targetScope = 'subscription' - -@minLength(1) -@maxLength(64) -@description('Name of the environment that can be used as part of naming resource convention') -param environmentName string - -@minLength(1) -@description('Primary location for all resources') -param location string - -var tags = { 'azd-env-name': environmentName } -var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) -var abbrs = loadJsonContent('./abbreviations.json') - -resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' = { - name: 'rg-${environmentName}' - location: location - tags: tags -} - -module containerApps './resources.bicep' = { - name: 'containerApps' - scope: rg - params: { - location: location - tags: tags - resourceToken: resourceToken - abbrs: abbrs - } -} - -output AZURE_LOCATION string = location -output AZURE_TENANT_ID string = tenant().tenantId -output AZURE_RESOURCE_GROUP string = rg.name -output AZURE_CONTAINER_REGISTRY_ENDPOINT string = containerApps.outputs.AZURE_CONTAINER_REGISTRY_ENDPOINT -output WEB_URL string = containerApps.outputs.WEB_URL diff --git a/evals/azure-deploy/fixture/container-apps-bicep/infra/main.parameters.json b/evals/azure-deploy/fixture/container-apps-bicep/infra/main.parameters.json deleted file mode 100644 index bc915c431..000000000 --- a/evals/azure-deploy/fixture/container-apps-bicep/infra/main.parameters.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "environmentName": { - "value": "${AZURE_ENV_NAME}" - }, - "location": { - "value": "${AZURE_LOCATION}" - } - } -} diff --git a/evals/azure-deploy/fixture/container-apps-bicep/infra/resources.bicep b/evals/azure-deploy/fixture/container-apps-bicep/infra/resources.bicep deleted file mode 100644 index 3df4c4a03..000000000 --- a/evals/azure-deploy/fixture/container-apps-bicep/infra/resources.bicep +++ /dev/null @@ -1,71 +0,0 @@ -@description('Location for all resources') -param location string = resourceGroup().location - -param tags object = {} -param resourceToken string -param abbrs object - -// AcrPull role definition GUID -var acrPullRoleId = '7f951dda-4ed3-4680-a7ca-43fe172d538d' - -resource containerRegistry 'Microsoft.ContainerRegistry/registries@2023-01-01-preview' = { - name: '${abbrs.containerRegistries}${resourceToken}' - location: location - tags: tags - sku: { name: 'Basic' } - properties: { adminUserEnabled: false } -} - -resource containerAppsEnv 'Microsoft.App/managedEnvironments@2023-05-01' = { - name: '${abbrs.managedEnvironments}${resourceToken}' - location: location - tags: tags - properties: {} -} - -resource containerApp 'Microsoft.App/containerApps@2023-05-01' = { - name: '${abbrs.containerApps}${resourceToken}' - location: location - tags: union(tags, { 'azd-service-name': 'web' }) - identity: { - type: 'SystemAssigned' - } - properties: { - managedEnvironmentId: containerAppsEnv.id - configuration: { - ingress: { - external: true - targetPort: 3000 - } - registries: [ - { - server: containerRegistry.properties.loginServer - identity: 'system' - } - ] - } - template: { - containers: [ - { - name: 'web' - // Placeholder public image for initial provisioning; azd will replace on first deploy - image: 'mcr.microsoft.com/azuredocs/containerapps-helloworld:latest' - resources: { cpu: json('0.5'), memory: '1Gi' } - } - ] - } - } -} - -resource roleAcrPull 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(containerRegistry.id, containerApp.id, subscriptionResourceId('Microsoft.Authorization/roleDefinitions', acrPullRoleId)) - scope: containerRegistry - properties: { - principalId: containerApp.identity.principalId - principalType: 'ServicePrincipal' - roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', acrPullRoleId) - } -} - -output AZURE_CONTAINER_REGISTRY_ENDPOINT string = containerRegistry.properties.loginServer -output WEB_URL string = 'https://${containerApp.properties.configuration.ingress.fqdn}' diff --git a/evals/azure-deploy/fixture/container-apps-bicep/package.json b/evals/azure-deploy/fixture/container-apps-bicep/package.json deleted file mode 100644 index c1b814669..000000000 --- a/evals/azure-deploy/fixture/container-apps-bicep/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "containerized-web-app", - "version": "1.0.0", - "scripts": { "start": "node app.js" } -} diff --git a/evals/azure-deploy/fixture/container-apps-terraform/Dockerfile b/evals/azure-deploy/fixture/container-apps-terraform/Dockerfile deleted file mode 100644 index 09b60793e..000000000 --- a/evals/azure-deploy/fixture/container-apps-terraform/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM node:20-alpine -WORKDIR /app -COPY package*.json ./ -RUN npm install --production -COPY . . -EXPOSE 3000 -CMD ["node", "app.js"] diff --git a/evals/azure-deploy/fixture/container-apps-terraform/app.js b/evals/azure-deploy/fixture/container-apps-terraform/app.js deleted file mode 100644 index bb3cb26c3..000000000 --- a/evals/azure-deploy/fixture/container-apps-terraform/app.js +++ /dev/null @@ -1,7 +0,0 @@ -const http = require('http'); -const port = process.env.PORT || 3000; -const server = http.createServer((req, res) => { - res.writeHead(200, { 'Content-Type': 'text/html' }); - res.end('

Containerized Web App

'); -}); -server.listen(port); diff --git a/evals/azure-deploy/fixture/container-apps-terraform/azure.yaml b/evals/azure-deploy/fixture/container-apps-terraform/azure.yaml deleted file mode 100644 index 71cb1a85a..000000000 --- a/evals/azure-deploy/fixture/container-apps-terraform/azure.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json - -name: containerized-web-app-tf -services: - web: - project: . - host: containerapp - docker: - path: ./Dockerfile -infra: - provider: terraform diff --git a/evals/azure-deploy/fixture/container-apps-terraform/infra/main.tf b/evals/azure-deploy/fixture/container-apps-terraform/infra/main.tf deleted file mode 100644 index 1fa76c4e4..000000000 --- a/evals/azure-deploy/fixture/container-apps-terraform/infra/main.tf +++ /dev/null @@ -1,125 +0,0 @@ -terraform { - required_providers { - azurerm = { - source = "hashicorp/azurerm" - version = "~> 3.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - backend "azurerm" {} -} - -provider "azurerm" { - features {} -} - -variable "environment_name" { - type = string - description = "Name of the azd environment" -} - -variable "location" { - type = string - description = "Azure region for all resources" -} - -# Image variable defaults to placeholder so provisioning succeeds before first azd deploy -variable "web_image" { - type = string - default = "mcr.microsoft.com/azuredocs/containerapps-helloworld:latest" - description = "Container image for the web service (replaced by azd on deploy)" -} - -resource "random_string" "resource_token" { - length = 13 - upper = false - special = false -} - -resource "azurerm_resource_group" "rg" { - name = "rg-${var.environment_name}" - location = var.location - tags = { "azd-env-name" = var.environment_name } -} - -resource "azurerm_container_registry" "acr" { - name = "cr${random_string.resource_token.result}" - resource_group_name = azurerm_resource_group.rg.name - location = azurerm_resource_group.rg.location - sku = "Basic" - admin_enabled = false - tags = { "azd-env-name" = var.environment_name } -} - -resource "azurerm_container_app_environment" "cae" { - name = "cae-${random_string.resource_token.result}" - resource_group_name = azurerm_resource_group.rg.name - location = azurerm_resource_group.rg.location - tags = { "azd-env-name" = var.environment_name } -} - -resource "azurerm_container_app" "web" { - name = "ca-${random_string.resource_token.result}" - container_app_environment_id = azurerm_container_app_environment.cae.id - resource_group_name = azurerm_resource_group.rg.name - revision_mode = "Single" - - identity { - type = "SystemAssigned" - } - - registry { - server = azurerm_container_registry.acr.login_server - identity = "System" - } - - ingress { - external_enabled = true - target_port = 3000 - traffic_weight { - percentage = 100 - latest_revision = true - } - } - - template { - container { - name = "web" - image = var.web_image - cpu = 0.5 - memory = "1Gi" - } - } - - tags = { - "azd-env-name" = var.environment_name - "azd-service-name" = "web" - } - - lifecycle { - ignore_changes = [ - template[0].container[0].image - ] - } -} - -resource "azurerm_role_assignment" "acr_pull" { - scope = azurerm_container_registry.acr.id - role_definition_name = "AcrPull" - principal_id = azurerm_container_app.web.identity[0].principal_id -} - -output "AZURE_LOCATION" { - value = var.location -} - -output "AZURE_CONTAINER_REGISTRY_ENDPOINT" { - value = azurerm_container_registry.acr.login_server -} - -output "WEB_URL" { - value = "https://${azurerm_container_app.web.ingress[0].fqdn}" -} diff --git a/evals/azure-deploy/fixture/container-apps-terraform/package.json b/evals/azure-deploy/fixture/container-apps-terraform/package.json deleted file mode 100644 index c1b814669..000000000 --- a/evals/azure-deploy/fixture/container-apps-terraform/package.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "containerized-web-app", - "version": "1.0.0", - "scripts": { "start": "node app.js" } -} diff --git a/evals/azure-deploy/fixture/dts-bicep/azure.yaml b/evals/azure-deploy/fixture/dts-bicep/azure.yaml deleted file mode 100644 index e113192a7..000000000 --- a/evals/azure-deploy/fixture/dts-bicep/azure.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json - -name: dts-workflow-app -services: - api: - project: . - language: js - host: function -infra: - provider: bicep diff --git a/evals/azure-deploy/fixture/dts-bicep/host.json b/evals/azure-deploy/fixture/dts-bicep/host.json deleted file mode 100644 index d2059a46b..000000000 --- a/evals/azure-deploy/fixture/dts-bicep/host.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "version": "2.0" -} diff --git a/evals/azure-deploy/fixture/dts-bicep/infra/main.bicep b/evals/azure-deploy/fixture/dts-bicep/infra/main.bicep deleted file mode 100644 index 5833e2a9f..000000000 --- a/evals/azure-deploy/fixture/dts-bicep/infra/main.bicep +++ /dev/null @@ -1,34 +0,0 @@ -targetScope = 'subscription' - -@minLength(1) -@maxLength(64) -@description('Name of the environment that can be used as part of naming resource convention') -param environmentName string - -@minLength(1) -@description('Primary location for all resources') -param location string - -var tags = { 'azd-env-name': environmentName } -var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) - -resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' = { - name: 'rg-${environmentName}' - location: location - tags: tags -} - -module dtsWorkflow './resources.bicep' = { - name: 'dtsWorkflow' - scope: rg - params: { - location: location - tags: tags - resourceToken: resourceToken - } -} - -output AZURE_LOCATION string = location -output AZURE_TENANT_ID string = tenant().tenantId -output AZURE_RESOURCE_GROUP string = rg.name -output API_URL string = dtsWorkflow.outputs.API_URL diff --git a/evals/azure-deploy/fixture/dts-bicep/infra/main.parameters.json b/evals/azure-deploy/fixture/dts-bicep/infra/main.parameters.json deleted file mode 100644 index bc915c431..000000000 --- a/evals/azure-deploy/fixture/dts-bicep/infra/main.parameters.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "environmentName": { - "value": "${AZURE_ENV_NAME}" - }, - "location": { - "value": "${AZURE_LOCATION}" - } - } -} diff --git a/evals/azure-deploy/fixture/dts-bicep/infra/resources.bicep b/evals/azure-deploy/fixture/dts-bicep/infra/resources.bicep deleted file mode 100644 index bf153e146..000000000 --- a/evals/azure-deploy/fixture/dts-bicep/infra/resources.bicep +++ /dev/null @@ -1,100 +0,0 @@ -@description('Location for all resources') -param location string = resourceGroup().location - -param tags object = {} -param resourceToken string - -// DTS built-in role: 0ad04412-c4d5-4796-b79c-f76d14c8d402 -var dtsContributorRoleId = '0ad04412-c4d5-4796-b79c-f76d14c8d402' - -resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = { - name: 'st${resourceToken}' - location: location - tags: tags - sku: { name: 'Standard_LRS' } - kind: 'StorageV2' -} - -resource appServicePlan 'Microsoft.Web/serverfarms@2022-09-01' = { - name: 'plan-${resourceToken}' - location: location - tags: tags - sku: { - name: 'Y1' - tier: 'Dynamic' - } - kind: 'functionapp' - properties: {} -} - -resource functionAppIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { - name: 'id-${resourceToken}' - location: location - tags: tags -} - -resource dtsScheduler 'Microsoft.DurableTask/schedulers@2024-10-01-preview' = { - name: 'dts-${resourceToken}' - location: location - tags: tags - sku: { - name: 'Dedicated' - capacity: 1 - } - properties: { - ipAllowlist: ['0.0.0.0/0'] - } -} - -resource dtsTaskHub 'Microsoft.DurableTask/schedulers/taskHubs@2024-10-01-preview' = { - parent: dtsScheduler - name: 'taskhub' - properties: {} -} - -resource dtsRoleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = { - name: guid(dtsScheduler.id, functionAppIdentity.id, subscriptionResourceId('Microsoft.Authorization/roleDefinitions', dtsContributorRoleId)) - scope: dtsScheduler - properties: { - principalId: functionAppIdentity.properties.principalId - principalType: 'ServicePrincipal' - roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', dtsContributorRoleId) - } -} - -resource functionApp 'Microsoft.Web/sites@2022-09-01' = { - name: 'func-${resourceToken}' - location: location - tags: union(tags, { 'azd-service-name': 'api' }) - kind: 'functionapp' - identity: { - type: 'UserAssigned' - userAssignedIdentities: { '${functionAppIdentity.id}': {} } - } - properties: { - serverFarmId: appServicePlan.id - siteConfig: { - appSettings: [ - { - name: 'AzureWebJobsStorage' - value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};EndpointSuffix=${environment().suffixes.storage};AccountKey=${storageAccount.listKeys().keys[0].value}' - } - { - name: 'FUNCTIONS_EXTENSION_VERSION' - value: '~4' - } - { - name: 'FUNCTIONS_WORKER_RUNTIME' - value: 'node' - } - { - name: 'DURABLE_TASK_SCHEDULER_CONNECTION_STRING' - value: 'Endpoint=${dtsScheduler.properties.endpoint};TaskHub=${dtsTaskHub.name};Authentication=ManagedIdentity;ClientID=${functionAppIdentity.properties.clientId}' - } - ] - } - httpsOnly: true - } -} - -output API_URL string = 'https://${functionApp.properties.defaultHostName}' diff --git a/evals/azure-deploy/fixture/dts-bicep/package.json b/evals/azure-deploy/fixture/dts-bicep/package.json deleted file mode 100644 index 77b2f7bdc..000000000 --- a/evals/azure-deploy/fixture/dts-bicep/package.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "order-processing-workflow", - "version": "1.0.0", - "description": "Multi-step order processing pipeline using Azure Durable Task Scheduler", - "main": "src/index.js", - "scripts": { - "start": "func start", - "build": "echo 'No build step required'" - }, - "dependencies": { - "@azure/durabletask": "^0.1.0", - "@azure/functions": "^4.0.0" - }, - "devDependencies": { - "azure-functions-core-tools": "^4.x" - }, - "engines": { - "node": ">=18.0.0" - } -} diff --git a/evals/azure-deploy/fixture/dts-bicep/src/functions/orderActivities.js b/evals/azure-deploy/fixture/dts-bicep/src/functions/orderActivities.js deleted file mode 100644 index 1c6ce69db..000000000 --- a/evals/azure-deploy/fixture/dts-bicep/src/functions/orderActivities.js +++ /dev/null @@ -1,29 +0,0 @@ -const { app } = require("@azure/functions"); - -app.generic("validateOrder", { - trigger: app.trigger.generic({ type: "activityTrigger" }), - handler: async (order) => { - if (!order.orderId || !order.amount || order.amount <= 0) { - return { success: false, reason: "Invalid order: missing orderId or invalid amount" }; - } - if (!order.customerEmail) { - return { success: false, reason: "Invalid order: missing customer email" }; - } - return { success: true }; - }, -}); - -app.generic("processPayment", { - trigger: app.trigger.generic({ type: "activityTrigger" }), - handler: async ({ orderId, amount }) => { - const paymentId = `pay-${orderId}-${Date.now()}`; - return { paymentId, orderId, amount, processedAt: new Date().toISOString() }; - }, -}); - -app.generic("sendConfirmation", { - trigger: app.trigger.generic({ type: "activityTrigger" }), - handler: async ({ orderId, email, paymentId }) => { - return { sent: true, orderId, email, paymentId }; - }, -}); diff --git a/evals/azure-deploy/fixture/dts-bicep/src/functions/orderOrchestrator.js b/evals/azure-deploy/fixture/dts-bicep/src/functions/orderOrchestrator.js deleted file mode 100644 index 2af10a39b..000000000 --- a/evals/azure-deploy/fixture/dts-bicep/src/functions/orderOrchestrator.js +++ /dev/null @@ -1,30 +0,0 @@ -const { app } = require("@azure/functions"); - -app.generic("orderProcessingOrchestrator", { - trigger: app.trigger.generic({ type: "orchestrationTrigger" }), - handler: async (ctx) => { - const order = ctx.getInput(); - - const validationResult = await ctx.callActivity("validateOrder", order); - if (!validationResult.success) { - return { status: "rejected", reason: validationResult.reason }; - } - - const paymentResult = await ctx.callActivity("processPayment", { - orderId: order.orderId, - amount: order.amount, - }); - - await ctx.callActivity("sendConfirmation", { - orderId: order.orderId, - email: order.customerEmail, - paymentId: paymentResult.paymentId, - }); - - return { - status: "completed", - orderId: order.orderId, - paymentId: paymentResult.paymentId, - }; - }, -}); diff --git a/evals/azure-deploy/fixture/dts-bicep/src/functions/startOrder.js b/evals/azure-deploy/fixture/dts-bicep/src/functions/startOrder.js deleted file mode 100644 index 317c65c8e..000000000 --- a/evals/azure-deploy/fixture/dts-bicep/src/functions/startOrder.js +++ /dev/null @@ -1,27 +0,0 @@ -const { app } = require("@azure/functions"); - -app.http("startOrder", { - methods: ["POST"], - authLevel: "function", - handler: async (request, context) => { - const order = await request.json(); - if (!order.orderId || !order.amount || !order.customerEmail) { - return { - status: 400, - jsonBody: { error: "Request must include orderId, amount, and customerEmail" }, - }; - } - - // In production, use DurableTaskSchedulerClient with the connection string. - // For local dev, use DurableClient from @azure/durable-functions. - context.log(`Starting order processing for orderId: ${order.orderId}`); - - return { - status: 202, - jsonBody: { - orderId: order.orderId, - message: "Order processing started", - }, - }; - }, -}); diff --git a/evals/azure-deploy/fixture/functions-bicep/azure.yaml b/evals/azure-deploy/fixture/functions-bicep/azure.yaml deleted file mode 100644 index 195db5403..000000000 --- a/evals/azure-deploy/fixture/functions-bicep/azure.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json - -name: serverless-http-api -services: - api: - project: . - language: js - host: function -infra: - provider: bicep diff --git a/evals/azure-deploy/fixture/functions-bicep/host.json b/evals/azure-deploy/fixture/functions-bicep/host.json deleted file mode 100644 index d2059a46b..000000000 --- a/evals/azure-deploy/fixture/functions-bicep/host.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "version": "2.0" -} diff --git a/evals/azure-deploy/fixture/functions-bicep/infra/main.bicep b/evals/azure-deploy/fixture/functions-bicep/infra/main.bicep deleted file mode 100644 index d4ec480cb..000000000 --- a/evals/azure-deploy/fixture/functions-bicep/infra/main.bicep +++ /dev/null @@ -1,34 +0,0 @@ -targetScope = 'subscription' - -@minLength(1) -@maxLength(64) -@description('Name of the environment that can be used as part of naming resource convention') -param environmentName string - -@minLength(1) -@description('Primary location for all resources') -param location string - -var tags = { 'azd-env-name': environmentName } -var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) - -resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' = { - name: 'rg-${environmentName}' - location: location - tags: tags -} - -module functions './resources.bicep' = { - name: 'functions' - scope: rg - params: { - location: location - tags: tags - resourceToken: resourceToken - } -} - -output AZURE_LOCATION string = location -output AZURE_TENANT_ID string = tenant().tenantId -output AZURE_RESOURCE_GROUP string = rg.name -output API_URL string = functions.outputs.API_URL diff --git a/evals/azure-deploy/fixture/functions-bicep/infra/main.parameters.json b/evals/azure-deploy/fixture/functions-bicep/infra/main.parameters.json deleted file mode 100644 index bc915c431..000000000 --- a/evals/azure-deploy/fixture/functions-bicep/infra/main.parameters.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "environmentName": { - "value": "${AZURE_ENV_NAME}" - }, - "location": { - "value": "${AZURE_LOCATION}" - } - } -} diff --git a/evals/azure-deploy/fixture/functions-bicep/infra/resources.bicep b/evals/azure-deploy/fixture/functions-bicep/infra/resources.bicep deleted file mode 100644 index 8a68cf8d9..000000000 --- a/evals/azure-deploy/fixture/functions-bicep/infra/resources.bicep +++ /dev/null @@ -1,58 +0,0 @@ -@description('Location for all resources') -param location string = resourceGroup().location - -param tags object = {} -param resourceToken string - -resource storageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = { - name: 'st${resourceToken}' - location: location - tags: tags - sku: { name: 'Standard_LRS' } - kind: 'StorageV2' -} - -resource appServicePlan 'Microsoft.Web/serverfarms@2022-09-01' = { - name: 'plan-${resourceToken}' - location: location - tags: tags - sku: { - name: 'Y1' - tier: 'Dynamic' - } - kind: 'functionapp' - properties: {} -} - -resource functionApp 'Microsoft.Web/sites@2022-09-01' = { - name: 'func-${resourceToken}' - location: location - tags: union(tags, { 'azd-service-name': 'api' }) - kind: 'functionapp' - properties: { - serverFarmId: appServicePlan.id - siteConfig: { - appSettings: [ - { - name: 'AzureWebJobsStorage' - value: 'DefaultEndpointsProtocol=https;AccountName=${storageAccount.name};EndpointSuffix=${environment().suffixes.storage};AccountKey=${storageAccount.listKeys().keys[0].value}' - } - { - name: 'FUNCTIONS_EXTENSION_VERSION' - value: '~4' - } - { - name: 'FUNCTIONS_WORKER_RUNTIME' - value: 'node' - } - { - name: 'WEBSITE_NODE_DEFAULT_VERSION' - value: '~20' - } - ] - } - httpsOnly: true - } -} - -output API_URL string = 'https://${functionApp.properties.defaultHostName}' diff --git a/evals/azure-deploy/fixture/functions-bicep/package.json b/evals/azure-deploy/fixture/functions-bicep/package.json deleted file mode 100644 index 3bcd4ef06..000000000 --- a/evals/azure-deploy/fixture/functions-bicep/package.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "serverless-http-api", - "version": "1.0.0", - "scripts": { - "start": "func start" - } -} diff --git a/evals/azure-deploy/fixture/functions-terraform/azure.yaml b/evals/azure-deploy/fixture/functions-terraform/azure.yaml deleted file mode 100644 index cc9a845c3..000000000 --- a/evals/azure-deploy/fixture/functions-terraform/azure.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json - -name: serverless-http-api-tf -services: - api: - project: . - language: js - host: function -infra: - provider: terraform diff --git a/evals/azure-deploy/fixture/functions-terraform/host.json b/evals/azure-deploy/fixture/functions-terraform/host.json deleted file mode 100644 index d2059a46b..000000000 --- a/evals/azure-deploy/fixture/functions-terraform/host.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "version": "2.0" -} diff --git a/evals/azure-deploy/fixture/functions-terraform/infra/main.tf b/evals/azure-deploy/fixture/functions-terraform/infra/main.tf deleted file mode 100644 index 97a3ec2ce..000000000 --- a/evals/azure-deploy/fixture/functions-terraform/infra/main.tf +++ /dev/null @@ -1,88 +0,0 @@ -terraform { - required_providers { - azurerm = { - source = "hashicorp/azurerm" - version = "~> 3.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - backend "azurerm" {} -} - -provider "azurerm" { - features {} -} - -variable "environment_name" { - type = string - description = "Name of the azd environment" -} - -variable "location" { - type = string - description = "Azure region for all resources" -} - -resource "random_string" "resource_token" { - length = 13 - upper = false - special = false -} - -resource "azurerm_resource_group" "rg" { - name = "rg-${var.environment_name}" - location = var.location - tags = { "azd-env-name" = var.environment_name } -} - -resource "azurerm_storage_account" "storage" { - name = "st${random_string.resource_token.result}" - resource_group_name = azurerm_resource_group.rg.name - location = azurerm_resource_group.rg.location - account_tier = "Standard" - account_replication_type = "LRS" -} - -resource "azurerm_service_plan" "plan" { - name = "plan-${random_string.resource_token.result}" - resource_group_name = azurerm_resource_group.rg.name - location = azurerm_resource_group.rg.location - os_type = "Linux" - sku_name = "Y1" -} - -resource "azurerm_linux_function_app" "func" { - name = "func-${random_string.resource_token.result}" - resource_group_name = azurerm_resource_group.rg.name - location = azurerm_resource_group.rg.location - service_plan_id = azurerm_service_plan.plan.id - storage_account_name = azurerm_storage_account.storage.name - storage_account_access_key = azurerm_storage_account.storage.primary_access_key - https_only = true - - site_config { - application_stack { - node_version = "20" - } - } - - app_settings = { - FUNCTIONS_WORKER_RUNTIME = "node" - } - - tags = { - "azd-env-name" = var.environment_name - "azd-service-name" = "api" - } -} - -output "AZURE_LOCATION" { - value = var.location -} - -output "API_URL" { - value = "https://${azurerm_linux_function_app.func.default_hostname}" -} diff --git a/evals/azure-deploy/fixture/swa-bicep/azure.yaml b/evals/azure-deploy/fixture/swa-bicep/azure.yaml deleted file mode 100644 index eb38ccc4d..000000000 --- a/evals/azure-deploy/fixture/swa-bicep/azure.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json - -name: whiteboard-app -services: - web: - project: . - host: staticwebapp -infra: - provider: bicep diff --git a/evals/azure-deploy/fixture/swa-bicep/index.html b/evals/azure-deploy/fixture/swa-bicep/index.html deleted file mode 100644 index 1f0bc04a7..000000000 --- a/evals/azure-deploy/fixture/swa-bicep/index.html +++ /dev/null @@ -1,8 +0,0 @@ - - -test-sample-app - -

test-sample-app

- - - diff --git a/evals/azure-deploy/fixture/swa-bicep/infra/main.bicep b/evals/azure-deploy/fixture/swa-bicep/infra/main.bicep deleted file mode 100644 index f1f26b2b7..000000000 --- a/evals/azure-deploy/fixture/swa-bicep/infra/main.bicep +++ /dev/null @@ -1,34 +0,0 @@ -targetScope = 'subscription' - -@minLength(1) -@maxLength(64) -@description('Name of the environment that can be used as part of naming resource convention') -param environmentName string - -@minLength(1) -@description('Primary location for all resources') -param location string - -var tags = { 'azd-env-name': environmentName } -var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) - -resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' = { - name: 'rg-${environmentName}' - location: location - tags: tags -} - -module web './resources.bicep' = { - name: 'web' - scope: rg - params: { - location: location - tags: tags - resourceToken: resourceToken - } -} - -output AZURE_LOCATION string = location -output AZURE_TENANT_ID string = tenant().tenantId -output AZURE_RESOURCE_GROUP string = rg.name -output STATIC_WEB_APP_URL string = web.outputs.STATIC_WEB_APP_URL diff --git a/evals/azure-deploy/fixture/swa-bicep/infra/main.parameters.json b/evals/azure-deploy/fixture/swa-bicep/infra/main.parameters.json deleted file mode 100644 index bc915c431..000000000 --- a/evals/azure-deploy/fixture/swa-bicep/infra/main.parameters.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "environmentName": { - "value": "${AZURE_ENV_NAME}" - }, - "location": { - "value": "${AZURE_LOCATION}" - } - } -} diff --git a/evals/azure-deploy/fixture/swa-bicep/infra/resources.bicep b/evals/azure-deploy/fixture/swa-bicep/infra/resources.bicep deleted file mode 100644 index 2f2de33e6..000000000 --- a/evals/azure-deploy/fixture/swa-bicep/infra/resources.bicep +++ /dev/null @@ -1,18 +0,0 @@ -@description('Location for all resources') -param location string = resourceGroup().location - -param tags object = {} -param resourceToken string - -resource staticWebApp 'Microsoft.Web/staticSites@2022-09-01' = { - name: 'swa-${resourceToken}' - location: location - tags: union(tags, { 'azd-service-name': 'web' }) - sku: { - name: 'Free' - tier: 'Free' - } - properties: {} -} - -output STATIC_WEB_APP_URL string = staticWebApp.properties.defaultHostname diff --git a/evals/azure-deploy/fixture/swa-terraform/azure.yaml b/evals/azure-deploy/fixture/swa-terraform/azure.yaml deleted file mode 100644 index 4d5e39d1b..000000000 --- a/evals/azure-deploy/fixture/swa-terraform/azure.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json - -name: whiteboard-app-tf -services: - web: - project: . - host: staticwebapp -infra: - provider: terraform diff --git a/evals/azure-deploy/fixture/swa-terraform/index.html b/evals/azure-deploy/fixture/swa-terraform/index.html deleted file mode 100644 index 1f0bc04a7..000000000 --- a/evals/azure-deploy/fixture/swa-terraform/index.html +++ /dev/null @@ -1,8 +0,0 @@ - - -test-sample-app - -

test-sample-app

- - - diff --git a/evals/azure-deploy/fixture/swa-terraform/infra/main.tf b/evals/azure-deploy/fixture/swa-terraform/infra/main.tf deleted file mode 100644 index bfdf82474..000000000 --- a/evals/azure-deploy/fixture/swa-terraform/infra/main.tf +++ /dev/null @@ -1,60 +0,0 @@ -terraform { - required_providers { - azurerm = { - source = "hashicorp/azurerm" - version = "~> 3.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.0" - } - } - backend "azurerm" {} -} - -provider "azurerm" { - features {} -} - -variable "environment_name" { - type = string - description = "Name of the azd environment" -} - -variable "location" { - type = string - description = "Azure region for all resources" -} - -resource "random_string" "resource_token" { - length = 13 - upper = false - special = false -} - -resource "azurerm_resource_group" "rg" { - name = "rg-${var.environment_name}" - location = var.location - tags = { "azd-env-name" = var.environment_name } -} - -resource "azurerm_static_web_app" "swa" { - name = "swa-${random_string.resource_token.result}" - resource_group_name = azurerm_resource_group.rg.name - location = azurerm_resource_group.rg.location - sku_tier = "Free" - sku_size = "Free" - - tags = { - "azd-env-name" = var.environment_name - "azd-service-name" = "web" - } -} - -output "AZURE_LOCATION" { - value = var.location -} - -output "STATIC_WEB_APP_URL" { - value = azurerm_static_web_app.swa.default_host_name -} From c071eb337f1154f2880a7ba73ea1d283043f758b Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Tue, 21 Jul 2026 15:53:27 -0700 Subject: [PATCH 15/21] increase all max_turns to 80 --- evals/azure-deploy/deploy-eval.yaml | 42 ++++++++++++++--------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/evals/azure-deploy/deploy-eval.yaml b/evals/azure-deploy/deploy-eval.yaml index 3b317c9a4..fe2d293f3 100644 --- a/evals/azure-deploy/deploy-eval.yaml +++ b/evals/azure-deploy/deploy-eval.yaml @@ -52,7 +52,7 @@ stimuli: - name: "Deploy SWA - Whiteboard App (Bicep)" prompt: "Create a static whiteboard web app and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -80,7 +80,7 @@ stimuli: - name: "Deploy SWA - Portfolio Website (Bicep)" prompt: "Create a static portfolio website and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -113,7 +113,7 @@ stimuli: - name: "Deploy App Service - Discussion Board (Bicep)" prompt: "Create a discussion board application and deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -141,7 +141,7 @@ stimuli: - name: "Deploy App Service - Todo List (Bicep)" prompt: "Create a todo list with frontend and API and deploy it to Azure App Service using my current subscription in the westus2 region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -174,7 +174,7 @@ stimuli: - name: "Deploy Azure Functions - Serverless HTTP API (Bicep)" prompt: "Create a serverless HTTP API using Azure Functions and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -201,7 +201,7 @@ stimuli: - name: "Deploy Azure Functions - Event-Driven (Bicep)" prompt: "Create an event-driven function app to process messages and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -228,7 +228,7 @@ stimuli: - name: "Deploy Azure Functions - Python Service Bus (Bicep)" prompt: "Create an azure python function app that takes input from a service bus trigger and does message processing and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -258,9 +258,9 @@ stimuli: # Jest: "creates and deploys workflow app with Durable Task Scheduler" - name: "Deploy DTS - Workflow App (Bicep)" - prompt: "Create a workflow app that orchestrates a multi-step order processing pipeline and deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." + prompt: "Create a workflow app that orchestrates a multi-step order processing pipeline. Make the app as simple as possible for demonstration purposes only. Then deploy it to Azure using my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -301,7 +301,7 @@ stimuli: - name: "Deploy Container Apps - Containerized Web App (Bicep)" prompt: "Create a containerized web application and deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -344,7 +344,7 @@ stimuli: - name: "Deploy Container Apps - Node.js App (Bicep)" prompt: "Create a simple containerized Node.js hello world app and deploy it to Azure Container Apps using my current subscription in the swedencentral region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -384,7 +384,7 @@ stimuli: - name: "Deploy SWA - Whiteboard App (Terraform)" prompt: "Create a static whiteboard web app and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -418,7 +418,7 @@ stimuli: - name: "Deploy SWA - Portfolio Website (Terraform)" prompt: "Create a static portfolio website and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -457,7 +457,7 @@ stimuli: - name: "Deploy App Service - Discussion Board (Terraform)" prompt: "Create a discussion board application and deploy it to Azure App Service, prefer Terraform over Bicep, in my current subscription in the westus2 region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -491,7 +491,7 @@ stimuli: - name: "Deploy App Service - Todo List (Terraform)" prompt: "Create a todo list with frontend and API and deploy to Azure App Service using Terraform in my current subscription in the westus2 region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -530,7 +530,7 @@ stimuli: - name: "Deploy Azure Functions - Serverless HTTP API (Terraform)" prompt: "Create a serverless HTTP API using Azure Functions and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -563,7 +563,7 @@ stimuli: - name: "Deploy Azure Functions - Event-Driven (Terraform)" prompt: "Create an event-driven function app to process messages and deploy it to Azure Functions using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -596,7 +596,7 @@ stimuli: - name: "Deploy Azure Functions - URL Shortener (Terraform)" prompt: "Create a URL shortener service using Azure Functions that creates short links and redirects users to the original URL and deploy it to Azure using Terraform infrastructure in my current subscription in the eastus2 region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -628,7 +628,7 @@ stimuli: - name: "Deploy Container Apps - Containerized Web App (Terraform)" prompt: "Create a containerized web application and deploy it to Azure Container Apps using terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -675,7 +675,7 @@ stimuli: - name: "Deploy Container Apps - Node.js App (Terraform)" prompt: "Create a simple containerized Node.js hello world app and deploy it to Azure Container Apps using Terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full @@ -710,7 +710,7 @@ stimuli: - name: "Deploy Container Apps - Social Media App (Terraform)" prompt: "Create a simple social media application with likes and comments and deploy to Azure using Terraform infrastructure in my current subscription in the swedencentral region. Use azd as the deployment tool." constraints: - max_turns: 50 + max_turns: 80 tags: type: integration tier: full From d441c96f481f73bb25ae0f89ad3ec593e14112ac Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Wed, 22 Jul 2026 15:57:21 -0700 Subject: [PATCH 16/21] relax IaC check path --- evals/azure-deploy/deploy-eval.yaml | 58 ++++++++++++++--------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/evals/azure-deploy/deploy-eval.yaml b/evals/azure-deploy/deploy-eval.yaml index fe2d293f3..0a510ea7f 100644 --- a/evals/azure-deploy/deploy-eval.yaml +++ b/evals/azure-deploy/deploy-eval.yaml @@ -285,11 +285,11 @@ stimuli: # Verify DTS infrastructure is present in the agent-generated workspace - type: file-matches config: - path: "infra/resources.bicep" + path: "infra/**/*.bicep" pattern: "(?i)Microsoft\\.DurableTask/schedulers" - type: file-matches config: - path: "infra/resources.bicep" + path: "infra/**/*.bicep" pattern: "0ad04412-c4d5-4796-b79c-f76d14c8d402" # ═══════════════════════════════════════════ @@ -325,19 +325,19 @@ stimuli: command: "(?i)azd\\s+(up|provision)\\b" - type: file-matches config: - path: "infra/resources.bicep" + path: "infra/**/*.bicep" pattern: "(?i)Microsoft\\.App/containerApps" - type: file-matches config: - path: "infra/resources.bicep" + path: "infra/**/*.bicep" pattern: "(?i)Microsoft\\.ContainerRegistry/registries" - type: file-matches config: - path: "infra/resources.bicep" + path: "infra/**/*.bicep" pattern: "(?i)SystemAssigned" - type: file-matches config: - path: "infra/resources.bicep" + path: "infra/**/*.bicep" pattern: "7f951dda-4ed3-4680-a7ca-43fe172d538d" # Jest: "creates simple containerized Node.js app" @@ -368,11 +368,11 @@ stimuli: command: "(?i)azd\\s+(up|provision)\\b" - type: file-matches config: - path: "infra/resources.bicep" + path: "infra/**/*.bicep" pattern: "(?i)Microsoft\\.App/containerApps" - type: file-matches config: - path: "infra/resources.bicep" + path: "infra/**/*.bicep" pattern: "7f951dda-4ed3-4680-a7ca-43fe172d538d" # ═══════════════════════════════════════════ @@ -409,10 +409,10 @@ stimuli: command: "(?i)azd\\s+(up|provision)\\b" - type: file-exists config: - path: "infra/*.tf" + path: "infra/**/*.tf" - type: file-not-exists config: - path: "infra/*.bicep" + path: "infra/**/*.bicep" # Jest: "creates static portfolio website with Terraform infrastructure" - name: "Deploy SWA - Portfolio Website (Terraform)" @@ -443,10 +443,10 @@ stimuli: command: "(?i)azd\\s+(up|provision)\\b" - type: file-exists config: - path: "infra/*.tf" + path: "infra/**/*.tf" - type: file-not-exists config: - path: "infra/*.bicep" + path: "infra/**/*.bicep" # ═══════════════════════════════════════════ # terraform-app-service-deploy @@ -482,10 +482,10 @@ stimuli: command: "(?i)azd\\s+(up|provision)\\b" - type: file-exists config: - path: "infra/*.tf" + path: "infra/**/*.tf" - type: file-not-exists config: - path: "infra/*.bicep" + path: "infra/**/*.bicep" # Jest: "creates todo list with frontend and API using Terraform" - name: "Deploy App Service - Todo List (Terraform)" @@ -516,10 +516,10 @@ stimuli: command: "(?i)azd\\s+(up|provision)\\b" - type: file-exists config: - path: "infra/*.tf" + path: "infra/**/*.tf" - type: file-not-exists config: - path: "infra/*.bicep" + path: "infra/**/*.bicep" # ═══════════════════════════════════════════ # terraform-azure-functions-deploy @@ -554,10 +554,10 @@ stimuli: command: "(?i)azd\\s+(up|provision)\\b" - type: file-exists config: - path: "infra/*.tf" + path: "infra/**/*.tf" - type: file-not-exists config: - path: "infra/*.bicep" + path: "infra/**/*.bicep" # Jest: "creates event-driven function app with Terraform" - name: "Deploy Azure Functions - Event-Driven (Terraform)" @@ -587,10 +587,10 @@ stimuli: command: "(?i)azd\\s+(up|provision)\\b" - type: file-exists config: - path: "infra/*.tf" + path: "infra/**/*.tf" - type: file-not-exists config: - path: "infra/*.bicep" + path: "infra/**/*.bicep" # Jest: "creates URL shortener service with Terraform infrastructure" - name: "Deploy Azure Functions - URL Shortener (Terraform)" @@ -652,23 +652,23 @@ stimuli: command: "(?i)azd\\s+(up|provision)\\b" - type: file-matches config: - path: "infra/main.tf" + path: "infra/**/*.tf" pattern: "(?i)azurerm_container_app[^_]" - type: file-matches config: - path: "infra/main.tf" + path: "infra/**/*.tf" pattern: "(?i)azurerm_container_app_environment" - type: file-matches config: - path: "infra/main.tf" + path: "infra/**/*.tf" pattern: "(?i)azurerm_container_registry" - type: file-matches config: - path: "infra/main.tf" + path: "infra/**/*.tf" pattern: "(?i)SystemAssigned" - type: file-matches config: - path: "infra/main.tf" + path: "infra/**/*.tf" pattern: "(?i)AcrPull" # Jest: "creates simple containerized Node.js app with Terraform" @@ -699,11 +699,11 @@ stimuli: command: "(?i)azd\\s+(up|provision)\\b" - type: file-matches config: - path: "infra/main.tf" + path: "infra/**/*.tf" pattern: "(?i)azurerm_container_app[^_]" - type: file-matches config: - path: "infra/main.tf" + path: "infra/**/*.tf" pattern: "(?i)AcrPull" # Jest: "creates social media application with Terraform infrastructure" @@ -734,11 +734,11 @@ stimuli: command: "(?i)azd\\s+(up|provision)\\b" - type: file-matches config: - path: "infra/main.tf" + path: "infra/**/*.tf" pattern: "(?i)azurerm_container_app[^_]" - type: file-matches config: - path: "infra/main.tf" + path: "infra/**/*.tf" pattern: "(?i)AcrPull" # ═══════════════════════════════════════════ From 217052da9133bcf39a4504a17ee060aad97970d7 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Tue, 28 Jul 2026 11:15:55 -0700 Subject: [PATCH 17/21] remove incorrect bicep check fix acr endpoint check location --- evals/azure-deploy/deploy-eval.yaml | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/evals/azure-deploy/deploy-eval.yaml b/evals/azure-deploy/deploy-eval.yaml index 0a510ea7f..07735d89a 100644 --- a/evals/azure-deploy/deploy-eval.yaml +++ b/evals/azure-deploy/deploy-eval.yaml @@ -976,23 +976,6 @@ stimuli: required: - name: "(?i)^(bash|powershell|pwsh)$" command: "(?i)azd\\s+(up|provision)\\b" - # Verify Container Apps two-phase Bicep pattern in the agent-generated workspace - - type: file-matches - config: - path: "**/*.bicep" - pattern: "(?i)Microsoft\\.App/containerApps" - - type: file-matches - config: - path: "**/*.bicep" - pattern: "(?i)Microsoft\\.ContainerRegistry/registries" - - type: file-matches - config: - path: "**/*.bicep" - pattern: "(?i)SystemAssigned" - - type: file-matches - config: - path: "**/*.bicep" - pattern: "7f951dda-4ed3-4680-a7ca-43fe172d538d" - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" @@ -1091,7 +1074,6 @@ stimuli: pattern: "(?i)fatal error|unhandled exception|stack trace" # Jest: "deploys aspire orleans-voting" - # Also soft-checks container deploy env vars (AZURE_CONTAINER_REGISTRY_ENDPOINT, etc.). - name: "Brownfield .NET Aspire - Orleans Voting" prompt: >- Please deploy this application to Azure. @@ -1133,9 +1115,10 @@ stimuli: required: - name: "(?i)^(bash|powershell|pwsh)$" command: "(?i)azd\\s+(up|provision)\\b" - # Soft-check: container deploy env vars expected in azure.yaml or bicep - - type: output-matches + # AZD env file should have the ACR + - type: file-matches config: + path: ".azure/**/*" pattern: "(?i)AZURE_CONTAINER_REGISTRY_ENDPOINT|AZURE_CONTAINER_REGISTRY_MANAGED_IDENTITY_ID|MANAGED_IDENTITY_CLIENT_ID" - type: output-not-matches config: From bef3c39a767d58ff5aabefa388cfa5ab41e4dbd7 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Tue, 28 Jul 2026 11:20:30 -0700 Subject: [PATCH 18/21] add deployed link grader --- evals/azure-deploy/deploy-eval.yaml | 98 ++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/evals/azure-deploy/deploy-eval.yaml b/evals/azure-deploy/deploy-eval.yaml index 07735d89a..a9457d10b 100644 --- a/evals/azure-deploy/deploy-eval.yaml +++ b/evals/azure-deploy/deploy-eval.yaml @@ -75,6 +75,9 @@ stimuli: required: - name: "(?i)^(bash|powershell|pwsh)$" command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "creates static portfolio website with bicep" - name: "Deploy SWA - Portfolio Website (Bicep)" @@ -103,6 +106,9 @@ stimuli: required: - name: "(?i)^(bash|powershell|pwsh)$" command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # ═══════════════════════════════════════════ # vanilla-app-service-deploy (Bicep) @@ -136,6 +142,9 @@ stimuli: required: - name: "(?i)^(bash|powershell|pwsh)$" command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "creates todo list with frontend and API" - name: "Deploy App Service - Todo List (Bicep)" @@ -164,6 +173,9 @@ stimuli: required: - name: "(?i)^(bash|powershell|pwsh)$" command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # ═══════════════════════════════════════════ # vanilla-azure-functions-deploy (Bicep) @@ -196,6 +208,9 @@ stimuli: required: - name: "(?i)^(bash|powershell|pwsh)$" command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "creates event-driven function app" - name: "Deploy Azure Functions - Event-Driven (Bicep)" @@ -223,6 +238,9 @@ stimuli: required: - name: "(?i)^(bash|powershell|pwsh)$" command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "creates Python function app with Service Bus trigger" - name: "Deploy Azure Functions - Python Service Bus (Bicep)" @@ -250,6 +268,9 @@ stimuli: required: - name: "(?i)^(bash|powershell|pwsh)$" command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # ═══════════════════════════════════════════ # durable-task-scheduler-deploy (Bicep) @@ -291,6 +312,9 @@ stimuli: config: path: "infra/**/*.bicep" pattern: "0ad04412-c4d5-4796-b79c-f76d14c8d402" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # ═══════════════════════════════════════════ # vanilla-azure-container-apps-deploy (Bicep) @@ -339,6 +363,9 @@ stimuli: config: path: "infra/**/*.bicep" pattern: "7f951dda-4ed3-4680-a7ca-43fe172d538d" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "creates simple containerized Node.js app" - name: "Deploy Container Apps - Node.js App (Bicep)" @@ -374,6 +401,9 @@ stimuli: config: path: "infra/**/*.bicep" pattern: "7f951dda-4ed3-4680-a7ca-43fe172d538d" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # ═══════════════════════════════════════════ # terraform-static-web-apps-deploy @@ -413,6 +443,9 @@ stimuli: - type: file-not-exists config: path: "infra/**/*.bicep" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "creates static portfolio website with Terraform infrastructure" - name: "Deploy SWA - Portfolio Website (Terraform)" @@ -447,6 +480,9 @@ stimuli: - type: file-not-exists config: path: "infra/**/*.bicep" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # ═══════════════════════════════════════════ # terraform-app-service-deploy @@ -486,6 +522,9 @@ stimuli: - type: file-not-exists config: path: "infra/**/*.bicep" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "creates todo list with frontend and API using Terraform" - name: "Deploy App Service - Todo List (Terraform)" @@ -520,6 +559,9 @@ stimuli: - type: file-not-exists config: path: "infra/**/*.bicep" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # ═══════════════════════════════════════════ # terraform-azure-functions-deploy @@ -558,6 +600,9 @@ stimuli: - type: file-not-exists config: path: "infra/**/*.bicep" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "creates event-driven function app with Terraform" - name: "Deploy Azure Functions - Event-Driven (Terraform)" @@ -591,6 +636,9 @@ stimuli: - type: file-not-exists config: path: "infra/**/*.bicep" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "creates URL shortener service with Terraform infrastructure" - name: "Deploy Azure Functions - URL Shortener (Terraform)" @@ -618,6 +666,9 @@ stimuli: required: - name: "(?i)^(bash|powershell|pwsh)$" command: "(?i)azd\\s+(up|provision)\\b" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # ═══════════════════════════════════════════ # terraform-azure-container-apps-deploy @@ -670,6 +721,9 @@ stimuli: config: path: "infra/**/*.tf" pattern: "(?i)AcrPull" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "creates simple containerized Node.js app with Terraform" - name: "Deploy Container Apps - Node.js App (Terraform)" @@ -705,6 +759,9 @@ stimuli: config: path: "infra/**/*.tf" pattern: "(?i)AcrPull" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "creates social media application with Terraform infrastructure" - name: "Deploy Container Apps - Social Media App (Terraform)" @@ -740,6 +797,9 @@ stimuli: config: path: "infra/**/*.tf" pattern: "(?i)AcrPull" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # ═══════════════════════════════════════════ # brownfield-dotnet @@ -791,6 +851,9 @@ stimuli: - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "deploys MvcMovie 10" - name: "Brownfield .NET - MvcMovie 10" @@ -838,6 +901,9 @@ stimuli: - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "deploys aspire azure functions" - name: "Brownfield .NET Aspire - Azure Functions" @@ -885,6 +951,9 @@ stimuli: - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "deploys aspire client apps integration" - name: "Brownfield .NET Aspire - Client Apps Integration" @@ -932,6 +1001,9 @@ stimuli: - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "deploys aspire container build" # Checks Container Apps two-phase Bicep pattern in the agent-generated workspace. @@ -979,6 +1051,9 @@ stimuli: - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "does not deploy aspire custom resources" # The agent MUST NOT attempt deployment — custom Aspire resources are unsupported. @@ -1072,6 +1147,9 @@ stimuli: - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "deploys aspire orleans-voting" - name: "Brownfield .NET Aspire - Orleans Voting" @@ -1123,6 +1201,9 @@ stimuli: - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # ═══════════════════════════════════════════ # brownfield-javascript @@ -1169,6 +1250,9 @@ stimuli: - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "deploys aspire with javascript" - name: "Brownfield .NET Aspire - JavaScript" @@ -1216,6 +1300,9 @@ stimuli: - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "deploys aspire with node" - name: "Brownfield .NET Aspire - Node" @@ -1263,6 +1350,9 @@ stimuli: - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # ═══════════════════════════════════════════ # brownfield-python @@ -1309,6 +1399,9 @@ stimuli: - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" # Jest: "deploys aspire with python" - name: "Brownfield .NET Aspire - Python" @@ -1355,4 +1448,7 @@ stimuli: command: "(?i)azd\\s+(up|provision)\\b" - type: output-not-matches config: - pattern: "(?i)fatal error|unhandled exception|stack trace" \ No newline at end of file + pattern: "(?i)fatal error|unhandled exception|stack trace" + - type: output-matches + config: + pattern: "https?://[\\w.-]+(\\.azurewebsites\\.net|\\.azurestaticapps\\.net|\\.azurecontainerapps\\.io|\\.web\\.core\\.windows\\.net)" From 0601b3737d56d6c9cfade1ed394e88c6f2553af0 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Thu, 30 Jul 2026 14:01:36 -0700 Subject: [PATCH 19/21] clear eval suites --- .../azure-deploy/deploy-eval.yaml | 2 +- evals/azure-skills/azure-deploy/eval.yaml | 156 ------------------ .../azure-deploy/output-eval.yaml | 2 +- 3 files changed, 2 insertions(+), 158 deletions(-) delete mode 100644 evals/azure-skills/azure-deploy/eval.yaml diff --git a/evals/azure-skills/azure-deploy/deploy-eval.yaml b/evals/azure-skills/azure-deploy/deploy-eval.yaml index a9457d10b..4c20c371f 100644 --- a/evals/azure-skills/azure-deploy/deploy-eval.yaml +++ b/evals/azure-skills/azure-deploy/deploy-eval.yaml @@ -16,7 +16,7 @@ # the deployment completed and the app is live. # NOTE: earlyTerminate tests must NOT use the `completed` grader. -name: azure-deploy-e2e-eval +name: azure-deploy-deploy-eval description: | End-to-end evaluation suite for the azure-deploy skill. Tests deployment execution quality across: diff --git a/evals/azure-skills/azure-deploy/eval.yaml b/evals/azure-skills/azure-deploy/eval.yaml deleted file mode 100644 index 65eedba00..000000000 --- a/evals/azure-skills/azure-deploy/eval.yaml +++ /dev/null @@ -1,156 +0,0 @@ -# Vally eval config — migrated from Waza -# Source: tests/azure-deploy/eval/eval.yaml + tasks/*.yaml -# Migration: Waza → Vally (issue #1817, Phase 1) -# -# Global graders (evaluate#125 workaround): has_output and no_runtime_failure -# are duplicated into every stimulus block below. - -name: azure-deploy-eval -description: | - Evaluation suite for the azure-deploy skill. - Tests deployment guidance quality with emphasis on: - - AVM+AZD pattern-module preference - - AVM fallback when no pattern module exists - - deploy-only routing (not prepare/validate) - -tags: - type: integration - skill: azure-deploy - -defaults: - runs: 1 - timeout: "7m" - executor: integration-test-agent-runner - model: claude-sonnet-4.6 - -scoring: - threshold: 0.8 - weights: - output-contains: 0.125 - output-matches: 0.125 - output-not-matches: 0.5 - completed: 0.25 - -stimuli: - # ── avm-order-bicep-001 ── - # Waza source: tasks/avm-order-bicep.yaml - # Validates deploy guidance prefers AVM+AZD pattern modules first - - name: "AVM+AZD Priority - Bicep Deploy" - prompt: | - My app is already prepared and validated. - Give me deploy guidance and module preference order for Bicep. - Prefer AVM+AZD patterns where available, with fallback to AVM resource modules and AVM utility modules. - Use azd as the deployment tool. - tags: - type: integration - skill: azure-deploy - tier: full - cost: llm - area: output - graders: - # Task: expected.output_contains - - type: output-contains - config: - substring: "AVM" - - type: output-contains - config: - substring: "deploy" - - type: output-contains - config: - substring: "pattern" - # Task grader: avm_pattern_first - - type: output-matches - config: - pattern: "(?i)AVM\\+AZD|AZD pattern|pattern modules" - # Task grader: includes_resource_and_utility_fallback - - type: output-matches - config: - pattern: "(?is)(AVM\\+AZD|AZD pattern|pattern modules).*resource modules.*utility modules" - # Global: has_output - - type: completed - # Global: no_runtime_failure - - type: output-not-matches - config: - pattern: "(?i)fatal error|exception occurred|crashed" - - # ── avm-fallback-no-pattern-001 ── - # Waza source: tasks/avm-fallback-no-pattern.yaml - # Validates fallback order when no AVM+AZD pattern module exists - - name: "AVM Fallback When No AZD Pattern" - prompt: | - I'm deploying with Bicep and there is no AVM+AZD pattern module for my scenario. - What module order should I follow if no pattern module exists and fallback must stay AVM resource modules then AVM utility modules? - Use azd as the deployment tool. - tags: - type: integration - skill: azure-deploy - tier: full - cost: llm - area: output - graders: - # Task: expected.output_contains - - type: output-contains - config: - substring: "AVM" - - type: output-contains - config: - substring: "resource" - - type: output-contains - config: - substring: "utility" - # Task grader: explicit_no_pattern_fallback - - type: output-matches - config: - pattern: "(?is)(no .*pattern module|if no .*pattern).*AVM.*resource.*AVM.*utility" - # Task grader: avoids_non_avm_fallback - - type: output-not-matches - config: - pattern: "(?i)fallback to non-AVM|use non-AVM modules" - # Global: has_output - - type: completed - # Global: no_runtime_failure - - type: output-not-matches - config: - pattern: "(?i)fatal error|exception occurred|crashed" - - # ── avm-azd-pattern-preference-001 ── - # Source: tests/azure-deploy/avm/integration.test.ts - # describe("avm-azd-pattern-preference") - # Validates that AZD pattern modules are preferred over AVM resource modules - # when setting up azd infrastructure with Bicep for a container app. - - name: "AZD Pattern Module Priority - Container App" - prompt: | - Set up azd infrastructure with Bicep for a container app. - Use AVM modules and prefer AZD pattern modules over resource modules. - Use azd as the deployment tool. - tags: - type: integration - skill: azure-deploy - tier: full - cost: llm - area: output - graders: - - type: output-contains - config: - substring: "AZD" - - type: output-contains - config: - substring: "pattern" - # Task grader: avm_azd_pattern_preferred — matches "AZD pattern", "AVM+AZD", or "pattern modules" - - type: output-matches - config: - pattern: "(?i)AZD\\s+pattern|AVM\\+AZD|pattern modules?" - # Task grader: context_present — AVM/Bicep/container/module present - - type: output-matches - config: - pattern: "(?i)(container|bicep|module)" - # Task grader: pattern_before_resource — pattern modules discussed before resource modules - - type: output-matches - config: - pattern: "(?is)(AZD pattern|AVM\\+AZD|pattern modules?).*resource modules?" - # Global: has_output - - type: completed - # Global: no_runtime_failure - - type: output-not-matches - config: - pattern: "(?i)fatal error|exception occurred|crashed" \ No newline at end of file diff --git a/evals/azure-skills/azure-deploy/output-eval.yaml b/evals/azure-skills/azure-deploy/output-eval.yaml index 65eedba00..e8711386c 100644 --- a/evals/azure-skills/azure-deploy/output-eval.yaml +++ b/evals/azure-skills/azure-deploy/output-eval.yaml @@ -5,7 +5,7 @@ # Global graders (evaluate#125 workaround): has_output and no_runtime_failure # are duplicated into every stimulus block below. -name: azure-deploy-eval +name: azure-deploy-output-eval description: | Evaluation suite for the azure-deploy skill. Tests deployment guidance quality with emphasis on: From 11e04d4cfbcba588669b43e8bd5eb74d1b86349f Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Thu, 30 Jul 2026 14:03:30 -0700 Subject: [PATCH 20/21] disallow provision command for no deploy test --- evals/azure-skills/azure-deploy/deploy-eval.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/azure-skills/azure-deploy/deploy-eval.yaml b/evals/azure-skills/azure-deploy/deploy-eval.yaml index 4c20c371f..eb03bc2b5 100644 --- a/evals/azure-skills/azure-deploy/deploy-eval.yaml +++ b/evals/azure-skills/azure-deploy/deploy-eval.yaml @@ -1097,7 +1097,7 @@ stimuli: config: disallowed: - name: "(?i)^(bash|powershell|pwsh)$" - command: "(?i)azd\\s+(up|deploy)\\b" + command: "(?i)azd\\s+(up|deploy|provision)\\b" - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" From 88b0fc792cc3a503d5fda4b38f2872e229761b6d Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Thu, 30 Jul 2026 14:13:16 -0700 Subject: [PATCH 21/21] properly dispatch azure-deploy test runs --- .github/workflows/test-all-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-all-integration.yml b/.github/workflows/test-all-integration.yml index 0efb83bc9..a1a34beda 100644 --- a/.github/workflows/test-all-integration.yml +++ b/.github/workflows/test-all-integration.yml @@ -156,7 +156,7 @@ jobs: core.setOutput("has_azure_deploy", hasAzureDeploy.toString()); // Build matrix excluding azure-deploy - const matrix = skills.filter(s => s !== "azure-deploy"); + const matrix = skills.filter(s => s !== "azure-skills/azure-deploy"); core.setOutput("matrix", JSON.stringify(matrix)); # Call the azure-deploy workflow when azure-deploy is in the skills list