From 68aba5433c7ee5f08e4804542f4f1e3b9642f528 Mon Sep 17 00:00:00 2001 From: "Tom Meschter (from Dev Box)" Date: Wed, 15 Jul 2026 15:38:26 -0700 Subject: [PATCH 1/2] feat: replace azure-validate AZCLI/Bicep validation steps with a shared script Fixes microsoft/GitHub-Copilot-for-Azure#2503. Adds validate-deployment.{sh,ps1} (bash + PowerShell) that runs the fixed az validation sequence (CLI present, authenticated, bicep build, deployment validate, what-if) with per-step PASS/FAIL, a what-if change summary, and a non-zero exit on failure. Both the AZCLI and Bicep recipes now reference this single shared helper instead of duplicating the inline command steps. Adds a standalone-Bicep eval fixture and stimulus so coverage actually exercises the script. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: a000b508-d45c-4d9e-85f9-394275303cac --- evals/azure-validate/e2e-eval.yaml | 47 +++++ .../fixture/bicep-cli/index.html | 15 ++ .../fixture/bicep-cli/infra/main.bicep | 32 +++ .../bicep-cli/infra/main.parameters.json | 12 ++ .../fixture/bicep-cli/infra/resources.bicep | 18 ++ .../references/recipes/azcli/README.md | 92 +++----- .../references/recipes/bicep/README.md | 77 +++---- .../recipes/scripts/validate-deployment.ps1 | 170 +++++++++++++++ .../recipes/scripts/validate-deployment.sh | 197 ++++++++++++++++++ 9 files changed, 553 insertions(+), 107 deletions(-) create mode 100644 evals/azure-validate/fixture/bicep-cli/index.html create mode 100644 evals/azure-validate/fixture/bicep-cli/infra/main.bicep create mode 100644 evals/azure-validate/fixture/bicep-cli/infra/main.parameters.json create mode 100644 evals/azure-validate/fixture/bicep-cli/infra/resources.bicep create mode 100644 plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.ps1 create mode 100755 plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.sh diff --git a/evals/azure-validate/e2e-eval.yaml b/evals/azure-validate/e2e-eval.yaml index 5f496e9bf..50142c0cd 100644 --- a/evals/azure-validate/e2e-eval.yaml +++ b/evals/azure-validate/e2e-eval.yaml @@ -178,6 +178,53 @@ stimuli: - name: "(?i)^(bash|powershell|pwsh)$" command: "(?i)azd provision --no-prompt" + # Jest: "runs the validate-deployment script for a standalone Bicep (az CLI) project" + # No azure.yaml → azure-prepare selects the Bicep/AZCLI recipe, whose validation is the + # shared validate-deployment.{sh,ps1} helper. The agent must RUN that script (not azd, + # not raw az deployment create). + - name: "Runs validate-deployment Script - Standalone Bicep (az CLI)" + prompt: "My static status page is ready. I deploy it directly with the Azure CLI (az deployment) using my Bicep templates in ./infra — I do NOT use azd. Please prepare and validate it for deployment to my current subscription in eastus2." + environment: + files: + - src: fixture/bicep-cli/index.html + dest: index.html + - src: fixture/bicep-cli/infra/main.bicep + dest: infra/main.bicep + - src: fixture/bicep-cli/infra/resources.bicep + dest: infra/resources.bicep + - src: fixture/bicep-cli/infra/main.parameters.json + dest: infra/main.parameters.json + constraints: + max_turns: 50 + tags: + type: integration + tier: full + cost: llm + area: behavior + skill: azure-validate + followUp: + - "Continue with recommended options until complete." + earlyTerminate: '[{"type":"skill-call","skill":"azure-deploy"},{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"validate-deployment\\.(sh|ps1)"},{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"az\\s+deployment\\b.*\\b(create|up)\\b"}]' + graders: + # azure-validate WAS invoked; azure-deploy was NOT + - type: skill-invocation + config: + required: + - azure-validate + disallowed: + - azure-deploy + # The shared validate-deployment script WAS run; no actual deployment happened + - type: tool-calls + config: + required: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)validate-deployment\\.(sh|ps1)" + disallowed: + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)azd\\s+(up|deploy)\\b" + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)az\\s+deployment\\b.*\\bcreate\\b" + # ═══════════════════════════════════════════ # brownfield-dotnet-validate # Source: describe("brownfield-dotnet-validate") diff --git a/evals/azure-validate/fixture/bicep-cli/index.html b/evals/azure-validate/fixture/bicep-cli/index.html new file mode 100644 index 000000000..51394e1b4 --- /dev/null +++ b/evals/azure-validate/fixture/bicep-cli/index.html @@ -0,0 +1,15 @@ + + +Status Page + +

Service Status

+
+

API

Operational

+

Web

Operational

+
+ + diff --git a/evals/azure-validate/fixture/bicep-cli/infra/main.bicep b/evals/azure-validate/fixture/bicep-cli/infra/main.bicep new file mode 100644 index 000000000..50db4acd2 --- /dev/null +++ b/evals/azure-validate/fixture/bicep-cli/infra/main.bicep @@ -0,0 +1,32 @@ +targetScope = 'subscription' + +@minLength(1) +@maxLength(64) +@description('Name of the environment that can be used as part of naming resource convention') +param environmentName string + +@minLength(1) +@description('Primary location for all resources') +param location string + +var tags = { 'azd-env-name': environmentName } +var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) + +resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' = { + name: 'rg-${environmentName}' + location: location + tags: tags +} + +module web './resources.bicep' = { + name: 'web' + scope: rg + params: { + location: location + tags: tags + resourceToken: resourceToken + } +} + +output AZURE_LOCATION string = location +output AZURE_TENANT_ID string = tenant().tenantId diff --git a/evals/azure-validate/fixture/bicep-cli/infra/main.parameters.json b/evals/azure-validate/fixture/bicep-cli/infra/main.parameters.json new file mode 100644 index 000000000..071681447 --- /dev/null +++ b/evals/azure-validate/fixture/bicep-cli/infra/main.parameters.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "environmentName": { + "value": "statuspage" + }, + "location": { + "value": "eastus2" + } + } +} diff --git a/evals/azure-validate/fixture/bicep-cli/infra/resources.bicep b/evals/azure-validate/fixture/bicep-cli/infra/resources.bicep new file mode 100644 index 000000000..2f2de33e6 --- /dev/null +++ b/evals/azure-validate/fixture/bicep-cli/infra/resources.bicep @@ -0,0 +1,18 @@ +@description('Location for all resources') +param location string = resourceGroup().location + +param tags object = {} +param resourceToken string + +resource staticWebApp 'Microsoft.Web/staticSites@2022-09-01' = { + name: 'swa-${resourceToken}' + location: location + tags: union(tags, { 'azd-service-name': 'web' }) + sku: { + name: 'Free' + tier: 'Free' + } + properties: {} +} + +output STATIC_WEB_APP_URL string = staticWebApp.properties.defaultHostname diff --git a/plugin/skills/azure-validate/references/recipes/azcli/README.md b/plugin/skills/azure-validate/references/recipes/azcli/README.md index b2553f381..e6f198f5b 100644 --- a/plugin/skills/azure-validate/references/recipes/azcli/README.md +++ b/plugin/skills/azure-validate/references/recipes/azcli/README.md @@ -9,84 +9,54 @@ Validation steps for Azure CLI deployments. ## Validation Steps -- [ ] 1. Azure CLI Installation -- [ ] 2. Authentication -- [ ] 3. Bicep Compilation -- [ ] 4. Template Validation -- [ ] 5. What-If Preview -- [ ] 6. Docker Build (if containerized) -- [ ] 7. Azure Policy Validation +- [ ] 1. Core Validation (CLI, auth, build, validate, what-if) — run [`validate-deployment` script](../scripts/validate-deployment.sh) +- [ ] 2. Docker Build (if containerized) +- [ ] 3. Azure Policy Validation ## Validation Details -### 1. Azure CLI Installation +### 1. Core Validation Script -Verify Azure CLI is installed: +The core validation checks are a fixed, deterministic sequence. Run the shared +**validate-deployment** helper instead of executing and parsing each command by hand. It +confirms the Azure CLI is installed and authenticated, compiles the Bicep template +(`az bicep build`), validates it against the target scope (`az deployment ... validate`), +and runs a what-if preview — printing a compact PASS/FAIL summary plus a what-if change +count (Create/Modify/Delete). -```bash -az version -``` - -**If not installed:** -``` -mcp_azure_mcp_extension_cli_install(cli-type: "az") -``` - -### 2. Authentication +- Bash: [`../scripts/validate-deployment.sh`](../scripts/validate-deployment.sh) +- PowerShell: [`../scripts/validate-deployment.ps1`](../scripts/validate-deployment.ps1) -```bash -az account show -``` +**Subscription scope:** -**If not logged in:** ```bash -az login +../scripts/validate-deployment.sh --scope sub --location ``` - -**Set subscription:** -```bash -az account set --subscription +```powershell +../scripts/validate-deployment.ps1 -Scope sub -Location ``` -### 3. Bicep Compilation +**Resource group scope:** ```bash -az bicep build --file ./infra/main.bicep +../scripts/validate-deployment.sh --scope group --resource-group ``` - -### 4. Template Validation - -```bash -# Subscription scope -az deployment sub validate \ - --location \ - --template-file ./infra/main.bicep \ - --parameters ./infra/main.parameters.json - -# Resource group scope -az deployment group validate \ - --resource-group \ - --template-file ./infra/main.bicep \ - --parameters ./infra/main.parameters.json +```powershell +../scripts/validate-deployment.ps1 -Scope group -ResourceGroup ``` -### 5. What-If Preview +Defaults: `--template ./infra/main.bicep`, `--parameters ./infra/main.parameters.json` +(skipped if absent). Pass `--subscription ` to target a specific subscription. -```bash -# Subscription scope -az deployment sub what-if \ - --location \ - --template-file ./infra/main.bicep \ - --parameters ./infra/main.parameters.json - -# Resource group scope -az deployment group what-if \ - --resource-group \ - --template-file ./infra/main.bicep \ - --parameters ./infra/main.parameters.json -``` +**Interpreting results:** + +- `OVERALL: PASS` — all five checks passed; record the summary in Section 7 (Validation Proof). +- Any step `FAIL` — the script prints the failing command's error. Remediate: + - **Authenticated** fails → `az login`, then `az account set --subscription `. + - **Azure CLI installed** fails → install via `mcp_azure_mcp_extension_cli_install(cli-type: "az")`. + - Otherwise see [Error handling](./errors.md). -### 6. Docker Build (if containerized) +### 2. Docker Build (if containerized) **Before building**, validate the Docker build context: @@ -105,7 +75,7 @@ npm install --package-lock-only docker build -t :test ./src/ ``` -### 7. Azure Policy Validation +### 3. Azure Policy Validation See [Policy Validation Guide](../../policy-validation.md) for instructions on retrieving and validating Azure policies for your subscription. diff --git a/plugin/skills/azure-validate/references/recipes/bicep/README.md b/plugin/skills/azure-validate/references/recipes/bicep/README.md index 3789938ed..4fb13b7f6 100644 --- a/plugin/skills/azure-validate/references/recipes/bicep/README.md +++ b/plugin/skills/azure-validate/references/recipes/bicep/README.md @@ -10,64 +10,52 @@ Validation steps for standalone Bicep deployments. ## Validation Steps -- [ ] 1. Bicep Compilation -- [ ] 2. Template Validation -- [ ] 3. What-If Preview -- [ ] 4. Authentication -- [ ] 5. Linting (optional) -- [ ] 6. Azure Policy Validation +- [ ] 1. Core Validation (build, validate, what-if, auth) — run [`validate-deployment` script](../scripts/validate-deployment.sh) +- [ ] 2. Linting (optional) +- [ ] 3. Azure Policy Validation ## Validation Details -### 1. Bicep Compilation +### 1. Core Validation Script -```bash -az bicep build --file ./infra/main.bicep -``` +The core validation checks are a fixed, deterministic sequence — identical to the AZCLI +recipe. Run the shared **validate-deployment** helper instead of executing and parsing each +command by hand. It confirms the Azure CLI is installed and authenticated, compiles the Bicep +template (`az bicep build`), validates it against the target scope (`az deployment ... +validate`), and runs a what-if preview — printing a compact PASS/FAIL summary plus a what-if +change count (Create/Modify/Delete). -**Pass:** No output (compiles cleanly) -**Fail:** Shows line numbers and errors +- Bash: [`../scripts/validate-deployment.sh`](../scripts/validate-deployment.sh) +- PowerShell: [`../scripts/validate-deployment.ps1`](../scripts/validate-deployment.ps1) -### 2. Template Validation +**Subscription scope:** ```bash -# Subscription scope -az deployment sub validate \ - --location \ - --template-file ./infra/main.bicep \ - --parameters ./infra/main.parameters.json - -# Resource group scope -az deployment group validate \ - --resource-group \ - --template-file ./infra/main.bicep \ - --parameters ./infra/main.parameters.json +../scripts/validate-deployment.sh --scope sub --location +``` +```powershell +../scripts/validate-deployment.ps1 -Scope sub -Location ``` -### 3. What-If Preview +**Resource group scope:** ```bash -az deployment sub what-if \ - --location \ - --template-file ./infra/main.bicep \ - --parameters ./infra/main.parameters.json -``` - -**Expected output:** +../scripts/validate-deployment.sh --scope group --resource-group ``` -Resource and property changes are indicated with these symbols: - + Create - ~ Modify - - Delete +```powershell +../scripts/validate-deployment.ps1 -Scope group -ResourceGroup ``` -### 4. Authentication +Defaults: `--template ./infra/main.bicep`, `--parameters ./infra/main.parameters.json` +(skipped if absent). Pass `--subscription ` to target a specific subscription. -```bash -az account show -``` +**Interpreting results:** + +- `OVERALL: PASS` — all checks passed; record the summary in Section 7 (Validation Proof). +- Any step `FAIL` — the script prints the failing command's error. If **Authenticated** fails, + run `az login`. Otherwise see [Error handling](./errors.md). -### 5. Linting (optional) +### 2. Linting (optional) Use Bicep linter rules: @@ -75,7 +63,7 @@ Use Bicep linter rules: az bicep lint --file ./infra/main.bicep ``` -### 6. Azure Policy Validation +### 3. Azure Policy Validation See [Policy Validation Guide](../../policy-validation.md) for instructions on retrieving and validating Azure policies for your subscription. @@ -83,10 +71,7 @@ See [Policy Validation Guide](../../policy-validation.md) for instructions on re | Check | Command | Pass | |-------|---------|------| -| Bicep compiles | `az bicep build` | ☐ | -| Template valid | `az deployment validate` | ☐ | -| What-if passes | `az deployment what-if` | ☐ | -| Auth valid | `az account show` | ☐ | +| Core validation (build, validate, what-if, auth) | `validate-deployment` script | ☐ | | Policies validated | MCP Policy tool | ☐ | ## References diff --git a/plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.ps1 b/plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.ps1 new file mode 100644 index 000000000..de43133cf --- /dev/null +++ b/plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.ps1 @@ -0,0 +1,170 @@ +<# +.SYNOPSIS + Runs the standard Azure CLI pre-deployment validation sequence for a Bicep + template and reports PASS/FAIL for each step. Shared by the AZCLI and Bicep + validation recipes. +.DESCRIPTION + Executes, in order: + 1. az version - Azure CLI is installed + 2. az account show - authenticated to Azure + 3. az bicep build - template compiles cleanly + 4. az deployment ... validate - template validates against the target scope + 5. az deployment ... what-if - preview changes (Create/Modify/Delete summary) + Emits a per-step PASS/FAIL summary and an OVERALL result. Exits 1 if any step fails. +.PARAMETER Scope + Deployment scope: 'sub' or 'group' (required). +.PARAMETER Location + Location (required when -Scope sub). +.PARAMETER ResourceGroup + Resource group name (required when -Scope group). +.PARAMETER Template + Bicep template path. Default: ./infra/main.bicep +.PARAMETER Parameters + Parameters file path. Default: ./infra/main.parameters.json + (skipped automatically if the file does not exist). +.PARAMETER Subscription + Subscription to target (optional). +.EXAMPLE + .\validate-deployment.ps1 -Scope sub -Location eastus +.EXAMPLE + .\validate-deployment.ps1 -Scope group -ResourceGroup my-rg ` + -Template ./infra/main.bicep -Parameters ./infra/main.parameters.json +#> +param( + [ValidateSet("sub", "group")][string]$Scope, + [string]$Location, + [string]$ResourceGroup, + [string]$Template = "./infra/main.bicep", + [string]$Parameters = "./infra/main.parameters.json", + [string]$Subscription +) + +# Validate arguments +if (-not $Scope) { + Write-Error "-Scope is required and must be 'sub' or 'group'." + exit 2 +} +if ($Scope -eq "sub" -and -not $Location) { + Write-Error "-Location is required when -Scope is 'sub'." + exit 2 +} +if ($Scope -eq "group" -and -not $ResourceGroup) { + Write-Error "-ResourceGroup is required when -Scope is 'group'." + exit 2 +} + +# Build shared argument arrays +$subArgs = @() +if ($Subscription) { $subArgs = @("--subscription", $Subscription) } + +$paramArgs = @() +if (Test-Path $Parameters) { + $paramArgs = @("--parameters", $Parameters) +} else { + Write-Host "NOTE: parameters file '$Parameters' not found; validating without --parameters." +} + +if ($Scope -eq "sub") { + $scopeTargetArgs = @("--location", $Location) + $scopeDesc = "subscription (location: $Location)" +} else { + $scopeTargetArgs = @("--resource-group", $ResourceGroup) + $scopeDesc = "resource group '$ResourceGroup'" +} + +# Track results +$steps = [System.Collections.ArrayList]@() +$overall = 0 + +function Add-Result([string]$Name, [string]$Result) { + [void]$steps.Add([PSCustomObject]@{ Step = $Name; Result = $Result }) + if ($Result -ne "PASS") { $script:overall = 1 } +} + +Write-Host "=== Azure deployment validation ===" +Write-Host "Template: $Template" +Write-Host "Scope: $scopeDesc" +Write-Host "" + +# Step 1: Azure CLI installed +Write-Host "--- Step 1: Azure CLI installed (az version) ---" +az version *> $null +if ($LASTEXITCODE -eq 0) { + Write-Host "PASS: Azure CLI is installed." + Add-Result "Azure CLI installed" "PASS" +} else { + Write-Host "FAIL: Azure CLI not found. Install it, then re-run." + Add-Result "Azure CLI installed" "FAIL" + Write-Host "" + Write-Host "=== Summary ===" + $steps | Format-Table -AutoSize | Out-String | Write-Host + Write-Host "OVERALL: FAIL" + exit 1 +} +Write-Host "" + +# Step 2: Authenticated +Write-Host "--- Step 2: Authenticated (az account show) ---" +$accountJson = az account show @subArgs -o json 2>$null +if ($accountJson) { + $accountName = ($accountJson | ConvertFrom-Json).name + Write-Host "PASS: Authenticated (subscription: $accountName)." + Add-Result "Authenticated" "PASS" +} else { + Write-Host "FAIL: Not logged in. Run 'az login' (and 'az account set --subscription ')." + Add-Result "Authenticated" "FAIL" +} +Write-Host "" + +# Step 3: Bicep compilation +Write-Host "--- Step 3: Bicep compilation (az bicep build) ---" +$buildOutput = az bicep build --file $Template 2>&1 +if ($LASTEXITCODE -eq 0) { + Write-Host "PASS: Template compiles cleanly." + Add-Result "Bicep compilation" "PASS" +} else { + Write-Host "FAIL: Bicep compilation errors:" + Write-Host ($buildOutput | Out-String) + Add-Result "Bicep compilation" "FAIL" +} +Write-Host "" + +# Step 4: Template validation +Write-Host "--- Step 4: Template validation (az deployment $Scope validate) ---" +$validateOutput = az deployment $Scope validate @scopeTargetArgs --template-file $Template @paramArgs @subArgs 2>&1 +if ($LASTEXITCODE -eq 0) { + Write-Host "PASS: Template validated against the target scope." + Add-Result "Template validation" "PASS" +} else { + Write-Host "FAIL: Template validation errors:" + Write-Host ($validateOutput | Out-String) + Add-Result "Template validation" "FAIL" +} +Write-Host "" + +# Step 5: What-if preview +Write-Host "--- Step 5: What-if preview (az deployment $Scope what-if) ---" +$whatifOutput = az deployment $Scope what-if @scopeTargetArgs --template-file $Template @paramArgs @subArgs 2>&1 +if ($LASTEXITCODE -eq 0) { + $lines = $whatifOutput | Out-String -Stream + $createCount = ($lines | Where-Object { $_ -match '^\s*\+ ' }).Count + $modifyCount = ($lines | Where-Object { $_ -match '^\s*~ ' }).Count + $deleteCount = ($lines | Where-Object { $_ -match '^\s*- ' }).Count + Write-Host "PASS: What-if completed. Changes -> Create: $createCount, Modify: $modifyCount, Delete: $deleteCount" + Add-Result "What-if preview" "PASS" +} else { + Write-Host "FAIL: What-if errors:" + Write-Host ($whatifOutput | Out-String) + Add-Result "What-if preview" "FAIL" +} +Write-Host "" + +# Summary +Write-Host "=== Summary ===" +$steps | Format-Table -AutoSize | Out-String | Write-Host +if ($overall -eq 0) { + Write-Host "OVERALL: PASS" +} else { + Write-Host "OVERALL: FAIL" +} +exit $overall diff --git a/plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.sh b/plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.sh new file mode 100755 index 000000000..33e2546e9 --- /dev/null +++ b/plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.sh @@ -0,0 +1,197 @@ +#!/usr/bin/env bash +# validate-deployment.sh +# Runs the standard Azure CLI pre-deployment validation sequence for a Bicep +# template and reports PASS/FAIL for each step. Shared by the AZCLI and Bicep +# validation recipes. +# +# Steps (in order): +# 1. az version - Azure CLI is installed +# 2. az account show - authenticated to Azure +# 3. az bicep build - template compiles cleanly +# 4. az deployment ... validate - template validates against the target scope +# 5. az deployment ... what-if - preview changes (with a Create/Modify/Delete summary) +# +# Usage: +# ./validate-deployment.sh --scope sub --location [options] +# ./validate-deployment.sh --scope group --resource-group [options] +# +# Options: +# --scope Deployment scope (required) +# --location Location (required when --scope sub) +# --resource-group Resource group (required when --scope group) +# --template Bicep template (default: ./infra/main.bicep) +# --parameters Parameters file (default: ./infra/main.parameters.json; +# skipped automatically if the file does not exist) +# --subscription Subscription to target (optional) +# +# Examples: +# ./validate-deployment.sh --scope sub --location eastus +# ./validate-deployment.sh --scope group --resource-group my-rg \ +# --template ./infra/main.bicep --parameters ./infra/main.parameters.json +# +# Exit code: 0 if every step passes, 1 otherwise. + +set -uo pipefail + +SCOPE="" +LOCATION="" +RESOURCE_GROUP="" +TEMPLATE="./infra/main.bicep" +PARAMETERS="./infra/main.parameters.json" +SUBSCRIPTION="" + +while [ $# -gt 0 ]; do + case "$1" in + --scope) SCOPE="${2:-}"; shift 2 ;; + --location) LOCATION="${2:-}"; shift 2 ;; + --resource-group) RESOURCE_GROUP="${2:-}"; shift 2 ;; + --template) TEMPLATE="${2:-}"; shift 2 ;; + --parameters) PARAMETERS="${2:-}"; shift 2 ;; + --subscription) SUBSCRIPTION="${2:-}"; shift 2 ;; + -h|--help) + grep '^#' "$0" | sed 's/^# \{0,1\}//' + exit 0 ;; + *) + echo "Unknown argument: $1" >&2 + exit 2 ;; + esac +done + +# Validate arguments +if [ "$SCOPE" != "sub" ] && [ "$SCOPE" != "group" ]; then + echo "ERROR: --scope must be 'sub' or 'group'." >&2 + exit 2 +fi +if [ "$SCOPE" = "sub" ] && [ -z "$LOCATION" ]; then + echo "ERROR: --location is required when --scope is 'sub'." >&2 + exit 2 +fi +if [ "$SCOPE" = "group" ] && [ -z "$RESOURCE_GROUP" ]; then + echo "ERROR: --resource-group is required when --scope is 'group'." >&2 + exit 2 +fi + +# Build shared argument arrays +SUB_ARGS=() +[ -n "$SUBSCRIPTION" ] && SUB_ARGS=(--subscription "$SUBSCRIPTION") + +PARAM_ARGS=() +if [ -f "$PARAMETERS" ]; then + PARAM_ARGS=(--parameters "$PARAMETERS") +else + echo "NOTE: parameters file '$PARAMETERS' not found; validating without --parameters." +fi + +if [ "$SCOPE" = "sub" ]; then + SCOPE_TARGET_ARGS=(--location "$LOCATION") + SCOPE_DESC="subscription (location: $LOCATION)" +else + SCOPE_TARGET_ARGS=(--resource-group "$RESOURCE_GROUP") + SCOPE_DESC="resource group '$RESOURCE_GROUP'" +fi + +# Track results +declare -a STEP_NAMES=() +declare -a STEP_RESULTS=() +OVERALL=0 + +record() { + STEP_NAMES+=("$1") + STEP_RESULTS+=("$2") + [ "$2" = "PASS" ] || OVERALL=1 +} + +echo "=== Azure deployment validation ===" +echo "Template: $TEMPLATE" +echo "Scope: $SCOPE_DESC" +echo "" + +# Step 1: Azure CLI installed +echo "--- Step 1: Azure CLI installed (az version) ---" +if az version >/dev/null 2>&1; then + echo "PASS: Azure CLI is installed." + record "Azure CLI installed" PASS +else + echo "FAIL: Azure CLI not found. Install it, then re-run." + record "Azure CLI installed" FAIL + # Nothing else can run without the CLI. + printf '\n=== Summary ===\n' + printf '%-28s %s\n' "${STEP_NAMES[0]}" "${STEP_RESULTS[0]}" + echo "OVERALL: FAIL" + exit 1 +fi +echo "" + +# Step 2: Authenticated +echo "--- Step 2: Authenticated (az account show) ---" +ACCOUNT_JSON=$(az account show "${SUB_ARGS[@]}" -o json 2>/dev/null) +if [ -n "$ACCOUNT_JSON" ]; then + ACCOUNT_NAME=$(echo "$ACCOUNT_JSON" | grep -o '"name"[^,]*' | head -1 | sed 's/.*: *"\(.*\)"/\1/') + echo "PASS: Authenticated (subscription: ${ACCOUNT_NAME:-unknown})." + record "Authenticated" PASS +else + echo "FAIL: Not logged in. Run 'az login' (and 'az account set --subscription ')." + record "Authenticated" FAIL +fi +echo "" + +# Step 3: Bicep compilation +echo "--- Step 3: Bicep compilation (az bicep build) ---" +BUILD_OUTPUT=$(az bicep build --file "$TEMPLATE" 2>&1) +BUILD_RC=$? +if [ $BUILD_RC -eq 0 ]; then + echo "PASS: Template compiles cleanly." + record "Bicep compilation" PASS +else + echo "FAIL: Bicep compilation errors:" + echo "$BUILD_OUTPUT" + record "Bicep compilation" FAIL +fi +echo "" + +# Step 4: Template validation +echo "--- Step 4: Template validation (az deployment $SCOPE validate) ---" +VALIDATE_OUTPUT=$(az deployment "$SCOPE" validate "${SCOPE_TARGET_ARGS[@]}" \ + --template-file "$TEMPLATE" \ + "${PARAM_ARGS[@]}" "${SUB_ARGS[@]}" 2>&1) +VALIDATE_RC=$? +if [ $VALIDATE_RC -eq 0 ]; then + echo "PASS: Template validated against the target scope." + record "Template validation" PASS +else + echo "FAIL: Template validation errors:" + echo "$VALIDATE_OUTPUT" + record "Template validation" FAIL +fi +echo "" + +# Step 5: What-if preview +echo "--- Step 5: What-if preview (az deployment $SCOPE what-if) ---" +WHATIF_OUTPUT=$(az deployment "$SCOPE" what-if "${SCOPE_TARGET_ARGS[@]}" \ + --template-file "$TEMPLATE" \ + "${PARAM_ARGS[@]}" "${SUB_ARGS[@]}" 2>&1) +WHATIF_RC=$? +if [ $WHATIF_RC -eq 0 ]; then + CREATE_COUNT=$(echo "$WHATIF_OUTPUT" | grep -c '^[[:space:]]*+ ') + MODIFY_COUNT=$(echo "$WHATIF_OUTPUT" | grep -c '^[[:space:]]*~ ') + DELETE_COUNT=$(echo "$WHATIF_OUTPUT" | grep -c '^[[:space:]]*- ') + echo "PASS: What-if completed. Changes -> Create: $CREATE_COUNT, Modify: $MODIFY_COUNT, Delete: $DELETE_COUNT" + record "What-if preview" PASS +else + echo "FAIL: What-if errors:" + echo "$WHATIF_OUTPUT" + record "What-if preview" FAIL +fi +echo "" + +# Summary +echo "=== Summary ===" +for i in "${!STEP_NAMES[@]}"; do + printf '%-28s %s\n' "${STEP_NAMES[$i]}" "${STEP_RESULTS[$i]}" +done +if [ $OVERALL -eq 0 ]; then + echo "OVERALL: PASS" +else + echo "OVERALL: FAIL" +fi +exit $OVERALL From d4155c6b1e758d2bb09bd7d43c7cc08be48f9c63 Mon Sep 17 00:00:00 2001 From: "Tom Meschter (from Dev Box)" Date: Thu, 16 Jul 2026 11:21:36 -0700 Subject: [PATCH 2/2] fix: address PR review feedback on validate-deployment scripts - sh: guard value-consuming options against missing values (exit 2) - sh: exclude shebang from --help output - sh: document exit code 2 in header comment - sh: use 'az account show --query name -o tsv' + exit code instead of grep/sed JSON parsing - ps1: remove redundant \ summary table; keep \ to drive exit code, set inline per step - ps1: document exit code 2; align auth check to --query/\0 - bicep README: include CLI presence in core-validation wording (checklist item + table row) Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: a000b508-d45c-4d9e-85f9-394275303cac --- .../references/recipes/bicep/README.md | 4 +- .../recipes/scripts/validate-deployment.ps1 | 37 ++++------- .../recipes/scripts/validate-deployment.sh | 61 ++++++++----------- 3 files changed, 37 insertions(+), 65 deletions(-) diff --git a/plugin/skills/azure-validate/references/recipes/bicep/README.md b/plugin/skills/azure-validate/references/recipes/bicep/README.md index 4fb13b7f6..7d28edbb8 100644 --- a/plugin/skills/azure-validate/references/recipes/bicep/README.md +++ b/plugin/skills/azure-validate/references/recipes/bicep/README.md @@ -10,7 +10,7 @@ Validation steps for standalone Bicep deployments. ## Validation Steps -- [ ] 1. Core Validation (build, validate, what-if, auth) — run [`validate-deployment` script](../scripts/validate-deployment.sh) +- [ ] 1. Core Validation (CLI, auth, build, validate, what-if) — run [`validate-deployment` script](../scripts/validate-deployment.sh) - [ ] 2. Linting (optional) - [ ] 3. Azure Policy Validation @@ -71,7 +71,7 @@ See [Policy Validation Guide](../../policy-validation.md) for instructions on re | Check | Command | Pass | |-------|---------|------| -| Core validation (build, validate, what-if, auth) | `validate-deployment` script | ☐ | +| Core validation (CLI, auth, build, validate, what-if) | `validate-deployment` script | ☐ | | Policies validated | MCP Policy tool | ☐ | ## References diff --git a/plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.ps1 b/plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.ps1 index de43133cf..f91cbe7ca 100644 --- a/plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.ps1 +++ b/plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.ps1 @@ -10,7 +10,9 @@ 3. az bicep build - template compiles cleanly 4. az deployment ... validate - template validates against the target scope 5. az deployment ... what-if - preview changes (Create/Modify/Delete summary) - Emits a per-step PASS/FAIL summary and an OVERALL result. Exits 1 if any step fails. + Emits per-step PASS/FAIL lines and an OVERALL result. + Exit codes: 0 = every step passed; 1 = a validation step failed; + 2 = usage / argument error. .PARAMETER Scope Deployment scope: 'sub' or 'group' (required). .PARAMETER Location @@ -72,15 +74,9 @@ if ($Scope -eq "sub") { $scopeDesc = "resource group '$ResourceGroup'" } -# Track results -$steps = [System.Collections.ArrayList]@() +# Track overall result (0 = all passed, 1 = at least one failure). $overall = 0 -function Add-Result([string]$Name, [string]$Result) { - [void]$steps.Add([PSCustomObject]@{ Step = $Name; Result = $Result }) - if ($Result -ne "PASS") { $script:overall = 1 } -} - Write-Host "=== Azure deployment validation ===" Write-Host "Template: $Template" Write-Host "Scope: $scopeDesc" @@ -91,13 +87,9 @@ Write-Host "--- Step 1: Azure CLI installed (az version) ---" az version *> $null if ($LASTEXITCODE -eq 0) { Write-Host "PASS: Azure CLI is installed." - Add-Result "Azure CLI installed" "PASS" } else { Write-Host "FAIL: Azure CLI not found. Install it, then re-run." - Add-Result "Azure CLI installed" "FAIL" Write-Host "" - Write-Host "=== Summary ===" - $steps | Format-Table -AutoSize | Out-String | Write-Host Write-Host "OVERALL: FAIL" exit 1 } @@ -105,14 +97,12 @@ Write-Host "" # Step 2: Authenticated Write-Host "--- Step 2: Authenticated (az account show) ---" -$accountJson = az account show @subArgs -o json 2>$null -if ($accountJson) { - $accountName = ($accountJson | ConvertFrom-Json).name +$accountName = az account show @subArgs --query name -o tsv 2>$null +if ($LASTEXITCODE -eq 0) { Write-Host "PASS: Authenticated (subscription: $accountName)." - Add-Result "Authenticated" "PASS" } else { Write-Host "FAIL: Not logged in. Run 'az login' (and 'az account set --subscription ')." - Add-Result "Authenticated" "FAIL" + $overall = 1 } Write-Host "" @@ -121,11 +111,10 @@ Write-Host "--- Step 3: Bicep compilation (az bicep build) ---" $buildOutput = az bicep build --file $Template 2>&1 if ($LASTEXITCODE -eq 0) { Write-Host "PASS: Template compiles cleanly." - Add-Result "Bicep compilation" "PASS" } else { Write-Host "FAIL: Bicep compilation errors:" Write-Host ($buildOutput | Out-String) - Add-Result "Bicep compilation" "FAIL" + $overall = 1 } Write-Host "" @@ -134,11 +123,10 @@ Write-Host "--- Step 4: Template validation (az deployment $Scope validate) ---" $validateOutput = az deployment $Scope validate @scopeTargetArgs --template-file $Template @paramArgs @subArgs 2>&1 if ($LASTEXITCODE -eq 0) { Write-Host "PASS: Template validated against the target scope." - Add-Result "Template validation" "PASS" } else { Write-Host "FAIL: Template validation errors:" Write-Host ($validateOutput | Out-String) - Add-Result "Template validation" "FAIL" + $overall = 1 } Write-Host "" @@ -151,17 +139,14 @@ if ($LASTEXITCODE -eq 0) { $modifyCount = ($lines | Where-Object { $_ -match '^\s*~ ' }).Count $deleteCount = ($lines | Where-Object { $_ -match '^\s*- ' }).Count Write-Host "PASS: What-if completed. Changes -> Create: $createCount, Modify: $modifyCount, Delete: $deleteCount" - Add-Result "What-if preview" "PASS" } else { Write-Host "FAIL: What-if errors:" Write-Host ($whatifOutput | Out-String) - Add-Result "What-if preview" "FAIL" + $overall = 1 } Write-Host "" -# Summary -Write-Host "=== Summary ===" -$steps | Format-Table -AutoSize | Out-String | Write-Host +# Overall result (drives the exit code) if ($overall -eq 0) { Write-Host "OVERALL: PASS" } else { diff --git a/plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.sh b/plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.sh index 33e2546e9..595406c9f 100755 --- a/plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.sh +++ b/plugin/skills/azure-validate/references/recipes/scripts/validate-deployment.sh @@ -29,10 +29,18 @@ # ./validate-deployment.sh --scope group --resource-group my-rg \ # --template ./infra/main.bicep --parameters ./infra/main.parameters.json # -# Exit code: 0 if every step passes, 1 otherwise. +# Exit codes: +# 0 - every validation step passed +# 1 - a validation step failed +# 2 - usage / argument error (unknown or valueless option, missing required flag) set -uo pipefail +# Ensure an option that consumes a value actually has one ($@ = remaining args). +need_val() { + [ "$#" -ge 2 ] || { echo "ERROR: $1 requires a value." >&2; exit 2; } +} + SCOPE="" LOCATION="" RESOURCE_GROUP="" @@ -42,14 +50,14 @@ SUBSCRIPTION="" while [ $# -gt 0 ]; do case "$1" in - --scope) SCOPE="${2:-}"; shift 2 ;; - --location) LOCATION="${2:-}"; shift 2 ;; - --resource-group) RESOURCE_GROUP="${2:-}"; shift 2 ;; - --template) TEMPLATE="${2:-}"; shift 2 ;; - --parameters) PARAMETERS="${2:-}"; shift 2 ;; - --subscription) SUBSCRIPTION="${2:-}"; shift 2 ;; + --scope) need_val "$@"; SCOPE="$2"; shift 2 ;; + --location) need_val "$@"; LOCATION="$2"; shift 2 ;; + --resource-group) need_val "$@"; RESOURCE_GROUP="$2"; shift 2 ;; + --template) need_val "$@"; TEMPLATE="$2"; shift 2 ;; + --parameters) need_val "$@"; PARAMETERS="$2"; shift 2 ;; + --subscription) need_val "$@"; SUBSCRIPTION="$2"; shift 2 ;; -h|--help) - grep '^#' "$0" | sed 's/^# \{0,1\}//' + grep '^#' "$0" | grep -v '^#!' | sed 's/^# \{0,1\}//' exit 0 ;; *) echo "Unknown argument: $1" >&2 @@ -90,17 +98,9 @@ else SCOPE_DESC="resource group '$RESOURCE_GROUP'" fi -# Track results -declare -a STEP_NAMES=() -declare -a STEP_RESULTS=() +# Track overall result (0 = all passed, 1 = at least one failure). OVERALL=0 -record() { - STEP_NAMES+=("$1") - STEP_RESULTS+=("$2") - [ "$2" = "PASS" ] || OVERALL=1 -} - echo "=== Azure deployment validation ===" echo "Template: $TEMPLATE" echo "Scope: $SCOPE_DESC" @@ -110,13 +110,10 @@ echo "" echo "--- Step 1: Azure CLI installed (az version) ---" if az version >/dev/null 2>&1; then echo "PASS: Azure CLI is installed." - record "Azure CLI installed" PASS else echo "FAIL: Azure CLI not found. Install it, then re-run." - record "Azure CLI installed" FAIL # Nothing else can run without the CLI. - printf '\n=== Summary ===\n' - printf '%-28s %s\n' "${STEP_NAMES[0]}" "${STEP_RESULTS[0]}" + echo "" echo "OVERALL: FAIL" exit 1 fi @@ -124,14 +121,11 @@ echo "" # Step 2: Authenticated echo "--- Step 2: Authenticated (az account show) ---" -ACCOUNT_JSON=$(az account show "${SUB_ARGS[@]}" -o json 2>/dev/null) -if [ -n "$ACCOUNT_JSON" ]; then - ACCOUNT_NAME=$(echo "$ACCOUNT_JSON" | grep -o '"name"[^,]*' | head -1 | sed 's/.*: *"\(.*\)"/\1/') +if ACCOUNT_NAME=$(az account show "${SUB_ARGS[@]}" --query name -o tsv 2>/dev/null); then echo "PASS: Authenticated (subscription: ${ACCOUNT_NAME:-unknown})." - record "Authenticated" PASS else echo "FAIL: Not logged in. Run 'az login' (and 'az account set --subscription ')." - record "Authenticated" FAIL + OVERALL=1 fi echo "" @@ -141,11 +135,10 @@ BUILD_OUTPUT=$(az bicep build --file "$TEMPLATE" 2>&1) BUILD_RC=$? if [ $BUILD_RC -eq 0 ]; then echo "PASS: Template compiles cleanly." - record "Bicep compilation" PASS else echo "FAIL: Bicep compilation errors:" echo "$BUILD_OUTPUT" - record "Bicep compilation" FAIL + OVERALL=1 fi echo "" @@ -157,11 +150,10 @@ VALIDATE_OUTPUT=$(az deployment "$SCOPE" validate "${SCOPE_TARGET_ARGS[@]}" \ VALIDATE_RC=$? if [ $VALIDATE_RC -eq 0 ]; then echo "PASS: Template validated against the target scope." - record "Template validation" PASS else echo "FAIL: Template validation errors:" echo "$VALIDATE_OUTPUT" - record "Template validation" FAIL + OVERALL=1 fi echo "" @@ -176,19 +168,14 @@ if [ $WHATIF_RC -eq 0 ]; then MODIFY_COUNT=$(echo "$WHATIF_OUTPUT" | grep -c '^[[:space:]]*~ ') DELETE_COUNT=$(echo "$WHATIF_OUTPUT" | grep -c '^[[:space:]]*- ') echo "PASS: What-if completed. Changes -> Create: $CREATE_COUNT, Modify: $MODIFY_COUNT, Delete: $DELETE_COUNT" - record "What-if preview" PASS else echo "FAIL: What-if errors:" echo "$WHATIF_OUTPUT" - record "What-if preview" FAIL + OVERALL=1 fi echo "" -# Summary -echo "=== Summary ===" -for i in "${!STEP_NAMES[@]}"; do - printf '%-28s %s\n' "${STEP_NAMES[$i]}" "${STEP_RESULTS[$i]}" -done +# Overall result if [ $OVERALL -eq 0 ]; then echo "OVERALL: PASS" else