From a169a962249fc3428b127e669729b1cdb6b702bf Mon Sep 17 00:00:00 2001 From: "Tom Meschter (from Dev Box)" Date: Tue, 21 Jul 2026 11:59:16 -0700 Subject: [PATCH 1/5] feat: add pod-evidence script for azure-diagnostics AKS pod failures Replace the repeated read-only AKS pod-failure evidence bundle (find failing pods, describe, logs + --previous, top, resource jsonpath) with a single cross-platform pod-evidence script (bash + PowerShell) that gathers and digests the invariant bundle into one labeled packet. Update pod-failures.md, aks-troubleshooting.md, and command-flows.md to reference the script with markdown links, sample invocations, and a short description, keeping the interpretation/decision tables in prose. Closes #2507 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bdfcb108-93f9-4399-954c-f96110af0428 --- .../scripts/pod-evidence.ps1 | 145 ++++++++++++++++++ .../azure-diagnostics/scripts/pod-evidence.sh | 145 ++++++++++++++++++ .../aks/aks-troubleshooting.md | 11 +- .../troubleshooting/aks/pod-failures.md | 75 +++------ .../aks/references/command-flows.md | 9 +- 5 files changed, 324 insertions(+), 61 deletions(-) create mode 100644 plugin/skills/azure-diagnostics/scripts/pod-evidence.ps1 create mode 100644 plugin/skills/azure-diagnostics/scripts/pod-evidence.sh diff --git a/plugin/skills/azure-diagnostics/scripts/pod-evidence.ps1 b/plugin/skills/azure-diagnostics/scripts/pod-evidence.ps1 new file mode 100644 index 000000000..05057575b --- /dev/null +++ b/plugin/skills/azure-diagnostics/scripts/pod-evidence.ps1 @@ -0,0 +1,145 @@ +<# +.SYNOPSIS + Collects the invariant, read-only AKS pod-failure evidence bundle and prints a + single labeled digest. +.DESCRIPTION + Gathers the same evidence bundle for one or more pods regardless of the symptom + (CrashLoopBackOff, OOMKilled, Pending, probe failures, ImagePullBackOff). For each + pod it collects and summarizes: + STATUS - READY / phase / restart count (kubectl get pod -o wide) + STATE - exit code, reason, last-state snippet (jsonpath over containerStatuses) + EVENTS - the Events section (kubectl describe pod) + LOGS - current container logs (tailed) (kubectl logs) + PREV LOGS - previous/crashed container logs (tailed)(kubectl logs --previous) + RESOURCES - requests/limits vs live usage (jsonpath + kubectl top pod) + + This script only GATHERS and DIGESTS evidence. It never mutates cluster state. + Interpreting the digest to pick a fix stays with the caller. +.PARAMETER Pod + Pod name (single-pod mode). Omit when using -AllFailing. +.PARAMETER Namespace + Namespace of the pod. Required in single-pod mode. In -AllFailing mode, limits the + scan to this namespace. +.PARAMETER AllFailing + Auto-select every pod not in Running/Succeeded phase (across all namespaces unless + -Namespace is given) and digest each. +.PARAMETER Tail + Number of log lines to show per stream. Default 50. +.EXAMPLE + .\pod-evidence.ps1 my-api-7d9f-abcde -Namespace prod +.EXAMPLE + .\pod-evidence.ps1 -AllFailing +.EXAMPLE + .\pod-evidence.ps1 -AllFailing -Namespace prod -Tail 100 +#> +param( + [Parameter(Position = 0)][string]$Pod, + [Alias("n")][string]$Namespace, + [switch]$AllFailing, + [int]$Tail = 50 +) + +# Best-effort: individual kubectl reads may fail (unreachable cluster, missing +# metrics-server, no previous logs). Keep the default "Continue" so a single failed +# read is suppressed via 2>$null and the digest proceeds instead of aborting. +$ErrorActionPreference = "Continue" + +if (-not (Get-Command kubectl -ErrorAction SilentlyContinue)) { + Write-Error "kubectl not found on PATH." + exit 1 +} + +function Digest-Pod { + param([string]$Ns, [string]$Name) + + Write-Host "==================================================================" + Write-Host "POD: $Name NAMESPACE: $Ns" + Write-Host "==================================================================" + + Write-Host "--- STATUS (ready / phase / restarts) ---" + $status = kubectl get pod $Name -n $Ns -o wide 2>&1 + if ($status) { $status | ForEach-Object { Write-Host "$_" } } else { Write-Host "(unable to get pod)" } + Write-Host "" + + Write-Host "--- STATE (exit code / reason / last state) ---" + $jp = '{range .status.containerStatuses[*]}container={.name}{"`n"} ready={.ready} restarts={.restartCount}{"`n"} current: waiting={.state.waiting.reason} running={.state.running.startedAt} terminated={.state.terminated.reason}(exit={.state.terminated.exitCode}){"`n"} lastState: terminated={.lastState.terminated.reason}(exit={.lastState.terminated.exitCode}) at {.lastState.terminated.finishedAt}{"`n"}{end}' + $state = kubectl get pod $Name -n $Ns -o jsonpath=$jp 2>$null + if ($state) { Write-Host $state } else { Write-Host "(no container status available)" } + Write-Host "" + + Write-Host "--- EVENTS ---" + $desc = kubectl describe pod $Name -n $Ns 2>$null + if ($desc) { + $idx = ($desc | Select-String -Pattern '^Events:' | Select-Object -First 1).LineNumber + if ($idx) { + $desc | Select-Object -Skip ($idx - 1) | Select-Object -First 25 | ForEach-Object { Write-Host "$_" } + } else { + Write-Host "(no Events section)" + } + } else { + Write-Host "(unable to describe pod)" + } + Write-Host "" + + Write-Host "--- LOGS (current, last $Tail lines) ---" + $logs = kubectl logs $Name -n $Ns --tail=$Tail 2>&1 + if ($logs) { $logs | ForEach-Object { Write-Host "$_" } } else { Write-Host "(no current logs)" } + Write-Host "" + + Write-Host "--- PREV LOGS (previous instance, last $Tail lines) ---" + $prev = kubectl logs $Name -n $Ns --previous --tail=$Tail 2>$null + if ($LASTEXITCODE -eq 0 -and $prev) { + Write-Host $prev + } else { + Write-Host "(no previous-instance logs - pod has not restarted or they were rotated)" + } + Write-Host "" + + Write-Host "--- RESOURCES (requests/limits vs live usage) ---" + Write-Host "requests/limits:" + $res = kubectl get pod $Name -n $Ns -o jsonpath='{range .spec.containers[*]} {.name}: requests={.resources.requests} limits={.resources.limits}{"`n"}{end}' 2>$null + if ($res) { Write-Host $res } else { Write-Host " (unable to read resources)" } + Write-Host "live usage:" + $top = kubectl top pod $Name -n $Ns 2>&1 + if ($top) { $top | ForEach-Object { Write-Host " $_" } } else { Write-Host " (metrics-server unavailable)" } + Write-Host "" +} + +if ($AllFailing) { + $scope = if ($Namespace) { " in namespace '$Namespace'" } else { "" } + Write-Host "pod-evidence: scanning for pods not in Running/Succeeded$scope..." + + $cols = "custom-columns=NS:.metadata.namespace,NAME:.metadata.name" + if ($Namespace) { + $rows = kubectl get pods -n $Namespace --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers -o $cols 2>$null + } else { + $rows = kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers -o $cols 2>$null + } + + $rows = @($rows | Where-Object { $_ -and $_.Trim() }) + if ($rows.Count -eq 0) { + Write-Host "No unhealthy pods found (all pods are Running or Succeeded)." + exit 0 + } + + Write-Host "Found $($rows.Count) unhealthy pod(s). Collecting evidence for each below." + Write-Host "" + foreach ($row in $rows) { + $parts = ($row -split '\s+') | Where-Object { $_ } + Digest-Pod -Ns $parts[0] -Name $parts[1] + } + Write-Host "pod-evidence: done. Reviewed $($rows.Count) failing pod(s) - use the STATE/EVENTS/LOGS above to pick a fix." +} else { + if (-not $Pod) { + Write-Error "A pod name is required (or use -AllFailing)." + exit 2 + } + if (-not $Namespace) { + Write-Error "-Namespace is required in single-pod mode." + exit 2 + } + Write-Host "pod-evidence: collecting the read-only evidence bundle for pod '$Pod' in namespace '$Namespace'." + Write-Host "" + Digest-Pod -Ns $Namespace -Name $Pod + Write-Host "pod-evidence: done. Use the STATE/EVENTS/LOGS/RESOURCES digest above to pick a fix." +} diff --git a/plugin/skills/azure-diagnostics/scripts/pod-evidence.sh b/plugin/skills/azure-diagnostics/scripts/pod-evidence.sh new file mode 100644 index 000000000..92a24e657 --- /dev/null +++ b/plugin/skills/azure-diagnostics/scripts/pod-evidence.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash +# pod-evidence.sh +# Collects the invariant, read-only AKS pod-failure evidence bundle for one or more +# pods and prints a single labeled digest. Works identically regardless of the pod +# symptom (CrashLoopBackOff, OOMKilled, Pending, probe failures, ImagePullBackOff). +# +# For each pod it gathers and summarizes: +# STATUS - READY / phase / restart count (kubectl get pod -o wide) +# STATE - exit code, reason, last-state snippet (jsonpath over containerStatuses) +# EVENTS - the Events section (kubectl describe pod) +# LOGS - current container logs (tailed) (kubectl logs) +# PREV LOGS - previous/crashed container logs (tailed)(kubectl logs --previous) +# RESOURCES - requests/limits vs live usage (jsonpath + kubectl top pod) +# +# This script only GATHERS and DIGESTS evidence. It never mutates cluster state. +# Interpreting the digest to pick a fix (exit-code / event / probe decision tables) +# stays with the caller. +# +# Usage: +# ./pod-evidence.sh -n [--tail ] +# ./pod-evidence.sh --all-failing [-n ] [--tail ] +# +# Options: +# -n, --namespace Namespace of the pod. Required in single-pod mode. +# In --all-failing mode, limits the scan to this namespace. +# --all-failing Auto-select every pod not in Running/Succeeded phase +# (across all namespaces unless -n is given) and digest each. +# --tail Number of log lines to show per stream (default 50). +# -h, --help Show this help. +# +# Examples: +# ./pod-evidence.sh my-api-7d9f-abcde -n prod +# ./pod-evidence.sh --all-failing +# ./pod-evidence.sh --all-failing -n prod --tail 100 + +set -euo pipefail + +POD="" +NAMESPACE="" +ALL_FAILING=false +TAIL=50 + +usage() { + sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//' +} + +while [ $# -gt 0 ]; do + case "$1" in + -n|--namespace) NAMESPACE="${2:?--namespace requires a value}"; shift 2 ;; + --all-failing) ALL_FAILING=true; shift ;; + --tail) TAIL="${2:?--tail requires a value}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + -*) echo "Unknown option: $1" >&2; usage; exit 2 ;; + *) POD="$1"; shift ;; + esac +done + +if ! command -v kubectl >/dev/null 2>&1; then + echo "ERROR: kubectl not found on PATH." >&2 + exit 1 +fi + +# Digest a single pod. Args: +digest_pod() { + local ns="$1" pod="$2" + + echo "==================================================================" + echo "POD: $pod NAMESPACE: $ns" + echo "==================================================================" + + echo "--- STATUS (ready / phase / restarts) ---" + kubectl get pod "$pod" -n "$ns" -o wide 2>&1 || echo "(unable to get pod)" + echo "" + + echo "--- STATE (exit code / reason / last state) ---" + kubectl get pod "$pod" -n "$ns" -o jsonpath='{range .status.containerStatuses[*]}container={.name}{"\n"} ready={.ready} restarts={.restartCount}{"\n"} current: waiting={.state.waiting.reason} running={.state.running.startedAt} terminated={.state.terminated.reason}(exit={.state.terminated.exitCode}){"\n"} lastState: terminated={.lastState.terminated.reason}(exit={.lastState.terminated.exitCode}) at {.lastState.terminated.finishedAt}{"\n"}{end}' 2>/dev/null \ + || echo "(no container status available)" + echo "" + + echo "--- EVENTS ---" + if kubectl describe pod "$pod" -n "$ns" >/dev/null 2>&1; then + kubectl describe pod "$pod" -n "$ns" 2>/dev/null | sed -n '/^Events:/,$p' | head -n 25 + else + echo "(unable to describe pod)" + fi + echo "" + + echo "--- LOGS (current, last $TAIL lines) ---" + kubectl logs "$pod" -n "$ns" --tail="$TAIL" 2>&1 || echo "(no current logs)" + echo "" + + echo "--- PREV LOGS (previous instance, last $TAIL lines) ---" + if kubectl logs "$pod" -n "$ns" --previous --tail="$TAIL" 2>/dev/null; then + : + else + echo "(no previous-instance logs - pod has not restarted or they were rotated)" + fi + echo "" + + echo "--- RESOURCES (requests/limits vs live usage) ---" + echo "requests/limits:" + kubectl get pod "$pod" -n "$ns" -o jsonpath='{range .spec.containers[*]} {.name}: requests={.resources.requests} limits={.resources.limits}{"\n"}{end}' 2>/dev/null \ + || echo " (unable to read resources)" + echo "live usage:" + kubectl top pod "$pod" -n "$ns" 2>&1 | sed 's/^/ /' || echo " (metrics-server unavailable)" + echo "" +} + +if [ "$ALL_FAILING" = true ]; then + echo "pod-evidence: scanning for pods not in Running/Succeeded${NAMESPACE:+ in namespace '$NAMESPACE'}..." + if [ -n "$NAMESPACE" ]; then + mapfile -t ROWS < <(kubectl get pods -n "$NAMESPACE" --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2>/dev/null) + else + mapfile -t ROWS < <(kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2>/dev/null) + fi + + if [ "${#ROWS[@]}" -eq 0 ]; then + echo "No unhealthy pods found (all pods are Running or Succeeded)." + exit 0 + fi + + echo "Found ${#ROWS[@]} unhealthy pod(s). Collecting evidence for each below." + echo "" + for row in "${ROWS[@]}"; do + [ -z "$row" ] && continue + ns="${row%% *}" + name="${row##* }" + digest_pod "$ns" "$name" + done + echo "pod-evidence: done. Reviewed ${#ROWS[@]} failing pod(s) - use the STATE/EVENTS/LOGS above to pick a fix." +else + if [ -z "$POD" ]; then + echo "ERROR: a pod name is required (or use --all-failing)." >&2 + usage + exit 2 + fi + if [ -z "$NAMESPACE" ]; then + echo "ERROR: -n/--namespace is required in single-pod mode." >&2 + exit 2 + fi + echo "pod-evidence: collecting the read-only evidence bundle for pod '$POD' in namespace '$NAMESPACE'." + echo "" + digest_pod "$NAMESPACE" "$POD" + echo "pod-evidence: done. Use the STATE/EVENTS/LOGS/RESOURCES digest above to pick a fix." +fi diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md index ce789bb02..623170b31 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md @@ -76,10 +76,17 @@ kubectl cluster-info kubectl get nodes -o wide kubectl get pods -n kube-system kubectl get events -A --sort-by=.lastTimestamp -kubectl describe pod -n -kubectl logs -n --previous ``` +For unhealthy pods, gather the full read-only evidence bundle (describe, current + previous logs, resources vs usage) with the pod-evidence script instead of running the commands one by one — [`../../scripts/pod-evidence.sh`](../../scripts/pod-evidence.sh) / [`../../scripts/pod-evidence.ps1`](../../scripts/pod-evidence.ps1): + +```bash +../../scripts/pod-evidence.sh -n +../../scripts/pod-evidence.sh --all-failing +``` + +See [pod-failures.md](pod-failures.md) for how to interpret the digest. + Keep these read-only unless the user explicitly asks for remediation. ## Guardrails diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md index 9a2c4e33a..1d03df41f 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md @@ -1,34 +1,28 @@ # Pod Failures & Application Issues -## Common Pod Diagnostic Commands +## Evidence Bundle Script + +For **any** pod symptom below, run the **pod-evidence** script to collect the same +read-only evidence bundle. Per pod it digests **STATUS**, **STATE** (exit code, reason, +last state), **EVENTS**, current/previous **LOGS**, and **RESOURCES** (requests/limits +vs `top`). It only gathers; interpret it with the tables. + +Bash [`../../scripts/pod-evidence.sh`](../../scripts/pod-evidence.sh) · PowerShell [`../../scripts/pod-evidence.ps1`](../../scripts/pod-evidence.ps1) ```bash -# List unhealthy pods across all namespaces -kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded -# All pods wide view -kubectl get pods -A -o wide -# Detailed pod status - events section is critical -kubectl describe pod -n -# Pod logs (current and previous crash) -kubectl logs -n -kubectl logs -n --previous +../../scripts/pod-evidence.sh -n # one pod +../../scripts/pod-evidence.sh --all-failing # all unhealthy pods ``` +PowerShell: `pod-evidence.ps1 -Namespace ` (`-AllFailing` scans all). + --- ## CrashLoopBackOff Pod starts, crashes, restarts with exponential backoff (10s, 20s, 40s... up to 5m). -**Diagnostics:** - -```bash -kubectl describe pod -n -# Check: Exit Code, Reason, Last State, Events - -kubectl logs -n --previous -# Shows stdout/stderr from the last crashed container -``` +**Diagnostics:** [pod-evidence](#evidence-bundle-script) → read **STATE** (exit code, reason, last state) and **PREV LOGS** (last crashed container). **Decision tree:** @@ -40,14 +34,7 @@ kubectl logs -n --previous | `139` | Segfault (SIGSEGV) | Binary compatibility issue or native code bug | | `143` | SIGTERM - graceful shutdown | Pod was terminated; check if liveness probe killed it | -**OOMKilled specifically:** - -```bash -kubectl describe pod -n | grep -A2 "Last State" -# Reason: OOMKilled -> container exceeded memory limit -``` - -Fix: increase `resources.limits.memory` or optimize application memory usage. Check `kubectl top pod -n ` for actual usage. +**OOMKilled specifically:** the **STATE** section shows `terminated=OOMKilled` and **RESOURCES** shows the memory limit vs live usage. Fix: increase `resources.limits.memory` or optimize application memory usage. **OOM kill tracing with Inspektor Gadget:** Use `trace_oomkill` (timeout 30) with `--k8s-namespace --k8s-podname ` to see which process was killed and memory at kill time. See [references/inspektor-gadget.md](references/inspektor-gadget.md). @@ -67,12 +54,7 @@ See [references/inspektor-gadget.md](references/inspektor-gadget.md). Pod can't pull the container image. -**Diagnostics:** - -```bash -kubectl describe pod -n -# Events section shows the exact pull error -``` +**Diagnostics:** [pod-evidence](#evidence-bundle-script) → read **EVENTS** for the exact pull error. | Error Message | Cause | Fix | | --------------------------------------- | ---------------------------- | -------------------------------------------------------------- | @@ -94,12 +76,7 @@ az aks check-acr -g -n --acr .azurecr.io Pod stays in `Pending` - scheduler can't place it. -**Diagnostics:** - -```bash -kubectl describe pod -n -# Events section shows why scheduling failed -``` +**Diagnostics:** [pod-evidence](#evidence-bundle-script) → read **EVENTS** for why scheduling failed. | Event Message | Cause | Fix | | ---------------------------------------------------------------------- | ----------------------------------- | --------------------------------------------------------------- | @@ -115,15 +92,7 @@ kubectl describe pod -n **Readiness probe failure** -> pod removed from Service endpoints (no traffic). **Liveness probe failure** -> pod killed and restarted. -**Diagnostics:** - -```bash -kubectl describe pod -n -# Look for: "Readiness probe failed" or "Liveness probe failed" in Events - -# Check the pod's READY column - must show n/n -kubectl get pod -n -``` +**Diagnostics:** [pod-evidence](#evidence-bundle-script) → **EVENTS** shows `Readiness/Liveness probe failed`; **STATUS** shows the READY column (must be n/n). | Symptom | Cause | Fix | | ------------------------------------ | ----------------------- | ---------------------------------------------------------- | @@ -137,15 +106,7 @@ kubectl get pod -n ## Resource Constraints (CPU/Memory) -**Check actual usage vs limits:** - -```bash -kubectl top pod -n -kubectl top pod -n --sort-by=memory - -# Compare with requests/limits -kubectl get pod -n -o jsonpath='{.spec.containers[*].resources}' -``` +**Check actual usage vs limits:** [pod-evidence](#evidence-bundle-script) → **RESOURCES** compares requests/limits against live `top` usage. To rank a namespace by memory: `kubectl top pod -n --sort-by=memory`. | Symptom | Cause | Fix | | -------------------------------- | --------------------------------------- | --------------------------------------------------- | diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md index 819724323..1e9a32a85 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md @@ -28,8 +28,13 @@ kubectl get nodes -o wide kubectl get pods -n kube-system kubectl get events -A --sort-by=.lastTimestamp kubectl get pods -n -kubectl describe pod -n -kubectl logs -n --previous +``` + +For pod detail and logs, gather the read-only evidence bundle (describe, current + previous logs, resources vs usage) with the pod-evidence script — [`../../../scripts/pod-evidence.sh`](../../../scripts/pod-evidence.sh) / [`../../../scripts/pod-evidence.ps1`](../../../scripts/pod-evidence.ps1): + +```bash +../../../scripts/pod-evidence.sh -n +../../../scripts/pod-evidence.sh --all-failing ``` ## Connectivity Flow From aee951150bb2f74bdd259dabcf05fe47b67f5fa9 Mon Sep 17 00:00:00 2001 From: "Tom Meschter (from Dev Box)" Date: Wed, 22 Jul 2026 10:05:08 -0700 Subject: [PATCH 2/5] fix: address PR review comments on pod-evidence scripts and docs - pod-evidence.sh: replace mapfile with a portable while-read loop (Bash 3.2 / macOS) and validate --tail is a positive integer - pod-evidence.ps1: use \0 for the STATUS section so failures deterministically print (unable to get pod) - docs: add relative path to the PowerShell example in pod-failures.md; add PowerShell invocation examples in aks-troubleshooting.md and command-flows.md Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bdfcb108-93f9-4399-954c-f96110af0428 --- .../azure-diagnostics/scripts/pod-evidence.ps1 | 2 +- .../azure-diagnostics/scripts/pod-evidence.sh | 14 ++++++++++++-- .../troubleshooting/aks/aks-troubleshooting.md | 4 ++++ .../troubleshooting/aks/pod-failures.md | 6 +++--- .../aks/references/command-flows.md | 4 ++++ 5 files changed, 24 insertions(+), 6 deletions(-) diff --git a/plugin/skills/azure-diagnostics/scripts/pod-evidence.ps1 b/plugin/skills/azure-diagnostics/scripts/pod-evidence.ps1 index 05057575b..a22750fdc 100644 --- a/plugin/skills/azure-diagnostics/scripts/pod-evidence.ps1 +++ b/plugin/skills/azure-diagnostics/scripts/pod-evidence.ps1 @@ -58,7 +58,7 @@ function Digest-Pod { Write-Host "--- STATUS (ready / phase / restarts) ---" $status = kubectl get pod $Name -n $Ns -o wide 2>&1 - if ($status) { $status | ForEach-Object { Write-Host "$_" } } else { Write-Host "(unable to get pod)" } + if ($LASTEXITCODE -eq 0 -and $status) { $status | ForEach-Object { Write-Host "$_" } } else { Write-Host "(unable to get pod)" } Write-Host "" Write-Host "--- STATE (exit code / reason / last state) ---" diff --git a/plugin/skills/azure-diagnostics/scripts/pod-evidence.sh b/plugin/skills/azure-diagnostics/scripts/pod-evidence.sh index 92a24e657..cffd27945 100644 --- a/plugin/skills/azure-diagnostics/scripts/pod-evidence.sh +++ b/plugin/skills/azure-diagnostics/scripts/pod-evidence.sh @@ -60,6 +60,11 @@ if ! command -v kubectl >/dev/null 2>&1; then exit 1 fi +case "$TAIL" in + ''|*[!0-9]*) echo "ERROR: --tail must be a positive integer (got '$TAIL')." >&2; exit 2 ;; + 0) echo "ERROR: --tail must be a positive integer (got '$TAIL')." >&2; exit 2 ;; +esac + # Digest a single pod. Args: digest_pod() { local ns="$1" pod="$2" @@ -108,11 +113,16 @@ digest_pod() { if [ "$ALL_FAILING" = true ]; then echo "pod-evidence: scanning for pods not in Running/Succeeded${NAMESPACE:+ in namespace '$NAMESPACE'}..." + # Portable row collection (avoids `mapfile`, which is unavailable in Bash 3.2 / macOS). + ROWS=() if [ -n "$NAMESPACE" ]; then - mapfile -t ROWS < <(kubectl get pods -n "$NAMESPACE" --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2>/dev/null) + SCAN_ARGS=(-n "$NAMESPACE") else - mapfile -t ROWS < <(kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2>/dev/null) + SCAN_ARGS=(-A) fi + while IFS= read -r line; do + [ -n "$line" ] && ROWS+=("$line") + done < <(kubectl get pods "${SCAN_ARGS[@]}" --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2>/dev/null) if [ "${#ROWS[@]}" -eq 0 ]; then echo "No unhealthy pods found (all pods are Running or Succeeded)." diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md index 623170b31..334af398b 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md @@ -84,6 +84,10 @@ For unhealthy pods, gather the full read-only evidence bundle (describe, current ../../scripts/pod-evidence.sh -n ../../scripts/pod-evidence.sh --all-failing ``` +```powershell +../../scripts/pod-evidence.ps1 -Namespace +../../scripts/pod-evidence.ps1 -AllFailing +``` See [pod-failures.md](pod-failures.md) for how to interpret the digest. diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md index 1d03df41f..f61d2e8cf 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md @@ -4,8 +4,8 @@ For **any** pod symptom below, run the **pod-evidence** script to collect the same read-only evidence bundle. Per pod it digests **STATUS**, **STATE** (exit code, reason, -last state), **EVENTS**, current/previous **LOGS**, and **RESOURCES** (requests/limits -vs `top`). It only gathers; interpret it with the tables. +last state), **EVENTS**, current/previous **LOGS**, and **RESOURCES** (requests vs +`top`). It only gathers; interpret with the tables. Bash [`../../scripts/pod-evidence.sh`](../../scripts/pod-evidence.sh) · PowerShell [`../../scripts/pod-evidence.ps1`](../../scripts/pod-evidence.ps1) @@ -14,7 +14,7 @@ Bash [`../../scripts/pod-evidence.sh`](../../scripts/pod-evidence.sh) · PowerSh ../../scripts/pod-evidence.sh --all-failing # all unhealthy pods ``` -PowerShell: `pod-evidence.ps1 -Namespace ` (`-AllFailing` scans all). +PowerShell: `../../scripts/pod-evidence.ps1 -Namespace ` (`-AllFailing` scans all). --- diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md index 1e9a32a85..65619343f 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md @@ -36,6 +36,10 @@ For pod detail and logs, gather the read-only evidence bundle (describe, current ../../../scripts/pod-evidence.sh -n ../../../scripts/pod-evidence.sh --all-failing ``` +```powershell +../../../scripts/pod-evidence.ps1 -Namespace +../../../scripts/pod-evidence.ps1 -AllFailing +``` ## Connectivity Flow From 2cac0b8b19b3c0d5c2b419039b71b9e7bd3f54c7 Mon Sep 17 00:00:00 2001 From: "Tom Meschter (from Dev Box)" Date: Thu, 23 Jul 2026 15:05:56 -0700 Subject: [PATCH 3/5] test: add eval asserting agent runs pod-evidence script for AKS pod failures Adds a response-quality stimulus to evals/azure-diagnostics/eval.yaml that verifies the agent invokes the read-only pod-evidence.{sh,ps1} script for a CrashLoopBackOff prompt (issue #2507). Uses a tool-calls grader to assert the shell invocation and an earlyTerminate tool-call-match guard to stop the run right after the attempt; completed grader omitted per early-terminate rule. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bdfcb108-93f9-4399-954c-f96110af0428 --- evals/azure-diagnostics/eval.yaml | 42 +++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/evals/azure-diagnostics/eval.yaml b/evals/azure-diagnostics/eval.yaml index 524fa3819..3201838ca 100644 --- a/evals/azure-diagnostics/eval.yaml +++ b/evals/azure-diagnostics/eval.yaml @@ -110,8 +110,6 @@ stimuli: - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" - - # ── vm-ssh-troubleshooting-prompt ── # Migrated from azure-compute ownership: VM connectivity incidents now route to azure-diagnostics. - name: "VM SSH refused troubleshooting" prompt: "I can't SSH into my Azure Linux VM. SSH says connection refused and I need help checking NSG or firewall issues." @@ -129,6 +127,46 @@ stimuli: disallowed: - azure-compute # Global: no_runtime_failure + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ── pod-evidence-script-invoked ── + # Response-quality check: for an AKS pod-failure prompt the skill should drive the + # agent to RUN the read-only pod-evidence evidence-bundle script rather than re-listing + # the raw kubectl describe/logs/top sequence (issue #2507). The `tool-calls` grader + # asserts the agent attempted to execute pod-evidence.{sh,ps1} via a shell tool. + # + # earlyTerminate fires the moment that shell call starts, so the agent is stopped right + # after the attempt (no follow-on turns). Note this is a guard/optimization, not a hard + # execution block: `tool-call-match` reacts to the `tool.execution_start` event, which + # the SDK emits only after the (auto-approved) tool has already launched. The script is + # read-only and, with no AKS cluster/kubectl context in CI, every kubectl call fails + # gracefully — so it is inert regardless. Because earlyTerminate is set, the `completed` + # grader is intentionally omitted (early-terminated runs always fail it by design). + - name: "Pod-evidence script invoked for CrashLoopBackOff" + prompt: "My AKS pod is stuck in CrashLoopBackOff. Gather the read-only failure evidence I need before I decide on a fix." + config: + runs: 1 + tags: + type: integration + tier: full + cost: llm + area: response-quality + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"^(bash|powershell|pwsh)$","argsPattern":"(?i)pod-evidence\\.(sh|ps1)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-diagnostics + - type: tool-calls + config: + required: + # Copilot CLI uses "powershell" tool for executing scripts on Windows + # and "bash" for executing scripts on other platforms. + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)pod-evidence\\.(sh|ps1)" + # Global: no_runtime_failure - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" \ No newline at end of file From f4f20d582e97b10d1cc267ef8529d08468748411 Mon Sep 17 00:00:00 2001 From: "Tom Meschter (from Dev Box)" Date: Tue, 28 Jul 2026 13:58:16 -0700 Subject: [PATCH 4/5] fix: harden pod-evidence scripts per PR review Address Copilot review on PR #2933: - sh: capture --all-failing scan via command substitution and check exit status so a kubectl failure exits 1 instead of being misreported as "no unhealthy pods found" under set -e. - sh: read the Events section with single-pass awk instead of piping describe into head, avoiding a SIGPIPE abort under set -o pipefail. - ps1: drop the redundant $ErrorActionPreference = "Continue" (already the default); keep the explanatory comment. - ps1: validate -Tail is a positive integer and exit 2 on bad input, matching the bash --tail check. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bdfcb108-93f9-4399-954c-f96110af0428 --- .../scripts/pod-evidence.ps1 | 11 ++++++--- .../azure-diagnostics/scripts/pod-evidence.sh | 23 +++++++++++++++---- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/plugins/azure-skills/skills/azure-diagnostics/scripts/pod-evidence.ps1 b/plugins/azure-skills/skills/azure-diagnostics/scripts/pod-evidence.ps1 index a22750fdc..ac8e6e5f3 100644 --- a/plugins/azure-skills/skills/azure-diagnostics/scripts/pod-evidence.ps1 +++ b/plugins/azure-skills/skills/azure-diagnostics/scripts/pod-evidence.ps1 @@ -40,15 +40,20 @@ param( ) # Best-effort: individual kubectl reads may fail (unreachable cluster, missing -# metrics-server, no previous logs). Keep the default "Continue" so a single failed -# read is suppressed via 2>$null and the digest proceeds instead of aborting. -$ErrorActionPreference = "Continue" +# metrics-server, no previous logs). PowerShell's default $ErrorActionPreference is +# "Continue", so a single failed read is suppressed via 2>$null and the digest proceeds +# instead of aborting — no explicit assignment needed. if (-not (Get-Command kubectl -ErrorAction SilentlyContinue)) { Write-Error "kubectl not found on PATH." exit 1 } +if ($Tail -le 0) { + Write-Error "-Tail must be a positive integer (got '$Tail')." + exit 2 +} + function Digest-Pod { param([string]$Ns, [string]$Name) diff --git a/plugins/azure-skills/skills/azure-diagnostics/scripts/pod-evidence.sh b/plugins/azure-skills/skills/azure-diagnostics/scripts/pod-evidence.sh index cffd27945..e1e8cd019 100644 --- a/plugins/azure-skills/skills/azure-diagnostics/scripts/pod-evidence.sh +++ b/plugins/azure-skills/skills/azure-diagnostics/scripts/pod-evidence.sh @@ -84,7 +84,11 @@ digest_pod() { echo "--- EVENTS ---" if kubectl describe pod "$pod" -n "$ns" >/dev/null 2>&1; then - kubectl describe pod "$pod" -n "$ns" 2>/dev/null | sed -n '/^Events:/,$p' | head -n 25 + # Read the whole describe output in a single awk pass (no `head` in the pipe: + # an early-exiting `head` would SIGPIPE the upstream kubectl and, under + # `set -o pipefail`, abort the script). awk consumes all input and prints only + # the first 25 lines of the Events section. + kubectl describe pod "$pod" -n "$ns" 2>/dev/null | awk '/^Events:/{f=1} f && n<25 {print; n++}' else echo "(unable to describe pod)" fi @@ -113,16 +117,27 @@ digest_pod() { if [ "$ALL_FAILING" = true ]; then echo "pod-evidence: scanning for pods not in Running/Succeeded${NAMESPACE:+ in namespace '$NAMESPACE'}..." - # Portable row collection (avoids `mapfile`, which is unavailable in Bash 3.2 / macOS). - ROWS=() + # Portable, set -e-safe row collection: capture output + exit status via command + # substitution (not process substitution, whose failures don't propagate under set -e) + # so a failed scan is reported as an error instead of a misleading "no unhealthy pods". + # Avoids `mapfile`, which is unavailable in Bash 3.2 / macOS. if [ -n "$NAMESPACE" ]; then SCAN_ARGS=(-n "$NAMESPACE") else SCAN_ARGS=(-A) fi + set +e + SCAN_OUT=$(kubectl get pods "${SCAN_ARGS[@]}" --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2>/dev/null) + SCAN_RC=$? + set -e + if [ "$SCAN_RC" -ne 0 ]; then + echo "ERROR: unable to list pods (kubectl exited $SCAN_RC). Check your cluster context and credentials." >&2 + exit 1 + fi + ROWS=() while IFS= read -r line; do [ -n "$line" ] && ROWS+=("$line") - done < <(kubectl get pods "${SCAN_ARGS[@]}" --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2>/dev/null) + done <<< "$SCAN_OUT" if [ "${#ROWS[@]}" -eq 0 ]; then echo "No unhealthy pods found (all pods are Running or Succeeded)." From 69747be858e702d4e785f94e88f6de56df38386e Mon Sep 17 00:00:00 2001 From: "Tom Meschter (from Dev Box)" Date: Tue, 4 Aug 2026 10:02:09 -0700 Subject: [PATCH 5/5] fix: use tool-call-result early-terminate in pod-evidence eval Per PR #2933 review (JasonYeMSFT): switch the pod-evidence-invoked stimulus's earlyTerminate from tool-call-match to tool-call-result so termination keys off the matched call's tool.execution_complete event (a confirmed invocation with a recorded result) rather than execution_start, which can be raced by termination before the result is captured. Update the accompanying comment to match. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: bdfcb108-93f9-4399-954c-f96110af0428 --- evals/azure-skills/azure-diagnostics/eval.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/evals/azure-skills/azure-diagnostics/eval.yaml b/evals/azure-skills/azure-diagnostics/eval.yaml index 3201838ca..d63fda46b 100644 --- a/evals/azure-skills/azure-diagnostics/eval.yaml +++ b/evals/azure-skills/azure-diagnostics/eval.yaml @@ -137,13 +137,15 @@ stimuli: # the raw kubectl describe/logs/top sequence (issue #2507). The `tool-calls` grader # asserts the agent attempted to execute pod-evidence.{sh,ps1} via a shell tool. # - # earlyTerminate fires the moment that shell call starts, so the agent is stopped right - # after the attempt (no follow-on turns). Note this is a guard/optimization, not a hard - # execution block: `tool-call-match` reacts to the `tool.execution_start` event, which - # the SDK emits only after the (auto-approved) tool has already launched. The script is - # read-only and, with no AKS cluster/kubectl context in CI, every kubectl call fails - # gracefully — so it is inert regardless. Because earlyTerminate is set, the `completed` - # grader is intentionally omitted (early-terminated runs always fail it by design). + # earlyTerminate stops the agent as soon as the pod-evidence call is observed, so there + # are no follow-on turns. It uses `tool-call-result`, which reacts to the matching call's + # `tool.execution_complete` event — i.e. the invocation is confirmed to have actually run + # and produced a result, which is the reliable signal (execution_start alone can be raced + # by termination before the result is recorded). This is a guard/optimization, not a hard + # execution block: the script is read-only and, with no AKS cluster/kubectl context in CI, + # every kubectl call fails gracefully — so it is inert regardless. Because earlyTerminate + # is set, the `completed` grader is intentionally omitted (early-terminated runs always + # fail it by design). - name: "Pod-evidence script invoked for CrashLoopBackOff" prompt: "My AKS pod is stuck in CrashLoopBackOff. Gather the read-only failure evidence I need before I decide on a fix." config: @@ -153,7 +155,7 @@ stimuli: tier: full cost: llm area: response-quality - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"^(bash|powershell|pwsh)$","argsPattern":"(?i)pod-evidence\\.(sh|ps1)"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"^(bash|powershell|pwsh)$","argsPattern":"(?i)pod-evidence\\.(sh|ps1)"}]' graders: - type: skill-invocation config: