diff --git a/evals/azure-skills/azure-diagnostics/eval.yaml b/evals/azure-skills/azure-diagnostics/eval.yaml index ebb0db78f..86a1c7f35 100644 --- a/evals/azure-skills/azure-diagnostics/eval.yaml +++ b/evals/azure-skills/azure-diagnostics/eval.yaml @@ -36,8 +36,8 @@ stimuli: tier: smoke cost: llm area: routing - requiredSkills: - - azure-diagnostics + requiredSkills: + - azure-diagnostics earlyTerminate: '[{"type":"skill-call","skill":"azure-diagnostics"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation @@ -59,8 +59,8 @@ stimuli: tier: full cost: llm area: routing - requiredSkills: - - azure-diagnostics + requiredSkills: + - azure-diagnostics earlyTerminate: '[{"type":"skill-call","skill":"azure-diagnostics"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation @@ -82,8 +82,8 @@ stimuli: tier: full cost: llm area: routing - requiredSkills: - - azure-diagnostics + requiredSkills: + - azure-diagnostics earlyTerminate: '[{"type":"skill-call","skill":"azure-diagnostics"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation @@ -104,8 +104,8 @@ stimuli: tier: full cost: llm area: routing - requiredSkills: - - azure-diagnostics + requiredSkills: + - azure-diagnostics earlyTerminate: '[{"type":"skill-call","skill":"azure-diagnostics"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation @@ -118,6 +118,47 @@ stimuli: - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" + # ── pod-evidence-script-invoked ── + # Response-quality check: for an AKS pod-failure prompt the skill should drive the + # agent to RUN the read-only pod-evidence evidence-bundle script rather than re-listing + # the raw kubectl describe/logs/top sequence (issue #2507). The `tool-calls` grader + # asserts the agent attempted to execute pod-evidence.{sh,ps1} via a shell tool. + # + # earlyTerminate stops the agent as soon as the pod-evidence call is observed, so there + # are no follow-on turns. It uses `tool-call-result`, which reacts to the matching call's + # `tool.execution_complete` event — i.e. the invocation is confirmed to have actually run + # and produced a result, which is the reliable signal (execution_start alone can be raced + # by termination before the result is recorded). This is a guard/optimization, not a hard + # execution block: the script is read-only and, with no AKS cluster/kubectl context in CI, + # every kubectl call fails gracefully — so it is inert regardless. Because earlyTerminate + # is set, the `completed` grader is intentionally omitted (early-terminated runs always + # fail it by design). + - name: "Pod-evidence script invoked for CrashLoopBackOff" + prompt: "My AKS pod is stuck in CrashLoopBackOff. Gather the read-only failure evidence I need before I decide on a fix." + config: + runs: 1 + tags: + type: integration + tier: full + cost: llm + area: response-quality + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"^(bash|powershell|pwsh)$","argsPattern":"(?i)pod-evidence\\.(sh|ps1)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-diagnostics + - type: tool-calls + config: + required: + # Copilot CLI uses "powershell" tool for executing scripts on Windows + # and "bash" for executing scripts on other platforms. + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)pod-evidence\\.(sh|ps1)" + # Global: no_runtime_failure + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" # ── messaging-namespace-connectivity-probe ── # Added with issue #2511: for the messaging "cannot connect at all" flow the @@ -173,8 +214,8 @@ stimuli: tier: full cost: llm area: routing - requiredSkills: - - azure-diagnostics + requiredSkills: + - azure-diagnostics earlyTerminate: '[{"type":"skill-call","skill":"azure-diagnostics"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation @@ -217,8 +258,8 @@ stimuli: tier: full cost: llm area: response-quality - requiredSkills: - - azure-diagnostics + requiredSkills: + - azure-diagnostics earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"aks-baseline\\.(ps1|sh)"}]' graders: - type: skill-invocation @@ -226,9 +267,8 @@ stimuli: required: - azure-diagnostics # The script's shell tool call completes (earlyTerminate fires on its - # *result*), so the built-in `tool-calls` grader can match it via the - # tool_result. `required` matches on completion; this is the same shape the - # azure-validate e2e suite uses to confirm a script was run. + # result), so the built-in `tool-calls` grader can match it via the + # tool_result. - type: tool-calls config: required: diff --git a/plugins/azure-skills/skills/azure-diagnostics/scripts/pod-evidence.ps1 b/plugins/azure-skills/skills/azure-diagnostics/scripts/pod-evidence.ps1 new file mode 100644 index 000000000..ac8e6e5f3 --- /dev/null +++ b/plugins/azure-skills/skills/azure-diagnostics/scripts/pod-evidence.ps1 @@ -0,0 +1,150 @@ +<# +.SYNOPSIS + Collects the invariant, read-only AKS pod-failure evidence bundle and prints a + single labeled digest. +.DESCRIPTION + Gathers the same evidence bundle for one or more pods regardless of the symptom + (CrashLoopBackOff, OOMKilled, Pending, probe failures, ImagePullBackOff). For each + pod it collects and summarizes: + STATUS - READY / phase / restart count (kubectl get pod -o wide) + STATE - exit code, reason, last-state snippet (jsonpath over containerStatuses) + EVENTS - the Events section (kubectl describe pod) + LOGS - current container logs (tailed) (kubectl logs) + PREV LOGS - previous/crashed container logs (tailed)(kubectl logs --previous) + RESOURCES - requests/limits vs live usage (jsonpath + kubectl top pod) + + This script only GATHERS and DIGESTS evidence. It never mutates cluster state. + Interpreting the digest to pick a fix stays with the caller. +.PARAMETER Pod + Pod name (single-pod mode). Omit when using -AllFailing. +.PARAMETER Namespace + Namespace of the pod. Required in single-pod mode. In -AllFailing mode, limits the + scan to this namespace. +.PARAMETER AllFailing + Auto-select every pod not in Running/Succeeded phase (across all namespaces unless + -Namespace is given) and digest each. +.PARAMETER Tail + Number of log lines to show per stream. Default 50. +.EXAMPLE + .\pod-evidence.ps1 my-api-7d9f-abcde -Namespace prod +.EXAMPLE + .\pod-evidence.ps1 -AllFailing +.EXAMPLE + .\pod-evidence.ps1 -AllFailing -Namespace prod -Tail 100 +#> +param( + [Parameter(Position = 0)][string]$Pod, + [Alias("n")][string]$Namespace, + [switch]$AllFailing, + [int]$Tail = 50 +) + +# Best-effort: individual kubectl reads may fail (unreachable cluster, missing +# metrics-server, no previous logs). PowerShell's default $ErrorActionPreference is +# "Continue", so a single failed read is suppressed via 2>$null and the digest proceeds +# instead of aborting — no explicit assignment needed. + +if (-not (Get-Command kubectl -ErrorAction SilentlyContinue)) { + Write-Error "kubectl not found on PATH." + exit 1 +} + +if ($Tail -le 0) { + Write-Error "-Tail must be a positive integer (got '$Tail')." + exit 2 +} + +function Digest-Pod { + param([string]$Ns, [string]$Name) + + Write-Host "==================================================================" + Write-Host "POD: $Name NAMESPACE: $Ns" + Write-Host "==================================================================" + + Write-Host "--- STATUS (ready / phase / restarts) ---" + $status = kubectl get pod $Name -n $Ns -o wide 2>&1 + if ($LASTEXITCODE -eq 0 -and $status) { $status | ForEach-Object { Write-Host "$_" } } else { Write-Host "(unable to get pod)" } + Write-Host "" + + Write-Host "--- STATE (exit code / reason / last state) ---" + $jp = '{range .status.containerStatuses[*]}container={.name}{"`n"} ready={.ready} restarts={.restartCount}{"`n"} current: waiting={.state.waiting.reason} running={.state.running.startedAt} terminated={.state.terminated.reason}(exit={.state.terminated.exitCode}){"`n"} lastState: terminated={.lastState.terminated.reason}(exit={.lastState.terminated.exitCode}) at {.lastState.terminated.finishedAt}{"`n"}{end}' + $state = kubectl get pod $Name -n $Ns -o jsonpath=$jp 2>$null + if ($state) { Write-Host $state } else { Write-Host "(no container status available)" } + Write-Host "" + + Write-Host "--- EVENTS ---" + $desc = kubectl describe pod $Name -n $Ns 2>$null + if ($desc) { + $idx = ($desc | Select-String -Pattern '^Events:' | Select-Object -First 1).LineNumber + if ($idx) { + $desc | Select-Object -Skip ($idx - 1) | Select-Object -First 25 | ForEach-Object { Write-Host "$_" } + } else { + Write-Host "(no Events section)" + } + } else { + Write-Host "(unable to describe pod)" + } + Write-Host "" + + Write-Host "--- LOGS (current, last $Tail lines) ---" + $logs = kubectl logs $Name -n $Ns --tail=$Tail 2>&1 + if ($logs) { $logs | ForEach-Object { Write-Host "$_" } } else { Write-Host "(no current logs)" } + Write-Host "" + + Write-Host "--- PREV LOGS (previous instance, last $Tail lines) ---" + $prev = kubectl logs $Name -n $Ns --previous --tail=$Tail 2>$null + if ($LASTEXITCODE -eq 0 -and $prev) { + Write-Host $prev + } else { + Write-Host "(no previous-instance logs - pod has not restarted or they were rotated)" + } + Write-Host "" + + Write-Host "--- RESOURCES (requests/limits vs live usage) ---" + Write-Host "requests/limits:" + $res = kubectl get pod $Name -n $Ns -o jsonpath='{range .spec.containers[*]} {.name}: requests={.resources.requests} limits={.resources.limits}{"`n"}{end}' 2>$null + if ($res) { Write-Host $res } else { Write-Host " (unable to read resources)" } + Write-Host "live usage:" + $top = kubectl top pod $Name -n $Ns 2>&1 + if ($top) { $top | ForEach-Object { Write-Host " $_" } } else { Write-Host " (metrics-server unavailable)" } + Write-Host "" +} + +if ($AllFailing) { + $scope = if ($Namespace) { " in namespace '$Namespace'" } else { "" } + Write-Host "pod-evidence: scanning for pods not in Running/Succeeded$scope..." + + $cols = "custom-columns=NS:.metadata.namespace,NAME:.metadata.name" + if ($Namespace) { + $rows = kubectl get pods -n $Namespace --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers -o $cols 2>$null + } else { + $rows = kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers -o $cols 2>$null + } + + $rows = @($rows | Where-Object { $_ -and $_.Trim() }) + if ($rows.Count -eq 0) { + Write-Host "No unhealthy pods found (all pods are Running or Succeeded)." + exit 0 + } + + Write-Host "Found $($rows.Count) unhealthy pod(s). Collecting evidence for each below." + Write-Host "" + foreach ($row in $rows) { + $parts = ($row -split '\s+') | Where-Object { $_ } + Digest-Pod -Ns $parts[0] -Name $parts[1] + } + Write-Host "pod-evidence: done. Reviewed $($rows.Count) failing pod(s) - use the STATE/EVENTS/LOGS above to pick a fix." +} else { + if (-not $Pod) { + Write-Error "A pod name is required (or use -AllFailing)." + exit 2 + } + if (-not $Namespace) { + Write-Error "-Namespace is required in single-pod mode." + exit 2 + } + Write-Host "pod-evidence: collecting the read-only evidence bundle for pod '$Pod' in namespace '$Namespace'." + Write-Host "" + Digest-Pod -Ns $Namespace -Name $Pod + Write-Host "pod-evidence: done. Use the STATE/EVENTS/LOGS/RESOURCES digest above to pick a fix." +} diff --git a/plugins/azure-skills/skills/azure-diagnostics/scripts/pod-evidence.sh b/plugins/azure-skills/skills/azure-diagnostics/scripts/pod-evidence.sh new file mode 100644 index 000000000..e1e8cd019 --- /dev/null +++ b/plugins/azure-skills/skills/azure-diagnostics/scripts/pod-evidence.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash +# pod-evidence.sh +# Collects the invariant, read-only AKS pod-failure evidence bundle for one or more +# pods and prints a single labeled digest. Works identically regardless of the pod +# symptom (CrashLoopBackOff, OOMKilled, Pending, probe failures, ImagePullBackOff). +# +# For each pod it gathers and summarizes: +# STATUS - READY / phase / restart count (kubectl get pod -o wide) +# STATE - exit code, reason, last-state snippet (jsonpath over containerStatuses) +# EVENTS - the Events section (kubectl describe pod) +# LOGS - current container logs (tailed) (kubectl logs) +# PREV LOGS - previous/crashed container logs (tailed)(kubectl logs --previous) +# RESOURCES - requests/limits vs live usage (jsonpath + kubectl top pod) +# +# This script only GATHERS and DIGESTS evidence. It never mutates cluster state. +# Interpreting the digest to pick a fix (exit-code / event / probe decision tables) +# stays with the caller. +# +# Usage: +# ./pod-evidence.sh -n [--tail ] +# ./pod-evidence.sh --all-failing [-n ] [--tail ] +# +# Options: +# -n, --namespace Namespace of the pod. Required in single-pod mode. +# In --all-failing mode, limits the scan to this namespace. +# --all-failing Auto-select every pod not in Running/Succeeded phase +# (across all namespaces unless -n is given) and digest each. +# --tail Number of log lines to show per stream (default 50). +# -h, --help Show this help. +# +# Examples: +# ./pod-evidence.sh my-api-7d9f-abcde -n prod +# ./pod-evidence.sh --all-failing +# ./pod-evidence.sh --all-failing -n prod --tail 100 + +set -euo pipefail + +POD="" +NAMESPACE="" +ALL_FAILING=false +TAIL=50 + +usage() { + sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//' +} + +while [ $# -gt 0 ]; do + case "$1" in + -n|--namespace) NAMESPACE="${2:?--namespace requires a value}"; shift 2 ;; + --all-failing) ALL_FAILING=true; shift ;; + --tail) TAIL="${2:?--tail requires a value}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + -*) echo "Unknown option: $1" >&2; usage; exit 2 ;; + *) POD="$1"; shift ;; + esac +done + +if ! command -v kubectl >/dev/null 2>&1; then + echo "ERROR: kubectl not found on PATH." >&2 + exit 1 +fi + +case "$TAIL" in + ''|*[!0-9]*) echo "ERROR: --tail must be a positive integer (got '$TAIL')." >&2; exit 2 ;; + 0) echo "ERROR: --tail must be a positive integer (got '$TAIL')." >&2; exit 2 ;; +esac + +# Digest a single pod. Args: +digest_pod() { + local ns="$1" pod="$2" + + echo "==================================================================" + echo "POD: $pod NAMESPACE: $ns" + echo "==================================================================" + + echo "--- STATUS (ready / phase / restarts) ---" + kubectl get pod "$pod" -n "$ns" -o wide 2>&1 || echo "(unable to get pod)" + echo "" + + echo "--- STATE (exit code / reason / last state) ---" + kubectl get pod "$pod" -n "$ns" -o jsonpath='{range .status.containerStatuses[*]}container={.name}{"\n"} ready={.ready} restarts={.restartCount}{"\n"} current: waiting={.state.waiting.reason} running={.state.running.startedAt} terminated={.state.terminated.reason}(exit={.state.terminated.exitCode}){"\n"} lastState: terminated={.lastState.terminated.reason}(exit={.lastState.terminated.exitCode}) at {.lastState.terminated.finishedAt}{"\n"}{end}' 2>/dev/null \ + || echo "(no container status available)" + echo "" + + echo "--- EVENTS ---" + if kubectl describe pod "$pod" -n "$ns" >/dev/null 2>&1; then + # Read the whole describe output in a single awk pass (no `head` in the pipe: + # an early-exiting `head` would SIGPIPE the upstream kubectl and, under + # `set -o pipefail`, abort the script). awk consumes all input and prints only + # the first 25 lines of the Events section. + kubectl describe pod "$pod" -n "$ns" 2>/dev/null | awk '/^Events:/{f=1} f && n<25 {print; n++}' + else + echo "(unable to describe pod)" + fi + echo "" + + echo "--- LOGS (current, last $TAIL lines) ---" + kubectl logs "$pod" -n "$ns" --tail="$TAIL" 2>&1 || echo "(no current logs)" + echo "" + + echo "--- PREV LOGS (previous instance, last $TAIL lines) ---" + if kubectl logs "$pod" -n "$ns" --previous --tail="$TAIL" 2>/dev/null; then + : + else + echo "(no previous-instance logs - pod has not restarted or they were rotated)" + fi + echo "" + + echo "--- RESOURCES (requests/limits vs live usage) ---" + echo "requests/limits:" + kubectl get pod "$pod" -n "$ns" -o jsonpath='{range .spec.containers[*]} {.name}: requests={.resources.requests} limits={.resources.limits}{"\n"}{end}' 2>/dev/null \ + || echo " (unable to read resources)" + echo "live usage:" + kubectl top pod "$pod" -n "$ns" 2>&1 | sed 's/^/ /' || echo " (metrics-server unavailable)" + echo "" +} + +if [ "$ALL_FAILING" = true ]; then + echo "pod-evidence: scanning for pods not in Running/Succeeded${NAMESPACE:+ in namespace '$NAMESPACE'}..." + # Portable, set -e-safe row collection: capture output + exit status via command + # substitution (not process substitution, whose failures don't propagate under set -e) + # so a failed scan is reported as an error instead of a misleading "no unhealthy pods". + # Avoids `mapfile`, which is unavailable in Bash 3.2 / macOS. + if [ -n "$NAMESPACE" ]; then + SCAN_ARGS=(-n "$NAMESPACE") + else + SCAN_ARGS=(-A) + fi + set +e + SCAN_OUT=$(kubectl get pods "${SCAN_ARGS[@]}" --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name 2>/dev/null) + SCAN_RC=$? + set -e + if [ "$SCAN_RC" -ne 0 ]; then + echo "ERROR: unable to list pods (kubectl exited $SCAN_RC). Check your cluster context and credentials." >&2 + exit 1 + fi + ROWS=() + while IFS= read -r line; do + [ -n "$line" ] && ROWS+=("$line") + done <<< "$SCAN_OUT" + + if [ "${#ROWS[@]}" -eq 0 ]; then + echo "No unhealthy pods found (all pods are Running or Succeeded)." + exit 0 + fi + + echo "Found ${#ROWS[@]} unhealthy pod(s). Collecting evidence for each below." + echo "" + for row in "${ROWS[@]}"; do + [ -z "$row" ] && continue + ns="${row%% *}" + name="${row##* }" + digest_pod "$ns" "$name" + done + echo "pod-evidence: done. Reviewed ${#ROWS[@]} failing pod(s) - use the STATE/EVENTS/LOGS above to pick a fix." +else + if [ -z "$POD" ]; then + echo "ERROR: a pod name is required (or use --all-failing)." >&2 + usage + exit 2 + fi + if [ -z "$NAMESPACE" ]; then + echo "ERROR: -n/--namespace is required in single-pod mode." >&2 + exit 2 + fi + echo "pod-evidence: collecting the read-only evidence bundle for pod '$POD' in namespace '$NAMESPACE'." + echo "" + digest_pod "$NAMESPACE" "$POD" + echo "pod-evidence: done. Use the STATE/EVENTS/LOGS/RESOURCES digest above to pick a fix." +fi diff --git a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md index 264e42a70..5592c3c59 100644 --- a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md +++ b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md @@ -84,10 +84,29 @@ When AKS-MCP cannot perform the baseline read, run the **[`aks-baseline`](../../ Then deep-dive on a specific pod as the digest indicates: ```bash +az aks show -g -n +az aks nodepool list -g --cluster-name +kubectl cluster-info +kubectl get nodes -o wide +kubectl get pods -n kube-system +kubectl get events -A --sort-by=.lastTimestamp kubectl describe pod -n kubectl logs -n --previous ``` +For unhealthy pods, gather the full read-only evidence bundle (describe, current + previous logs, resources vs usage) with the pod-evidence script instead of running the commands one by one — [`../../scripts/pod-evidence.sh`](../../scripts/pod-evidence.sh) / [`../../scripts/pod-evidence.ps1`](../../scripts/pod-evidence.ps1): + +```bash +../../scripts/pod-evidence.sh -n +../../scripts/pod-evidence.sh --all-failing +``` +```powershell +../../scripts/pod-evidence.ps1 -Namespace +../../scripts/pod-evidence.ps1 -AllFailing +``` + +See [pod-failures.md](pod-failures.md) for how to interpret the digest. + Keep these read-only unless the user explicitly asks for remediation. ## Guardrails diff --git a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md index 9a2c4e33a..f61d2e8cf 100644 --- a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md +++ b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md @@ -1,34 +1,28 @@ # Pod Failures & Application Issues -## Common Pod Diagnostic Commands +## Evidence Bundle Script + +For **any** pod symptom below, run the **pod-evidence** script to collect the same +read-only evidence bundle. Per pod it digests **STATUS**, **STATE** (exit code, reason, +last state), **EVENTS**, current/previous **LOGS**, and **RESOURCES** (requests vs +`top`). It only gathers; interpret with the tables. + +Bash [`../../scripts/pod-evidence.sh`](../../scripts/pod-evidence.sh) · PowerShell [`../../scripts/pod-evidence.ps1`](../../scripts/pod-evidence.ps1) ```bash -# List unhealthy pods across all namespaces -kubectl get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded -# All pods wide view -kubectl get pods -A -o wide -# Detailed pod status - events section is critical -kubectl describe pod -n -# Pod logs (current and previous crash) -kubectl logs -n -kubectl logs -n --previous +../../scripts/pod-evidence.sh -n # one pod +../../scripts/pod-evidence.sh --all-failing # all unhealthy pods ``` +PowerShell: `../../scripts/pod-evidence.ps1 -Namespace ` (`-AllFailing` scans all). + --- ## CrashLoopBackOff Pod starts, crashes, restarts with exponential backoff (10s, 20s, 40s... up to 5m). -**Diagnostics:** - -```bash -kubectl describe pod -n -# Check: Exit Code, Reason, Last State, Events - -kubectl logs -n --previous -# Shows stdout/stderr from the last crashed container -``` +**Diagnostics:** [pod-evidence](#evidence-bundle-script) → read **STATE** (exit code, reason, last state) and **PREV LOGS** (last crashed container). **Decision tree:** @@ -40,14 +34,7 @@ kubectl logs -n --previous | `139` | Segfault (SIGSEGV) | Binary compatibility issue or native code bug | | `143` | SIGTERM - graceful shutdown | Pod was terminated; check if liveness probe killed it | -**OOMKilled specifically:** - -```bash -kubectl describe pod -n | grep -A2 "Last State" -# Reason: OOMKilled -> container exceeded memory limit -``` - -Fix: increase `resources.limits.memory` or optimize application memory usage. Check `kubectl top pod -n ` for actual usage. +**OOMKilled specifically:** the **STATE** section shows `terminated=OOMKilled` and **RESOURCES** shows the memory limit vs live usage. Fix: increase `resources.limits.memory` or optimize application memory usage. **OOM kill tracing with Inspektor Gadget:** Use `trace_oomkill` (timeout 30) with `--k8s-namespace --k8s-podname ` to see which process was killed and memory at kill time. See [references/inspektor-gadget.md](references/inspektor-gadget.md). @@ -67,12 +54,7 @@ See [references/inspektor-gadget.md](references/inspektor-gadget.md). Pod can't pull the container image. -**Diagnostics:** - -```bash -kubectl describe pod -n -# Events section shows the exact pull error -``` +**Diagnostics:** [pod-evidence](#evidence-bundle-script) → read **EVENTS** for the exact pull error. | Error Message | Cause | Fix | | --------------------------------------- | ---------------------------- | -------------------------------------------------------------- | @@ -94,12 +76,7 @@ az aks check-acr -g -n --acr .azurecr.io Pod stays in `Pending` - scheduler can't place it. -**Diagnostics:** - -```bash -kubectl describe pod -n -# Events section shows why scheduling failed -``` +**Diagnostics:** [pod-evidence](#evidence-bundle-script) → read **EVENTS** for why scheduling failed. | Event Message | Cause | Fix | | ---------------------------------------------------------------------- | ----------------------------------- | --------------------------------------------------------------- | @@ -115,15 +92,7 @@ kubectl describe pod -n **Readiness probe failure** -> pod removed from Service endpoints (no traffic). **Liveness probe failure** -> pod killed and restarted. -**Diagnostics:** - -```bash -kubectl describe pod -n -# Look for: "Readiness probe failed" or "Liveness probe failed" in Events - -# Check the pod's READY column - must show n/n -kubectl get pod -n -``` +**Diagnostics:** [pod-evidence](#evidence-bundle-script) → **EVENTS** shows `Readiness/Liveness probe failed`; **STATUS** shows the READY column (must be n/n). | Symptom | Cause | Fix | | ------------------------------------ | ----------------------- | ---------------------------------------------------------- | @@ -137,15 +106,7 @@ kubectl get pod -n ## Resource Constraints (CPU/Memory) -**Check actual usage vs limits:** - -```bash -kubectl top pod -n -kubectl top pod -n --sort-by=memory - -# Compare with requests/limits -kubectl get pod -n -o jsonpath='{.spec.containers[*].resources}' -``` +**Check actual usage vs limits:** [pod-evidence](#evidence-bundle-script) → **RESOURCES** compares requests/limits against live `top` usage. To rank a namespace by memory: `kubectl top pod -n --sort-by=memory`. | Symptom | Cause | Fix | | -------------------------------- | --------------------------------------- | --------------------------------------------------- | diff --git a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md index 90f9e8915..e96846694 100644 --- a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md +++ b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md @@ -27,11 +27,25 @@ Check API reachability -> inspect nodes -> inspect kube-system -> inspect events CLI fallback when AKS-MCP cannot perform the Kubernetes baseline read — the same **[`aks-baseline`](../../../scripts/aks-baseline.sh)** script also covers node readiness, unhealthy pods, kube-system health, and recent warning events. Pass `--namespace` to include an affected namespace, then deep-dive on a specific pod: ```bash -# bash -./scripts/aks-baseline.sh -g -n --namespace +kubectl cluster-info +kubectl get nodes -o wide +kubectl get pods -n kube-system +kubectl get events -A --sort-by=.lastTimestamp +kubectl get pods -n +``` + +For pod detail and logs, gather the read-only evidence bundle (describe, current + previous logs, resources vs usage) with the pod-evidence script — [`../../../scripts/pod-evidence.sh`](../../../scripts/pod-evidence.sh) / [`../../../scripts/pod-evidence.ps1`](../../../scripts/pod-evidence.ps1): + +```bash +../../../scripts/pod-evidence.sh -n +../../../scripts/pod-evidence.sh --all-failing kubectl describe pod -n kubectl logs -n --previous ``` +```powershell +../../../scripts/pod-evidence.ps1 -Namespace +../../../scripts/pod-evidence.ps1 -AllFailing +``` ```powershell # PowerShell