diff --git a/evals/azure-skills/azure-diagnostics/eval.yaml b/evals/azure-skills/azure-diagnostics/eval.yaml index bf7879c31..88d41d321 100644 --- a/evals/azure-skills/azure-diagnostics/eval.yaml +++ b/evals/azure-skills/azure-diagnostics/eval.yaml @@ -36,8 +36,8 @@ stimuli: tier: smoke cost: llm area: routing - requiredSkills: - - azure-diagnostics + requiredSkills: + - azure-diagnostics earlyTerminate: '[{"type":"skill-call","skill":"azure-diagnostics"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation @@ -59,8 +59,8 @@ stimuli: tier: full cost: llm area: routing - requiredSkills: - - azure-diagnostics + requiredSkills: + - azure-diagnostics earlyTerminate: '[{"type":"skill-call","skill":"azure-diagnostics"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation @@ -82,8 +82,8 @@ stimuli: tier: full cost: llm area: routing - requiredSkills: - - azure-diagnostics + requiredSkills: + - azure-diagnostics earlyTerminate: '[{"type":"skill-call","skill":"azure-diagnostics"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation @@ -104,8 +104,8 @@ stimuli: tier: full cost: llm area: routing - requiredSkills: - - azure-diagnostics + requiredSkills: + - azure-diagnostics earlyTerminate: '[{"type":"skill-call","skill":"azure-diagnostics"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation @@ -128,8 +128,8 @@ stimuli: tier: full cost: llm area: routing - requiredSkills: - - azure-diagnostics + requiredSkills: + - azure-diagnostics earlyTerminate: '[{"type":"skill-call","skill":"azure-diagnostics"},{"type":"tool-call-count","count":3}]' graders: - type: skill-invocation @@ -143,6 +143,42 @@ stimuli: config: pattern: "(?i)fatal error|unhandled exception|stack trace" + # ── inspektor-gadget-run-ig-script ── + # Exercises the run-ig helper script for Inspektor Gadget without a live + # cluster. earlyTerminate uses tool-call-result so the run stops once the + # run-ig invocation completes: with no kubectl/cluster in CI the underlying + # `kubectl debug` fails instantly and harmlessly, and the completed tool call + # is fully recorded for the grader to confirm the script was invoked. + # NOTE: early-terminated runs must NOT use the `completed` grader. + - name: "Inspektor Gadget run-ig script invocation" + prompt: "I'm troubleshooting DNS failures on AKS node aks-nodepool1-12345678-vmss000000. Use the azure-diagnostics Inspektor Gadget helper script to run a trace_dns gadget on that node." + config: + runs: 1 + tags: + type: integration + tier: full + cost: llm + area: behavior + requiredSkills: + - azure-diagnostics + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh","argsPattern":"run-ig\\.(sh|ps1)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-diagnostics + # The agent actually invoked the run-ig helper script. + - type: tool-calls + config: + required: + # Copilot CLI uses "powershell" on Windows, "bash" on other platforms. + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)run-ig\\.(sh|ps1)" + # Global: no_runtime_failure + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + # ═══════════════════════════════════════════ # Script execution tests # ═══════════════════════════════════════════ @@ -172,8 +208,8 @@ stimuli: tier: full cost: llm area: response-quality - requiredSkills: - - azure-diagnostics + requiredSkills: + - azure-diagnostics earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh|run_in_terminal","argsPattern":"aks-baseline\\.(ps1|sh)"}]' graders: - type: skill-invocation diff --git a/plugins/azure-skills/skills/azure-diagnostics/scripts/run-ig.ps1 b/plugins/azure-skills/skills/azure-diagnostics/scripts/run-ig.ps1 new file mode 100644 index 000000000..ab2acc972 --- /dev/null +++ b/plugins/azure-skills/skills/azure-diagnostics/scripts/run-ig.ps1 @@ -0,0 +1,184 @@ +<# +.SYNOPSIS + Runs an Inspektor Gadget (IG) trace on an AKS node via `kubectl debug`. + +.DESCRIPTION + Handles the mechanical, error-prone assembly of the IG invocation: + - resolves the target node from a pod (or takes a node directly) + - injects the pinned IG image + version + - applies the correct default -Timeout for the gadget type + - adds the k8s namespace/pod/container filters + - handles the special `tcpdump` gadget (pcap-ng output piped to tcpdump) + + The privileged debug pod requires explicit user approval and appropriate RBAC. + Use -DryRun to print the assembled command without running it. + +.PARAMETER Gadget + Gadget to run, e.g. trace_dns, snapshot_socket, tcpdump (required). + +.PARAMETER Pod + Pod name; the node is resolved automatically. + +.PARAMETER Namespace + Namespace of the pod (required with -Pod). + +.PARAMETER Node + Run directly against a node (node-wide scope). + +.PARAMETER Container + Scope to a specific container. + +.PARAMETER Timeout + Override the gadget-type default timeout (seconds). + +.PARAMETER Filter + Extra IG flags, passed through verbatim (e.g. -Filter --max-entries,20). + +.PARAMETER Pf + tcpdump packet filter (tcpdump gadget only, e.g. "port 80"). + +.PARAMETER IgVersion + Override the pinned IG image tag. + +.PARAMETER DryRun + Print the assembled command; do not execute. + +.EXAMPLE + ./run-ig.ps1 -Gadget trace_dns -Pod web-0 -Namespace default + +.EXAMPLE + ./run-ig.ps1 -Gadget snapshot_process -Node aks-nodepool1-1234 + +.EXAMPLE + ./run-ig.ps1 -Gadget tcpdump -Pod web-0 -Namespace default -Pf "port 80" + +.EXAMPLE + ./run-ig.ps1 -Gadget traceloop -Pod web-0 -Namespace default -Filter --syscall-filters,open,connect + +.EXAMPLE + ./run-ig.ps1 -Gadget trace_dns -Pod web-0 -Namespace default -DryRun +#> +[CmdletBinding()] +param( + [string]$Gadget, + [string]$Pod, + [Alias('Ns')] + [string]$Namespace, + [string]$Node, + [string]$Container, + [int]$Timeout, + [string[]]$Filter, + [string]$Pf, + # Pinned IG image tag. Bump this default (and run-ig.sh) to update the IG version. + [string]$IgVersion = 'v0.51.0', + [switch]$DryRun +) + +$IgImageRepo = 'mcr.microsoft.com/oss/v2/inspektor-gadget/ig' + +if (-not $Gadget) { + Write-Error 'Provide -Gadget (e.g. trace_dns, snapshot_socket, tcpdump).' + exit 2 +} +if (-not $Node -and -not $Pod) { + Write-Error 'Provide either -Node or -Pod -Namespace .' + exit 2 +} +if ($Pod -and -not $Namespace) { + Write-Error '-Pod requires -Namespace .' + exit 2 +} +if ($Pf -and $Gadget -ne 'tcpdump') { + Write-Error '-Pf is only valid for the tcpdump gadget.' + exit 2 +} + +# Default timeout by gadget type, inferred from the gadget name prefix. +# snapshot_* / top_* -> 5s (point-in-time / quick aggregate) +# trace_* / profile_* / tcpdump -> 30s (streaming / sampling) +function Get-DefaultTimeout([string]$g) { + switch -Wildcard ($g) { + 'snapshot_*' { return 5 } + 'top_*' { return 5 } + 'trace_*' { return 30 } + 'profile_*' { return 30 } + 'tcpdump' { return 30 } + default { return 30 } # unknown gadget: use the safer streaming default + } +} + +if (-not $PSBoundParameters.ContainsKey('Timeout') -or $Timeout -le 0) { + $Timeout = Get-DefaultTimeout $Gadget +} + +# Resolve the node name from the pod when not given directly. +if (-not $Node) { + $Node = ((& kubectl get pod $Pod -n $Namespace -o "jsonpath={.spec.nodeName}" 2>$null) | Out-String).Trim() + if (-not $Node) { + Write-Error "Could not resolve node for pod '$Pod' in namespace '$Namespace'." + exit 1 + } +} + +$IgImage = "${IgImageRepo}:${IgVersion}" + +# Assemble the k8s scoping filters. +$filters = @() +if ($Namespace) { $filters += @('--k8s-namespace', $Namespace) } +if ($Pod) { $filters += @('--k8s-podname', $Pod) } +if ($Container) { $filters += @('--k8s-containername', $Container) } + +# Base kubectl debug invocation. +$debug = @('debug', '--profile=sysadmin', "node/$Node", '--attach', '--quiet', "--image=$IgImage", '--') + +if ($Gadget -eq 'tcpdump') { + # tcpdump emits raw pcap-ng; pipe through tcpdump for readable output when available. + $igCmd = @('ig', 'run', "tcpdump:$IgVersion", '-o', 'pcap-ng') + $filters + @('--timeout', "$Timeout") + if ($Pf) { $igCmd += @('--pf', $Pf) } + if ($Filter) { $igCmd += $Filter } +} +else { + $igCmd = @('ig', 'run', "${Gadget}:$IgVersion", '-o', 'json') + $filters + @('--timeout', "$Timeout") + if ($Filter) { $igCmd += $Filter } +} + +$fullArgs = $debug + $igCmd + +# Pretty-print a shell-quoted version of the command for display. +function Format-Cmd([string[]]$parts) { + ($parts | ForEach-Object { + if ($_ -match '\s') { '"' + $_ + '"' } else { $_ } + }) -join ' ' +} + +$displayCmd = 'kubectl ' + (Format-Cmd $fullArgs) + +# The tcpdump gadget is only piped through `tcpdump` when that binary is present. +# Reflect the real behavior in the displayed command so -DryRun does not mislead. +$tcpdumpAvail = $Gadget -eq 'tcpdump' -and [bool](Get-Command tcpdump -ErrorAction SilentlyContinue) +if ($tcpdumpAvail) { + $displayCmd = "$displayCmd | tcpdump -nvr -" +} + +Write-Host "Gadget: $Gadget" +Write-Host "Node: $Node" +Write-Host "Timeout: ${Timeout}s" +Write-Host "Image: $IgImage" +Write-Host "Command: $displayCmd" +if ($Gadget -eq 'tcpdump' -and -not $tcpdumpAvail) { + Write-Host 'Note: tcpdump not found; emitting raw pcap-ng to stdout.' +} + +if ($DryRun) { + Write-Host '(dry-run: command not executed)' + exit 0 +} + +Write-Host "Ran gadget $Gadget on node $Node (timeout ${Timeout}s)" + +if ($tcpdumpAvail) { + & kubectl @fullArgs | & tcpdump -nvr - +} +else { + & kubectl @fullArgs +} diff --git a/plugins/azure-skills/skills/azure-diagnostics/scripts/run-ig.sh b/plugins/azure-skills/skills/azure-diagnostics/scripts/run-ig.sh new file mode 100644 index 000000000..c1286d813 --- /dev/null +++ b/plugins/azure-skills/skills/azure-diagnostics/scripts/run-ig.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# run-ig.sh +# Runs an Inspektor Gadget (IG) trace on an AKS node via `kubectl debug`. +# +# Handles the mechanical, error-prone assembly of the IG invocation: +# - resolves the target node from a pod (or takes a node directly) +# - injects the pinned IG image + version +# - applies the correct default --timeout for the gadget type +# - adds the k8s namespace/pod/container filters +# - handles the special `tcpdump` gadget (pcap-ng output piped to tcpdump) +# +# The privileged debug pod requires explicit user approval and appropriate RBAC. +# Use --dry-run to print the assembled command without running it. +# +# Usage: +# ./run-ig.sh --gadget (--pod --ns | --node ) [options] +# +# Options: +# --gadget Gadget to run, e.g. trace_dns, snapshot_socket, tcpdump (required) +# --pod Pod name; the node is resolved automatically +# --ns Namespace of the pod (required with --pod) +# --node Run directly against a node (node-wide scope) +# --container Scope to a specific container +# --timeout Override the gadget-type default timeout +# --filter Extra IG flag, repeatable (e.g. --filter --max-entries --filter 20) +# --pf "" tcpdump packet filter (tcpdump gadget only, e.g. "port 80") +# --ig-version Override the pinned IG image tag (default below) +# --dry-run Print the assembled command; do not execute +# +# Examples: +# ./run-ig.sh --gadget trace_dns --pod web-0 --ns default +# ./run-ig.sh --gadget snapshot_process --node aks-nodepool1-1234 +# ./run-ig.sh --gadget tcpdump --pod web-0 --ns default --pf "port 80" +# ./run-ig.sh --gadget traceloop --pod web-0 --ns default --filter --syscall-filters --filter open,connect +# ./run-ig.sh --gadget trace_dns --pod web-0 --ns default --dry-run + +set -euo pipefail + +# Pinned IG image tag. Bump this line (and run-ig.ps1) to update the IG version. +IG_VERSION="v0.51.0" +IG_IMAGE_REPO="mcr.microsoft.com/oss/v2/inspektor-gadget/ig" + +GADGET="" +POD="" +NS="" +NODE="" +CONTAINER="" +TIMEOUT="" +PF="" +DRY_RUN="false" +EXTRA_FILTERS=() + +usage() { + # Print the leading comment block (from line 2) as help, stopping at the + # first non-comment line so script code is never echoed. + awk 'NR>1 && /^#/ { sub(/^# ?/, ""); print; next } NR>1 { exit }' "$0" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --gadget) GADGET="${2:?--gadget requires a value}"; shift 2;; + --pod) POD="${2:?--pod requires a value}"; shift 2;; + --ns|--namespace) NS="${2:?--ns requires a value}"; shift 2;; + --node) NODE="${2:?--node requires a value}"; shift 2;; + --container) CONTAINER="${2:?--container requires a value}"; shift 2;; + --timeout) TIMEOUT="${2:?--timeout requires a value}"; shift 2;; + --filter) EXTRA_FILTERS+=("${2:?--filter requires a value}"); shift 2;; + --pf) PF="${2:?--pf requires a value}"; shift 2;; + --ig-version) IG_VERSION="${2:?--ig-version requires a value}"; shift 2;; + --dry-run) DRY_RUN="true"; shift;; + -h|--help) usage; exit 0;; + *) echo "Unknown argument: $1" >&2; usage >&2; exit 2;; + esac +done + +if [[ -z "$GADGET" ]]; then + echo "Error: --gadget is required." >&2 + exit 2 +fi + +if [[ -z "$NODE" && -z "$POD" ]]; then + echo "Error: provide either --node or --pod --ns ." >&2 + exit 2 +fi + +if [[ -n "$POD" && -z "$NS" ]]; then + echo "Error: --pod requires --ns ." >&2 + exit 2 +fi + +# Default timeout by gadget type, inferred from the gadget name prefix. +# snapshot_* / top_* -> 5s (point-in-time / quick aggregate) +# trace_* / profile_* / tcpdump -> 30s (streaming / sampling) +default_timeout() { + case "$1" in + snapshot_*|top_*) echo 5;; + trace_*|profile_*|tcpdump) echo 30;; + *) echo 30;; # unknown gadget: use the safer streaming default + esac +} + +if [[ -z "$TIMEOUT" ]]; then + TIMEOUT="$(default_timeout "$GADGET")" +fi + +# Resolve the node name from the pod when not given directly. +if [[ -z "$NODE" ]]; then + NODE="$(kubectl get pod "$POD" -n "$NS" -o jsonpath='{.spec.nodeName}')" + if [[ -z "$NODE" ]]; then + echo "Error: could not resolve node for pod '$POD' in namespace '$NS'." >&2 + exit 1 + fi +fi + +IG_IMAGE="${IG_IMAGE_REPO}:${IG_VERSION}" + +# Assemble the k8s scoping filters. +FILTERS=() +[[ -n "$NS" ]] && FILTERS+=(--k8s-namespace "$NS") +[[ -n "$POD" ]] && FILTERS+=(--k8s-podname "$POD") +[[ -n "$CONTAINER" ]] && FILTERS+=(--k8s-containername "$CONTAINER") + +# Base kubectl debug invocation. +DEBUG=(kubectl debug --profile=sysadmin "node/${NODE}" --attach --quiet --image="$IG_IMAGE" --) + +if [[ "$GADGET" == "tcpdump" ]]; then + # tcpdump emits raw pcap-ng; pipe through tcpdump for readable output when available. + IG_CMD=(ig run "tcpdump:${IG_VERSION}" -o pcap-ng "${FILTERS[@]}" --timeout "$TIMEOUT") + [[ -n "$PF" ]] && IG_CMD+=(--pf "$PF") + [[ ${#EXTRA_FILTERS[@]} -gt 0 ]] && IG_CMD+=("${EXTRA_FILTERS[@]}") +else + if [[ -n "$PF" ]]; then + echo "Error: --pf is only valid for the tcpdump gadget." >&2 + exit 2 + fi + IG_CMD=(ig run "${GADGET}:${IG_VERSION}" -o json "${FILTERS[@]}" --timeout "$TIMEOUT") + [[ ${#EXTRA_FILTERS[@]} -gt 0 ]] && IG_CMD+=("${EXTRA_FILTERS[@]}") +fi + +FULL_CMD=("${DEBUG[@]}" "${IG_CMD[@]}") + +# Pretty-print a shell-quoted version of the command for display. +quote_cmd() { + local out="" + local a + for a in "$@"; do + if [[ "$a" =~ [[:space:]] ]]; then + out+="\"$a\" " + else + out+="$a " + fi + done + echo "${out% }" +} + +DISPLAY_CMD="$(quote_cmd "${FULL_CMD[@]}")" + +# The tcpdump gadget is only piped through `tcpdump` when that binary is present. +# Reflect the real behavior in the displayed command so --dry-run does not mislead. +TCPDUMP_AVAIL="false" +if [[ "$GADGET" == "tcpdump" ]] && command -v tcpdump >/dev/null 2>&1; then + TCPDUMP_AVAIL="true" + DISPLAY_CMD="$DISPLAY_CMD | tcpdump -nvr -" +fi + +echo "Gadget: $GADGET" >&2 +echo "Node: $NODE" >&2 +echo "Timeout: ${TIMEOUT}s" >&2 +echo "Image: $IG_IMAGE" >&2 +echo "Command: $DISPLAY_CMD" >&2 +if [[ "$GADGET" == "tcpdump" && "$TCPDUMP_AVAIL" == "false" ]]; then + echo "Note: tcpdump not found; emitting raw pcap-ng to stdout." >&2 +fi + +if [[ "$DRY_RUN" == "true" ]]; then + echo "(dry-run: command not executed)" >&2 + exit 0 +fi + +echo "Ran gadget $GADGET on node $NODE (timeout ${TIMEOUT}s)" >&2 + +if [[ "$TCPDUMP_AVAIL" == "true" ]]; then + "${FULL_CMD[@]}" | tcpdump -nvr - +else + "${FULL_CMD[@]}" +fi diff --git a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md index 264e42a70..b4d316092 100644 --- a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md +++ b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md @@ -20,7 +20,7 @@ Primary AKS troubleshooting guide for incidents routed from [../../SKILL.md](../ When gathering AKS diagnostic evidence, prefer `mcp_azure_mcp_aks`, then the smallest discovered AKS-MCP tool that fits the read, then supporting Azure tools such as `mcp_azure_mcp_applens`, `mcp_azure_mcp_monitor`, or `mcp_azure_mcp_resourcehealth`. Use raw `az aks` and `kubectl` only when the AKS-MCP surface cannot perform the needed check. -When standard diagnostics do not reveal root cause, use **Inspektor Gadget** for real-time, low-level node and pod observability (DNS traces, TCP traces, process snapshots, file access traces). See [references/inspektor-gadget.md](references/inspektor-gadget.md) for the gadget catalog, command patterns, and symptom-to-gadget mapping. +When standard diagnostics do not reveal root cause, use **Inspektor Gadget** for real-time, low-level node and pod observability (DNS traces, TCP traces, process snapshots, file access traces). See [references/inspektor-gadget.md](references/inspektor-gadget.md) for the gadget catalog, the `run-ig` script, and symptom-to-gadget mapping. See [references/aks-mcp.md](references/aks-mcp.md), [references/structured-input-modes.md](references/structured-input-modes.md), [references/command-flows.md](references/command-flows.md) diff --git a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/networking.md b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/networking.md index e3e7936d1..b5fd1ed39 100644 --- a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/networking.md +++ b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/networking.md @@ -32,11 +32,11 @@ Pods that are running but not Ready are removed from Endpoints. Check `kubectl g **Deep diagnostics with Inspektor Gadget** (when the above checks are inconclusive): -Use the [IG base command pattern](references/inspektor-gadget.md) with `--k8s-namespace --k8s-podname ` and these gadgets: +Use [`scripts/run-ig.sh`](references/inspektor-gadget.md) (or `run-ig.ps1`) with `--pod --ns ` and these gadgets: -- `snapshot_socket` (timeout 5) — check what ports the pod is listening on -- `trace_tcp` (timeout 30) — trace connect/accept/close events -- `trace_tcpretrans` (timeout 30) — packet retransmissions +- `snapshot_socket` — check what ports the pod is listening on +- `trace_tcp` — trace connect/accept/close events +- `trace_tcpretrans` — packet retransmissions See [references/inspektor-gadget.md](references/inspektor-gadget.md). @@ -81,7 +81,7 @@ Custom VNet DNS must forward `.cluster.local` to the CoreDNS ClusterIP and other **Deep diagnostics with Inspektor Gadget** (when the above checks are inconclusive): -Use the [IG base command pattern](references/inspektor-gadget.md) with `--k8s-namespace --k8s-podname ` and `trace_dns` (timeout 30). Key signals: `rcode=3` (NXDOMAIN), `rcode=2` (SERVFAIL), high `latency` values, queries going to unexpected destinations. +Use [`scripts/run-ig.sh`](references/inspektor-gadget.md) (or `run-ig.ps1`) with `--pod --ns ` and `trace_dns`. Key signals: `rcode=3` (NXDOMAIN), `rcode=2` (SERVFAIL), high `latency` values, queries going to unexpected destinations. See [references/inspektor-gadget.md](references/inspektor-gadget.md). diff --git a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/node-issues.md b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/node-issues.md index 12767cf93..9e69be29e 100644 --- a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/node-issues.md +++ b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/node-issues.md @@ -107,7 +107,7 @@ Common culprit: high-volume container logs accumulating in `/var/log/containers` **Deep diagnostics with Inspektor Gadget** (PID pressure or unknown process load): -Use `snapshot_process` (timeout 5) to list all processes on the node. For node-wide scope, omit pod filters. See [references/inspektor-gadget.md](references/inspektor-gadget.md). +Use `scripts/run-ig.sh --gadget snapshot_process --node ` (or `run-ig.ps1`) to list all processes on the node. For node-wide scope, use `--node` (no pod filters). See [references/inspektor-gadget.md](references/inspektor-gadget.md). --- diff --git a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md index 9a2c4e33a..9b8c0d884 100644 --- a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md +++ b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md @@ -49,17 +49,15 @@ kubectl describe pod -n | grep -A2 "Last State" Fix: increase `resources.limits.memory` or optimize application memory usage. Check `kubectl top pod -n ` for actual usage. -**OOM kill tracing with Inspektor Gadget:** Use `trace_oomkill` (timeout 30) with `--k8s-namespace --k8s-podname ` to see which process was killed and memory at kill time. See [references/inspektor-gadget.md](references/inspektor-gadget.md). +**OOM kill tracing with Inspektor Gadget:** Run `trace_oomkill` for the pod to see which process was killed and memory at kill time: `scripts/run-ig.sh --gadget trace_oomkill --pod --ns ` (or `run-ig.ps1`). **Deep diagnostics with Inspektor Gadget** (when logs and describe are inconclusive): -Use the [IG base command pattern](references/inspektor-gadget.md) with `--k8s-namespace --k8s-podname ` and these gadgets: +Use [`scripts/run-ig.sh`](references/inspektor-gadget.md) (or `run-ig.ps1`) with `--pod --ns ` and these gadgets: -- `trace_exec` (timeout 30) — see what the container executes at startup -- `trace_open` (timeout 30) — find missing configs/secrets (retval -2 = ENOENT, -13 = EACCES) -- `snapshot_process` (timeout 5) — list running processes in the pod - -See [references/inspektor-gadget.md](references/inspektor-gadget.md). +- `trace_exec` — see what the container executes at startup +- `trace_open` — find missing configs/secrets (retval -2 = ENOENT, -13 = EACCES) +- `snapshot_process` — list running processes in the pod --- diff --git a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md index 90f9e8915..cf625eb30 100644 --- a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md +++ b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md @@ -85,7 +85,7 @@ kubectl describe quota -n ## Deep Diagnostics Flow (Inspektor Gadget) ```text -Standard diagnostics inconclusive -> resolve target node -> select gadget from symptom-to-gadget map -> run IG command with namespace/pod filters -> interpret output -> correlate with prior evidence +Standard diagnostics inconclusive -> select gadget from symptom-to-gadget map -> run `scripts/run-ig.sh` (or `run-ig.ps1`; resolves node, applies timeout) -> interpret output -> correlate with prior evidence ``` Use when steps 1–3 of the evidence order (Azure-side, Kubernetes-side, and detector evidence) do not reveal root cause. See [inspektor-gadget.md](inspektor-gadget.md) for the full gadget catalog and command patterns. diff --git a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/references/inspektor-gadget.md b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/references/inspektor-gadget.md index 381a63e05..214caa30f 100644 --- a/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/references/inspektor-gadget.md +++ b/plugins/azure-skills/skills/azure-diagnostics/troubleshooting/aks/references/inspektor-gadget.md @@ -1,45 +1,49 @@ # Inspektor Gadget (IG) Reference -Use Inspektor Gadget for real-time, low-level node/pod diagnostics when `kubectl` is insufficient. +Use Inspektor Gadget for low-level node/pod diagnostics when `kubectl` is insufficient. -## IG Version +## Run Script -`` = `v0.51.0` — substitute this exact tag (with `v` prefix) wherever `` appears. Bump this line only. - -## Base Command Pattern +Invoke gadgets with the `run-ig` script ([`scripts/run-ig.sh`](../../../scripts/run-ig.sh) / +[`scripts/run-ig.ps1`](../../../scripts/run-ig.ps1)). It resolves the node from the pod, injects +the pinned IG image/version, applies the default `--timeout` for the gadget type, adds the k8s +filters, and handles the `tcpdump` variant. You still choose **which** gadget (see the +Symptom-to-Gadget Map) and interpret the output. ```bash -kubectl debug --profile=sysadmin node/ --attach --quiet \ - --image=mcr.microsoft.com/oss/v2/inspektor-gadget/ig: \ - -- ig run : -o json --timeout [filters...] +./scripts/run-ig.sh --gadget trace_dns --pod --ns # node auto-resolved +./scripts/run-ig.sh --gadget snapshot_process --node # node-wide +./scripts/run-ig.sh --gadget trace_dns --pod --ns --dry-run +``` +```powershell +.\scripts\run-ig.ps1 -Gadget trace_dns -Pod -Namespace ``` -Always set `--timeout` after `--` to cap runtime. Use `--timeout 5` for snapshot/top, `--timeout 30` for trace/profile. - -> **Note:** IG uses `kubectl debug --profile=sysadmin` (privileged debug pod). Only run with explicit user approval and appropriate RBAC. - -**Required:** Resolve the node name first: +**Options** (bash flags below; PowerShell uses PascalCase equivalents: `-Gadget`, `-Pod`, +`-Namespace`/`-Ns`, `-Node`, `-Container`, `-Timeout`, `-Filter`, `-Pf`, `-IgVersion`, +`-DryRun`): `--gadget` (required); target `--pod`/`--ns` **or** `--node`; `--container`; +`--timeout ` override; `--filter ` (repeatable IG-flag passthrough, e.g. +`--filter --max-entries --filter 20`); `--pf ""` (tcpdump only); `--ig-version `; +`--dry-run`. Default timeout by gadget name: `snapshot_*`/`top_*` → 5s, +`trace_*`/`profile_*`/`tcpdump` → 30s. Returns the gadget JSON (pcap-ng for tcpdump) plus a +`Ran gadget X on node Y` summary. IG version is pinned to `v0.51.0` in the scripts. -```bash -kubectl get pod -n -o jsonpath='{.spec.nodeName}' -``` +> **Approval required:** IG uses `kubectl debug --profile=sysadmin` (a privileged debug pod). +> **Ask the user before running the script** and confirm RBAC; use `--dry-run` to preview. ## Common Filters +The k8s scope filters and `--timeout` are set by the script. Pass any other IG flag below via +its repeatable `--filter`, e.g. `--filter --max-entries --filter 20`. + | Filter | Description | |---|---| -| `--k8s-namespace ` | Scope to a Kubernetes namespace | -| `--k8s-podname ` | Scope to a specific pod | -| `--k8s-containername ` | Scope to a specific container | -| `--timeout ` | Cap streaming duration for trace/profile gadgets | | `--max-entries ` | Max entries per batch for top/profile gadgets | | `--map-fetch-interval ` | Map fetch interval for top (except `top_process`) and profile gadgets (default `1000ms`) | | `--interval ` | Reporting interval for `top_process` only (e.g. `5s`) | | `--syscall-filters ` | Comma-separated syscalls for `traceloop` (e.g. `open,connect,accept`). **Always specify** to limit data volume | -> **Tip:** For top/profile, set `--map-fetch-interval` ≤ half of `--timeout` to collect at least one batch. E.g. `--timeout 2 --map-fetch-interval 1000ms --max-entries 20`. -> -> **Note:** `top_process` uses `--interval` instead of `--map-fetch-interval`. E.g. `--timeout 10 --interval 5s --max-entries 20`. +> **Tip:** For top/profile, keep `--map-fetch-interval` ≤ half of `--timeout` to collect ≥1 batch. `top_process` uses `--interval` instead of `--map-fetch-interval`. ## Gadget Catalog @@ -57,18 +61,14 @@ kubectl get pod -n -o jsonpath='{.spec.nodeName}' #### tcpdump gadget -Outputs raw pcap-ng data. Pipe to `tcpdump` for readable output: +Run via `--gadget tcpdump`; the script sets `-o pcap-ng` and pipes to `tcpdump -nvr -` when +available. Use `--pf ""` for tcpdump filters (e.g., `port 80`, `host 10.0.0.1`); `--pf` +is only valid for the `tcpdump` gadget. ```bash -kubectl debug --profile=sysadmin node/ --attach --quiet \ - --image=mcr.microsoft.com/oss/v2/inspektor-gadget/ig: \ - -- ig run tcpdump: -o pcap-ng --k8s-namespace --k8s-podname \ - --timeout 30 --pf "port 80" \ - | tcpdump -nvr - +./scripts/run-ig.sh --gadget tcpdump --pod --ns --pf "port 80" ``` -Use `--pf ""` for tcpdump filters (e.g., `port 80`, `host 10.0.0.1`). Output must be `-o pcap-ng` (not `-o json`). - ### Process & Workload | Gadget | Type | What It Does | When To Use | @@ -130,6 +130,6 @@ Use `--pf ""` for tcpdump filters (e.g., `port 80`, `host 10.0.0.1`). Outp ## Guardrails - IG gadgets are **read-only** — they do not modify cluster or application state. -- Resolve the correct node name before running any IG command. -- Always set `--timeout` to cap runtime. Prefer snapshot/top for quick checks; trace/profile for behavior over time. +- Invoke gadgets through `run-ig` (`scripts/run-ig.sh` / `scripts/run-ig.ps1`); it resolves the node and applies the correct timeout. **Ask the user before running it** (privileged debug pod). +- The script picks the default `--timeout` by gadget type. Prefer snapshot/top for quick checks; trace/profile for behavior over time. Override with `--timeout` when needed. - For reproduction: launch a trace gadget first, then reproduce the problem. The debug pod persists after the gadget exits, so run `kubectl logs ` to retrieve the captured output afterward.