From 00b854004c6217df13d705fa3d6146ae38690567 Mon Sep 17 00:00:00 2001 From: "Tom Meschter (from Dev Box)" Date: Tue, 21 Jul 2026 11:54:57 -0700 Subject: [PATCH 1/4] feat(azure-diagnostics): add run-ig script for Inspektor Gadget invocation Replace the inline kubectl debug ... ig run command assembly in the IG reference with cross-platform run-ig.sh / run-ig.ps1 scripts. The scripts resolve the node from a pod, inject the pinned IG image/version, apply the gadget-type default timeout, add k8s filters, and handle the tcpdump variant. A --dry-run flag prints the assembled command. Update inspektor-gadget.md and the caller docs (pod-failures, networking, node-issues, command-flows, aks-troubleshooting) to reference the script instead of hand-built commands. Fixes #2508 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 92c8ec36-b76b-4825-896f-02598b79135e --- .../azure-diagnostics/scripts/run-ig.ps1 | 176 ++++++++++++++++++ .../azure-diagnostics/scripts/run-ig.sh | 176 ++++++++++++++++++ .../aks/aks-troubleshooting.md | 2 +- .../troubleshooting/aks/networking.md | 10 +- .../troubleshooting/aks/node-issues.md | 2 +- .../troubleshooting/aks/pod-failures.md | 10 +- .../aks/references/command-flows.md | 2 +- .../aks/references/inspektor-gadget.md | 61 +++--- 8 files changed, 396 insertions(+), 43 deletions(-) create mode 100644 plugin/skills/azure-diagnostics/scripts/run-ig.ps1 create mode 100644 plugin/skills/azure-diagnostics/scripts/run-ig.sh diff --git a/plugin/skills/azure-diagnostics/scripts/run-ig.ps1 b/plugin/skills/azure-diagnostics/scripts/run-ig.ps1 new file mode 100644 index 000000000..722eb3807 --- /dev/null +++ b/plugin/skills/azure-diagnostics/scripts/run-ig.ps1 @@ -0,0 +1,176 @@ +<# +.SYNOPSIS + Runs an Inspektor Gadget (IG) trace on an AKS node via `kubectl debug`. + +.DESCRIPTION + Handles the mechanical, error-prone assembly of the IG invocation: + - resolves the target node from a pod (or takes a node directly) + - injects the pinned IG image + version + - applies the correct default -Timeout for the gadget type + - adds the k8s namespace/pod/container filters + - handles the special `tcpdump` gadget (pcap-ng output piped to tcpdump) + + The privileged debug pod requires explicit user approval and appropriate RBAC. + Use -DryRun to print the assembled command without running it. + +.PARAMETER Gadget + Gadget to run, e.g. trace_dns, snapshot_socket, tcpdump (required). + +.PARAMETER Pod + Pod name; the node is resolved automatically. + +.PARAMETER Namespace + Namespace of the pod (required with -Pod). + +.PARAMETER Node + Run directly against a node (node-wide scope). + +.PARAMETER Container + Scope to a specific container. + +.PARAMETER Timeout + Override the gadget-type default timeout (seconds). + +.PARAMETER Filter + Extra IG flags, passed through verbatim (e.g. -Filter --max-entries,20). + +.PARAMETER Pf + tcpdump packet filter (tcpdump gadget only, e.g. "port 80"). + +.PARAMETER IgVersion + Override the pinned IG image tag. + +.PARAMETER DryRun + Print the assembled command; do not execute. + +.EXAMPLE + ./run-ig.ps1 -Gadget trace_dns -Pod web-0 -Namespace default + +.EXAMPLE + ./run-ig.ps1 -Gadget snapshot_process -Node aks-nodepool1-1234 + +.EXAMPLE + ./run-ig.ps1 -Gadget tcpdump -Pod web-0 -Namespace default -Pf "port 80" + +.EXAMPLE + ./run-ig.ps1 -Gadget traceloop -Pod web-0 -Namespace default -Filter --syscall-filters,open,connect + +.EXAMPLE + ./run-ig.ps1 -Gadget trace_dns -Pod web-0 -Namespace default -DryRun +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$Gadget, + [string]$Pod, + [Alias('Ns')] + [string]$Namespace, + [string]$Node, + [string]$Container, + [int]$Timeout, + [string[]]$Filter, + [string]$Pf, + # Pinned IG image tag. Bump this default (and run-ig.sh) to update the IG version. + [string]$IgVersion = 'v0.51.0', + [switch]$DryRun +) + +$ErrorActionPreference = 'Stop' + +$IgImageRepo = 'mcr.microsoft.com/oss/v2/inspektor-gadget/ig' + +if (-not $Node -and -not $Pod) { + Write-Error 'Provide either -Node or -Pod -Namespace .' + exit 2 +} +if ($Pod -and -not $Namespace) { + Write-Error '-Pod requires -Namespace .' + exit 2 +} +if ($Pf -and $Gadget -ne 'tcpdump') { + Write-Error '-Pf is only valid for the tcpdump gadget.' + exit 2 +} + +# Default timeout by gadget type, inferred from the gadget name prefix. +# snapshot_* / top_* -> 5s (point-in-time / quick aggregate) +# trace_* / profile_* / tcpdump -> 30s (streaming / sampling) +function Get-DefaultTimeout([string]$g) { + switch -Wildcard ($g) { + 'snapshot_*' { return 5 } + 'top_*' { return 5 } + 'trace_*' { return 30 } + 'profile_*' { return 30 } + 'tcpdump' { return 30 } + default { return 30 } # unknown gadget: use the safer streaming default + } +} + +if (-not $PSBoundParameters.ContainsKey('Timeout') -or $Timeout -le 0) { + $Timeout = Get-DefaultTimeout $Gadget +} + +# Resolve the node name from the pod when not given directly. +if (-not $Node) { + $Node = (& kubectl get pod $Pod -n $Namespace -o "jsonpath={.spec.nodeName}").Trim() + if (-not $Node) { + Write-Error "Could not resolve node for pod '$Pod' in namespace '$Namespace'." + exit 1 + } +} + +$IgImage = "${IgImageRepo}:${IgVersion}" + +# Assemble the k8s scoping filters. +$filters = @() +if ($Namespace) { $filters += @('--k8s-namespace', $Namespace) } +if ($Pod) { $filters += @('--k8s-podname', $Pod) } +if ($Container) { $filters += @('--k8s-containername', $Container) } + +# Base kubectl debug invocation. +$debug = @('debug', '--profile=sysadmin', "node/$Node", '--attach', '--quiet', "--image=$IgImage", '--') + +if ($Gadget -eq 'tcpdump') { + # tcpdump emits raw pcap-ng; pipe through tcpdump for readable output when available. + $igCmd = @('ig', 'run', "tcpdump:$IgVersion", '-o', 'pcap-ng') + $filters + @('--timeout', "$Timeout") + if ($Pf) { $igCmd += @('--pf', $Pf) } + if ($Filter) { $igCmd += $Filter } +} +else { + $igCmd = @('ig', 'run', "${Gadget}:$IgVersion", '-o', 'json') + $filters + @('--timeout', "$Timeout") + if ($Filter) { $igCmd += $Filter } +} + +$fullArgs = $debug + $igCmd + +# Pretty-print a shell-quoted version of the command for display. +function Format-Cmd([string[]]$parts) { + ($parts | ForEach-Object { + if ($_ -match '\s') { '"' + $_ + '"' } else { $_ } + }) -join ' ' +} + +$displayCmd = 'kubectl ' + (Format-Cmd $fullArgs) +if ($Gadget -eq 'tcpdump') { + $displayCmd = "$displayCmd | tcpdump -nvr -" +} + +Write-Host "Gadget: $Gadget" +Write-Host "Node: $Node" +Write-Host "Timeout: ${Timeout}s" +Write-Host "Image: $IgImage" +Write-Host "Command: $displayCmd" + +if ($DryRun) { + Write-Host '(dry-run: command not executed)' + exit 0 +} + +Write-Host "Ran gadget $Gadget on node $Node (timeout ${Timeout}s)" + +if ($Gadget -eq 'tcpdump' -and (Get-Command tcpdump -ErrorAction SilentlyContinue)) { + & kubectl @fullArgs | & tcpdump -nvr - +} +else { + & kubectl @fullArgs +} diff --git a/plugin/skills/azure-diagnostics/scripts/run-ig.sh b/plugin/skills/azure-diagnostics/scripts/run-ig.sh new file mode 100644 index 000000000..4c38ffc70 --- /dev/null +++ b/plugin/skills/azure-diagnostics/scripts/run-ig.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +# run-ig.sh +# Runs an Inspektor Gadget (IG) trace on an AKS node via `kubectl debug`. +# +# Handles the mechanical, error-prone assembly of the IG invocation: +# - resolves the target node from a pod (or takes a node directly) +# - injects the pinned IG image + version +# - applies the correct default --timeout for the gadget type +# - adds the k8s namespace/pod/container filters +# - handles the special `tcpdump` gadget (pcap-ng output piped to tcpdump) +# +# The privileged debug pod requires explicit user approval and appropriate RBAC. +# Use --dry-run to print the assembled command without running it. +# +# Usage: +# ./run-ig.sh --gadget (--pod --ns | --node ) [options] +# +# Options: +# --gadget Gadget to run, e.g. trace_dns, snapshot_socket, tcpdump (required) +# --pod Pod name; the node is resolved automatically +# --ns Namespace of the pod (required with --pod) +# --node Run directly against a node (node-wide scope) +# --container Scope to a specific container +# --timeout Override the gadget-type default timeout +# --filter Extra IG flag, repeatable (e.g. --filter --max-entries --filter 20) +# --pf "" tcpdump packet filter (tcpdump gadget only, e.g. "port 80") +# --ig-version Override the pinned IG image tag (default below) +# --dry-run Print the assembled command; do not execute +# +# Examples: +# ./run-ig.sh --gadget trace_dns --pod web-0 --ns default +# ./run-ig.sh --gadget snapshot_process --node aks-nodepool1-1234 +# ./run-ig.sh --gadget tcpdump --pod web-0 --ns default --pf "port 80" +# ./run-ig.sh --gadget traceloop --pod web-0 --ns default --filter --syscall-filters --filter open,connect +# ./run-ig.sh --gadget trace_dns --pod web-0 --ns default --dry-run + +set -euo pipefail + +# Pinned IG image tag. Bump this line (and run-ig.ps1) to update the IG version. +IG_VERSION="v0.51.0" +IG_IMAGE_REPO="mcr.microsoft.com/oss/v2/inspektor-gadget/ig" + +GADGET="" +POD="" +NS="" +NODE="" +CONTAINER="" +TIMEOUT="" +PF="" +DRY_RUN="false" +EXTRA_FILTERS=() + +usage() { + sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//' +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --gadget) GADGET="${2:?--gadget requires a value}"; shift 2;; + --pod) POD="${2:?--pod requires a value}"; shift 2;; + --ns|--namespace) NS="${2:?--ns requires a value}"; shift 2;; + --node) NODE="${2:?--node requires a value}"; shift 2;; + --container) CONTAINER="${2:?--container requires a value}"; shift 2;; + --timeout) TIMEOUT="${2:?--timeout requires a value}"; shift 2;; + --filter) EXTRA_FILTERS+=("${2:?--filter requires a value}"); shift 2;; + --pf) PF="${2:?--pf requires a value}"; shift 2;; + --ig-version) IG_VERSION="${2:?--ig-version requires a value}"; shift 2;; + --dry-run) DRY_RUN="true"; shift;; + -h|--help) usage; exit 0;; + *) echo "Unknown argument: $1" >&2; usage >&2; exit 2;; + esac +done + +if [[ -z "$GADGET" ]]; then + echo "Error: --gadget is required." >&2 + exit 2 +fi + +if [[ -z "$NODE" && -z "$POD" ]]; then + echo "Error: provide either --node or --pod --ns ." >&2 + exit 2 +fi + +if [[ -n "$POD" && -z "$NS" ]]; then + echo "Error: --pod requires --ns ." >&2 + exit 2 +fi + +# Default timeout by gadget type, inferred from the gadget name prefix. +# snapshot_* / top_* -> 5s (point-in-time / quick aggregate) +# trace_* / profile_* / tcpdump -> 30s (streaming / sampling) +default_timeout() { + case "$1" in + snapshot_*|top_*) echo 5;; + trace_*|profile_*|tcpdump) echo 30;; + *) echo 30;; # unknown gadget: use the safer streaming default + esac +} + +if [[ -z "$TIMEOUT" ]]; then + TIMEOUT="$(default_timeout "$GADGET")" +fi + +# Resolve the node name from the pod when not given directly. +if [[ -z "$NODE" ]]; then + NODE="$(kubectl get pod "$POD" -n "$NS" -o jsonpath='{.spec.nodeName}')" + if [[ -z "$NODE" ]]; then + echo "Error: could not resolve node for pod '$POD' in namespace '$NS'." >&2 + exit 1 + fi +fi + +IG_IMAGE="${IG_IMAGE_REPO}:${IG_VERSION}" + +# Assemble the k8s scoping filters. +FILTERS=() +[[ -n "$NS" ]] && FILTERS+=(--k8s-namespace "$NS") +[[ -n "$POD" ]] && FILTERS+=(--k8s-podname "$POD") +[[ -n "$CONTAINER" ]] && FILTERS+=(--k8s-containername "$CONTAINER") + +# Base kubectl debug invocation. +DEBUG=(kubectl debug --profile=sysadmin "node/${NODE}" --attach --quiet --image="$IG_IMAGE" --) + +if [[ "$GADGET" == "tcpdump" ]]; then + # tcpdump emits raw pcap-ng; pipe through tcpdump for readable output when available. + IG_CMD=(ig run "tcpdump:${IG_VERSION}" -o pcap-ng "${FILTERS[@]}" --timeout "$TIMEOUT") + [[ -n "$PF" ]] && IG_CMD+=(--pf "$PF") + [[ ${#EXTRA_FILTERS[@]} -gt 0 ]] && IG_CMD+=("${EXTRA_FILTERS[@]}") +else + if [[ -n "$PF" ]]; then + echo "Error: --pf is only valid for the tcpdump gadget." >&2 + exit 2 + fi + IG_CMD=(ig run "${GADGET}:${IG_VERSION}" -o json "${FILTERS[@]}" --timeout "$TIMEOUT") + [[ ${#EXTRA_FILTERS[@]} -gt 0 ]] && IG_CMD+=("${EXTRA_FILTERS[@]}") +fi + +FULL_CMD=("${DEBUG[@]}" "${IG_CMD[@]}") + +# Pretty-print a shell-quoted version of the command for display. +quote_cmd() { + local out="" + local a + for a in "$@"; do + if [[ "$a" =~ [[:space:]] ]]; then + out+="\"$a\" " + else + out+="$a " + fi + done + echo "${out% }" +} + +DISPLAY_CMD="$(quote_cmd "${FULL_CMD[@]}")" +if [[ "$GADGET" == "tcpdump" ]]; then + DISPLAY_CMD="$DISPLAY_CMD | tcpdump -nvr -" +fi + +echo "Gadget: $GADGET" >&2 +echo "Node: $NODE" >&2 +echo "Timeout: ${TIMEOUT}s" >&2 +echo "Image: $IG_IMAGE" >&2 +echo "Command: $DISPLAY_CMD" >&2 + +if [[ "$DRY_RUN" == "true" ]]; then + echo "(dry-run: command not executed)" >&2 + exit 0 +fi + +echo "Ran gadget $GADGET on node $NODE (timeout ${TIMEOUT}s)" >&2 + +if [[ "$GADGET" == "tcpdump" ]] && command -v tcpdump >/dev/null 2>&1; then + "${FULL_CMD[@]}" | tcpdump -nvr - +else + "${FULL_CMD[@]}" +fi diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md index ce789bb02..48aad0ed0 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/aks-troubleshooting.md @@ -20,7 +20,7 @@ Primary AKS troubleshooting guide for incidents routed from [../../SKILL.md](../ When gathering AKS diagnostic evidence, prefer `mcp_azure_mcp_aks`, then the smallest discovered AKS-MCP tool that fits the read, then supporting Azure tools such as `mcp_azure_mcp_applens`, `mcp_azure_mcp_monitor`, or `mcp_azure_mcp_resourcehealth`. Use raw `az aks` and `kubectl` only when the AKS-MCP surface cannot perform the needed check. -When standard diagnostics do not reveal root cause, use **Inspektor Gadget** for real-time, low-level node and pod observability (DNS traces, TCP traces, process snapshots, file access traces). See [references/inspektor-gadget.md](references/inspektor-gadget.md) for the gadget catalog, command patterns, and symptom-to-gadget mapping. +When standard diagnostics do not reveal root cause, use **Inspektor Gadget** for real-time, low-level node and pod observability (DNS traces, TCP traces, process snapshots, file access traces). See [references/inspektor-gadget.md](references/inspektor-gadget.md) for the gadget catalog, the `run-ig` script, and symptom-to-gadget mapping. See [references/aks-mcp.md](references/aks-mcp.md), [references/structured-input-modes.md](references/structured-input-modes.md), [references/command-flows.md](references/command-flows.md) diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/networking.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/networking.md index e3e7936d1..2056c9812 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/networking.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/networking.md @@ -32,11 +32,11 @@ Pods that are running but not Ready are removed from Endpoints. Check `kubectl g **Deep diagnostics with Inspektor Gadget** (when the above checks are inconclusive): -Use the [IG base command pattern](references/inspektor-gadget.md) with `--k8s-namespace --k8s-podname ` and these gadgets: +Use the [`run-ig` script](references/inspektor-gadget.md) with `--pod --ns ` and these gadgets: -- `snapshot_socket` (timeout 5) — check what ports the pod is listening on -- `trace_tcp` (timeout 30) — trace connect/accept/close events -- `trace_tcpretrans` (timeout 30) — packet retransmissions +- `snapshot_socket` — check what ports the pod is listening on +- `trace_tcp` — trace connect/accept/close events +- `trace_tcpretrans` — packet retransmissions See [references/inspektor-gadget.md](references/inspektor-gadget.md). @@ -81,7 +81,7 @@ Custom VNet DNS must forward `.cluster.local` to the CoreDNS ClusterIP and other **Deep diagnostics with Inspektor Gadget** (when the above checks are inconclusive): -Use the [IG base command pattern](references/inspektor-gadget.md) with `--k8s-namespace --k8s-podname ` and `trace_dns` (timeout 30). Key signals: `rcode=3` (NXDOMAIN), `rcode=2` (SERVFAIL), high `latency` values, queries going to unexpected destinations. +Use the [`run-ig` script](references/inspektor-gadget.md) with `--pod --ns ` and `trace_dns`. Key signals: `rcode=3` (NXDOMAIN), `rcode=2` (SERVFAIL), high `latency` values, queries going to unexpected destinations. See [references/inspektor-gadget.md](references/inspektor-gadget.md). diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/node-issues.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/node-issues.md index 12767cf93..34303e126 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/node-issues.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/node-issues.md @@ -107,7 +107,7 @@ Common culprit: high-volume container logs accumulating in `/var/log/containers` **Deep diagnostics with Inspektor Gadget** (PID pressure or unknown process load): -Use `snapshot_process` (timeout 5) to list all processes on the node. For node-wide scope, omit pod filters. See [references/inspektor-gadget.md](references/inspektor-gadget.md). +Use `run-ig --gadget snapshot_process --node ` to list all processes on the node. For node-wide scope, use `--node` (no pod filters). See [references/inspektor-gadget.md](references/inspektor-gadget.md). --- diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md index 9a2c4e33a..5b4e668b3 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md @@ -49,15 +49,15 @@ kubectl describe pod -n | grep -A2 "Last State" Fix: increase `resources.limits.memory` or optimize application memory usage. Check `kubectl top pod -n ` for actual usage. -**OOM kill tracing with Inspektor Gadget:** Use `trace_oomkill` (timeout 30) with `--k8s-namespace --k8s-podname ` to see which process was killed and memory at kill time. See [references/inspektor-gadget.md](references/inspektor-gadget.md). +**OOM kill tracing with Inspektor Gadget:** Run `trace_oomkill` for the pod to see which process was killed and memory at kill time: `run-ig --gadget trace_oomkill --pod --ns `. See [references/inspektor-gadget.md](references/inspektor-gadget.md). **Deep diagnostics with Inspektor Gadget** (when logs and describe are inconclusive): -Use the [IG base command pattern](references/inspektor-gadget.md) with `--k8s-namespace --k8s-podname ` and these gadgets: +Use the [`run-ig` script](references/inspektor-gadget.md) with `--pod --ns ` and these gadgets: -- `trace_exec` (timeout 30) — see what the container executes at startup -- `trace_open` (timeout 30) — find missing configs/secrets (retval -2 = ENOENT, -13 = EACCES) -- `snapshot_process` (timeout 5) — list running processes in the pod +- `trace_exec` — see what the container executes at startup +- `trace_open` — find missing configs/secrets (retval -2 = ENOENT, -13 = EACCES) +- `snapshot_process` — list running processes in the pod See [references/inspektor-gadget.md](references/inspektor-gadget.md). diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md index 819724323..4546121e3 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md @@ -79,7 +79,7 @@ kubectl describe quota -n ## Deep Diagnostics Flow (Inspektor Gadget) ```text -Standard diagnostics inconclusive -> resolve target node -> select gadget from symptom-to-gadget map -> run IG command with namespace/pod filters -> interpret output -> correlate with prior evidence +Standard diagnostics inconclusive -> select gadget from symptom-to-gadget map -> run `run-ig` (resolves node, applies timeout) -> interpret output -> correlate with prior evidence ``` Use when steps 1–3 of the evidence order (Azure-side, Kubernetes-side, and detector evidence) do not reveal root cause. See [inspektor-gadget.md](inspektor-gadget.md) for the full gadget catalog and command patterns. diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/references/inspektor-gadget.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/references/inspektor-gadget.md index 381a63e05..d356a7e43 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/references/inspektor-gadget.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/references/inspektor-gadget.md @@ -1,37 +1,42 @@ # Inspektor Gadget (IG) Reference -Use Inspektor Gadget for real-time, low-level node/pod diagnostics when `kubectl` is insufficient. +Use Inspektor Gadget for low-level node/pod diagnostics when `kubectl` is insufficient. -## IG Version +## Run Script -`` = `v0.51.0` — substitute this exact tag (with `v` prefix) wherever `` appears. Bump this line only. - -## Base Command Pattern +Invoke gadgets with the `run-ig` script ([`scripts/run-ig.sh`](../../../scripts/run-ig.sh) / +[`scripts/run-ig.ps1`](../../../scripts/run-ig.ps1)). It resolves the node from the pod, injects +the pinned IG image/version, applies the default `--timeout` for the gadget type, adds the k8s +filters, and handles the `tcpdump` variant. You still choose **which** gadget (see the +Symptom-to-Gadget Map) and interpret the output. ```bash -kubectl debug --profile=sysadmin node/ --attach --quiet \ - --image=mcr.microsoft.com/oss/v2/inspektor-gadget/ig: \ - -- ig run : -o json --timeout [filters...] +./scripts/run-ig.sh --gadget trace_dns --pod --ns # node auto-resolved +./scripts/run-ig.sh --gadget snapshot_process --node # node-wide +./scripts/run-ig.sh --gadget trace_dns --pod --ns --dry-run +``` +```powershell +.\scripts\run-ig.ps1 -Gadget trace_dns -Pod -Namespace ``` -Always set `--timeout` after `--` to cap runtime. Use `--timeout 5` for snapshot/top, `--timeout 30` for trace/profile. - -> **Note:** IG uses `kubectl debug --profile=sysadmin` (privileged debug pod). Only run with explicit user approval and appropriate RBAC. - -**Required:** Resolve the node name first: +**Options:** `--gadget` (required); target `--pod`/`--ns` **or** `--node`; `--container`; +`--timeout ` override; `--filter ` (repeatable IG-flag passthrough, e.g. +`--filter --max-entries --filter 20`); `--pf ""` (tcpdump only); `--ig-version `; +`--dry-run`. Default timeout by gadget name: `snapshot_*`/`top_*` → 5s, +`trace_*`/`profile_*`/`tcpdump` → 30s. Returns the gadget JSON (pcap-ng for tcpdump) plus a +`Ran gadget X on node Y` summary. IG version is pinned to `v0.51.0` in the scripts. -```bash -kubectl get pod -n -o jsonpath='{.spec.nodeName}' -``` +> **Approval required:** IG uses `kubectl debug --profile=sysadmin` (a privileged debug pod). +> **Ask the user before running the script** and confirm RBAC; use `--dry-run` to preview. ## Common Filters +`--k8s-namespace`/`--k8s-podname`/`--k8s-containername` and `--timeout` are set by the script +(from `--pod`/`--ns`/`--container`/`--timeout`). Pass any other IG flag below via the script's +repeatable `--filter`, e.g. `--filter --max-entries --filter 20`. + | Filter | Description | |---|---| -| `--k8s-namespace ` | Scope to a Kubernetes namespace | -| `--k8s-podname ` | Scope to a specific pod | -| `--k8s-containername ` | Scope to a specific container | -| `--timeout ` | Cap streaming duration for trace/profile gadgets | | `--max-entries ` | Max entries per batch for top/profile gadgets | | `--map-fetch-interval ` | Map fetch interval for top (except `top_process`) and profile gadgets (default `1000ms`) | | `--interval ` | Reporting interval for `top_process` only (e.g. `5s`) | @@ -57,18 +62,14 @@ kubectl get pod -n -o jsonpath='{.spec.nodeName}' #### tcpdump gadget -Outputs raw pcap-ng data. Pipe to `tcpdump` for readable output: +Run via `--gadget tcpdump`; the script sets `-o pcap-ng` and pipes to `tcpdump -nvr -` when +available. Use `--pf ""` for tcpdump filters (e.g., `port 80`, `host 10.0.0.1`); `--pf` +is only valid for the `tcpdump` gadget. ```bash -kubectl debug --profile=sysadmin node/ --attach --quiet \ - --image=mcr.microsoft.com/oss/v2/inspektor-gadget/ig: \ - -- ig run tcpdump: -o pcap-ng --k8s-namespace --k8s-podname \ - --timeout 30 --pf "port 80" \ - | tcpdump -nvr - +./scripts/run-ig.sh --gadget tcpdump --pod --ns --pf "port 80" ``` -Use `--pf ""` for tcpdump filters (e.g., `port 80`, `host 10.0.0.1`). Output must be `-o pcap-ng` (not `-o json`). - ### Process & Workload | Gadget | Type | What It Does | When To Use | @@ -130,6 +131,6 @@ Use `--pf ""` for tcpdump filters (e.g., `port 80`, `host 10.0.0.1`). Outp ## Guardrails - IG gadgets are **read-only** — they do not modify cluster or application state. -- Resolve the correct node name before running any IG command. -- Always set `--timeout` to cap runtime. Prefer snapshot/top for quick checks; trace/profile for behavior over time. +- Invoke gadgets through `run-ig` (`scripts/run-ig.sh` / `scripts/run-ig.ps1`); it resolves the node and applies the correct timeout. **Ask the user before running it** (privileged debug pod). +- The script picks the default `--timeout` by gadget type. Prefer snapshot/top for quick checks; trace/profile for behavior over time. Override with `--timeout` when needed. - For reproduction: launch a trace gadget first, then reproduce the problem. The debug pod persists after the gadget exits, so run `kubectl logs ` to retrieve the captured output afterward. From 7c7d2f33e0002a4b081ffc085f306f04ca881764 Mon Sep 17 00:00:00 2001 From: "Tom Meschter (from Dev Box)" Date: Wed, 22 Jul 2026 11:15:11 -0700 Subject: [PATCH 2/4] fix(azure-diagnostics): address PR review feedback on run-ig scripts - run-ig.ps1: drop mandatory -Gadget param (avoid interactive prompt in non-interactive use); validate explicitly and exit with a clear message. - run-ig.ps1: remove \Continue='Stop' so Write-Error no longer terminates; guard pod node-resolution against empty/failed kubectl output. - Both scripts: only show the '| tcpdump -nvr -' pipe in the displayed command when tcpdump is present, so --dry-run matches real behavior; note when raw pcap-ng is emitted instead. - run-ig.sh: make usage() print only the leading comment block (no script code). - Docs: reference scripts/run-ig.sh (or run-ig.ps1) explicitly instead of a bare run-ig; note PowerShell PascalCase parameter names. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 92c8ec36-b76b-4825-896f-02598b79135e --- .../azure-diagnostics/scripts/run-ig.ps1 | 20 +++++++++++++------ .../azure-diagnostics/scripts/run-ig.sh | 16 ++++++++++++--- .../troubleshooting/aks/networking.md | 4 ++-- .../troubleshooting/aks/node-issues.md | 2 +- .../troubleshooting/aks/pod-failures.md | 4 ++-- .../aks/references/command-flows.md | 2 +- .../aks/references/inspektor-gadget.md | 13 ++++++------ 7 files changed, 39 insertions(+), 22 deletions(-) diff --git a/plugin/skills/azure-diagnostics/scripts/run-ig.ps1 b/plugin/skills/azure-diagnostics/scripts/run-ig.ps1 index 722eb3807..ab2acc972 100644 --- a/plugin/skills/azure-diagnostics/scripts/run-ig.ps1 +++ b/plugin/skills/azure-diagnostics/scripts/run-ig.ps1 @@ -60,7 +60,6 @@ #> [CmdletBinding()] param( - [Parameter(Mandatory = $true)] [string]$Gadget, [string]$Pod, [Alias('Ns')] @@ -75,10 +74,12 @@ param( [switch]$DryRun ) -$ErrorActionPreference = 'Stop' - $IgImageRepo = 'mcr.microsoft.com/oss/v2/inspektor-gadget/ig' +if (-not $Gadget) { + Write-Error 'Provide -Gadget (e.g. trace_dns, snapshot_socket, tcpdump).' + exit 2 +} if (-not $Node -and -not $Pod) { Write-Error 'Provide either -Node or -Pod -Namespace .' exit 2 @@ -112,7 +113,7 @@ if (-not $PSBoundParameters.ContainsKey('Timeout') -or $Timeout -le 0) { # Resolve the node name from the pod when not given directly. if (-not $Node) { - $Node = (& kubectl get pod $Pod -n $Namespace -o "jsonpath={.spec.nodeName}").Trim() + $Node = ((& kubectl get pod $Pod -n $Namespace -o "jsonpath={.spec.nodeName}" 2>$null) | Out-String).Trim() if (-not $Node) { Write-Error "Could not resolve node for pod '$Pod' in namespace '$Namespace'." exit 1 @@ -151,7 +152,11 @@ function Format-Cmd([string[]]$parts) { } $displayCmd = 'kubectl ' + (Format-Cmd $fullArgs) -if ($Gadget -eq 'tcpdump') { + +# The tcpdump gadget is only piped through `tcpdump` when that binary is present. +# Reflect the real behavior in the displayed command so -DryRun does not mislead. +$tcpdumpAvail = $Gadget -eq 'tcpdump' -and [bool](Get-Command tcpdump -ErrorAction SilentlyContinue) +if ($tcpdumpAvail) { $displayCmd = "$displayCmd | tcpdump -nvr -" } @@ -160,6 +165,9 @@ Write-Host "Node: $Node" Write-Host "Timeout: ${Timeout}s" Write-Host "Image: $IgImage" Write-Host "Command: $displayCmd" +if ($Gadget -eq 'tcpdump' -and -not $tcpdumpAvail) { + Write-Host 'Note: tcpdump not found; emitting raw pcap-ng to stdout.' +} if ($DryRun) { Write-Host '(dry-run: command not executed)' @@ -168,7 +176,7 @@ if ($DryRun) { Write-Host "Ran gadget $Gadget on node $Node (timeout ${Timeout}s)" -if ($Gadget -eq 'tcpdump' -and (Get-Command tcpdump -ErrorAction SilentlyContinue)) { +if ($tcpdumpAvail) { & kubectl @fullArgs | & tcpdump -nvr - } else { diff --git a/plugin/skills/azure-diagnostics/scripts/run-ig.sh b/plugin/skills/azure-diagnostics/scripts/run-ig.sh index 4c38ffc70..c1286d813 100644 --- a/plugin/skills/azure-diagnostics/scripts/run-ig.sh +++ b/plugin/skills/azure-diagnostics/scripts/run-ig.sh @@ -51,7 +51,9 @@ DRY_RUN="false" EXTRA_FILTERS=() usage() { - sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//' + # Print the leading comment block (from line 2) as help, stopping at the + # first non-comment line so script code is never echoed. + awk 'NR>1 && /^#/ { sub(/^# ?/, ""); print; next } NR>1 { exit }' "$0" } while [[ $# -gt 0 ]]; do @@ -152,7 +154,12 @@ quote_cmd() { } DISPLAY_CMD="$(quote_cmd "${FULL_CMD[@]}")" -if [[ "$GADGET" == "tcpdump" ]]; then + +# The tcpdump gadget is only piped through `tcpdump` when that binary is present. +# Reflect the real behavior in the displayed command so --dry-run does not mislead. +TCPDUMP_AVAIL="false" +if [[ "$GADGET" == "tcpdump" ]] && command -v tcpdump >/dev/null 2>&1; then + TCPDUMP_AVAIL="true" DISPLAY_CMD="$DISPLAY_CMD | tcpdump -nvr -" fi @@ -161,6 +168,9 @@ echo "Node: $NODE" >&2 echo "Timeout: ${TIMEOUT}s" >&2 echo "Image: $IG_IMAGE" >&2 echo "Command: $DISPLAY_CMD" >&2 +if [[ "$GADGET" == "tcpdump" && "$TCPDUMP_AVAIL" == "false" ]]; then + echo "Note: tcpdump not found; emitting raw pcap-ng to stdout." >&2 +fi if [[ "$DRY_RUN" == "true" ]]; then echo "(dry-run: command not executed)" >&2 @@ -169,7 +179,7 @@ fi echo "Ran gadget $GADGET on node $NODE (timeout ${TIMEOUT}s)" >&2 -if [[ "$GADGET" == "tcpdump" ]] && command -v tcpdump >/dev/null 2>&1; then +if [[ "$TCPDUMP_AVAIL" == "true" ]]; then "${FULL_CMD[@]}" | tcpdump -nvr - else "${FULL_CMD[@]}" diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/networking.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/networking.md index 2056c9812..b5fd1ed39 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/networking.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/networking.md @@ -32,7 +32,7 @@ Pods that are running but not Ready are removed from Endpoints. Check `kubectl g **Deep diagnostics with Inspektor Gadget** (when the above checks are inconclusive): -Use the [`run-ig` script](references/inspektor-gadget.md) with `--pod --ns ` and these gadgets: +Use [`scripts/run-ig.sh`](references/inspektor-gadget.md) (or `run-ig.ps1`) with `--pod --ns ` and these gadgets: - `snapshot_socket` — check what ports the pod is listening on - `trace_tcp` — trace connect/accept/close events @@ -81,7 +81,7 @@ Custom VNet DNS must forward `.cluster.local` to the CoreDNS ClusterIP and other **Deep diagnostics with Inspektor Gadget** (when the above checks are inconclusive): -Use the [`run-ig` script](references/inspektor-gadget.md) with `--pod --ns ` and `trace_dns`. Key signals: `rcode=3` (NXDOMAIN), `rcode=2` (SERVFAIL), high `latency` values, queries going to unexpected destinations. +Use [`scripts/run-ig.sh`](references/inspektor-gadget.md) (or `run-ig.ps1`) with `--pod --ns ` and `trace_dns`. Key signals: `rcode=3` (NXDOMAIN), `rcode=2` (SERVFAIL), high `latency` values, queries going to unexpected destinations. See [references/inspektor-gadget.md](references/inspektor-gadget.md). diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/node-issues.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/node-issues.md index 34303e126..9e69be29e 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/node-issues.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/node-issues.md @@ -107,7 +107,7 @@ Common culprit: high-volume container logs accumulating in `/var/log/containers` **Deep diagnostics with Inspektor Gadget** (PID pressure or unknown process load): -Use `run-ig --gadget snapshot_process --node ` to list all processes on the node. For node-wide scope, use `--node` (no pod filters). See [references/inspektor-gadget.md](references/inspektor-gadget.md). +Use `scripts/run-ig.sh --gadget snapshot_process --node ` (or `run-ig.ps1`) to list all processes on the node. For node-wide scope, use `--node` (no pod filters). See [references/inspektor-gadget.md](references/inspektor-gadget.md). --- diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md index 5b4e668b3..b614a8543 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/pod-failures.md @@ -49,11 +49,11 @@ kubectl describe pod -n | grep -A2 "Last State" Fix: increase `resources.limits.memory` or optimize application memory usage. Check `kubectl top pod -n ` for actual usage. -**OOM kill tracing with Inspektor Gadget:** Run `trace_oomkill` for the pod to see which process was killed and memory at kill time: `run-ig --gadget trace_oomkill --pod --ns `. See [references/inspektor-gadget.md](references/inspektor-gadget.md). +**OOM kill tracing with Inspektor Gadget:** Run `trace_oomkill` for the pod to see which process was killed and memory at kill time: `scripts/run-ig.sh --gadget trace_oomkill --pod --ns ` (or `run-ig.ps1`). See [references/inspektor-gadget.md](references/inspektor-gadget.md). **Deep diagnostics with Inspektor Gadget** (when logs and describe are inconclusive): -Use the [`run-ig` script](references/inspektor-gadget.md) with `--pod --ns ` and these gadgets: +Use [`scripts/run-ig.sh`](references/inspektor-gadget.md) (or `run-ig.ps1`) with `--pod --ns ` and these gadgets: - `trace_exec` — see what the container executes at startup - `trace_open` — find missing configs/secrets (retval -2 = ENOENT, -13 = EACCES) diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md index 4546121e3..aa8d6328d 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/references/command-flows.md @@ -79,7 +79,7 @@ kubectl describe quota -n ## Deep Diagnostics Flow (Inspektor Gadget) ```text -Standard diagnostics inconclusive -> select gadget from symptom-to-gadget map -> run `run-ig` (resolves node, applies timeout) -> interpret output -> correlate with prior evidence +Standard diagnostics inconclusive -> select gadget from symptom-to-gadget map -> run `scripts/run-ig.sh` (or `run-ig.ps1`; resolves node, applies timeout) -> interpret output -> correlate with prior evidence ``` Use when steps 1–3 of the evidence order (Azure-side, Kubernetes-side, and detector evidence) do not reveal root cause. See [inspektor-gadget.md](inspektor-gadget.md) for the full gadget catalog and command patterns. diff --git a/plugin/skills/azure-diagnostics/troubleshooting/aks/references/inspektor-gadget.md b/plugin/skills/azure-diagnostics/troubleshooting/aks/references/inspektor-gadget.md index d356a7e43..214caa30f 100644 --- a/plugin/skills/azure-diagnostics/troubleshooting/aks/references/inspektor-gadget.md +++ b/plugin/skills/azure-diagnostics/troubleshooting/aks/references/inspektor-gadget.md @@ -19,7 +19,9 @@ Symptom-to-Gadget Map) and interpret the output. .\scripts\run-ig.ps1 -Gadget trace_dns -Pod -Namespace ``` -**Options:** `--gadget` (required); target `--pod`/`--ns` **or** `--node`; `--container`; +**Options** (bash flags below; PowerShell uses PascalCase equivalents: `-Gadget`, `-Pod`, +`-Namespace`/`-Ns`, `-Node`, `-Container`, `-Timeout`, `-Filter`, `-Pf`, `-IgVersion`, +`-DryRun`): `--gadget` (required); target `--pod`/`--ns` **or** `--node`; `--container`; `--timeout ` override; `--filter ` (repeatable IG-flag passthrough, e.g. `--filter --max-entries --filter 20`); `--pf ""` (tcpdump only); `--ig-version `; `--dry-run`. Default timeout by gadget name: `snapshot_*`/`top_*` → 5s, @@ -31,9 +33,8 @@ Symptom-to-Gadget Map) and interpret the output. ## Common Filters -`--k8s-namespace`/`--k8s-podname`/`--k8s-containername` and `--timeout` are set by the script -(from `--pod`/`--ns`/`--container`/`--timeout`). Pass any other IG flag below via the script's -repeatable `--filter`, e.g. `--filter --max-entries --filter 20`. +The k8s scope filters and `--timeout` are set by the script. Pass any other IG flag below via +its repeatable `--filter`, e.g. `--filter --max-entries --filter 20`. | Filter | Description | |---|---| @@ -42,9 +43,7 @@ repeatable `--filter`, e.g. `--filter --max-entries --filter 20`. | `--interval ` | Reporting interval for `top_process` only (e.g. `5s`) | | `--syscall-filters ` | Comma-separated syscalls for `traceloop` (e.g. `open,connect,accept`). **Always specify** to limit data volume | -> **Tip:** For top/profile, set `--map-fetch-interval` ≤ half of `--timeout` to collect at least one batch. E.g. `--timeout 2 --map-fetch-interval 1000ms --max-entries 20`. -> -> **Note:** `top_process` uses `--interval` instead of `--map-fetch-interval`. E.g. `--timeout 10 --interval 5s --max-entries 20`. +> **Tip:** For top/profile, keep `--map-fetch-interval` ≤ half of `--timeout` to collect ≥1 batch. `top_process` uses `--interval` instead of `--map-fetch-interval`. ## Gadget Catalog From 9ea17bd0ea476040b919b112b999f076966f974b Mon Sep 17 00:00:00 2001 From: "Tom Meschter (from Dev Box)" Date: Thu, 23 Jul 2026 13:26:35 -0700 Subject: [PATCH 3/4] test(azure-diagnostics): add eval exercising run-ig script invocation Adds an integration stimulus that drives the agent to invoke the run-ig Inspektor Gadget helper script and early-terminates on the tool-call-match the instant the script is invoked, so the privileged kubectl debug never executes. No live cluster required. Grades skill invocation plus a tool-calls check confirming the script was actually called. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 92c8ec36-b76b-4825-896f-02598b79135e --- evals/azure-diagnostics/eval.yaml | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/evals/azure-diagnostics/eval.yaml b/evals/azure-diagnostics/eval.yaml index 524fa3819..f5d0fc6d2 100644 --- a/evals/azure-diagnostics/eval.yaml +++ b/evals/azure-diagnostics/eval.yaml @@ -129,6 +129,41 @@ stimuli: disallowed: - azure-compute # Global: no_runtime_failure + - type: output-not-matches + config: + pattern: "(?i)fatal error|unhandled exception|stack trace" + + # ── inspektor-gadget-run-ig-script ── + # Exercises the run-ig helper script for Inspektor Gadget without a live + # cluster: earlyTerminate fires on the tool-call-match the instant the agent + # tries to run scripts/run-ig.sh|.ps1, so the privileged `kubectl debug` never + # actually executes. The recorded execution_start still lets the tool-calls + # grader confirm the attempt. NOTE: early-terminated runs must NOT use the + # `completed` grader. + - name: "Inspektor Gadget run-ig script invocation" + prompt: "I'm troubleshooting DNS failures on AKS node aks-nodepool1-12345678-vmss000000. Use the azure-diagnostics Inspektor Gadget helper script to run a trace_dns gadget on that node." + config: + runs: 1 + tags: + type: integration + tier: full + cost: llm + area: behavior + earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh","argsPattern":"run-ig\\.(sh|ps1)"}]' + graders: + - type: skill-invocation + config: + required: + - azure-diagnostics + # The agent actually invoked the run-ig helper script (recorded at + # execution_start before earlyTerminate aborted the run). + - type: tool-calls + config: + required: + # Copilot CLI uses "powershell" on Windows, "bash" on other platforms. + - name: "(?i)^(bash|powershell|pwsh)$" + command: "(?i)run-ig\\.(sh|ps1)" + # Global: no_runtime_failure - type: output-not-matches config: pattern: "(?i)fatal error|unhandled exception|stack trace" \ No newline at end of file From 69c7ba71623232a5b2e4e951c74131dd58b71ac0 Mon Sep 17 00:00:00 2001 From: "Tom Meschter (from Dev Box)" Date: Tue, 4 Aug 2026 11:18:20 -0700 Subject: [PATCH 4/4] test(azure-diagnostics): use tool-call-result earlyTerminate for run-ig eval Address review feedback: switch the run-ig stimulus earlyTerminate from tool-call-match to tool-call-result. Terminating on the completed tool call reliably records the invocation for the grader; with no cluster/kubectl in CI the underlying kubectl debug fails instantly and harmlessly. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 92c8ec36-b76b-4825-896f-02598b79135e --- evals/azure-skills/azure-diagnostics/eval.yaml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/evals/azure-skills/azure-diagnostics/eval.yaml b/evals/azure-skills/azure-diagnostics/eval.yaml index f5d0fc6d2..52d1da5ef 100644 --- a/evals/azure-skills/azure-diagnostics/eval.yaml +++ b/evals/azure-skills/azure-diagnostics/eval.yaml @@ -135,11 +135,11 @@ stimuli: # ── inspektor-gadget-run-ig-script ── # Exercises the run-ig helper script for Inspektor Gadget without a live - # cluster: earlyTerminate fires on the tool-call-match the instant the agent - # tries to run scripts/run-ig.sh|.ps1, so the privileged `kubectl debug` never - # actually executes. The recorded execution_start still lets the tool-calls - # grader confirm the attempt. NOTE: early-terminated runs must NOT use the - # `completed` grader. + # cluster. earlyTerminate uses tool-call-result so the run stops once the + # run-ig invocation completes: with no kubectl/cluster in CI the underlying + # `kubectl debug` fails instantly and harmlessly, and the completed tool call + # is fully recorded for the grader to confirm the script was invoked. + # NOTE: early-terminated runs must NOT use the `completed` grader. - name: "Inspektor Gadget run-ig script invocation" prompt: "I'm troubleshooting DNS failures on AKS node aks-nodepool1-12345678-vmss000000. Use the azure-diagnostics Inspektor Gadget helper script to run a trace_dns gadget on that node." config: @@ -149,14 +149,13 @@ stimuli: tier: full cost: llm area: behavior - earlyTerminate: '[{"type":"tool-call-match","toolPattern":"bash|powershell|pwsh","argsPattern":"run-ig\\.(sh|ps1)"}]' + earlyTerminate: '[{"type":"tool-call-result","toolPattern":"bash|powershell|pwsh","argsPattern":"run-ig\\.(sh|ps1)"}]' graders: - type: skill-invocation config: required: - azure-diagnostics - # The agent actually invoked the run-ig helper script (recorded at - # execution_start before earlyTerminate aborted the run). + # The agent actually invoked the run-ig helper script. - type: tool-calls config: required: