diff --git a/.github/instructions/skill-files.instructions.md b/.github/instructions/skill-files.instructions.md index 7a2eb04ac..307c55b44 100644 --- a/.github/instructions/skill-files.instructions.md +++ b/.github/instructions/skill-files.instructions.md @@ -17,7 +17,7 @@ description: "Detailed description including trigger phrases and use cases." license: MIT metadata: author: Microsoft - version: "1.0.0" + version: "0.0.0-placeholder" --- ``` @@ -25,7 +25,7 @@ metadata: - **description**: 1-1024 characters, explain WHAT the skill does and WHEN to use it. Include trigger phrases. - **license**: Required for all skills. Use `MIT` unless there is a documented exception. - **metadata.author**: Recommended value is `Microsoft`. -- **metadata.version**: Semver format (`X.Y.Z`). Set to `"1.0.0"` for new skills. For skills under `plugin/`, versions are stamped automatically at build time by NBGV — use `"0.0.0-placeholder"` in source. For skills elsewhere (e.g., `.github/skills/`), set a real version and bump it in the same PR that modifies the skill. +- **metadata.version**: Set to `"0.0.0-placeholder"` for new skills. For skills under `plugins/`, versions are stamped automatically at build time by NBGV — use `"0.0.0-placeholder"` in source. For skills elsewhere (e.g., `.github/skills/`), set a real X.Y.Z version and bump it in the same PR that modifies the skill. ## Size Limits @@ -34,10 +34,12 @@ Keep the main SKILL.md concise. Move detailed documentation to files under the ` ## Required Sections 1. **Quick Reference** - Summary table with key properties (MCP tools, CLI commands, best for) -2. **When to Use This Skill** - Clear list of activation scenarios -3. **MCP Tools** - Table of available MCP commands with parameters -4. **Workflow/Steps** - Numbered or phased step-by-step processes -5. **Error Handling** - Table of errors, messages, and remediation +2. **Workflow/Steps** - Numbered or phased step-by-step processes + +## Optional Sections + +1. **Prerequisite** - Expected environmental conditions for the skill to operate (e.g. files in the workspace, local CLI tools, type of projects, etc.) +2. **Error Handling** - Table of errors, messages, and remediation. When the workflow is complicated and has a high chance of getting errors, use this section to provide troubleshooting guidance. Simple workflows don't need an explicit error handling section. ## Formatting Standards diff --git a/hooks/scripts/track-telemetry.ps1 b/hooks/scripts/track-telemetry.ps1 index c453b54b6..1dd6f1b70 100644 --- a/hooks/scripts/track-telemetry.ps1 +++ b/hooks/scripts/track-telemetry.ps1 @@ -74,7 +74,8 @@ # - toolArgs.path / toolArgs.filePath (Copilot CLI) # - tool_input.filePath / tool_input.file_path / tool_input.path (Claude Code / VS Code) # -# Recognized azure-skills install paths: +# Recognized install paths (one set per plugin, see $pathPatterns below): +# azure-skills: # - .copilot/installed-plugins//azure/skills/... # ( is the marketplace/catalog folder the plugin was # installed under, e.g. "awesome-copilot" — it does not necessarily @@ -82,6 +83,11 @@ # - .claude/plugins/cache/azure-skills/azure//skills/... # - .claude/plugins/cache/claude-plugins-official/azure//skills/... # - .vscode/agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills/... +# azure-kusto-graph-skills: +# - .copilot/installed-plugins//azure-kusto-graph-skills/skills/... +# - .claude/plugins/cache/azure-skills/azure-kusto-graph-skills//skills/... +# - .vscode/agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-kusto-graph-skills/skills/... +# shared: # - .agents/skills/... # # If the path matches AND is not a SKILL.md file, the relative path after @@ -289,14 +295,33 @@ function Get-ToolInputPath { # === STEP 2: Determine what to track for azmcp === -# Azure-skills path patterns per client (used for SKILL.md and file-reference matching) +# Path patterns per client, one block per plugin (used for SKILL.md and +# file-reference matching). When onboarding another plugin, add a new block by +# swapping both the catalog/plugin segments (e.g. "azure" and "azure-skills") +# for the new plugin's name. + +# --- azure-skills plugin --- +# The Copilot CLI pattern wildcards the catalog/marketplace folder name +# (e.g. "awesome-copilot") since it does not necessarily match the plugin's +# own name ("azure"). $pathPatternCopilot = '\.copilot/installed-plugins/[^/]+/azure/skills/' $pathPatternClaude = '\.claude/plugins/cache/(azure-skills|claude-plugins-official)/azure/[0-9.]+/skills/' $pathPatternVscodeAgentPlugins = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-skills/skills/' + +# --- azure-kusto-graph-skills plugin --- +$pathPatternCopilotKustoGraph = '\.copilot/installed-plugins/[^/]+/azure-kusto-graph-skills/skills/' +$pathPatternClaudeKustoGraph = '\.claude/plugins/cache/azure-skills/azure-kusto-graph-skills/[0-9.]+/skills/' +$pathPatternVscodeAgentPluginsKustoGraph = 'agent-plugins/github\.com/microsoft/azure-skills/\.github/plugins/azure-kusto-graph-skills/skills/' + +# --- shared across all plugins --- $pathPatternAgentsSkills = '\.agents/skills/' # Put the path patterns into an array for easier iteration -$pathPatterns = @($pathPatternCopilot, $pathPatternClaude, $pathPatternVscodeAgentPlugins, $pathPatternAgentsSkills) +$pathPatterns = @( + $pathPatternCopilot, $pathPatternClaude, $pathPatternVscodeAgentPlugins, + $pathPatternCopilotKustoGraph, $pathPatternClaudeKustoGraph, $pathPatternVscodeAgentPluginsKustoGraph, + $pathPatternAgentsSkills +) # If $env:AZURE_SKILLS_PLUGIN_ROOT is set, add it to the path patterns for local skill development if ($env:AZURE_SKILLS_PLUGIN_ROOT) { diff --git a/hooks/scripts/track-telemetry.sh b/hooks/scripts/track-telemetry.sh index a3c85ceb1..0b1f9506c 100755 --- a/hooks/scripts/track-telemetry.sh +++ b/hooks/scripts/track-telemetry.sh @@ -76,7 +76,8 @@ # - toolArgs.path / toolArgs.filePath (Copilot CLI) # - tool_input.filePath / tool_input.file_path / tool_input.path (Claude Code / VS Code) # -# Recognized azure-skills install paths: +# Recognized install paths (one set per plugin, see is_azure_skills_path): +# azure-skills: # - .copilot/installed-plugins//azure/skills/... # ( is the marketplace/catalog folder the plugin was # installed under, e.g. "awesome-copilot" — it does not necessarily @@ -84,6 +85,11 @@ # - .claude/plugins/cache/azure-skills/azure//skills/... # - .claude/plugins/cache/claude-plugins-official/azure//skills/... # - .vscode/agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills/... +# azure-kusto-graph-skills: +# - .copilot/installed-plugins//azure-kusto-graph-skills/skills/... +# - .claude/plugins/cache/azure-skills/azure-kusto-graph-skills//skills/... +# - .vscode/agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-kusto-graph-skills/skills/... +# shared: # - .agents/skills/... # # If the path matches AND is not a SKILL.md file, the relative path after @@ -296,15 +302,30 @@ fi # === STEP 2: Determine what to track for azmcp === -# Check if a path matches any known azure-skills folder structure -# Returns 0 (true) if matched, 1 (false) otherwise +# Check if a path matches any known plugin skills folder structure. + # Each plugin has its own block below — when onboarding another plugin, add a new + # block by swapping both the catalog/plugin segments (e.g. "azure" and + # "azure-skills") for the new plugin. Returns 0 (true) if matched, 1 (false) otherwise. is_azure_skills_path() { local p="$1" + + # --- azure-skills plugin --- + # The Copilot CLI pattern wildcards the catalog/marketplace folder name + # (e.g. "awesome-copilot") since it does not necessarily match the + # plugin's own name ("azure"). [[ "$p" == *".copilot/installed-plugins/"*"/azure/skills/"* ]] && return 0 [[ "$p" == *".claude/plugins/cache/azure-skills/azure/"*"/skills/"* ]] && return 0 [[ "$p" == *".claude/plugins/cache/claude-plugins-official/azure/"*"/skills/"* ]] && return 0 [[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-skills/skills/"* ]] && return 0 + + # --- azure-kusto-graph-skills plugin --- + [[ "$p" == *".copilot/installed-plugins/"*"/azure-kusto-graph-skills/skills/"* ]] && return 0 + [[ "$p" == *".claude/plugins/cache/azure-skills/azure-kusto-graph-skills/"*"/skills/"* ]] && return 0 + [[ "$p" == *"agent-plugins/github.com/microsoft/azure-skills/.github/plugins/azure-kusto-graph-skills/skills/"* ]] && return 0 + + # --- shared across all plugins --- [[ "$p" == *".agents/skills/"* ]] && return 0 + # Local plugin development: match paths under AZURE_SKILLS_PLUGIN_ROOT/skills/ # (e.g. when loading a local plugin via `--plugin-dir`) if [ -n "$AZURE_SKILLS_PLUGIN_ROOT" ]; then @@ -431,3 +452,4 @@ fi # Output success to stdout (required by hooks) return_success + diff --git a/pipelines/scripts/Invoke-RunBenchmarks.ps1 b/pipelines/scripts/Invoke-RunBenchmarks.ps1 index 23ce67d6a..aa5937af1 100644 --- a/pipelines/scripts/Invoke-RunBenchmarks.ps1 +++ b/pipelines/scripts/Invoke-RunBenchmarks.ps1 @@ -172,7 +172,7 @@ "--agent", "github-copilot-cli", "--benchmark", $Benchmark, "--model", $m, - "--encrypted-env", "CAPI_INTEGRATION_ID CAPI_HMAC_KEY", + "--encrypted-env", "CAPI_INTEGRATION_ID", "CAPI_HMAC_KEY", "--env", "USE_COPILOT_CLI_VERSION", "--dataset", (Join-Path $targetDir "metadata.csv"), "--tag", "org=CoreAI Cloud and Tools", diff --git a/plugins/azure-skills/skills/microsoft-foundry/SKILL.md b/plugins/azure-skills/skills/microsoft-foundry/SKILL.md index 98314c6c3..f16db3917 100644 --- a/plugins/azure-skills/skills/microsoft-foundry/SKILL.md +++ b/plugins/azure-skills/skills/microsoft-foundry/SKILL.md @@ -54,6 +54,7 @@ This skill includes specialized sub-skills for specific workflows. **When a sub- | **observe** | Evaluate agent quality, run batch evals, analyze failures, optimize prompts, improve agent instructions, compare versions, set up CI/CD monitoring, and enable continuous production evaluation | [observe](foundry-agent/observe/observe.md) | | **trace** | Query traces, analyze latency/failures, correlate eval results to specific responses via App Insights `customEvents` | [trace](foundry-agent/trace/trace.md) | | **troubleshoot** | View hosted agent logs, query telemetry, diagnose failures | [troubleshoot](foundry-agent/troubleshoot/troubleshoot.md) | +| **validate** | Use only when the user explicitly asks to use this validation sub-skill or to validate Microsoft Foundry hosted-agent code against best practices. Never invoke it proactively or add it to another workflow. | [validate](foundry-agent/validate/validate.md) | | **create (quick start)** | Create a new hosted Foundry agent from scratch end-to-end — scaffold, provision or use an existing Foundry project, deploy, and smoke-test. Do not use for any work on existing code. For anything not covered by the quickstart, use **create**. | [create/quick-start-hosted.md](foundry-agent/create/quick-start-hosted.md) | | **create** | Use when the standard end-to-end happy path (quick start) doesn't fit. Create a new Foundry agent, update code of an existing agent, continue development of an existing agent, wire connections at scaffold time, use advanced setup or A2A (Agent2Agent), or recover from a failed quickstart run. | [create](foundry-agent/create/create-hosted.md) | | **agent-optimizer** | Make existing Python hosted-agent code optimization-ready, configure eval.yaml, run Agent Optimizer jobs, apply candidates locally, and deploy through azd after review. | [agent-optimizer](foundry-agent/agent-optimizer/agent-optimizer.md) | diff --git a/plugins/azure-skills/skills/microsoft-foundry/foundry-agent/validate/references/default-rules.yaml b/plugins/azure-skills/skills/microsoft-foundry/foundry-agent/validate/references/default-rules.yaml index 5c6443f8e..4ac5b3f80 100644 --- a/plugins/azure-skills/skills/microsoft-foundry/foundry-agent/validate/references/default-rules.yaml +++ b/plugins/azure-skills/skills/microsoft-foundry/foundry-agent/validate/references/default-rules.yaml @@ -1,4 +1,4 @@ -version: 4.3.0 +version: 1.0.0 scope: Repository review rules, not certification controls. rules: @@ -14,3 +14,77 @@ rules: statusCriteria: pass: Local configuration places every MCP server in a Foundry Toolbox and the agent uses its consumer endpoint, or no local Toolbox definition exists and the agent clearly consumes an externally managed Toolbox endpoint. No code path accesses an MCP server endpoint directly. fail: Local configuration places an MCP server outside Toolbox, or any hosted-agent code path accesses an MCP server endpoint directly instead of a Toolbox consumer endpoint. + + - id: OBS-001 + title: Enable supported tracing for hosted agents + level: recommendation + rationale: Foundry tracing provides end-to-end visibility into agent invocations, model operations, and tool calls. Hosted-agent protocol libraries can emit OpenTelemetry automatically when project monitoring or an OTLP exporter is configured, so explicit application instrumentation is not always required. + guidance: + - "https://learn.microsoft.com/azure/foundry/agents/how-to/configure-hosted-agent-telemetry" + - "https://learn.microsoft.com/azure/foundry/observability/how-to/trace-agent-setup" + when: Apply to every hosted agent configured with host azure.ai.agent. + checks: >- + Inspect azure.yaml, agent configuration, infrastructure, dependencies, and observability documentation. Accept any supported tracing path with an export destination: Foundry project monitoring connected to Application Insights, automatic instrumentation supplied by the hosted-agent library or a framework supported by the Microsoft OpenTelemetry distribution and configured for Application Insights or OTLP export, or explicit OpenTelemetry instrumentation with a configured exporter. Do not require application OpenTelemetry code when the hosted-agent protocol library or Foundry server-side tracing supplies it. Do not treat use of a hosting or instrumentation library alone as proof that telemetry is exported. Treat project-level monitoring that cannot be inspected from the repository as missing evidence, not as proof that tracing is disabled. + statusCriteria: + pass: Repository evidence identifies both a supported tracing path and an Application Insights or OTLP export destination, and the configuration does not contradict or disable that path. + fail: Repository evidence explicitly disables all applicable tracing or telemetry export without documenting an alternative, or claims tracing is enabled while its repository-managed configuration is contradictory. + inconclusive: Tracing may be supplied by remotely managed Foundry project monitoring or another external configuration that cannot be established from repository evidence. + + - id: SDK-001 + title: Use consistent Microsoft Agent Framework dependencies + level: recommendation + rationale: Microsoft Agent Framework is the recommended orchestration framework for applicable Foundry hosted-agent scenarios, but other supported frameworks and custom implementations remain valid choices. This rule checks repository-level dependency consistency rather than requiring the latest package or proving complete runtime compatibility. + guidance: + - "https://learn.microsoft.com/azure/foundry/how-to/develop/sdk-overview#agent-framework" + - "https://learn.microsoft.com/azure/foundry/agents/quickstarts/quickstart-deploy-own-code#choose-your-framework" + when: Apply only when the hosted agent declares or imports Microsoft Agent Framework; otherwise skip this rule. + checks: >- + Inspect dependency manifests, lock files, central package-management files, and Microsoft Agent Framework imports under the agent root. Verify that the project declares the official Agent Framework package family and that package declarations, effective versions when determinable, and source imports are internally consistent. An exact declaration or fully resolvable central version declaration can establish the effective direct dependency version. Do not execute tests, inspect dependency caches or generated results, infer API availability from an uninstalled package, or require the latest package version. + statusCriteria: + pass: The official Agent Framework package family is declared, the effective direct dependency version can be determined from repository configuration, and source imports are consistent with that package family. + fail: Repository evidence explicitly shows an unrelated package presented as Agent Framework, conflicting effective dependency versions for the deployed agent, or imports inconsistent with the declared package family. + inconclusive: The effective package or version is affected by unresolved ranges, unresolved central version management, environment-specific resolution, or other indirection, or imports cannot be mapped reliably to the declared package family. + + - id: AGT-001 + title: Do not instruct agents to bypass controls or fabricate success + level: warning + rationale: Operative agent instructions must not authorize fabricated outcomes, unrestricted consequential actions, or bypasses of runtime controls and required approvals. + guidance: + - "https://learn.microsoft.com/azure/foundry/agents/concepts/tool-best-practice" + - "https://learn.microsoft.com/azure/foundry/responsible-ai/agents/transparency-note" + when: Apply when the repository stores or references instructions that govern the hosted agent at runtime; otherwise skip this rule. + checks: >- + Identify instructions that are actually loaded or referenced by the hosted agent, distinguishing them from documentation, comments, tests, examples, and security counterexamples. Inspect the operative instructions for language that requires or permits the agent to fabricate results, represent failed operations as successful, bypass authorization or approval controls, or perform unrestricted consequential actions. Do not treat ordinary error recovery, retries, fallback behavior, or clearly bounded automation as a violation. + statusCriteria: + pass: Operative instructions do not require or permit fabrication, control or approval bypass, treating failure as success, or unrestricted consequential action. + fail: Operative instructions explicitly require or permit at least one of those behaviors. + inconclusive: Potentially unsafe wording is ambiguous, or repository evidence cannot establish whether the identified instructions are operative. + + - id: CFG-001 + title: Do not override platform-managed runtime configuration + level: warning + rationale: Foundry injects platform-managed runtime values into hosted agents. Redefining reserved FOUNDRY_* or AGENT_* variables can shadow those values and break authentication, routing, telemetry, or agent lifecycle behavior. + guidance: + - "https://learn.microsoft.com/azure/foundry/agents/how-to/configure-hosted-agent-env-variables#review-platform-environment-variables" + when: Apply when hosted-agent runtime or deployment configuration is present. + checks: >- + Inspect azure.yaml, agent configuration, container configuration, deployment scripts, and source code that writes environment variables for the deployed process. Verify that FOUNDRY_* and AGENT_* values supplied by the platform are consumed but are not declared, assigned, or overwritten by repository-managed deployed runtime configuration. Include deprecated user-defined variables such as FOUNDRY_TOOLBOX_ENDPOINT. Reading a platform-managed variable is valid. Do not fail a local-only development environment file unless repository evidence shows that it is committed as deployed configuration or consumed by the deployed runtime. Do not use this rule for general required-setting validation, credentials, endpoint literals, or other configuration without a reserved prefix. + statusCriteria: + pass: Repository-managed deployed runtime configuration does not declare, assign, or overwrite FOUNDRY_* or AGENT_* variables; any local-only development values are not consumed by the deployed runtime. + fail: Repository evidence explicitly declares, assigns, or overwrites a reserved FOUNDRY_* or AGENT_* variable in deployed hosted-agent runtime configuration. + inconclusive: Repository evidence cannot establish whether a local or externally generated configuration is consumed by the deployed runtime or could shadow a platform-managed value. + + - id: PROTO-001 + title: Keep declared protocols consistent with runtime handlers + level: warning + rationale: A Foundry hosted agent must implement each protocol endpoint declared by its deployment configuration. A mismatch prevents the platform from invoking the agent through that protocol. + guidance: + - "https://learn.microsoft.com/azure/foundry/agents/concepts/hosted-agent-contract#protocol-endpoints" + - "https://learn.microsoft.com/azure/foundry/agents/how-to/add-protocol-adapter" + when: Apply when the hosted-agent service declares the Responses or Invocations protocol in azure.yaml; otherwise skip this rule. + checks: >- + Inspect the target azure.ai.agent service, its locally resolvable configuration references, dependency declarations, and deployed entry point. For each Responses or Invocations value in that service's azure.yaml protocols[].protocol, verify that the deployed entry point provides the matching endpoint and handler through an official protocol adapter or a valid custom implementation. When the project declares a specific hosted-agent protocol SDK, verify that the entry point uses that SDK's corresponding adapter or host, while accepting other documented registration patterns. Consistent examples include responses with ResponsesHostServer, ResponsesAgentServerHost, or a custom POST /responses handler, and invocations with InvocationsHostServer, InvocationAgentServerHost, or a custom POST /invocations handler; these examples are not an exhaustive class-name allowlist. Inspect only these two protocols and runtime paths that can be fully resolved from files under the agent root. Do not execute the server or infer dynamically registered routes. Failure to locate or resolve an adapter, route, or handler is inconclusive, not fail. + statusCriteria: + pass: Every declared Responses or Invocations protocol has either a statically identifiable official adapter and handler or a statically identifiable custom endpoint and handler, with no explicit protocol mismatch. + fail: Repository evidence fully resolves the deployed entry point and explicitly shows that a declared Responses or Invocations protocol is mapped to a different endpoint or handler, or that the complete static route registration omits the declared protocol endpoint. + inconclusive: An adapter, route, handler, deployed entry point, local configuration reference, framework registration, or dynamically constructed route cannot be resolved sufficiently to map every declared Responses or Invocations protocol to a valid implementation path. diff --git a/tests/package-lock.json b/tests/package-lock.json index cf37d5808..8265e7ff7 100644 --- a/tests/package-lock.json +++ b/tests/package-lock.json @@ -16,7 +16,7 @@ "@types/jest": "^30.0.0", "@types/node": "^25.9.3", "cross-env": "^10.1.0", - "eslint": "^10.8.1", + "eslint": "^10.9.0", "eslint-import-resolver-typescript": "^4.4.4", "eslint-plugin-import-x": "^4.17.1", "eslint-plugin-jest": "^29.16.1", @@ -4135,9 +4135,9 @@ } }, "node_modules/eslint": { - "version": "10.8.1", - "resolved": "https://ms-feed-2.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/eslint/-/eslint-10.8.1.tgz", - "integrity": "sha1-+zfVFMGbbdWy1rcBaf0m/d+peWc=", + "version": "10.9.0", + "resolved": "https://ms-feed-25.pkgs.visualstudio.com/1es-public/_packaging/npm-public/npm/registry/eslint/-/eslint-10.9.0.tgz", + "integrity": "sha1-PYYGigbGx4FhpAYuaYdNOPWdSYs=", "dev": true, "license": "MIT", "workspaces": [ diff --git a/tests/package.json b/tests/package.json index 86f072c2d..21cb0f842 100644 --- a/tests/package.json +++ b/tests/package.json @@ -29,7 +29,7 @@ "@types/jest": "^30.0.0", "@types/node": "^25.9.3", "cross-env": "^10.1.0", - "eslint": "^10.8.1", + "eslint": "^10.9.0", "eslint-import-resolver-typescript": "^4.4.4", "eslint-plugin-import-x": "^4.17.1", "eslint-plugin-jest": "^29.16.1",