diff --git a/.github/ISSUE_TEMPLATE/skill_extension_request.yml b/.github/ISSUE_TEMPLATE/skill_extension_request.yml index 205e2838f..163e4d66a 100644 --- a/.github/ISSUE_TEMPLATE/skill_extension_request.yml +++ b/.github/ISSUE_TEMPLATE/skill_extension_request.yml @@ -21,7 +21,7 @@ body: id: existing-skill attributes: label: Existing Skill to Extend - description: Which skill do you want to extend? (See `plugin/skills/`.) + description: Which skill do you want to extend? (See `plugins/*/skills/`.) placeholder: ex. azure-prepare validations: required: true diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 5c05a1eb0..893f527ef 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -97,7 +97,7 @@ SKIP_INTEGRATION_TESTS=true npm test -- --testPathPatterns= ### Steps -1. **Create the skill directory**: `plugin/skills//` +1. **Create the skill directory**: `plugins//skills//` 2. **Add `version.json`**: ```json diff --git a/.github/instructions/skill-files.instructions.md b/.github/instructions/skill-files.instructions.md index 5383396f4..7a2eb04ac 100644 --- a/.github/instructions/skill-files.instructions.md +++ b/.github/instructions/skill-files.instructions.md @@ -1,5 +1,5 @@ --- -applyTo: plugin/skills/**/SKILL.md +applyTo: plugins/*/skills/**/SKILL.md --- # Skill File Authoring Guidelines diff --git a/.github/skills/investigate-integration-test/SKILL.md b/.github/skills/investigate-integration-test/SKILL.md index 92152c780..e6cfb0b07 100644 --- a/.github/skills/investigate-integration-test/SKILL.md +++ b/.github/skills/investigate-integration-test/SKILL.md @@ -22,5 +22,5 @@ Investigates a failing integration test given a GitHub issue in `microsoft/GitHu 1. Read the GitHub issue. 2. Download the test logs and artifacts from the linked run. 3. Look through the logs/artifacts and analyze the test with the prompt specified in the issue to diagnose the failure. -4. Examine the relevant skills under `plugin/skills` for context. +4. Examine the relevant skills under `plugins/*/skills` for context. 5. Offer a suggested fix for each identified problem. Do not implement any fixes without the user's approval. diff --git a/.github/skills/sensei/SKILL.md b/.github/skills/sensei/SKILL.md index a96165df4..de363567e 100644 --- a/.github/skills/sensei/SKILL.md +++ b/.github/skills/sensei/SKILL.md @@ -41,7 +41,7 @@ When user says "sensei help" or asks how to use sensei, show this: For each skill, execute this loop until the frontmatter aligns with convention, have thorough routing tests AND routing tests pass: -1. **READ** - Load `plugin/skills/{skill-name}/SKILL.md`, and vally eval suites in `evals/{skill-name}/*.yaml`. +1. **READ** - Load `plugins/{plugin}/skills/{skill-name}/SKILL.md`, and vally eval suites in `evals/{skill-name}/*.yaml`. 2. **VERIFY** - Compare the skill frontmatter with conventions in [CONVENTIONS](references/CONVENTIONS.md). If the skill's frontmatter violates any written convention, notify the user and propose a fix to align the frontmatter with the convention. 3. **SCAFFOLD** - If `evals/{skill-name}/` doesn't exist, follow instructions in `vally-eval` skill to scaffold a set of routing tests. The routing tests test if the skill can be invoked for target user prompts. Generate user prompts that match the target scenario of the skill's description. 4. **IMPROVE** - If the skill description doesn't already have WHEN: triggers, add them. @@ -52,7 +52,7 @@ For each skill, execute this loop until the frontmatter aligns with convention, ## Constraints -- Only modify `plugin/skills/` - these are the Azure skills used by Copilot +- Only modify `plugins/{plugin}/skills/` - these are the Azure skills used by Copilot - Files in `.github/skills/` should be left as is - Max 5 iterations per skill before moving on diff --git a/.github/skills/skill-authoring/references/token-budgets.md b/.github/skills/skill-authoring/references/token-budgets.md index ea2fb08c6..c3366448c 100644 --- a/.github/skills/skill-authoring/references/token-budgets.md +++ b/.github/skills/skill-authoring/references/token-budgets.md @@ -31,7 +31,7 @@ Update links: `[guide](references/large-guide.md)` → `[guide](references/large ```bash cd scripts -npm run tokens -- check plugin/skills/my-skill/SKILL.md +npm run tokens -- check plugins/{plugin}/skills/my-skill/SKILL.md ``` ## Why Token Limits Matter diff --git a/.github/skills/skill-reviewer/SKILL.md b/.github/skills/skill-reviewer/SKILL.md index ba706e8d5..3ca67d003 100644 --- a/.github/skills/skill-reviewer/SKILL.md +++ b/.github/skills/skill-reviewer/SKILL.md @@ -13,11 +13,11 @@ Performs thorough, structured code reviews of skill PRs — severity-classified ## When to Use -- Reviewing a PR that adds or modifies a skill under `plugin/skills/` or `.github/skills/` +- Reviewing a PR that adds or modifies a skill under `plugins/{plugin}/skills/` or `.github/skills/` - Checking skill compliance before submitting a PR - Auditing an existing skill for quality issues -> 💡 **Note:** `.github/skills/` meta-skills have different conventions — checklist sections 8-9 apply only to `plugin/skills/` service skills. +> 💡 **Note:** `.github/skills/` meta-skills have different conventions — checklist sections 8-9 apply only to `plugins/{plugin}/skills/` service skills. ## Review Workflow diff --git a/.github/skills/skill-reviewer/references/review-checklist.md b/.github/skills/skill-reviewer/references/review-checklist.md index b3ae97f32..d36e54a2f 100644 --- a/.github/skills/skill-reviewer/references/review-checklist.md +++ b/.github/skills/skill-reviewer/references/review-checklist.md @@ -27,7 +27,7 @@ Flag files that exceed limits. For large files, recommend splitting by category ## 3. Required SKILL.md Sections -### Service skills (`plugin/skills/`) +### Service skills (`plugins/{plugin}/skills/`) Every service skill SKILL.md must contain: diff --git a/.github/workflows/test-all-integration.yml b/.github/workflows/test-all-integration.yml index 62d043206..95661ab5b 100644 --- a/.github/workflows/test-all-integration.yml +++ b/.github/workflows/test-all-integration.yml @@ -376,10 +376,12 @@ jobs: env: TOKEN_USAGE_STORAGE_ACCOUNT: ${{ vars.REPORT_STORAGE_ACCOUNT }} TOKEN_USAGE_TABLE_NAME: ${{ vars.TOKEN_USAGE_TABLE || 'integrationtokenusage' }} - SKILL: ${{ matrix.skill }} + SKILL_INPUT: ${{ matrix.skill }} BRANCH: ${{ github.ref_name }} RUN_ID: ${{ github.run_id }} - run: npm run upload:token-usage + run: | + export SKILL="${SKILL_INPUT##*/}" + npm run upload:token-usage # Upload per-run tool usage (one row per tool call) to the Azure Table that # powers the dashboard's per-run tool review. Runs for both scheduled and manual runs. @@ -389,7 +391,9 @@ jobs: env: TOOL_USAGE_STORAGE_ACCOUNT: ${{ vars.REPORT_STORAGE_ACCOUNT }} TOOL_USAGE_TABLE_NAME: ${{ vars.TOOL_USAGE_TABLE || 'integrationtoolusage' }} - SKILL: ${{ matrix.skill }} + SKILL_INPUT: ${{ matrix.skill }} BRANCH: ${{ github.ref_name }} RUN_ID: ${{ github.run_id }} - run: npm run upload:tool-usage + run: | + export SKILL="${SKILL_INPUT##*/}" + npm run upload:tool-usage diff --git a/dashboard/src/skills/App.tsx b/dashboard/src/skills/App.tsx index d2f474981..ba08b7405 100644 --- a/dashboard/src/skills/App.tsx +++ b/dashboard/src/skills/App.tsx @@ -58,16 +58,16 @@ const REPO_BLOB_BASE = /** * Build a link to a skill's SKILL.md source file on GitHub. * - * The frontmatter collector validates the built `output/skills/` tree, so the + * The frontmatter collector validates the built `output/{plugin}/skills/` tree, so the * reported path may be prefixed with `output/`. That directory is git-ignored, - * so we normalize it back to the `plugin/skills/` source path. Returns null + * so we normalize it back to the `plugins/{plugin}/skills/` source path. Returns null * when no usable SKILL.md path is available. */ export function skillMdUrl(path: string): string | null { const normalized = path.replace(/\\/g, "/").trim(); if (!normalized.endsWith("/SKILL.md")) return null; - const sourcePath = normalized.replace(/^output\/skills\//, "plugin/skills/"); - if (!sourcePath.startsWith("plugin/skills/")) return null; + const sourcePath = normalized.replace(/^output\/([a-zA-Z-]+)\/skills\//, "plugins/$1/skills/"); + if (!sourcePath.startsWith("plugins/")) return null; return `${REPO_BLOB_BASE}/${sourcePath}`; } @@ -82,7 +82,7 @@ interface HealthData { function isPluginSkillPath(pathValue: string): boolean { const normalized = pathValue.replace(/\\/g, "/"); - return normalized.startsWith("output/skills/") || normalized.startsWith("plugin/skills/"); + return /output\/[a-zA-Z-]+\/skills/.test(normalized) || /plugins\/[a-zA-Z-]+\/skills/.test(normalized); } /** Extract plugin skills (with descriptions) from the frontmatter category. */ diff --git a/dashboard/src/skills/__tests__/skillLink.test.ts b/dashboard/src/skills/__tests__/skillLink.test.ts index d2abc3bff..393185f52 100644 --- a/dashboard/src/skills/__tests__/skillLink.test.ts +++ b/dashboard/src/skills/__tests__/skillLink.test.ts @@ -5,28 +5,28 @@ const BASE = "https://github.com/microsoft/GitHub-Copilot-for-Azure/blob/main"; describe("skillMdUrl", () => { it("maps a built output path back to the plugin source", () => { - expect(skillMdUrl("output/skills/azure-deploy/SKILL.md")).toBe( - `${BASE}/plugin/skills/azure-deploy/SKILL.md`, + expect(skillMdUrl("output/azure-skills/skills/azure-deploy/SKILL.md")).toBe( + `${BASE}/plugins/azure-skills/skills/azure-deploy/SKILL.md`, ); }); it("passes through a plugin source path unchanged", () => { - expect(skillMdUrl("plugin/skills/azure-prepare/SKILL.md")).toBe( - `${BASE}/plugin/skills/azure-prepare/SKILL.md`, + expect(skillMdUrl("plugins/azure-skills/skills/azure-prepare/SKILL.md")).toBe( + `${BASE}/plugins/azure-skills/skills/azure-prepare/SKILL.md`, ); }); it("handles nested skill folders", () => { expect( - skillMdUrl("output/skills/microsoft-foundry/foundry-agent/create/SKILL.md"), + skillMdUrl("output/azure-skills/skills/microsoft-foundry/foundry-agent/create/SKILL.md"), ).toBe( - `${BASE}/plugin/skills/microsoft-foundry/foundry-agent/create/SKILL.md`, + `${BASE}/plugins/azure-skills/skills/microsoft-foundry/foundry-agent/create/SKILL.md`, ); }); it("normalizes backslash separators", () => { - expect(skillMdUrl("output\\skills\\azure-deploy\\SKILL.md")).toBe( - `${BASE}/plugin/skills/azure-deploy/SKILL.md`, + expect(skillMdUrl("output\\azure-skills\\skills\\azure-deploy\\SKILL.md")).toBe( + `${BASE}/plugins/azure-skills/skills/azure-deploy/SKILL.md`, ); }); @@ -35,7 +35,7 @@ describe("skillMdUrl", () => { }); it("returns null for a non-SKILL.md path", () => { - expect(skillMdUrl("plugin/skills/azure-deploy/references/foo.md")).toBeNull(); + expect(skillMdUrl("plugins/azure-skills/skills/azure-deploy/references/foo.md")).toBeNull(); }); it("returns null for a path outside the skills tree", () => { @@ -52,7 +52,7 @@ describe("skillsFromHealthData", () => { { name: "azure-deploy", metadata: { - path: "output/skills/azure-deploy/SKILL.md", + path: "output/azure-skills/skills/azure-deploy/SKILL.md", description: "Deploy skill", }, }, @@ -66,9 +66,9 @@ describe("skillsFromHealthData", () => { }); expect(skills).toHaveLength(1); expect(skills[0].name).toBe("azure-deploy"); - expect(skills[0].path).toBe("output/skills/azure-deploy/SKILL.md"); + expect(skills[0].path).toBe("output/azure-skills/skills/azure-deploy/SKILL.md"); expect(skillMdUrl(skills[0].path)).toBe( - `${BASE}/plugin/skills/azure-deploy/SKILL.md`, + `${BASE}/plugins/azure-skills/skills/azure-deploy/SKILL.md`, ); }); }); diff --git a/evals/azure-skills/azure-enterprise-infra-planner/eval.yaml b/evals/azure-skills/azure-enterprise-infra-planner/eval.yaml index cc06f036a..f4c5c2761 100644 --- a/evals/azure-skills/azure-enterprise-infra-planner/eval.yaml +++ b/evals/azure-skills/azure-enterprise-infra-planner/eval.yaml @@ -578,11 +578,6 @@ stimuli: pattern: "(?i)fatal error|unhandled exception|stack trace" # ── response-jumpbox-nsgs-internal-lb ── - # Jest: integration.test.ts → response-quality "generates infrastructure-plan.json with expected resources" - # Assertions: skill-invocation + file-exists(.azure/insights.json, infrastructure-plan.json) + JSON schema markers - # + networking+compute resource types in plan + no IaC files generated (Bicep/Terraform/tfvars). - # followUp 2 explicitly halts the agent at the Phase 6 IaC Gate - # (see plugin/skills/azure-enterprise-infra-planner/references/phases/6-generate-iac.md). - name: "Response - Jumpbox + NSGs + Internal LB (plan only)" prompt: "Provision a jumpbox VM for secure management, establish NSGs for each tier, and connect tiers using internal Azure Load Balancer. Assume all defaults to make the plan." tags: diff --git a/scripts/src/dashboard/__tests__/collectors/frontmatter.test.ts b/scripts/src/dashboard/__tests__/collectors/frontmatter.test.ts index 43c426c30..e037ae61d 100644 --- a/scripts/src/dashboard/__tests__/collectors/frontmatter.test.ts +++ b/scripts/src/dashboard/__tests__/collectors/frontmatter.test.ts @@ -71,7 +71,7 @@ describe("parseFrontmatterJson", () => { skills: [ { name: "bad-skill", - path: "plugin/skills/bad-skill/SKILL.md", + path: "plugins/azure-skills/skills/bad-skill/SKILL.md", status: "fail", errors: ["[name-format] name uses uppercase"], warnings: [], @@ -95,7 +95,7 @@ describe("parseFrontmatterJson", () => { skills: [ { name: "warn-skill", - path: "plugin/skills/warn-skill/SKILL.md", + path: "plugins/azure-skills/skills/warn-skill/SKILL.md", status: "warn", errors: [], warnings: ["[description-length] description is very long"], @@ -118,7 +118,7 @@ describe("parseFrontmatterJson", () => { skills: [ { name: "multi-issue", - path: "plugin/skills/multi-issue/SKILL.md", + path: "plugins/azure-skills/skills/multi-issue/SKILL.md", status: "fail", errors: ["[name-format] bad name", "[no-xml-tags] has XML"], warnings: ["[description-length] too long"], @@ -140,7 +140,7 @@ describe("parseFrontmatterJson", () => { skills: [ { name: "counted", - path: "plugin/skills/counted/SKILL.md", + path: "plugins/azure-skills/skills/counted/SKILL.md", status: "fail", errors: ["err1", "err2"], warnings: ["warn1"], @@ -169,7 +169,7 @@ describe("parseFrontmatterJson", () => { skills: [ { name: "verbatim", - path: "plugin/skills/verbatim/SKILL.md", + path: "plugins/azure-skills/skills/verbatim/SKILL.md", status: "pass", errors: [], warnings: [], @@ -244,7 +244,7 @@ describe("frontmatterCollector.collect", () => { skills: [ { name: "fail-skill", - path: "plugin/skills/fail-skill/SKILL.md", + path: "plugins/azure-skills/skills/fail-skill/SKILL.md", status: "fail", errors: ["problem"], warnings: [], @@ -323,9 +323,9 @@ describe("frontmatterCollector.collect", () => { const { join } = await import("node:path"); const root = mkdtempSync(join(tmpdir(), "fm-filecount-")); - // Built skill: output/skills/skill-a with 2 own files + a nested sub-skill + // Built skill: output/{plugin}/skills/skill-a with 2 own files + a nested sub-skill // that contributes 2 more, so the recursive count is 4. - const skillDir = join(root, "output", "skills", "skill-a"); + const skillDir = join(root, "output", "azure-skills", "skills", "skill-a"); const nestedDir = join(skillDir, "nested"); mkdirSync(nestedDir, { recursive: true }); writeFileSync(join(skillDir, "SKILL.md"), "a"); @@ -337,7 +337,7 @@ describe("frontmatterCollector.collect", () => { skills: [ { name: "skill-a", - path: "output/skills/skill-a/SKILL.md", + path: "output/azure-skills/skills/skill-a/SKILL.md", status: "pass", errors: [], warnings: [], @@ -368,7 +368,7 @@ describe("frontmatterCollector.collect", () => { skills: [ { name: "gone", - path: "output/skills/gone/SKILL.md", + path: "output/azure-skills/skills/gone/SKILL.md", status: "pass", errors: [], warnings: [], diff --git a/scripts/src/dashboard/__tests__/collectors/references.test.ts b/scripts/src/dashboard/__tests__/collectors/references.test.ts index 58caeffd9..d7c572445 100644 --- a/scripts/src/dashboard/__tests__/collectors/references.test.ts +++ b/scripts/src/dashboard/__tests__/collectors/references.test.ts @@ -47,7 +47,7 @@ describe("parseReferencesJson", () => { const raw = makeReferencesJson({ references: [ { - source: "plugin/skills/foo/SKILL.md", + source: "plugins/azure-skills/skills/foo/SKILL.md", target: "./missing.md", status: "broken", message: "Target does not exist: ./missing.md", @@ -61,7 +61,7 @@ describe("parseReferencesJson", () => { expect(report.status).toBe("fail"); expect(report.summary.failed).toBe(1); expect(report.items).toHaveLength(1); - expect(report.items[0].name).toBe("plugin/skills/foo/SKILL.md"); + expect(report.items[0].name).toBe("plugins/azure-skills/skills/foo/SKILL.md"); expect(report.items[0].status).toBe("fail"); expect(report.items[0].message).toContain("missing.md"); }); @@ -70,7 +70,7 @@ describe("parseReferencesJson", () => { const raw = makeReferencesJson({ references: [ { - source: "plugin/skills/bar/references/orphan.md", + source: "plugins/azure-skills/skills/bar/references/orphan.md", target: "bar/SKILL.md", status: "warning", message: "File exists in references directory but is not linked", diff --git a/scripts/src/shared/skill-helper.ts b/scripts/src/shared/skill-helper.ts index f4855ac69..be4135bc5 100644 --- a/scripts/src/shared/skill-helper.ts +++ b/scripts/src/shared/skill-helper.ts @@ -2,7 +2,7 @@ * Skill Utility * * Shared helpers for loading, listing, and parsing SKILL.md files from the - * plugin/skills directory. All frontmatter parsing goes through + * plugins//skills/ directory. All frontmatter parsing goes through * `parseSkillContent` which normalises line endings, validates `---` * delimiters, and exposes the raw YAML source. */ @@ -144,7 +144,7 @@ const pluginDirnameMap = new Map([ /** * Load a skill by name. * - * Reads the SKILL.md file from `plugin/skills/` and parses it + * Reads the SKILL.md file from `plugins//skills/` and parses it * via `parseSkillContent`. Throws when the file is missing or contains * no valid frontmatter. */ diff --git a/scripts/src/tokens/__tests__/types.test.ts b/scripts/src/tokens/__tests__/types.test.ts index 1b0000b85..403d79158 100644 --- a/scripts/src/tokens/__tests__/types.test.ts +++ b/scripts/src/tokens/__tests__/types.test.ts @@ -169,6 +169,6 @@ describe("matchesPattern", () => { it("handles paths without leading slash", () => { expect(matchesPattern("SKILL.md", "SKILL.md")).toBe(true); - expect(matchesPattern("plugin/skills/SKILL.md", "SKILL.md")).toBe(true); + expect(matchesPattern("plugins/azure-skills/skills/SKILL.md", "SKILL.md")).toBe(true); }); }); diff --git a/scripts/src/tokens/__tests__/utils.test.ts b/scripts/src/tokens/__tests__/utils.test.ts index fc3494462..d3d19c18a 100644 --- a/scripts/src/tokens/__tests__/utils.test.ts +++ b/scripts/src/tokens/__tests__/utils.test.ts @@ -76,18 +76,18 @@ describe("getLimitForFile", () => { "references/**/*.md": 1500 }, overrides: { - "plugin/skills/special/SKILL.md": 5000 + "plugins/azure-skills/skills/special/SKILL.md": 5000 } }; it("returns override limit for exact match", () => { - const result = getLimitForFile("plugin/skills/special/SKILL.md", mockConfig, "/root"); + const result = getLimitForFile("plugins/azure-skills/skills/special/SKILL.md", mockConfig, "/root"); expect(result.limit).toBe(5000); - expect(result.pattern).toBe("plugin/skills/special/SKILL.md"); + expect(result.pattern).toBe("plugins/azure-skills/skills/special/SKILL.md"); }); it("returns specific pattern limit for SKILL.md files", () => { - const result = getLimitForFile("plugin/skills/my-skill/SKILL.md", mockConfig, "/root"); + const result = getLimitForFile("plugins/azure-skills/skills/my-skill/SKILL.md", mockConfig, "/root"); expect(result.limit).toBe(3500); expect(result.pattern).toBe("SKILL.md"); }); @@ -119,7 +119,7 @@ describe("getLimitForFile", () => { overrides: {} }; - const result = getLimitForFile("plugin/skills/my-skill/SKILL.md", configWithMultiplePatterns, "/root"); + const result = getLimitForFile("plugins/azure-skills/skills/my-skill/SKILL.md", configWithMultiplePatterns, "/root"); // SKILL.md exact filename match wins over globstar pattern due to specificity scoring // (no wildcards = +10000 points) expect(result.limit).toBe(3500); diff --git a/tests/AGENTS.md b/tests/AGENTS.md index 8da7b6854..242a02fb0 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -11,7 +11,7 @@ When a user asks to scaffold, create, or add tests for a skill, follow these ste Skills in azure-skills plugin use [vally](https://microsoft.github.io/vally/get-started/) to run integration tests that run prompts against an LLM Agent and evaluate the outcome. The vally-eval skill provides the knowledge on where to add the test code, how to run the tests and where to collect the test results. Combine the instructions in vally-eval skill, the official documentation of vally and the rest of the instructions in this file to learn how to write vally eval suites for azure-skills plugin. ### Step 2: Read the skill's SKILL.md -Load the file at `plugin/skills/{skill-name}/SKILL.md` to understand: +Load the file at `plugins/{plugin}/skills/{skill-name}/SKILL.md` to understand: - The skill's name and description (from frontmatter) - What the skill does (from content) - What Azure services/tools it references diff --git a/tests/jest.config.ts b/tests/jest.config.ts index 593900466..bc3772dee 100644 --- a/tests/jest.config.ts +++ b/tests/jest.config.ts @@ -37,8 +37,8 @@ const config: Config = { // Coverage configuration collectCoverageFrom: [ - "../output/skills/**/*.js", - "../output/skills/**/*.ts", + "../output/*/skills/**/*.js", + "../output/*/skills/**/*.ts", "!**/node_modules/**", "!**/_template/**" ], diff --git a/tests/utils/skill-loader.ts b/tests/utils/skill-loader.ts index 680bd3d4d..930bf0112 100644 --- a/tests/utils/skill-loader.ts +++ b/tests/utils/skill-loader.ts @@ -1,7 +1,7 @@ /** * Skill Loader Utility * - * Loads and parses SKILL.md files from the output/skills directory. + * Loads and parses SKILL.md files from the output/{plugin}/skills directory. * Extracts frontmatter metadata and content. */