From 11b4739e26bb6da7355659e8edf3824825618b4b Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Wed, 6 May 2026 15:08:09 -0700 Subject: [PATCH 1/4] Comparison deploy tests --- .github/workflows/pr.yml | 13 +++++++++++-- .github/workflows/test-all-integration.yml | 18 ++++++++++++++---- .github/workflows/test-azure-deploy.yml | 6 ++++++ tests/skills.json | 5 +++-- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 766832091..1e0237f8c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -268,9 +268,18 @@ jobs: hasError = true; } - // 2. Integration test schedule coverages all skills + // 2. Integration test schedule coverages all skills. + // Only consider the cron schedules referenced by .github/workflows/test-all-integration.yml. + // Other entries in integrationTestSchedule (e.g. Sunday comparison runs) don't count + // toward the coverage check. + const requiredCronSchedules = [ + '0 5 * * 2-6', + '0 8 * * 2-6', + '0 12 * * 2-6', + ]; let scheduledSkills = []; - Object.values(skillsJson.integrationTestSchedule).forEach(value => { + requiredCronSchedules.forEach(cron => { + const value = skillsJson.integrationTestSchedule[cron]; if (value) { value.split(',').forEach(s => scheduledSkills.push(s)); } diff --git a/.github/workflows/test-all-integration.yml b/.github/workflows/test-all-integration.yml index 989da5466..742c9aa60 100644 --- a/.github/workflows/test-all-integration.yml +++ b/.github/workflows/test-all-integration.yml @@ -18,6 +18,9 @@ on: - cron: '0 8 * * 2-6' # 4:00 AM PST Tue-Sat (12:00 PM UTC Tue-Sat) - cron: '0 12 * * 2-6' + # 4:00 AM PST Sun (12:00 PM UTC Sun) + # Comparison test runs that don't use skills + - cron: '0 12 * * 7' workflow_dispatch: inputs: skills: @@ -51,10 +54,10 @@ on: # Customize the inputs for each scheduled run by modifying these env vars. # Skills for each schedule are defined in tests/skills.json under integrationTestSchedule. env: - SCHEDULED_DEBUG_21: 'false' # debug flag for 9:00 PM PST run - SCHEDULED_DEBUG_00: 'false' # debug flag for 12:00 AM PST run - SCHEDULED_DEBUG_04: 'false' # debug flag for 4:00 AM PST run - + SCHEDULED_DEBUG_21: 'false' # debug flag for 9:00 PM PST run (Weekday) + SCHEDULED_DEBUG_00: 'false' # debug flag for 12:00 AM PST run (Weekday) + SCHEDULED_DEBUG_04: 'false' # debug flag for 4:00 AM PST run (Weekday) + SCHEDULED_DEBUG_SUN: 'false' # debug flag for 4:00 AM PST run (Sunday) jobs: resolve-inputs: name: Resolve workflow inputs @@ -64,6 +67,7 @@ jobs: skills: ${{ steps.resolve.outputs.skills }} deploy-test-pattern: ${{ steps.resolve.outputs.deploy-test-pattern }} debug: ${{ steps.resolve.outputs.debug }} + no-skills: ${{ steps.resolve.outputs.no-skills }} steps: - name: Checkout repository if: github.event_name == 'schedule' @@ -113,6 +117,10 @@ jobs: "0 12 * * 2-6") echo "debug=$SCHEDULED_DEBUG_04" >> "$GITHUB_OUTPUT" ;; + "0 12 * * 7") + echo "debug=$SCHEDULED_DEBUG_SUN" >> "$GITHUB_OUTPUT" + echo "no-skills=true" >> "$GITHUB_OUTPUT" + ;; *) echo "Unknown schedule: $CRON" exit 1 @@ -168,6 +176,7 @@ jobs: test-pattern: ${{ needs.resolve-inputs.outputs.deploy-test-pattern }} debug: ${{ needs.resolve-inputs.outputs.debug == 'true' }} publish-reports: ${{ github.event_name == 'schedule' }} + no-skills: ${{ needs.resolve-inputs.outputs.no-skills == 'true' }} secrets: COPILOT_CLI_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} @@ -282,6 +291,7 @@ jobs: MODEL_OVERRIDE: ${{ inputs.model-override }} SKILL_TEST_PATTERN: ${{ inputs.skill-test-pattern }} SKILL: ${{ matrix.skill }} + NO_SKILLS: ${{ needs.resolve-inputs.outputs.no-skills == 'true' && 'true' || '' }} run: | echo test with $MODEL_OVERRIDE # Handle azure-ai vs azure-aigateway prefix collision diff --git a/.github/workflows/test-azure-deploy.yml b/.github/workflows/test-azure-deploy.yml index 6c2be5bec..5ae5a83b6 100644 --- a/.github/workflows/test-azure-deploy.yml +++ b/.github/workflows/test-azure-deploy.yml @@ -50,6 +50,11 @@ on: required: false type: boolean default: false + no-skills: + description: 'Whether to set NO_SKILLS=true for jest tests (skip loading skills)' + required: false + type: boolean + default: false secrets: COPILOT_CLI_TOKEN: required: true @@ -189,6 +194,7 @@ jobs: DEBUG: ${{ inputs.debug && '1' || '' }} MODEL_OVERRIDE: ${{ inputs.model-override }} TEST_GROUP: ${{ matrix.test-group }} + NO_SKILLS: ${{ inputs.no-skills && 'true' || '' }} run: | echo test with $MODEL_OVERRIDE npm run test:integration -- azure-deploy "$TEST_GROUP" diff --git a/tests/skills.json b/tests/skills.json index 8a1bb42bb..d507e0939 100644 --- a/tests/skills.json +++ b/tests/skills.json @@ -30,6 +30,7 @@ "integrationTestSchedule": { "0 5 * * 2-6": "microsoft-foundry", "0 8 * * 2-6": "azure-deploy", - "0 12 * * 2-6": "airunway-aks-setup,appinsights-instrumentation,azure-ai,azure-aigateway,azure-cloud-migrate,azure-compliance,azure-compute,azure-cost,azure-diagnostics,azure-enterprise-infra-planner,azure-hosted-copilot-sdk,azure-kubernetes,azure-kusto,azure-messaging,azure-prepare,azure-quotas,azure-rbac,azure-resource-lookup,azure-resource-visualizer,azure-storage,azure-upgrade,azure-validate,entra-agent-id,entra-app-registration" + "0 12 * * 2-6": "airunway-aks-setup,appinsights-instrumentation,azure-ai,azure-aigateway,azure-cloud-migrate,azure-compliance,azure-compute,azure-cost,azure-diagnostics,azure-enterprise-infra-planner,azure-hosted-copilot-sdk,azure-kubernetes,azure-kusto,azure-messaging,azure-prepare,azure-quotas,azure-rbac,azure-resource-lookup,azure-resource-visualizer,azure-storage,azure-upgrade,azure-validate,entra-agent-id,entra-app-registration", + "0 12 * * 7": "azure-deploy" } -} +} \ No newline at end of file From b7d326d7cfa33adb54aefa873e93148058fb6b09 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Wed, 6 May 2026 15:25:05 -0700 Subject: [PATCH 2/4] Skip steps that no longer make sense without skills --- .github/workflows/test-all-integration.yml | 4 ++-- tests/azure-deploy/integration.test.ts | 3 ++- tests/utils/evaluate.ts | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-all-integration.yml b/.github/workflows/test-all-integration.yml index 742c9aa60..caf19d8da 100644 --- a/.github/workflows/test-all-integration.yml +++ b/.github/workflows/test-all-integration.yml @@ -57,7 +57,7 @@ env: SCHEDULED_DEBUG_21: 'false' # debug flag for 9:00 PM PST run (Weekday) SCHEDULED_DEBUG_00: 'false' # debug flag for 12:00 AM PST run (Weekday) SCHEDULED_DEBUG_04: 'false' # debug flag for 4:00 AM PST run (Weekday) - SCHEDULED_DEBUG_SUN: 'false' # debug flag for 4:00 AM PST run (Sunday) + SCHEDULED_DEBUG_NO_SKILLS: 'false' # debug flag for 4:00 AM PST run (Sunday) jobs: resolve-inputs: name: Resolve workflow inputs @@ -118,7 +118,7 @@ jobs: echo "debug=$SCHEDULED_DEBUG_04" >> "$GITHUB_OUTPUT" ;; "0 12 * * 7") - echo "debug=$SCHEDULED_DEBUG_SUN" >> "$GITHUB_OUTPUT" + echo "debug=$SCHEDULED_DEBUG_NO_SKILLS" >> "$GITHUB_OUTPUT" echo "no-skills=true" >> "$GITHUB_OUTPUT" ;; *) diff --git a/tests/azure-deploy/integration.test.ts b/tests/azure-deploy/integration.test.ts index 6fab7b2e7..a394958cc 100644 --- a/tests/azure-deploy/integration.test.ts +++ b/tests/azure-deploy/integration.test.ts @@ -43,7 +43,8 @@ const pseudoRandomResourceGroupNameSystemPromptModifier = { describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => { const agent = useAgentRunner(); - describe("skill-invocation", () => { + const describeSkillInvocation = process.env.NO_SKILLS === "true" ? describe.skip : describe; + describeSkillInvocation("skill-invocation", () => { const followUp = ["Continue with recommended options until complete."]; test("invokes azure-deploy skill for deployment prompt", async () => { await withTestResult(async ({ setSkillInvocationRate }) => { diff --git a/tests/utils/evaluate.ts b/tests/utils/evaluate.ts index 6cc22c65b..a4f90569e 100644 --- a/tests/utils/evaluate.ts +++ b/tests/utils/evaluate.ts @@ -300,6 +300,9 @@ export function isToolCalled(metadata: AgentMetadata, toolName: string, argument } export function softCheckSkill(agentMetadata: AgentMetadata, skillName: string): void { + if (process.env.NO_SKILLS === "true") { + return; + } const isSkillUsed = isSkillInvoked(agentMetadata, skillName); if (!isSkillUsed) { From da48f231c0f868b8e236a06eb0344731cf9dfe70 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Wed, 6 May 2026 16:03:50 -0700 Subject: [PATCH 3/4] PR feedback --- .github/workflows/pr.yml | 2 +- .github/workflows/test-all-integration.yml | 4 ++-- tests/skills.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 1e0237f8c..de98616e5 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -268,7 +268,7 @@ jobs: hasError = true; } - // 2. Integration test schedule coverages all skills. + // 2. Integration test schedule covers all skills. // Only consider the cron schedules referenced by .github/workflows/test-all-integration.yml. // Other entries in integrationTestSchedule (e.g. Sunday comparison runs) don't count // toward the coverage check. diff --git a/.github/workflows/test-all-integration.yml b/.github/workflows/test-all-integration.yml index caf19d8da..ac02d8c32 100644 --- a/.github/workflows/test-all-integration.yml +++ b/.github/workflows/test-all-integration.yml @@ -20,7 +20,7 @@ on: - cron: '0 12 * * 2-6' # 4:00 AM PST Sun (12:00 PM UTC Sun) # Comparison test runs that don't use skills - - cron: '0 12 * * 7' + - cron: '0 12 * * 0' workflow_dispatch: inputs: skills: @@ -117,7 +117,7 @@ jobs: "0 12 * * 2-6") echo "debug=$SCHEDULED_DEBUG_04" >> "$GITHUB_OUTPUT" ;; - "0 12 * * 7") + "0 12 * * 0") echo "debug=$SCHEDULED_DEBUG_NO_SKILLS" >> "$GITHUB_OUTPUT" echo "no-skills=true" >> "$GITHUB_OUTPUT" ;; diff --git a/tests/skills.json b/tests/skills.json index d507e0939..541da04bb 100644 --- a/tests/skills.json +++ b/tests/skills.json @@ -31,6 +31,6 @@ "0 5 * * 2-6": "microsoft-foundry", "0 8 * * 2-6": "azure-deploy", "0 12 * * 2-6": "airunway-aks-setup,appinsights-instrumentation,azure-ai,azure-aigateway,azure-cloud-migrate,azure-compliance,azure-compute,azure-cost,azure-diagnostics,azure-enterprise-infra-planner,azure-hosted-copilot-sdk,azure-kubernetes,azure-kusto,azure-messaging,azure-prepare,azure-quotas,azure-rbac,azure-resource-lookup,azure-resource-visualizer,azure-storage,azure-upgrade,azure-validate,entra-agent-id,entra-app-registration", - "0 12 * * 7": "azure-deploy" + "0 12 * * 0": "azure-deploy" } } \ No newline at end of file From dafea7a358968d5148c1499d00505ed9efa40897 Mon Sep 17 00:00:00 2001 From: Chunan Ye Date: Thu, 7 May 2026 14:34:28 -0700 Subject: [PATCH 4/4] Extract the comparison test into a new dedicated workflow --- .github/workflows/test-all-integration.yml | 18 ++----- .../workflows/test-comparison-no-skills.yml | 51 +++++++++++++++++++ tests/skills.json | 3 +- 3 files changed, 56 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/test-comparison-no-skills.yml diff --git a/.github/workflows/test-all-integration.yml b/.github/workflows/test-all-integration.yml index ac02d8c32..989da5466 100644 --- a/.github/workflows/test-all-integration.yml +++ b/.github/workflows/test-all-integration.yml @@ -18,9 +18,6 @@ on: - cron: '0 8 * * 2-6' # 4:00 AM PST Tue-Sat (12:00 PM UTC Tue-Sat) - cron: '0 12 * * 2-6' - # 4:00 AM PST Sun (12:00 PM UTC Sun) - # Comparison test runs that don't use skills - - cron: '0 12 * * 0' workflow_dispatch: inputs: skills: @@ -54,10 +51,10 @@ on: # Customize the inputs for each scheduled run by modifying these env vars. # Skills for each schedule are defined in tests/skills.json under integrationTestSchedule. env: - SCHEDULED_DEBUG_21: 'false' # debug flag for 9:00 PM PST run (Weekday) - SCHEDULED_DEBUG_00: 'false' # debug flag for 12:00 AM PST run (Weekday) - SCHEDULED_DEBUG_04: 'false' # debug flag for 4:00 AM PST run (Weekday) - SCHEDULED_DEBUG_NO_SKILLS: 'false' # debug flag for 4:00 AM PST run (Sunday) + SCHEDULED_DEBUG_21: 'false' # debug flag for 9:00 PM PST run + SCHEDULED_DEBUG_00: 'false' # debug flag for 12:00 AM PST run + SCHEDULED_DEBUG_04: 'false' # debug flag for 4:00 AM PST run + jobs: resolve-inputs: name: Resolve workflow inputs @@ -67,7 +64,6 @@ jobs: skills: ${{ steps.resolve.outputs.skills }} deploy-test-pattern: ${{ steps.resolve.outputs.deploy-test-pattern }} debug: ${{ steps.resolve.outputs.debug }} - no-skills: ${{ steps.resolve.outputs.no-skills }} steps: - name: Checkout repository if: github.event_name == 'schedule' @@ -117,10 +113,6 @@ jobs: "0 12 * * 2-6") echo "debug=$SCHEDULED_DEBUG_04" >> "$GITHUB_OUTPUT" ;; - "0 12 * * 0") - echo "debug=$SCHEDULED_DEBUG_NO_SKILLS" >> "$GITHUB_OUTPUT" - echo "no-skills=true" >> "$GITHUB_OUTPUT" - ;; *) echo "Unknown schedule: $CRON" exit 1 @@ -176,7 +168,6 @@ jobs: test-pattern: ${{ needs.resolve-inputs.outputs.deploy-test-pattern }} debug: ${{ needs.resolve-inputs.outputs.debug == 'true' }} publish-reports: ${{ github.event_name == 'schedule' }} - no-skills: ${{ needs.resolve-inputs.outputs.no-skills == 'true' }} secrets: COPILOT_CLI_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} @@ -291,7 +282,6 @@ jobs: MODEL_OVERRIDE: ${{ inputs.model-override }} SKILL_TEST_PATTERN: ${{ inputs.skill-test-pattern }} SKILL: ${{ matrix.skill }} - NO_SKILLS: ${{ needs.resolve-inputs.outputs.no-skills == 'true' && 'true' || '' }} run: | echo test with $MODEL_OVERRIDE # Handle azure-ai vs azure-aigateway prefix collision diff --git a/.github/workflows/test-comparison-no-skills.yml b/.github/workflows/test-comparison-no-skills.yml new file mode 100644 index 000000000..7dc17ae80 --- /dev/null +++ b/.github/workflows/test-comparison-no-skills.yml @@ -0,0 +1,51 @@ +# Comparison integration tests (no skills). +# +# Runs azure-deploy deployment tests with NO_SKILLS=true on Sundays as a +# baseline / comparison run against the regular weekday integration tests. +# This workflow has its own name so downstream automation (e.g. the +# "Analyze Test Run" agentic workflow that listens to "Integration Tests - all") +# does not get triggered by the Sunday baseline runs. + +name: Integration Tests - comparison (no skills) + +permissions: + id-token: write + contents: read + +on: + schedule: + # 4:00 AM PST Sun (12:00 PM UTC Sun) — comparison runs that don't use skills + - cron: "0 12 * * 0" + workflow_dispatch: + inputs: + model-override: + description: "Model to use for testing" + required: false + type: choice + options: + - claude-sonnet-4.5 + - claude-opus-4.5 + test-pattern: + description: 'Optional: Comma separated patterns by name or describe block. If empty, all tests will be run.' + required: false + type: string + default: "" + debug: + description: "Whether to set DEBUG=1 for jest tests" + required: false + type: boolean + default: false + +jobs: + azure-deploy: + name: Integration – azure-deploy (no skills) + if: github.repository == 'microsoft/GitHub-Copilot-for-Azure' + uses: ./.github/workflows/test-azure-deploy.yml + with: + model-override: ${{ inputs.model-override }} + test-pattern: ${{ inputs.test-pattern }} + debug: ${{ inputs.debug || false }} + publish-reports: ${{ github.event_name == 'schedule' }} + no-skills: true + secrets: + COPILOT_CLI_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} diff --git a/tests/skills.json b/tests/skills.json index 541da04bb..7fbcd6aa7 100644 --- a/tests/skills.json +++ b/tests/skills.json @@ -30,7 +30,6 @@ "integrationTestSchedule": { "0 5 * * 2-6": "microsoft-foundry", "0 8 * * 2-6": "azure-deploy", - "0 12 * * 2-6": "airunway-aks-setup,appinsights-instrumentation,azure-ai,azure-aigateway,azure-cloud-migrate,azure-compliance,azure-compute,azure-cost,azure-diagnostics,azure-enterprise-infra-planner,azure-hosted-copilot-sdk,azure-kubernetes,azure-kusto,azure-messaging,azure-prepare,azure-quotas,azure-rbac,azure-resource-lookup,azure-resource-visualizer,azure-storage,azure-upgrade,azure-validate,entra-agent-id,entra-app-registration", - "0 12 * * 0": "azure-deploy" + "0 12 * * 2-6": "airunway-aks-setup,appinsights-instrumentation,azure-ai,azure-aigateway,azure-cloud-migrate,azure-compliance,azure-compute,azure-cost,azure-diagnostics,azure-enterprise-infra-planner,azure-hosted-copilot-sdk,azure-kubernetes,azure-kusto,azure-messaging,azure-prepare,azure-quotas,azure-rbac,azure-resource-lookup,azure-resource-visualizer,azure-storage,azure-upgrade,azure-validate,entra-agent-id,entra-app-registration" } } \ No newline at end of file