Run comparison integration tests without skills

.github/workflows/pr.yml

@@ -268,9 +268,18 @@ jobs:
                 hasError = true;
             }
 
-            // 2. Integration test schedule coverages all skills
+            // 2. Integration test schedule covers all skills.
+            // Only consider the cron schedules referenced by .github/workflows/test-all-integration.yml.
+            // Other entries in integrationTestSchedule (e.g. Sunday comparison runs) don't count
+            // toward the coverage check.
+            const requiredCronSchedules = [
+                '0 5 * * 2-6',
+                '0 8 * * 2-6',
+                '0 12 * * 2-6',
+            ];
             let scheduledSkills = [];
-            Object.values(skillsJson.integrationTestSchedule).forEach(value => {
+            requiredCronSchedules.forEach(cron => {
+                const value = skillsJson.integrationTestSchedule[cron];
                 if (value) {
                     value.split(',').forEach(s => scheduledSkills.push(s));
                 }

.github/workflows/test-azure-deploy.yml

@@ -50,6 +50,11 @@ on:
         required: false
         type: boolean
         default: false
+      no-skills:
+        description: 'Whether to set NO_SKILLS=true for jest tests (skip loading skills)'
+        required: false
+        type: boolean
+        default: false
     secrets:
       COPILOT_CLI_TOKEN:
         required: true
@@ -189,6 +194,7 @@ jobs:
           DEBUG: ${{ inputs.debug && '1' || '' }}
           MODEL_OVERRIDE: ${{ inputs.model-override }}
           TEST_GROUP: ${{ matrix.test-group }}
+          NO_SKILLS: ${{ inputs.no-skills && 'true' || '' }}
         run: |
           echo test with $MODEL_OVERRIDE
           npm run test:integration -- azure-deploy "$TEST_GROUP"

.github/workflows/test-comparison-no-skills.yml

@@ -0,0 +1,51 @@
+# Comparison integration tests (no skills).
+#
+# Runs azure-deploy deployment tests with NO_SKILLS=true on Sundays as a
+# baseline / comparison run against the regular weekday integration tests.
+# This workflow has its own name so downstream automation (e.g. the
+# "Analyze Test Run" agentic workflow that listens to "Integration Tests - all")
+# does not get triggered by the Sunday baseline runs.
+
+name: Integration Tests - comparison (no skills)
+
+permissions:
+  id-token: write
+  contents: read
+
+on:
+  schedule:
+    # 4:00 AM PST Sun (12:00 PM UTC Sun) — comparison runs that don't use skills
+    - cron: "0 12 * * 0"
+  workflow_dispatch:
+    inputs:
+      model-override:
+        description: "Model to use for testing"
+        required: false
+        type: choice
+        options:
+          - claude-sonnet-4.5
+          - claude-opus-4.5
+      test-pattern:
+        description: 'Optional: Comma separated patterns by name or describe block. If empty, all tests will be run.'
+        required: false
+        type: string
+        default: ""
+      debug:
+        description: "Whether to set DEBUG=1 for jest tests"
+        required: false
+        type: boolean
+        default: false
+
+jobs:
+  azure-deploy:
+    name: Integration – azure-deploy (no skills)
+    if: github.repository == 'microsoft/GitHub-Copilot-for-Azure'
+    uses: ./.github/workflows/test-azure-deploy.yml
+    with:
+      model-override: ${{ inputs.model-override }}
+      test-pattern: ${{ inputs.test-pattern }}
+      debug: ${{ inputs.debug || false }}
+      publish-reports: ${{ github.event_name == 'schedule' }}
+      no-skills: true
+    secrets:
+      COPILOT_CLI_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }}

tests/azure-deploy/integration.test.ts

@@ -43,7 +43,8 @@ const pseudoRandomResourceGroupNameSystemPromptModifier = {
 
 describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => {
   const agent = useAgentRunner();
-  describe("skill-invocation", () => {
+  const describeSkillInvocation = process.env.NO_SKILLS === "true" ? describe.skip : describe;
+  describeSkillInvocation("skill-invocation", () => {
     const followUp = ["Continue with recommended options until complete."];
     test("invokes azure-deploy skill for deployment prompt", async () => {
       await withTestResult(async ({ setSkillInvocationRate }) => {

tests/skills.json

@@ -32,4 +32,4 @@
         "0 8 * * 2-6": "azure-deploy",
         "0 12 * * 2-6": "airunway-aks-setup,appinsights-instrumentation,azure-ai,azure-aigateway,azure-cloud-migrate,azure-compliance,azure-compute,azure-cost,azure-diagnostics,azure-enterprise-infra-planner,azure-hosted-copilot-sdk,azure-kubernetes,azure-kusto,azure-messaging,azure-prepare,azure-quotas,azure-rbac,azure-resource-lookup,azure-resource-visualizer,azure-storage,azure-upgrade,azure-validate,entra-agent-id,entra-app-registration"
     }
-}
+}

tests/utils/evaluate.ts

@@ -300,6 +300,9 @@ export function isToolCalled(metadata: AgentMetadata, toolName: string, argument
 }
 
 export function softCheckSkill(agentMetadata: AgentMetadata, skillName: string): void {
+  if (process.env.NO_SKILLS === "true") {
+    return;
+  }
   const isSkillUsed = isSkillInvoked(agentMetadata, skillName);
 
   if (!isSkillUsed) {