diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 766832091..de98616e5 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -268,9 +268,18 @@ jobs: hasError = true; } - // 2. Integration test schedule coverages all skills + // 2. Integration test schedule covers all skills. + // Only consider the cron schedules referenced by .github/workflows/test-all-integration.yml. + // Other entries in integrationTestSchedule (e.g. Sunday comparison runs) don't count + // toward the coverage check. + const requiredCronSchedules = [ + '0 5 * * 2-6', + '0 8 * * 2-6', + '0 12 * * 2-6', + ]; let scheduledSkills = []; - Object.values(skillsJson.integrationTestSchedule).forEach(value => { + requiredCronSchedules.forEach(cron => { + const value = skillsJson.integrationTestSchedule[cron]; if (value) { value.split(',').forEach(s => scheduledSkills.push(s)); } diff --git a/.github/workflows/test-azure-deploy.yml b/.github/workflows/test-azure-deploy.yml index 6c2be5bec..5ae5a83b6 100644 --- a/.github/workflows/test-azure-deploy.yml +++ b/.github/workflows/test-azure-deploy.yml @@ -50,6 +50,11 @@ on: required: false type: boolean default: false + no-skills: + description: 'Whether to set NO_SKILLS=true for jest tests (skip loading skills)' + required: false + type: boolean + default: false secrets: COPILOT_CLI_TOKEN: required: true @@ -189,6 +194,7 @@ jobs: DEBUG: ${{ inputs.debug && '1' || '' }} MODEL_OVERRIDE: ${{ inputs.model-override }} TEST_GROUP: ${{ matrix.test-group }} + NO_SKILLS: ${{ inputs.no-skills && 'true' || '' }} run: | echo test with $MODEL_OVERRIDE npm run test:integration -- azure-deploy "$TEST_GROUP" diff --git a/.github/workflows/test-comparison-no-skills.yml b/.github/workflows/test-comparison-no-skills.yml new file mode 100644 index 000000000..7dc17ae80 --- /dev/null +++ b/.github/workflows/test-comparison-no-skills.yml @@ -0,0 +1,51 @@ +# Comparison integration tests (no skills). +# +# Runs azure-deploy deployment tests with NO_SKILLS=true on Sundays as a +# baseline / comparison run against the regular weekday integration tests. +# This workflow has its own name so downstream automation (e.g. the +# "Analyze Test Run" agentic workflow that listens to "Integration Tests - all") +# does not get triggered by the Sunday baseline runs. + +name: Integration Tests - comparison (no skills) + +permissions: + id-token: write + contents: read + +on: + schedule: + # 4:00 AM PST Sun (12:00 PM UTC Sun) — comparison runs that don't use skills + - cron: "0 12 * * 0" + workflow_dispatch: + inputs: + model-override: + description: "Model to use for testing" + required: false + type: choice + options: + - claude-sonnet-4.5 + - claude-opus-4.5 + test-pattern: + description: 'Optional: Comma separated patterns by name or describe block. If empty, all tests will be run.' + required: false + type: string + default: "" + debug: + description: "Whether to set DEBUG=1 for jest tests" + required: false + type: boolean + default: false + +jobs: + azure-deploy: + name: Integration – azure-deploy (no skills) + if: github.repository == 'microsoft/GitHub-Copilot-for-Azure' + uses: ./.github/workflows/test-azure-deploy.yml + with: + model-override: ${{ inputs.model-override }} + test-pattern: ${{ inputs.test-pattern }} + debug: ${{ inputs.debug || false }} + publish-reports: ${{ github.event_name == 'schedule' }} + no-skills: true + secrets: + COPILOT_CLI_TOKEN: ${{ secrets.COPILOT_CLI_TOKEN }} diff --git a/tests/azure-deploy/integration.test.ts b/tests/azure-deploy/integration.test.ts index 6fab7b2e7..a394958cc 100644 --- a/tests/azure-deploy/integration.test.ts +++ b/tests/azure-deploy/integration.test.ts @@ -43,7 +43,8 @@ const pseudoRandomResourceGroupNameSystemPromptModifier = { describeIntegration(`${SKILL_NAME}_ - Integration Tests`, () => { const agent = useAgentRunner(); - describe("skill-invocation", () => { + const describeSkillInvocation = process.env.NO_SKILLS === "true" ? describe.skip : describe; + describeSkillInvocation("skill-invocation", () => { const followUp = ["Continue with recommended options until complete."]; test("invokes azure-deploy skill for deployment prompt", async () => { await withTestResult(async ({ setSkillInvocationRate }) => { diff --git a/tests/skills.json b/tests/skills.json index 8a1bb42bb..7fbcd6aa7 100644 --- a/tests/skills.json +++ b/tests/skills.json @@ -32,4 +32,4 @@ "0 8 * * 2-6": "azure-deploy", "0 12 * * 2-6": "airunway-aks-setup,appinsights-instrumentation,azure-ai,azure-aigateway,azure-cloud-migrate,azure-compliance,azure-compute,azure-cost,azure-diagnostics,azure-enterprise-infra-planner,azure-hosted-copilot-sdk,azure-kubernetes,azure-kusto,azure-messaging,azure-prepare,azure-quotas,azure-rbac,azure-resource-lookup,azure-resource-visualizer,azure-storage,azure-upgrade,azure-validate,entra-agent-id,entra-app-registration" } -} +} \ No newline at end of file diff --git a/tests/utils/evaluate.ts b/tests/utils/evaluate.ts index 6cc22c65b..a4f90569e 100644 --- a/tests/utils/evaluate.ts +++ b/tests/utils/evaluate.ts @@ -300,6 +300,9 @@ export function isToolCalled(metadata: AgentMetadata, toolName: string, argument } export function softCheckSkill(agentMetadata: AgentMetadata, skillName: string): void { + if (process.env.NO_SKILLS === "true") { + return; + } const isSkillUsed = isSkillInvoked(agentMetadata, skillName); if (!isSkillUsed) {