diff --git a/.github/workflows/microsoft-foundry-e2e-eval-comment.yml b/.github/workflows/microsoft-foundry-e2e-eval-comment.yml deleted file mode 100644 index a05b88288..000000000 --- a/.github/workflows/microsoft-foundry-e2e-eval-comment.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: Microsoft Foundry E2E Eval Comment Dispatch - -on: - issue_comment: - types: [created] - -permissions: {} - -jobs: - dispatch: - name: Dispatch Microsoft Foundry E2E eval - if: > - github.repository == 'microsoft/GitHub-Copilot-for-Azure' && - github.event.issue.pull_request && - contains(github.event.comment.body, '/foundry-e2e') - runs-on: ubuntu-latest - permissions: - actions: write - contents: read - issues: write - pull-requests: write - - steps: - - name: Dispatch eval workflow - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - with: - script: | - const { data: collaboratorPermission } = - await github.rest.repos.getCollaboratorPermissionLevel({ - owner: context.repo.owner, - repo: context.repo.repo, - username: context.payload.comment.user.login, - }); - - const allowedPermissions = new Set(['admin', 'maintain', 'write']); - const permission = collaboratorPermission.permission; - if (!allowedPermissions.has(permission)) { - throw new Error(`Commenter has ${permission} permission; write, maintain, or admin permission is required.`); - } - - const prNumber = context.payload.issue.number; - const { data: pullRequest } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber, - }); - if (pullRequest.state !== 'open') { - throw new Error(`PR #${prNumber} is ${pullRequest.state}; only open PRs can trigger this workflow.`); - } - - const body = context.payload.comment.body; - const commandPattern = /^\/foundry-e2e(?:\s+e2e-branch=([A-Za-z0-9._/-]+))?$/; - const commandLine = body - .split(/\r?\n/) - .map((line) => line.trim()) - .find((line) => - /^\/foundry-e2e\b/.test(line) - ); - - if (!commandLine) { - throw new Error('No exact Microsoft Foundry E2E command line found.'); - } - - const commandMatch = commandPattern.exec(commandLine); - if (!commandMatch) { - throw new Error( - 'Invalid command. Use "/foundry-e2e" or "/foundry-e2e e2e-branch=".' - ); - } - - const ref = commandMatch[1] || 'main'; - if ( - ref.includes('..') || - ref.startsWith('/') || - ref.endsWith('/') || - ref.endsWith('.') || - ref.includes('@{') || - ref.split('/').some((part) => part.startsWith('.') || part.endsWith('.lock')) - ) { - throw new Error(`Invalid branch/ref: ${ref}`); - } - - await github.rest.repos.getBranch({ - owner: context.repo.owner, - repo: context.repo.repo, - branch: ref, - }); - - await github.rest.actions.createWorkflowDispatch({ - owner: context.repo.owner, - repo: context.repo.repo, - workflow_id: 'microsoft-foundry-e2e-eval.yml', - ref, - inputs: { - target_pr_id: prNumber.toString(), - }, - }); - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.issue.number, - body: `Dispatched Microsoft Foundry E2E eval on \`${ref}\` with \`target_pr_id=${prNumber}\`.`, - }); diff --git a/.github/workflows/microsoft-foundry-e2e-eval.yml b/.github/workflows/microsoft-foundry-e2e-eval.yml deleted file mode 100644 index ba63a61fc..000000000 --- a/.github/workflows/microsoft-foundry-e2e-eval.yml +++ /dev/null @@ -1,265 +0,0 @@ -name: Microsoft Foundry E2E Evaluations - -on: - workflow_dispatch: - inputs: - target_pr_id: - description: "Target PR number (optional)" - required: false - type: string - runs: - description: "Number of Vally trials per stimulus" - required: false - default: 1 - type: number - report-in-pr: - description: "Post report to target PR" - required: false - default: true - type: boolean - -permissions: {} - -jobs: - eval: - name: Run Microsoft Foundry E2E evals - runs-on: windows-latest - permissions: - id-token: write - contents: read - issues: write - pull-requests: write - - steps: - - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - fetch-depth: 0 - - - name: Override Microsoft Foundry skill from target PR head - if: ${{ inputs.target_pr_id != '' }} - shell: bash - env: - TARGET_PR_ID: ${{ inputs.target_pr_id }} - run: | - set -euo pipefail - - if [[ ! "${TARGET_PR_ID}" =~ ^[0-9]+$ ]]; then - echo "::error::target_pr_id must be a pull request number." - exit 1 - fi - - pr_head_ref="refs/remotes/origin/target-pr-${TARGET_PR_ID}-head" - echo "Fetching PR #${TARGET_PR_ID} head from refs/pull/${TARGET_PR_ID}/head." - git fetch --no-tags --depth=1 origin "+refs/pull/${TARGET_PR_ID}/head:${pr_head_ref}" - rm -rf plugin/skills/microsoft-foundry - git restore --source="${pr_head_ref}" --staged --worktree -- plugin/skills/microsoft-foundry - - echo "Using plugin/skills/microsoft-foundry from PR #${TARGET_PR_ID}:" - git log -1 --oneline "${pr_head_ref}" - git status --short plugin/skills/microsoft-foundry - - - name: Set up Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: "22" - cache: "npm" - cache-dependency-path: | - package-lock.json - scripts/package-lock.json - tests/package-lock.json - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.12" - - - name: Install Azure Developer CLI - uses: Azure/setup-azd@634ad924cf8baef2257898ba5663be8d19f15aca # v2 - - - name: Install azd AI agent extension - run: azd extension install azure.ai.agents - - - name: Login to Azure using OIDC - uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0 - with: - client-id: ${{ secrets.FOUNDRY_E2E_EVAL_AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.FOUNDRY_E2E_EVAL_AZURE_TENANT_ID }} - subscription-id: ${{ secrets.FOUNDRY_E2E_EVAL_AZURE_SUBSCRIPTION_ID }} - - # azure/login only fetches the OIDC access token once. - # If the eval runs long enough and the token expires, later Azure CLI or azd calls can fail with authentication errors. - # This refreshes the cached OIDC access token periodically so the long-running Foundry E2E eval has a valid token. - # Learn more at https://github.com/Azure/login/issues/372 - # Refresh interval is sub-5-minutes because the OIDC token's lifetime is - # 5 minutes. Using 240s avoids seeing invalid tokens due to clock skew. - - name: Fetch OIDC token every 240 seconds - shell: bash - env: - AZURE_CLIENT_ID: ${{ secrets.FOUNDRY_E2E_EVAL_AZURE_CLIENT_ID }} - AZURE_TENANT_ID: ${{ secrets.FOUNDRY_E2E_EVAL_AZURE_TENANT_ID }} - run: | - while true; do - token=$(curl -s -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=api://AzureADTokenExchange" | jq .value -r) - az login --service-principal -u "$AZURE_CLIENT_ID" -t "$AZURE_TENANT_ID" --federated-token "$token" --output none - sleep 240 - done & - - - name: Check Azure CLI login - shell: bash - run: | - set -euo pipefail - current_subscription="$(az account show --query id -o tsv)" - test "$current_subscription" = "${{ secrets.FOUNDRY_E2E_EVAL_AZURE_SUBSCRIPTION_ID }}" - az account show --output table - - - name: Configure azd authentication - shell: bash - run: | - set -euo pipefail - azd config set auth.useAzCliAuth "true" - azd config set defaults.subscription "${{ secrets.FOUNDRY_E2E_EVAL_AZURE_SUBSCRIPTION_ID }}" - - - name: Install repository dependencies - run: npm ci - - - name: Install test dependencies - working-directory: tests - run: npm ci - - - name: Build plugin output - run: npm run build - - - name: Remove unrelated skills - shell: bash - run: | - set -euo pipefail - - skills_dir="output/skills" - if [[ ! -d "${skills_dir}/microsoft-foundry" ]]; then - echo "::error::${skills_dir}/microsoft-foundry not found." - exit 1 - fi - - find "${skills_dir}" -mindepth 1 -maxdepth 1 -type d ! -name "microsoft-foundry" -exec rm -rf -- {} + - - - name: Run Microsoft Foundry Vally evals - working-directory: tests - shell: bash - env: - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - VALLY_RUNS: ${{ inputs.runs }} - VALLY_RUNNER_DISABLE_AZURE_MCP: "true" - run: | - set -euo pipefail - - if [[ ! "${VALLY_RUNS}" =~ ^[1-9][0-9]*$ ]]; then - echo "::error::runs must be a positive integer." - exit 1 - fi - - npm run test:vally -- \ - --suite foundry-e2e \ - --runs "${VALLY_RUNS}" \ - --workers 1 \ - --junit \ - --threshold 0.8 - - - name: Upload Vally results - id: upload-vally-results - if: always() - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: microsoft-foundry-e2e-eval-results - path: | - tests/results/ - tests/reports/ - retention-days: 30 - - - name: Install Copilot CLI - if: always() - shell: bash - run: npm install -g @github/copilot - - - name: Generate Microsoft Foundry E2E eval report with Copilot CLI - if: always() - continue-on-error: true - shell: bash - env: - ARTIFACT_URL: ${{ steps.upload-vally-results.outputs.artifact-url }} - COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} - COPILOT_MODEL: 'claude-sonnet-4.6' - COPILOT_PROMPT: tests/prompts/microsoft-foundry-e2e-eval-report.prompt.md - REPORT_MD: ${{ runner.temp }}/microsoft-foundry-e2e-report/report.md - run: copilot -C "${GITHUB_WORKSPACE}" --model "${COPILOT_MODEL}" --disable-builtin-mcps --no-custom-instructions --no-ask-user --allow-all-tools --secret-env-vars=COPILOT_GITHUB_TOKEN --no-color --silent -p "$(cat "${COPILOT_PROMPT}")" - - - name: Publish Microsoft Foundry E2E eval report - id: publish-e2e-report - if: always() - shell: bash - env: - REPORT_MD: ${{ runner.temp }}/microsoft-foundry-e2e-report/report.md - run: | - set -euo pipefail - - if [[ -z "${REPORT_MD}" ]]; then - REPORT_MD="${RUNNER_TEMP}/microsoft-foundry-e2e-report/report.md" - fi - - mkdir -p "$(dirname "${REPORT_MD}")" - - if [[ ! -s "${REPORT_MD}" ]]; then - echo "# Microsoft Foundry E2E Evaluation Report" > "${REPORT_MD}" - echo >> "${REPORT_MD}" - echo "Report generation did not produce output." >> "${REPORT_MD}" - fi - - cat "${REPORT_MD}" - cat "${REPORT_MD}" >> "${GITHUB_STEP_SUMMARY}" - echo "report_md=${REPORT_MD}" >> "${GITHUB_OUTPUT}" - - - name: Comment Microsoft Foundry E2E eval results on PR - if: ${{ always() && inputs.target_pr_id != '' && inputs['report-in-pr'] }} - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - env: - TARGET_PR_ID: ${{ inputs.target_pr_id }} - REPORT_MD: ${{ steps.publish-e2e-report.outputs.report_md }} - JOB_STATUS: ${{ job.status }} - with: - script: | - const fs = require('fs'); - - const targetPrId = (process.env.TARGET_PR_ID ?? '').trim(); - if (!/^[0-9]+$/.test(targetPrId)) { - throw new Error(`target_pr_id must be a pull request number; got ${targetPrId}.`); - } - - const serverUrl = process.env.GITHUB_SERVER_URL || 'https://github.com'; - const runUrl = `${serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const reportPath = (process.env.REPORT_MD ?? '').trim(); - const jobStatus = (process.env.JOB_STATUS ?? 'unknown').trim(); - let report = 'Microsoft Foundry E2E eval report was not generated.'; - - if (reportPath && fs.existsSync(reportPath)) { - report = fs.readFileSync(reportPath, 'utf8').trim(); - } - - const maxReportLength = 50000; - if (report.length > maxReportLength) { - report = `${report.slice(0, maxReportLength)}\n\n...truncated. See the workflow run or artifact for the full eval report.`; - } - - const lines = [ - `Microsoft Foundry E2E eval finished for PR #${targetPrId}.`, - '', - `Workflow run: ${runUrl}`, - `Job status: ${jobStatus}`, - ]; - lines.push('', report); - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: Number(targetPrId), - body: lines.join('\n'), - }); diff --git a/tests/prompts/microsoft-foundry-e2e-eval-report.prompt.md b/tests/prompts/microsoft-foundry-e2e-eval-report.prompt.md deleted file mode 100644 index 091aa3882..000000000 --- a/tests/prompts/microsoft-foundry-e2e-eval-report.prompt.md +++ /dev/null @@ -1,267 +0,0 @@ -# Microsoft Foundry E2E Evaluation Report Prompt - -You are generating the final Markdown report for a Microsoft Foundry E2E Vally evaluation workflow. - -## Overall Guidance - -Create the report as a Markdown file at the path specified by the `REPORT_MD` environment variable. Create parent directories if needed. Do not print the report to stdout; write the complete report to `REPORT_MD`. After writing and verifying the file, the final chat response should only say `Report written to ` followed by the report path. - -In CI, Vally writes `eval-results.md` and `results.jsonl` under timestamped subdirectories of `tests/results/`; the CI runner uses the same repository-relative path. Find all `results.jsonl` files under `tests/results/`, and merge every `eval-results.md` found under `tests/results/` into the combined Raw Results table. Analyze `results.jsonl` directly. Do not invent numbers. - -Only read existing result files in the current working directory and write the final Markdown file to `REPORT_MD`. Do not scan sibling repositories or user directories. Do not run Vally, tests, package install commands, deployment commands, `azd`, `git`, or any command that creates a new evaluation run. Do not create or modify any file except `REPORT_MD` and its parent directory. - -The report file must contain Markdown only. Do not wrap the report in a code fence. Use normal Markdown pipe tables, not terminal box-drawing tables. Do not include preamble, progress narration, raw event logs, or free-form stage notes outside the requested sections. - -Only analyze test records in `results.jsonl` where `trajectory.stimulus.name` starts with `Golden Path`. Ignore all other stimuli and ignore the final `run-summary` record. Determine each trial's model from `trajectory.metadata.model` first, then from `trajectory.metrics.tokenUsage.model`, then from the last `token_usage` event with `data.model`. - -## Output Report Structure - -The output report must use this section order: - -1. `# Microsoft Foundry E2E Evaluation Report` -2. `## Golden Path Result` -3. `## Golden Path Time Cost` -4. `## Golden Path Token Cost` -5. `## Golden Path Model Performance` -6. `## Download` -7. `## Raw Results` - -Do not add other top-level sections. Do not create a `## Links` section. The schemas below show the required shape and example formatting; calculate the actual values from the input files. - -## Section: Golden Path Result - -Purpose: highlight the Golden Path result first, without mixing in non-Golden Path stimuli. - -Guidance: - -- Include only Golden Path trials. -- This section is the Sonnet 4.6 baseline: include only Golden Path trials whose model is `claude-sonnet-4.6`. -- Report the overall Golden Path outcome as `PASS` only if every Golden Path trial passed; otherwise report `FAIL`. -- Add one table row per Golden Path trial, in chronological trial order. -- Use `-` in `Notes` for passed trials. For failed trials, keep the note short and based on the eval result. - -Schema example: - -```markdown -# Microsoft Foundry E2E Evaluation Report - -## Golden Path Result - -**Outcome:** PASS - -**Golden Path trials analyzed:** 2 - -**Passed:** 2 - -**Failed:** 0 - -| Run | Stimulus | Result | Notes | -|---|---|---|---| -| Run 1 | Golden Path - create and deploy Foundry agent | PASS | - | -| Run 2 | Golden Path - create and deploy Foundry agent | PASS | - | -``` - -## Section: Golden Path Time Cost - -Purpose: show Golden Path runtime first as an overall average, then as per-run stage timing. - -Guidance: - -- Runtime is measured from the first `user_message` event to the last `assistant_message` event for each Golden Path trial. -- This section is the Sonnet 4.6 baseline: include only Golden Path trials whose model is `claude-sonnet-4.6`. -- `Total average runtime` must equal the average of the per-run `Total` row values. -- Each stage duration is the AI-driven full wall-clock time for that stage: start when the AI begins working on that stage, and end when the AI completes that stage and moves to the next stage. Include AI reasoning, command execution, waiting, result inspection, retries, and verification within the stage. -- Each run column's stage durations must sum exactly to that run's `Total` row. Assign all elapsed wall-clock time to exactly one stage. -- Report time in `x min Y s` format. Round seconds to an integer. If shorter than 1 minute, report only `Y s`. -- Always include spaces before units: use `54 s`, not `54s`; use `22 min 45 s`, not `22 min 45s`. -- Use the event timeline and event content semantically to divide each Golden Path trial into stages. Do not rely on one exact tool name or one exact command string. -- If there is only one Golden Path trial, use the single-trial schema with `Stage` and `Average` columns. Do not use a `Run 1` column. -- If there are multiple Golden Path trials, use one `Run N` column per Golden Path trial. Do not add an `Average` column. -- Always include the final `Total` row. -- Use `N/A` when a stage did not happen in that trial. - -Main stages: - -- Collect prerequisite info for agent creation -- Scaffold agent code and customize for B2B -- Foundry resources creation -- Test agent locally -- Deploy agent to Foundry -- Test agent by remote invocation -- Eval suite -- Final Output - -Notes: - -- `Foundry resources creation` starts when the AI begins declaring or working on Foundry project/resource creation, `azd provision`, or equivalent `azd provision` tool-call signals, and ends when `azd provision` has fully completed successfully. -- `Test agent locally` includes creating the local virtual environment, installing `uv`, installing project packages from requirements or equivalent package files, starting the local agent server, and invoking the local agent to verify it responds. - -Single-trial schema example: - -```markdown -## Golden Path Time Cost - -1 Golden Path trials analyzed. - -**Total average runtime:** 15 min 43 s - -| Stage | Average | -|---|---:| -| Collect prerequisite info for agent creation | 54 s | -| Scaffold agent code and customize for B2B | 1 min 13 s | -| Foundry resources creation | 1 min 30 s | -| Test agent locally | 5 min 10 s | -| Deploy agent to Foundry | 1 min 50 s | -| Test agent by remote invocation | 4 min 54 s | -| Eval suite | N/A | -| Final Output | 12 s | -| Total | 15 min 43 s | -``` - -Multi-trial schema example: - -```markdown -## Golden Path Time Cost - -2 Golden Path trials analyzed. - -**Total average runtime:** 16 min 7 s - -| Stage | Run 1 | Run 2 | -|---|---:|---:| -| Collect prerequisite info for agent creation | 55 s | 4 min 33 s | -| Scaffold agent code and customize for B2B | 1 min 13 s | 3 min 1 s | -| Foundry resources creation | 1 min 30 s | 1 min 50 s | -| Test agent locally | 5 min 9 s | 3 min 18 s | -| Deploy agent to Foundry | 1 min 50 s | 2 min 55 s | -| Test agent by remote invocation | 4 min 54 s | 36 s | -| Eval suite | N/A | 5 s | -| Final Output | 12 s | 13 s | -| Total | 15 min 43 s | 16 min 31 s | -``` - -## Section: Golden Path Token Cost - -Purpose: show average token usage and AI credit cost for Golden Path trials only. - -### Pricing Table - -Use this GitHub Copilot pricing table. Prices are USD per 1M tokens. The `Model` column uses the Copilot CLI model id. OpenAI and Microsoft models do not have a separate cache write price in GitHub Copilot pricing, so use `N/A` for `Cache write` and treat cache write tokens as regular input tokens when calculating cost. - -| Model | Input | Cached input | Cache write | Output | -|---|---:|---:|---:|---:| -| claude-opus-4.8 | $5.00 | $0.50 | $6.25 | $25.00 | -| claude-sonnet-4.6 | $3.00 | $0.30 | $3.75 | $15.00 | -| claude-sonnet-5 | $2.00 | $0.20 | $2.50 | $10.00 | -| gpt-5.3-codex | $1.75 | $0.175 | N/A | $14.00 | -| gpt-5-mini | $0.25 | $0.025 | N/A | $2.00 | -| mai-code-1-flash | $0.75 | $0.075 | N/A | $4.50 | - -Guidance: - -This section is the Sonnet 4.6 baseline: include only Golden Path trials whose model is `claude-sonnet-4.6`. - -Step 1: calculate average token usage. - -- Calculate token usage directly from `trajectory.metrics.tokenUsage`. -- Each Golden Path trial has complete token usage fields: `inputTokens`, `cacheReadTokens`, `cacheWriteTokens`, and `outputTokens`. -- Average these four fields across Golden Path trials only: `inputTokens`, `cacheReadTokens`, `cacheWriteTokens`, and `outputTokens`. -- Calculate `Total tokens` as average `inputTokens` plus average `outputTokens`. -- Round token counts to integers and format them with thousands separators. - -Step 2: calculate average AIC. - -- `100` AI credits equals `$1.00`. -- Determine the model used by each Golden Path trial from `trajectory.metrics.tokenUsage.model`, then use the matching model row in the Pricing Table for `inputRate`, `cachedInputRate`, `cacheWriteRate`, and `outputRate`. -- Calculate `averageAic = ((((averageInputTokens - averageCacheReadTokens - averageCacheWriteTokens) * inputRate) + (averageCacheReadTokens * cachedInputRate) + (averageCacheWriteTokens * cacheWriteRate) + (averageOutputTokens * outputRate)) / 1,000,000) * 100`. -- If `Cache write` is `N/A` for the model, calculate `averageAic = ((((averageInputTokens - averageCacheReadTokens) * inputRate) + (averageCacheReadTokens * cachedInputRate) + (averageOutputTokens * outputRate)) / 1,000,000) * 100`. -- Format `Average AIC` with two decimal places. - -Schema example: - -```markdown -## Golden Path Token Cost - -2 Golden Path trials analyzed. - -| Metric | Average | -|---|---:| -| Input tokens | 120,000 | -| cacheReadTokens | 30,000 | -| cacheWriteTokens | 5,000 | -| Output tokens | 8,000 | -| Total tokens | 128,000 | -| Average AIC | 8.40 | -``` - -## Section: Golden Path Model Performance - -Purpose: compare Golden Path performance across every model run in the workflow. - -Guidance: - -- Include every Golden Path trial from every `results.jsonl` file found under `tests/results/`. -- Group trials by model. If a model has multiple Golden Path trials across files or repeated `--runs`, average all of that model's Golden Path trials. -- Runtime for each trial is measured from the first `user_message` event to the last `assistant_message` event, using the same timing rule as `## Golden Path Time Cost`. -- Use token usage from `trajectory.metrics.tokenUsage` to calculate `Avg total tokens` and `Avg AIC`; do not include input/cache/output token detail columns in this table. -- `Avg total tokens` is average `inputTokens` plus average `outputTokens`. -- Calculate `Avg AIC` using the Pricing Table and the same cost formula as `## Golden Path Token Cost`. -- Round `Avg total tokens` to an integer and format it with thousands separators. -- Format average runtime in `x min Y s` format. -- Order rows by the `VALLY_MODELS` environment variable if it is available. Otherwise, order rows by the first chronological appearance of each model in the result files. -- Use `N/A` for unavailable metrics or missing pricing rows. Format `Avg AIC` with two decimal places when it is available. - -Schema example: - -```markdown -## Golden Path Model Performance - -N models analyzed. - -| Model | Avg total tokens | Avg time cost | Avg AIC | -|---|---:|---:|---:| -| claude-opus-4.8 | 139,000 | 18 min 22 s | 70.25 | -| claude-sonnet-4.6 | 128,000 | 15 min 43 s | 40.28 | -| gpt-5.3-codex | 125,500 | 17 min 9 s | 26.11 | -``` - -## Section: Download - -Purpose: provide the Vally artifact download link using the existing workflow style. - -Guidance: - -- If the `ARTIFACT_URL` environment variable is available and non-empty, include exactly `[Download Vally results artifact](${ARTIFACT_URL})`. -- If `ARTIFACT_URL` is missing or empty, include exactly `Vally results artifact URL is unavailable.` -- Do not include workflow links here. - -Schema example: - -```markdown -## Download - -[Download Vally results artifact](https://example.com/artifact) -``` - -## Section: Raw Results - -Purpose: keep the original Vally summary available, but put it last so Golden Path analysis is emphasized first. - -Guidance: - -- Include the results table from every `eval-results.md` under `tests/results/`, and merge them into one combined table so all models appear together. -- Make sure every row shows its model by keeping (or adding) a `Model` column. -- If a source file starts with a `## Eval Results` heading, omit that heading so `## Raw Results` remains the final top-level report section. - -Schema example: - -```markdown -## Raw Results - -| Stimulus | Skills | Model | Graders | Pass Rate | Duration | Tokens | Verdict | -|---|---|---|---|---|---|---|---| -| Golden Path - create and deploy Foundry agent | `microsoft-foundry` | claude-sonnet-4.6 | ✅ skill-invocation 1/1
✅ completed 1/1 | 1/1 | 15m 43s | 128,000 | ✅ | -| Golden Path - create and deploy Foundry agent | `microsoft-foundry` | claude-opus-4.8 | ✅ skill-invocation 1/1
✅ completed 1/1 | 1/1 | 18m 22s | 139,000 | ✅ | -``` - -Before finishing, verify that `REPORT_MD` exists and contains all required report sections.