From 13a36e63f51b5d3022c921e24dbb4fda2bf2e127 Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Fri, 1 May 2026 19:49:11 -0400 Subject: [PATCH 1/3] Add two new models to the nightly msbench run Co-authored-by: Copilot --- pipelines/scripts/Invoke-RunBenchmarks.ps1 | 585 +++++++++++---------- 1 file changed, 297 insertions(+), 288 deletions(-) diff --git a/pipelines/scripts/Invoke-RunBenchmarks.ps1 b/pipelines/scripts/Invoke-RunBenchmarks.ps1 index d47314adc..019d77eac 100644 --- a/pipelines/scripts/Invoke-RunBenchmarks.ps1 +++ b/pipelines/scripts/Invoke-RunBenchmarks.ps1 @@ -1,288 +1,297 @@ -<# -.SYNOPSIS - Installs MSBench CLI and runs Azure benchmarks. - -.DESCRIPTION - This script runs in Azure DevOps under an AzureCLI@2 task with federated authentication. - Feed authentication is handled by a preceding PipAuthenticate@1 task that sets - PIP_EXTRA_INDEX_URL for the azure-sdk/internal/MicrosoftSweBench feed. - The script retrieves a GitHub PAT from KeyVault, clones the msbench-benchmarks repo, - installs MSBench CLI, and invokes for each model: - msbench-cli run --agent github-copilot-cli --benchmark --model --no-wait - - Run IDs are extracted from the output and set as the pipeline output variable RUN_IDS. - - MSBench CLI reference: - - https://github.com/devdiv-microsoft/MicrosoftSweBench/wiki - -.PARAMETER Benchmark - Benchmark identifier. Default: azure.skill - -.PARAMETER OutputPath - Directory path where run_ids.json and timestamp.txt will be saved. - -.PARAMETER StorageAccountName - Required. The Azure Storage account name to upload run artifacts to. - -.PARAMETER ContainerName - Required. The blob container name to upload run artifacts to. - -.LINK - https://github.com/devdiv-microsoft/MicrosoftSweBench/wiki -#> - - param( - [string]$Benchmark = "azure.skill", - [string[]]$Model = @( - "claude-sonnet-4.5-autodev-test", - "claude-opus-4.5-autodev-test", - "gpt-5.2-codex-autodev-test", - "gpt-5.2-autodev-test", - "gemini-2.5-pro-autodev-test" - ), - [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$OutputPath, - [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$StorageAccountName, - [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$ContainerName - ) - - Set-StrictMode -Version Latest - $ErrorActionPreference = "Stop" - - if (!$Benchmark) { - throw "Benchmark parameter is required." - } - - if (!$Model -or $Model.Count -eq 0) { - throw "Model parameter is required." - } - - $vaultName = "kv-msbench-eval-azuremcp" - $secretName = "azure-eval-gh-pat" - - Write-Host "Benchmark: $Benchmark" - Write-Host "Models: $($Model -join ', ')" - Write-Host "Output Path: $OutputPath" - $pipelineRun = $env:TF_BUILD -eq "True" - - # --- Retrieve GitHub PAT from KeyVault --- - try { - Write-Host "Retrieving GitHub PAT from KeyVault $vaultName secret $secretName" - $pat = az keyvault secret show --vault-name $vaultName --name $secretName --query value -o tsv - - if (!$pat) { - throw "Secret $secretName not found in KeyVault $vaultName." - } - - $env:GITHUB_MCP_SERVER_TOKEN = $pat - - # Log the PAT as a secret variable to avoid exposing it in logs - if ($pipelineRun) { - Write-Host "##vso[task.setsecret]$pat" - } - } - catch { - throw "Failed to retrieve GitHub PAT from KeyVault: $_" - } - - # --- Feed auth is handled by the PipAuthenticate@1 pipeline task --- - # PipAuthenticate sets PIP_EXTRA_INDEX_URL for the azure-sdk/internal/MicrosoftSweBench feed. - if ($env:PIP_EXTRA_INDEX_URL) { - Write-Host "PIP_EXTRA_INDEX_URL is set (feed auth configured by PipAuthenticate task)" - } else { - Write-Warning "PIP_EXTRA_INDEX_URL is not set. Feed authentication may fail. Ensure PipAuthenticate@1 runs before this script." - } - - $pythonCommand = Get-Command python - Write-Host "Using python from: $($pythonCommand.Path). Version: $(python --version 2>&1)" - - Write-Host "Install/upgrade pip" - python -m pip install --upgrade pip - if ($LASTEXITCODE -ne 0) { - throw "pip install/upgrade failed with exit code $LASTEXITCODE" - } - - Write-Host "Installing/upgrading MSBench CLI" - python -m pip install msbench-cli --no-input - if ($LASTEXITCODE -ne 0) { - throw "pip install msbench-cli failed with exit code $LASTEXITCODE" - } - - Write-Host "MSBench CLI version" - & 'msbench-cli' version - if ($LASTEXITCODE -ne 0) { - throw "msbench-cli version failed with exit code $LASTEXITCODE" - } - - Write-Host "Checking database used by MSBench CLI" - & 'msbench-cli' database - if ($LASTEXITCODE -ne 0) { - throw "msbench-cli database failed with exit code $LASTEXITCODE" - } - - # --- Clone repo and cd to working directory --- - $msbenchRepo = "https://devdiv@dev.azure.com/devdiv/OnlineServices/_git/msbench-benchmarks" - $repoName = "msbench-benchmarks" - - $cloneDir = Join-Path $PWD $repoName - - if (Test-Path $cloneDir) { - Write-Host "Removing existing directory $cloneDir" - Remove-Item -Recurse -Force $cloneDir - } - - Write-Host "Cloning $msbenchRepo into $cloneDir" - # ADO resource id for Azure Repos is 499b84ac-1321-427f-aa17-267ca6975798 - $token = az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798 --query accessToken -o tsv - if ($pipelineRun) { - Write-Host "##vso[task.setsecret]$token" - } - - git -c http.extraheader="AUTHORIZATION: bearer $token" ` - clone --depth 1 $msbenchRepo $cloneDir - if ($LASTEXITCODE -ne 0) { - throw "git clone failed with exit code $LASTEXITCODE" - } - - $targetDir = Join-Path $cloneDir "curation/benchmarks/azure" - if (!(Test-Path $targetDir)) { - throw "Working directory '$targetDir' does not exist after clone." - } - - Write-Host "Changing directory to $targetDir" - Set-Location $targetDir - - $failedModels = @() - $runIds = @() - - foreach ($m in $Model) { - Write-Host "`n=== Running benchmark for model: $m ===" - - $runArgs = @( - "run", - "--agent", "github-copilot-cli", - "--benchmark", $Benchmark, - "--model", $m, - "--env", "GITHUB_MCP_SERVER_TOKEN", - "--dataset", (Join-Path $targetDir "metadata.csv"), - "--tag", "org=CoreAI Cloud and Tools", - "--no-wait" - ) - - Write-Host "Running: msbench-cli $($runArgs -join ' ')" - $cmdOutput = & 'msbench-cli' @runArgs 2>&1 - $msbenchExitCode = $LASTEXITCODE - $cmdOutput | ForEach-Object { Write-Host $_ } - - if ($msbenchExitCode -ne 0) { - Write-Warning "msbench-cli run failed for model '$m' with exit code $msbenchExitCode" - $failedModels += $m - } else { - # Extract run_id from output lines like "run_id=22914845268" - $foundRunId = $false - foreach ($line in $cmdOutput) { - if ($line -match 'run_id=(\d+)') { - $runId = $Matches[1] - Write-Host "Extracted run_id=$runId for model $m" - $runIds += $runId - $foundRunId = $true - break - } - } - if (-not $foundRunId) { - Write-Warning "No run_id found in output for model '$m'" - $failedModels += $m - } - } - } - - if ($failedModels.Count -gt 0) { - throw "msbench-cli run failed for models: $($failedModels -join ', ')" - } - - # Set pipeline output variable with collected run IDs - $runIds = $runIds | Select-Object -Unique - $runIdsValue = $runIds -join ',' - Write-Host "Collected run IDs: $runIdsValue" - if ($pipelineRun -and $runIds.Count -gt 0) { - Write-Host "##vso[task.setvariable variable=RUN_IDS;isoutput=true]$runIdsValue" - } - - New-Item -Path $OutputPath -ItemType Directory -Force | Out-Null - $jsonPath = Join-Path $OutputPath "run_ids.json" - - Write-Host "Saving run IDs to $jsonPath" - $runIds | ConvertTo-Json -AsArray | Out-File -FilePath $jsonPath -Encoding utf8 - - $timestampPath = Join-Path $OutputPath "timestamp.txt" - $timestamp = (Get-Date).ToUniversalTime().ToString("yyyy-MM-dd") - Write-Host "Saving timestamp to ${timestampPath}: $timestamp" - $timestamp | Out-File -FilePath $timestampPath -Encoding utf8 -NoNewline - - # --- Upload run_ids.json and timestamp.txt to Azure Blob Storage --- - Write-Host "`n--- Uploading run artifacts to Azure Blob Storage ---" - foreach ($file in @($jsonPath, $timestampPath)) { - $fileName = [System.IO.Path]::GetFileName($file) - $blobPath = "$timestamp/$fileName" - Write-Host "Uploading $fileName -> $ContainerName/$blobPath" - $azArgs = @( - "storage", "blob", "upload", - "--account-name", $StorageAccountName, - "--container-name", $ContainerName, - "--name", $blobPath, - "--file", $file, - "--auth-mode", "login", - "--overwrite" - ) - az @azArgs - if ($LASTEXITCODE -ne 0) { - Write-Warning "Failed to upload $fileName to blob storage" - } - } - - # --- Append timestamp to ToBeProcessed file --- - $toBeProcessedBlob = "ToBeProcessed" - $toBeProcessedLocal = Join-Path $OutputPath "ToBeProcessed" - - Write-Host "Checking for existing $toBeProcessedBlob in $ContainerName" - $downloadArgs = @( - "storage", "blob", "download", - "--account-name", $StorageAccountName, - "--container-name", $ContainerName, - "--name", $toBeProcessedBlob, - "--file", $toBeProcessedLocal, - "--auth-mode", "login" - ) - $null = az @downloadArgs 2>&1 - - if ($LASTEXITCODE -ne 0) { - Write-Host "$toBeProcessedBlob not found, creating new file" - "" | Out-File -FilePath $toBeProcessedLocal -Encoding utf8 -NoNewline - } - - $existingDates = @(Get-Content -Path $toBeProcessedLocal -ErrorAction SilentlyContinue | Where-Object { $_.Trim() -ne "" }) - if ($existingDates -notcontains $timestamp) { - Write-Host "Appending $timestamp to $toBeProcessedBlob" - $existingDates += $timestamp - } else { - Write-Host "$timestamp already in $toBeProcessedBlob, skipping" - } - - ($existingDates -join "`n") + "`n" | Out-File -FilePath $toBeProcessedLocal -Encoding utf8 -NoNewline - - $uploadArgs = @( - "storage", "blob", "upload", - "--account-name", $StorageAccountName, - "--container-name", $ContainerName, - "--name", $toBeProcessedBlob, - "--file", $toBeProcessedLocal, - "--auth-mode", "login", - "--overwrite" - ) - az @uploadArgs - if ($LASTEXITCODE -ne 0) { - Write-Warning "Failed to upload $toBeProcessedBlob to blob storage" - } else { - Write-Host "Updated $toBeProcessedBlob with timestamp $timestamp" - } - - Write-Host "`nAll $($Model.Count) model runs completed successfully." +<# +.SYNOPSIS + Installs MSBench CLI and runs Azure benchmarks. + +.DESCRIPTION + This script runs in Azure DevOps under an AzureCLI@2 task with federated authentication. + Feed authentication is handled by a preceding PipAuthenticate@1 task that sets + PIP_EXTRA_INDEX_URL for the azure-sdk/internal/MicrosoftSweBench feed. + The script retrieves a GitHub PAT from KeyVault, clones the msbench-benchmarks repo, + installs MSBench CLI, and invokes for each model: + msbench-cli run --agent github-copilot-cli --benchmark --model --no-wait + + Run IDs are extracted from the output and set as the pipeline output variable RUN_IDS. + + MSBench CLI reference: + - https://github.com/devdiv-microsoft/MicrosoftSweBench/wiki + +.PARAMETER Benchmark + Benchmark identifier. Default: azure.skill + +.PARAMETER OutputPath + Directory path where run_ids.json and timestamp.txt will be saved. + +.PARAMETER StorageAccountName + Required. The Azure Storage account name to upload run artifacts to. + +.PARAMETER ContainerName + Required. The blob container name to upload run artifacts to. + +.LINK + https://github.com/devdiv-microsoft/MicrosoftSweBench/wiki +#> + + param( + [string]$Benchmark = "azure.skill", + [string[]]$Model = @( + "claude-sonnet-4.5-autodev-test", + "claude-opus-4.5-autodev-test", + "gpt-5.2-codex-autodev-test", + "gpt-5.2-autodev-test", + "gemini-2.5-pro-autodev-test", + "claude-opus-4.7", + "gpt-5.5" + ), + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$OutputPath, + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$StorageAccountName, + [Parameter(Mandatory=$true)][ValidateNotNullOrEmpty()][string]$ContainerName + ) + + Set-StrictMode -Version Latest + $ErrorActionPreference = "Stop" + + if (!$Benchmark) { + throw "Benchmark parameter is required." + } + + if (!$Model -or $Model.Count -eq 0) { + throw "Model parameter is required." + } + + Write-Host "Benchmark: $Benchmark" + Write-Host "Models: $($Model -join ', ')" + Write-Host "Output Path: $OutputPath" + $pipelineRun = $env:TF_BUILD -eq "True" + + $vaultName = "kv-msbench-eval-azuremcp" + $secretNameGhPAT = "azure-eval-gh-pat" + $secretNameCAPIID= "azure-mcp-eval-capi-id" + $secretNameCAPIHMAC = "azure-mcp-eval-capi-hmac" + + # --- Retrieve GitHub PAT from KeyVault --- + try { + Write-Host "Retrieving GitHub PAT from KeyVault $vaultName secret $secretNameGhPAT" + $pat = az keyvault secret show --vault-name $vaultName --name $secretNameGhPAT --query value -o tsv + $capiId = az keyvault secret show --vault-name $vaultName --name $secretNameCAPIID --query value -o tsv + $capiHmac = az keyvault secret show --vault-name $vaultName --name $secretNameCAPIHMAC --query value -o tsv + + if (!$pat) { + throw "Secret $secretNameGhPAT not found in KeyVault $vaultName." + } + + $env:GITHUB_MCP_SERVER_TOKEN = $pat + $env:CAPI_INTEGRATION_ID = $capiId + $env:CAPI_HMAC_KEY = $capiHmac + # Log the secrets as secret variables to avoid exposing them in logs + if ($pipelineRun) { + Write-Host "##vso[task.setsecret]$pat" + Write-Host "##vso[task.setsecret]$capiId" + Write-Host "##vso[task.setsecret]$capiHmac" + } + } + catch { + throw "Failed to retrieve GitHub PAT from KeyVault: $_" + } + + # --- Feed auth is handled by the PipAuthenticate@1 pipeline task --- + # PipAuthenticate sets PIP_EXTRA_INDEX_URL for the azure-sdk/internal/MicrosoftSweBench feed. + if ($env:PIP_EXTRA_INDEX_URL) { + Write-Host "PIP_EXTRA_INDEX_URL is set (feed auth configured by PipAuthenticate task)" + } else { + Write-Warning "PIP_EXTRA_INDEX_URL is not set. Feed authentication may fail. Ensure PipAuthenticate@1 runs before this script." + } + + $pythonCommand = Get-Command python + Write-Host "Using python from: $($pythonCommand.Path). Version: $(python --version 2>&1)" + + Write-Host "Install/upgrade pip" + python -m pip install --upgrade pip + if ($LASTEXITCODE -ne 0) { + throw "pip install/upgrade failed with exit code $LASTEXITCODE" + } + + Write-Host "Installing/upgrading MSBench CLI" + python -m pip install msbench-cli --no-input + if ($LASTEXITCODE -ne 0) { + throw "pip install msbench-cli failed with exit code $LASTEXITCODE" + } + + Write-Host "MSBench CLI version" + & 'msbench-cli' version + if ($LASTEXITCODE -ne 0) { + throw "msbench-cli version failed with exit code $LASTEXITCODE" + } + + Write-Host "Checking database used by MSBench CLI" + & 'msbench-cli' database + if ($LASTEXITCODE -ne 0) { + throw "msbench-cli database failed with exit code $LASTEXITCODE" + } + + # --- Clone repo and cd to working directory --- + $msbenchRepo = "https://devdiv@dev.azure.com/devdiv/OnlineServices/_git/msbench-benchmarks" + $repoName = "msbench-benchmarks" + + $cloneDir = Join-Path $PWD $repoName + + if (Test-Path $cloneDir) { + Write-Host "Removing existing directory $cloneDir" + Remove-Item -Recurse -Force $cloneDir + } + + Write-Host "Cloning $msbenchRepo into $cloneDir" + # ADO resource id for Azure Repos is 499b84ac-1321-427f-aa17-267ca6975798 + $token = az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798 --query accessToken -o tsv + if ($pipelineRun) { + Write-Host "##vso[task.setsecret]$token" + } + + git -c http.extraheader="AUTHORIZATION: bearer $token" ` + clone --depth 1 $msbenchRepo $cloneDir + if ($LASTEXITCODE -ne 0) { + throw "git clone failed with exit code $LASTEXITCODE" + } + + $targetDir = Join-Path $cloneDir "curation/benchmarks/azure" + if (!(Test-Path $targetDir)) { + throw "Working directory '$targetDir' does not exist after clone." + } + + Write-Host "Changing directory to $targetDir" + Set-Location $targetDir + + $failedModels = @() + $runIds = @() + + foreach ($m in $Model) { + Write-Host "`n=== Running benchmark for model: $m ===" + + $runArgs = @( + "run", + "--agent", "github-copilot-cli", + "--benchmark", $Benchmark, + "--model", $m, + "--env", "GITHUB_MCP_SERVER_TOKEN CAPI_INTEGRATION_ID CAPI_HMAC_KEY", + "--dataset", (Join-Path $targetDir "metadata.csv"), + "--tag", "org=CoreAI Cloud and Tools", + "--no-wait" + ) + + Write-Host "Running: msbench-cli $($runArgs -join ' ')" + $cmdOutput = & 'msbench-cli' @runArgs 2>&1 + $msbenchExitCode = $LASTEXITCODE + $cmdOutput | ForEach-Object { Write-Host $_ } + + if ($msbenchExitCode -ne 0) { + Write-Warning "msbench-cli run failed for model '$m' with exit code $msbenchExitCode" + $failedModels += $m + } else { + # Extract run_id from output lines like "run_id=22914845268" + $foundRunId = $false + foreach ($line in $cmdOutput) { + if ($line -match 'run_id=(\d+)') { + $runId = $Matches[1] + Write-Host "Extracted run_id=$runId for model $m" + $runIds += $runId + $foundRunId = $true + break + } + } + if (-not $foundRunId) { + Write-Warning "No run_id found in output for model '$m'" + $failedModels += $m + } + } + } + + if ($failedModels.Count -gt 0) { + throw "msbench-cli run failed for models: $($failedModels -join ', ')" + } + + # Set pipeline output variable with collected run IDs + $runIds = $runIds | Select-Object -Unique + $runIdsValue = $runIds -join ',' + Write-Host "Collected run IDs: $runIdsValue" + if ($pipelineRun -and $runIds.Count -gt 0) { + Write-Host "##vso[task.setvariable variable=RUN_IDS;isoutput=true]$runIdsValue" + } + + New-Item -Path $OutputPath -ItemType Directory -Force | Out-Null + $jsonPath = Join-Path $OutputPath "run_ids.json" + + Write-Host "Saving run IDs to $jsonPath" + $runIds | ConvertTo-Json -AsArray | Out-File -FilePath $jsonPath -Encoding utf8 + + $timestampPath = Join-Path $OutputPath "timestamp.txt" + $timestamp = (Get-Date).ToUniversalTime().ToString("yyyy-MM-dd") + Write-Host "Saving timestamp to ${timestampPath}: $timestamp" + $timestamp | Out-File -FilePath $timestampPath -Encoding utf8 -NoNewline + + # --- Upload run_ids.json and timestamp.txt to Azure Blob Storage --- + Write-Host "`n--- Uploading run artifacts to Azure Blob Storage ---" + foreach ($file in @($jsonPath, $timestampPath)) { + $fileName = [System.IO.Path]::GetFileName($file) + $blobPath = "$timestamp/$fileName" + Write-Host "Uploading $fileName -> $ContainerName/$blobPath" + $azArgs = @( + "storage", "blob", "upload", + "--account-name", $StorageAccountName, + "--container-name", $ContainerName, + "--name", $blobPath, + "--file", $file, + "--auth-mode", "login", + "--overwrite" + ) + az @azArgs + if ($LASTEXITCODE -ne 0) { + Write-Warning "Failed to upload $fileName to blob storage" + } + } + + # --- Append timestamp to ToBeProcessed file --- + $toBeProcessedBlob = "ToBeProcessed" + $toBeProcessedLocal = Join-Path $OutputPath "ToBeProcessed" + + Write-Host "Checking for existing $toBeProcessedBlob in $ContainerName" + $downloadArgs = @( + "storage", "blob", "download", + "--account-name", $StorageAccountName, + "--container-name", $ContainerName, + "--name", $toBeProcessedBlob, + "--file", $toBeProcessedLocal, + "--auth-mode", "login" + ) + $null = az @downloadArgs 2>&1 + + if ($LASTEXITCODE -ne 0) { + Write-Host "$toBeProcessedBlob not found, creating new file" + "" | Out-File -FilePath $toBeProcessedLocal -Encoding utf8 -NoNewline + } + + $existingDates = @(Get-Content -Path $toBeProcessedLocal -ErrorAction SilentlyContinue | Where-Object { $_.Trim() -ne "" }) + if ($existingDates -notcontains $timestamp) { + Write-Host "Appending $timestamp to $toBeProcessedBlob" + $existingDates += $timestamp + } else { + Write-Host "$timestamp already in $toBeProcessedBlob, skipping" + } + + ($existingDates -join "`n") + "`n" | Out-File -FilePath $toBeProcessedLocal -Encoding utf8 -NoNewline + + $uploadArgs = @( + "storage", "blob", "upload", + "--account-name", $StorageAccountName, + "--container-name", $ContainerName, + "--name", $toBeProcessedBlob, + "--file", $toBeProcessedLocal, + "--auth-mode", "login", + "--overwrite" + ) + az @uploadArgs + if ($LASTEXITCODE -ne 0) { + Write-Warning "Failed to upload $toBeProcessedBlob to blob storage" + } else { + Write-Host "Updated $toBeProcessedBlob with timestamp $timestamp" + } + + Write-Host "`nAll $($Model.Count) model runs completed successfully." From 0fbe052dec82f5b49dc44a5192697e758b332c6f Mon Sep 17 00:00:00 2001 From: Fan Yang Date: Fri, 1 May 2026 19:56:15 -0400 Subject: [PATCH 2/3] Address review feedback Co-authored-by: Copilot --- pipelines/scripts/Invoke-RunBenchmarks.ps1 | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pipelines/scripts/Invoke-RunBenchmarks.ps1 b/pipelines/scripts/Invoke-RunBenchmarks.ps1 index 019d77eac..a2a602d8d 100644 --- a/pipelines/scripts/Invoke-RunBenchmarks.ps1 +++ b/pipelines/scripts/Invoke-RunBenchmarks.ps1 @@ -65,7 +65,7 @@ $vaultName = "kv-msbench-eval-azuremcp" $secretNameGhPAT = "azure-eval-gh-pat" - $secretNameCAPIID= "azure-mcp-eval-capi-id" + $secretNameCAPIID = "azure-mcp-eval-capi-id" $secretNameCAPIHMAC = "azure-mcp-eval-capi-hmac" # --- Retrieve GitHub PAT from KeyVault --- @@ -78,6 +78,12 @@ if (!$pat) { throw "Secret $secretNameGhPAT not found in KeyVault $vaultName." } + if (!$capiId) { + throw "Secret $secretNameCAPIID not found in KeyVault $vaultName." + } + if (!$capiHmac) { + throw "Secret $secretNameCAPIHMAC not found in KeyVault $vaultName." + } $env:GITHUB_MCP_SERVER_TOKEN = $pat $env:CAPI_INTEGRATION_ID = $capiId From 0fcb71b737cd057132d103a7357d829a524ad972 Mon Sep 17 00:00:00 2001 From: Fan Yang <52458914+fanyang-mono@users.noreply.github.com> Date: Fri, 1 May 2026 19:56:27 -0400 Subject: [PATCH 3/3] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- pipelines/scripts/Invoke-RunBenchmarks.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pipelines/scripts/Invoke-RunBenchmarks.ps1 b/pipelines/scripts/Invoke-RunBenchmarks.ps1 index a2a602d8d..0c0bbb656 100644 --- a/pipelines/scripts/Invoke-RunBenchmarks.ps1 +++ b/pipelines/scripts/Invoke-RunBenchmarks.ps1 @@ -6,8 +6,9 @@ This script runs in Azure DevOps under an AzureCLI@2 task with federated authentication. Feed authentication is handled by a preceding PipAuthenticate@1 task that sets PIP_EXTRA_INDEX_URL for the azure-sdk/internal/MicrosoftSweBench feed. - The script retrieves a GitHub PAT from KeyVault, clones the msbench-benchmarks repo, - installs MSBench CLI, and invokes for each model: + The run requires both a GitHub PAT retrieved from KeyVault and the CAPI integration + credentials/environment variables expected by MSBench for the selected agent. The script + clones the msbench-benchmarks repo, installs MSBench CLI, and invokes for each model: msbench-cli run --agent github-copilot-cli --benchmark --model --no-wait Run IDs are extracted from the output and set as the pipeline output variable RUN_IDS.