From 7974f6e75b359f9c2eeae4e5ca87722dccb332f7 Mon Sep 17 00:00:00 2001 From: Dima Birenbaum Date: Sun, 18 Oct 2026 14:58:46 +0300 Subject: [PATCH] fix(container-mapping): support Docker 29 events Use structured Docker event actor IDs and surface container mapping failures as non-blocking warnings. Normalize CLI output line endings so Windows reports keep individual event and image rows. Add regression coverage and update the container mapping documentation. Refs #172. --- README.md | 1 + .../v1/command-executor.ts | 2 +- .../v1/container-mapping.ts | 4 +- .../v1/command-executor.tests.ts | 19 ++++ .../v1/container-mapping.tests.ts | 88 ++++++++++++++----- test/MicrosoftSecurityDevOps/v1/msdoMock.ts | 26 +++++- test/testCommon.ts | 1 + 7 files changed, 116 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 987362e..a99967f 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ An extension for Azure DevOps that contributes a build task to run the [Microsof * Build breaks and more * Captures the [container images pushed in a build run](https://learn.microsoft.com/azure/defender-for-cloud/container-image-mapping) * In Azure DevOps, this requires the [Microsoft Defender for DevOps Container Mapping extension](https://marketplace.visualstudio.com/items?itemName=ms-securitydevops.ms-dfd-code-to-cloud) to extract metadata from pipelines, such as the container's digest ID and name, for connecting DevOps entities with their related cloud resources. This extension is automatically shared and installed with organizations that are [connected to Microsoft Defender for Cloud](https://learn.microsoft.com/azure/defender-for-cloud/quickstart-onboard-devops). If an organization is not connected to Defender for Cloud, there is **no way** to configure this functionality as it does not work with manual modification of the Microsoft Security DevOps extension. Attempting to configure it through Microsoft Security DevOps may cause unexpected issues. + * The injected Container Mapping task reads push events using Docker's structured `Actor.ID` field, including on Docker 29. Mapping failures produce pipeline warnings without failing the build. ## Basic diff --git a/src/MicrosoftSecurityDevOps/v1/command-executor.ts b/src/MicrosoftSecurityDevOps/v1/command-executor.ts index dcd54f9..f574114 100644 --- a/src/MicrosoftSecurityDevOps/v1/command-executor.ts +++ b/src/MicrosoftSecurityDevOps/v1/command-executor.ts @@ -127,7 +127,7 @@ export class CommandExecutor implements IExecutor { */ private getCleanedOutput: () => string = () => { var cleanedOutput = []; - Buffer.concat(this.chunks).toString().split(os.EOL).forEach((line) => { + Buffer.concat(this.chunks).toString().split(/\r?\n/).forEach((line) => { if (line) { cleanedOutput.push(line); } diff --git a/src/MicrosoftSecurityDevOps/v1/container-mapping.ts b/src/MicrosoftSecurityDevOps/v1/container-mapping.ts index 0d557be..6e55020 100644 --- a/src/MicrosoftSecurityDevOps/v1/container-mapping.ts +++ b/src/MicrosoftSecurityDevOps/v1/container-mapping.ts @@ -48,7 +48,7 @@ export class ContainerMapping implements IMicrosoftSecurityDevOps { // Initialize the commands let dockerVersionCmd = new CommandExecutor('docker', '--version'); - let eventsCmd = new CommandExecutor('docker', `events --since ${startTime} --until ${new Date().toISOString()} --filter event=push --filter type=image --format ID={{.ID}}`); + let eventsCmd = new CommandExecutor('docker', `events --since ${startTime} --until ${new Date().toISOString()} --filter event=push --filter type=image --format ID={{.Actor.ID}}`); let imagesCmd = new CommandExecutor('docker', 'images --format CreatedAt={{.CreatedAt}}::Repo={{.Repository}}::Tag={{.Tag}}::Digest={{.Digest}}'); // Execute all commands in parallel @@ -236,7 +236,7 @@ export class ContainerMapping implements IMicrosoftSecurityDevOps { } catch (error) { // Log the error - writeToOutStream("Error in Container Mapping: " + error); + tl.warning("Error in Container Mapping: " + error); } finally { // End the collapsible section diff --git a/test/MicrosoftSecurityDevOps/v1/command-executor.tests.ts b/test/MicrosoftSecurityDevOps/v1/command-executor.tests.ts index 18a81c4..6ee0368 100644 --- a/test/MicrosoftSecurityDevOps/v1/command-executor.tests.ts +++ b/test/MicrosoftSecurityDevOps/v1/command-executor.tests.ts @@ -3,8 +3,10 @@ import * as path from 'path'; import os from 'os'; import * as sinon from 'sinon'; import mockery from 'mockery'; +import * as Q from 'q'; import * as assert from 'assert'; import * as mtr from 'azure-pipelines-task-lib/mock-toolrunner'; +import { IExecOptions } from 'azure-pipelines-task-lib/toolrunner'; const MockToolRunner = mtr.ToolRunner; describe('Command Executor tests', function () { @@ -68,4 +70,21 @@ describe('Command Executor tests', function () { assert.equal(cleanedOutput, testDockerVersion, "Command line should be removed"); done(); }); + + [ + { name: 'LF', output: 'ID=first\n\nID=second\n' }, + { name: 'CRLF', output: 'ID=first\r\n\r\nID=second\r\n' }, + { name: 'mixed', output: 'ID=first\r\n\nID=second\n' } + ].forEach(({ name, output }) => { + it(`should normalize ${name} output to platform line endings`, async function() { + sinon.stub(MockToolRunner.prototype, 'exec').callsFake((options: IExecOptions) => { + options.outStream.write(output); + return Q.resolve(0); + }); + const executorMock = new executor.CommandExecutor('testTool', 'testArg'); + const result = await executorMock.execute(); + assert.equal(result.code, 0); + assert.equal(result.output, ['ID=first', 'ID=second'].join(os.EOL)); + }); + }); }); \ No newline at end of file diff --git a/test/MicrosoftSecurityDevOps/v1/container-mapping.tests.ts b/test/MicrosoftSecurityDevOps/v1/container-mapping.tests.ts index f0f93a7..af5ff0b 100644 --- a/test/MicrosoftSecurityDevOps/v1/container-mapping.tests.ts +++ b/test/MicrosoftSecurityDevOps/v1/container-mapping.tests.ts @@ -1,6 +1,7 @@ import { stagingDirectory, TestConstants } from '../../testCommon'; import * as path from 'path'; import * as assert from 'assert'; +import * as os from 'os'; import * as ttm from 'azure-pipelines-task-lib/mock-test'; import { TaskLibAnswerExecResult } from 'azure-pipelines-task-lib/mock-answer'; const helpers = require(path.join(stagingDirectory, 'MicrosoftSecurityDevOps', 'v1', 'msdo-helpers')); @@ -9,6 +10,13 @@ let CommandType = helpers.CommandType; const conMapTestFile = path.join(__dirname, 'msdoMock.js'); +async function runContainerMappingTask(): Promise { + const tr = new ttm.MockTestRunner(conMapTestFile); + tr.nodePath = process.execPath; + await tr.runAsync(); + return tr; +} + describe('Container Mapping tests', function () { var curTime = new Date().toISOString(); @@ -22,32 +30,40 @@ describe('Container Mapping tests', function () { // Reset the environment variables for each test. this.afterEach(() => { delete process.env[TestConstants.MockResponse]; + delete process.env[TestConstants.MockResponses]; delete process.env[helpers.Constants.PreJobStartTime]; delete process.env[TestConstants.InputPrefix + Inputs.CommandType]; }); - it('should process pre-job', function(done: Mocha.Done) { + it('should process pre-job', async function() { process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PreJob; - let tr: ttm.MockTestRunner = new ttm.MockTestRunner(conMapTestFile); - tr.run(); + const tr = await runContainerMappingTask(); assert.equal(tr.succeeded, true, 'should have succeeded:\n' + tr.stdout); assert.equal(tr.stdOutContained('vso[task.setvariable variable='+ helpers.Constants.PreJobStartTime +';'), true, "variable not set:\n" + tr.stdout); - done(); }); - it('should process post-job', function(done: Mocha.Done) { + it('should process post-job', async function() { process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PostJob; process.env[helpers.Constants.PreJobStartTime] = curTime; - let tr: ttm.MockTestRunner = new ttm.MockTestRunner(conMapTestFile); - tr.run(); + const tr = await runContainerMappingTask(); assert.equal(tr.succeeded, true, 'should have succeeded:\n' + tr.stdout); + assert.equal(tr.warningIssues.length, 0, 'should not warn on successful collection:\n' + tr.stdout); assert.equal(tr.stdOutContained('##vso[task.debug]'+ helpers.Constants.PreJobStartTime +'='+curTime), true, "Task should have been processed:\n" + tr.stdout); - done(); }); - it('should always succeed if pre or post job', function(done: Mocha.Done) { + it('should use structured actor IDs and preserve Docker event filters', async function() { + process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PostJob; + process.env[helpers.Constants.PreJobStartTime] = curTime; + const tr = await runContainerMappingTask(); + const eventsCommand = tr.stdout.split(/\r?\n/).find(line => line.startsWith('Mock command: docker events ')); + assert.ok(eventsCommand, 'Docker events command not captured:\n' + tr.stdout); + assert.match(eventsCommand, /^Mock command: docker events --since \S+ --until \S+ --filter event=push --filter type=image --format ID=\{\{\.Actor\.ID\}\}$/); + assert.equal(eventsCommand.includes(`--since ${curTime} --until `), true, 'should preserve the pre-job start time'); + }); + + it('should always succeed if pre or post job', async function() { process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PostJob; let mockResponse: TaskLibAnswerExecResult = { code: 100, @@ -55,37 +71,67 @@ describe('Container Mapping tests', function () { }; process.env[TestConstants.MockResponse] = JSON.stringify(mockResponse); process.env[helpers.Constants.PreJobStartTime] = curTime; - let tr: ttm.MockTestRunner = new ttm.MockTestRunner(conMapTestFile); - tr.run(); + const tr = await runContainerMappingTask(); assert.equal(tr.succeeded, true, 'should have succeeded:\n' + tr.stdout); + assert.equal(tr.warningIssues.length, 1, 'should warn without failing:\n' + tr.stdout); assert.equal(tr.stdOutContained('Exit Code: 100'), true, "Exit code not present:\n" + tr.stdout); - done(); }); - it('should encode post-job output', function(done: Mocha.Done) { + it('should warn without failing when Docker events cannot be fetched', async function() { + process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PostJob; + process.env[helpers.Constants.PreJobStartTime] = curTime; + const errorMessage = "Error parsing format: can't evaluate field ID in type *events.Message"; + process.env[TestConstants.MockResponses] = JSON.stringify({ + events: { code: 64, stdout: errorMessage } + }); + const tr = await runContainerMappingTask(); + assert.equal(tr.succeeded, true, 'should have succeeded:\n' + tr.stdout); + assert.equal(tr.warningIssues.length, 1, 'should emit one pipeline warning:\n' + tr.stdout); + assert.equal(tr.warningIssues[0].includes('Unable to fetch Docker events: ' + errorMessage), true); + assert.equal(tr.stdOutContained('Mock HTTP request body: {"dockerVersion":'), false, 'should not upload a failed collection'); + }); + + it('should preserve multiple Docker events in the report and encoded output', async function() { + process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PostJob; + process.env[helpers.Constants.PreJobStartTime] = curTime; + const events = ['ID=registry.example.com/app:first', 'ID=registry.example.com/app:second']; + const eventsOutput = events.join(os.EOL); + process.env[TestConstants.MockResponses] = JSON.stringify({ + events: { code: 0, stdout: eventsOutput } + }); + const tr = await runContainerMappingTask(); + const expectedReport = { + dockerVersion: TestConstants.Success, + dockerEvents: events, + dockerImages: [TestConstants.Success] + }; + assert.equal(tr.succeeded, true, 'should have succeeded:\n' + tr.stdout); + assert.equal(tr.warningIssues.length, 0, 'should not warn:\n' + tr.stdout); + assert.equal(tr.stdOutContained('Mock HTTP request body: ' + JSON.stringify(expectedReport)), true, 'report should preserve event lines:\n' + tr.stdout); + assert.equal(tr.stdOutContained(helpers.getEncodedContent(TestConstants.Success, eventsOutput, TestConstants.Success)), true, 'encoded output should preserve event lines:\n' + tr.stdout); + }); + + it('should encode post-job output', async function() { process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PostJob; process.env[helpers.Constants.PreJobStartTime] = curTime; - let tr: ttm.MockTestRunner = new ttm.MockTestRunner(conMapTestFile); - tr.run(); + const tr = await runContainerMappingTask(); var expectedEncodedOutput = helpers.getEncodedContent(TestConstants.Success, TestConstants.Success, TestConstants.Success); assert.equal(tr.succeeded, true, 'should have succeeded:\n' + tr.stdout); assert.equal(tr.stdOutContained(expectedEncodedOutput), true, "Output should have been encoded:\n" + tr.stdout); - done(); }); - it('should not fetch images if no events', function(done: Mocha.Done) { + it('should preserve the empty-events result without warning', async function() { process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PostJob; process.env[helpers.Constants.PreJobStartTime] = curTime; let mockResponse: TaskLibAnswerExecResult = { code: 0 }; process.env[TestConstants.MockResponse] = JSON.stringify(mockResponse); - let tr: ttm.MockTestRunner = new ttm.MockTestRunner(conMapTestFile); - tr.run(); + const tr = await runContainerMappingTask(); var expectedEncodedOutput = helpers.getEncodedContent(undefined, '', ''); assert.equal(tr.succeeded, true, 'should have succeeded:\n' + tr.stdout); - assert.equal(tr.stdOutContained('code=NoDockerEvents'), true, "Issue not reported:\n" + tr.stdout); + assert.equal(tr.warningIssues.length, 0, 'should not warn when there are no push events:\n' + tr.stdout); + assert.equal(tr.stdOutContained('type=NoDockerEvents;name=NoDockerEvents;'), true, "Issue not reported:\n" + tr.stdout); assert.equal(tr.stdOutContained(expectedEncodedOutput), true, "Encoded output not present:\n" + tr.stdout); - done(); }); }); \ No newline at end of file diff --git a/test/MicrosoftSecurityDevOps/v1/msdoMock.ts b/test/MicrosoftSecurityDevOps/v1/msdoMock.ts index 82f0de0..b2ecd66 100644 --- a/test/MicrosoftSecurityDevOps/v1/msdoMock.ts +++ b/test/MicrosoftSecurityDevOps/v1/msdoMock.ts @@ -2,6 +2,7 @@ import { stagingDirectory, TestConstants } from '../../testCommon'; import { TaskLibAnswerExecResult } from 'azure-pipelines-task-lib/mock-answer'; import * as tmrm from 'azure-pipelines-task-lib/mock-run'; import * as path from 'path'; +import { EventEmitter } from 'events'; let taskPath = path.join(stagingDirectory, 'MicrosoftSecurityDevOps', 'v1', 'index.js'); @@ -14,10 +15,19 @@ let successResponse: TaskLibAnswerExecResult = { let response: string = process.env[TestConstants.MockResponse]; let mockResponse: TaskLibAnswerExecResult = response ? JSON.parse(response) : successResponse; +let responses: string = process.env[TestConstants.MockResponses]; +let mockResponses: { [command: string]: TaskLibAnswerExecResult } = responses ? JSON.parse(responses) : {}; class ExecutorMock { + private readonly response: TaskLibAnswerExecResult; + + constructor(toolName: string, argLine: string) { + console.log(`Mock command: ${toolName} ${argLine}`); + this.response = mockResponses[argLine.split(' ')[0]] || mockResponse; + } + public async execute() { - return Promise.resolve({code: mockResponse.code, output: mockResponse.stdout}); + return Promise.resolve({code: this.response.code, output: this.response.stdout}); } public static removeCommandFromOutput(output: string) : string { @@ -29,4 +39,18 @@ tmr.registerMock('./command-executor', { CommandExecutor: ExecutorMock }); +tmr.registerMock('https', { + request: (_uri: string, _options: object, callback: (response: EventEmitter) => void) => { + const response = Object.assign(new EventEmitter(), { statusCode: 200, statusMessage: 'OK' }); + return Object.assign(new EventEmitter(), { + write: (data: string) => console.log('Mock HTTP request body: ' + data), + end: () => { + callback(response); + response.emit('data', JSON.stringify({ oidcToken: 'mock-token' })); + response.emit('end'); + } + }); + } +}); + tmr.run(); diff --git a/test/testCommon.ts b/test/testCommon.ts index cf418b4..dece447 100644 --- a/test/testCommon.ts +++ b/test/testCommon.ts @@ -9,5 +9,6 @@ export enum TestConstants { Success = 'Success', TaskTestTrace = 'TASK_TEST_TRACE', MockResponse = 'MOCK_RESPONSE', + MockResponses = 'MOCK_RESPONSES', InputPrefix = 'INPUT_' }; \ No newline at end of file