container-mapping: fix Docker 29 event collection

README.md

@@ -11,6 +11,7 @@ An extension for Azure DevOps that contributes a build task to run the [Microsof
 * Build breaks and more
 * Captures the [container images pushed in a build run](https://learn.microsoft.com/azure/defender-for-cloud/container-image-mapping)
   * In Azure DevOps, this requires the [Microsoft Defender for DevOps Container Mapping extension](https://marketplace.visualstudio.com/items?itemName=ms-securitydevops.ms-dfd-code-to-cloud) to extract metadata from pipelines, such as the container's digest ID and name, for connecting DevOps entities with their related cloud resources. This extension is automatically shared and installed with organizations that are [connected to Microsoft Defender for Cloud](https://learn.microsoft.com/azure/defender-for-cloud/quickstart-onboard-devops). If an organization is not connected to Defender for Cloud, there is **no way** to configure this functionality as it does not work with manual modification of the Microsoft Security DevOps extension. Attempting to configure it through Microsoft Security DevOps may cause unexpected issues.
+  * The injected Container Mapping task reads push events using Docker's structured `Actor.ID` field, including on Docker 29. Mapping failures produce pipeline warnings without failing the build.
 
 ## Basic
 

src/MicrosoftSecurityDevOps/v1/command-executor.ts

@@ -127,7 +127,7 @@ export class CommandExecutor implements IExecutor {
      */
     private getCleanedOutput: () => string = () => {
         var cleanedOutput = [];
-        Buffer.concat(this.chunks).toString().split(os.EOL).forEach((line) => {
+        Buffer.concat(this.chunks).toString().split(/\r?\n/).forEach((line) => {
             if (line) {
                 cleanedOutput.push(line);
             }

src/MicrosoftSecurityDevOps/v1/container-mapping.ts

@@ -48,7 +48,7 @@ export class ContainerMapping implements IMicrosoftSecurityDevOps {
 
         // Initialize the commands 
         let dockerVersionCmd = new CommandExecutor('docker', '--version');
-        let eventsCmd = new CommandExecutor('docker', `events --since ${startTime} --until ${new Date().toISOString()} --filter event=push --filter type=image --format ID={{.ID}}`);
+        let eventsCmd = new CommandExecutor('docker', `events --since ${startTime} --until ${new Date().toISOString()} --filter event=push --filter type=image --format ID={{.Actor.ID}}`);
         let imagesCmd = new CommandExecutor('docker', 'images --format CreatedAt={{.CreatedAt}}::Repo={{.Repository}}::Tag={{.Tag}}::Digest={{.Digest}}');
 
         // Execute all commands in parallel
@@ -236,7 +236,7 @@ export class ContainerMapping implements IMicrosoftSecurityDevOps {
         }
         catch (error) {
             // Log the error
-            writeToOutStream("Error in Container Mapping: " + error);
+            tl.warning("Error in Container Mapping: " + error);
         }
         finally {
             // End the collapsible section

test/MicrosoftSecurityDevOps/v1/command-executor.tests.ts

@@ -3,8 +3,10 @@ import * as path from 'path';
 import os from 'os';
 import * as sinon from 'sinon';
 import mockery from 'mockery';
+import * as Q from 'q';
 import * as assert from 'assert';
 import * as mtr from 'azure-pipelines-task-lib/mock-toolrunner';
+import { IExecOptions } from 'azure-pipelines-task-lib/toolrunner';
 const MockToolRunner = mtr.ToolRunner;
 
 describe('Command Executor tests', function () {
@@ -68,4 +70,21 @@ describe('Command Executor tests', function () {
         assert.equal(cleanedOutput, testDockerVersion, "Command line should be removed");
         done();
     });
+
+    [
+        { name: 'LF', output: 'ID=first\n\nID=second\n' },
+        { name: 'CRLF', output: 'ID=first\r\n\r\nID=second\r\n' },
+        { name: 'mixed', output: 'ID=first\r\n\nID=second\n' }
+    ].forEach(({ name, output }) => {
+        it(`should normalize ${name} output to platform line endings`, async function() {
+            sinon.stub(MockToolRunner.prototype, 'exec').callsFake((options: IExecOptions) => {
+                options.outStream.write(output);
+                return Q.resolve(0);
+            });
+            const executorMock = new executor.CommandExecutor('testTool', 'testArg');
+            const result = await executorMock.execute();
+            assert.equal(result.code, 0);
+            assert.equal(result.output, ['ID=first', 'ID=second'].join(os.EOL));
+        });
+    });
 });

test/MicrosoftSecurityDevOps/v1/container-mapping.tests.ts

@@ -1,6 +1,7 @@
 import { stagingDirectory, TestConstants } from '../../testCommon';
 import * as path from 'path';
 import * as assert from 'assert';
+import * as os from 'os';
 import * as ttm from 'azure-pipelines-task-lib/mock-test';
 import { TaskLibAnswerExecResult } from 'azure-pipelines-task-lib/mock-answer';
 const helpers = require(path.join(stagingDirectory, 'MicrosoftSecurityDevOps', 'v1', 'msdo-helpers'));
@@ -9,6 +10,13 @@ let CommandType = helpers.CommandType;
 
 const conMapTestFile = path.join(__dirname, 'msdoMock.js');
 
+async function runContainerMappingTask(): Promise<ttm.MockTestRunner> {
+    const tr = new ttm.MockTestRunner(conMapTestFile);
+    tr.nodePath = process.execPath;
+    await tr.runAsync();
+    return tr;
+}
+
 describe('Container Mapping tests', function () {
 
     var curTime = new Date().toISOString();
@@ -22,70 +30,108 @@ describe('Container Mapping tests', function () {
     // Reset the environment variables for each test.
     this.afterEach(() => {
         delete process.env[TestConstants.MockResponse];
+        delete process.env[TestConstants.MockResponses];
         delete process.env[helpers.Constants.PreJobStartTime];
         delete process.env[TestConstants.InputPrefix + Inputs.CommandType];
     }); 
 
-    it('should process pre-job', function(done: Mocha.Done) {        
+    it('should process pre-job', async function() {
         process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PreJob;
-        let tr: ttm.MockTestRunner = new ttm.MockTestRunner(conMapTestFile);
-        tr.run();
+        const tr = await runContainerMappingTask();
         assert.equal(tr.succeeded, true, 'should have succeeded:\n' + tr.stdout);
         assert.equal(tr.stdOutContained('vso[task.setvariable variable='+ helpers.Constants.PreJobStartTime +';'), true, 
             "variable not set:\n" + tr.stdout);
-        done();
     });
 
-    it('should process post-job', function(done: Mocha.Done) {
+    it('should process post-job', async function() {
         process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PostJob;
         process.env[helpers.Constants.PreJobStartTime] = curTime; 
-        let tr: ttm.MockTestRunner = new ttm.MockTestRunner(conMapTestFile);
-        tr.run();
+        const tr = await runContainerMappingTask();
         assert.equal(tr.succeeded, true, 'should have succeeded:\n' + tr.stdout);
+        assert.equal(tr.warningIssues.length, 0, 'should not warn on successful collection:\n' + tr.stdout);
         assert.equal(tr.stdOutContained('##vso[task.debug]'+ helpers.Constants.PreJobStartTime +'='+curTime), true, 
         "Task should have been processed:\n" + tr.stdout);
-        done();
     });
 
-    it('should always succeed if pre or post job', function(done: Mocha.Done) {
+    it('should use structured actor IDs and preserve Docker event filters', async function() {
+        process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PostJob;
+        process.env[helpers.Constants.PreJobStartTime] = curTime;
+        const tr = await runContainerMappingTask();
+        const eventsCommand = tr.stdout.split(/\r?\n/).find(line => line.startsWith('Mock command: docker events '));
+        assert.ok(eventsCommand, 'Docker events command not captured:\n' + tr.stdout);
+        assert.match(eventsCommand, /^Mock command: docker events --since \S+ --until \S+ --filter event=push --filter type=image --format ID=\{\{\.Actor\.ID\}\}$/);
+        assert.equal(eventsCommand.includes(`--since ${curTime} --until `), true, 'should preserve the pre-job start time');
+    });
+
+    it('should always succeed if pre or post job', async function() {
         process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PostJob;
         let mockResponse: TaskLibAnswerExecResult = {
             code: 100,
             stdout: TestConstants.Error + " Exit Code: 100"
         };
         process.env[TestConstants.MockResponse] = JSON.stringify(mockResponse);
         process.env[helpers.Constants.PreJobStartTime] = curTime;
-        let tr: ttm.MockTestRunner = new ttm.MockTestRunner(conMapTestFile);
-        tr.run();
+        const tr = await runContainerMappingTask();
         assert.equal(tr.succeeded, true, 'should have succeeded:\n' + tr.stdout);
+        assert.equal(tr.warningIssues.length, 1, 'should warn without failing:\n' + tr.stdout);
         assert.equal(tr.stdOutContained('Exit Code: 100'), true, "Exit code not present:\n" + tr.stdout);
-        done();
     });
 
-    it('should encode post-job output', function(done: Mocha.Done) {
+    it('should warn without failing when Docker events cannot be fetched', async function() {
+        process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PostJob;
+        process.env[helpers.Constants.PreJobStartTime] = curTime;
+        const errorMessage = "Error parsing format: can't evaluate field ID in type *events.Message";
+        process.env[TestConstants.MockResponses] = JSON.stringify({
+            events: { code: 64, stdout: errorMessage }
+        });
+        const tr = await runContainerMappingTask();
+        assert.equal(tr.succeeded, true, 'should have succeeded:\n' + tr.stdout);
+        assert.equal(tr.warningIssues.length, 1, 'should emit one pipeline warning:\n' + tr.stdout);
+        assert.equal(tr.warningIssues[0].includes('Unable to fetch Docker events: ' + errorMessage), true);
+        assert.equal(tr.stdOutContained('Mock HTTP request body: {"dockerVersion":'), false, 'should not upload a failed collection');
+    });
+
+    it('should preserve multiple Docker events in the report and encoded output', async function() {
+        process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PostJob;
+        process.env[helpers.Constants.PreJobStartTime] = curTime;
+        const events = ['ID=registry.example.com/app:first', 'ID=registry.example.com/app:second'];
+        const eventsOutput = events.join(os.EOL);
+        process.env[TestConstants.MockResponses] = JSON.stringify({
+            events: { code: 0, stdout: eventsOutput }
+        });
+        const tr = await runContainerMappingTask();
+        const expectedReport = {
+            dockerVersion: TestConstants.Success,
+            dockerEvents: events,
+            dockerImages: [TestConstants.Success]
+        };
+        assert.equal(tr.succeeded, true, 'should have succeeded:\n' + tr.stdout);
+        assert.equal(tr.warningIssues.length, 0, 'should not warn:\n' + tr.stdout);
+        assert.equal(tr.stdOutContained('Mock HTTP request body: ' + JSON.stringify(expectedReport)), true, 'report should preserve event lines:\n' + tr.stdout);
+        assert.equal(tr.stdOutContained(helpers.getEncodedContent(TestConstants.Success, eventsOutput, TestConstants.Success)), true, 'encoded output should preserve event lines:\n' + tr.stdout);
+    });
+
+    it('should encode post-job output', async function() {
         process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PostJob;
         process.env[helpers.Constants.PreJobStartTime] = curTime;
-        let tr: ttm.MockTestRunner = new ttm.MockTestRunner(conMapTestFile);
-        tr.run();
+        const tr = await runContainerMappingTask();
         var expectedEncodedOutput = helpers.getEncodedContent(TestConstants.Success, TestConstants.Success, TestConstants.Success);
         assert.equal(tr.succeeded, true, 'should have succeeded:\n' + tr.stdout);
         assert.equal(tr.stdOutContained(expectedEncodedOutput), true, "Output should have been encoded:\n" + tr.stdout);
-        done();
     });
 
-    it('should not fetch images if no events', function(done: Mocha.Done) {
+    it('should preserve the empty-events result without warning', async function() {
         process.env[TestConstants.InputPrefix + Inputs.CommandType] = CommandType.PostJob;
         process.env[helpers.Constants.PreJobStartTime] = curTime;
         let mockResponse: TaskLibAnswerExecResult = {
             code: 0
         };
         process.env[TestConstants.MockResponse] = JSON.stringify(mockResponse);
-        let tr: ttm.MockTestRunner = new ttm.MockTestRunner(conMapTestFile);
-        tr.run();
+        const tr = await runContainerMappingTask();
         var expectedEncodedOutput = helpers.getEncodedContent(undefined, '', '');
         assert.equal(tr.succeeded, true, 'should have succeeded:\n' + tr.stdout);
-        assert.equal(tr.stdOutContained('code=NoDockerEvents'), true, "Issue not reported:\n" + tr.stdout);
+        assert.equal(tr.warningIssues.length, 0, 'should not warn when there are no push events:\n' + tr.stdout);
+        assert.equal(tr.stdOutContained('type=NoDockerEvents;name=NoDockerEvents;'), true, "Issue not reported:\n" + tr.stdout);
         assert.equal(tr.stdOutContained(expectedEncodedOutput), true, "Encoded output not present:\n" + tr.stdout);
-        done();
     });
 });

test/MicrosoftSecurityDevOps/v1/msdoMock.ts

@@ -2,6 +2,7 @@ import { stagingDirectory, TestConstants } from '../../testCommon';
 import { TaskLibAnswerExecResult } from 'azure-pipelines-task-lib/mock-answer';
 import * as tmrm from 'azure-pipelines-task-lib/mock-run';
 import * as path from 'path';
+import { EventEmitter } from 'events';
 
 let taskPath = path.join(stagingDirectory, 'MicrosoftSecurityDevOps', 'v1', 'index.js');
 
@@ -14,10 +15,19 @@ let successResponse: TaskLibAnswerExecResult = {
 
 let response: string = process.env[TestConstants.MockResponse];
 let mockResponse: TaskLibAnswerExecResult =  response ? JSON.parse(response) : successResponse; 
+let responses: string = process.env[TestConstants.MockResponses];
+let mockResponses: { [command: string]: TaskLibAnswerExecResult } = responses ? JSON.parse(responses) : {};
 
 class ExecutorMock {
+    private readonly response: TaskLibAnswerExecResult;
+
+    constructor(toolName: string, argLine: string) {
+        console.log(`Mock command: ${toolName} ${argLine}`);
+        this.response = mockResponses[argLine.split(' ')[0]] || mockResponse;
+    }
+
     public async execute() {
-        return Promise.resolve({code: mockResponse.code, output: mockResponse.stdout});
+        return Promise.resolve({code: this.response.code, output: this.response.stdout});
     }
 
     public static removeCommandFromOutput(output: string) : string {
@@ -29,4 +39,18 @@ tmr.registerMock('./command-executor', {
     CommandExecutor: ExecutorMock
 });
 
+tmr.registerMock('https', {
+    request: (_uri: string, _options: object, callback: (response: EventEmitter) => void) => {
+        const response = Object.assign(new EventEmitter(), { statusCode: 200, statusMessage: 'OK' });
+        return Object.assign(new EventEmitter(), {
+            write: (data: string) => console.log('Mock HTTP request body: ' + data),
+            end: () => {
+                callback(response);
+                response.emit('data', JSON.stringify({ oidcToken: 'mock-token' }));
+                response.emit('end');
+            }
+        });
+    }
+});
+
 tmr.run();

test/testCommon.ts

@@ -9,5 +9,6 @@ export enum TestConstants {
     Success = 'Success',
     TaskTestTrace = 'TASK_TEST_TRACE',
     MockResponse = 'MOCK_RESPONSE',
+    MockResponses = 'MOCK_RESPONSES',
     InputPrefix = 'INPUT_'
 };