Skip to content

Commit 931bc4c

Browse files
Copilotedburns
Copilot
and
edburns
authored
Port shouldUseReplacedIdentitySectionInResponse E2E test to dotnet, go, nodejs, python, and rust (github#1691)
* Initial plan * Port shouldUseReplacedIdentitySectionInResponse E2E test to all 5 non-Java languages Co-authored-by: edburns <75821+edburns@users.noreply.github.com> * fix(python): fix ruff format and lint violations in system_message_sections e2e test Co-authored-by: edburns <75821+edburns@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: edburns <75821+edburns@users.noreply.github.com> Co-authored-by: Ed Burns <edburns@microsoft.com>
1 parent 30db42f commit 931bc4c

6 files changed

Lines changed: 243 additions & 0 deletions

File tree

‎dotnet/test/E2E/SystemMessageSectionsE2ETests.cs‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
*--------------------------------------------------------------------------------------------*/
4+
5+
using GitHub.Copilot.Rpc;
6+
using GitHub.Copilot.Test.Harness;
7+
using Xunit;
8+
using Xunit.Abstractions;
9+
10+
namespace GitHub.Copilot.Test.E2E;
11+
12+
public class SystemMessageSectionsE2ETests(E2ETestFixture fixture, ITestOutputHelper output) : E2ETestBase(fixture, "system_message_sections", output)
13+
{
14+
[Fact]
15+
public async Task Should_Use_Replaced_Identity_Section_In_Response()
16+
{
17+
var session = await CreateSessionAsync(new SessionConfig
18+
{
19+
OnPermissionRequest = PermissionHandler.ApproveAll,
20+
SystemMessage = new SystemMessageConfig
21+
{
22+
Mode = SystemMessageMode.Customize,
23+
Sections = new Dictionary<SystemMessageSection, SectionOverride>
24+
{
25+
[SystemMessageSection.Identity] = new SectionOverride
26+
{
27+
Action = SectionOverrideAction.Replace,
28+
Content = "You are a helpful gardening assistant called Botanica. You only answer questions about plants and gardening."
29+
}
30+
}
31+
}
32+
});
33+
34+
await session.SendAsync(new MessageOptions { Prompt = "Who are you?" });
35+
var response = await TestHelper.GetFinalAssistantMessageAsync(session);
36+
37+
Assert.NotNull(response);
38+
var content = response.Data.Content.ToLowerInvariant();
39+
Assert.True(
40+
content.Contains("botanica") || content.Contains("garden") || content.Contains("plant"),
41+
$"Expected response to reflect the replaced identity section, but got: {response.Data.Content}");
42+
}
43+
}

‎go/internal/e2e/system_message_sections_e2e_test.go‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
4+
package e2e
5+
6+
import (
7+
"strings"
8+
"testing"
9+
10+
copilot "github.com/github/copilot-sdk/go"
11+
"github.com/github/copilot-sdk/go/internal/e2e/testharness"
12+
)
13+
14+
func TestSystemMessageSectionsE2E(t *testing.T) {
15+
ctx := testharness.NewTestContext(t)
16+
client := ctx.NewClient()
17+
t.Cleanup(func() { client.ForceStop() })
18+
19+
t.Run("should_use_replaced_identity_section_in_response", func(t *testing.T) {
20+
ctx.ConfigureForTest(t)
21+
22+
session, err := client.CreateSession(t.Context(), &copilot.SessionConfig{
23+
OnPermissionRequest: copilot.PermissionHandler.ApproveAll,
24+
SystemMessage: &copilot.SystemMessageConfig{
25+
Mode: "customize",
26+
Sections: map[string]copilot.SectionOverride{
27+
"identity": {
28+
Action: copilot.SectionActionReplace,
29+
Content: "You are a helpful gardening assistant called Botanica. You only answer questions about plants and gardening.",
30+
},
31+
},
32+
},
33+
})
34+
if err != nil {
35+
t.Fatalf("Failed to create session: %v", err)
36+
}
37+
38+
response, err := session.SendAndWait(t.Context(), copilot.MessageOptions{
39+
Prompt: "Who are you?",
40+
})
41+
if err != nil {
42+
t.Fatalf("Failed to send message: %v", err)
43+
}
44+
if response == nil {
45+
t.Fatal("Expected a response from the assistant")
46+
}
47+
48+
ad, ok := response.Data.(*copilot.AssistantMessageData)
49+
if !ok {
50+
t.Fatalf("Expected AssistantMessageData, got %T", response.Data)
51+
}
52+
content := strings.ToLower(ad.Content)
53+
if !strings.Contains(content, "botanica") && !strings.Contains(content, "garden") && !strings.Contains(content, "plant") {
54+
t.Errorf("Expected response to reflect the replaced identity section, but got: %s", ad.Content)
55+
}
56+
})
57+
}

‎nodejs/test/e2e/system_message_sections.e2e.test.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
*--------------------------------------------------------------------------------------------*/
4+
5+
import { describe, expect, it } from "vitest";
6+
import { approveAll } from "../../src/index.js";
7+
import { createSdkTestContext } from "./harness/sdkTestContext.js";
8+
9+
describe("System message sections", async () => {
10+
const { copilotClient: client } = await createSdkTestContext();
11+
12+
it("should_use_replaced_identity_section_in_response", async () => {
13+
const session = await client.createSession({
14+
onPermissionRequest: approveAll,
15+
systemMessage: {
16+
mode: "customize",
17+
sections: {
18+
identity: {
19+
action: "replace",
20+
content:
21+
"You are a helpful gardening assistant called Botanica. You only answer questions about plants and gardening.",
22+
},
23+
},
24+
},
25+
});
26+
27+
const response = await session.sendAndWait({ prompt: "Who are you?" });
28+
29+
expect(response).not.toBeNull();
30+
const content = response!.data.content.toLowerCase();
31+
expect(
32+
content.includes("botanica") || content.includes("garden") || content.includes("plant"),
33+
`Expected response to reflect the replaced identity section, but got: ${response!.data.content}`
34+
).toBe(true);
35+
36+
await session.disconnect();
37+
});
38+
});

‎python/e2e/test_system_message_sections_e2e.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
"""
2+
Copyright (c) Microsoft Corporation.
3+
4+
Tests for system message sections functionality
5+
"""
6+
7+
import pytest
8+
9+
from copilot.session import PermissionHandler
10+
11+
from .testharness import E2ETestContext
12+
13+
pytestmark = pytest.mark.asyncio(loop_scope="module")
14+
15+
16+
class TestSystemMessageSections:
17+
async def test_should_use_replaced_identity_section_in_response(self, ctx: E2ETestContext):
18+
"""Test that replacing the identity section causes the assistant to adopt a new persona"""
19+
session = await ctx.client.create_session(
20+
system_message={
21+
"mode": "customize",
22+
"sections": {
23+
"identity": {
24+
"action": "replace",
25+
"content": (
26+
"You are a helpful gardening assistant called Botanica."
27+
" You only answer questions about plants and gardening."
28+
),
29+
},
30+
},
31+
},
32+
on_permission_request=PermissionHandler.approve_all,
33+
)
34+
35+
response = await session.send_and_wait("Who are you?")
36+
37+
assert response is not None, "Expected a response from the assistant"
38+
content = response.data.content.lower()
39+
assert "botanica" in content or "garden" in content or "plant" in content, (
40+
f"Expected response to reflect the replaced identity section,"
41+
f" but got: {response.data.content}"
42+
)
43+
44+
await session.disconnect()

‎rust/tests/e2e.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ mod subagent_hooks;
119119
mod support;
120120
#[path = "e2e/suspend.rs"]
121121
mod suspend;
122+
#[path = "e2e/system_message_sections.rs"]
123+
mod system_message_sections;
122124
#[path = "e2e/system_message_transform.rs"]
123125
mod system_message_transform;
124126
#[path = "e2e/telemetry.rs"]

‎rust/tests/e2e/system_message_sections.rs‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
use std::collections::HashMap;
2+
3+
use github_copilot_sdk::{SectionOverride, SystemMessageConfig};
4+
5+
use super::support::{assistant_message_content, with_e2e_context};
6+
7+
#[tokio::test]
8+
async fn should_use_replaced_identity_section_in_response() {
9+
with_e2e_context(
10+
"system_message_sections",
11+
"should_use_replaced_identity_section_in_response",
12+
|ctx| {
13+
Box::pin(async move {
14+
ctx.set_default_copilot_user();
15+
let mut sections = HashMap::new();
16+
sections.insert(
17+
"identity".to_string(),
18+
SectionOverride {
19+
action: Some("replace".to_string()),
20+
content: Some(
21+
"You are a helpful gardening assistant called Botanica. \
22+
You only answer questions about plants and gardening."
23+
.to_string(),
24+
),
25+
},
26+
);
27+
let client = ctx.start_client().await;
28+
let session = client
29+
.create_session(
30+
ctx.approve_all_session_config().with_system_message(
31+
SystemMessageConfig::new()
32+
.with_mode("customize")
33+
.with_sections(sections),
34+
),
35+
)
36+
.await
37+
.expect("create session");
38+
39+
let answer = session
40+
.send_and_wait("Who are you?")
41+
.await
42+
.expect("send")
43+
.expect("assistant message");
44+
let content = assistant_message_content(&answer).to_lowercase();
45+
assert!(
46+
content.contains("botanica")
47+
|| content.contains("garden")
48+
|| content.contains("plant"),
49+
"Expected response to reflect the replaced identity section, but got: {}",
50+
assistant_message_content(&answer)
51+
);
52+
53+
session.disconnect().await.expect("disconnect session");
54+
client.stop().await.expect("stop client");
55+
})
56+
},
57+
)
58+
.await;
59+
}

0 commit comments

Comments
 (0)