Skip to content

Commit 8c05750

Browse files
giles17Copilot
giles17
and
Copilot
authored
Python: Forward GitHub Copilot input attachments as inline blobs (microsoft#7300)
* Python: Forward GitHub Copilot input attachments as inline blobs The Python GitHubCopilotAgent built the prompt from message text only, so DataContent (images/documents) passed on input was silently dropped. The .NET provider already forwards these as attachments. Map input data content to the Copilot SDK's inline BlobAttachment (base64, no temp files) in both the streaming and non-streaming send paths. Data content without a media type is dropped with a warning instead of silently. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ac85c429-4115-42ef-a18a-f576e3cf03f7 * Python: Handle non-base64 data URIs and fix attachment docstring Address PR review feedback: - Guard _get_data_bytes_as_str against ContentError so a non-base64 data: URI (which _validate_uri still classifies as type="data") is skipped with a warning instead of failing the entire Copilot request. - Correct the docstring: remote URIs and non-base64 data URIs are neither attached nor added to the prompt (the prompt is built from text content only). - Add tests for the non-base64 data URI path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ac85c429-4115-42ef-a18a-f576e3cf03f7 * Python: Fix flaky attachment test under telemetry The end-to-end non-base64 data URI test failed in CI because GitHubCopilotAgent's telemetry layer serializes message content (observability._to_otel_part -> _get_data_bytes_as_str), which raises ContentError on a non-base64 data: URI before the attachment code runs. That is an unrelated core-observability limitation, not attachment behavior. Use RawGitHubCopilotAgent (no telemetry layer) for that test so it isolates the provider's send path. The direct helper test still covers the ContentError guard. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ac85c429-4115-42ef-a18a-f576e3cf03f7 --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: ac85c429-4115-42ef-a18a-f576e3cf03f7
1 parent 0d5c0f8 commit 8c05750

2 files changed

Lines changed: 186 additions & 6 deletions

File tree

‎python/packages/github_copilot/agent_framework_github_copilot/_agent.py‎

Lines changed: 62 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,12 @@
3030
)
3131
from agent_framework._settings import load_settings
3232
from agent_framework._tools import FunctionTool, ToolTypes
33-
from agent_framework._types import AgentRunInputs, normalize_tools
34-
from agent_framework.exceptions import AgentException
33+
from agent_framework._types import (
34+
AgentRunInputs,
35+
_get_data_bytes_as_str, # pyright: ignore[reportPrivateUsage]
36+
normalize_tools,
37+
)
38+
from agent_framework.exceptions import AgentException, ContentError
3539
from agent_framework.observability import AgentTelemetryLayer
3640

3741
if sys.version_info >= (3, 11):
@@ -47,6 +51,8 @@
4751
from copilot import CopilotClient, CopilotSession, RuntimeConnection
4852
from copilot.generated.rpc import PermissionDecisionUserNotAvailable
4953
from copilot.session import (
54+
Attachment,
55+
BlobAttachment,
5056
MCPServerConfig,
5157
PermissionRequestResult,
5258
PreToolUseHandler,
@@ -656,10 +662,11 @@ def usage_event_handler(event: SessionEvent) -> None:
656662
prompt = "\n".join([message.text for message in context_messages])
657663
if session_context.instructions:
658664
prompt = "\n".join(session_context.instructions) + "\n" + prompt
665+
attachments = self._prepare_attachments_for_copilot(context_messages)
659666

660667
unsubscribe = copilot_session.on(usage_event_handler)
661668
try:
662-
response_event = await copilot_session.send_and_wait(prompt, timeout=timeout)
669+
response_event = await copilot_session.send_and_wait(prompt, attachments=attachments, timeout=timeout)
663670
except Exception as ex:
664671
raise AgentException(f"GitHub Copilot request failed: {ex}") from ex
665672
finally:
@@ -762,6 +769,7 @@ async def _stream_updates(
762769
prompt = "\n".join([message.text for message in context_messages])
763770
if session_context.instructions:
764771
prompt = "\n".join(session_context.instructions) + "\n" + prompt
772+
attachments = self._prepare_attachments_for_copilot(context_messages)
765773

766774
queue: asyncio.Queue[AgentResponseUpdate | Exception | None] = asyncio.Queue()
767775

@@ -844,7 +852,7 @@ def event_handler(event: SessionEvent) -> None:
844852
unsubscribe = copilot_session.on(event_handler)
845853

846854
try:
847-
await copilot_session.send(prompt)
855+
await copilot_session.send(prompt, attachments=attachments)
848856

849857
while (item := await queue.get()) is not None:
850858
if isinstance(item, Exception):
@@ -916,6 +924,56 @@ def _prepare_system_message(
916924
elif opts_system_message is not None:
917925
opts["system_message"] = opts_system_message
918926

927+
@staticmethod
928+
def _prepare_attachments_for_copilot(messages: Sequence[Message]) -> list[Attachment] | None:
929+
"""Convert inline binary message content into Copilot SDK attachments.
930+
931+
Scans the outgoing messages for ``data`` content (binary payloads such as
932+
images or documents carried as base64 data URIs) and maps each one to an
933+
inline ``blob`` attachment understood by the Copilot SDK.
934+
935+
Only base64 ``data:`` content is forwarded as an attachment. Other content
936+
is not turned into an attachment: text content is already carried in the
937+
prompt, while remote URIs (for example ``https://`` links) and malformed or
938+
non-base64 ``data:`` URIs are skipped -- they are neither attached nor added
939+
to the prompt.
940+
941+
Args:
942+
messages: The messages being sent to the Copilot session.
943+
944+
Returns:
945+
A list of Copilot ``Attachment`` objects, or ``None`` when the messages
946+
contain no attachable binary content.
947+
"""
948+
attachments: list[Attachment] = []
949+
for message in messages:
950+
for content in message.contents:
951+
if content.type != "data":
952+
continue
953+
try:
954+
data_str = _get_data_bytes_as_str(content)
955+
except ContentError:
956+
logger.warning(
957+
"Skipping GitHub Copilot attachment with an unsupported data URI; "
958+
"only base64-encoded 'data:' URIs can be forwarded as attachments."
959+
)
960+
continue
961+
if not data_str:
962+
continue
963+
if not content.media_type:
964+
logger.warning(
965+
"Dropping GitHub Copilot attachment with no media type; the Copilot SDK "
966+
"requires a MIME type for inline binary content."
967+
)
968+
continue
969+
blob: BlobAttachment = {
970+
"type": "blob",
971+
"data": data_str,
972+
"mimeType": content.media_type,
973+
}
974+
attachments.append(blob)
975+
return attachments or None
976+
919977
def _prepare_tools(
920978
self,
921979
tools: Sequence[ToolTypes | CopilotTool],

‎python/packages/github_copilot/tests/test_github_copilot_agent.py‎

Lines changed: 124 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
# ruff: noqa: E402
44

5+
import base64
56
import inspect
67
import os
78
import unittest.mock
@@ -37,7 +38,7 @@
3738
)
3839
from copilot.tools import ToolInvocation, ToolResult
3940

40-
from agent_framework_github_copilot import GitHubCopilotAgent, GitHubCopilotOptions
41+
from agent_framework_github_copilot import GitHubCopilotAgent, GitHubCopilotOptions, RawGitHubCopilotAgent
4142

4243

4344
def copilot_options(options: GitHubCopilotOptions) -> GitHubCopilotOptions:
@@ -3351,10 +3352,131 @@ def load_skill(skill_name: str) -> str:
33513352
assert "load_skill" in tool_names
33523353

33533354

3355+
class TestGitHubCopilotAttachments:
3356+
"""Tests for forwarding inline binary message content as Copilot attachments."""
3357+
3358+
async def test_data_content_forwarded_as_blob_attachment(
3359+
self,
3360+
mock_client: MagicMock,
3361+
mock_session: MagicMock,
3362+
assistant_message_event: SessionEvent,
3363+
) -> None:
3364+
"""Non-streaming: DataContent is sent to the SDK as an inline blob attachment."""
3365+
mock_session.send_and_wait.return_value = assistant_message_event
3366+
image_bytes = b"\x89PNG\r\n\x1a\n-fake-image"
3367+
message = Message(
3368+
role="user",
3369+
contents=[
3370+
Content.from_text("Describe this image"),
3371+
Content.from_data(data=image_bytes, media_type="image/png"),
3372+
],
3373+
)
3374+
3375+
agent = GitHubCopilotAgent(client=mock_client)
3376+
await agent.run(message)
3377+
3378+
attachments = mock_session.send_and_wait.call_args.kwargs["attachments"]
3379+
assert attachments is not None
3380+
assert len(attachments) == 1
3381+
assert attachments[0]["type"] == "blob"
3382+
assert attachments[0]["mimeType"] == "image/png"
3383+
assert base64.b64decode(attachments[0]["data"]) == image_bytes
3384+
3385+
async def test_data_content_forwarded_as_blob_attachment_streaming(
3386+
self,
3387+
mock_client: MagicMock,
3388+
mock_session: MagicMock,
3389+
session_idle_event: SessionEvent,
3390+
) -> None:
3391+
"""Streaming: DataContent is sent to the SDK as an inline blob attachment."""
3392+
3393+
def mock_on(handler: Any) -> Any:
3394+
handler(session_idle_event)
3395+
return lambda: None
3396+
3397+
mock_session.on = mock_on
3398+
image_bytes = b"\x89PNG\r\n\x1a\n-fake-image"
3399+
message = Message(
3400+
role="user",
3401+
contents=[
3402+
Content.from_text("Describe this image"),
3403+
Content.from_data(data=image_bytes, media_type="image/png"),
3404+
],
3405+
)
3406+
3407+
agent = GitHubCopilotAgent(client=mock_client)
3408+
async for _ in agent.run(message, stream=True):
3409+
pass
3410+
3411+
attachments = mock_session.send.call_args.kwargs["attachments"]
3412+
assert attachments is not None
3413+
assert len(attachments) == 1
3414+
assert attachments[0]["type"] == "blob"
3415+
assert attachments[0]["mimeType"] == "image/png"
3416+
assert base64.b64decode(attachments[0]["data"]) == image_bytes
3417+
3418+
async def test_text_only_message_sends_no_attachments(
3419+
self,
3420+
mock_client: MagicMock,
3421+
mock_session: MagicMock,
3422+
assistant_message_event: SessionEvent,
3423+
) -> None:
3424+
"""A text-only message results in no attachments being forwarded."""
3425+
mock_session.send_and_wait.return_value = assistant_message_event
3426+
3427+
agent = GitHubCopilotAgent(client=mock_client)
3428+
await agent.run("Just text, no attachments")
3429+
3430+
assert mock_session.send_and_wait.call_args.kwargs["attachments"] is None
3431+
3432+
def test_prepare_attachments_skips_data_without_media_type(self) -> None:
3433+
"""Data content lacking a media type is dropped rather than sent without a MIME type."""
3434+
content = Content.from_data(data=b"payload", media_type="application/octet-stream")
3435+
content.media_type = None
3436+
message = Message(role="user", contents=[content])
3437+
3438+
attachments = GitHubCopilotAgent._prepare_attachments_for_copilot([message])
3439+
3440+
assert attachments is None
3441+
3442+
async def test_non_base64_data_uri_is_skipped_not_raised(
3443+
self,
3444+
mock_client: MagicMock,
3445+
mock_session: MagicMock,
3446+
assistant_message_event: SessionEvent,
3447+
) -> None:
3448+
"""A non-base64 ``data:`` URI is skipped by the send path instead of failing the request.
3449+
3450+
Uses ``RawGitHubCopilotAgent`` (no telemetry layer) to isolate the provider's own
3451+
attachment handling. The telemetry layer in ``GitHubCopilotAgent`` independently
3452+
serializes message content and would trip a separate core limitation on this
3453+
contrived input, which is unrelated to attachment forwarding.
3454+
"""
3455+
mock_session.send_and_wait.return_value = assistant_message_event
3456+
# ``Content.from_uri`` classifies this as type="data" but it is not base64-encoded,
3457+
# so extracting its bytes raises ContentError internally.
3458+
non_base64 = Content.from_uri("data:text/plain,hello")
3459+
assert non_base64.type == "data"
3460+
message = Message(role="user", contents=[Content.from_text("hi"), non_base64])
3461+
3462+
agent = RawGitHubCopilotAgent(client=mock_client)
3463+
# Should complete without raising.
3464+
await agent.run(message)
3465+
3466+
assert mock_session.send_and_wait.call_args.kwargs["attachments"] is None
3467+
3468+
def test_prepare_attachments_skips_non_base64_data_uri(self) -> None:
3469+
"""The helper drops a non-base64 ``data:`` URI rather than raising ContentError."""
3470+
message = Message(role="user", contents=[Content.from_uri("data:text/plain,hello")])
3471+
3472+
attachments = GitHubCopilotAgent._prepare_attachments_for_copilot([message])
3473+
3474+
assert attachments is None
3475+
3476+
33543477
# ---------------------------------------------------------------------------
33553478
# Integration tests — require COPILOT_GITHUB_TOKEN env var
33563479
# ---------------------------------------------------------------------------
3357-
33583480
skip_if_copilot_integration_tests_disabled = pytest.mark.skipif(
33593481
os.getenv("COPILOT_GITHUB_TOKEN", "") == "",
33603482
reason="No COPILOT_GITHUB_TOKEN provided; skipping integration tests.",

0 commit comments

Comments
 (0)