Skip to content

Commit 2c49f50

Browse files
rogerbarretorogerbarreto
authored
.NET: Preserve streamed annotations in Foundry hosted responses (microsoft#7984)
* .NET: Preserve streamed response annotations Capture annotations delivered separately from text and emit supported citation types through hosted Responses streams. Add local and live streaming and non-streaming coverage for web search and Azure AI Search. Copilot-Session: 0c67da26-d12e-47ab-aac4-53e251ef6edf * .NET: Refine streamed annotation correlation Copilot-Session: 0c67da26-d12e-47ab-aac4-53e251ef6edf * .NET: Align annotation event with OpenAI 2.12 Use the renamed output text annotation event so item correlation remains available after the SDK upgrade. Copilot-Session: 0c67da26-d12e-47ab-aac4-53e251ef6edf * .NET: Update annotation integration tests for SDK changes Use the renamed strongly typed annotation event and unwrap project connection results after the Projects 3 upgrade. Copilot-Session: 0c67da26-d12e-47ab-aac4-53e251ef6edf * .NET: Use hosted Search index in CI Use the dedicated hosted-agent Search index configuration for Foundry hosted integration tests. Copilot-Session: 0c67da26-d12e-47ab-aac4-53e251ef6edf --------- Copilot-Session: 0c67da26-d12e-47ab-aac4-53e251ef6edf
1 parent cc8c1fa commit 2c49f50

14 files changed

Lines changed: 1248 additions & 30 deletions

File tree

‎.github/workflows/dotnet-build-and-test.yml‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -600,12 +600,9 @@ jobs:
600600
env:
601601
AZURE_AI_PROJECT_ENDPOINT: ${{ vars.IT_HOSTED_AGENT_PROJECT_ENDPOINT }}
602602
AZURE_AI_MODEL_DEPLOYMENT_NAME: ${{ vars.IT_HOSTED_AGENT_MODEL_DEPLOYMENT_NAME }}
603-
# Azure AI Search (for the azure-search-rag scenario). Reuses the integration
604-
# environment secrets shared with python-sample-validation.yml. The index is
605-
# provisioned out of band; see dotnet/tests/Foundry.Hosting.IntegrationTests/README.md
606-
# for the required schema and seed content.
603+
# Use the dedicated hosted-agent Search index for both Search scenarios.
607604
AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }}
608-
AZURE_SEARCH_INDEX_NAME: ${{ secrets.AZURE_SEARCH_INDEX_NAME }}
605+
AZURE_SEARCH_INDEX_NAME: ${{ vars.FOUNDRY_HOSTED_AGENT_SEARCH_INDEX_NAME }}
609606
# IT_HOSTED_AGENT_IMAGE was exported into $GITHUB_ENV by the previous step.
610607

611608
# This final job is required to satisfy the merge queue. It must only run (or succeed) if no tests failed

‎dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs‎

Lines changed: 115 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,14 @@
1515
using Microsoft.Agents.AI.Workflows;
1616
using Microsoft.Extensions.AI;
1717
using MeaiTextContent = Microsoft.Extensions.AI.TextContent;
18+
using OpenAIContainerFileCitationMessageAnnotation = OpenAI.Responses.ContainerFileCitationMessageAnnotation;
19+
using OpenAIFileCitationMessageAnnotation = OpenAI.Responses.FileCitationMessageAnnotation;
20+
using OpenAIFilePathMessageAnnotation = OpenAI.Responses.FilePathMessageAnnotation;
21+
using OpenAIMessageResponseItem = OpenAI.Responses.MessageResponseItem;
22+
using OpenAIStreamingResponseOutputItemAddedUpdate = OpenAI.Responses.StreamingResponseOutputItemAddedUpdate;
23+
using OpenAIStreamingResponseOutputItemDoneUpdate = OpenAI.Responses.StreamingResponseOutputItemDoneUpdate;
24+
using OpenAIStreamingResponseOutputTextAnnotationAddedUpdate = OpenAI.Responses.StreamingResponseOutputTextAnnotationAddedUpdate;
25+
using OpenAIStreamingResponseOutputTextDeltaUpdate = OpenAI.Responses.StreamingResponseOutputTextDeltaUpdate;
1826

1927
namespace Microsoft.Agents.AI.Foundry.Hosting;
2028

@@ -100,13 +108,14 @@ public static async IAsyncEnumerable<ResponseStreamEvent> ConvertUpdatesToEvents
100108
continue;
101109
}
102110

111+
string? currentMessageId = ResolveMessageId(update);
103112
foreach (var content in update.Contents)
104113
{
105114
switch (content)
106115
{
107116
case MeaiTextContent textContent:
108117
{
109-
if (!IsSameMessage(update.MessageId, previousMessageId) && currentMessageBuilder is not null)
118+
if (!IsSameMessage(currentMessageId, previousMessageId) && currentMessageBuilder is not null)
110119
{
111120
foreach (var evt in CloseCurrentMessage(currentMessageBuilder, currentTextBuilder, accumulatedText, accumulatedAnnotations))
112121
{
@@ -119,7 +128,10 @@ public static async IAsyncEnumerable<ResponseStreamEvent> ConvertUpdatesToEvents
119128
accumulatedAnnotations = null;
120129
}
121130

122-
previousMessageId = update.MessageId;
131+
if (currentMessageId is { Length: > 0 })
132+
{
133+
previousMessageId = currentMessageId;
134+
}
123135

124136
if (currentMessageBuilder is null)
125137
{
@@ -138,14 +150,6 @@ public static async IAsyncEnumerable<ResponseStreamEvent> ConvertUpdatesToEvents
138150
yield return currentTextBuilder!.EmitDelta(textContent.Text);
139151
}
140152

141-
if (textContent.Annotations is { Count: > 0 })
142-
{
143-
foreach (var sdkAnnotation in ConvertToSdkAnnotations(textContent.Annotations))
144-
{
145-
(accumulatedAnnotations ??= []).Add(sdkAnnotation);
146-
}
147-
}
148-
149153
break;
150154
}
151155

@@ -336,6 +340,27 @@ public static async IAsyncEnumerable<ResponseStreamEvent> ConvertUpdatesToEvents
336340
default:
337341
break;
338342
}
343+
344+
var isTextContent = content is MeaiTextContent;
345+
var isAnnotationOnlyContentForCurrentMessage =
346+
content.GetType() == typeof(AIContent) &&
347+
IsSameAnnotationMessage(currentMessageId, previousMessageId);
348+
349+
// MEAI OpenAI sends streaming citations in a separate annotation-only AIContent after
350+
// the text deltas. Accumulate them because AgentServer emits annotations only after
351+
// output_text.done, and de-duplicate providers that report the same citation twice.
352+
if ((isTextContent || isAnnotationOnlyContentForCurrentMessage)
353+
&& content.Annotations is { Count: > 0 }
354+
&& currentMessageBuilder is not null)
355+
{
356+
foreach (var sdkAnnotation in ConvertToSdkAnnotations(content.Annotations))
357+
{
358+
if (accumulatedAnnotations?.Any(existing => AreEquivalentAnnotations(existing, sdkAnnotation)) is not true)
359+
{
360+
(accumulatedAnnotations ??= []).Add(sdkAnnotation);
361+
}
362+
}
363+
}
339364
}
340365
}
341366

@@ -385,16 +410,93 @@ private static IEnumerable<ResponseStreamEvent> CloseCurrentMessage(
385410
private static bool IsSameMessage(string? currentId, string? previousId) =>
386411
currentId is not { Length: > 0 } || previousId is not { Length: > 0 } || currentId == previousId;
387412

413+
private static bool IsSameAnnotationMessage(string? currentId, string? previousId) =>
414+
currentId is { Length: > 0 }
415+
? currentId == previousId
416+
: previousId is not { Length: > 0 };
417+
418+
private static string? ResolveMessageId(AgentResponseUpdate update)
419+
{
420+
if (update.MessageId is { Length: > 0 })
421+
{
422+
return update.MessageId;
423+
}
424+
425+
object? rawRepresentation = update.RawRepresentation is ChatResponseUpdate chatUpdate
426+
? chatUpdate.RawRepresentation
427+
: update.RawRepresentation;
428+
429+
return rawRepresentation switch
430+
{
431+
OpenAIStreamingResponseOutputTextAnnotationAddedUpdate annotationAdded => annotationAdded.ItemId,
432+
OpenAIStreamingResponseOutputTextDeltaUpdate textDelta => textDelta.ItemId,
433+
OpenAIStreamingResponseOutputItemAddedUpdate { Item: OpenAIMessageResponseItem message } => message.Id,
434+
OpenAIStreamingResponseOutputItemDoneUpdate { Item: OpenAIMessageResponseItem message } => message.Id,
435+
_ => null,
436+
};
437+
}
438+
439+
private static bool AreEquivalentAnnotations(Annotation left, Annotation right) =>
440+
(left, right) switch
441+
{
442+
(UrlCitationBody leftUrl, UrlCitationBody rightUrl) =>
443+
leftUrl.Url == rightUrl.Url &&
444+
leftUrl.StartIndex == rightUrl.StartIndex &&
445+
leftUrl.EndIndex == rightUrl.EndIndex &&
446+
leftUrl.Title == rightUrl.Title,
447+
(FileCitationBody leftFile, FileCitationBody rightFile) =>
448+
leftFile.FileId == rightFile.FileId &&
449+
leftFile.Index == rightFile.Index &&
450+
leftFile.Filename == rightFile.Filename,
451+
(ContainerFileCitationBody leftContainerFile, ContainerFileCitationBody rightContainerFile) =>
452+
leftContainerFile.ContainerId == rightContainerFile.ContainerId &&
453+
leftContainerFile.FileId == rightContainerFile.FileId &&
454+
leftContainerFile.StartIndex == rightContainerFile.StartIndex &&
455+
leftContainerFile.EndIndex == rightContainerFile.EndIndex &&
456+
leftContainerFile.Filename == rightContainerFile.Filename,
457+
(FilePath leftFilePath, FilePath rightFilePath) =>
458+
leftFilePath.FileId == rightFilePath.FileId &&
459+
leftFilePath.Index == rightFilePath.Index,
460+
_ => false,
461+
};
462+
388463
/// <summary>
389464
/// Converts MEAI <see cref="AIAnnotation"/> instances to Responses SDK <see cref="Annotation"/> objects.
390-
/// Only <see cref="CitationAnnotation"/> with a URL and at least one <see cref="TextSpanAnnotatedRegion"/>
391-
/// with explicit start/end indices is converted; all other shapes are skipped.
465+
/// Only supported <see cref="CitationAnnotation"/> shapes are converted; all others are skipped.
392466
/// </summary>
393467
private static IEnumerable<Annotation> ConvertToSdkAnnotations(IList<AIAnnotation> annotations)
394468
{
395469
foreach (var ann in annotations)
396470
{
397-
if (ann is not CitationAnnotation citation || citation.Url is null)
471+
if (ann is not CitationAnnotation citation)
472+
{
473+
continue;
474+
}
475+
476+
if (citation.RawRepresentation is OpenAIContainerFileCitationMessageAnnotation containerFileCitation)
477+
{
478+
yield return new ContainerFileCitationBody(
479+
containerFileCitation.ContainerId,
480+
containerFileCitation.FileId,
481+
containerFileCitation.StartIndex,
482+
containerFileCitation.EndIndex,
483+
containerFileCitation.Filename);
484+
continue;
485+
}
486+
487+
if (citation.RawRepresentation is OpenAIFileCitationMessageAnnotation fileCitation)
488+
{
489+
yield return new FileCitationBody(fileCitation.FileId, fileCitation.Index, fileCitation.Filename);
490+
continue;
491+
}
492+
493+
if (citation.RawRepresentation is OpenAIFilePathMessageAnnotation filePath)
494+
{
495+
yield return new FilePath(filePath.FileId, filePath.Index);
496+
continue;
497+
}
498+
499+
if (citation.Url is null)
398500
{
399501
continue;
400502
}

‎dotnet/src/Shared/IntegrationTests/TestSettings.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ internal static class TestSettings
2222
public const string AzureAIProjectEndpoint = "AZURE_AI_PROJECT_ENDPOINT";
2323

2424
// Azure AI Search (Foundry.Hosting integration tests, RAG scenario)
25+
public const string AzureSearchConnectionId = "AZURE_SEARCH_CONNECTION_ID";
26+
public const string AzureSearchConnectionName = "AZURE_SEARCH_CONNECTION_NAME";
2527
public const string AzureSearchEndpoint = "AZURE_SEARCH_ENDPOINT";
2628
public const string AzureSearchIndexName = "AZURE_SEARCH_INDEX_NAME";
2729

‎dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/Program.cs‎

Lines changed: 61 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.ComponentModel;
44
using Azure;
55
using Azure.AI.Projects;
6+
using Azure.AI.Projects.Agents;
67
using Azure.Identity;
78
using Azure.Search.Documents;
89
using Azure.Search.Documents.Models;
@@ -43,6 +44,8 @@
4344
"custom-storage" => CreateCustomStorageAgent(projectClient, deployment),
4445
"memory" => await CreateMemoryAgentAsync(projectClient, deployment).ConfigureAwait(false),
4546
"azure-search-rag" => CreateAzureSearchRagAgent(projectClient, deployment),
47+
"azure-search-tool-annotations" => CreateAzureSearchToolAnnotationsAgent(projectClient, deployment),
48+
"web-search-annotations" => CreateWebSearchAnnotationsAgent(projectClient, deployment),
4649
"session-files" => CreateSessionFilesAgent(projectClient, deployment),
4750
"agent-skills" => CreateAgentSkillsAgent(projectClient, deployment),
4851
"user-identity" => CreateUserIdentityAgent(projectClient, deployment),
@@ -77,14 +80,12 @@
7780
options.SteerableConversations = scenario == "steerable-long-running";
7881
});
7982

80-
// toolbox-oauth-consent scenario: pre-register a Foundry toolbox whose tool source is fronted by a
81-
// per-user OAuth connection. IT_TOOLBOX_NAME names that toolbox (the fixture sets it). With the
82-
// startup-deferral fix the container stays routable even though the toolbox cannot enumerate without
83-
// a consented user, and the first user request surfaces an oauth_consent_request.
84-
var consentToolboxName = Environment.GetEnvironmentVariable("IT_TOOLBOX_NAME");
85-
if (!string.IsNullOrEmpty(consentToolboxName))
83+
// Scenarios that consume a project toolbox set IT_TOOLBOX_NAME through their fixture.
84+
// The hosting bridge resolves the toolbox through its MCP endpoint and adds its tools to every request.
85+
var toolboxName = Environment.GetEnvironmentVariable("IT_TOOLBOX_NAME");
86+
if (!string.IsNullOrEmpty(toolboxName))
8687
{
87-
builder.Services.AddFoundryToolboxes(credential, consentToolboxName);
88+
builder.Services.AddFoundryToolboxes(credential, toolboxName);
8889
}
8990

9091
var app = builder.Build();
@@ -207,6 +208,59 @@ static AIAgent CreateAzureSearchRagAgent(AIProjectClient client, string deployme
207208
});
208209
}
209210

211+
static AIAgent CreateWebSearchAnnotationsAgent(AIProjectClient client, string deployment) =>
212+
client.AsAIAgent(new ChatClientAgentOptions
213+
{
214+
Name = "web-search-annotations-agent",
215+
Description = "Hosted web search annotation test agent.",
216+
ChatOptions = new ChatOptions
217+
{
218+
ModelId = deployment,
219+
Instructions = """
220+
Answer with current information from the web search results.
221+
Include citations for the sources used in the answer.
222+
""",
223+
Tools = [new HostedWebSearchTool()],
224+
ToolMode = ChatToolMode.RequireAny,
225+
},
226+
});
227+
228+
static AIAgent CreateAzureSearchToolAnnotationsAgent(AIProjectClient client, string deployment)
229+
{
230+
var connectionId = Environment.GetEnvironmentVariable("AZURE_SEARCH_CONNECTION_ID")
231+
?? throw new InvalidOperationException(
232+
"AZURE_SEARCH_CONNECTION_ID is not set for IT_SCENARIO=azure-search-tool-annotations.");
233+
var indexName = Environment.GetEnvironmentVariable("AZURE_SEARCH_INDEX_NAME")
234+
?? throw new InvalidOperationException(
235+
"AZURE_SEARCH_INDEX_NAME is not set for IT_SCENARIO=azure-search-tool-annotations.");
236+
var searchTool = FoundryAITool.CreateAzureAISearchTool(new AzureAISearchToolOptions(
237+
[
238+
new AzureAISearchToolIndex
239+
{
240+
ProjectConnectionId = connectionId,
241+
IndexName = indexName,
242+
QueryType = AzureAISearchQueryType.Simple,
243+
TopK = 3,
244+
}
245+
]));
246+
247+
return client.AsAIAgent(new ChatClientAgentOptions
248+
{
249+
Name = "azure-search-tool-annotations-agent",
250+
Description = "Azure AI Search hosted tool annotation test agent.",
251+
ChatOptions = new ChatOptions
252+
{
253+
ModelId = deployment,
254+
Instructions = """
255+
Answer only from the Azure AI Search results.
256+
Include citations for the sources used in the answer.
257+
""",
258+
Tools = [searchTool],
259+
ToolMode = ChatToolMode.RequireAny,
260+
},
261+
});
262+
}
263+
210264
static Func<string, CancellationToken, Task<IEnumerable<TextSearchProvider.TextSearchResult>>>
211265
CreateAzureSearchAdapter(SearchClient client, int top = 3) =>
212266
async (query, cancellationToken) =>

0 commit comments

Comments
 (0)