Skip to content

Commit 3604ba7

Browse files
eavanvalkenburgeavanvalkenburg
authored
Python: Normalize chat finish reasons (microsoft#7105)
* Python: Normalize chat finish reasons Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 45b65bfa-8e36-47b0-99d9-ec58ec60ace1 * Preserve Copilot finish reasons Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 45b65bfa-8e36-47b0-99d9-ec58ec60ace1 * fix claude finish reasons
1 parent 3ab2630 commit 3604ba7

14 files changed

Lines changed: 170 additions & 23 deletions

File tree

‎python/packages/ag-ui/agent_framework_ag_ui/_event_converters.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,6 @@ def _handle_run_error(self, event: dict[str, Any]) -> ChatResponseUpdate:
246246

247247
return ChatResponseUpdate(
248248
role="assistant",
249-
finish_reason="content_filter",
250249
contents=[
251250
Content.from_error(
252251
message=error_message,

‎python/packages/ag-ui/tests/ag_ui/test_event_converters.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,7 @@ def test_run_error_event(self) -> None:
328328

329329
assert update is not None
330330
assert update.role == "assistant"
331-
assert update.finish_reason == "content_filter"
331+
assert update.finish_reason is None
332332
assert len(update.contents) == 1
333333
assert update.contents[0].message == "Connection timeout"
334334
assert update.contents[0].error_code == "RUN_ERROR"

‎python/packages/bedrock/agent_framework_bedrock/_chat_client.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,7 @@ class BedrockChatOptions(ChatOptions[ResponseModelT], Generic[ResponseModelT], t
207207
"max_tokens": "length",
208208
"length": "length",
209209
"content_filtered": "content_filter",
210+
"guardrail_intervened": "content_filter",
210211
"tool_use": "tool_calls",
211212
}
212213

@@ -760,10 +761,10 @@ def _parse_message_contents(self, content_blocks: Sequence[dict[str, Any]]) -> l
760761
logger.debug("Ignoring unsupported Bedrock content block: %s", block)
761762
return contents
762763

763-
def _map_finish_reason(self, reason: str | None) -> FinishReasonLiteral | None:
764+
def _map_finish_reason(self, reason: str | None) -> str | None:
764765
if not reason:
765766
return None
766-
return FINISH_REASON_MAP.get(reason.lower())
767+
return FINISH_REASON_MAP.get(reason.lower(), reason)
767768

768769
def _prepare_output_config(self, response_format: Any | None) -> dict[str, Any] | None:
769770
"""Convert response_format into the AWS Bedrock outputConfig wire format.

‎python/packages/bedrock/tests/test_bedrock_settings.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,12 @@ def test_process_response_parses_tool_use_and_result() -> None:
107107
assert chat_response.finish_reason == client._map_finish_reason("tool_use")
108108

109109

110+
def test_map_finish_reason_maps_guardrail_intervention() -> None:
111+
client = _build_client()
112+
113+
assert client._map_finish_reason("guardrail_intervened") == "content_filter"
114+
115+
110116
def test_process_response_parses_tool_result() -> None:
111117
client = _build_client()
112118
response = {

‎python/packages/claude/agent_framework_claude/_agent.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,14 @@
6868

6969
logger = logging.getLogger("agent_framework.claude")
7070

71+
FINISH_REASON_MAP: dict[str, str] = {
72+
"end_turn": "stop",
73+
"stop_sequence": "stop",
74+
"max_tokens": "length",
75+
"tool_use": "tool_calls",
76+
"refusal": "content_filter",
77+
}
78+
7179

7280
# Name of the in-process MCP server that hosts Agent Framework tools.
7381
# FunctionTool instances are converted to SDK MCP tools and served
@@ -843,7 +851,9 @@ async def _get_stream(
843851
}.items()
844852
if isinstance(value, int)
845853
})
846-
finish_reason = message.stop_reason
854+
finish_reason = (
855+
FINISH_REASON_MAP.get(message.stop_reason, message.stop_reason) if message.stop_reason else None
856+
)
847857
if usage_details or finish_reason:
848858
yield AgentResponseUpdate(
849859
contents=[Content.from_usage(usage_details, raw_representation=message)]

‎python/packages/claude/tests/test_claude_agent.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,13 @@ async def test_run_captures_session_id(self) -> None:
271271
await agent.run("Hello", session=session)
272272
assert session.service_session_id == "test-session-id"
273273

274-
async def test_run_captures_result_message_usage_and_finish_reason(self) -> None:
274+
@pytest.mark.parametrize(
275+
("stop_reason", "expected_finish_reason"),
276+
[("end_turn", "stop"), ("pause_turn", "pause_turn")],
277+
)
278+
async def test_run_captures_result_message_usage_and_finish_reason(
279+
self, stop_reason: str, expected_finish_reason: str
280+
) -> None:
275281
"""Test that ResultMessage metadata is propagated to the final AgentResponse."""
276282
from claude_agent_sdk import AssistantMessage, ResultMessage, TextBlock
277283
from claude_agent_sdk.types import StreamEvent
@@ -296,7 +302,7 @@ async def test_run_captures_result_message_usage_and_finish_reason(self) -> None
296302
is_error=False,
297303
num_turns=1,
298304
session_id="test-session-id",
299-
stop_reason="end_turn",
305+
stop_reason=stop_reason,
300306
usage={
301307
"input_tokens": 42,
302308
"output_tokens": 18,
@@ -311,7 +317,7 @@ async def test_run_captures_result_message_usage_and_finish_reason(self) -> None
311317
agent = ClaudeAgent()
312318
response = await agent.run("Hello")
313319

314-
assert response.finish_reason == "end_turn"
320+
assert response.finish_reason == expected_finish_reason
315321
assert response.usage_details == {
316322
"input_token_count": 42,
317323
"output_token_count": 18,
@@ -465,7 +471,7 @@ async def test_run_stream_final_response_captures_usage_and_finish_reason(self)
465471
pass
466472
response = await stream.get_final_response()
467473

468-
assert response.finish_reason == "max_tokens"
474+
assert response.finish_reason == "length"
469475
assert response.usage_details == {
470476
"input_token_count": 7,
471477
"output_token_count": 9,

‎python/packages/core/agent_framework/observability.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1614,11 +1614,15 @@ async def _finalize_stream() -> None:
16141614
and response.messages
16151615
and span.is_recording()
16161616
):
1617+
finish_reason = cast(
1618+
"FinishReason | None",
1619+
response.finish_reason if response.finish_reason in FINISH_REASON_MAP else None,
1620+
)
16171621
_capture_messages(
16181622
span=span,
16191623
provider_name=provider_name,
16201624
messages=response.messages,
1621-
finish_reason=response.finish_reason, # type: ignore[arg-type]
1625+
finish_reason=finish_reason,
16221626
output=True,
16231627
)
16241628
except Exception as exception:

‎python/packages/github_copilot/agent_framework_github_copilot/_agent.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -635,8 +635,11 @@ def usage_event_handler(event: SessionEvent) -> None:
635635
parsed_usage_details = self._parse_usage_details_from_copilot(event.data)
636636
if parsed_usage_details:
637637
usage_details = add_usage_details(usage_details, parsed_usage_details)
638-
if event.data.finish_reason:
639-
finish_reason = event.data.finish_reason
638+
event_finish_reason = (
639+
"content_filter" if event.data.content_filter_triggered else event.data.finish_reason
640+
)
641+
if event_finish_reason:
642+
finish_reason = event_finish_reason
640643
if event.data.model:
641644
model = event.data.model
642645
else:
@@ -780,7 +783,7 @@ def event_handler(event: SessionEvent) -> None:
780783
)
781784
return
782785
usage_details = self._parse_usage_details_from_copilot(event.data)
783-
finish_reason = event.data.finish_reason or None
786+
finish_reason = "content_filter" if event.data.content_filter_triggered else event.data.finish_reason
784787
model = event.data.model or None
785788
if usage_details or finish_reason or model:
786789
update = AgentResponseUpdate(

‎python/packages/github_copilot/tests/test_github_copilot_agent.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -488,11 +488,18 @@ async def test_run_captures_assistant_usage_event(
488488
input_tokens=5,
489489
output_tokens=2,
490490
finish_reason="length",
491+
content_filter_triggered=True,
491492
),
492493
id=uuid4(),
493494
timestamp=datetime.now(timezone.utc),
494495
type=SessionEventType.ASSISTANT_USAGE,
495496
)
497+
empty_usage_event = SessionEvent(
498+
data=AssistantUsageData(model="gpt-5.1-mini"),
499+
id=uuid4(),
500+
timestamp=datetime.now(timezone.utc),
501+
type=SessionEventType.ASSISTANT_USAGE,
502+
)
496503
usage_handler: Any = None
497504

498505
def mock_on(handler: Any) -> Any:
@@ -503,6 +510,7 @@ def mock_on(handler: Any) -> Any:
503510
async def mock_send_and_wait(*args: Any, **kwargs: Any) -> SessionEvent:
504511
usage_handler(usage_event)
505512
usage_handler(second_usage_event)
513+
usage_handler(empty_usage_event)
506514
return assistant_message_event
507515

508516
mock_session.on = mock_on
@@ -511,7 +519,7 @@ async def mock_send_and_wait(*args: Any, **kwargs: Any) -> SessionEvent:
511519
agent = GitHubCopilotAgent(client=mock_client)
512520
response = await agent.run("Hello")
513521

514-
assert response.finish_reason == "length"
522+
assert response.finish_reason == "content_filter"
515523
assert response.usage_details == {
516524
"input_token_count": 125,
517525
"output_token_count": 42,
@@ -596,7 +604,7 @@ async def test_run_streaming_captures_assistant_usage_event(
596604
model="gpt-5.1-mini",
597605
input_tokens=10,
598606
output_tokens=4,
599-
finish_reason="stop",
607+
finish_reason="provider_specific_reason",
600608
)
601609
usage_event = SessionEvent(
602610
data=usage_data,
@@ -620,7 +628,7 @@ def mock_on(handler: Any) -> Any:
620628
response = await stream.get_final_response()
621629

622630
assert response.text == "Hello"
623-
assert response.finish_reason == "stop"
631+
assert response.finish_reason == "provider_specific_reason"
624632
assert response.usage_details == {
625633
"input_token_count": 10,
626634
"output_token_count": 4,

‎python/packages/ollama/agent_framework_ollama/_chat_client.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
ChatResponse,
2525
ChatResponseUpdate,
2626
Content,
27+
FinishReason,
2728
FunctionInvocationConfiguration,
2829
FunctionInvocationLayer,
2930
FunctionTool,
@@ -542,6 +543,11 @@ def _parse_contents_from_ollama(self, response: OllamaChatResponse) -> list[Cont
542543
contents.extend(tool_calls)
543544
return contents
544545

546+
def _get_finish_reason_from_ollama(self, response: OllamaChatResponse) -> FinishReason | None:
547+
if response.message.tool_calls:
548+
return FinishReason("tool_calls")
549+
return FinishReason(response.done_reason) if response.done_reason else None
550+
545551
def _parse_streaming_response_from_ollama(self, response: OllamaChatResponse) -> ChatResponseUpdate:
546552
contents = self._parse_contents_from_ollama(response)
547553
finish_reason = None
@@ -561,7 +567,7 @@ def _parse_streaming_response_from_ollama(self, response: OllamaChatResponse) ->
561567
)
562568
if usage_details:
563569
contents.append(Content.from_usage(usage_details, raw_representation=response))
564-
finish_reason = response.done_reason if response.done_reason in ("stop", "length") else None
570+
finish_reason = self._get_finish_reason_from_ollama(response)
565571
return ChatResponseUpdate(
566572
contents=contents,
567573
role="assistant",
@@ -590,7 +596,7 @@ def _parse_response_from_ollama(
590596
if isinstance(value, int)
591597
}
592598
)
593-
finish_reason = response.done_reason if response.done_reason in ("stop", "length") else None
599+
finish_reason = self._get_finish_reason_from_ollama(response)
594600

595601
return ChatResponse(
596602
messages=[Message(role="assistant", contents=contents)],

0 commit comments

Comments
 (0)