Skip to content

Commit fb2fc6b

Browse files
moonbox3moonbox3
authored
Python: Fix ChatCompletionAgent intermediate message ordering for invoke and invoke_stream (microsoft#12000)
### Motivation and Context When using the `on_intermediate_message` callback, we are returning messages to the caller in an incorrect order. While using function calling, the order is: 1. Provide user input to agent that requires a tool call 2. FunctionCallContent chunks returned via callback 3. FunctionResultContent returned via callback 4. Final output (StreamingTextContent/TextContent) from agent returned via invoke_stream/invoke What we are really seeing is: 1. Provide user input to agent that requires a tool call 2. Final output (StreamingTextContent/TextContent) from agent returned via invoke_stream/invoke 3. FunctionCallContent chunks returned via callback 4. FunctionResultContent returned via callback The callback needs a bit of adjusting -- for the streaming case it must emit messages as part of handling the AsyncGen response_list loop. <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> ### Description Fix the ordering of the intermediate callback so messages aren't received out of order. - Adds unit tests to check message order - Adjusts some samples - Closes microsoft#11998 <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [X] The code builds clean without any errors or warnings - [X] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [X] All unit tests pass, and I have added new tests where possible - [X] I didn't break anyone 😄
1 parent fa91988 commit fb2fc6b

4 files changed

Lines changed: 203 additions & 91 deletions

File tree

‎python/samples/concepts/agents/chat_completion_agent/chat_completion_agent_message_callback.py‎

Lines changed: 26 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,18 @@ def get_item_price(
4343
intermediate_steps: list[ChatMessageContent] = []
4444

4545

46+
# This callback function will be called for each intermediate message
47+
# Which will allow one to handle FunctionCallContent and FunctionResultContent
48+
# If the callback is not provided, the agent will return the final response
49+
# with no intermediate tool call steps.
4650
async def handle_intermediate_steps(message: ChatMessageContent) -> None:
47-
intermediate_steps.append(message)
51+
for item in message.items or []:
52+
if isinstance(item, FunctionCallContent):
53+
print(f"Function Call:> {item.name} with arguments: {item.arguments}")
54+
elif isinstance(item, FunctionResultContent):
55+
print(f"Function Result:> {item.result} for function: {item.name}")
56+
else:
57+
print(f"{message.role}: {message.content}")
4858

4959

5060
async def main() -> None:
@@ -77,43 +87,26 @@ async def main() -> None:
7787
print(f"# {response.role}: {response}")
7888
thread = response.thread
7989

80-
# Print the intermediate steps
81-
print("\nIntermediate Steps:")
82-
for msg in intermediate_steps:
83-
if any(isinstance(item, FunctionResultContent) for item in msg.items):
84-
for fr in msg.items:
85-
if isinstance(fr, FunctionResultContent):
86-
print(f"Function Result:> {fr.result} for function: {fr.name}")
87-
elif any(isinstance(item, FunctionCallContent) for item in msg.items):
88-
for fcc in msg.items:
89-
if isinstance(fcc, FunctionCallContent):
90-
print(f"Function Call:> {fcc.name} with arguments: {fcc.arguments}")
91-
else:
92-
print(f"{msg.role}: {msg.content}")
90+
"""
91+
Sample Output:
9392
94-
# Sample output:
9593
# User: 'Hello'
96-
# AuthorRole.ASSISTANT: Hello! How can I assist you today? If you have any questions about the menu,
97-
# feel free to ask!
94+
# AuthorRole.ASSISTANT: Hi there! How can I assist you today?
9895
# User: 'What is the special soup?'
99-
# AuthorRole.ASSISTANT: The special soup is Clam Chowder. Would you like to know more about it or
100-
# anything else on the menu?
96+
Function Call:> MenuPlugin-get_specials with arguments: {}
97+
Function Result:>
98+
Special Soup: Clam Chowder
99+
Special Salad: Cobb Salad
100+
Special Drink: Chai Tea
101+
for function: MenuPlugin-get_specials
102+
# AuthorRole.ASSISTANT: The special soup today is Clam Chowder. Would you like to know anything else from the menu?
101103
# User: 'How much does that cost?'
102-
# AuthorRole.ASSISTANT: The Clam Chowder costs $9.99. If you have any other questions or need further
103-
# information, feel free to ask!
104+
Function Call:> MenuPlugin-get_item_price with arguments: {"menu_item":"Clam Chowder"}
105+
Function Result:> $9.99 for function: MenuPlugin-get_item_price
106+
# AuthorRole.ASSISTANT: The Clam Chowder costs $9.99. Would you like to know more about the menu or anything else?
104107
# User: 'Thank you'
105-
# AuthorRole.ASSISTANT: You're welcome! If you have any more questions in the future, don't hesitate
106-
# to ask. Have a great day!
107-
#
108-
# Intermediate Steps:
109-
# Function Call:> MenuPlugin-get_specials with arguments: {}
110-
# Function Result:>
111-
# Special Soup: Clam Chowder
112-
# Special Salad: Cobb Salad
113-
# Special Drink: Chai Tea
114-
# for function: MenuPlugin-get_specials
115-
# Function Call:> MenuPlugin-get_item_price with arguments: {"menu_item":"Clam Chowder"}
116-
# Function Result:> $9.99 for function: MenuPlugin-get_item_price
108+
# AuthorRole.ASSISTANT: You're welcome! If you have any more questions, feel free to ask. Enjoy your day!
109+
"""
117110

118111

119112
if __name__ == "__main__":

‎python/samples/concepts/agents/chat_completion_agent/chat_completion_agent_message_callback_streaming.py‎

Lines changed: 29 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,18 @@ def get_item_price(
4040
intermediate_steps: list[ChatMessageContent] = []
4141

4242

43+
# This callback function will be called for each intermediate message
44+
# Which will allow one to handle FunctionCallContent and FunctionResultContent
45+
# If the callback is not provided, the agent will return the final response
46+
# with no intermediate tool call steps.
4347
async def handle_streaming_intermediate_steps(message: ChatMessageContent) -> None:
44-
intermediate_steps.append(message)
48+
for item in message.items or []:
49+
if isinstance(item, FunctionCallContent):
50+
print(f"Function Call:> {item.name} with arguments: {item.arguments}")
51+
elif isinstance(item, FunctionResultContent):
52+
print(f"Function Result:> {item.result} for function: {item.name}")
53+
else:
54+
print(f"{message.role}: {message.content}")
4555

4656

4757
async def main() -> None:
@@ -66,57 +76,39 @@ async def main() -> None:
6676

6777
for user_input in user_inputs:
6878
print(f"\n# User: '{user_input}'")
69-
first_chunk = True
7079
async for response in agent.invoke_stream(
7180
messages=user_input,
7281
thread=thread,
7382
on_intermediate_message=handle_streaming_intermediate_steps,
7483
):
75-
if first_chunk:
76-
print(f"# {response.name}: ", end="", flush=True)
77-
first_chunk = False
7884
if response.content:
7985
print(response.content, end="", flush=True)
8086
thread = response.thread
8187
print()
8288

83-
# Print the intermediate steps
84-
print("\nIntermediate Steps:")
85-
for msg in intermediate_steps:
86-
if any(isinstance(item, FunctionResultContent) for item in msg.items):
87-
for fr in msg.items:
88-
if isinstance(fr, FunctionResultContent):
89-
print(f"Function Result:> {fr.result} for function: {fr.name}")
90-
elif any(isinstance(item, FunctionCallContent) for item in msg.items):
91-
for fcc in msg.items:
92-
if isinstance(fcc, FunctionCallContent):
93-
print(f"Function Call:> {fcc.name} with arguments: {fcc.arguments}")
94-
else:
95-
print(f"{msg.role}: {msg.content}")
89+
"""
90+
Sample Output:
9691
97-
# Sample output:
9892
# User: 'Hello'
99-
# AuthorRole.ASSISTANT: Hello! How can I assist you today? If you have any questions about the menu,
100-
# feel free to ask!
93+
Hello! How can I assist you today?
94+
10195
# User: 'What is the special soup?'
102-
# AuthorRole.ASSISTANT: The special soup is Clam Chowder. Would you like to know more about it or
103-
# anything else on the menu?
96+
Function Call:> MenuPlugin-get_specials with arguments: {}
97+
Function Result:>
98+
Special Soup: Clam Chowder
99+
Special Salad: Cobb Salad
100+
Special Drink: Chai Tea
101+
for function: MenuPlugin-get_specials
102+
The special soup today is Clam Chowder. Is there anything else you'd like to know?
103+
104104
# User: 'How much does that cost?'
105-
# AuthorRole.ASSISTANT: The Clam Chowder costs $9.99. If you have any other questions or need further
106-
# information, feel free to ask!
105+
Function Call:> MenuPlugin-get_item_price with arguments: {"menu_item":"Clam Chowder"}
106+
Function Result:> $9.99 for function: MenuPlugin-get_item_price
107+
The Clam Chowder costs $9.99. Would you like to know anything else about the menu?
108+
107109
# User: 'Thank you'
108-
# AuthorRole.ASSISTANT: You're welcome! If you have any more questions in the future, don't hesitate
109-
# to ask. Have a great day!
110-
#
111-
# Intermediate Steps:
112-
# Function Call:> MenuPlugin-get_specials with arguments: {}
113-
# Function Result:>
114-
# Special Soup: Clam Chowder
115-
# Special Salad: Cobb Salad
116-
# Special Drink: Chai Tea
117-
# for function: MenuPlugin-get_specials
118-
# Function Call:> MenuPlugin-get_item_price with arguments: {"menu_item":"Clam Chowder"}
119-
# Function Result:> $9.99 for function: MenuPlugin-get_item_price
110+
You're welcome! If you have any more questions, feel free to ask. Have a great day!
111+
"""
120112

121113

122114
if __name__ == "__main__":

‎python/semantic_kernel/agents/chat_completion/chat_completion_agent.py‎

Lines changed: 34 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -415,6 +415,8 @@ async def invoke_stream(
415415

416416
role = None
417417
response_builder: list[str] = []
418+
start_idx = len(agent_chat_history)
419+
418420
async for response_list in responses:
419421
for response in response_list:
420422
role = response.role
@@ -430,12 +432,19 @@ async def invoke_stream(
430432
):
431433
yield AgentResponseItem(message=response, thread=thread)
432434

433-
await self._capture_mutated_messages(
434-
agent_chat_history,
435-
message_count_before_completion,
436-
thread,
437-
on_intermediate_message,
438-
)
435+
# Drain newly added tool messages since last index to maintain
436+
# correct order and avoid duplicates
437+
new_messages = await self._drain_mutated_messages(
438+
agent_chat_history,
439+
start_idx,
440+
thread,
441+
)
442+
# resets start_idx to the latest length of agent_chat_history.
443+
start_idx = len(agent_chat_history)
444+
445+
if on_intermediate_message:
446+
for message in new_messages:
447+
await on_intermediate_message(message)
439448

440449
if role != AuthorRole.TOOL:
441450
# Tool messages will be automatically added to the chat history by the auto function invocation loop
@@ -478,6 +487,7 @@ async def _inner_invoke(
478487
kernel=kernel,
479488
arguments=arguments,
480489
)
490+
start_idx = len(agent_chat_history)
481491

482492
message_count_before_completion = len(agent_chat_history)
483493

@@ -495,13 +505,18 @@ async def _inner_invoke(
495505
f"with message count: {message_count_before_completion}."
496506
)
497507

498-
await self._capture_mutated_messages(
508+
# Drain newly added tool messages since last index to maintain
509+
# correct order and avoid duplicates
510+
new_msgs = await self._drain_mutated_messages(
499511
agent_chat_history,
500-
message_count_before_completion,
512+
start_idx,
501513
thread,
502-
on_intermediate_message,
503514
)
504515

516+
if on_intermediate_message:
517+
for msg in new_msgs:
518+
await on_intermediate_message(msg)
519+
505520
for response in responses:
506521
response.name = self.name
507522
if response.role != AuthorRole.TOOL:
@@ -540,18 +555,16 @@ async def _get_chat_completion_service_and_settings(
540555

541556
return chat_completion_service, settings
542557

543-
async def _capture_mutated_messages(
558+
async def _drain_mutated_messages(
544559
self,
545-
agent_chat_history: ChatHistory,
560+
history: ChatHistory,
546561
start: int,
547562
thread: ChatHistoryAgentThread,
548-
on_intermediate_message: Callable[[ChatMessageContent], Awaitable[None]] | None = None,
549-
) -> None:
550-
"""Capture mutated messages related function calling/tools."""
551-
for message_index in range(start, len(agent_chat_history)):
552-
message = agent_chat_history[message_index] # type: ignore
553-
message.name = self.name
554-
await thread.on_new_message(message)
555-
556-
if on_intermediate_message:
557-
await on_intermediate_message(message)
563+
) -> list[ChatMessageContent]:
564+
"""Return messages appended to history after start and push them to thread."""
565+
drained: list[ChatMessageContent] = []
566+
for i in range(start, len(history)):
567+
msg: ChatMessageContent = history[i] # type: ignore
568+
await thread.on_new_message(msg)
569+
drained.append(msg)
570+
return drained

0 commit comments

Comments
 (0)