Skip to content

Commit 81e89f0

Browse files
moonbox3moonbox3
authored
Python: Improve agent integration tests (microsoft#11475)
### Motivation and Context The current integration tests for agents work fine; however, if any errors are encountered due to server-side hiccups, the instant failure cause the whole test run to fail. This PR creates an `agent_test_base` that provides some retry logic, as well as a standardized way for agent test classes to use the underlying methods/protocol. <!-- Thank you for your contribution to the semantic-kernel repo! Please help reviewers and future users, providing the following information: 1. Why is this change required? 2. What problem does it solve? 3. What scenario does it contribute to? 4. If it fixes an open issue, please link to the issue here. --> ### Description Improve the agent integration tests. - The OpenAI Responses API is throwing 500s for file search right now. Adding `xfail` for the time being. Filed an [issue](openai/openai-python#2298) on them. - Handle errors when calling for a response for the OpenAI Responses API -- throw a specific agent except or content exception, if received from Azure OpenAI. - Removed the `OpenAIResponsesAgent` `invoke` and `invoke_stream` web_search tests. We have coverage for `get_responses`. The web search is flaky sometimes. <!-- Describe your changes, the overall approach, the underlying design. These notes will help understanding how your code works. Thanks! --> ### Contribution Checklist <!-- Before submitting this PR, please make sure: --> - [X] The code builds clean without any errors or warnings - [X] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [X] All unit tests pass, and I have added new tests where possible - [X] I didn't break anyone 😄
1 parent d72d74d commit 81e89f0

8 files changed

Lines changed: 498 additions & 202 deletions

File tree

‎python/semantic_kernel/agents/open_ai/responses_agent_thread_actions.py‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from functools import reduce
77
from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar, cast
88

9+
from openai import BadRequestError
910
from openai._streaming import AsyncStream
1011
from openai.types.responses import ResponseFunctionToolCall
1112
from openai.types.responses.response import Response
@@ -28,6 +29,7 @@
2829
merge_function_results,
2930
)
3031
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
32+
from semantic_kernel.connectors.ai.open_ai.exceptions.content_filter_ai_exception import ContentFilterAIException
3133
from semantic_kernel.contents.annotation_content import AnnotationContent
3234
from semantic_kernel.contents.chat_history import ChatHistory
3335
from semantic_kernel.contents.chat_message_content import CMC_ITEM_TYPES, ChatMessageContent
@@ -41,6 +43,7 @@
4143
from semantic_kernel.contents.utils.author_role import AuthorRole
4244
from semantic_kernel.contents.utils.status import Status
4345
from semantic_kernel.exceptions.agent_exceptions import (
46+
AgentExecutionException,
4447
AgentInvokeException,
4548
)
4649
from semantic_kernel.functions.kernel_arguments import KernelArguments
@@ -481,15 +484,31 @@ async def _get_response(
481484
response_options: dict | None = None,
482485
stream: bool = False,
483486
) -> Response | AsyncStream[ResponseStreamEvent]:
484-
response: Response = await agent.client.responses.create(
485-
input=cls._prepare_chat_history_for_request(chat_history),
486-
instructions=merged_instructions or agent.instructions,
487-
previous_response_id=previous_response_id,
488-
store=store_output_enabled,
489-
tools=tools, # type: ignore
490-
stream=stream,
491-
**response_options,
492-
)
487+
try:
488+
response: Response = await agent.client.responses.create(
489+
input=cls._prepare_chat_history_for_request(chat_history),
490+
instructions=merged_instructions or agent.instructions,
491+
previous_response_id=previous_response_id,
492+
store=store_output_enabled,
493+
tools=tools, # type: ignore
494+
stream=stream,
495+
**response_options,
496+
)
497+
except BadRequestError as ex:
498+
if ex.code == "content_filter":
499+
raise ContentFilterAIException(
500+
f"{type(agent)} encountered a content error",
501+
ex,
502+
) from ex
503+
raise AgentExecutionException(
504+
f"{type(agent)} failed to complete the request",
505+
ex,
506+
) from ex
507+
except Exception as ex:
508+
raise AgentExecutionException(
509+
f"{type(agent)} service failed to complete the request",
510+
ex,
511+
) from ex
493512
if response is None:
494513
raise AgentInvokeException("Response is None")
495514
return response

‎python/tests/integration/agents/__init__.py‎

Whitespace-only changes.

‎python/tests/integration/agents/agent_test_base.py‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
# Copyright (c) Microsoft. All rights reserved.
2+
3+
import asyncio
4+
from collections.abc import AsyncIterator, Awaitable, Callable
5+
from typing import Any, Generic, Protocol, TypeVar
6+
7+
from semantic_kernel.agents.agent import Agent, AgentResponseItem, AgentThread
8+
from semantic_kernel.contents import ChatMessageContent
9+
10+
DEFAULT_MAX_ATTEMPTS = 3
11+
DEFAULT_BACKOFF_SECONDS = 1
12+
13+
14+
class ChatResponseProtocol(Protocol):
15+
"""Represents a single response item returned by the agent."""
16+
17+
@property
18+
def message(self) -> ChatMessageContent: ...
19+
20+
@property
21+
def thread(self) -> AgentThread | None: ...
22+
23+
24+
class ChatAgentProtocol(Protocol):
25+
"""Protocol describing the common agent interface used by the tests."""
26+
27+
async def get_response(
28+
self, messages: str | list[str] | None, thread: object | None = None
29+
) -> ChatResponseProtocol: ...
30+
31+
def invoke(
32+
self, messages: str | list[str] | None, thread: object | None = None
33+
) -> AsyncIterator[ChatResponseProtocol]: ...
34+
35+
def invoke_stream(
36+
self, messages: str | list[str] | None, thread: object | None = None
37+
) -> AsyncIterator[ChatResponseProtocol]: ...
38+
39+
40+
TAgent = TypeVar("TAgent", bound=ChatAgentProtocol)
41+
42+
43+
async def run_with_retry(
44+
coro: Callable[..., Awaitable[Any]],
45+
*args,
46+
attempts: int = DEFAULT_MAX_ATTEMPTS,
47+
backoff_seconds: float = DEFAULT_BACKOFF_SECONDS,
48+
**kwargs,
49+
) -> AgentResponseItem[ChatMessageContent]:
50+
"""
51+
Execute an async callable with retry/backoff logic.
52+
53+
Args:
54+
coro: The async function to call
55+
args: Positional args to pass to the function
56+
attempts: How many times to attempt before giving up
57+
backoff_seconds: The initial backoff in seconds, doubled after each failure
58+
kwargs: Keyword args to pass to the function
59+
60+
Returns:
61+
Whatever the async function returns
62+
63+
Raises:
64+
Exception: If the function fails after the specified number of attempts
65+
"""
66+
delay = backoff_seconds
67+
for attempt in range(1, attempts + 1):
68+
try:
69+
return await coro(*args, **kwargs)
70+
except Exception:
71+
if attempt == attempts:
72+
raise
73+
await asyncio.sleep(delay)
74+
delay *= 2
75+
raise RuntimeError("Unexpected error: run_with_retry exit.")
76+
77+
78+
class AgentTestBase(Generic[TAgent]):
79+
"""Common test base that wraps all agent invocation patterns with retry logic.
80+
81+
Each integration test can inherit from this or use its methods directly.
82+
"""
83+
84+
async def get_response_with_retry(
85+
self,
86+
agent: Agent,
87+
messages: str | list[str] | None,
88+
thread: Any | None = None,
89+
attempts: int = DEFAULT_MAX_ATTEMPTS,
90+
backoff_seconds: float = DEFAULT_BACKOFF_SECONDS,
91+
) -> AgentResponseItem[ChatMessageContent]:
92+
"""Wraps agent.get_response(...) in run_with_retry."""
93+
return await run_with_retry(
94+
agent.get_response, messages=messages, thread=thread, attempts=attempts, backoff_seconds=backoff_seconds
95+
)
96+
97+
async def get_invoke_with_retry(
98+
self,
99+
agent: Any,
100+
messages: str | list[str] | None,
101+
thread: Any | None = None,
102+
attempts: int = DEFAULT_MAX_ATTEMPTS,
103+
backoff_seconds: float = DEFAULT_BACKOFF_SECONDS,
104+
) -> list[AgentResponseItem[ChatMessageContent]]:
105+
"""Wraps agent.invoke(...) in run_with_retry.
106+
107+
Collects generator results in a list before returning them.
108+
"""
109+
return await run_with_retry(
110+
self._collect_from_invoke,
111+
agent,
112+
messages,
113+
thread=thread,
114+
attempts=attempts,
115+
backoff_seconds=backoff_seconds,
116+
)
117+
118+
async def get_invoke_stream_with_retry(
119+
self,
120+
agent: Any,
121+
messages: str | list[str] | None,
122+
thread: Any | None = None,
123+
attempts: int = DEFAULT_MAX_ATTEMPTS,
124+
backoff_seconds: float = DEFAULT_BACKOFF_SECONDS,
125+
) -> list[AgentResponseItem[ChatMessageContent]]:
126+
"""Wraps agent.invoke_stream(...) in run_with_retry.
127+
128+
Collects streaming results in a list before returning them."""
129+
return await run_with_retry(
130+
self._collect_from_invoke_stream,
131+
agent,
132+
messages,
133+
thread=thread,
134+
attempts=attempts,
135+
backoff_seconds=backoff_seconds,
136+
)
137+
138+
async def _collect_from_invoke(
139+
self, agent: Agent, messages: str | list[str] | None, thread: Any | None = None
140+
) -> list[AgentResponseItem[ChatMessageContent]]:
141+
results: list[AgentResponseItem[ChatMessageContent]] = []
142+
async for response in agent.invoke(messages=messages, thread=thread):
143+
results.append(response)
144+
return results
145+
146+
async def _collect_from_invoke_stream(
147+
self, agent: Agent, messages: str | list[str] | None, thread: Any | None = None
148+
) -> list[AgentResponseItem[ChatMessageContent]]:
149+
results: list[AgentResponseItem[ChatMessageContent]] = []
150+
async for response in agent.invoke_stream(messages=messages, thread=thread):
151+
results.append(response)
152+
return results

0 commit comments

Comments
 (0)