Skip to content

Commit 3b27cd3

Browse files
eavanvalkenburgeavanvalkenburg
authored
Python: Add native computer use to Responses clients (microsoft#8751)
* feat(python): add native computer use support to Responses clients * fix(python): parse computer outputs with OpenAI 2.x type stubs * Fix computer-use workflow review findings * Preserve mixed computer results in file history and fix typing * Remove web search MCP readme examples and fix OpenAI test typing
1 parent be01dec commit 3b27cd3

27 files changed

Lines changed: 2775 additions & 228 deletions

File tree

‎python/packages/core/AGENTS.md‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,10 @@ agent_framework/
7070
- **`ChatResponseUpdate`** - Streaming response update
7171
- **`AgentResponse`** / **`AgentResponseUpdate`** - Agent-level response wrappers
7272
- **`Content`** - Base class for message content (text, function calls, images, etc.)
73+
- **Computer use** - `ComputerSafetyCheck` and the `Content.from_computer_tool_call` / `from_computer_tool_result`
74+
constructors are experimental under `COMPUTER_USE`; the rest of `Content` retains its existing stage. Computer
75+
results can omit screenshots in core; OpenAI-based connectors require them when converting to Responses items.
76+
Completed call/result pairs remain in the transcript but are not user-input requests.
7377
- **`ChatOptions`** - TypedDict for chat request options
7478

7579
### Tools (`_tools.py`)
@@ -163,6 +167,9 @@ The vector store API is experimental under the shared `VECTOR_STORES` feature ID
163167
- **`HistoryProvider`** - Base class for conversation history storage
164168
- **`InMemoryHistoryProvider`** - Built-in session-state history provider for local runs
165169
- **`FileHistoryProvider`** - Experimental append-only file-backed history provider; msgspec JSON Lines is the default and `serialization_format="msgpack"` uses length-prefixed binary MessagePack records. Custom `dumps`/`loads` remain as deprecated JSON-only compatibility hooks and emit `DeprecationWarning` when supplied.
170+
- **Mixed computer/function workflow history** - The default `HistoryProvider.after_run` defers completed local
171+
function results for loadable providers that store inputs until the computer reply arrives, so history records
172+
them once in call order. Custom `after_run` implementations must handle this themselves.
166173

167174
### Skills (`_skills.py`)
168175

@@ -296,7 +303,8 @@ The vector store API is experimental under the shared `VECTOR_STORES` feature ID
296303
requests without synthesizing responses, recursively releases nested executor correlation, resumes executors whose
297304
remaining requests were already answered, accepts the same request-scoped tools and invocation/client kwargs needed
298305
by that continuation, can atomically restore a supplied checkpoint before cancellation, and returns the resulting
299-
`WorkflowRunResult`.
306+
`WorkflowRunResult`. Cancelling a computer request also cancels the other pending requests in that agent's batch,
307+
retains already resolved sibling evidence in terminal output, and resets its session before new input.
300308
- **`WorkflowBuilder`** - Fluent API for building workflows, including explicit
301309
`output_from` / `intermediate_output_from` selection for caller-facing emissions. `output_from`
302310
is an allow-list for **Workflow Output**; unselected executor payloads are hidden unless

‎python/packages/core/agent_framework/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,7 @@
254254
"ChatOptions",
255255
"ChatResponse",
256256
"ChatResponseUpdate",
257+
"ComputerSafetyCheck",
257258
"Content",
258259
"ContinuationToken",
259260
"Embedding",
@@ -476,6 +477,7 @@
476477
"ClassSkill",
477478
"CompactionProvider",
478479
"CompactionStrategy",
480+
"ComputerSafetyCheck",
479481
"Content",
480482
"ContextProvider",
481483
"ContextWindowCompactionStrategy",

‎python/packages/core/agent_framework/__init__.pyi‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,7 @@ from ._types import (
220220
ChatOptions,
221221
ChatResponse,
222222
ChatResponseUpdate,
223+
ComputerSafetyCheck,
223224
Content,
224225
ContinuationToken,
225226
Embedding,
@@ -431,6 +432,7 @@ __all__ = [
431432
"ClassSkill",
432433
"CompactionProvider",
433434
"CompactionStrategy",
435+
"ComputerSafetyCheck",
434436
"Content",
435437
"ContextProvider",
436438
"ContextWindowCompactionStrategy",

‎python/packages/core/agent_framework/_compaction.py‎

Lines changed: 37 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747

4848
_TOOL_CALL_CONTENT_TYPES: Final[set[str]] = {
4949
"function_call",
50+
"computer_tool_call",
5051
"mcp_server_tool_call",
5152
"code_interpreter_tool_call",
5253
"shell_tool_call",
@@ -146,20 +147,25 @@ def _is_reasoning_only_assistant(message: Message) -> bool:
146147
return all(content.type == "text_reasoning" for content in message.contents)
147148

148149

149-
def _unambiguous_function_call_result_pairs(messages: Sequence[Message]) -> list[tuple[int, int]]:
150-
unmatched_declaration_indices: dict[str, list[int]] = {}
150+
def _unambiguous_tool_call_result_pairs(messages: Sequence[Message]) -> list[tuple[int, int]]:
151+
unmatched_declaration_indices: dict[tuple[str, str], list[int]] = {}
151152
pairs: list[tuple[int, int]] = []
152153

153154
for message_index, message in enumerate(messages):
154155
if message.role not in ("assistant", "tool"):
155156
continue
156157
for content in message.contents:
157-
if message.role == "assistant" and content.type == "function_call" and content.call_id:
158-
unmatched_declaration_indices.setdefault(content.call_id, []).append(message_index)
158+
if (
159+
message.role == "assistant"
160+
and content.type in ("function_call", "computer_tool_call")
161+
and content.call_id
162+
):
163+
unmatched_declaration_indices.setdefault((content.type, content.call_id), []).append(message_index)
159164
continue
160-
if content.type != "function_result" or not content.call_id:
165+
if content.type not in ("function_result", "computer_tool_result") or not content.call_id:
161166
continue
162-
candidates = unmatched_declaration_indices.get(content.call_id)
167+
call_type = "function_call" if content.type == "function_result" else "computer_tool_call"
168+
candidates = unmatched_declaration_indices.get((call_type, content.call_id))
163169
if candidates is None or len(candidates) != 1:
164170
continue
165171
pairs.append((candidates.pop(), message_index))
@@ -191,8 +197,8 @@ def _group_id_for(message: Message, group_index: int) -> str:
191197
return f"group_index_{group_index}"
192198

193199

194-
def _link_function_call_result_spans(messages: Sequence[Message], spans: list[dict[str, Any]]) -> None:
195-
"""Link non-adjacent function results to unambiguous declaration occurrences."""
200+
def _link_tool_call_result_spans(messages: Sequence[Message], spans: list[dict[str, Any]]) -> None:
201+
"""Link non-adjacent tool results to unambiguous declaration occurrences."""
196202
if len(spans) < 2:
197203
return
198204

@@ -222,7 +228,7 @@ def union(left: int, right: int) -> bool:
222228
return True
223229

224230
linked = False
225-
for declaration_message_index, result_message_index in _unambiguous_function_call_result_pairs(messages):
231+
for declaration_message_index, result_message_index in _unambiguous_tool_call_result_pairs(messages):
226232
declaration_span_index = span_by_message_index[declaration_message_index]
227233
result_span_index = span_by_message_index[result_message_index]
228234
if declaration_span_index < result_span_index and union(result_span_index, declaration_span_index):
@@ -260,7 +266,7 @@ def group_messages(
260266
Returns:
261267
Ordered list of lightweight span dicts with keys:
262268
``group_id``, ``kind``, ``start_index``, ``end_index``, ``has_reasoning``.
263-
Non-contiguous function-call declaration and result spans share a group id.
269+
Non-contiguous function or computer call/result spans share a group id.
264270
"""
265271
_ensure_message_ids(messages, id_offset=id_offset, reserved_ids=reserved_ids)
266272
spans: list[dict[str, Any]] = []
@@ -365,7 +371,7 @@ def group_messages(
365371
i += 1
366372
group_index += 1
367373

368-
_link_function_call_result_spans(messages, spans)
374+
_link_tool_call_result_spans(messages, spans)
369375
return spans
370376

371377

@@ -828,19 +834,24 @@ def _reannotation_start(messages: Sequence[Message], index: int) -> int:
828834
return previous_index
829835

830836

831-
def _function_pair_reannotation_start(messages: Sequence[Message], start_index: int) -> int:
832-
unmatched_declaration_indices: dict[str, list[int]] = {}
837+
def _tool_pair_reannotation_start(messages: Sequence[Message], start_index: int) -> int:
838+
unmatched_declaration_indices: dict[tuple[str, str], list[int]] = {}
833839
matching_indices: list[int] = []
834840
for message_index, message in enumerate(messages):
835841
if message.role not in ("assistant", "tool"):
836842
continue
837843
for content in message.contents:
838-
if message.role == "assistant" and content.type == "function_call" and content.call_id:
839-
unmatched_declaration_indices.setdefault(content.call_id, []).append(message_index)
844+
if (
845+
message.role == "assistant"
846+
and content.type in ("function_call", "computer_tool_call")
847+
and content.call_id
848+
):
849+
unmatched_declaration_indices.setdefault((content.type, content.call_id), []).append(message_index)
840850
continue
841-
if content.type != "function_result" or not content.call_id:
851+
if content.type not in ("function_result", "computer_tool_result") or not content.call_id:
842852
continue
843-
candidates = unmatched_declaration_indices.get(content.call_id)
853+
call_type = "function_call" if content.type == "function_result" else "computer_tool_call"
854+
candidates = unmatched_declaration_indices.get((call_type, content.call_id))
844855
if not candidates:
845856
continue
846857
if message_index >= start_index:
@@ -894,7 +905,7 @@ def annotate_message_groups(
894905
start_index = min(candidate_starts)
895906

896907
start_index = _reannotation_start(messages, start_index)
897-
start_index = _function_pair_reannotation_start(messages, start_index)
908+
start_index = _tool_pair_reannotation_start(messages, start_index)
898909

899910
# Linked groups can be non-contiguous, so the last prefix message does not
900911
# necessarily carry the highest group index.
@@ -1585,6 +1596,14 @@ def _format_summary_content(content: Content) -> str:
15851596
if content.call_id:
15861597
call += f" [call_id={content.call_id}]"
15871598
return call
1599+
if content.type == "computer_tool_call":
1600+
actions = _tool_result_text(content.actions) if content.actions is not None else "no actions"
1601+
call_id_suffix = f" [call_id={content.call_id}]" if content.call_id else ""
1602+
return f"computer_tool_call: {actions}{call_id_suffix}"
1603+
if content.type == "computer_tool_result":
1604+
call_id_suffix = f" [call_id={content.call_id}]" if content.call_id else ""
1605+
screenshot_label = ": screenshot" if content.screenshot is not None else ""
1606+
return f"computer_tool_result{screenshot_label}{call_id_suffix}"
15881607
if content.type == "function_result":
15891608
result_text = _format_summary_result_items(content.items) if content.items else ""
15901609
if not result_text:

‎python/packages/core/agent_framework/_feature_stage.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ class ExperimentalFeature(str, Enum):
5151
"""
5252

5353
AGENT_HOOKS = "AGENT_HOOKS"
54+
COMPUTER_USE = "COMPUTER_USE"
5455
DECLARATIVE_AGENTS = "DECLARATIVE_AGENTS"
5556
EVALS = "EVALS"
5657
FILE_HISTORY = "FILE_HISTORY"

‎python/packages/core/agent_framework/_sessions.py‎

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@
6868

6969
MESSAGE_INJECTION_PENDING_MESSAGES_STATE_KEY: str = "message_injection.pending_messages"
7070
_MESSAGE_INJECTION_LOCK = threading.Lock()
71+
_WORKFLOW_DEFER_COMPUTER_FUNCTION_RESULTS_KEY = "workflow.defer_computer_function_results"
7172

7273
JsonDumps: TypeAlias = Callable[[Any], str | bytes]
7374
JsonLoads: TypeAlias = Callable[[str | bytes], Any]
@@ -939,6 +940,53 @@ def _filter_approval_control_messages(messages: Sequence[Message]) -> list[Messa
939940
return filtered_messages
940941

941942

943+
def _paired_local_function_results(contents: Sequence[Content]) -> dict[int, Content]:
944+
"""Match completed local results to their function-call occurrences."""
945+
results_by_call_id: dict[str, deque[Content]] = {}
946+
for content in contents:
947+
if content.type == "function_result" and content.call_id:
948+
results_by_call_id.setdefault(content.call_id, deque()).append(content)
949+
pairs: dict[int, Content] = {}
950+
for content in contents:
951+
if (
952+
content.type == "function_call"
953+
and not content.informational_only
954+
and content.id
955+
and content.call_id
956+
and (results := results_by_call_id.get(content.call_id))
957+
):
958+
pairs[id(content)] = results.popleft()
959+
return pairs
960+
961+
962+
def _without_deferred_workflow_function_results(messages: Sequence[Message]) -> list[Message]:
963+
"""Persist mixed computer turns without results staged for workflow resume."""
964+
start = next(
965+
(
966+
index
967+
for index, message in enumerate(messages)
968+
if any(content.type == "computer_tool_call" and content.user_input_request for content in message.contents)
969+
),
970+
None,
971+
)
972+
if start is None:
973+
return list(messages)
974+
contents = [content for message in messages[start:] for content in message.contents]
975+
deferred_ids = {id(result) for result in _paired_local_function_results(contents).values()}
976+
if not deferred_ids:
977+
return list(messages)
978+
stored = list(messages[:start])
979+
for message in messages[start:]:
980+
kept = [content for content in message.contents if id(content) not in deferred_ids]
981+
if len(kept) == len(message.contents):
982+
stored.append(message)
983+
elif kept:
984+
copied = copy.copy(message)
985+
copied.contents = kept
986+
stored.append(copied)
987+
return stored
988+
989+
942990
class HistoryProvider(ContextProvider):
943991
"""Base class for conversation history storage providers.
944992
@@ -1069,7 +1117,14 @@ async def after_run(
10691117
if self.store_inputs:
10701118
messages_to_store.extend(context.input_messages)
10711119
if self.store_outputs and context.response and context.response.messages:
1072-
messages_to_store.extend(context.response.messages)
1120+
output_messages = context.response.messages
1121+
if (
1122+
self.load_messages
1123+
and self.store_inputs
1124+
and session.state.get(_WORKFLOW_DEFER_COMPUTER_FUNCTION_RESULTS_KEY)
1125+
):
1126+
output_messages = _without_deferred_workflow_function_results(output_messages)
1127+
messages_to_store.extend(output_messages)
10731128
if messages_to_store:
10741129
await self.save_messages(context.session_id, messages_to_store, state=state)
10751130

‎python/packages/core/agent_framework/_tools.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4845,7 +4845,12 @@ async def _process_model_function_calls(
48454845
had_errors=execution.had_errors,
48464846
max_errors=max_errors,
48474847
)
4848-
if execution.should_terminate:
4848+
has_pending_computer_call = any(
4849+
content.type == "computer_tool_call" and content.user_input_request
4850+
for message in response.messages
4851+
for content in message.contents
4852+
)
4853+
if execution.should_terminate or has_pending_computer_call:
48494854
processing_result.action = "return"
48504855
if processing_result.action == "return":
48514856
returned_approval_requests = [

0 commit comments

Comments
 (0)