Skip to content

Commit ee3c518

Browse files
donggyun112donggyun112
committed
fix(models): stop LiteLlm from running tool calls quoted in answer text
When a provider returns no structured tool_calls, LiteLlm turned every {"name": ..., "arguments": ...} object found anywhere in the answer text into a function call, including one the model only quoted from a README, an API document or a retrieved page. The agent then ran that tool and the quoted text disappeared from the answer. Treat text as tool calls only when it opens with them, allowing leading whitespace and the <tool_call> and code-fence wrappers models emit, and consume those wrappers with the call. Parse streamed text for calls only once it is complete, since a single delta cannot tell a call from a quote.
1 parent 3fd932d commit ee3c518

2 files changed

Lines changed: 170 additions & 40 deletions

File tree

‎src/google/adk/models/lite_llm.py‎

Lines changed: 42 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2064,6 +2064,16 @@ def _parse_deepseek_tool_calls_from_text(
20642064
return tool_calls, remainder or None
20652065

20662066

2067+
_INLINE_CALL_OPEN_RE = re.compile(r"\s*(?:<tool_call>|```(?:json)?)?\s*")
2068+
_INLINE_CALL_CLOSE_RE = re.compile(r"\s*(?:</tool_call>|```)?\s*")
2069+
2070+
2071+
def _skip_past(pattern: re.Pattern[str], text: str, pos: int) -> int:
2072+
"""Returns the index just past `pattern` matched at `pos`, or `pos`."""
2073+
match = pattern.match(text, pos)
2074+
return match.end() if match else pos
2075+
2076+
20672077
def _parse_tool_calls_from_text(
20682078
text_block: str,
20692079
) -> tuple[list[ChatCompletionMessageToolCall], Optional[str]]:
@@ -2085,41 +2095,35 @@ def _parse_tool_calls_from_text(
20852095
return tool_calls, extra_remainder
20862096
return ds_tool_calls, None
20872097

2088-
remainder_segments = []
2098+
# Only calls the text opens with count, optionally in the wrappers models
2099+
# emit them in. JSON after any prose is quoted text: turning it into a call
2100+
# would run a tool the model only showed.
20892101
cursor = 0
2090-
text_length = len(text_block)
2091-
2092-
while cursor < text_length:
2093-
brace_index = text_block.find("{", cursor)
2094-
if brace_index == -1:
2095-
remainder_segments.append(text_block[cursor:])
2102+
while True:
2103+
start = _skip_past(_INLINE_CALL_OPEN_RE, text_block, cursor)
2104+
if not text_block.startswith("{", start):
20962105
break
2097-
2098-
remainder_segments.append(text_block[cursor:brace_index])
20992106
try:
2100-
candidate, end = _JSON_DECODER.raw_decode(text_block, brace_index)
2107+
candidate, end = _JSON_DECODER.raw_decode(text_block, start)
21012108
except json.JSONDecodeError:
2102-
remainder_segments.append(text_block[brace_index])
2103-
cursor = brace_index + 1
2104-
continue
2105-
2109+
break
21062110
tool_call = _build_tool_call_from_json_dict(
21072111
candidate, index=len(tool_calls)
21082112
)
2109-
if tool_call:
2110-
tool_calls.append(tool_call)
2111-
else:
2112-
remainder_segments.append(text_block[brace_index:end])
2113-
cursor = end
2113+
if not tool_call:
2114+
break
2115+
tool_calls.append(tool_call)
2116+
cursor = _skip_past(_INLINE_CALL_CLOSE_RE, text_block, end)
21142117

2115-
remainder = "".join(segment for segment in remainder_segments if segment)
2116-
remainder = remainder.strip()
2117-
2118-
return tool_calls, remainder or None
2118+
if not tool_calls:
2119+
return tool_calls, text_block.strip() or None
2120+
return tool_calls, text_block[cursor:].strip() or None
21192121

21202122

21212123
def _split_message_content_and_tool_calls(
21222124
message: Message,
2125+
*,
2126+
parse_inline_tool_calls: bool = True,
21232127
) -> tuple[Optional[OpenAIMessageContent], list[ChatCompletionMessageToolCall]]:
21242128
"""Returns message content and tool calls, parsing inline JSON when needed."""
21252129
existing_tool_calls = message.get("tool_calls") or []
@@ -2130,7 +2134,11 @@ def _split_message_content_and_tool_calls(
21302134

21312135
# LiteLLM responses either provide structured tool_calls or inline JSON, not
21322136
# both. When tool_calls are present we trust them and skip the fallback parser.
2133-
if normalized_tool_calls or not isinstance(content, str):
2137+
if (
2138+
normalized_tool_calls
2139+
or not isinstance(content, str)
2140+
or not parse_inline_tool_calls
2141+
):
21342142
return content, normalized_tool_calls
21352143

21362144
fallback_tool_calls, remainder = _parse_tool_calls_from_text(content)
@@ -2493,11 +2501,15 @@ def _has_meaningful_signal(message: Message | Delta | None) -> bool:
24932501
reasoning_parts: List[types.Part] = []
24942502

24952503
if message is not None:
2496-
# Both Delta and Message support dict-like .get() access
2504+
# Both Delta and Message support dict-like .get() access. A delta is too
2505+
# little text to tell a tool call from one quoted mid-answer, so streamed
2506+
# text is parsed for calls once, as a whole, when the stream finalizes.
24972507
(
24982508
message_content,
24992509
tool_calls,
2500-
) = _split_message_content_and_tool_calls(message)
2510+
) = _split_message_content_and_tool_calls(
2511+
message, parse_inline_tool_calls=message_field != "delta"
2512+
)
25012513
reasoning_value = _extract_reasoning_value(message)
25022514
if reasoning_value:
25032515
reasoning_parts = _convert_reasoning_value_to_parts(reasoning_value)
@@ -2687,7 +2699,10 @@ def _message_to_generate_content_response(
26872699
)
26882700
if thought_parts:
26892701
parts.extend(thought_parts)
2690-
message_content, tool_calls = _split_message_content_and_tool_calls(message)
2702+
# A partial message is one streamed delta; see _model_response_to_chunk.
2703+
message_content, tool_calls = _split_message_content_and_tool_calls(
2704+
message, parse_inline_tool_calls=not is_partial
2705+
)
26912706
if isinstance(message_content, str) and message_content:
26922707
parts.append(types.Part.from_text(text=message_content))
26932708

‎tests/unittests/models/test_litellm.py‎

Lines changed: 128 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4151,7 +4151,6 @@ async def test_thought_signature_round_trip():
41514151
def test_parse_tool_calls_from_text_multiple_calls():
41524152
text = (
41534153
'{"name":"alpha","arguments":{"value":1}}\n'
4154-
"Some filler text "
41554154
'{"id":"custom","name":"beta","arguments":{"timezone":"Asia/Taipei"}} '
41564155
"ignored suffix"
41574156
)
@@ -4164,7 +4163,21 @@ def test_parse_tool_calls_from_text_multiple_calls():
41644163
assert json.loads(tool_calls[1].function.arguments) == {
41654164
"timezone": "Asia/Taipei"
41664165
}
4167-
assert remainder == "Some filler text ignored suffix"
4166+
assert remainder == "ignored suffix"
4167+
4168+
4169+
def test_parse_tool_calls_from_text_stops_at_text_between_calls():
4170+
"""A tool call that follows prose is quoted text, not another call."""
4171+
beta = '{"name":"beta","arguments":{"timezone":"Asia/Taipei"}}'
4172+
text = (
4173+
'{"name":"alpha","arguments":{"value":1}}\n'
4174+
f"Some filler text {beta} ignored suffix"
4175+
)
4176+
4177+
tool_calls, remainder = _parse_tool_calls_from_text(text)
4178+
4179+
assert [call.function.name for call in tool_calls] == ["alpha"]
4180+
assert remainder == f"Some filler text {beta} ignored suffix"
41684181

41694182

41704183
def test_parse_tool_calls_from_text_invalid_json_returns_remainder():
@@ -4281,7 +4294,7 @@ def test_parse_tool_calls_from_text_mixed_formats():
42814294
"""DeepSeek tokens + standard inline JSON in the same text."""
42824295
ds_part = _ds_wrapped(_ds_tool_call("ds_func", '{"a": 1}'))
42834296
standard_part = '{"name": "std_func", "arguments": {"b": 2}}'
4284-
text = ds_part + " some text " + standard_part
4297+
text = ds_part + "\n" + standard_part + " some text"
42854298
tool_calls, remainder = _parse_tool_calls_from_text(text)
42864299
assert len(tool_calls) == 2
42874300
assert tool_calls[0].function.name == "ds_func"
@@ -4313,15 +4326,40 @@ def test_extract_json_from_deepseek_args_invalid_fence_returns_none():
43134326
assert _extract_json_from_deepseek_args('```json\n{"a": 1,}\n```') is None
43144327

43154328

4316-
def test_split_message_content_and_tool_calls_inline_text():
4317-
message = {
4318-
"role": "assistant",
4319-
"content": (
4320-
'Intro {"name":"alpha","arguments":{"value":1}} trailing content'
4321-
),
4322-
}
4329+
def test_split_message_content_keeps_tool_call_json_quoted_in_text():
4330+
"""JSON shaped like a tool call inside prose stays text, not a call."""
4331+
text = (
4332+
"The README shows this example request:\n"
4333+
'{"name":"delete_file","arguments":{"path":"/data/prod.db"}}\n'
4334+
"It is used to remove files."
4335+
)
4336+
message = {"role": "assistant", "content": text}
4337+
4338+
content, tool_calls = _split_message_content_and_tool_calls(message)
4339+
4340+
assert tool_calls == []
4341+
assert content == text
4342+
4343+
4344+
@pytest.mark.parametrize(
4345+
"text",
4346+
[
4347+
'\n{"name": "alpha", "arguments": {"value": 1}}\n',
4348+
(
4349+
'<tool_call>\n{"name": "alpha", "arguments": {"value":'
4350+
" 1}}\n</tool_call>"
4351+
),
4352+
'```json\n{"name": "alpha", "arguments": {"value": 1}}\n```',
4353+
],
4354+
ids=["bare", "tool_call_tags", "code_fence"],
4355+
)
4356+
def test_split_message_content_parses_text_that_is_a_tool_call(text):
4357+
"""A message whose text is a tool call becomes that tool call."""
4358+
message = {"role": "assistant", "content": text}
4359+
43234360
content, tool_calls = _split_message_content_and_tool_calls(message)
4324-
assert content == "Intro trailing content"
4361+
4362+
assert content is None
43254363
assert len(tool_calls) == 1
43264364
assert tool_calls[0].function.name == "alpha"
43274365
assert json.loads(tool_calls[0].function.arguments) == {"value": 1}
@@ -5061,7 +5099,7 @@ def test_to_litellm_role():
50615099
"message": {
50625100
"role": "assistant",
50635101
"content": (
5064-
'Intro {"id":"call_2","name":"alpha",'
5102+
'{"id":"call_2","name":"alpha",'
50655103
'"arguments":{"foo":"bar"}} wrap'
50665104
),
50675105
},
@@ -5073,7 +5111,7 @@ def test_to_litellm_role():
50735111
},
50745112
),
50755113
[
5076-
TextChunk(text="Intro wrap"),
5114+
TextChunk(text="wrap"),
50775115
FunctionChunk(
50785116
id="call_2",
50795117
name="alpha",
@@ -6219,6 +6257,83 @@ async def test_streaming_inline_tool_call_malformed_arguments(
62196257
assert "test_function" in final_response.error_message
62206258

62216259

6260+
def _text_stream(*deltas: str) -> list[ModelResponseStream]:
6261+
"""Streams each text as one delta, then a stop-only chunk."""
6262+
chunks = [
6263+
ModelResponseStream(
6264+
choices=[
6265+
StreamingChoices(
6266+
finish_reason=None,
6267+
delta=Delta(role="assistant", content=text),
6268+
)
6269+
]
6270+
)
6271+
for text in deltas
6272+
]
6273+
chunks.append(
6274+
ModelResponseStream(
6275+
choices=[
6276+
StreamingChoices(
6277+
finish_reason="stop",
6278+
delta=Delta(role="assistant", content=""),
6279+
)
6280+
]
6281+
)
6282+
)
6283+
return chunks
6284+
6285+
6286+
@pytest.mark.asyncio
6287+
async def test_streaming_text_quoting_a_tool_call_is_not_a_call(
6288+
mock_completion, lite_llm_instance
6289+
):
6290+
"""Prose that quotes a tool call in its own delta streams back as text."""
6291+
call_json = '{"name": "test_function", "arguments": {"test_arg": "x"}}'
6292+
mock_completion.return_value = iter(
6293+
_text_stream("The README shows this example: ", call_json, " Done.")
6294+
)
6295+
6296+
responses = [
6297+
response
6298+
async for response in lite_llm_instance.generate_content_async(
6299+
LLM_REQUEST_WITH_FUNCTION_DECLARATION, stream=True
6300+
)
6301+
]
6302+
6303+
assert not [
6304+
part
6305+
for response in responses
6306+
for part in response.content.parts
6307+
if part.function_call
6308+
]
6309+
final_text = "".join(part.text for part in responses[-1].content.parts)
6310+
assert final_text == f"The README shows this example: {call_json} Done."
6311+
6312+
6313+
@pytest.mark.asyncio
6314+
async def test_streaming_text_that_is_a_tool_call_becomes_a_call(
6315+
mock_completion, lite_llm_instance
6316+
):
6317+
"""A streamed message whose text is a tool call ends as that call."""
6318+
mock_completion.return_value = iter(
6319+
_text_stream(
6320+
'{"name": "test_function", ',
6321+
'"arguments": {"test_arg": "x"}}',
6322+
)
6323+
)
6324+
6325+
responses = [
6326+
response
6327+
async for response in lite_llm_instance.generate_content_async(
6328+
LLM_REQUEST_WITH_FUNCTION_DECLARATION, stream=True
6329+
)
6330+
]
6331+
6332+
function_call = responses[-1].content.parts[0].function_call
6333+
assert function_call.name == "test_function"
6334+
assert function_call.args == {"test_arg": "x"}
6335+
6336+
62226337
@pytest.mark.asyncio
62236338
async def test_streaming_tool_call_complete_with_length_finish_reason(
62246339
mock_completion, lite_llm_instance

0 commit comments

Comments
 (0)