Skip to content

Commit fe8497a

Browse files
authored
Python: Preserve MessageText values during formula preprocessing (microsoft#8366)
* Python: Preserve MessageText values during formula preprocessing Process only original formula text and bind extracted messages through temporary state instead of embedding message content in formula source. Avoid temporary-name collisions and cover literal handling, nested calls, and cleanup regressions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Python: Respect quoted identifiers and comments in MessageText scanning Share quoted-token and comment handling between formula scanning and call-boundary matching. Cover identifier escapes and comment boundaries with preprocessing and real-PowerFx regressions, and replace the test append-return lambda with a typed helper for mypy. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Python: Wait for retired output cleanup in Hyperlight test Assert the rejected sandbox entry is evicted, then drain asynchronous disposal before checking that its output directory is gone. Avoid racing a directory listing against background deletion without changing runtime behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Python: Drop local Hyperlight test fix in favor of upstream Revert 6d8ca2d. The cleanup test is now fixed upstream; remove our independent correction before merging main. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Python: Preserve MessageText expressions in interpolated strings Traverse only original expression positions so interpolation expressions are preprocessed without reinterpreting literal text or message values. Move quoted-token handling to module scope and add focused regression coverage for the new interpolation contexts. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 6129a8c commit fe8497a

3 files changed

Lines changed: 408 additions & 104 deletions

File tree

‎python/packages/declarative/agent_framework_declarative/_workflows/_declarative_base.py‎

Lines changed: 116 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
import re
3232
import sys
3333
import uuid
34-
from collections.abc import Mapping
34+
from collections.abc import Iterator, Mapping
3535
from dataclasses import dataclass, field
3636
from decimal import Decimal as _Decimal
3737
from enum import Enum
@@ -67,6 +67,73 @@
6767
_SAFE_PATH_SEGMENT_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_]*$")
6868

6969

70+
def _skip_powerfx_opaque_token(formula: str, start: int) -> int:
71+
"""Return the end of an ordinary quoted token or comment, or start if neither."""
72+
quote = formula[start]
73+
if quote in ('"', "'"):
74+
pos = start + 1
75+
while pos < len(formula):
76+
if formula[pos] == quote:
77+
if pos + 1 < len(formula) and formula[pos + 1] == quote:
78+
pos += 2
79+
continue
80+
return pos + 1
81+
pos += 1
82+
return pos
83+
if formula.startswith("//", start):
84+
pos = start + 2
85+
while pos < len(formula) and formula[pos] not in "\r\n":
86+
pos += 1
87+
return pos
88+
if formula.startswith("/*", start):
89+
end = formula.find("*/", start + 2)
90+
return len(formula) if end == -1 else end + 2
91+
return start
92+
93+
94+
def _iter_powerfx_expression_indices(formula: str) -> Iterator[int]:
95+
"""Yield code positions, excluding literals/comments but including interpolation expressions."""
96+
# None denotes interpolated text; integers track record braces in expression sections.
97+
scopes: list[int | None] = [0]
98+
cursor = 0
99+
while cursor < len(formula):
100+
depth = scopes[-1]
101+
char = formula[cursor]
102+
if depth is None:
103+
if formula.startswith(('""', "{{", "}}"), cursor):
104+
cursor += 2
105+
continue
106+
if char == "{":
107+
scopes.append(0)
108+
elif char == '"':
109+
scopes.pop()
110+
cursor += 1
111+
continue
112+
113+
if formula.startswith('$"', cursor):
114+
scopes.append(None)
115+
cursor += 2
116+
continue
117+
118+
token_end = _skip_powerfx_opaque_token(formula, cursor)
119+
if token_end != cursor:
120+
cursor = token_end
121+
continue
122+
123+
if char == "{":
124+
scopes[-1] = depth + 1
125+
elif char == "}":
126+
if depth > 0:
127+
scopes[-1] = depth - 1
128+
elif len(scopes) > 1:
129+
scopes.pop()
130+
cursor += 1
131+
continue
132+
133+
yield cursor
134+
cursor += 1
135+
136+
70137
@dataclass(frozen=True)
71138
class DeclarativeEnvConfig:
72139
"""Configuration that populates the PowerFx ``Env`` symbol for a workflow.
@@ -667,11 +734,11 @@ def _preprocess_custom_functions(self, formula: str, temp_writes: list[tuple[str
667734
When they appear nested inside other functions (e.g., Upper(MessageText(...))),
668735
we need to evaluate them first and replace with the result.
669736
670-
For long strings (>500 chars), the result is stored in a temporary state variable
671-
to avoid exceeding PowerFx's 1000 character expression limit. This is a limitation
672-
of the Python PowerFx wrapper (powerfx package), which doesn't expose the
673-
MaximumExpressionLength configuration that the .NET PowerFxConfig provides.
674-
The .NET implementation defaults to 10,000 characters, while Python defaults to 1,000.
737+
Results are stored in temporary state variables so untrusted message text is
738+
passed to PowerFx as data rather than inserted into formula source.
739+
Temporary names avoid existing Local keys and references in the original formula.
740+
Literal text, quoted identifiers, and comments are left untouched; expression
741+
sections inside PowerFx interpolated strings are preprocessed as code.
675742
676743
Args:
677744
formula: The PowerFx formula to pre-process
@@ -684,90 +751,53 @@ def _preprocess_custom_functions(self, formula: str, temp_writes: list[tuple[str
684751
Returns:
685752
The rewritten formula.
686753
"""
687-
import re
688-
689-
# Threshold for storing in state vs embedding as literal.
690-
# The Python PowerFx wrapper defaults to a 1000 char expression limit (vs 10,000 in .NET).
691-
# We use 500 to leave room for the rest of the expression around the replaced value.
692-
MAX_INLINE_LENGTH = 500
693-
694754
temp_var_counter = 0
695-
696-
# Custom functions that need pre-processing: (regex pattern, handler)
697-
custom_functions = [
698-
(r"MessageText\(", self._eval_and_replace_message_text),
699-
]
700-
701-
for pattern, handler in custom_functions:
702-
# Find all occurrences of the custom function
703-
while True:
704-
match = re.search(pattern, formula)
705-
if not match:
755+
reserved_names = {name.casefold() for name in self.get_state_data().get("Local", {})}
756+
# Reserve formula references too, so previously undefined names stay undefined.
757+
folded_formula = formula.casefold()
758+
function_name = "MessageText"
759+
call_prefix = f"{function_name}("
760+
result: list[str] = []
761+
copied_until = 0
762+
positions = _iter_powerfx_expression_indices(formula)
763+
764+
for cursor in positions:
765+
if not formula.startswith(call_prefix, cursor):
766+
continue
767+
768+
paren_start = cursor + len(function_name)
769+
depth = 1
770+
for pos in positions:
771+
if pos <= paren_start:
772+
continue
773+
char = formula[pos]
774+
if char == "(":
775+
depth += 1
776+
elif char == ")":
777+
depth -= 1
778+
if depth == 0:
706779
break
780+
else:
781+
break
707782

708-
# Find the matching closing parenthesis
709-
start = match.start()
710-
paren_start = match.end() - 1 # Position of opening (
711-
depth = 1
712-
pos = paren_start + 1
713-
in_string = False
714-
escape_next = False
715-
716-
while pos < len(formula) and depth > 0:
717-
char = formula[pos]
718-
if escape_next:
719-
escape_next = False
720-
pos += 1
721-
continue
722-
if char == "\\":
723-
escape_next = True
724-
pos += 1
725-
continue
726-
if char == '"' and not escape_next:
727-
in_string = not in_string
728-
elif not in_string:
729-
if char == "(":
730-
depth += 1
731-
elif char == ")":
732-
depth -= 1
733-
pos += 1
734-
735-
if depth != 0:
736-
# Malformed expression, skip
783+
inner_expr = formula[paren_start + 1 : pos]
784+
replacement = self._eval_and_replace_message_text(inner_expr)
785+
while True:
786+
temp_var_name = f"_TempMessageText{temp_var_counter}"
787+
temp_var_counter += 1
788+
folded_name = temp_var_name.casefold()
789+
if folded_name not in reserved_names and folded_name not in folded_formula:
737790
break
738-
739-
# Extract the inner expression (between parentheses)
740-
end = pos
741-
inner_expr = formula[paren_start + 1 : end - 1]
742-
743-
# Evaluate and get replacement
744-
replacement = handler(inner_expr)
745-
746-
# Replace in formula
747-
if isinstance(replacement, str):
748-
if len(replacement) > MAX_INLINE_LENGTH:
749-
# Store long results in an underscore-prefixed temp key;
750-
# record the prior value so eval() can restore it.
751-
temp_var_name = f"_TempMessageText{temp_var_counter}"
752-
temp_var_counter += 1
753-
temp_var_path = f"Local.{temp_var_name}"
754-
temp_writes.append((temp_var_path, self.get(temp_var_path, default=self._MISSING)))
755-
self.set(temp_var_path, replacement)
756-
replacement_str = temp_var_path
757-
logger.debug(
758-
f"Stored long MessageText result ({len(replacement)} chars) "
759-
f"in temp variable {temp_var_name}"
760-
)
761-
else:
762-
# Short strings can be embedded directly
763-
escaped = replacement.replace('"', '""')
764-
replacement_str = f'"{escaped}"'
765-
else:
766-
replacement_str = str(replacement) if replacement is not None else '""'
767-
768-
formula = formula[:start] + replacement_str + formula[end:]
769-
770-
return formula
791+
temp_var_path = f"Local.{temp_var_name}"
792+
temp_writes.append((temp_var_path, self.get(temp_var_path, default=self._MISSING)))
793+
self.set(temp_var_path, replacement)
794+
result.append(formula[copied_until:cursor])
795+
result.append(temp_var_path)
796+
logger.debug(f"Stored MessageText result ({len(replacement)} chars) in temp variable {temp_var_name}")
797+
copied_until = pos + 1
798+
799+
result.append(formula[copied_until:])
800+
return "".join(result)
771801

772802
def _eval_and_replace_message_text(self, inner_expr: str) -> str:
773803
"""Evaluate MessageText() and return the text result.

0 commit comments

Comments
 (0)