Skip to content

Commit 8aef825

Browse files
MeiSiristhebestMeiSiristhebest
committed
fix(python): return empty list for whitespace-only input in text_chunker
When text is trimmed to empty, return [], True immediately to preserve compatibility with main and prevent emitting empty string chunks. Also make early-exit regression assertions more resilient.
1 parent 571012a commit 8aef825

2 files changed

Lines changed: 11 additions & 1 deletion

File tree

‎python/semantic_kernel/text/text_chunker.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,8 @@ def _split_str(
235235

236236
if trim:
237237
text = text.strip()
238+
if not text:
239+
return [], True
238240

239241
text_as_is = [text]
240242

‎python/tests/unit/text/test_text_chunker.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -557,7 +557,15 @@ def counting_split_list(*args, **kwargs):
557557
text = "\n".join(f"Line number {i} is short." for i in range(30))
558558
lines = tc.split_plaintext_lines(text, max_token_per_line=50)
559559

560-
assert len(lines) == 4
560+
assert len(lines) > 0
561561
# Tier 1 (['\n', '\r']) completes all splits. Remaining 9 tiers should not execute _split_list.
562562
assert call_counts["split_list"] == 0
563563
assert call_counts["split_str"] > 0
564+
565+
566+
def test_split_lines_whitespace_only():
567+
"""Test that whitespace-only input returns an empty list without preserving empty strings."""
568+
assert split_plaintext_lines(" ", 5) == []
569+
assert split_plaintext_lines("\n\n", 5) == []
570+
assert split_markdown_lines("\n\n", 5) == []
571+
assert split_plaintext_paragraph([" "], 5) == []

0 commit comments

Comments
 (0)