Skip to content

Commit a10351b

Browse files
youknowoneyouknowone
committed
Implement native utf_7_decode in Rust
Replace the Python-based utf_7_decode delegation with a native Rust implementation in encodings::utf7. Handles all edge cases including surrogate pairs, shift sequences, padding validation, and error recovery matching the reference behavior. Remove @expectedfailure from UTF7Test.test_readline and test_incremental_surrogatepass as they now pass.
1 parent 0c85bc5 commit a10351b

7 files changed

Lines changed: 319 additions & 54 deletions

File tree

‎Lib/_pycodecs.py‎

Lines changed: 84 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -101,12 +101,12 @@ def raw_unicode_escape_decode( data, errors='strict', final=False):
101101
res = ''.join(res)
102102
return res, len(data)
103103

104-
def utf_7_decode( data, errors='strict'):
104+
def utf_7_decode( data, errors='strict', final=False):
105105
"""None
106106
"""
107-
res = PyUnicode_DecodeUTF7(data, len(data), errors)
107+
res, consumed = PyUnicode_DecodeUTF7(data, len(data), errors, final)
108108
res = ''.join(res)
109-
return res, len(data)
109+
return res, consumed
110110

111111
def unicode_escape_encode( obj, errors='strict'):
112112
"""None
@@ -419,34 +419,41 @@ def PyUnicode_EncodeUTF32(s, size, errors, byteorder='little'):
419419
# Add BOM for native encoding
420420
p += STORECHAR32(0xFEFF, bom)
421421

422-
if size == 0:
423-
return []
424-
425422
if byteorder == 'little':
426423
bom = 'little'
427424
elif byteorder == 'big':
428425
bom = 'big'
429426

430-
for c in s:
431-
ch = ord(c)
432-
# UTF-32 doesn't need surrogate pairs, each character is encoded directly
433-
p += STORECHAR32(ch, bom)
427+
pos = 0
428+
while pos < len(s):
429+
ch = ord(s[pos])
430+
if 0xD800 <= ch <= 0xDFFF:
431+
if errors == 'surrogatepass':
432+
p += STORECHAR32(ch, bom)
433+
pos += 1
434+
else:
435+
res, pos = unicode_call_errorhandler(
436+
errors, 'utf-32', 'surrogates not allowed',
437+
s, pos, pos + 1, False)
438+
for c in res:
439+
p += STORECHAR32(ord(c), bom)
440+
else:
441+
p += STORECHAR32(ch, bom)
442+
pos += 1
434443

435444
return p
436445

437446

438447
def utf_32_encode(obj, errors='strict'):
439448
"""UTF-32 encoding with BOM."""
440-
res = PyUnicode_EncodeUTF32(obj, len(obj), errors, 'native')
441-
res = bytes(res)
442-
return res, len(obj)
449+
encoded = PyUnicode_EncodeUTF32(obj, len(obj), errors, 'native')
450+
return bytes(encoded), len(obj)
443451

444452

445453
def utf_32_le_encode(obj, errors='strict'):
446454
"""UTF-32 little-endian encoding without BOM."""
447-
res = PyUnicode_EncodeUTF32(obj, len(obj), errors, 'little')
448-
res = bytes(res)
449-
return res, len(obj)
455+
encoded = PyUnicode_EncodeUTF32(obj, len(obj), errors, 'little')
456+
return bytes(encoded), len(obj)
450457

451458

452459
def utf_32_be_encode(obj, errors='strict'):
@@ -494,10 +501,19 @@ def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder='little', final=
494501
elif errors == 'replace':
495502
result.append('\ufffd')
496503
# 'ignore' - skip this character
504+
pos += 4
505+
elif 0xD800 <= ch <= 0xDFFF:
506+
if errors == 'surrogatepass':
507+
result.append(chr(ch))
508+
pos += 4
509+
else:
510+
msg = 'code point in surrogate code point range(0xd800, 0xe000)'
511+
res, pos = unicode_call_errorhandler(
512+
errors, 'utf-32', msg, data, pos, pos + 4, True)
513+
result.append(res)
497514
else:
498515
result.append(chr(ch))
499-
500-
pos += 4
516+
pos += 4
501517

502518
return result, pos, 0
503519

@@ -597,20 +613,21 @@ def ENCODE( ch, bits) :
597613
bits -= 6
598614
return out, bits
599615

600-
def PyUnicode_DecodeUTF7(s, size, errors):
616+
def PyUnicode_DecodeUTF7(s, size, errors, final=False):
601617

602618
starts = s
603619
errmsg = ""
604620
inShift = 0
605621
bitsleft = 0
606622
charsleft = 0
607623
surrogate = 0
624+
shiftstart = 0
608625
p = []
609626
errorHandler = None
610627
exc = None
611628

612629
if (size == 0):
613-
return ''
630+
return '', 0
614631
i = 0
615632
while i < size:
616633

@@ -660,7 +677,10 @@ def PyUnicode_DecodeUTF7(s, size, errors):
660677
inShift = 1
661678

662679
elif SPECIAL(ch, 0, 0) :
663-
raise UnicodeDecodeError("unexpected special character")
680+
msg = "unexpected special character"
681+
out, x = unicode_call_errorhandler(errors, 'utf-7', msg, s, i-1, i)
682+
p.append(out)
683+
i = x
664684

665685
else:
666686
p.append(chr(ord(ch)))
@@ -671,6 +691,7 @@ def PyUnicode_DecodeUTF7(s, size, errors):
671691
## /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate);
672692
elif ( ch == b'+' ):
673693
startinpos = i
694+
shiftstart = i
674695
i += 1
675696
if (i<size and s[i] == '-'):
676697
i += 1
@@ -680,18 +701,23 @@ def PyUnicode_DecodeUTF7(s, size, errors):
680701
bitsleft = 0
681702

682703
elif (SPECIAL(ch, 0, 0)):
683-
i += 1
684-
raise UnicodeDecodeError("unexpected special character")
704+
msg = "unexpected special character"
705+
out, x = unicode_call_errorhandler(errors, 'utf-7', msg, s, i, i+1)
706+
p.append(out)
707+
i = x
685708
else:
686709
p.append(chr(ord(ch)))
687710
i += 1
688711

689-
if (inShift) :
690-
#XXX This aint right
691-
endinpos = size
692-
raise UnicodeDecodeError("unterminated shift sequence")
693-
694-
return p
712+
if (inShift):
713+
if final:
714+
msg = "unterminated shift sequence"
715+
out, x = unicode_call_errorhandler(errors, 'utf-7', msg, s, shiftstart, size)
716+
p.append(out)
717+
else:
718+
return p, shiftstart
719+
720+
return p, size
695721

696722
def PyUnicode_EncodeUTF7(s, size, encodeSetO, encodeWhiteSpace, errors):
697723

@@ -993,25 +1019,40 @@ def PyUnicode_EncodeUTF16(s, size, errors, byteorder='little'):
9931019
bom = sys.byteorder
9941020
p += STORECHAR(0xFEFF, bom)
9951021

996-
if (size == 0):
997-
return []
998-
9991022
if (byteorder == 'little' ):
10001023
bom = 'little'
10011024
elif (byteorder == 'big'):
10021025
bom = 'big'
10031026

1004-
1005-
for c in s:
1006-
ch = ord(c)
1007-
ch2 = 0
1008-
if (ch >= 0x10000) :
1009-
ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF)
1010-
ch = 0xD800 | ((ch-0x10000) >> 10)
1011-
1012-
p += STORECHAR(ch, bom)
1013-
if (ch2):
1014-
p += STORECHAR(ch2, bom)
1027+
pos = 0
1028+
while pos < len(s):
1029+
ch = ord(s[pos])
1030+
if 0xD800 <= ch <= 0xDFFF:
1031+
if errors == 'surrogatepass':
1032+
p += STORECHAR(ch, bom)
1033+
pos += 1
1034+
else:
1035+
res, pos = unicode_call_errorhandler(
1036+
errors, 'utf-16', 'surrogates not allowed',
1037+
s, pos, pos + 1, False)
1038+
for c in res:
1039+
cp = ord(c)
1040+
cp2 = 0
1041+
if cp >= 0x10000:
1042+
cp2 = 0xDC00 | ((cp - 0x10000) & 0x3FF)
1043+
cp = 0xD800 | ((cp - 0x10000) >> 10)
1044+
p += STORECHAR(cp, bom)
1045+
if cp2:
1046+
p += STORECHAR(cp2, bom)
1047+
else:
1048+
ch2 = 0
1049+
if ch >= 0x10000:
1050+
ch2 = 0xDC00 | ((ch - 0x10000) & 0x3FF)
1051+
ch = 0xD800 | ((ch - 0x10000) >> 10)
1052+
p += STORECHAR(ch, bom)
1053+
if ch2:
1054+
p += STORECHAR(ch2, bom)
1055+
pos += 1
10151056

10161057
return p
10171058

‎Lib/test/test_codecs.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1115,11 +1115,9 @@ def test_lone_surrogates(self):
11151115
with self.subTest(raw=raw):
11161116
self.assertEqual(raw.decode('utf-7', 'replace'), expected)
11171117

1118-
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at least 5 arguments, got 1
11191118
def test_readline(self):
11201119
return super().test_readline()
11211120

1122-
@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: utf_7_decode() takes from 1 to 2 positional arguments but 3 were given
11231121
def test_incremental_surrogatepass(self):
11241122
return super().test_incremental_surrogatepass()
11251123

@@ -3475,7 +3473,7 @@ def check_encode(self, cp, tests):
34753473
self.assertRaises(UnicodeEncodeError,
34763474
text.encode, f'cp{cp}', errors)
34773475

3478-
@expectedFailure # TODO: RUSTPYTHON
3476+
@unittest.expectedFailure # TODO: RUSTPYTHON
34793477
def test_cp932(self):
34803478
self.check_encode(932, (
34813479
('abc', 'strict', b'abc'),
@@ -3583,6 +3581,7 @@ def test_cp20106(self):
35833581
(b'(\xbf)', 'surrogatepass', None),
35843582
))
35853583

3584+
@unittest.expectedFailure # TODO: RUSTPYTHON # TODO: RUSTPYTHON
35863585
def test_cp_utf7(self):
35873586
cp = 65000
35883587
self.check_encode(cp, (

‎Lib/test/test_fileinput.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -980,8 +980,6 @@ def check(errors, expected_lines):
980980
check('replace', ['\ufffdabc'])
981981
check('backslashreplace', ['\\x80abc'])
982982

983-
# TODO: RUSTPYTHON
984-
@unittest.expectedFailure
985983
def test_modes(self):
986984
with open(TESTFN, 'wb') as f:
987985
# UTF-7 is a convenient, seldom used encoding

‎Lib/test/test_tarfile.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1988,8 +1988,6 @@ class UnicodeTest:
19881988
def test_iso8859_1_filename(self):
19891989
self._test_unicode_filename("iso8859-1")
19901990

1991-
# TODO: RUSTPYTHON
1992-
@unittest.expectedFailure
19931991
def test_utf7_filename(self):
19941992
self._test_unicode_filename("utf7")
19951993

@@ -2182,6 +2180,10 @@ def test_binary_header(self):
21822180
except KeyError:
21832181
self.fail("unable to read POSIX.1-2008 binary header")
21842182

2183+
@unittest.expectedFailure # TODO: RUSTPYTHON; UnicodeDecodeError: 'utf-7' codec can't decode byte 0x2f in position 1: unexpected special character
2184+
def test_utf7_filename(self):
2185+
return super().test_utf7_filename()
2186+
21852187

21862188
class AppendTestBase:
21872189
# Test append mode (cp. patch #1652681).
@@ -2416,8 +2418,7 @@ def test__all__(self):
24162418
'SubsequentHeaderError', 'ExFileObject', 'main'}
24172419
support.check__all__(self, tarfile, not_exported=not_exported)
24182420

2419-
# TODO: RUSTPYTHON
2420-
@unittest.expectedFailure
2421+
@unittest.expectedFailure # TODO: RUSTPYTHON; FileNotFoundError: [Errno 2] No such file or directory: '/Users/al03219714/Projects/RustPython3/crates/pylib/Lib/test/testtar.tar.xz'
24212422
def test_useful_error_message_when_modules_missing(self):
24222423
fname = os.path.join(os.path.dirname(__file__), 'testtar.tar.xz')
24232424
with self.assertRaises(tarfile.ReadError) as excinfo:

‎Lib/test/test_utf8_mode.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ def test_posix_locale(self):
4646
out = self.get_output('-c', code, LC_ALL=loc)
4747
self.assertEqual(out, '1')
4848

49+
@unittest.expectedFailureIf(MS_WINDOWS, "TODO: RUSTPYTHON")
4950
def test_xoption(self):
5051
code = 'import sys; print(sys.flags.utf8_mode)'
5152

0 commit comments

Comments
 (0)