Skip to content

Better codecs and fix lots of test_io and other expectedFailures #6997

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 4 commits into from
Feb 6, 2026
Merged

Better codecs and fix lots of test_io and other expectedFailures #6997

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
afea165
Fix test_io expectedFailures
youknowone Feb 4, 2026
e948314
Update test_io from v3.14.3
Feb 5, 2026
258ac74
fix io
youknowone Feb 5, 2026
6e09d1b
win codecs
youknowone Feb 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Fix test_io expectedFailures
  • Loading branch information
@youknowone
youknowone committed Feb 5, 2026
commit afea16569be9b4d590402d1df1f387cdfc26f2b7
582 changes: 350 additions & 232 deletions Lib/_pycodecs.py
View file
Open in desktop

Large diffs are not rendered by default.

3 changes: 1 addition & 2 deletions Lib/test/test_bz2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -730,7 +730,7 @@ def testOpenBytesFilename(self):
self.assertEqual(f.read(), self.DATA)
self.assertEqual(f.name, str_filename)

@unittest.expectedFailure # TODO: RUSTPYTHON
@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: <FakePath 'Z:\\TEMP\\tmphoipjcen'> != 'Z:\\TEMP\\tmphoipjcen'
def testOpenPathLikeFilename(self):
filename = FakePath(self.filename)
with BZ2File(filename, "wb") as f:
Expand Down Expand Up @@ -1189,7 +1189,6 @@ def test_encoding_error_handler(self):
as f:
self.assertEqual(f.read(), "foobar")

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_newline(self):
# Test with explicit newline (universal newline mode disabled).
text = self.TEXT.decode("ascii")
Expand Down
70 changes: 6 additions & 64 deletions Lib/test/test_codecs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -465,7 +465,6 @@ class UTF32Test(ReadTest, unittest.TestCase):
b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m'
b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m')

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'codecs' has no attribute 'utf_32_ex_decode'. Did you mean: 'utf_16_ex_decode'?
def test_only_one_bom(self):
_,_,reader,writer = codecs.lookup(self.encoding)
# encode some stream
Expand All @@ -481,7 +480,6 @@ def test_only_one_bom(self):
f = reader(s)
self.assertEqual(f.read(), "spamspam")

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'codecs' has no attribute 'utf_32_ex_decode'. Did you mean: 'utf_16_ex_decode'?
def test_badbom(self):
s = io.BytesIO(4*b"\xff")
f = codecs.getreader(self.encoding)(s)
Expand All @@ -491,7 +489,6 @@ def test_badbom(self):
f = codecs.getreader(self.encoding)(s)
self.assertRaises(UnicodeDecodeError, f.read)

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'codecs' has no attribute 'utf_32_ex_decode'. Did you mean: 'utf_16_ex_decode'?
def test_partial(self):
self.check_partial(
"\x00\xff\u0100\uffff\U00010000",
Expand Down Expand Up @@ -523,7 +520,6 @@ def test_partial(self):
]
)

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_handlers(self):
self.assertEqual(('\ufffd', 1),
codecs.utf_32_decode(b'\x01', 'replace', True))
Expand All @@ -534,7 +530,6 @@ def test_errors(self):
self.assertRaises(UnicodeDecodeError, codecs.utf_32_decode,
b"\xff", "strict", True)

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'codecs' has no attribute 'utf_32_ex_decode'. Did you mean: 'utf_16_ex_decode'?
def test_decoder_state(self):
self.check_state_handling_decode(self.encoding,
"spamspam", self.spamle)
Expand All @@ -551,35 +546,24 @@ def test_issue8941(self):
self.assertEqual('\U00010000' * 1024,
codecs.utf_32_decode(encoded_be)[0])

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_lone_surrogates(self):
return super().test_lone_surrogates()

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'codecs' has no attribute 'utf_32_ex_decode'. Did you mean: 'utf_16_ex_decode'?
def test_bug1098990_a(self):
return super().test_bug1098990_a()

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'codecs' has no attribute 'utf_32_ex_decode'. Did you mean: 'utf_16_ex_decode'?
def test_bug1098990_b(self):
return super().test_bug1098990_b()

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'codecs' has no attribute 'utf_32_ex_decode'. Did you mean: 'utf_16_ex_decode'?
def test_bug1175396(self):
return super().test_bug1175396()

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'codecs' has no attribute 'utf_32_ex_decode'. Did you mean: 'utf_16_ex_decode'?
def test_incremental_surrogatepass(self):
return super().test_incremental_surrogatepass()

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'codecs' has no attribute 'utf_32_ex_decode'. Did you mean: 'utf_16_ex_decode'?
def test_mixed_readline_and_read(self):
return super().test_mixed_readline_and_read()

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'codecs' has no attribute 'utf_32_ex_decode'. Did you mean: 'utf_16_ex_decode'?
def test_readline(self):
return super().test_readline()

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'codecs' has no attribute 'utf_32_ex_decode'. Did you mean: 'utf_16_ex_decode'?
def test_readlinequeue(self):
return super().test_readlinequeue()

Expand Down Expand Up @@ -636,10 +620,6 @@ def test_issue8941(self):
self.assertEqual('\U00010000' * 1024,
codecs.utf_32_le_decode(encoded)[0])

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_lone_surrogates(self):
return super().test_lone_surrogates()




Expand Down Expand Up @@ -693,10 +673,6 @@ def test_issue8941(self):
self.assertEqual('\U00010000' * 1024,
codecs.utf_32_be_decode(encoded)[0])

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_lone_surrogates(self):
return super().test_lone_surrogates()




Expand Down Expand Up @@ -739,7 +715,6 @@ def test_badbom(self):
f = codecs.getreader(self.encoding)(s)
self.assertRaises(UnicodeDecodeError, f.read)

@unittest.expectedFailure # TODO: RUSTPYTHON; UnicodeDecodeError: 'utf-16' codec can't decode bytes in position 0-1: unexpected end of data
def test_partial(self):
self.check_partial(
"\x00\xff\u0100\uffff\U00010000",
Expand All @@ -761,7 +736,6 @@ def test_partial(self):
]
)

@unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: index out of range
def test_handlers(self):
self.assertEqual(('\ufffd', 1),
codecs.utf_16_decode(b'\x01', 'replace', True))
Expand Down Expand Up @@ -805,11 +779,6 @@ def test_invalid_modes(self):
self.assertIn("can't have text and binary mode at once",
str(cm.exception))

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_lone_surrogates(self):
return super().test_lone_surrogates()

@unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: index out of range
def test_incremental_surrogatepass(self):
return super().test_incremental_surrogatepass()

Expand All @@ -819,7 +788,6 @@ class UTF16LETest(ReadTest, unittest.TestCase):
encoding = "utf-16-le"
ill_formed_sequence = b"\x80\xdc"

@unittest.expectedFailure # TODO: RUSTPYTHON; UnicodeDecodeError: 'utf-16' codec can't decode bytes in position 0-1: unexpected end of data
def test_partial(self):
self.check_partial(
"\x00\xff\u0100\uffff\U00010000",
Expand All @@ -839,7 +807,6 @@ def test_partial(self):
]
)

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_errors(self):
tests = [
(b'\xff', '\ufffd'),
Expand All @@ -861,11 +828,6 @@ def test_nonbmp(self):
self.assertEqual(b'\x00\xd8\x03\xde'.decode(self.encoding),
"\U00010203")

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_lone_surrogates(self):
return super().test_lone_surrogates()

@unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: index out of range
def test_incremental_surrogatepass(self):
return super().test_incremental_surrogatepass()

Expand All @@ -874,7 +836,6 @@ class UTF16BETest(ReadTest, unittest.TestCase):
encoding = "utf-16-be"
ill_formed_sequence = b"\xdc\x80"

@unittest.expectedFailure # TODO: RUSTPYTHON; UnicodeDecodeError: 'utf-16' codec can't decode bytes in position 0-1: unexpected end of data
def test_partial(self):
self.check_partial(
"\x00\xff\u0100\uffff\U00010000",
Expand All @@ -894,7 +855,6 @@ def test_partial(self):
]
)

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_errors(self):
tests = [
(b'\xff', '\ufffd'),
Expand All @@ -916,11 +876,6 @@ def test_nonbmp(self):
self.assertEqual(b'\xd8\x00\xde\x03'.decode(self.encoding),
"\U00010203")

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_lone_surrogates(self):
return super().test_lone_surrogates()

@unittest.expectedFailure # TODO: RUSTPYTHON; UnicodeDecodeError: 'utf-16' codec can't decode bytes in position 0-1: unexpected end of data
def test_incremental_surrogatepass(self):
return super().test_incremental_surrogatepass()

Expand Down Expand Up @@ -970,7 +925,6 @@ def test_decode_error(self):
self.assertEqual(data.decode(self.encoding, error_handler),
expected)

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_lone_surrogates(self):
super().test_lone_surrogates()
# not sure if this is making sense for
Expand Down Expand Up @@ -1023,7 +977,6 @@ def test_incremental_errors(self):
class UTF7Test(ReadTest, unittest.TestCase):
encoding = "utf-7"

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_ascii(self):
# Set D (directly encoded characters)
set_d = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
Expand All @@ -1050,7 +1003,6 @@ def test_ascii(self):
b'+AAAAAQACAAMABAAFAAYABwAIAAsADAAOAA8AEAARABIAEwAU'
b'ABUAFgAXABgAGQAaABsAHAAdAB4AHwBcAH4Afw-')

@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at least 5 arguments, got 1
def test_partial(self):
self.check_partial(
'a+-b\x00c\x80d\u0100e\U00010000f',
Expand Down Expand Up @@ -1090,7 +1042,6 @@ def test_partial(self):
]
)

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_errors(self):
tests = [
(b'\xffb', '\ufffdb'),
Expand Down Expand Up @@ -1121,7 +1072,6 @@ def test_errors(self):
raw, 'strict', True)
self.assertEqual(raw.decode('utf-7', 'replace'), expected)

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_nonbmp(self):
self.assertEqual('\U000104A0'.encode(self.encoding), b'+2AHcoA-')
self.assertEqual('\ud801\udca0'.encode(self.encoding), b'+2AHcoA-')
Expand All @@ -1137,7 +1087,6 @@ def test_nonbmp(self):
self.assertEqual(b'+IKwgrNgB3KA'.decode(self.encoding),
'\u20ac\u20ac\U000104A0')

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_lone_surrogates(self):
tests = [
(b'a+2AE-b', 'a\ud801b'),
Expand All @@ -1158,15 +1107,9 @@ def test_lone_surrogates(self):
with self.subTest(raw=raw):
self.assertEqual(raw.decode('utf-7', 'replace'), expected)

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_bug1175396(self):
return super().test_bug1175396()

@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected at least 5 arguments, got 1
def test_readline(self):
return super().test_readline()

@unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: utf_7_decode() takes from 1 to 2 positional arguments but 3 were given
def test_incremental_surrogatepass(self):
return super().test_incremental_surrogatepass()

Expand Down Expand Up @@ -3062,7 +3005,6 @@ def test_latin1(self):


class BomTest(unittest.TestCase):
@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: module 'codecs' has no attribute 'utf_32_ex_decode'. Did you mean: 'utf_16_ex_decode'?
def test_seek0(self):
data = "1234567890"
tests = ("utf-16",
Expand Down Expand Up @@ -3457,7 +3399,7 @@ def test_invalid_code_page(self):
self.assertRaises(OSError, codecs.code_page_encode, 123, 'a')
self.assertRaises(OSError, codecs.code_page_decode, 123, b'a')

@unittest.expectedFailure # TODO: RUSTPYTHON
@unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
def test_code_page_name(self):
self.assertRaisesRegex(UnicodeEncodeError, 'cp932',
codecs.code_page_encode, 932, '\xff')
Expand Down Expand Up @@ -3524,7 +3466,7 @@ def check_encode(self, cp, tests):
self.assertRaises(UnicodeEncodeError,
text.encode, f'cp{cp}', errors)

@unittest.expectedFailure # TODO: RUSTPYTHON
@unittest.expectedFailure # TODO: RUSTPYTHON
def test_cp932(self):
self.check_encode(932, (
('abc', 'strict', b'abc'),
Expand Down Expand Up @@ -3559,7 +3501,7 @@ def test_cp932(self):
(b'\x81\x00abc', 'backslashreplace', '\\x81\x00abc'),
))

@unittest.expectedFailure # TODO: RUSTPYTHON
@unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
def test_cp1252(self):
self.check_encode(1252, (
('abc', 'strict', b'abc'),
Expand Down Expand Up @@ -3633,7 +3575,7 @@ def test_cp20106(self):
(b'(\xbf)', 'surrogatepass', None),
))

@unittest.expectedFailure # TODO: RUSTPYTHON
@unittest.expectedFailure # TODO: RUSTPYTHON # TODO: RUSTPYTHON
def test_cp_utf7(self):
cp = 65000
self.check_encode(cp, (
Expand All @@ -3654,7 +3596,7 @@ def test_cp_utf7(self):
(b'[\xff]', 'strict', '[\xff]'),
))

@unittest.expectedFailure # TODO: RUSTPYTHON
@unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
def test_multibyte_encoding(self):
self.check_decode(932, (
(b'\x84\xe9\x80', 'ignore', '\u9a3e'),
Expand Down Expand Up @@ -3688,7 +3630,7 @@ def test_code_page_decode_flags(self):
self.assertEqual(codecs.code_page_decode(42, b'abc'),
('\uf061\uf062\uf063', 3))

@unittest.expectedFailure # TODO: RUSTPYTHON
@unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
def test_incremental(self):
decoded = codecs.code_page_decode(932, b'\x82', 'strict', False)
self.assertEqual(decoded, ('', 0))
Expand Down
2 changes: 0 additions & 2 deletions Lib/test/test_fileinput.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -980,8 +980,6 @@ def check(errors, expected_lines):
check('replace', ['\ufffdabc'])
check('backslashreplace', ['\\x80abc'])

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_modes(self):
with open(TESTFN, 'wb') as f:
# UTF-7 is a convenient, seldom used encoding
Expand Down
1 change: 0 additions & 1 deletion Lib/test/test_gzip.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1036,7 +1036,6 @@ def test_encoding_error_handler(self):
as f:
self.assertEqual(f.read(), "foobar")

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_newline(self):
# Test with explicit newline (universal newline mode disabled).
uncompressed = data1.decode("ascii") * 50
Expand Down
Loading