Skip to content

Commit 3a50e70

Browse files
author
Victor Stinner
committed
Issue #12281: Rewrite the MBCS codec to handle correctly replace and ignore
error handlers on all Windows versions. The MBCS codec is now supporting all error handlers, instead of only replace to encode and ignore to decode.
1 parent 1e73a24 commit 3a50e70

8 files changed

Lines changed: 888 additions & 147 deletions

File tree

‎Doc/library/codecs.rst‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1280,12 +1280,13 @@ functions can be used directly if desired.
12801280
.. module:: encodings.mbcs
12811281
:synopsis: Windows ANSI codepage
12821282

1283-
Encode operand according to the ANSI codepage (CP_ACP). This codec only
1284-
supports ``'strict'`` and ``'replace'`` error handlers to encode, and
1285-
``'strict'`` and ``'ignore'`` error handlers to decode.
1283+
Encode operand according to the ANSI codepage (CP_ACP).
12861284

12871285
Availability: Windows only.
12881286

1287+
.. versionchanged:: 3.3
1288+
Support any error handler.
1289+
12891290
.. versionchanged:: 3.2
12901291
Before 3.2, the *errors* argument was ignored; ``'replace'`` was always used
12911292
to encode, and ``'ignore'`` to decode.

‎Doc/whatsnew/3.3.rst‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,11 @@ The :mod:`array` module supports the :c:type:`long long` type using ``q`` and
197197
codecs
198198
------
199199

200+
The :mod:`~encodings.mbcs` codec has be rewritten to handle correclty
201+
``replace`` and ``ignore`` error handlers on all Windows versions. The
202+
:mod:`~encodings.mbcs` codec is now supporting all error handlers, instead of
203+
only ``replace`` to encode and ``ignore`` to decode.
204+
200205
Multibyte CJK decoders now resynchronize faster. They only ignore the first
201206
byte of an invalid byte sequence. For example, ``b'\xff\n'.decode('gb2312',
202207
'replace')`` now returns a ``\n`` after the replacement character.

‎Include/unicodeobject.h‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1466,18 +1466,32 @@ PyAPI_FUNC(PyObject*) PyUnicode_DecodeMBCSStateful(
14661466
Py_ssize_t *consumed /* bytes consumed */
14671467
);
14681468

1469+
PyAPI_FUNC(PyObject*) PyUnicode_DecodeCodePageStateful(
1470+
int code_page, /* code page number */
1471+
const char *string, /* encoded string */
1472+
Py_ssize_t length, /* size of string */
1473+
const char *errors, /* error handling */
1474+
Py_ssize_t *consumed /* bytes consumed */
1475+
);
1476+
14691477
PyAPI_FUNC(PyObject*) PyUnicode_AsMBCSString(
14701478
PyObject *unicode /* Unicode object */
14711479
);
14721480

14731481
#ifndef Py_LIMITED_API
14741482
PyAPI_FUNC(PyObject*) PyUnicode_EncodeMBCS(
14751483
const Py_UNICODE *data, /* Unicode char buffer */
1476-
Py_ssize_t length, /* Number of Py_UNICODE chars to encode */
1484+
Py_ssize_t length, /* number of Py_UNICODE chars to encode */
14771485
const char *errors /* error handling */
14781486
);
14791487
#endif
14801488

1489+
PyAPI_FUNC(PyObject*) PyUnicode_EncodeCodePage(
1490+
int code_page, /* code page number */
1491+
PyObject *unicode, /* Unicode object */
1492+
const char *errors /* error handling */
1493+
);
1494+
14811495
#endif /* HAVE_MBCS */
14821496

14831497
/* --- Decimal Encoder ---------------------------------------------------- */

‎Lib/test/test_codecs.py‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1744,6 +1744,203 @@ def test_readline(self):
17441744
self.assertEqual(sout, b"\x80")
17451745

17461746

1747+
class CodePageTest(unittest.TestCase):
1748+
CP_UTF8 = 65001
1749+
vista_or_later = (sys.getwindowsversion().major >= 6)
1750+
1751+
def test_invalid_code_page(self):
1752+
self.assertRaises(ValueError, codecs.code_page_encode, -1, 'a')
1753+
self.assertRaises(ValueError, codecs.code_page_decode, -1, b'a')
1754+
self.assertRaises(WindowsError, codecs.code_page_encode, 123, 'a')
1755+
self.assertRaises(WindowsError, codecs.code_page_decode, 123, b'a')
1756+
1757+
def test_code_page_name(self):
1758+
self.assertRaisesRegex(UnicodeEncodeError, 'cp932',
1759+
codecs.code_page_encode, 932, '\xff')
1760+
self.assertRaisesRegex(UnicodeDecodeError, 'cp932',
1761+
codecs.code_page_decode, 932, b'\x81\x00')
1762+
self.assertRaisesRegex(UnicodeDecodeError, 'CP_UTF8',
1763+
codecs.code_page_decode, self.CP_UTF8, b'\xff')
1764+
1765+
def check_decode(self, cp, tests):
1766+
for raw, errors, expected in tests:
1767+
if expected is not None:
1768+
try:
1769+
decoded = codecs.code_page_decode(cp, raw, errors)
1770+
except UnicodeDecodeError as err:
1771+
self.fail('Unable to decode %a from "cp%s" with '
1772+
'errors=%r: %s' % (raw, cp, errors, err))
1773+
self.assertEqual(decoded[0], expected,
1774+
'%a.decode("cp%s", %r)=%a != %a'
1775+
% (raw, cp, errors, decoded[0], expected))
1776+
# assert 0 <= decoded[1] <= len(raw)
1777+
self.assertGreaterEqual(decoded[1], 0)
1778+
self.assertLessEqual(decoded[1], len(raw))
1779+
else:
1780+
self.assertRaises(UnicodeDecodeError,
1781+
codecs.code_page_decode, cp, raw, errors)
1782+
1783+
def check_encode(self, cp, tests):
1784+
for text, errors, expected in tests:
1785+
if expected is not None:
1786+
try:
1787+
encoded = codecs.code_page_encode(cp, text, errors)
1788+
except UnicodeEncodeError as err:
1789+
self.fail('Unable to encode %a to "cp%s" with '
1790+
'errors=%r: %s' % (text, cp, errors, err))
1791+
self.assertEqual(encoded[0], expected,
1792+
'%a.encode("cp%s", %r)=%a != %a'
1793+
% (text, cp, errors, encoded[0], expected))
1794+
self.assertEqual(encoded[1], len(text))
1795+
else:
1796+
self.assertRaises(UnicodeEncodeError,
1797+
codecs.code_page_encode, cp, text, errors)
1798+
1799+
def test_cp932(self):
1800+
self.check_encode(932, (
1801+
('abc', 'strict', b'abc'),
1802+
('\uff44\u9a3e', 'strict', b'\x82\x84\xe9\x80'),
1803+
# not encodable
1804+
('\xff', 'strict', None),
1805+
('[\xff]', 'ignore', b'[]'),
1806+
('[\xff]', 'replace', b'[y]'),
1807+
('[\u20ac]', 'replace', b'[?]'),
1808+
))
1809+
tests = [
1810+
(b'abc', 'strict', 'abc'),
1811+
(b'\x82\x84\xe9\x80', 'strict', '\uff44\u9a3e'),
1812+
# invalid bytes
1813+
(b'\xff', 'strict', None),
1814+
(b'\xff', 'ignore', ''),
1815+
(b'\xff', 'replace', '\ufffd'),
1816+
(b'\x81\x00abc', 'strict', None),
1817+
(b'\x81\x00abc', 'ignore', '\x00abc'),
1818+
]
1819+
if self.vista_or_later:
1820+
tests.append((b'\x81\x00abc', 'replace', '\ufffd\x00abc'))
1821+
else:
1822+
tests.append((b'\x81\x00abc', 'replace', '\x00\x00abc'))
1823+
self.check_decode(932, tests)
1824+
1825+
def test_cp1252(self):
1826+
self.check_encode(1252, (
1827+
('abc', 'strict', b'abc'),
1828+
('\xe9\u20ac', 'strict', b'\xe9\x80'),
1829+
('\xff', 'strict', b'\xff'),
1830+
('\u0141', 'strict', None),
1831+
('\u0141', 'ignore', b''),
1832+
('\u0141', 'replace', b'L'),
1833+
))
1834+
self.check_decode(1252, (
1835+
(b'abc', 'strict', 'abc'),
1836+
(b'\xe9\x80', 'strict', '\xe9\u20ac'),
1837+
(b'\xff', 'strict', '\xff'),
1838+
))
1839+
1840+
def test_cp_utf7(self):
1841+
cp = 65000
1842+
self.check_encode(cp, (
1843+
('abc', 'strict', b'abc'),
1844+
('\xe9\u20ac', 'strict', b'+AOkgrA-'),
1845+
('\U0010ffff', 'strict', b'+2//f/w-'),
1846+
('\udc80', 'strict', b'+3IA-'),
1847+
('\ufffd', 'strict', b'+//0-'),
1848+
))
1849+
self.check_decode(cp, (
1850+
(b'abc', 'strict', 'abc'),
1851+
(b'+AOkgrA-', 'strict', '\xe9\u20ac'),
1852+
(b'+2//f/w-', 'strict', '\U0010ffff'),
1853+
(b'+3IA-', 'strict', '\udc80'),
1854+
(b'+//0-', 'strict', '\ufffd'),
1855+
# invalid bytes
1856+
(b'[+/]', 'strict', '[]'),
1857+
(b'[\xff]', 'strict', '[\xff]'),
1858+
))
1859+
1860+
def test_cp_utf8(self):
1861+
cp = self.CP_UTF8
1862+
1863+
tests = [
1864+
('abc', 'strict', b'abc'),
1865+
('\xe9\u20ac', 'strict', b'\xc3\xa9\xe2\x82\xac'),
1866+
('\U0010ffff', 'strict', b'\xf4\x8f\xbf\xbf'),
1867+
]
1868+
if self.vista_or_later:
1869+
tests.append(('\udc80', 'strict', None))
1870+
tests.append(('\udc80', 'ignore', b''))
1871+
tests.append(('\udc80', 'replace', b'?'))
1872+
else:
1873+
tests.append(('\udc80', 'strict', b'\xed\xb2\x80'))
1874+
self.check_encode(cp, tests)
1875+
1876+
tests = [
1877+
(b'abc', 'strict', 'abc'),
1878+
(b'\xc3\xa9\xe2\x82\xac', 'strict', '\xe9\u20ac'),
1879+
(b'\xf4\x8f\xbf\xbf', 'strict', '\U0010ffff'),
1880+
(b'\xef\xbf\xbd', 'strict', '\ufffd'),
1881+
(b'[\xc3\xa9]', 'strict', '[\xe9]'),
1882+
# invalid bytes
1883+
(b'[\xff]', 'strict', None),
1884+
(b'[\xff]', 'ignore', '[]'),
1885+
(b'[\xff]', 'replace', '[\ufffd]'),
1886+
]
1887+
if self.vista_or_later:
1888+
tests.extend((
1889+
(b'[\xed\xb2\x80]', 'strict', None),
1890+
(b'[\xed\xb2\x80]', 'ignore', '[]'),
1891+
(b'[\xed\xb2\x80]', 'replace', '[\ufffd\ufffd\ufffd]'),
1892+
))
1893+
else:
1894+
tests.extend((
1895+
(b'[\xed\xb2\x80]', 'strict', '[\udc80]'),
1896+
))
1897+
self.check_decode(cp, tests)
1898+
1899+
def test_error_handlers(self):
1900+
self.check_encode(932, (
1901+
('\xff', 'backslashreplace', b'\\xff'),
1902+
('\xff', 'xmlcharrefreplace', b'&#255;'),
1903+
))
1904+
self.check_decode(932, (
1905+
(b'\xff', 'surrogateescape', '\udcff'),
1906+
))
1907+
if self.vista_or_later:
1908+
self.check_encode(self.CP_UTF8, (
1909+
('\udc80', 'surrogatepass', b'\xed\xb2\x80'),
1910+
))
1911+
1912+
def test_multibyte_encoding(self):
1913+
self.check_decode(932, (
1914+
(b'\x84\xe9\x80', 'ignore', '\u9a3e'),
1915+
(b'\x84\xe9\x80', 'replace', '\ufffd\u9a3e'),
1916+
))
1917+
self.check_decode(self.CP_UTF8, (
1918+
(b'\xff\xf4\x8f\xbf\xbf', 'ignore', '\U0010ffff'),
1919+
(b'\xff\xf4\x8f\xbf\xbf', 'replace', '\ufffd\U0010ffff'),
1920+
))
1921+
if self.vista_or_later:
1922+
self.check_encode(self.CP_UTF8, (
1923+
('[\U0010ffff\uDC80]', 'ignore', b'[\xf4\x8f\xbf\xbf]'),
1924+
('[\U0010ffff\uDC80]', 'replace', b'[\xf4\x8f\xbf\xbf?]'),
1925+
))
1926+
1927+
def test_incremental(self):
1928+
decoded = codecs.code_page_decode(932,
1929+
b'\xe9\x80\xe9', 'strict',
1930+
False)
1931+
self.assertEqual(decoded, ('\u9a3e', 2))
1932+
1933+
decoded = codecs.code_page_decode(932,
1934+
b'\xe9\x80\xe9\x80', 'strict',
1935+
False)
1936+
self.assertEqual(decoded, ('\u9a3e\u9a3e', 4))
1937+
1938+
decoded = codecs.code_page_decode(932,
1939+
b'abc', 'strict',
1940+
False)
1941+
self.assertEqual(decoded, ('abc', 3))
1942+
1943+
17471944
def test_main():
17481945
support.run_unittest(
17491946
UTF32Test,
@@ -1772,6 +1969,7 @@ def test_main():
17721969
SurrogateEscapeTest,
17731970
BomTest,
17741971
TransformCodecTest,
1972+
CodePageTest,
17751973
)
17761974

17771975

‎Misc/NEWS‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ What's New in Python 3.3 Alpha 1?
1010
Core and Builtins
1111
-----------------
1212

13+
- Issue #12281: Rewrite the MBCS codec to handle correctly replace and ignore
14+
error handlers on all Windows versions. The MBCS codec is now supporting all
15+
error handlers, instead of only replace to encode and ignore to decode.
16+
1317
- Issue #13188: When called without an explicit traceback argument,
1418
generator.throw() now gets the traceback from the passed exception's
1519
``__traceback__`` attribute. Patch by Petri Lehtinen.

‎Modules/_codecsmodule.c‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -612,6 +612,31 @@ mbcs_decode(PyObject *self,
612612
return codec_tuple(decoded, consumed);
613613
}
614614

615+
static PyObject *
616+
code_page_decode(PyObject *self,
617+
PyObject *args)
618+
{
619+
Py_buffer pbuf;
620+
const char *errors = NULL;
621+
int final = 0;
622+
Py_ssize_t consumed;
623+
PyObject *decoded = NULL;
624+
int code_page;
625+
626+
if (!PyArg_ParseTuple(args, "iy*|zi:code_page_decode",
627+
&code_page, &pbuf, &errors, &final))
628+
return NULL;
629+
consumed = pbuf.len;
630+
631+
decoded = PyUnicode_DecodeCodePageStateful(code_page,
632+
pbuf.buf, pbuf.len, errors,
633+
final ? NULL : &consumed);
634+
PyBuffer_Release(&pbuf);
635+
if (decoded == NULL)
636+
return NULL;
637+
return codec_tuple(decoded, consumed);
638+
}
639+
615640
#endif /* HAVE_MBCS */
616641

617642
/* --- Encoder ------------------------------------------------------------ */
@@ -1011,6 +1036,29 @@ mbcs_encode(PyObject *self,
10111036
return v;
10121037
}
10131038

1039+
static PyObject *
1040+
code_page_encode(PyObject *self,
1041+
PyObject *args)
1042+
{
1043+
PyObject *str, *v;
1044+
const char *errors = NULL;
1045+
int code_page;
1046+
1047+
if (!PyArg_ParseTuple(args, "iO|z:code_page_encode",
1048+
&code_page, &str, &errors))
1049+
return NULL;
1050+
1051+
str = PyUnicode_FromObject(str);
1052+
if (str == NULL)
1053+
return NULL;
1054+
v = codec_tuple(PyUnicode_EncodeCodePage(code_page,
1055+
str,
1056+
errors),
1057+
PyUnicode_GET_LENGTH(str));
1058+
Py_DECREF(str);
1059+
return v;
1060+
}
1061+
10141062
#endif /* HAVE_MBCS */
10151063

10161064
/* --- Error handler registry --------------------------------------------- */
@@ -1101,6 +1149,8 @@ static PyMethodDef _codecs_functions[] = {
11011149
#ifdef HAVE_MBCS
11021150
{"mbcs_encode", mbcs_encode, METH_VARARGS},
11031151
{"mbcs_decode", mbcs_decode, METH_VARARGS},
1152+
{"code_page_encode", code_page_encode, METH_VARARGS},
1153+
{"code_page_decode", code_page_decode, METH_VARARGS},
11041154
#endif
11051155
{"register_error", register_error, METH_VARARGS,
11061156
register_error__doc__},

0 commit comments

Comments
 (0)