Skip to content

Commit 88d8fb6

Browse files
Issue python#13916: Disallowed the surrogatepass error handler for non UTF-*
encodings.
1 parent 8e4efbe commit 88d8fb6

3 files changed

Lines changed: 35 additions & 4 deletions

File tree

‎Lib/test/test_codecs.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2807,6 +2807,9 @@ def test_cp932(self):
28072807
('[\u20ac]', 'replace', b'[?]'),
28082808
('[\xff]', 'backslashreplace', b'[\\xff]'),
28092809
('[\xff]', 'xmlcharrefreplace', b'[ÿ]'),
2810+
('\udcff', 'strict', None),
2811+
('[\udcff]', 'surrogateescape', b'[\xff]'),
2812+
('[\udcff]', 'surrogatepass', None),
28102813
))
28112814
self.check_decode(932, (
28122815
(b'abc', 'strict', 'abc'),
@@ -2816,6 +2819,7 @@ def test_cp932(self):
28162819
(b'[\xff]', 'ignore', '[]'),
28172820
(b'[\xff]', 'replace', '[\ufffd]'),
28182821
(b'[\xff]', 'surrogateescape', '[\udcff]'),
2822+
(b'[\xff]', 'surrogatepass', None),
28192823
(b'\x81\x00abc', 'strict', None),
28202824
(b'\x81\x00abc', 'ignore', '\x00abc'),
28212825
(b'\x81\x00abc', 'replace', '\ufffd\x00abc'),
@@ -2826,14 +2830,23 @@ def test_cp1252(self):
28262830
('abc', 'strict', b'abc'),
28272831
('\xe9\u20ac', 'strict', b'\xe9\x80'),
28282832
('\xff', 'strict', b'\xff'),
2833+
# test error handlers
28292834
('\u0141', 'strict', None),
28302835
('\u0141', 'ignore', b''),
28312836
('\u0141', 'replace', b'L'),
2837+
('\udc98', 'surrogateescape', b'\x98'),
2838+
('\udc98', 'surrogatepass', None),
28322839
))
28332840
self.check_decode(1252, (
28342841
(b'abc', 'strict', 'abc'),
28352842
(b'\xe9\x80', 'strict', '\xe9\u20ac'),
28362843
(b'\xff', 'strict', '\xff'),
2844+
# invalid bytes
2845+
(b'[\x98]', 'strict', None),
2846+
(b'[\x98]', 'ignore', '[]'),
2847+
(b'[\x98]', 'replace', '[\ufffd]'),
2848+
(b'[\x98]', 'surrogateescape', '[\udc98]'),
2849+
(b'[\x98]', 'surrogatepass', None),
28372850
))
28382851

28392852
def test_cp_utf7(self):

‎Misc/NEWS‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,9 @@ Core and Builtins
8484
Library
8585
-------
8686

87+
- Issue #13916: Disallowed the surrogatepass error handler for non UTF-*
88+
encodings.
89+
8790
- Issue #20998: Fixed re.fullmatch() of repeated single character pattern
8891
with ignore case. Original patch by Matthew Barnett.
8992

‎Python/codecs.c‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -901,6 +901,7 @@ PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc)
901901
}
902902
}
903903

904+
#define ENC_UNKNOWN -1
904905
#define ENC_UTF8 0
905906
#define ENC_UTF16BE 1
906907
#define ENC_UTF16LE 2
@@ -916,7 +917,11 @@ get_standard_encoding(const char *encoding, int *bytelength)
916917
encoding += 3;
917918
if (*encoding == '-' || *encoding == '_' )
918919
encoding++;
919-
if (encoding[0] == '1' && encoding[1] == '6') {
920+
if (encoding[0] == '8' && encoding[1] == '\0') {
921+
*bytelength = 3;
922+
return ENC_UTF8;
923+
}
924+
else if (encoding[0] == '1' && encoding[1] == '6') {
920925
encoding += 2;
921926
*bytelength = 2;
922927
if (*encoding == '\0') {
@@ -955,9 +960,7 @@ get_standard_encoding(const char *encoding, int *bytelength)
955960
}
956961
}
957962
}
958-
/* utf-8 */
959-
*bytelength = 3;
960-
return ENC_UTF8;
963+
return ENC_UNKNOWN;
961964
}
962965

963966
/* This handler is declared static until someone demonstrates
@@ -994,6 +997,12 @@ PyCodec_SurrogatePassErrors(PyObject *exc)
994997
}
995998
code = get_standard_encoding(encoding, &bytelength);
996999
Py_DECREF(encode);
1000+
if (code == ENC_UNKNOWN) {
1001+
/* Not supported, fail with original exception */
1002+
PyErr_SetObject(PyExceptionInstance_Class(exc), exc);
1003+
Py_DECREF(object);
1004+
return NULL;
1005+
}
9971006

9981007
res = PyBytes_FromStringAndSize(NULL, bytelength*(end-start));
9991008
if (!res) {
@@ -1068,6 +1077,12 @@ PyCodec_SurrogatePassErrors(PyObject *exc)
10681077
}
10691078
code = get_standard_encoding(encoding, &bytelength);
10701079
Py_DECREF(encode);
1080+
if (code == ENC_UNKNOWN) {
1081+
/* Not supported, fail with original exception */
1082+
PyErr_SetObject(PyExceptionInstance_Class(exc), exc);
1083+
Py_DECREF(object);
1084+
return NULL;
1085+
}
10711086

10721087
/* Try decoding a single surrogate character. If
10731088
there are more, let the codec call us again. */

0 commit comments

Comments
 (0)