Skip to content

gh-126004: fix positions handling in codecs.replace_errors #127674

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
picnixz merged 17 commits into from
Jan 23, 2025
Merged

gh-126004: fix positions handling in codecs.replace_errors #127674

Changes from 1 commit
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
6dbcdea
fix `codecs.replace_errors`
picnixz Dec 6, 2024
193c051
blurb
picnixz Dec 6, 2024
a9140f5
fix tests
picnixz Dec 6, 2024
98db6a2
fix tests
picnixz Dec 6, 2024
37ffb43
cosmetic changes
picnixz Dec 6, 2024
4c20312
Merge remote-tracking branch 'upstream/main' into fix/codecs/replace-…
picnixz Jan 3, 2025
4fe70a0
use internal `_PyUnicodeError_GetParams` helper
picnixz Jan 3, 2025
24bb030
fixup
picnixz Jan 3, 2025
5fa095b
Merge branch 'main' into fix/codecs/replace-errors-126004
picnixz Jan 14, 2025
f1c6339
Merge branch 'main' into fix/codecs/replace-errors-126004
picnixz Jan 21, 2025
31235a7
update usages of `_PyUnicodeError_GetParams`
picnixz Jan 21, 2025
7626ceb
amend some cosmetic changes to be consistent
picnixz Jan 21, 2025
fbe1930
Merge branch 'main' into fix/codecs/replace-errors-126004
picnixz Jan 22, 2025
306b8f6
remove unnecessary cast and comment
picnixz Jan 22, 2025
d71b3a5
use memset instead of a for-loop
picnixz Jan 22, 2025
39951be
Revert "use memset instead of a for-loop"
picnixz Jan 22, 2025
b8ea97b
Merge branch 'main' into fix/codecs/replace-errors-126004
picnixz Jan 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
update usages of _PyUnicodeError_GetParams
  • Loading branch information
@picnixz
picnixz committed Jan 21, 2025
commit 31235a71254dca0a4b91c6cc79c14ed25b77e7b6
35 changes: 14 additions & 21 deletions Python/codecs.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -700,52 +700,48 @@ PyObject *PyCodec_IgnoreErrors(PyObject *exc)
}


PyObject *PyCodec_ReplaceErrors(PyObject *exc)
PyObject *
PyCodec_ReplaceErrors(PyObject *exc)
{
Py_ssize_t start, end;
Py_ssize_t start, end, slen;

if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeEncodeError)) {
if (_PyUnicodeError_GetParams(exc, NULL, NULL, &start, &end, false) < 0) {
if (_PyUnicodeError_GetParams(exc, NULL, NULL,
&start, &end, &slen, false) < 0) {
return NULL;
}
if (end <= start) {
goto oob;
}
Py_ssize_t len = end - start;
PyObject *res = PyUnicode_New(len, '?');
PyObject *res = PyUnicode_New(slen, '?');
if (res == NULL) {
return NULL;
}
assert(PyUnicode_KIND(res) == PyUnicode_1BYTE_KIND);
Py_UCS1 *outp = PyUnicode_1BYTE_DATA(res);
memset(outp, (int)'?', sizeof(Py_UCS1) * len);
memset(outp, (int)'?', sizeof(Py_UCS1) * slen);
Comment thread
picnixz marked this conversation as resolved.
Outdated
assert(_PyUnicode_CheckConsistency(res, 1));
return Py_BuildValue("(Nn)", res, end);
}
else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeDecodeError)) {
// _PyUnicodeError_GetParams() is slightly faster than the public getter
Comment thread
picnixz marked this conversation as resolved.
Outdated
if (_PyUnicodeError_GetParams(exc, NULL, NULL, NULL, &end, true) < 0) {
if (_PyUnicodeError_GetParams(exc, NULL, NULL,
NULL, &end, NULL, true) < 0) {
return NULL;
}
return Py_BuildValue("(Cn)",
(int)Py_UNICODE_REPLACEMENT_CHARACTER,
end);
}
else if (PyObject_TypeCheck(exc, (PyTypeObject *)PyExc_UnicodeTranslateError)) {
if (_PyUnicodeError_GetParams(exc, NULL, NULL, &start, &end, false) < 0) {
if (_PyUnicodeError_GetParams(exc, NULL, NULL,
&start, &end, &slen, false) < 0) {
return NULL;
}
if (end <= start) {
goto oob;
}
Py_ssize_t len = end - start;
PyObject *res = PyUnicode_New(len, Py_UNICODE_REPLACEMENT_CHARACTER);
PyObject *res = PyUnicode_New(slen, Py_UNICODE_REPLACEMENT_CHARACTER);
if (res == NULL) {
return NULL;
}
assert(PyUnicode_KIND(res) == PyUnicode_2BYTE_KIND);
assert(slen == 0 || PyUnicode_KIND(res) == PyUnicode_2BYTE_KIND);
Py_UCS2 *outp = PyUnicode_2BYTE_DATA(res);
for (Py_ssize_t i = 0; i < len; ++i) {
for (Py_ssize_t i = 0; i < slen; ++i) {
outp[i] = Py_UNICODE_REPLACEMENT_CHARACTER;
}
assert(_PyUnicode_CheckConsistency(res, 1));
Expand All @@ -755,9 +751,6 @@ PyObject *PyCodec_ReplaceErrors(PyObject *exc)
wrong_exception_type(exc);
return NULL;
}

oob:
return Py_BuildValue("(Nn)", Py_GetConstant(Py_CONSTANT_EMPTY_STR), end);
}

PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc)
Expand Down