Skip to content

Commit 45b9c2e

Browse files
devyubindevyubin
authored
Replace malformed unicode error helper calls with the _real variants (#8359)
* Replace malformed unicode error helper calls with the _real variants new_unicode_decode_error / new_unicode_encode_error build the exception from a bare message without running the initializer, so the result has none of the five attributes a unicode error must carry and str() renders as an empty string. Convert the non-Windows call sites in csv, socket, getlogin and the fs-path decoders to the _real constructors, passing the source bytes/str and the failing offset from the captured Utf8Error. The Windows-gated sites (nt, mbcs/oem codecs) and the sites whose source object is not reachable (uname, array) are left for follow-ups. Assisted-by: Claude Code:claude-fable-5 * Report the full invalid UTF-8 span in the converted decode errors The conversions used valid_up_to() + 1 for the decode-error end offset, which under-reports multi-byte invalid sequences. Take the span from the Utf8Error instead: valid_up_to() + error_len(), or the input length for a truncated sequence (error_len() == None), matching CPython. Assisted-by: Claude Code:claude-opus-4-8
1 parent 3562c7c commit 45b9c2e

5 files changed

Lines changed: 85 additions & 22 deletions

File tree

‎crates/stdlib/src/csv.rs‎

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,21 @@ mod _csv {
6060
vm.new_exception_msg(super::_csv::error(vm), msg.into())
6161
}
6262

63+
fn new_not_utf8_error(
64+
vm: &VirtualMachine,
65+
bytes: &[u8],
66+
err: core::str::Utf8Error,
67+
) -> PyBaseExceptionRef {
68+
vm.new_unicode_decode_error_real(
69+
vm.ctx.new_str("utf-8"),
70+
vm.ctx.new_bytes(bytes.to_vec()),
71+
err.valid_up_to(),
72+
err.error_len()
73+
.map_or(bytes.len(), |n| err.valid_up_to() + n),
74+
vm.ctx.new_str("csv not utf8"),
75+
)
76+
}
77+
6378
#[pyattr]
6479
#[pyclass(module = "csv", name = "Dialect")]
6580
#[derive(Debug, PyPayload, Clone, Copy)]
@@ -1111,8 +1126,8 @@ mod _csv {
11111126
{
11121127
return Ok(vm.ctx.none());
11131128
}
1114-
let field = core::str::from_utf8(&field)
1115-
.map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
1129+
let field =
1130+
core::str::from_utf8(&field).map_err(|e| new_not_utf8_error(vm, &field, e))?;
11161131
Ok(vm.ctx.new_str(field).into())
11171132
})
11181133
.collect()
@@ -1248,7 +1263,7 @@ mod _csv {
12481263
prev_end = end;
12491264
let s = core::str::from_utf8(&buffer[range.clone()])
12501265
// not sure if this is possible - the input was all strings
1251-
.map_err(|_e| vm.new_unicode_decode_error("csv not utf8"))?;
1266+
.map_err(|e| new_not_utf8_error(vm, &buffer[range.clone()], e))?;
12521267

12531268
// TODO: RUSTPYTHON; Incomplete implementation
12541269
if let QuoteStyle::Nonnumeric = zelf.dialect.quoting {
@@ -1423,8 +1438,8 @@ mod _csv {
14231438
}
14241439

14251440
write_lineterminator(&mut output, self.dialect.lineterminator);
1426-
let s = core::str::from_utf8(&output)
1427-
.map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
1441+
let s =
1442+
core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?;
14281443
self.write.call((s,), vm)
14291444
}
14301445

@@ -1469,8 +1484,8 @@ mod _csv {
14691484

14701485
write_lineterminator(&mut output, self.dialect.lineterminator);
14711486

1472-
let s = core::str::from_utf8(&output)
1473-
.map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
1487+
let s =
1488+
core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?;
14741489

14751490
self.write.call((s,), vm)
14761491
}
@@ -1518,8 +1533,8 @@ mod _csv {
15181533

15191534
write_lineterminator(&mut output, self.dialect.lineterminator);
15201535

1521-
let s = core::str::from_utf8(&output)
1522-
.map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
1536+
let s =
1537+
core::str::from_utf8(&output).map_err(|e| new_not_utf8_error(vm, &output, e))?;
15231538

15241539
self.write.call((s,), vm)
15251540
}
@@ -1593,7 +1608,7 @@ mod _csv {
15931608
}
15941609

15951610
let s = core::str::from_utf8(&buffer[..buffer_offset])
1596-
.map_err(|_| vm.new_unicode_decode_error("csv not utf8"))?;
1611+
.map_err(|e| new_not_utf8_error(vm, &buffer[..buffer_offset], e))?;
15971612

15981613
self.write.call((s,), vm)
15991614
}

‎crates/stdlib/src/socket.rs‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2612,8 +2612,15 @@ mod _socket {
26122612
}
26132613
Some(ArgStrOrBytesLike::Buf(b)) => {
26142614
let bytes = b.borrow_buf();
2615-
let host_str = core::str::from_utf8(&bytes)
2616-
.map_err(|_| vm.new_unicode_decode_error("host bytes is not utf8"))?;
2615+
let host_str = core::str::from_utf8(&bytes).map_err(|e| {
2616+
vm.new_unicode_decode_error_real(
2617+
vm.ctx.new_str("utf-8"),
2618+
vm.ctx.new_bytes(bytes.to_vec()),
2619+
e.valid_up_to(),
2620+
e.error_len().map_or(bytes.len(), |n| e.valid_up_to() + n),
2621+
vm.ctx.new_str("host bytes is not utf8"),
2622+
)
2623+
})?;
26172624
Some(host_str.to_owned())
26182625
}
26192626
None => None,
@@ -2627,14 +2634,35 @@ mod _socket {
26272634
ArgStrOrBytesLike::Str(s) => {
26282635
// For str, check for surrogates and raise UnicodeEncodeError if found
26292636
s.to_str()
2630-
.ok_or_else(|| vm.new_unicode_encode_error("surrogates not allowed"))?
2637+
.ok_or_else(|| {
2638+
let start = s
2639+
.as_wtf8()
2640+
.code_points()
2641+
.position(|c| c.to_char().is_none())
2642+
.unwrap();
2643+
vm.new_unicode_encode_error_real(
2644+
vm.ctx.new_str("utf-8"),
2645+
(*s).clone(),
2646+
start,
2647+
start + 1,
2648+
vm.ctx.new_str("surrogates not allowed"),
2649+
)
2650+
})?
26312651
.to_owned()
26322652
}
26332653
ArgStrOrBytesLike::Buf(b) => {
26342654
// For bytes, check if it's valid UTF-8
26352655
let bytes = b.borrow_buf();
26362656
core::str::from_utf8(&bytes)
2637-
.map_err(|_| vm.new_unicode_decode_error("port is not utf8"))?
2657+
.map_err(|e| {
2658+
vm.new_unicode_decode_error_real(
2659+
vm.ctx.new_str("utf-8"),
2660+
vm.ctx.new_bytes(bytes.to_vec()),
2661+
e.valid_up_to(),
2662+
e.error_len().map_or(bytes.len(), |n| e.valid_up_to() + n),
2663+
vm.ctx.new_str("port is not utf8"),
2664+
)
2665+
})?
26382666
.to_owned()
26392667
}
26402668
};

‎crates/vm/src/function/fspath.rs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,15 @@ impl FsPath {
125125
}
126126

127127
pub fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> {
128-
rustpython_host_env::os::bytes_as_os_str(b)
129-
.map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8"))
128+
rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| {
129+
vm.new_unicode_decode_error_real(
130+
vm.ctx.new_str("utf-8"),
131+
vm.ctx.new_bytes(b.to_vec()),
132+
e.valid_up_to(),
133+
e.error_len().map_or(b.len(), |n| e.valid_up_to() + n),
134+
vm.ctx.new_str("can't decode path for utf-8"),
135+
)
136+
})
130137
}
131138
}
132139

‎crates/vm/src/stdlib/os.rs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -130,8 +130,15 @@ pub(super) struct FollowSymlinks(
130130

131131
#[cfg(not(windows))]
132132
fn bytes_as_os_str<'a>(b: &'a [u8], vm: &VirtualMachine) -> PyResult<&'a std::ffi::OsStr> {
133-
rustpython_host_env::os::bytes_as_os_str(b)
134-
.map_err(|_| vm.new_unicode_decode_error("can't decode path for utf-8"))
133+
rustpython_host_env::os::bytes_as_os_str(b).map_err(|e| {
134+
vm.new_unicode_decode_error_real(
135+
vm.ctx.new_str("utf-8"),
136+
vm.ctx.new_bytes(b.to_vec()),
137+
e.valid_up_to(),
138+
e.error_len().map_or(b.len(), |n| e.valid_up_to() + n),
139+
vm.ctx.new_str("can't decode path for utf-8"),
140+
)
141+
})
135142
}
136143

137144
pub(crate) fn warn_if_bool_fd(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<()> {

‎crates/vm/src/stdlib/posix.rs‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1731,10 +1731,16 @@ pub mod module {
17311731
let Some(login) = rustpython_host_env::posix::getlogin() else {
17321732
return Err(vm.new_os_error("unable to determine login name"));
17331733
};
1734-
login
1735-
.to_str()
1736-
.map(|s| s.to_owned())
1737-
.map_err(|e| vm.new_unicode_decode_error(format!("unable to decode login name: {e}")))
1734+
login.to_str().map(|s| s.to_owned()).map_err(|e| {
1735+
vm.new_unicode_decode_error_real(
1736+
vm.ctx.new_str("utf-8"),
1737+
vm.ctx.new_bytes(login.as_bytes().to_vec()),
1738+
e.valid_up_to(),
1739+
e.error_len()
1740+
.map_or(login.as_bytes().len(), |n| e.valid_up_to() + n),
1741+
vm.ctx.new_str("unable to decode login name"),
1742+
)
1743+
})
17381744
}
17391745

17401746
// cfg from nix

0 commit comments

Comments
 (0)