Skip to content

Commit 4bd9545

Browse files
authored
Leave a byte that starts no UTF-8 sequence at the end of a console write (#8621)
* Leave a byte that starts no UTF-8 sequence at the end of a console write `find_last_utf8_boundary` read every byte at or above 0xc0 as the start of a sequence, giving 0xf8..=0xff an expected length of 4 and cutting the tail back to before it. `_find_last_utf8_boundary` stops the four-byte range at 0xf8 and returns the full length for anything above it, leaving the byte for MultiByteToWideChar to answer with U+FFFD. `write_console_utf8` therefore reported fewer bytes written than it was given whenever one of those bytes fell in the last three, and for a buffer that began with one it cut the length to 0, converted nothing and returned Ok(0) -- a write that never advances. The scan also ran a fourth iteration the C loop does not; the arms happen to answer the same there, but the bound is now the same 3. * Drop the expectedFailure on test_winconsoleio's test_write The marker went stale with the boundary fix in the parent commit. The test writes `b'\xff'*10` and `data + b'\xff'` to CONOUT$ and asserts the returned count, which is the case `find_last_utf8_boundary` was answering wrong, so regrtest now reports it as an unexpected success.
1 parent a783ae8 commit 4bd9545

2 files changed

Lines changed: 59 additions & 7 deletions

File tree

‎Lib/test/test_winconsoleio.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,6 @@ def test_write_empty_data(self):
142142
with ConIO('CONOUT$', 'w') as f:
143143
self.assertEqual(f.write(b''), 0)
144144

145-
@unittest.expectedFailureIfWindows("TODO: RUSTPYTHON")
146145
@requires_resource('console')
147146
def test_write(self):
148147
testcases = []

‎crates/host_env/src/nt.rs‎

Lines changed: 59 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1724,27 +1724,41 @@ fn copy_from_small_buf(buf: &mut [u8; 4], dest: &mut [u8]) -> usize {
17241724
n
17251725
}
17261726

1727+
/// The length to cut `buf` down to so it ends on a whole UTF-8 sequence:
1728+
/// `_find_last_utf8_boundary` in `Modules/_io/winconsoleio.c`.
1729+
///
1730+
/// Only the last three bytes can begin a sequence the buffer does not also
1731+
/// end, so the scan stops there. A byte from `0xf8` up starts nothing at all
1732+
/// -- the four-byte sequences end at `0xf7` -- and an invalid byte is left in
1733+
/// place for the decoder to answer with U+FFFD, the same as any other byte
1734+
/// that is not the start of an incomplete sequence.
17271735
fn find_last_utf8_boundary(buf: &[u8], len: usize) -> usize {
17281736
let len = len.min(buf.len());
1729-
for count in 1..=4.min(len) {
1737+
for count in 1..=3.min(len) {
17301738
let c = buf[len - count];
17311739
if c < 0x80 {
1740+
// No starting byte found.
17321741
return len;
17331742
}
17341743
if c >= 0xc0 {
1735-
let expected = if c < 0xe0 {
1736-
2
1744+
let incomplete = if c < 0xe0 {
1745+
// 2-byte sequence
1746+
count < 2
17371747
} else if c < 0xf0 {
1738-
3
1748+
// 3-byte sequence
1749+
count < 3
17391750
} else {
1740-
4
1751+
// 4-byte sequence, or a byte that starts no sequence.
1752+
c < 0xf8
17411753
};
1742-
if count < expected {
1754+
if incomplete {
17431755
return len - count;
17441756
}
1757+
// Either complete or invalid sequence.
17451758
return len;
17461759
}
17471760
}
1761+
// Either a complete 4-byte sequence or an invalid one.
17481762
len
17491763
}
17501764

@@ -2157,3 +2171,42 @@ pub fn execve(
21572171
Ok(())
21582172
}
21592173
}
2174+
2175+
#[cfg(test)]
2176+
mod find_last_utf8_boundary_tests {
2177+
use super::find_last_utf8_boundary;
2178+
2179+
#[test]
2180+
fn keeps_a_complete_tail() {
2181+
assert_eq!(find_last_utf8_boundary(b"abc", 3), 3);
2182+
// 2-, 3- and 4-byte sequences, each ending exactly at `len`.
2183+
assert_eq!(find_last_utf8_boundary("a\u{a2}".as_bytes(), 3), 3);
2184+
assert_eq!(find_last_utf8_boundary("a\u{20ac}".as_bytes(), 4), 4);
2185+
assert_eq!(find_last_utf8_boundary("a\u{10348}".as_bytes(), 5), 5);
2186+
}
2187+
2188+
#[test]
2189+
fn cuts_an_incomplete_tail() {
2190+
assert_eq!(find_last_utf8_boundary(&[b'a', 0xc2], 2), 1);
2191+
assert_eq!(find_last_utf8_boundary(&[b'a', 0xe2, 0x82], 3), 1);
2192+
assert_eq!(find_last_utf8_boundary(&[b'a', 0xf0, 0x90, 0x8d], 4), 1);
2193+
}
2194+
2195+
#[test]
2196+
fn leaves_a_byte_that_starts_no_sequence() {
2197+
// 0xf8..=0xff begin nothing, so the tail is whole as it stands and
2198+
// the decoder answers each byte with U+FFFD.
2199+
for c in 0xf8..=0xffu8 {
2200+
assert_eq!(find_last_utf8_boundary(&[b'a', c], 2), 2, "{c:#04x}");
2201+
assert_eq!(find_last_utf8_boundary(&[c], 1), 1, "{c:#04x}");
2202+
}
2203+
// A continuation byte alone is not a start either.
2204+
assert_eq!(find_last_utf8_boundary(&[b'a', 0x80, 0x80, 0x80], 4), 4);
2205+
}
2206+
2207+
#[test]
2208+
fn handles_an_empty_buffer() {
2209+
assert_eq!(find_last_utf8_boundary(b"", 0), 0);
2210+
assert_eq!(find_last_utf8_boundary(b"ab", 9), 2);
2211+
}
2212+
}

0 commit comments

Comments
 (0)