Skip to content

Commit 699913d

Browse files
youknowoneyouknowone
authored
Align codec, call, exception, and Unix module behavior with CPython (#8647)
* Collect consecutive charmap encoding failures Pass one contiguous run of unmappable characters to the codec error handler, and use the full run when a text replacement is itself unmappable. Assisted-by: Codex:gpt-5 * Leave conversions out of standalone format specs A conversion belongs to a replacement field rather than the format specification consumed by format(), so stop accepting and storing it in FormatSpec. Assisted-by: Codex:gpt-5 * Decode empty byte strings without looking up a codec Match PyUnicode_FromEncodedObject by returning the shared empty string before resolving the requested encoding for empty bytes and bytearray inputs. Assisted-by: Codex:gpt-5 * Validate call shape before converting builtin arguments Reject unknown keywords and excess arguments before conversions can raise a different error. Align Python-function positional-only handling, use declared list arguments in heapq, keep delegated codecs positional-only, and preserve CPython parameter names and validation order. Assisted-by: Codex:gpt-5 * Align attribute and exception initialization errors Give __dict__ and type __qualname__ their CPython diagnostics, preserve OSError filename fields correctly, accept seven-item SyntaxError details, and report Unicode error constructor arity before converting values. Assisted-by: Codex:gpt-5 * Expose missing Unix and Darwin module APIs Add the platform constants exported by fcntl, mmap, select, socket, syslog, termios, stat, errno, time, and _sre. Implement posix.killpg and resource.getpagesize, and expose resource.error as OSError. Assisted-by: Codex:gpt-5 * Honor the configured Python CPU count Parse -X cpu_count and PYTHON_CPU_COUNT, expose the stored value through sys._get_cpu_count_config, and let os.cpu_count prefer it over the host count. Also publish getsizeof with its CPython parameter name. Assisted-by: Codex:gpt-5 * Fix CPU count and WASI CI coverage Remove the obsolete expected-failure marker now that CPU count overrides pass, and avoid exporting FASYNC where WASI libc has no O_ASYNC constant. Assisted-by: Codex:gpt-5 * self review
1 parent 9b5e550 commit 699913d

38 files changed

Lines changed: 593 additions & 331 deletions

‎.cspell.dict/cpython.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -293,3 +293,4 @@ XSETREF
293293
xibufferview
294294
xstat
295295
XXPRIME
296+
ENOTCAPABLE

‎Lib/_pycodecs.py‎

Lines changed: 37 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1687,6 +1687,39 @@ def charmapencode_output(c, mapping):
16871687
raise TypeError("character mapping must return integer, None or str")
16881688

16891689

1690+
def charmap_encoding_error(p, size, inpos, mapping, errors, res):
1691+
# /* find all unencodable characters */
1692+
collendpos = inpos + 1
1693+
while collendpos < size:
1694+
try:
1695+
charmapencode_output(ord(p[collendpos]), mapping)
1696+
except KeyError:
1697+
collendpos += 1
1698+
else:
1699+
break
1700+
1701+
replacement, newpos = unicode_call_errorhandler(
1702+
errors,
1703+
"charmap",
1704+
"character maps to <undefined>",
1705+
p,
1706+
inpos,
1707+
collendpos,
1708+
False,
1709+
)
1710+
if isinstance(replacement, bytes):
1711+
res += list(replacement)
1712+
else:
1713+
for y in replacement:
1714+
try:
1715+
res += charmapencode_output(ord(y), mapping)
1716+
except KeyError:
1717+
raise UnicodeEncodeError(
1718+
"charmap", p, inpos, collendpos, "character maps to <undefined>"
1719+
)
1720+
return newpos
1721+
1722+
16901723
def PyUnicode_EncodeCharmap(p, size, mapping="latin-1", errors="strict"):
16911724
## /* the following variable is used for caching string comparisons
16921725
## * -1=not initialized, 0=unknown, 1=strict, 2=replace,
@@ -1703,29 +1736,11 @@ def PyUnicode_EncodeCharmap(p, size, mapping="latin-1", errors="strict"):
17031736
# /* try to encode it */
17041737
try:
17051738
x = charmapencode_output(ord(p[inpos]), mapping)
1706-
res += x
17071739
except KeyError:
1708-
x = unicode_call_errorhandler(
1709-
errors,
1710-
"charmap",
1711-
"character maps to <undefined>",
1712-
p,
1713-
inpos,
1714-
inpos + 1,
1715-
False,
1716-
)
1717-
replacement = x[0]
1718-
if isinstance(replacement, bytes):
1719-
res += list(replacement)
1720-
else:
1721-
try:
1722-
for y in replacement:
1723-
res += charmapencode_output(ord(y), mapping)
1724-
except KeyError:
1725-
raise UnicodeEncodeError(
1726-
"charmap", p, inpos, inpos + 1, "character maps to <undefined>"
1727-
)
1728-
inpos += 1
1740+
inpos = charmap_encoding_error(p, size, inpos, mapping, errors, res)
1741+
else:
1742+
res += x
1743+
inpos += 1
17291744
return res
17301745

17311746

‎Lib/test/test_cmd_line.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1192,7 +1192,6 @@ def test_cmd_dedent_failcase(self):
11921192
code = template.replace('-', '\t').replace('+', ' ')
11931193
assert_python_failure('-c', code)
11941194

1195-
@unittest.expectedFailure # TODO: RUSTPYTHON
11961195
def test_cpu_count(self):
11971196
code = "import os; print(os.cpu_count(), os.process_cpu_count())"
11981197
res = assert_python_ok('-X', 'cpu_count=4321', '-c', code)

‎Lib/test/test_inspect/test_inspect.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2274,7 +2274,6 @@ def test_multiple_features(self):
22742274
'(4,[5,6])]), q=0, **collections.UserDict('
22752275
'y=9, z=10)')
22762276

2277-
@unittest.expectedFailure # TODO: RUSTPYTHON; + <lambda>() got an unexpected keyword argument 'x'
22782277
def test_errors(self):
22792278
f0 = self.makeCallable('')
22802279
f1 = self.makeCallable('a, b')

‎Lib/test/test_resource.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,6 @@ def __getitem__(self, key):
197197

198198
resource.setrlimit(resource.RLIMIT_CPU, BadSequence())
199199

200-
@unittest.expectedFailure # TODO: RUSTPYTHON; module 'resource' has no attribute 'getpagesize'
201200
def test_pagesize(self):
202201
pagesize = resource.getpagesize()
203202
self.assertIsInstance(pagesize, int)

‎crates/common/src/format.rs‎

Lines changed: 1 addition & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -39,18 +39,6 @@ pub enum FormatConversion {
3939
Ascii = b'b',
4040
}
4141

42-
impl FormatParse for FormatConversion {
43-
fn parse(text: &Wtf8) -> (Option<Self>, &Wtf8) {
44-
let Some(conversion) = Self::from_string(text) else {
45-
return (None, text);
46-
};
47-
let mut chars = text.code_points();
48-
chars.next(); // Consume the bang
49-
chars.next(); // Consume one r,s,a char
50-
(Some(conversion), chars.as_wtf8())
51-
}
52-
}
53-
5442
impl FormatConversion {
5543
#[must_use]
5644
pub fn from_char(c: CodePoint) -> Option<Self> {
@@ -61,15 +49,6 @@ impl FormatConversion {
6149
_ => None,
6250
}
6351
}
64-
65-
fn from_string(text: &Wtf8) -> Option<Self> {
66-
let mut chars = text.code_points();
67-
if chars.next()? != '!' {
68-
return None;
69-
}
70-
71-
Self::from_char(chars.next()?)
72-
}
7352
}
7453

7554
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
@@ -222,7 +201,6 @@ impl FormatParse for FormatType {
222201

223202
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
224203
pub struct FormatSpec {
225-
conversion: Option<FormatConversion>,
226204
fill: Option<CodePoint>,
227205
align: Option<FormatAlign>,
228206
align_specified: bool,
@@ -351,8 +329,6 @@ impl FormatSpec {
351329
}
352330

353331
fn _parse(text: &Wtf8) -> Result<Self, FormatSpecError> {
354-
// get_integer in CPython
355-
let (conversion, text) = FormatConversion::parse(text);
356332
let (mut fill, mut align, text) = parse_fill_and_align(text);
357333
let align_specified = align.is_some();
358334
let (sign, text) = FormatSign::parse(text);
@@ -381,7 +357,6 @@ impl FormatSpec {
381357
}
382358

383359
Ok(Self {
384-
conversion,
385360
fill,
386361
align,
387362
align_specified,
@@ -835,7 +810,6 @@ impl FormatSpec {
835810
/// parses to. Written as a destructure so a new field cannot be forgotten here.
836811
fn is_empty(&self) -> bool {
837812
let Self {
838-
conversion,
839813
fill,
840814
align,
841815
align_specified,
@@ -848,8 +822,7 @@ impl FormatSpec {
848822
frac_grouping_option,
849823
format_type,
850824
} = self;
851-
conversion.is_none()
852-
&& fill.is_none()
825+
fill.is_none()
853826
&& align.is_none()
854827
&& !align_specified
855828
&& sign.is_none()
@@ -1717,7 +1690,6 @@ mod tests {
17171690
#[test]
17181691
fn width_only() {
17191692
let expected = Ok(FormatSpec {
1720-
conversion: None,
17211693
fill: None,
17221694
align: None,
17231695
align_specified: false,
@@ -1736,7 +1708,6 @@ mod tests {
17361708
#[test]
17371709
fn fill_and_width() {
17381710
let expected = Ok(FormatSpec {
1739-
conversion: None,
17401711
fill: Some('<'.into()),
17411712
align: Some(FormatAlign::Right),
17421713
align_specified: true,
@@ -1755,7 +1726,6 @@ mod tests {
17551726
#[test]
17561727
fn all() {
17571728
let expected = Ok(FormatSpec {
1758-
conversion: None,
17591729
fill: Some('<'.into()),
17601730
align: Some(FormatAlign::Right),
17611731
align_specified: true,

‎crates/host_env/src/fcntl.rs‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,28 @@ pub use libc::{F_FULLFSYNC, F_NOCACHE};
1919
#[cfg(target_os = "freebsd")]
2020
pub use libc::{F_DUP2FD, F_DUP2FD_CLOEXEC};
2121

22-
#[cfg(any(target_os = "android", target_os = "linux"))]
22+
#[cfg(any(target_os = "android", target_os = "linux", target_vendor = "apple"))]
2323
pub use libc::{F_OFD_GETLK, F_OFD_SETLK, F_OFD_SETLKW};
2424

25+
/// `FASYNC` is what `fcntl.h` calls the flag `O_ASYNC` stands for.
26+
#[cfg(not(target_os = "wasi"))]
27+
pub const FASYNC: libc::c_int = libc::O_ASYNC;
28+
29+
#[cfg(target_vendor = "apple")]
30+
pub use libc::F_RDAHEAD;
31+
32+
/// The commands `fcntl.h` carries that libc does not name.
33+
#[cfg(target_vendor = "apple")]
34+
mod darwin_fcntl {
35+
pub const F_SETNOSIGPIPE: libc::c_int = 73;
36+
pub const F_GETNOSIGPIPE: libc::c_int = 74;
37+
pub const F_SETLEASE: libc::c_int = 106;
38+
pub const F_GETLEASE: libc::c_int = 107;
39+
}
40+
41+
#[cfg(target_vendor = "apple")]
42+
pub use darwin_fcntl::{F_GETLEASE, F_GETNOSIGPIPE, F_SETLEASE, F_SETNOSIGPIPE};
43+
2544
#[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))]
2645
pub use libc::{
2746
F_ADD_SEALS, F_GET_SEALS, F_GETLEASE, F_GETPIPE_SZ, F_NOTIFY, F_SEAL_GROW, F_SEAL_SEAL,

‎crates/host_env/src/mmap.rs‎

Lines changed: 76 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,84 @@ use std::io;
77

88
#[cfg(unix)]
99
pub use libc::{
10-
MADV_DONTNEED, MADV_NORMAL, MADV_RANDOM, MADV_SEQUENTIAL, MADV_WILLNEED, MAP_ANON,
11-
MAP_ANONYMOUS, MAP_PRIVATE, MAP_SHARED, PROT_EXEC, PROT_READ, PROT_WRITE,
10+
MADV_DONTNEED, MADV_NORMAL, MADV_RANDOM, MADV_SEQUENTIAL, MADV_WILLNEED, PROT_EXEC, PROT_READ,
11+
PROT_WRITE,
1212
};
1313

14+
#[cfg(unix)]
15+
bitflags::bitflags! {
16+
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
17+
pub struct MapFlags: libc::c_int {
18+
const MAP_SHARED = libc::MAP_SHARED;
19+
const MAP_PRIVATE = libc::MAP_PRIVATE;
20+
const MAP_ANON = libc::MAP_ANON;
21+
const MAP_ANONYMOUS = libc::MAP_ANONYMOUS;
22+
23+
#[cfg(all(target_os = "linux", target_arch = "x86_64", target_env = "gnu"))]
24+
const MAP_DENYWRITE = libc::MAP_DENYWRITE;
25+
#[cfg(all(target_os = "linux", target_arch = "x86_64", target_env = "gnu"))]
26+
const MAP_EXECUTABLE = libc::MAP_EXECUTABLE;
27+
#[cfg(all(target_os = "linux", target_arch = "x86_64", target_env = "gnu"))]
28+
const MAP_POPULATE = libc::MAP_POPULATE;
29+
30+
#[cfg(any(target_os = "linux", target_os = "openbsd", target_os = "netbsd"))]
31+
const MAP_STACK = libc::MAP_STACK;
32+
33+
#[cfg(target_vendor = "apple")]
34+
const MAP_NORESERVE = libc::MAP_NORESERVE;
35+
#[cfg(target_vendor = "apple")]
36+
const MAP_NOEXTEND = libc::MAP_NOEXTEND;
37+
#[cfg(target_vendor = "apple")]
38+
const MAP_HASSEMAPHORE = libc::MAP_HASSEMAPHORE;
39+
#[cfg(target_vendor = "apple")]
40+
const MAP_NOCACHE = libc::MAP_NOCACHE;
41+
#[cfg(target_vendor = "apple")]
42+
const MAP_JIT = libc::MAP_JIT;
43+
// libc does not name these `sys/mman.h` flags.
44+
#[cfg(target_vendor = "apple")]
45+
const MAP_RESILIENT_CODESIGN = 0x2000;
46+
#[cfg(target_vendor = "apple")]
47+
const MAP_RESILIENT_MEDIA = 0x4000;
48+
#[cfg(target_vendor = "apple")]
49+
const MAP_32BIT = 0x8000;
50+
#[cfg(target_vendor = "apple")]
51+
const MAP_TRANSLATED_ALLOW_EXECUTE = 0x20000;
52+
#[cfg(target_vendor = "apple")]
53+
const MAP_UNIX03 = 0x40000;
54+
#[cfg(target_vendor = "apple")]
55+
const MAP_TPRO = 0x80000;
56+
}
57+
}
58+
59+
#[cfg(unix)]
60+
macro_rules! map_flag_consts {
61+
($($name:ident),+ $(,)?) => {
62+
$(pub const $name: libc::c_int = MapFlags::$name.bits();)+
63+
};
64+
}
65+
66+
#[cfg(unix)]
67+
map_flag_consts! {
68+
MAP_SHARED, MAP_PRIVATE, MAP_ANON, MAP_ANONYMOUS,
69+
}
70+
71+
#[cfg(all(target_os = "linux", target_arch = "x86_64", target_env = "gnu"))]
72+
map_flag_consts! {
73+
MAP_DENYWRITE, MAP_EXECUTABLE, MAP_POPULATE,
74+
}
75+
76+
#[cfg(any(target_os = "linux", target_os = "openbsd", target_os = "netbsd"))]
77+
map_flag_consts! {
78+
MAP_STACK,
79+
}
80+
81+
#[cfg(target_vendor = "apple")]
82+
map_flag_consts! {
83+
MAP_HASSEMAPHORE, MAP_JIT, MAP_NOCACHE, MAP_NOEXTEND, MAP_NORESERVE,
84+
MAP_RESILIENT_CODESIGN, MAP_RESILIENT_MEDIA, MAP_32BIT,
85+
MAP_TRANSLATED_ALLOW_EXECUTE, MAP_UNIX03, MAP_TPRO,
86+
}
87+
1488
#[cfg(target_os = "macos")]
1589
pub use libc::{MADV_FREE_REUSABLE, MADV_FREE_REUSE};
1690

@@ -50,12 +124,6 @@ pub use libc::{
50124
))]
51125
pub use libc::MADV_SOFT_OFFLINE;
52126

53-
#[cfg(all(target_os = "linux", target_arch = "x86_64", target_env = "gnu"))]
54-
pub use libc::{MAP_DENYWRITE, MAP_EXECUTABLE, MAP_POPULATE};
55-
56-
#[cfg(any(target_os = "linux", target_os = "openbsd", target_os = "netbsd"))]
57-
pub use libc::MAP_STACK;
58-
59127
#[cfg(target_os = "freebsd")]
60128
pub use libc::{MADV_AUTOSYNC, MADV_CORE, MADV_NOCORE, MADV_NOSYNC, MADV_PROTECT};
61129

‎crates/host_env/src/posix.rs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -948,6 +948,20 @@ pub fn kill(pid: i32, sig: i32) -> std::io::Result<()> {
948948
}
949949
}
950950

951+
/// `killpg(2)`: signal every process in a process group.
952+
///
953+
/// Not `kill(-pgid, sig)`: that spelling reads a negative pid, and `kill`
954+
/// takes its argument from Python where a caller can already pass one, so the
955+
/// two are not interchangeable at this layer.
956+
pub fn killpg(pgid: i32, sig: i32) -> std::io::Result<()> {
957+
let ret = unsafe { libc::killpg(pgid, sig) };
958+
if ret == -1 {
959+
Err(std::io::Error::last_os_error())
960+
} else {
961+
Ok(())
962+
}
963+
}
964+
951965
#[cfg(not(any(target_os = "wasi", target_os = "android")))]
952966
pub fn setuid(uid: u32) -> std::io::Result<()> {
953967
nix::unistd::setuid(uid_from_raw(uid)).map_err(std::io::Error::from)

‎crates/host_env/src/select.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@ use core::mem::MaybeUninit;
22
use std::io;
33

44
#[cfg(unix)]
5-
pub use libc::{EINTR, FD_SETSIZE, POLLERR, POLLHUP, POLLIN, POLLNVAL, POLLOUT, POLLPRI};
5+
pub use libc::{
6+
EINTR, FD_SETSIZE, PIPE_BUF, POLLERR, POLLHUP, POLLIN, POLLNVAL, POLLOUT, POLLPRI, POLLRDBAND,
7+
POLLRDNORM, POLLWRBAND, POLLWRNORM,
8+
};
69

710
#[cfg(any(target_os = "linux", target_os = "android", target_os = "redox"))]
811
pub use libc::{

0 commit comments

Comments
 (0)