Skip to content

Commit 41d2d58

Browse files
joshuamegnauth54joshuamegnauth54
committed
host_env: os.replace for Windows
`rename` and `replace` are the same for Unixes but different for Windows in Python. POSIX's `rename` atomically replaces its target; there is no `replace` in POSIX. `renameat2` with `RENAME_NOREPLACE` atomically checks if a file exists and renames if it doesn't, but Python's `rename` and `replace` predate `renameat2`. For Windows, `rename` acts like the `RENAME_NOREPLACE` flag while `replace` functions like `rename`. I implemented both using the Windows API. Both implementations follow CPython's code by using `MoveFileExW`. There are modern Windows APIs that may be worth using in the future. Rust's standard library prefers the modern APIs but falls back to `MoveFileExW` for deprecated Windows versions.
1 parent 045a6d5 commit 41d2d58

7 files changed

Lines changed: 182 additions & 74 deletions

File tree

‎crates/host_env/src/lib.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,8 @@ pub mod posix;
5555
#[cfg(windows)]
5656
#[path = "posix_windows.rs"]
5757
pub mod posix;
58+
#[cfg(any(unix, target_os = "wasi"))]
59+
pub mod posix_unix_wasi;
5860
#[cfg(unix)]
5961
pub mod pwd;
6062
#[cfg(unix)]

‎crates/host_env/src/os.rs‎

Lines changed: 0 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@ use core::ffi::CStr;
99
use core::str::Utf8Error;
1010
#[cfg(windows)]
1111
use core::time::Duration;
12-
#[cfg(unix)]
13-
use rustix::fd::AsFd;
1412
use std::{
1513
env,
1614
ffi::{OsStr, OsString},
@@ -267,41 +265,6 @@ pub fn copy_file_range(
267265
rustix::fs::copy_file_range(src, offset_src, dst, offset_dst, count)
268266
}
269267

270-
#[cfg(not(unix))]
271-
pub fn rename(
272-
from: impl AsRef<std::path::Path>,
273-
from_fd: Option<crt_fd::Borrowed<'_>>,
274-
to: impl AsRef<std::path::Path>,
275-
to_fd: Option<crt_fd::Borrowed<'_>>,
276-
) -> io::Result<()> {
277-
if from_fd.is_none() && to_fd.is_none() {
278-
// TODO: Rust's implementation always overwrites the file so ensure consistency between
279-
// operating systems. We need to use windows-sys directly to distinguish between
280-
// os.rename and os.replace.
281-
std::fs::rename(from, to)
282-
} else {
283-
core::hint::cold_path();
284-
Err(io::Error::new(
285-
io::ErrorKind::Unsupported,
286-
"renameat is not available on this platform",
287-
))
288-
}
289-
}
290-
291-
#[cfg(unix)]
292-
pub fn rename(
293-
from: impl AsRef<std::path::Path>,
294-
from_fd: Option<crt_fd::Borrowed<'_>>,
295-
to: impl AsRef<std::path::Path>,
296-
to_fd: Option<crt_fd::Borrowed<'_>>,
297-
) -> io::Result<()> {
298-
let from = from.as_ref();
299-
let from_fd = from_fd.as_ref().map_or(rustix::fs::CWD, AsFd::as_fd);
300-
let to = to.as_ref();
301-
let to_fd = to_fd.as_ref().map_or(rustix::fs::CWD, AsFd::as_fd);
302-
rustix::fs::renameat(from_fd, from, to_fd, to).map_err(Into::into)
303-
}
304-
305268
#[cfg(windows)]
306269
pub fn seek_fd(
307270
fd: crt_fd::Borrowed<'_>,

‎crates/host_env/src/posix.rs‎

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ use std::os::fd::FromRawFd;
1010
use std::os::fd::{AsFd, AsRawFd, BorrowedFd, IntoRawFd, OwnedFd};
1111
use std::path::Path;
1212

13-
use crate::crt_fd;
13+
pub use super::posix_unix_wasi::*;
1414

1515
pub struct UnameInfo {
1616
pub sysname: String,
@@ -176,16 +176,6 @@ pub fn fcopyfile(in_fd: i32, out_fd: i32, flags: u32) -> std::io::Result<()> {
176176
}
177177
}
178178

179-
#[cfg(any(unix, target_os = "wasi"))]
180-
pub fn make_dir(
181-
dir_fd: Option<crt_fd::Borrowed<'_>>,
182-
path: &impl AsRef<Path>,
183-
mode: libc::mode_t,
184-
) -> std::io::Result<()> {
185-
let dir_fd = dir_fd.as_ref().map_or(rustix::fs::CWD, AsFd::as_fd);
186-
rustix::fs::mkdirat(dir_fd, path.as_ref(), mode.into()).map_err(Into::into)
187-
}
188-
189179
#[cfg(unix)]
190180
pub fn link_paths(src: &CStr, dst: &CStr, follow_symlinks: bool) -> std::io::Result<()> {
191181
let flags = if follow_symlinks {

‎crates/host_env/src/posix_unix_wasi.rs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
//! Common POSIX implementations across Unix and WASI.
2+
3+
use std::{io, path::Path};
4+
5+
use rustix::{fd::AsFd, fs};
6+
7+
use crate::crt_fd;
8+
9+
/// https://pubs.opengroup.org/onlinepubs/9799919799/functions/mkdir.html
10+
pub fn make_dir(
11+
dir_fd: Option<crt_fd::Borrowed<'_>>,
12+
path: &impl AsRef<Path>,
13+
mode: fs::RawMode,
14+
) -> io::Result<()> {
15+
let dir_fd = dir_fd.as_ref().map_or(fs::CWD, AsFd::as_fd);
16+
fs::mkdirat(dir_fd, path.as_ref(), mode.into()).map_err(Into::into)
17+
}
18+
19+
/// https://pubs.opengroup.org/onlinepubs/9799919799/functions/rename.html
20+
pub fn rename(
21+
from: impl AsRef<Path>,
22+
from_fd: Option<crt_fd::Borrowed<'_>>,
23+
to: impl AsRef<Path>,
24+
to_fd: Option<crt_fd::Borrowed<'_>>,
25+
) -> io::Result<()> {
26+
let from = from.as_ref();
27+
let from_fd = from_fd.as_ref().map_or(fs::CWD, AsFd::as_fd);
28+
let to = to.as_ref();
29+
let to_fd = to_fd.as_ref().map_or(fs::CWD, AsFd::as_fd);
30+
fs::renameat(from_fd, from, to_fd, to).map_err(Into::into)
31+
}
32+
33+
/// https://docs.python.org/3/library/os.html#os.replace
34+
///
35+
/// Atomically replace `to` with `from`.
36+
/// POSIX's rename already atomically replaces targets, so this function just forwards to [`rename`].
37+
#[inline]
38+
pub fn replace(
39+
from: impl AsRef<Path>,
40+
from_fd: Option<crt_fd::Borrowed<'_>>,
41+
to: impl AsRef<Path>,
42+
to_fd: Option<crt_fd::Borrowed<'_>>,
43+
) -> io::Result<()> {
44+
rename(from, from_fd, to, to_fd)
45+
}

‎crates/host_env/src/posix_wasi.rs‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,9 @@ use core::{ffi::CStr, time::Duration};
33
use rustix::fd::AsFd;
44
use std::{ffi::OsStr, io, path::Path};
55

6-
use crate::{crt_fd, os::CheckLibcResult};
6+
pub use super::posix_unix_wasi::*;
77

8-
pub fn make_dir(
9-
dir_fd: Option<crt_fd::Borrowed<'_>>,
10-
path: &impl AsRef<Path>,
11-
mode: libc::mode_t,
12-
) -> std::io::Result<()> {
13-
let dir_fd = dir_fd.as_ref().map_or(rustix::fs::CWD, AsFd::as_fd);
14-
rustix::fs::mkdirat(dir_fd, path.as_ref(), mode.into()).map_err(Into::into)
15-
}
8+
use crate::{crt_fd, os::CheckLibcResult};
169

1710
pub fn remove_dir_at(dir_fd: i32, path: &CStr) -> io::Result<()> {
1811
unsafe { libc::unlinkat(dir_fd, path.as_ptr(), libc::AT_REMOVEDIR) }.check_libc_neg()?;

‎crates/host_env/src/posix_windows.rs‎

Lines changed: 80 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,96 @@
44
//! these syscalls, but they can be emulated with a mix of the Windows API and the Rust standard
55
//! library, the latter of which calls the former.
66
7+
use core::hint::cold_path;
78
use std::{fs, io, path::Path};
89

10+
use widestring::WideCString;
11+
use windows_sys::Win32::{
12+
Foundation::FALSE,
13+
Storage::FileSystem::{MOVE_FILE_FLAGS, MOVEFILE_REPLACE_EXISTING, MoveFileExW},
14+
};
15+
916
use crate::crt_fd;
1017

11-
#[expect(non_camel_case_types)]
12-
pub type mode_t = u32;
18+
pub type RawMode = u32;
1319

1420
pub fn make_dir(
1521
dir_fd: Option<crt_fd::Borrowed<'_>>,
1622
path: &impl AsRef<Path>,
17-
_mode: mode_t,
23+
_mode: RawMode,
1824
) -> io::Result<()> {
1925
debug_assert!(dir_fd.is_none());
2026
// TODO: On Windows, Python has an override if the mode is 0o700
2127
fs::create_dir(path)
2228
}
29+
30+
/// https://pubs.opengroup.org/onlinepubs/9799919799/functions/rename.html
31+
#[inline]
32+
pub fn rename(
33+
from: impl AsRef<Path>,
34+
#[cfg_attr(not(debug_assertions), expect(unused_variables))] from_fd: Option<
35+
crt_fd::Borrowed<'_>,
36+
>,
37+
to: impl AsRef<Path>,
38+
#[cfg_attr(not(debug_assertions), expect(unused_variables))] to_fd: Option<
39+
crt_fd::Borrowed<'_>,
40+
>,
41+
) -> io::Result<()> {
42+
debug_assert!(from_fd.is_none());
43+
debug_assert!(to_fd.is_none());
44+
45+
rename_impl(from, to, 0)
46+
}
47+
48+
/// https://docs.python.org/3/library/os.html#os.replace
49+
///
50+
/// Atomically replace `to` with `from`.
51+
#[inline]
52+
pub fn replace(
53+
from: impl AsRef<Path>,
54+
#[cfg_attr(not(debug_assertions), expect(unused_variables))] from_fd: Option<
55+
crt_fd::Borrowed<'_>,
56+
>,
57+
to: impl AsRef<Path>,
58+
#[cfg_attr(not(debug_assertions), expect(unused_variables))] to_fd: Option<
59+
crt_fd::Borrowed<'_>,
60+
>,
61+
) -> io::Result<()> {
62+
debug_assert!(from_fd.is_none());
63+
debug_assert!(to_fd.is_none());
64+
65+
rename_impl(from, to, MOVEFILE_REPLACE_EXISTING)
66+
}
67+
68+
fn rename_impl(
69+
from: impl AsRef<Path>,
70+
to: impl AsRef<Path>,
71+
flags: MOVE_FILE_FLAGS,
72+
) -> io::Result<()> {
73+
let from = WideCString::from_os_str(from)
74+
.map_err(io::Error::other)?
75+
.into_vec_with_nul();
76+
let to = WideCString::from_os_str(to)
77+
.map_err(io::Error::other)?
78+
.into_vec_with_nul();
79+
80+
// SAFETY:
81+
// * from and to are NUL terminated wide strings
82+
let success = unsafe {
83+
// Rust's [`std::fs::rename`] is more complicated than CPython's. Rust attempts to use modern APIs
84+
// where available, such as `FileRenameInfoEx`, which better map to POSIX. CPython simply
85+
// calls MoveFileExW so we'll do that for parity. However, it may be better to use the new
86+
// APIs and fall back if possible, especially if they're faster.
87+
//
88+
// Unlike POSIX's rename, MoveFileExW does not automatically move between volumes.
89+
// This is expected behavior in CPython.
90+
MoveFileExW(from.as_ptr(), to.as_ptr(), flags)
91+
};
92+
93+
if success != FALSE {
94+
Ok(())
95+
} else {
96+
cold_path();
97+
Err(io::Error::last_os_error())
98+
}
99+
}

‎crates/vm/src/stdlib/os.rs‎

Lines changed: 52 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,11 @@ impl<'fd> DirFd<'fd, 1> {
7878

7979
impl<const AVAILABLE: usize> FromArgs for DirFd<'_, AVAILABLE> {
8080
fn from_args(vm: &VirtualMachine, args: &mut FuncArgs) -> Result<Self, ArgumentError> {
81-
let fd = match args.take_keyword("dir_fd") {
81+
let fd = match args
82+
.take_keyword("dir_fd")
83+
.or_else(|| args.take_keyword("src_dir_fd"))
84+
.or_else(|| args.take_keyword("dst_dir_fd"))
85+
{
8286
Some(o) if vm.is_none(&o) => Ok(DEFAULT_DIR_FD),
8387
None => Ok(DEFAULT_DIR_FD),
8488
Some(o) => {
@@ -195,7 +199,7 @@ pub(super) mod _os {
195199
const UTIME_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox")));
196200
pub(crate) const SYMLINK_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox")));
197201
pub(crate) const UNLINK_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox")));
198-
const RENAME_DIR_FD: bool = cfg!(unix);
202+
const RENAME_DIR_FD: bool = cfg!(any(unix, target_os = "wasi"));
199203
const RMDIR_DIR_FD: bool = cfg!(not(any(windows, target_os = "redox")));
200204
const SCANDIR_FD: bool = cfg!(all(unix, not(target_os = "redox")));
201205

@@ -1382,14 +1386,15 @@ pub(super) mod _os {
13821386
src: PyObjectRef,
13831387
#[pyarg(positional)]
13841388
dst: PyObjectRef,
1385-
#[pyarg(any, default)]
1386-
src_dir_fd: OptionalArg<crt_fd::Borrowed<'fd>>,
1387-
#[pyarg(any, default)]
1388-
dst_dir_fd: OptionalArg<crt_fd::Borrowed<'fd>>,
1389+
#[pyarg(flatten)]
1390+
#[cfg_attr(not(any(unix, target_os = "wasi")), expect(unused_variables))]
1391+
src_dir_fd: DirFd<'fd, { RENAME_DIR_FD as usize }>,
1392+
#[pyarg(flatten)]
1393+
#[cfg_attr(not(any(unix, target_os = "wasi")), expect(unused_variables))]
1394+
dst_dir_fd: DirFd<'fd, { RENAME_DIR_FD as usize }>,
13891395
}
13901396

13911397
#[pyfunction]
1392-
#[pyfunction(name = "replace")]
13931398
fn rename(args: RenameArgs<'_>, vm: &VirtualMachine) -> PyResult<()> {
13941399
let src = PathConverter::new()
13951400
.function("rename")
@@ -1400,13 +1405,46 @@ pub(super) mod _os {
14001405
.argument("dst")
14011406
.try_path(args.dst, vm)?;
14021407

1403-
crate::host_env::os::rename(
1404-
&src,
1405-
args.src_dir_fd.into_option(),
1406-
&dst,
1407-
args.dst_dir_fd.into_option(),
1408-
)
1409-
.map_err(|err| {
1408+
#[cfg(any(unix, target_os = "wasi"))]
1409+
let src_dir_fd = args.src_dir_fd.get_opt();
1410+
#[cfg(not(any(unix, target_os = "wasi")))]
1411+
let src_dir_fd = None;
1412+
1413+
#[cfg(any(unix, target_os = "wasi"))]
1414+
let dst_dir_fd = args.dst_dir_fd.get_opt();
1415+
#[cfg(not(any(unix, target_os = "wasi")))]
1416+
let dst_dir_fd = None;
1417+
1418+
crate::host_env::posix::rename(&src, src_dir_fd, &dst, dst_dir_fd).map_err(|err| {
1419+
let builder = err.to_os_error_builder(vm);
1420+
let builder = builder.filename(src.filename(vm));
1421+
let builder = builder.filename2(dst.filename(vm));
1422+
builder.build(vm).upcast()
1423+
})
1424+
}
1425+
1426+
#[pyfunction]
1427+
fn replace(args: RenameArgs<'_>, vm: &VirtualMachine) -> PyResult<()> {
1428+
let src = PathConverter::new()
1429+
.function("replace")
1430+
.argument("src")
1431+
.try_path(args.src, vm)?;
1432+
let dst = PathConverter::new()
1433+
.function("replace")
1434+
.argument("dst")
1435+
.try_path(args.dst, vm)?;
1436+
1437+
#[cfg(any(unix, target_os = "wasi"))]
1438+
let src_dir_fd = args.src_dir_fd.get_opt();
1439+
#[cfg(not(any(unix, target_os = "wasi")))]
1440+
let src_dir_fd = None;
1441+
1442+
#[cfg(any(unix, target_os = "wasi"))]
1443+
let dst_dir_fd = args.dst_dir_fd.get_opt();
1444+
#[cfg(not(any(unix, target_os = "wasi")))]
1445+
let dst_dir_fd = None;
1446+
1447+
crate::host_env::posix::replace(&src, src_dir_fd, &dst, dst_dir_fd).map_err(|err| {
14101448
let builder = err.to_os_error_builder(vm);
14111449
let builder = builder.filename(src.filename(vm));
14121450
let builder = builder.filename2(dst.filename(vm));

0 commit comments

Comments
 (0)