Skip to content

Commit ee4e720

Browse files
Add more unicode functions to c-api (#8044)
* Add more unicode functions to c-api * Update crates/capi/src/unicodeobject.rs
1 parent a575121 commit ee4e720

1 file changed

Lines changed: 145 additions & 5 deletions

File tree

‎crates/capi/src/unicodeobject.rs‎

Lines changed: 145 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
use crate::PyObject;
21
use crate::object::define_py_check;
3-
use crate::pystate::with_vm;
2+
use crate::{PyObject, pystate::with_vm};
43
use core::ffi::{CStr, c_char, c_int};
54
use core::ptr::NonNull;
65
use core::slice;
@@ -92,6 +91,103 @@ pub unsafe extern "C" fn PyUnicode_AsEncodedString(
9291
})
9392
}
9493

94+
#[unsafe(no_mangle)]
95+
pub unsafe extern "C" fn PyUnicode_AsUTF8String(unicode: *mut PyObject) -> *mut PyObject {
96+
with_vm(|vm| {
97+
let unicode = unsafe { &*unicode }
98+
.try_downcast_ref::<PyStr>(vm)?
99+
.to_owned();
100+
vm.state
101+
.codec_registry
102+
.encode_text(unicode, "utf-8", None, vm)
103+
})
104+
}
105+
106+
#[unsafe(no_mangle)]
107+
pub unsafe extern "C" fn PyUnicode_DecodeFSDefaultAndSize(
108+
s: *const c_char,
109+
size: isize,
110+
) -> *mut PyObject {
111+
with_vm(|vm| {
112+
let size: usize = size
113+
.try_into()
114+
.map_err(|_| vm.new_system_error("size must be non-negative"))?;
115+
116+
let bytes = if s.is_null() {
117+
if size != 0 {
118+
return Err(vm.new_system_error(
119+
"PyUnicode_DecodeFSDefaultAndSize called with null data and non-zero size",
120+
));
121+
}
122+
&[][..]
123+
} else {
124+
unsafe { slice::from_raw_parts(s.cast::<u8>(), size) }
125+
};
126+
127+
vm.state.codec_registry.decode_text(
128+
vm.ctx.new_bytes(bytes.to_vec()).into(),
129+
vm.fs_encoding().as_str(),
130+
Some(vm.fs_encode_errors().to_owned()),
131+
vm,
132+
)
133+
})
134+
}
135+
136+
#[unsafe(no_mangle)]
137+
pub unsafe extern "C" fn PyUnicode_EncodeFSDefault(unicode: *mut PyObject) -> *mut PyObject {
138+
with_vm(|vm| {
139+
let unicode = unsafe { &*unicode }
140+
.try_downcast_ref::<PyStr>(vm)?
141+
.to_owned();
142+
vm.state.codec_registry.encode_text(
143+
unicode,
144+
vm.fs_encoding().as_str(),
145+
Some(vm.fs_encode_errors().to_owned()),
146+
vm,
147+
)
148+
})
149+
}
150+
151+
#[unsafe(no_mangle)]
152+
pub unsafe extern "C" fn PyUnicode_FromEncodedObject(
153+
obj: *mut PyObject,
154+
encoding: *const c_char,
155+
errors: *const c_char,
156+
) -> *mut PyObject {
157+
with_vm(|vm| {
158+
let obj = unsafe { &*obj };
159+
160+
if obj.downcast_ref::<PyStr>().is_some() {
161+
return Err(vm.new_type_error("decoding str is not supported".to_owned()));
162+
}
163+
164+
let encoding = if encoding.is_null() {
165+
"utf-8"
166+
} else {
167+
unsafe { CStr::from_ptr(encoding) }
168+
.to_str()
169+
.map_err(|_| vm.new_system_error("encoding must be valid UTF-8"))?
170+
};
171+
let errors = if errors.is_null() {
172+
None
173+
} else {
174+
let errors = unsafe { CStr::from_ptr(errors) }
175+
.to_str()
176+
.map_err(|_| vm.new_system_error("errors must be valid UTF-8"))?;
177+
Some(vm.ctx.new_utf8_str(errors))
178+
};
179+
180+
obj.try_bytes_like(vm, |b| {
181+
vm.state.codec_registry.decode_text(
182+
vm.ctx.new_bytes(b.to_vec()).into(),
183+
encoding,
184+
errors,
185+
vm,
186+
)
187+
})?
188+
})
189+
}
190+
95191
#[unsafe(no_mangle)]
96192
pub unsafe extern "C" fn PyUnicode_InternInPlace(string: *mut *mut PyObject) {
97193
with_vm(|vm| {
@@ -131,12 +227,17 @@ pub unsafe extern "C" fn PyUnicode_EqualToUTF8AndSize(
131227

132228
#[cfg(false)]
133229
mod tests {
230+
use std::ffi::{OsStr, OsString};
231+
134232
use pyo3::intern;
135233
use pyo3::prelude::*;
136-
use pyo3::types::PyString;
234+
use pyo3::types::{PyBytes, PyString, PyStringMethods};
235+
236+
#[cfg(unix)]
237+
use std::os::unix::ffi::OsStrExt;
137238

138239
#[test]
139-
fn test_unicode() {
240+
fn unicode() {
140241
Python::attach(|py| {
141242
let string = PyString::new(py, "Hello, World!");
142243
assert!(string.is_instance_of::<PyString>());
@@ -146,9 +247,48 @@ mod tests {
146247
}
147248

148249
#[test]
149-
fn test_intern_str() {
250+
fn intern_str() {
150251
Python::attach(|py| {
151252
let _string = intern!(py, "Hello, World!");
152253
})
153254
}
255+
256+
#[test]
257+
fn encode_utf8_via_wrapper() {
258+
Python::attach(|py| {
259+
let s = PyString::new(py, "h\u{00E9}llo");
260+
let encoded = s.encode_utf8().unwrap();
261+
assert_eq!(encoded.as_bytes(), "h\u{00E9}llo".as_bytes());
262+
})
263+
}
264+
265+
#[test]
266+
fn from_encoded_object_bytes() {
267+
Python::attach(|py| {
268+
let src = PyBytes::new(py, b"h\xC3\xA9llo");
269+
let s = PyString::from_encoded_object(src.as_any(), None, None).unwrap();
270+
assert_eq!(s.to_str().unwrap(), "h\u{00E9}llo");
271+
})
272+
}
273+
274+
#[cfg(unix)]
275+
#[test]
276+
fn fs_default_roundtrip_non_utf8_unix() {
277+
Python::attach(|py| {
278+
let original = OsStr::from_bytes(&[b'f', b'o', 0x80]);
279+
let py_str = original.into_pyobject(py).unwrap();
280+
let roundtrip: OsString = py_str.extract().unwrap();
281+
assert_eq!(roundtrip.as_os_str().as_bytes(), original.as_bytes());
282+
})
283+
}
284+
285+
#[test]
286+
fn fs_default_roundtrip_utf8() {
287+
Python::attach(|py| {
288+
let original = OsStr::new("hello.txt");
289+
let py_str = original.into_pyobject(py).unwrap();
290+
let roundtrip: OsString = py_str.extract().unwrap();
291+
assert_eq!(roundtrip, original);
292+
})
293+
}
154294
}

0 commit comments

Comments
 (0)