Skip to content

Commit fdb08aa

Browse files
youknowoneyouknowone
committed
Replace shallow Unicode wrappers with re-exports
Address review feedback on the extraction. Functions that only forwarded to the shared crate become `use ... as` re-exports rather than hand-written wrappers: - literal: is_printable is dropped; escape.rs calls rustpython_unicode::classify::is_repr_printable directly. - unicode::identifier::is_continue re-exports is_xid_continue. - unicode::data::lookup_character re-exports unicode_names2::character. - Drop the unused repr(u8) on DecompositionType. The SRE character-class and case predicates move back into sre_engine::string (their pre-extraction home) instead of living in a unicode::regex module that only sre_engine and the vm _sre binding used; is_uni_digit/is_uni_alnum still delegate to rustpython_unicode::classify. engine.rs and _sre consume them from sre_engine::string again. The unicode crate's tests move from a single root tests module in lib.rs into per-module test submodules (case, classify, data, identifier, normalize). Assisted-by: Claude
1 parent fe74871 commit fdb08aa

11 files changed

Lines changed: 185 additions & 257 deletions

File tree

‎crates/literal/src/char.rs‎

Lines changed: 0 additions & 12 deletions
This file was deleted.

‎crates/literal/src/escape.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -204,7 +204,7 @@ impl UnicodeEscape<'_> {
204204
'\\' | '\t' | '\r' | '\n' => 2,
205205
ch if ch < ' ' || ch as u32 == 0x7f => 4, // \xHH
206206
ch if ch.is_ascii() => 1,
207-
ch if crate::char::is_printable(ch) => {
207+
ch if rustpython_unicode::classify::is_repr_printable(ch) => {
208208
// max = std::cmp::max(ch, max);
209209
ch.len_utf8()
210210
}
@@ -238,7 +238,7 @@ impl UnicodeEscape<'_> {
238238
ch if ch.is_ascii() => {
239239
write!(formatter, "\\x{:02x}", ch as u8)
240240
}
241-
ch if crate::char::is_printable(ch) => formatter.write_char(ch),
241+
ch if rustpython_unicode::classify::is_repr_printable(ch) => formatter.write_char(ch),
242242
'\0'..='\u{ff}' => {
243243
write!(formatter, "\\x{:02x}", ch as u32)
244244
}

‎crates/literal/src/lib.rs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
extern crate alloc;
44

5-
pub mod char;
65
pub mod complex;
76
pub mod escape;
87
pub mod float;

‎crates/sre_engine/src/string.rs‎

Lines changed: 88 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
use rustpython_unicode as unicode;
21
use rustpython_wtf8::Wtf8;
32

43
#[derive(Debug, Clone, Copy)]
@@ -333,70 +332,142 @@ const fn utf8_is_cont_byte(byte: u8) -> bool {
333332
/// Mask of the value bits of a continuation byte.
334333
const CONT_MASK: u8 = 0b0011_1111;
335334

335+
// Character-class and case predicates for the SRE engine.
336+
//
337+
// Every predicate takes a raw `u32` code point (SRE decodes strings into `u32`s,
338+
// including lone surrogates) and returns whether it belongs to the class.
339+
// ASCII-mode predicates only ever consider byte values; Unicode-mode predicates
340+
// consult the shared property tables in `rustpython_unicode::classify`.
341+
342+
const UNDERSCORE: u32 = '_' as u32;
343+
344+
const fn is_py_ascii_whitespace(b: u8) -> bool {
345+
matches!(b, b'\t' | b'\n' | b'\x0C' | b'\r' | b' ' | b'\x0B')
346+
}
347+
336348
#[inline]
337349
pub(crate) fn is_word(ch: u32) -> bool {
338-
unicode::regex::is_word(ch)
350+
ch == UNDERSCORE || u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric())
339351
}
352+
340353
#[inline]
341354
pub(crate) fn is_space(ch: u32) -> bool {
342-
unicode::regex::is_space(ch)
355+
u8::try_from(ch).is_ok_and(is_py_ascii_whitespace)
343356
}
357+
344358
#[inline]
345359
pub(crate) fn is_digit(ch: u32) -> bool {
346-
unicode::regex::is_digit(ch)
360+
u8::try_from(ch).is_ok_and(|x| x.is_ascii_digit())
347361
}
362+
348363
#[inline]
349364
pub(crate) fn is_loc_alnum(ch: u32) -> bool {
350-
unicode::regex::is_loc_alnum(ch)
365+
// FIXME: Ignore the locales
366+
u8::try_from(ch).is_ok_and(|x| x.is_ascii_alphanumeric())
351367
}
368+
352369
#[inline]
353370
pub(crate) fn is_loc_word(ch: u32) -> bool {
354-
ch == '_' as u32 || is_loc_alnum(ch)
371+
ch == UNDERSCORE || is_loc_alnum(ch)
355372
}
373+
356374
#[inline]
357375
pub(crate) const fn is_linebreak(ch: u32) -> bool {
358-
unicode::regex::is_linebreak(ch)
376+
ch == '\n' as u32
359377
}
378+
360379
#[inline]
361380
#[must_use]
362381
pub fn lower_ascii(ch: u32) -> u32 {
363-
unicode::regex::lower_ascii(ch)
382+
u8::try_from(ch).map_or(ch, |x| x.to_ascii_lowercase() as u32)
364383
}
384+
365385
#[inline]
366386
pub(crate) fn lower_locate(ch: u32) -> u32 {
367-
unicode::regex::lower_locate(ch)
387+
// FIXME: Ignore the locales
388+
lower_ascii(ch)
368389
}
390+
369391
#[inline]
370392
pub(crate) fn upper_locate(ch: u32) -> u32 {
371-
unicode::regex::upper_locate(ch)
393+
// FIXME: Ignore the locales
394+
u8::try_from(ch).map_or(ch, |x| x.to_ascii_uppercase() as u32)
372395
}
396+
373397
#[inline]
374398
pub(crate) fn is_uni_digit(ch: u32) -> bool {
375-
unicode::regex::is_uni_digit(ch)
399+
// SRE_UNI_IS_DIGIT matches Unicode decimal digits (Py_UNICODE_ISDECIMAL),
400+
// not just ASCII 0-9.
401+
char::try_from(ch).is_ok_and(rustpython_unicode::classify::is_decimal)
376402
}
403+
377404
#[inline]
378405
pub(crate) fn is_uni_space(ch: u32) -> bool {
379-
unicode::regex::is_uni_space(ch)
406+
// TODO: check with cpython
407+
is_space(ch)
408+
|| matches!(
409+
ch,
410+
0x0009
411+
| 0x000A
412+
| 0x000B
413+
| 0x000C
414+
| 0x000D
415+
| 0x001C
416+
| 0x001D
417+
| 0x001E
418+
| 0x001F
419+
| 0x0020
420+
| 0x0085
421+
| 0x00A0
422+
| 0x1680
423+
| 0x2000
424+
| 0x2001
425+
| 0x2002
426+
| 0x2003
427+
| 0x2004
428+
| 0x2005
429+
| 0x2006
430+
| 0x2007
431+
| 0x2008
432+
| 0x2009
433+
| 0x200A
434+
| 0x2028
435+
| 0x2029
436+
| 0x202F
437+
| 0x205F
438+
| 0x3000
439+
)
380440
}
441+
381442
#[inline]
382443
pub(crate) const fn is_uni_linebreak(ch: u32) -> bool {
383-
unicode::regex::is_uni_linebreak(ch)
444+
matches!(
445+
ch,
446+
0x000A | 0x000B | 0x000C | 0x000D | 0x001C | 0x001D | 0x001E | 0x0085 | 0x2028 | 0x2029
447+
)
384448
}
449+
385450
#[inline]
386451
pub(crate) fn is_uni_alnum(ch: u32) -> bool {
387-
unicode::regex::is_uni_alnum(ch)
452+
// TODO: check with cpython
453+
char::try_from(ch).is_ok_and(rustpython_unicode::classify::is_alnum)
388454
}
455+
389456
#[inline]
390457
pub(crate) fn is_uni_word(ch: u32) -> bool {
391-
ch == '_' as u32 || is_uni_alnum(ch)
458+
ch == UNDERSCORE || is_uni_alnum(ch)
392459
}
460+
393461
#[inline]
394462
#[must_use]
395463
pub fn lower_unicode(ch: u32) -> u32 {
396-
unicode::regex::lower_unicode(ch)
464+
// TODO: check with cpython
465+
char::try_from(ch).map_or(ch, |x| x.to_lowercase().next().unwrap() as u32)
397466
}
467+
398468
#[inline]
399469
#[must_use]
400470
pub fn upper_unicode(ch: u32) -> u32 {
401-
unicode::regex::upper_unicode(ch)
471+
// TODO: check with cpython
472+
char::try_from(ch).map_or(ch, |x| x.to_uppercase().next().unwrap() as u32)
402473
}

‎crates/unicode/src/case.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,3 +55,15 @@ impl core::fmt::Write for FmtWriter<'_> {
5555
Ok(())
5656
}
5757
}
58+
59+
#[cfg(test)]
60+
mod tests {
61+
use super::casefold_str;
62+
63+
#[test]
64+
fn casefold_full_mappings() {
65+
// ß case-folds to "ss"
66+
assert_eq!(casefold_str("ß"), "ss");
67+
assert_eq!(casefold_str("Σ"), "σ");
68+
}
69+
}

‎crates/unicode/src/classify.rs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,3 +91,26 @@ pub fn is_repr_printable(c: char) -> bool {
9191
| GeneralCategory::Unassigned
9292
)
9393
}
94+
95+
#[cfg(test)]
96+
mod tests {
97+
use super::{is_decimal, is_digit, is_numeric};
98+
99+
#[test]
100+
fn numeric_type_chain_holds() {
101+
// isdecimal ⊂ isdigit ⊂ isnumeric
102+
for c in ('\0'..='\u{2FFFF}').filter_map(|c| char::from_u32(c as u32)) {
103+
if is_decimal(c) {
104+
assert!(is_digit(c), "{c:?} decimal but not digit");
105+
}
106+
if is_digit(c) {
107+
assert!(is_numeric(c), "{c:?} digit but not numeric");
108+
}
109+
}
110+
assert!(is_decimal('5'));
111+
assert!(!is_decimal('²'));
112+
assert!(is_digit('²'));
113+
assert!(!is_digit('⅓'));
114+
assert!(is_numeric('⅓'));
115+
}
116+
}

‎crates/unicode/src/data.rs‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ include!(concat!(
2828
));
2929

3030
#[derive(Clone, Copy)]
31-
#[repr(u8)]
3231
enum DecompositionType {
3332
#[allow(unused)]
3433
Canonical,
@@ -126,10 +125,7 @@ pub fn unicode_version() -> String {
126125
}
127126

128127
/// Look up a character by its Unicode name (`unicodedata.lookup`).
129-
#[must_use]
130-
pub fn lookup_character(name: &str) -> Option<char> {
131-
unicode_names2::character(name)
132-
}
128+
pub use unicode_names2::character as lookup_character;
133129

134130
/// The Unicode name of `ch` (`unicodedata.name`), if any.
135131
#[must_use]
@@ -346,3 +342,33 @@ impl Ucd {
346342
}
347343
}
348344
}
345+
346+
#[cfg(test)]
347+
mod tests {
348+
use rustpython_wtf8::CodePoint;
349+
350+
use super::{Ucd, character_name, lookup_character};
351+
352+
fn cp(ch: char) -> CodePoint {
353+
CodePoint::from(ch)
354+
}
355+
356+
#[test]
357+
fn data_queries_match_unicodedata_behavior() {
358+
let ucd = Ucd::new(true);
359+
assert_eq!(ucd.category(cp('A')), "Lu");
360+
assert_eq!(ucd.category(CodePoint::from_u32(0xD800).unwrap()), "Cs");
361+
assert_eq!(lookup_character("SNOWMAN"), Some('☃'));
362+
assert_eq!(character_name('☃').as_deref(), Some("SNOWMAN"));
363+
assert_eq!(ucd.decimal(cp('५')), Some(5));
364+
assert_eq!(ucd.digit(cp('²')), Some(2));
365+
let third = ucd.numeric(cp('⅓')).unwrap();
366+
assert!((third - 1.0 / 3.0).abs() < 1e-6, "got {third}");
367+
}
368+
369+
#[test]
370+
fn ucd_3_2_0_view_differs_from_modern() {
371+
let legacy = Ucd::new(false);
372+
assert_eq!(legacy.unidata_version(), "3.2.0");
373+
}
374+
}

‎crates/unicode/src/identifier.rs‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,17 @@ pub fn is_start(c: char) -> bool {
2121
}
2222

2323
/// Whether `c` may continue a Python identifier: `XID_Continue`.
24-
#[must_use]
25-
pub fn is_continue(c: char) -> bool {
26-
is_xid_continue(c)
24+
pub use is_xid_continue as is_continue;
25+
26+
#[cfg(test)]
27+
mod tests {
28+
use super::{is_continue, is_start};
29+
30+
#[test]
31+
fn identifier_predicates() {
32+
assert!(is_start('_'));
33+
assert!(is_start('가'));
34+
assert!(!is_start('1'));
35+
assert!(is_continue('1'));
36+
}
2737
}

0 commit comments

Comments
 (0)