Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 8 additions & 24 deletions crates/vm/src/builtins/float.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use crate::{
types::{AsNumber, Callable, Comparable, Constructor, Hashable, PyComparisonOp, Representable},
};

use core::cell::Cell;
use core::cell::UnsafeCell;
use core::ptr::NonNull;
use malachite_bigint::{BigInt, ToBigInt};
use num_complex::Complex64;
Expand All @@ -36,7 +36,7 @@ impl PyFloat {
}

thread_local! {
static FLOAT_FREELIST: Cell<crate::object::FreeList<PyFloat>> = const { Cell::new(crate::object::FreeList::new()) };
static FLOAT_FREELIST: UnsafeCell<crate::object::FreeList<PyFloat>> = const { UnsafeCell::new(crate::object::FreeList::new()) };
}

impl PyPayload for PyFloat {
Expand All @@ -48,34 +48,18 @@ impl PyPayload for PyFloat {
ctx.types.float_type
}

fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef {
vm.ctx.new_float(self.value).into()
}

#[inline]
unsafe fn freelist_push(obj: *mut PyObject) -> bool {
FLOAT_FREELIST
.try_with(|fl| {
let mut list = fl.take();
let stored = if list.len() < Self::MAX_FREELIST {
list.push(obj);
true
} else {
false
};
fl.set(list);
stored
})
.unwrap_or(false)
unsafe { crate::object::FreeList::push_local(&FLOAT_FREELIST, obj) }
}

#[inline]
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
FLOAT_FREELIST
.try_with(|fl| {
let mut list = fl.take();
let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) });
fl.set(list);
result
})
.ok()
.flatten()
crate::object::FreeList::pop_local(&FLOAT_FREELIST)
}
}

Expand Down
28 changes: 4 additions & 24 deletions crates/vm/src/builtins/int.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ use crate::{
types::{AsNumber, Comparable, Constructor, Hashable, PyComparisonOp, Representable},
};
use alloc::fmt;
use core::cell::Cell;
use core::cell::UnsafeCell;
use core::ops::{Neg, Not};
use core::ptr::NonNull;
use malachite_bigint::{BigInt, Sign};
Expand Down Expand Up @@ -52,7 +52,7 @@ where

// spell-checker:ignore MAXFREELIST
thread_local! {
static INT_FREELIST: Cell<crate::object::FreeList<PyInt>> = const { Cell::new(crate::object::FreeList::new()) };
static INT_FREELIST: UnsafeCell<crate::object::FreeList<PyInt>> = const { UnsafeCell::new(crate::object::FreeList::new()) };
}

impl PyPayload for PyInt {
Expand All @@ -70,32 +70,12 @@ impl PyPayload for PyInt {

#[inline]
unsafe fn freelist_push(obj: *mut PyObject) -> bool {
INT_FREELIST
.try_with(|fl| {
let mut list = fl.take();
let stored = if list.len() < Self::MAX_FREELIST {
list.push(obj);
true
} else {
false
};
fl.set(list);
stored
})
.unwrap_or(false)
unsafe { crate::object::FreeList::push_local(&INT_FREELIST, obj) }
}

#[inline]
unsafe fn freelist_pop(_payload: &Self) -> Option<NonNull<PyObject>> {
INT_FREELIST
.try_with(|fl| {
let mut list = fl.take();
let result = list.pop().map(|p| unsafe { NonNull::new_unchecked(p) });
fl.set(list);
result
})
.ok()
.flatten()
crate::object::FreeList::pop_local(&INT_FREELIST)
}
}

Expand Down
105 changes: 105 additions & 0 deletions crates/vm/src/object/core.rs
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,29 @@ pub(super) unsafe fn default_dealloc<T: PyPayload>(obj: *mut PyObject) {
unsafe { trashcan::end() };
}
}
/// Dealloc for payloads that have a freelist and no `tp_clear`. An instance
/// of exactly the base class with nothing attached — no `__del__`, weakrefs,
/// dict or member slots, GC tracking or QSBR publication — needs nothing from
/// `default_dealloc` but its freelist push, so it gets there directly instead
/// of through the `__del__`/weakref/trashcan/clear machinery. Anything else
/// takes `default_dealloc`, which re-checks all of it.
pub(super) unsafe fn freelist_dealloc<T: PyPayload>(obj: *mut PyObject) {
let obj_ref = unsafe { &*(obj as *const PyObject) };
let typ = obj_ref.class();
let plain = core::ptr::eq(typ, T::class(crate::vm::Context::genesis()))
&& typ.slots.del.load().is_none()
&& !typ.slots.flags.intersects(
crate::types::PyTypeFlags::HAS_DICT | crate::types::PyTypeFlags::HAS_WEAKREF,
)
&& typ.slots.member_count == 0
&& !obj_ref.is_gc_tracked()
&& !obj_ref.0.ref_count.is_published();
if plain && unsafe { T::freelist_push(obj) } {
return;
}
unsafe { default_dealloc::<T>(obj) }
}

pub(super) unsafe fn debug_obj<T: PyPayload + core::fmt::Debug>(
x: &PyObject,
f: &mut fmt::Formatter<'_>,
Expand Down Expand Up @@ -1394,6 +1417,53 @@ impl<T: PyPayload> Drop for FreeList<T> {
}
}

impl<T: PyPayload> FreeList<T> {
/// Push a dead husk onto this thread's freelist in `key`, unless it is
/// full or the thread-local is already gone.
///
/// The list is edited in place rather than moved out of a `Cell` and back,
/// which copied the whole `Vec` twice per object. Nothing between taking
/// the `&mut` and dropping it can re-enter: `Vec::push` only calls the
/// allocator, never Python code or another freelist.
///
/// # Safety
/// Same contract as [`PyPayload::freelist_push`].
#[inline]
pub(crate) unsafe fn push_local(
key: &'static std::thread::LocalKey<core::cell::UnsafeCell<Self>>,
obj: *mut PyObject,
) -> bool {
key.try_with(|cell| {
// SAFETY: see above; no other borrow of this cell is live.
let list = unsafe { &mut *cell.get() };
if list.items.len() < T::MAX_FREELIST {
list.items.push(obj);
true
} else {
false
}
})
.unwrap_or(false)
}

/// Pop a husk from this thread's freelist in `key`, editing it in place
/// (see [`Self::push_local`]).
#[inline]
pub(crate) fn pop_local(
key: &'static std::thread::LocalKey<core::cell::UnsafeCell<Self>>,
) -> Option<NonNull<PyObject>> {
key.try_with(|cell| {
// SAFETY: `Vec::pop` cannot re-enter; no other borrow is live.
let list = unsafe { &mut *cell.get() };
list.items
.pop()
.map(|p| unsafe { NonNull::new_unchecked(p) })
})
.ok()
.flatten()
}
}

impl<T: PyPayload> core::ops::Deref for FreeList<T> {
type PyObject>;
fn deref(&self) -> &Self::Target {
Expand Down Expand Up @@ -2746,6 +2816,41 @@ impl<T: PyPayload + crate::object::MaybeTraverse + core::fmt::Debug> PyRef<T> {

Self { ptr }
}

/// `new_ref` for an instance of exactly `T::class(ctx)`, a static type
/// whose instances carry no dict. `default_dealloc` only pushes instances
/// of exactly that class onto `T`'s freelist, so a reused husk already
/// points at it: the dict and heap-type checks, the type swap and the
/// type reference clone `new_ref` does for an arbitrary class are skipped.
#[inline(always)]
pub(crate) fn new_exact_ref(payload: T, ctx: &crate::vm::Context) -> Self {
let class = T::class(ctx);
debug_assert!(class.heaptype_ext.is_none());
debug_assert!(
!class
.slots
.flags
.has_feature(crate::types::PyTypeFlags::HAS_DICT)
);
let Some(cached) = (unsafe { T::freelist_pop(&payload) }) else {
return Self::new_ref(payload, class.to_owned(), None);
};
let inner = cached.as_ptr() as *mut PyInner<T>;
unsafe {
core::ptr::write(&mut (*inner).ref_count, RefCount::new());
(*inner).gc_bits.store(0, Ordering::Relaxed);
core::ptr::drop_in_place(&mut (*inner).payload);
core::ptr::write(&mut (*inner).payload, payload);
debug_assert!(core::ptr::eq(&*(*inner).typ, class));
}
let ptr = unsafe { NonNull::new_unchecked(inner.cast::<Py<T>>()) };
if <T as crate::object::MaybeTraverse>::HAS_TRAVERSE && !T::NEW_REF_UNTRACKED {
unsafe {
crate::gc_state::track_new_object(ptr.cast());
}
}
Self { ptr }
}
}

impl<T: crate::class::PySubclass + core::fmt::Debug> PyRef<T>
Expand Down
10 changes: 8 additions & 2 deletions crates/vm/src/object/traverse_object.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use crate::{
PyObject, PyObjectRef,
object::{
Erased, InstanceDict, MaybeTraverse, PyInner, PyObjectPayload, debug_obj, default_dealloc,
try_clear_obj, try_traverse_obj,
freelist_dealloc, try_clear_obj, try_traverse_obj,
},
};

Expand All @@ -26,7 +26,13 @@ impl PyObjVTable {
pub(super) const fn of<T: PyObjectPayload>() -> &'static Self {
&Self {
typeid: T::PAYLOAD_TYPE_ID,
dealloc: default_dealloc::<T>,
dealloc: const {
if T::HAS_FREELIST && !T::HAS_CLEAR {
freelist_dealloc::<T>
} else {
default_dealloc::<T>
}
},
debug: debug_obj::<T>,
trace: const {
if T::HAS_TRAVERSE {
Expand Down
4 changes: 2 additions & 2 deletions crates/vm/src/vm/context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -469,7 +469,7 @@ impl Context {
let inner_idx = (i - Self::INT_CACHE_POOL_MIN) as usize;
return self.int_cache_pool[inner_idx].clone();
}
PyInt::from(i).into_ref(self)
PyRef::new_exact_ref(PyInt::from(i), self)
}

/// Borrow a cached small integer whose lifetime is tied to this context.
Expand All @@ -493,7 +493,7 @@ impl Context {

#[inline]
pub fn new_float(&self, value: f64) -> PyRef<PyFloat> {
PyFloat::from(value).into_ref(self)
PyRef::new_exact_ref(PyFloat::from(value), self)
}

#[inline]
Expand Down
92 changes: 92 additions & 0 deletions extra_tests/snippets/vm_freelist.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
import gc
import weakref

# Objects of exactly int/float/complex/range are recycled through per-type
# freelists; everything else about deallocation must behave as before.


## Recycled objects carry the right value and type


def churn_ints(n):
total = 0
for i in range(1000, 1000 + n):
total += i
return total


assert churn_ints(5000) == sum(range(1000, 6000))
seen = [i for i in range(250, 270)]
assert seen == list(range(250, 270))
for i in range(-5, 257):
assert i is int(str(i)), i

floats = [x * 1.5 for x in range(2000)]
assert floats[-1] == 1999 * 1.5
assert all(type(f) is float for f in floats)
complexes = [complex(i, -i) for i in range(2000)]
assert complexes[123] == complex(123, -123)
ranges = [range(i, i + 3) for i in range(2000)]
assert ranges[77] == range(77, 80) and list(ranges[5]) == [5, 6, 7]
del floats, complexes, ranges


## Subclass instances take the full path: __del__ and weakrefs still work


class DelInt(int):
deleted = 0

def __del__(self):
DelInt.deleted += 1


for i in range(100):
DelInt(i + 1000)
gc.collect()
assert DelInt.deleted == 100, DelInt.deleted


class RefFloat(float):
pass


callbacks = []
obj = RefFloat(12345.5)
ref = weakref.ref(obj, lambda r: callbacks.append(r))
del obj
gc.collect()
assert ref() is None
assert len(callbacks) == 1


class DelFloat(float):
deleted = 0

def __del__(self):
DelFloat.deleted += 1


for i in range(50):
DelFloat(i + 0.5)
gc.collect()
assert DelFloat.deleted == 50, DelFloat.deleted


## Subclass husks never come back as plain ints


class Plain(int):
pass


for i in range(500):
Plain(i + 1000)
for i in range(1000, 1500):
x = i * 1
assert type(x) is int, type(x)
assert x == i

# bool shares int's payload; its singletons are untouched by the churn.
assert True + 0 == 1 and type(True + 0) is int
assert type(True) is bool and True is (1 == 1)
Loading