Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 44 additions & 12 deletions crates/vm/src/builtins/bytearray.rs
Original file line number Diff line number Diff line change
Expand Up @@ -140,14 +140,35 @@ impl PyByteArray {

pub fn _delitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<()> {
match SequenceIndex::try_from_borrowed_object::<NameBytearray>(vm, needle)? {
SequenceIndex::Int(i) => self.try_resizable(vm)?.elements.delitem_by_index(vm, i),
SequenceIndex::Int(i) => self.delitem_by_index(i, vm),
SequenceIndex::Slice(slice) => {
// TODO: delete 0 elements don't need resizable
self.try_resizable(vm)?.elements.delitem_by_slice(vm, slice)
}
}
}

fn delitem_by_index(&self, index: isize, vm: &VirtualMachine) -> PyResult<()> {
// Validate before export rejection when the bytes are available without
// waiting. A codec callback may already hold a read guard with a writer
// queued, so a blocking read here could deadlock before that rejection.
let valid = self
.inner
.try_read()
.map(|inner| inner.elements.wrap_index(index).is_some());
if valid == Some(false) {
return Err(vm.new_index_error("bytearray index out of range"));
}
let mut inner = self.try_resizable(vm)?;
// Another thread may have resized the bytearray between the locks.
let index = inner
.elements
.wrap_index(index)
.ok_or_else(|| vm.new_index_error("bytearray index out of range"))?;
inner.elements.remove(index);
Ok(())
}

fn irepeat(zelf: &Py<Self>, n: isize, vm: &VirtualMachine) -> PyResult<()> {
if n == 1 {
return Ok(());
Expand All @@ -173,11 +194,6 @@ impl PyByteArray {
fn inner(&self) -> PyDetachingRwLockReadGuard<'_, PyBytesInner> {
self.inner.read()
}
#[inline]
fn inner_mut(&self) -> PyDetachingRwLockWriteGuard<'_, PyBytesInner> {
self.inner.write()
}

fn __len__(&self) -> usize {
self.borrow_buf().len()
}
Expand Down Expand Up @@ -781,9 +797,18 @@ impl Initializer for PyByteArray {
type Args = ByteInnerNewOptions;

fn init(zelf: &Py<Self>, options: Self::Args, vm: &VirtualMachine) -> PyResult<()> {
// First unpack bytearray and *then* get a lock to set it.
let mut inner = options.get_inner(bytearray_from_object, vm)?;
core::mem::swap(&mut *zelf.inner_mut(), &mut inner);
// Reject exports and clear existing contents before converting source.
zelf.try_resizable(vm)?.elements = Vec::new();

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Reject reentrant initialization without self-deadlocking

When target + other reaches a Python other.__buffer__ that calls target.__init__(object()) (and catches the exception), AsSequence::concat is still holding target.inner() for reading. This new eager try_resizable then waits for the same thread's write lock, so the expression hangs; in the parent, invalid-source conversion raised TypeError before any write lock was taken and concatenation completed. Mark the outer borrow as an export so reentry gets BufferError, or otherwise avoid taking this write lock reentrantly.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Reject initializer reentry instead of waiting on the read lock

The new concat buffering removes the earlier target + other deadlock, but target == other still calls zelf.inner().cmp(...) with a read guard while other.__buffer__ runs. If that callback calls target.__init__(object()) and catches the exception, this eager try_resizable waits forever for the same thread's read guard; the parent converted the invalid source first and returned TypeError instead. Mark outer callback borrows such as comparison and starts/endswith as exports, or make this guard fail rather than wait reentrantly.

Useful? React with 👍 / 👎.

// Conversion can call Python, so do not hold the lock across it and
// check again for exports acquired by a callback or another thread.
let (mut inner, zero_count) =
options.get_inner_with_zero_count(bytearray_from_object, vm)?;
// A zero integer count performs no further writes, including when its
// __index__ callback changed this bytearray or acquired an export.
if zero_count {
return Ok(());
Comment on lines +935 to +936

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Release the previous allocation for zero-count initialization

When a large existing bytearray is reinitialized with an integer (or int-like value) of zero, this early return follows Vec::clear(), which preserves capacity, so bytearray.__alloc__() continues to expose the old allocation and potentially gigabytes remain held for the object's lifetime. The old implementation swapped in the newly created empty inner and released that storage; replace the backing vector while performing the initial clear so the zero-count callback behavior is kept without retaining the old allocation.

Useful? React with 👍 / 👎.

}
core::mem::swap(&mut *zelf.try_resizable(vm)?, &mut inner);
Comment on lines +935 to +938

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Treat source-less initialization as complete after clearing

When bytearray.__init__() is called without a source and another thread acquires a memoryview after the initial clear, get_inner_with_zero_count leaves this flag false, so the second resize guard raises BufferError. The call has already discarded the old contents and has no remaining write to perform, making a completed clear appear to fail; mark the missing-source case as no-further-write or keep the initial guard through that callback-free path.

Useful? React with 👍 / 👎.

Ok(())
}
}
Expand Down Expand Up @@ -841,9 +866,11 @@ impl AsBuffer for PyByteArray {
}

fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
// Keep the length stable until PyBuffer::new retains the export.
let inner = zelf.inner();
Ok(PyBuffer::new(
zelf.to_owned().into(),
BufferDescriptor::simple(zelf.__len__(), false),
BufferDescriptor::simple(inner.elements.len(), false),
&BUFFER_METHODS,
))
}
Expand All @@ -855,7 +882,12 @@ impl BufferResizeGuard for PyByteArray {
fn try_resizable_opt(&self) -> Option<Self::Resizable<'_>> {
// An export is a borrow someone else still holds, so it is answered
// before the lock rather than by waiting on it.
(self.exports.load(Ordering::SeqCst) == 0).then(|| self.inner.write())
if self.exports.load(Ordering::SeqCst) != 0 {
return None;
}
let inner = self.inner.write();
// An export may have been acquired while we waited for the lock.
(self.exports.load(Ordering::SeqCst) == 0).then_some(inner)
}
}

Expand Down Expand Up @@ -907,7 +939,7 @@ impl AsSequence for PyByteArray {
if let Some(value) = value {
zelf._setitem_by_index(i, &value, vm)
} else {
zelf.borrow_buf_mut().delitem_by_index(vm, i)
zelf.delitem_by_index(i, vm)
}
}),
contains: atomic_func!(|seq, other, vm| {
Expand Down
19 changes: 16 additions & 3 deletions crates/vm/src/bytes_inner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -225,12 +225,24 @@ impl ByteInnerNewOptions {
/// read: `bytes()` and `bytearray()` differ in whether they ask it how long
/// it is.
pub fn get_inner(self, from_object: FromObject, vm: &VirtualMachine) -> PyResult<PyBytesInner> {
match (self.source, self.encoding, self.errors) {
self.get_inner_with_zero_count(from_object, vm)
.map(|(inner, _)| inner)
}

pub(crate) fn get_inner_with_zero_count(
self,
from_object: FromObject,
vm: &VirtualMachine,
) -> PyResult<(PyBytesInner, bool)> {
let mut zero_count = false;
let inner = match (self.source, self.encoding, self.errors) {
(OptionalArg::Present(obj), OptionalArg::Missing, OptionalArg::Missing) => {
// Try __index__ first to handle int-like objects that might raise custom exceptions
if let Some(index_result) = obj.try_index_opt(vm) {
match index_result {
Ok(index) => Self::get_value_from_size(&index, vm),
Ok(index) => Self::get_value_from_size(&index, vm).inspect(|inner| {
zero_count = inner.elements.is_empty();
}),
Err(e) => {
// Only propagate non-TypeError exceptions
// TypeError means the object doesn't support __index__, so fall back
Expand Down Expand Up @@ -266,7 +278,8 @@ impl ByteInnerNewOptions {
(OptionalArg::Present(_), OptionalArg::Missing, OptionalArg::Present(_)) => {
Err(vm.new_type_error(STRING_WITHOUT_ENCODING.to_owned()))
}
}
}?;
Ok((inner, zero_count))
}
}

Expand Down
11 changes: 7 additions & 4 deletions crates/vm/src/stdlib/_io.rs
Original file line number Diff line number Diff line change
Expand Up @@ -762,10 +762,13 @@ mod _io {
)));
}
let n = n as usize;
let mut bytes = b.borrow_buf_mut();
bytes.truncate(n);
// FIXME: try to use Arc::unwrap on the bytearray to get at the inner buffer
bytes.clone().to_pyobject(vm)
let mut inner = b.try_resizable(vm)?;
let bytes = &mut inner.elements;
bytes
.try_reserve_exact(n.saturating_sub(bytes.len()))
.map_err(|_| vm.no_memory_error())?;
bytes.resize(n, 0);
core::mem::take(bytes).to_pyobject(vm)
}
})
} else {
Expand Down
136 changes: 136 additions & 0 deletions extra_tests/snippets/builtin_bytearray_exports.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
"""Resizing paths must respect outstanding bytearray exports."""

import io

from testutils import assert_raises

for args in ((), (b"abc",), (b"x",), (b"",), (object(),)):

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Remove the prohibited test-logic additions

Remove this new regression snippet and the two sibling builtin_bytearray_* snippets: the repository's critical test policy allows test-file edits only for adding or removing expected-failure markers, whereas these files introduce new assertions, control flow, and test data. Keeping them in this change violates the contribution rules even if the tests pass.

AGENTS.md reference: AGENTS.md:L273-L279

Useful? React with 👍 / 👎.

value = bytearray(b"abc")
with memoryview(value) as view:
with assert_raises(BufferError):
value.__init__(*args)
assert value == b"abc"
assert view.tobytes() == b"abc"
value.__init__(b"released")
assert value == b"released"

value = bytearray(b"abc")
with assert_raises(TypeError):
value.__init__(object())
assert value == b""
value = bytearray(b"abc")
value.__init__(value)
assert value == b""


class ExportDuringInit:
def __init__(self, count, payload=b""):
self.count = count
self.payload = payload
self.calls = 0

def __index__(self):
self.calls += 1
value.extend(self.payload)
self.view = memoryview(value)
return self.count


value = bytearray(b"abc")
source = ExportDuringInit(2)
with assert_raises(BufferError):
value.__init__(source)
assert value == b""
assert source.view.tobytes() == b""
assert source.calls == 1
source.view.release()
value.extend(b"released")
assert value == b"released"

value = bytearray(b"abc")
source = ExportDuringInit(0)
value.__init__(source)
assert value == b""
assert source.view.tobytes() == b""
source.view.release()

value = bytearray(b"abc")
source = ExportDuringInit(0, b"xy")
value.__init__(source)
assert value == b"xy"
assert source.view.tobytes() == b"xy"
assert source.calls == 1
source.view.release()


class FailingIndex:
def __index__(self):
assert value == b""
raise ValueError("conversion failed")


value = bytearray(b"abc")
with assert_raises(ValueError):
value.__init__(FailingIndex())
assert value == b""

for key in (0, slice(0, 1)):
value = bytearray(b"abc")
with memoryview(value) as view:
with assert_raises(BufferError):
del value[key]
assert view.tobytes() == b"abc"
del value[key]
assert value == b"bc"


class Reader(io.RawIOBase):
def __init__(self, count, retain, payload=b"abcd"):
self.count = count
self.retain = retain
self.payload = payload

def readinto(self, buffer):
buffer[:] = self.payload
self.buffer = buffer
if self.retain:
self.view = memoryview(buffer)
return self.count


for count in (0, 2, 4):
reader = Reader(count, False)
assert reader.read(4) == b"abcd"[:count]
assert reader.buffer == b""

reader = Reader(count, True)
with assert_raises(BufferError):
reader.read(4)
assert reader.view.tobytes() == b"abcd"
reader.view.release()
reader.buffer.clear()

for count in (-1, 5):
reader = Reader(count, True)
with assert_raises(ValueError):
reader.read(4)
assert reader.view.tobytes() == b"abcd"
reader.view.release()

reader = Reader(None, True)
assert reader.read(4) is None
assert reader.view.tobytes() == b"abcd"
reader.view.release()

# The callback can shrink the temporary buffer, so slicing by the original
# requested size without checking the current buffer is not safe.
reader = Reader(2, False, b"xy")
assert reader.read(4) == b"xy"
assert reader.buffer == b""

reader = Reader(4, False, b"xy")
result = reader.read(4)
assert len(result) == 4
assert result[:2] == b"xy"
# The contents of the newly grown region are unspecified by CPython.
assert reader.buffer == b""
70 changes: 70 additions & 0 deletions extra_tests/snippets/builtin_bytearray_init_exports.py

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do not make multiple tset files. move 5 files into builtin_bytearray.py

Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
"""Initializer storage and integer-deletion ordering with buffer exports."""

from testutils import assert_raises


class Zero:
def __init__(self, value):
self.value = value
self.observations = []

def __index__(self):
self.observations.append((len(self.value), self.value.__alloc__()))
return 0


# Final CPython rejects even an empty exported initializer before conversion.
for kind in ("missing", "integer", "callback"):
value = bytearray()
source = Zero(value)
with memoryview(value) as view:
with assert_raises(BufferError):
if kind == "missing":
value.__init__()
elif kind == "integer":
value.__init__(0)
else:
value.__init__(source)
assert value == b""
assert view.tobytes() == b""
assert source.observations == []

# Clearing an initializer must release its old storage before callbacks and
# before the successful zero-count early return.
for kind in ("missing", "integer", "callback"):
value = bytearray(b"x" * 64)
source = Zero(value)
assert value.__alloc__() > 0
if kind == "missing":
value.__init__()
elif kind == "integer":
value.__init__(0)
else:
value.__init__(source)
assert source.observations == [(0, 0)]
assert value == b""
assert value.__alloc__() == 0

# An invalid integer index is rejected before checking whether resizing is
# blocked, and neither error path may alter the exported bytes.
for initial, indices in ((b"abc", (3, -4)), (b"", (0, -1))):
for index in indices:
value = bytearray(initial)
with memoryview(value) as view:
try:
del value[index]
except IndexError as exc:
assert str(exc) == "bytearray index out of range"
else:
raise AssertionError("invalid index was accepted")
assert value == initial
assert view.tobytes() == initial

for index in (0, -1):
value = bytearray(b"abc")
with memoryview(value) as view:
with assert_raises(BufferError):
del value[index]
assert view.tobytes() == b"abc"
del value[index]
assert value == (b"bc" if index == 0 else b"ab")
Loading
Loading