Repository navigation
Serialize bytearray exports with resizing #9087
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
d0819a2
fcd4f4c
9fdae0a
a0fec1f
f163e13
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -140,14 +140,35 @@ impl PyByteArray { | |
|
|
||
| pub fn _delitem(&self, needle: &PyObject, vm: &VirtualMachine) -> PyResult<()> { | ||
| match SequenceIndex::try_from_borrowed_object::<NameBytearray>(vm, needle)? { | ||
| SequenceIndex::Int(i) => self.try_resizable(vm)?.elements.delitem_by_index(vm, i), | ||
| SequenceIndex::Int(i) => self.delitem_by_index(i, vm), | ||
| SequenceIndex::Slice(slice) => { | ||
| // TODO: delete 0 elements don't need resizable | ||
| self.try_resizable(vm)?.elements.delitem_by_slice(vm, slice) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| fn delitem_by_index(&self, index: isize, vm: &VirtualMachine) -> PyResult<()> { | ||
| // Validate before export rejection when the bytes are available without | ||
| // waiting. A codec callback may already hold a read guard with a writer | ||
| // queued, so a blocking read here could deadlock before that rejection. | ||
| let valid = self | ||
| .inner | ||
| .try_read() | ||
| .map(|inner| inner.elements.wrap_index(index).is_some()); | ||
| if valid == Some(false) { | ||
| return Err(vm.new_index_error("bytearray index out of range")); | ||
| } | ||
| let mut inner = self.try_resizable(vm)?; | ||
| // Another thread may have resized the bytearray between the locks. | ||
| let index = inner | ||
| .elements | ||
| .wrap_index(index) | ||
| .ok_or_else(|| vm.new_index_error("bytearray index out of range"))?; | ||
| inner.elements.remove(index); | ||
| Ok(()) | ||
| } | ||
|
|
||
| fn irepeat(zelf: &Py<Self>, n: isize, vm: &VirtualMachine) -> PyResult<()> { | ||
| if n == 1 { | ||
| return Ok(()); | ||
|
|
@@ -173,11 +194,6 @@ impl PyByteArray { | |
| fn inner(&self) -> PyDetachingRwLockReadGuard<'_, PyBytesInner> { | ||
| self.inner.read() | ||
| } | ||
| #[inline] | ||
| fn inner_mut(&self) -> PyDetachingRwLockWriteGuard<'_, PyBytesInner> { | ||
| self.inner.write() | ||
| } | ||
|
|
||
| fn __len__(&self) -> usize { | ||
| self.borrow_buf().len() | ||
| } | ||
|
|
@@ -781,9 +797,18 @@ impl Initializer for PyByteArray { | |
| type Args = ByteInnerNewOptions; | ||
|
|
||
| fn init(zelf: &Py<Self>, options: Self::Args, vm: &VirtualMachine) -> PyResult<()> { | ||
| // First unpack bytearray and *then* get a lock to set it. | ||
| let mut inner = options.get_inner(bytearray_from_object, vm)?; | ||
| core::mem::swap(&mut *zelf.inner_mut(), &mut inner); | ||
| // Reject exports and clear existing contents before converting source. | ||
| zelf.try_resizable(vm)?.elements = Vec::new(); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
The new concat buffering removes the earlier Useful? React with 👍 / 👎. |
||
| // Conversion can call Python, so do not hold the lock across it and | ||
| // check again for exports acquired by a callback or another thread. | ||
| let (mut inner, zero_count) = | ||
| options.get_inner_with_zero_count(bytearray_from_object, vm)?; | ||
| // A zero integer count performs no further writes, including when its | ||
| // __index__ callback changed this bytearray or acquired an export. | ||
| if zero_count { | ||
| return Ok(()); | ||
|
Comment on lines
+935
to
+936
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When a large existing bytearray is reinitialized with an integer (or int-like value) of zero, this early return follows Useful? React with 👍 / 👎. |
||
| } | ||
| core::mem::swap(&mut *zelf.try_resizable(vm)?, &mut inner); | ||
|
Comment on lines
+935
to
+938
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When Useful? React with 👍 / 👎. |
||
| Ok(()) | ||
| } | ||
| } | ||
|
|
@@ -841,9 +866,11 @@ impl AsBuffer for PyByteArray { | |
| } | ||
|
|
||
| fn as_buffer(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> { | ||
| // Keep the length stable until PyBuffer::new retains the export. | ||
| let inner = zelf.inner(); | ||
| Ok(PyBuffer::new( | ||
| zelf.to_owned().into(), | ||
| BufferDescriptor::simple(zelf.__len__(), false), | ||
| BufferDescriptor::simple(inner.elements.len(), false), | ||
| &BUFFER_METHODS, | ||
| )) | ||
| } | ||
|
|
@@ -855,7 +882,12 @@ impl BufferResizeGuard for PyByteArray { | |
| fn try_resizable_opt(&self) -> Option<Self::Resizable<'_>> { | ||
| // An export is a borrow someone else still holds, so it is answered | ||
| // before the lock rather than by waiting on it. | ||
| (self.exports.load(Ordering::SeqCst) == 0).then(|| self.inner.write()) | ||
| if self.exports.load(Ordering::SeqCst) != 0 { | ||
| return None; | ||
| } | ||
| let inner = self.inner.write(); | ||
| // An export may have been acquired while we waited for the lock. | ||
| (self.exports.load(Ordering::SeqCst) == 0).then_some(inner) | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -907,7 +939,7 @@ impl AsSequence for PyByteArray { | |
| if let Some(value) = value { | ||
| zelf._setitem_by_index(i, &value, vm) | ||
| } else { | ||
| zelf.borrow_buf_mut().delitem_by_index(vm, i) | ||
| zelf.delitem_by_index(i, vm) | ||
| } | ||
| }), | ||
| contains: atomic_func!(|seq, other, vm| { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| """Resizing paths must respect outstanding bytearray exports.""" | ||
|
|
||
| import io | ||
|
|
||
| from testutils import assert_raises | ||
|
|
||
| for args in ((), (b"abc",), (b"x",), (b"",), (object(),)): | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Remove this new regression snippet and the two sibling AGENTS.md reference: AGENTS.md:L273-L279 Useful? React with 👍 / 👎. |
||
| value = bytearray(b"abc") | ||
| with memoryview(value) as view: | ||
| with assert_raises(BufferError): | ||
| value.__init__(*args) | ||
| assert value == b"abc" | ||
| assert view.tobytes() == b"abc" | ||
| value.__init__(b"released") | ||
| assert value == b"released" | ||
|
|
||
| value = bytearray(b"abc") | ||
| with assert_raises(TypeError): | ||
| value.__init__(object()) | ||
| assert value == b"" | ||
| value = bytearray(b"abc") | ||
| value.__init__(value) | ||
| assert value == b"" | ||
|
|
||
|
|
||
| class ExportDuringInit: | ||
| def __init__(self, count, payload=b""): | ||
| self.count = count | ||
| self.payload = payload | ||
| self.calls = 0 | ||
|
|
||
| def __index__(self): | ||
| self.calls += 1 | ||
| value.extend(self.payload) | ||
| self.view = memoryview(value) | ||
| return self.count | ||
|
|
||
|
|
||
| value = bytearray(b"abc") | ||
| source = ExportDuringInit(2) | ||
| with assert_raises(BufferError): | ||
| value.__init__(source) | ||
| assert value == b"" | ||
| assert source.view.tobytes() == b"" | ||
| assert source.calls == 1 | ||
| source.view.release() | ||
| value.extend(b"released") | ||
| assert value == b"released" | ||
|
|
||
| value = bytearray(b"abc") | ||
| source = ExportDuringInit(0) | ||
| value.__init__(source) | ||
| assert value == b"" | ||
| assert source.view.tobytes() == b"" | ||
| source.view.release() | ||
|
|
||
| value = bytearray(b"abc") | ||
| source = ExportDuringInit(0, b"xy") | ||
| value.__init__(source) | ||
| assert value == b"xy" | ||
| assert source.view.tobytes() == b"xy" | ||
| assert source.calls == 1 | ||
| source.view.release() | ||
|
|
||
|
|
||
| class FailingIndex: | ||
| def __index__(self): | ||
| assert value == b"" | ||
| raise ValueError("conversion failed") | ||
|
|
||
|
|
||
| value = bytearray(b"abc") | ||
| with assert_raises(ValueError): | ||
| value.__init__(FailingIndex()) | ||
| assert value == b"" | ||
|
|
||
| for key in (0, slice(0, 1)): | ||
| value = bytearray(b"abc") | ||
| with memoryview(value) as view: | ||
| with assert_raises(BufferError): | ||
| del value[key] | ||
| assert view.tobytes() == b"abc" | ||
| del value[key] | ||
| assert value == b"bc" | ||
|
|
||
|
|
||
| class Reader(io.RawIOBase): | ||
| def __init__(self, count, retain, payload=b"abcd"): | ||
| self.count = count | ||
| self.retain = retain | ||
| self.payload = payload | ||
|
|
||
| def readinto(self, buffer): | ||
| buffer[:] = self.payload | ||
| self.buffer = buffer | ||
| if self.retain: | ||
| self.view = memoryview(buffer) | ||
| return self.count | ||
|
|
||
|
|
||
| for count in (0, 2, 4): | ||
| reader = Reader(count, False) | ||
| assert reader.read(4) == b"abcd"[:count] | ||
| assert reader.buffer == b"" | ||
|
|
||
| reader = Reader(count, True) | ||
| with assert_raises(BufferError): | ||
| reader.read(4) | ||
| assert reader.view.tobytes() == b"abcd" | ||
| reader.view.release() | ||
| reader.buffer.clear() | ||
|
|
||
| for count in (-1, 5): | ||
| reader = Reader(count, True) | ||
| with assert_raises(ValueError): | ||
| reader.read(4) | ||
| assert reader.view.tobytes() == b"abcd" | ||
| reader.view.release() | ||
|
|
||
| reader = Reader(None, True) | ||
| assert reader.read(4) is None | ||
| assert reader.view.tobytes() == b"abcd" | ||
| reader.view.release() | ||
|
|
||
| # The callback can shrink the temporary buffer, so slicing by the original | ||
| # requested size without checking the current buffer is not safe. | ||
| reader = Reader(2, False, b"xy") | ||
| assert reader.read(4) == b"xy" | ||
| assert reader.buffer == b"" | ||
|
|
||
| reader = Reader(4, False, b"xy") | ||
| result = reader.read(4) | ||
| assert len(result) == 4 | ||
| assert result[:2] == b"xy" | ||
| # The contents of the newly grown region are unspecified by CPython. | ||
| assert reader.buffer == b"" | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. do not make multiple tset files. move 5 files into builtin_bytearray.py |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| """Initializer storage and integer-deletion ordering with buffer exports.""" | ||
|
|
||
| from testutils import assert_raises | ||
|
|
||
|
|
||
| class Zero: | ||
| def __init__(self, value): | ||
| self.value = value | ||
| self.observations = [] | ||
|
|
||
| def __index__(self): | ||
| self.observations.append((len(self.value), self.value.__alloc__())) | ||
| return 0 | ||
|
|
||
|
|
||
| # Final CPython rejects even an empty exported initializer before conversion. | ||
| for kind in ("missing", "integer", "callback"): | ||
| value = bytearray() | ||
| source = Zero(value) | ||
| with memoryview(value) as view: | ||
| with assert_raises(BufferError): | ||
| if kind == "missing": | ||
| value.__init__() | ||
| elif kind == "integer": | ||
| value.__init__(0) | ||
| else: | ||
| value.__init__(source) | ||
| assert value == b"" | ||
| assert view.tobytes() == b"" | ||
| assert source.observations == [] | ||
|
|
||
| # Clearing an initializer must release its old storage before callbacks and | ||
| # before the successful zero-count early return. | ||
| for kind in ("missing", "integer", "callback"): | ||
| value = bytearray(b"x" * 64) | ||
| source = Zero(value) | ||
| assert value.__alloc__() > 0 | ||
| if kind == "missing": | ||
| value.__init__() | ||
| elif kind == "integer": | ||
| value.__init__(0) | ||
| else: | ||
| value.__init__(source) | ||
| assert source.observations == [(0, 0)] | ||
| assert value == b"" | ||
| assert value.__alloc__() == 0 | ||
|
|
||
| # An invalid integer index is rejected before checking whether resizing is | ||
| # blocked, and neither error path may alter the exported bytes. | ||
| for initial, indices in ((b"abc", (3, -4)), (b"", (0, -1))): | ||
| for index in indices: | ||
| value = bytearray(initial) | ||
| with memoryview(value) as view: | ||
| try: | ||
| del value[index] | ||
| except IndexError as exc: | ||
| assert str(exc) == "bytearray index out of range" | ||
| else: | ||
| raise AssertionError("invalid index was accepted") | ||
| assert value == initial | ||
| assert view.tobytes() == initial | ||
|
|
||
| for index in (0, -1): | ||
| value = bytearray(b"abc") | ||
| with memoryview(value) as view: | ||
| with assert_raises(BufferError): | ||
| del value[index] | ||
| assert view.tobytes() == b"abc" | ||
| del value[index] | ||
| assert value == (b"bc" if index == 0 else b"ab") |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When
target + otherreaches a Pythonother.__buffer__that callstarget.__init__(object())(and catches the exception),AsSequence::concatis still holdingtarget.inner()for reading. This new eagertry_resizablethen waits for the same thread's write lock, so the expression hangs; in the parent, invalid-source conversion raisedTypeErrorbefore any write lock was taken and concatenation completed. Mark the outer borrow as an export so reentry getsBufferError, or otherwise avoid taking this write lock reentrantly.Useful? React with 👍 / 👎.