Skip to content

Commit 3e0ec59

Browse files
committed
only allow one thread at a time to use ZstdCompressor
1 parent ddc5c7c commit 3e0ec59

3 files changed

Lines changed: 109 additions & 0 deletions

File tree

‎c-ext/_pyzstd_atomics.h‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
* Provides wrappers around C11 standard library atomics and MSVC intrinsics
3+
* to provide basic atomic load and store functionality. This is based on
4+
* code in CPython's pyatomic.h, pyatomic_std.h, and pyatomic_msc.h
5+
*
6+
* Adapted from:
7+
* - numpy/_core/src/common/npy_atomic.h
8+
* - cpython/Include/cpython/pyatomic.h
9+
*/
10+
11+
#ifndef PYZSTD_ATOMICS_H
12+
#define PYZSTD_ATOMICS_H
13+
14+
#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \
15+
&& !defined(__STDC_NO_ATOMICS__)
16+
// TODO: support C++ atomics as well if this header is ever needed in C++
17+
#include <stdatomic.h>
18+
#include <stdint.h>
19+
#define STDC_ATOMICS
20+
#elif _MSC_VER
21+
#include <intrin.h>
22+
#define MSC_ATOMICS
23+
#if !defined(_M_X64) && !defined(_M_IX86) && !defined(_M_ARM64)
24+
#error "Unsupported MSVC build configuration, neither x86 or ARM"
25+
#endif
26+
#elif defined(__GNUC__) && (__GNUC__ > 4)
27+
#define GCC_ATOMICS
28+
#elif defined(__clang__)
29+
#if __has_builtin(__atomic_load)
30+
#define GCC_ATOMICS
31+
#endif
32+
#else
33+
#error "no supported atomic implementation for this platform/compiler"
34+
#endif
35+
36+
37+
static inline int8_t
38+
pyzstd_atomic_load_int8(const int8_t *obj) {
39+
#ifdef STDC_ATOMICS
40+
return (int8_t)atomic_load((const _Atomic(int8_t)*)obj);
41+
#elif defined(MSC_ATOMICS)
42+
#if defined(_M_X64) || defined(_M_IX86)
43+
return *(volatile int8_t *)obj;
44+
#else // defined(_M_ARM64)
45+
return (int8_t)__ldar8((unsigned __int8 volatile *)obj);
46+
#endif
47+
#elif defined(GCC_ATOMICS)
48+
return __atomic_load_n(obj, __ATOMIC_SEQ_CST);
49+
#endif
50+
}
51+
52+
static inline void
53+
pyzstd_atomic_store_int8(int8_t *obj, int8_t value) {
54+
#ifdef STDC_ATOMICS
55+
atomic_store((_Atomic(int8_t)*)obj, value);
56+
#elif defined(MSC_ATOMICS)
57+
_InterlockedExchange8((volatile char *)obj, (char)value);
58+
#elif defined(GCC_ATOMICS)
59+
__atomic_store_n(obj, value, __ATOMIC_SEQ_CST);
60+
#endif
61+
}
62+
63+
static inline int
64+
pyzstd_atomic_compare_exchange_int8(int8_t *obj, int8_t expected, int8_t desired) {
65+
#ifdef STDC_ATOMICS
66+
return atomic_compare_exchange_strong((_Atomic(int8_t)*)obj,
67+
&expected, desired);
68+
#elif defined(MSC_ATOMICS)
69+
int8_t initial = (int8_t)_InterlockedCompareExchange8(
70+
(volatile char *)obj,
71+
(char)value,
72+
(char)expected);
73+
if (initial == *expected) {
74+
return 1;
75+
}
76+
*expected = initial;
77+
return 0;
78+
#elif defined(GCC_ATOMICS)
79+
return __atomic_compare_exchange_n(obj, &expected, desired, 0,
80+
__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
81+
#endif
82+
}
83+
84+
#undef MSC_ATOMICS
85+
#undef STDC_ATOMICS
86+
#undef GCC_ATOMICS
87+
88+
#endif // PYZSTD_ATOMICS_H

‎c-ext/compressor.c‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
*/
88

99
#include "python-zstandard.h"
10+
#include "_pyzstd_atomics.h"
1011

1112
extern PyObject *ZstdError;
1213

@@ -238,6 +239,8 @@ static int ZstdCompressor_init(ZstdCompressor *self, PyObject *args,
238239
Py_INCREF(dict);
239240
}
240241

242+
self->in_use = 0;
243+
241244
if (setup_cctx(self)) {
242245
return -1;
243246
}
@@ -522,6 +525,14 @@ static PyObject *ZstdCompressor_compress(ZstdCompressor *self, PyObject *args,
522525
return NULL;
523526
}
524527

528+
if (pyzstd_atomic_load_int8(&self->in_use)) {
529+
goto concurrent_use;
530+
}
531+
532+
if (!pyzstd_atomic_compare_exchange_int8(&self->in_use, 0, 1)) {
533+
goto concurrent_use;
534+
}
535+
525536
ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
526537

527538
destSize = ZSTD_compressBound(source.len);
@@ -569,8 +580,16 @@ static PyObject *ZstdCompressor_compress(ZstdCompressor *self, PyObject *args,
569580
Py_SET_SIZE(output, outBuffer.pos);
570581

571582
finally:
583+
pyzstd_atomic_store_int8(&self->in_use, 0);
572584
PyBuffer_Release(&source);
573585
return output;
586+
587+
concurrent_use:
588+
PyErr_SetString(ZstdError, "concurrent use is not allowed. "
589+
"See https://python-zstandard.readthedocs.io"
590+
"/en/latest/api_usage.html"
591+
"#thread-and-object-reuse-safety");
592+
return NULL;
574593
}
575594

576595
static ZstdCompressionObj *ZstdCompressor_compressobj(ZstdCompressor *self,

‎c-ext/python-zstandard.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,8 @@ typedef struct {
104104
ZSTD_CCtx *cctx;
105105
/* Compression parameters in use. */
106106
ZSTD_CCtx_params *params;
107+
/* Is this compressor being used by a thread? */
108+
int8_t in_use;
107109
} ZstdCompressor;
108110

109111
extern PyTypeObject *ZstdCompressorType;

0 commit comments

Comments
 (0)