Skip to content

Commit f30d8d8

Browse files
colesburycolesbury
committed
Add pyatomic.h
1 parent dbb23ad commit f30d8d8

8 files changed

Lines changed: 2431 additions & 3 deletions

File tree

‎Include/internal/pycore_atomic.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
#ifndef Py_ATOMIC_H
2-
#define Py_ATOMIC_H
1+
#ifndef Py_INTERNAL_ATOMIC_H
2+
#define Py_INTERNAL_ATOMIC_H
33
#ifdef __cplusplus
44
extern "C" {
55
#endif
@@ -554,4 +554,4 @@ typedef struct _Py_atomic_int {
554554
#ifdef __cplusplus
555555
}
556556
#endif
557-
#endif /* Py_ATOMIC_H */
557+
#endif /* Py_INTERNAL_ATOMIC_H */

‎Include/object.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
#ifndef Py_OBJECT_H
22
#define Py_OBJECT_H
3+
4+
#include "pyatomic.h"
5+
36
#ifdef __cplusplus
47
extern "C" {
58
#endif

‎Include/pyatomic.h‎

Lines changed: 361 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,361 @@
1+
#ifndef Py_ATOMIC_H
2+
#define Py_ATOMIC_H
3+
4+
static inline int
5+
_Py_atomic_add_int(volatile int *address, int value);
6+
7+
static inline int32_t
8+
_Py_atomic_add_int32(volatile int32_t *address, int32_t value);
9+
10+
static inline int64_t
11+
_Py_atomic_add_int64(volatile int64_t *address, int64_t value);
12+
13+
static inline intptr_t
14+
_Py_atomic_add_intptr(volatile intptr_t *address, intptr_t value);
15+
16+
static inline uint8_t
17+
_Py_atomic_add_uint8(volatile uint8_t *address, uint8_t value);
18+
19+
static inline uint32_t
20+
_Py_atomic_add_uint32(volatile uint32_t *address, uint32_t value);
21+
22+
static inline uint64_t
23+
_Py_atomic_add_uint64(volatile uint64_t *address, uint64_t value);
24+
25+
static inline uintptr_t
26+
_Py_atomic_add_uintptr(volatile uintptr_t *address, uintptr_t value);
27+
28+
static inline Py_ssize_t
29+
_Py_atomic_add_ssize(volatile Py_ssize_t *address, Py_ssize_t value);
30+
31+
32+
static inline int
33+
_Py_atomic_compare_exchange_int(volatile int *address, int expected, int value);
34+
35+
static inline int
36+
_Py_atomic_compare_exchange_int32(volatile int32_t *address, int32_t expected, int32_t value);
37+
38+
static inline int
39+
_Py_atomic_compare_exchange_int64(volatile int64_t *address, int64_t expected, int64_t value);
40+
41+
static inline int
42+
_Py_atomic_compare_exchange_intptr(volatile intptr_t *address, intptr_t expected, intptr_t value);
43+
44+
static inline int
45+
_Py_atomic_compare_exchange_uint(volatile unsigned int *address, unsigned int expected, unsigned int value);
46+
47+
static inline int
48+
_Py_atomic_compare_exchange_uint8(volatile uint8_t *address, uint8_t expected, uint8_t value);
49+
50+
static inline int
51+
_Py_atomic_compare_exchange_uint32(volatile uint32_t *address, uint32_t expected, uint32_t value);
52+
53+
static inline int
54+
_Py_atomic_compare_exchange_uint64(volatile uint64_t *address, uint64_t expected, uint64_t value);
55+
56+
static inline int
57+
_Py_atomic_compare_exchange_uintptr(volatile uintptr_t *address, uintptr_t expected, uintptr_t value);
58+
59+
static inline int
60+
_Py_atomic_compare_exchange_ssize(volatile Py_ssize_t *address, Py_ssize_t expected, Py_ssize_t value);
61+
62+
static inline int
63+
_Py_atomic_compare_exchange_ptr(volatile void *address, void *expected, void *value);
64+
65+
66+
static inline int
67+
_Py_atomic_exchange_int(volatile int *address, int value);
68+
69+
static inline int32_t
70+
_Py_atomic_exchange_int32(volatile int32_t *address, int32_t value);
71+
72+
static inline int64_t
73+
_Py_atomic_exchange_int64(volatile int64_t *address, int64_t value);
74+
75+
static inline intptr_t
76+
_Py_atomic_exchange_intptr(volatile intptr_t *address, intptr_t value);
77+
78+
static inline uint8_t
79+
_Py_atomic_exchange_uint8(volatile uint8_t *address, uint8_t value);
80+
81+
static inline uint32_t
82+
_Py_atomic_exchange_uint32(volatile uint32_t *address, uint32_t value);
83+
84+
static inline uint64_t
85+
_Py_atomic_exchange_uint64(volatile uint64_t *address, uint64_t value);
86+
87+
static inline uintptr_t
88+
_Py_atomic_exchange_uintptr(volatile uintptr_t *address, uintptr_t value);
89+
90+
static inline void *
91+
_Py_atomic_exchange_ptr(volatile void *address, void *value);
92+
93+
94+
static inline uint8_t
95+
_Py_atomic_and_uint8(volatile uint8_t *address, uint8_t value);
96+
97+
static inline uint32_t
98+
_Py_atomic_and_uint32(volatile uint32_t *address, uint32_t value);
99+
100+
static inline uint64_t
101+
_Py_atomic_and_uint64(volatile uint64_t *address, uint64_t value);
102+
103+
static inline uintptr_t
104+
_Py_atomic_and_uintptr(volatile uintptr_t *address, uintptr_t value);
105+
106+
107+
static inline uint8_t
108+
_Py_atomic_or_uint8(volatile uint8_t *address, uint8_t value);
109+
110+
static inline uint32_t
111+
_Py_atomic_or_uint32(volatile uint32_t *address, uint32_t value);
112+
113+
static inline uint64_t
114+
_Py_atomic_or_uint64(volatile uint64_t *address, uint64_t value);
115+
116+
static inline uintptr_t
117+
_Py_atomic_or_uintptr(volatile uintptr_t *address, uintptr_t value);
118+
119+
120+
static inline int
121+
_Py_atomic_load_int(const volatile int *address);
122+
123+
static inline int32_t
124+
_Py_atomic_load_int32(const volatile int32_t *address);
125+
126+
static inline int64_t
127+
_Py_atomic_load_int64(const volatile int64_t *address);
128+
129+
static inline intptr_t
130+
_Py_atomic_load_intptr(const volatile intptr_t *address);
131+
132+
static inline uint8_t
133+
_Py_atomic_load_uint8(const volatile uint8_t *address);
134+
135+
static inline uint16_t
136+
_Py_atomic_load_uint16(const volatile uint16_t *address);
137+
138+
static inline uint32_t
139+
_Py_atomic_load_uint32(const volatile uint32_t *address);
140+
141+
static inline uint64_t
142+
_Py_atomic_load_uint64(const volatile uint64_t *address);
143+
144+
static inline uintptr_t
145+
_Py_atomic_load_uintptr(const volatile uintptr_t *address);
146+
147+
static inline unsigned int
148+
_Py_atomic_load_uint(const volatile unsigned int *address);
149+
150+
static inline unsigned long
151+
_Py_atomic_load_ulong(const volatile unsigned long *address);
152+
153+
static inline Py_ssize_t
154+
_Py_atomic_load_ssize(const volatile Py_ssize_t *address);
155+
156+
static inline void *
157+
_Py_atomic_load_ptr(const volatile void *address);
158+
159+
160+
static inline int
161+
_Py_atomic_load_int_relaxed(const volatile int *address);
162+
163+
static inline int8_t
164+
_Py_atomic_load_int8_relaxed(const volatile int8_t *address);
165+
166+
static inline int16_t
167+
_Py_atomic_load_int16_relaxed(const volatile int16_t *address);
168+
169+
static inline int32_t
170+
_Py_atomic_load_int32_relaxed(const volatile int32_t *address);
171+
172+
static inline int64_t
173+
_Py_atomic_load_int64_relaxed(const volatile int64_t *address);
174+
175+
static inline intptr_t
176+
_Py_atomic_load_intptr_relaxed(const volatile intptr_t *address);
177+
178+
static inline uint8_t
179+
_Py_atomic_load_uint8_relaxed(const volatile uint8_t *address);
180+
181+
static inline uint16_t
182+
_Py_atomic_load_uint16_relaxed(const volatile uint16_t *address);
183+
184+
static inline uint32_t
185+
_Py_atomic_load_uint32_relaxed(const volatile uint32_t *address);
186+
187+
static inline uint64_t
188+
_Py_atomic_load_uint64_relaxed(const volatile uint64_t *address);
189+
190+
static inline uintptr_t
191+
_Py_atomic_load_uintptr_relaxed(const volatile uintptr_t *address);
192+
193+
static inline unsigned int
194+
_Py_atomic_load_uint_relaxed(const volatile unsigned int *address);
195+
196+
static inline unsigned long
197+
_Py_atomic_load_ulong_relaxed(const volatile unsigned long *address);
198+
199+
static inline Py_ssize_t
200+
_Py_atomic_load_ssize_relaxed(const volatile Py_ssize_t *address);
201+
202+
static inline void *
203+
_Py_atomic_load_ptr_relaxed(const volatile void *address);
204+
205+
206+
static inline void
207+
_Py_atomic_store_int(volatile int *address, int value);
208+
209+
static inline void
210+
_Py_atomic_store_int32(volatile int32_t *address, int32_t value);
211+
212+
static inline void
213+
_Py_atomic_store_int64(volatile int64_t *address, int64_t value);
214+
215+
static inline void
216+
_Py_atomic_store_intptr(volatile intptr_t *address, intptr_t value);
217+
218+
static inline void
219+
_Py_atomic_store_uint8(volatile uint8_t *address, uint8_t value);
220+
221+
static inline void
222+
_Py_atomic_store_uint16(volatile uint16_t *address, uint16_t value);
223+
224+
static inline void
225+
_Py_atomic_store_uint32(volatile uint32_t *address, uint32_t value);
226+
227+
static inline void
228+
_Py_atomic_store_uint64(volatile uint64_t *address, uint64_t value);
229+
230+
static inline void
231+
_Py_atomic_store_uintptr(volatile uintptr_t *address, uintptr_t value);
232+
233+
static inline void
234+
_Py_atomic_store_uint(volatile unsigned int *address, unsigned int value);
235+
236+
static inline void
237+
_Py_atomic_store_ulong(volatile unsigned long *address, unsigned long value);
238+
239+
static inline void
240+
_Py_atomic_store_ptr(volatile void *address, void *value);
241+
242+
static inline void
243+
_Py_atomic_store_ssize(volatile Py_ssize_t* address, Py_ssize_t value);
244+
245+
246+
static inline void
247+
_Py_atomic_store_int_relaxed(volatile int *address, int value);
248+
249+
static inline void
250+
_Py_atomic_store_int8_relaxed(volatile int8_t *address, int8_t value);
251+
252+
static inline void
253+
_Py_atomic_store_int16_relaxed(volatile int16_t *address, int16_t value);
254+
255+
static inline void
256+
_Py_atomic_store_int32_relaxed(volatile int32_t *address, int32_t value);
257+
258+
static inline void
259+
_Py_atomic_store_int64_relaxed(volatile int64_t *address, int64_t value);
260+
261+
static inline void
262+
_Py_atomic_store_intptr_relaxed(volatile intptr_t *address, intptr_t value);
263+
264+
static inline void
265+
_Py_atomic_store_uint8_relaxed(volatile uint8_t* address, uint8_t value);
266+
267+
static inline void
268+
_Py_atomic_store_uint16_relaxed(volatile uint16_t *address, uint16_t value);
269+
270+
static inline void
271+
_Py_atomic_store_uint32_relaxed(volatile uint32_t *address, uint32_t value);
272+
273+
static inline void
274+
_Py_atomic_store_uint64_relaxed(volatile uint64_t *address, uint64_t value);
275+
276+
static inline void
277+
_Py_atomic_store_uintptr_relaxed(volatile uintptr_t *address, uintptr_t value);
278+
279+
static inline void
280+
_Py_atomic_store_uint_relaxed(volatile unsigned int *address, unsigned int value);
281+
282+
static inline void
283+
_Py_atomic_store_ulong_relaxed(volatile unsigned long *address, unsigned long value);
284+
285+
static inline void
286+
_Py_atomic_store_ptr_relaxed(volatile void *address, void *value);
287+
288+
static inline void
289+
_Py_atomic_store_ssize_relaxed(volatile Py_ssize_t *address, Py_ssize_t value);
290+
291+
292+
static inline void
293+
_Py_atomic_store_uint64_release(volatile uint64_t *address, uint64_t value);
294+
295+
static inline void
296+
_Py_atomic_store_ptr_release(volatile void *address, void *value);
297+
298+
299+
static inline void
300+
_Py_atomic_fence_seq_cst(void);
301+
302+
static inline void
303+
_Py_atomic_fence_release(void);
304+
305+
static inline int
306+
_Py_atomic_uintptr_is_zero(uintptr_t *address)
307+
{
308+
#if defined(__GNUC__) && defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(__x86_64__)
309+
int out;
310+
__asm__ (
311+
"cmpq\t$0, %[address]"
312+
: "=@ccz" (out)
313+
: [address] "m" (*address)
314+
: );
315+
return out;
316+
#else
317+
return _Py_atomic_load_uintptr_relaxed(address) == 0;
318+
#endif
319+
}
320+
321+
static inline int
322+
_Py_atomic_compare_uintptr_relaxed(uintptr_t *address, uintptr_t value)
323+
{
324+
// GCC and clang generate an unecessary `mov` instead of using
325+
// `cmp` with a memory operand. The inline assembly avoids this.
326+
#if defined(__GNUC__) && defined(__GCC_ASM_FLAG_OUTPUTS__) && defined(__x86_64__)
327+
int out;
328+
__asm__ (
329+
"cmp %[value], %[address]"
330+
: "=@ccz" (out)
331+
: [value] "r"(value), [address] "m" (*address)
332+
: );
333+
return out;
334+
#else
335+
return _Py_atomic_load_uintptr_relaxed(address) == value;
336+
#endif
337+
}
338+
339+
#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))
340+
#define _Py_HAVE_GCC_BUILTIN_ATOMICS 1
341+
#elif defined(__clang__)
342+
#if __has_builtin(__atomic_load)
343+
#define _Py_HAVE_GCC_BUILTIN_ATOMICS 1
344+
#endif
345+
#endif
346+
347+
#if defined(_Py_HAVE_GCC_BUILTIN_ATOMICS)
348+
#define Py_ATOMIC_GCC_H
349+
#include "pyatomic_gcc.h"
350+
#elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
351+
#define Py_ATOMIC_STD_H
352+
#include "pyatomic_std.h"
353+
#elif defined(_MSC_VER)
354+
#define Py_ATOMIC_MSC_H
355+
#include "pyatomic_msc.h"
356+
#else
357+
#error "define pyatomic for this platform"
358+
#endif
359+
360+
#endif /* Py_ATOMIC_H */
361+

0 commit comments

Comments
 (0)