From d63eccb09d7b2b1e0538b1d4aff3d9cadb5df648 Mon Sep 17 00:00:00 2001 From: dpdani Date: Wed, 21 Aug 2024 18:08:09 +0200 Subject: [PATCH 01/31] gh-117657: TSAN Fix races in `PyMember_Get` and `PyMember_Set`, for C extensions --- Lib/test/libregrtest/tsan.py | 1 + Lib/test/test_free_threading/test_slots.py | 228 +++++++++++++++++++++ 2 files changed, 229 insertions(+) diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index dd18ae2584f5d8..7f36449d9060d7 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -25,6 +25,7 @@ 'test_threading_local', 'test_threadsignals', 'test_weakref', + 'test_free_threading.test_slots', ] diff --git a/Lib/test/test_free_threading/test_slots.py b/Lib/test/test_free_threading/test_slots.py index 758f74f54d0b56..710e74665aeb44 100644 --- a/Lib/test/test_free_threading/test_slots.py +++ b/Lib/test/test_free_threading/test_slots.py @@ -1,3 +1,4 @@ +import _testcapi import threading from test.support import threading_helper from unittest import TestCase @@ -41,3 +42,230 @@ def reader(): assert 0 <= eggs <= iters run_in_threads([writer, reader, reader, reader]) + + def test_T_BOOL(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + # different code paths for True and False + spam_old.T_BOOL = True + spam_new.T_BOOL = True + spam_old.T_BOOL = False + spam_new.T_BOOL = False + + def reader(): + for _ in range(1_000): + spam_old.T_BOOL + spam_new.T_BOOL + + run_in_threads([writer, reader]) + + def test_T_BYTE(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_BYTE = 0 + spam_new.T_BYTE = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_BYTE + spam_new.T_BYTE + + run_in_threads([writer, reader]) + + def test_T_UBYTE(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_UBYTE = 0 + spam_new.T_UBYTE = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_UBYTE + spam_new.T_UBYTE + + run_in_threads([writer, reader]) + + def test_T_SHORT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_SHORT = 0 + spam_new.T_SHORT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_SHORT + spam_new.T_SHORT + + run_in_threads([writer, reader]) + + def test_T_USHORT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_USHORT = 0 + spam_new.T_USHORT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_USHORT + spam_new.T_USHORT + + run_in_threads([writer, reader]) + + def test_T_INT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_INT = 0 + spam_new.T_INT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_INT + spam_new.T_INT + + run_in_threads([writer, reader]) + + def test_T_UINT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_UINT = 0 + spam_new.T_UINT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_UINT + spam_new.T_UINT + + run_in_threads([writer, reader]) + + def test_T_LONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_LONG = 0 + spam_new.T_LONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_LONG + spam_new.T_LONG + + run_in_threads([writer, reader]) + + def test_T_ULONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_ULONG = 0 + spam_new.T_ULONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_ULONG + spam_new.T_ULONG + + run_in_threads([writer, reader]) + + def test_T_PYSSIZET(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_PYSSIZET = 0 + spam_new.T_PYSSIZET = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_PYSSIZET + spam_new.T_PYSSIZET + + run_in_threads([writer, reader]) + + def test_T_FLOAT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_FLOAT = 0.0 + spam_new.T_FLOAT = 0.0 + + def reader(): + for _ in range(1_000): + spam_old.T_FLOAT + spam_new.T_FLOAT + + run_in_threads([writer, reader]) + + def test_T_DOUBLE(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_DOUBLE = 0.0 + spam_new.T_DOUBLE = 0.0 + + def reader(): + for _ in range(1_000): + spam_old.T_DOUBLE + spam_new.T_DOUBLE + + run_in_threads([writer, reader]) + + def test_T_LONGLONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_LONGLONG = 0 + spam_new.T_LONGLONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_LONGLONG + spam_new.T_LONGLONG + + run_in_threads([writer, reader]) + + def test_T_ULONGLONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_ULONGLONG = 0 + spam_new.T_ULONGLONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_ULONGLONG + spam_new.T_ULONGLONG + + run_in_threads([writer, reader]) From c2d3d411d5d06b839db18255bcd26879aa4ac78f Mon Sep 17 00:00:00 2001 From: dpdani Date: Wed, 21 Aug 2024 22:56:02 +0200 Subject: [PATCH 02/31] exercise race in T_CHAR as well --- Lib/test/test_free_threading/test_slots.py | 16 ++++++++++++++++ Modules/_testcapi/structmember.c | 8 ++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_free_threading/test_slots.py b/Lib/test/test_free_threading/test_slots.py index 710e74665aeb44..a3b9f4b0175ae7 100644 --- a/Lib/test/test_free_threading/test_slots.py +++ b/Lib/test/test_free_threading/test_slots.py @@ -269,3 +269,19 @@ def reader(): spam_new.T_ULONGLONG run_in_threads([writer, reader]) + + def test_T_CHAR(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_CHAR = "c" + spam_new.T_CHAR = "c" + + def reader(): + for _ in range(1_000): + spam_old.T_CHAR + spam_new.T_CHAR + + run_in_threads([writer, reader]) diff --git a/Modules/_testcapi/structmember.c b/Modules/_testcapi/structmember.c index 096eaecd40855f..f136a25e316d46 100644 --- a/Modules/_testcapi/structmember.c +++ b/Modules/_testcapi/structmember.c @@ -22,6 +22,7 @@ typedef struct { char inplace_member[6]; long long longlong_member; unsigned long long ulonglong_member; + char char_member; } all_structmembers; typedef struct { @@ -46,6 +47,7 @@ static struct PyMemberDef test_members_newapi[] = { {"T_STRING_INPLACE", Py_T_STRING_INPLACE, offsetof(test_structmembers, structmembers.inplace_member), 0, NULL}, {"T_LONGLONG", Py_T_LONGLONG, offsetof(test_structmembers, structmembers.longlong_member), 0, NULL}, {"T_ULONGLONG", Py_T_ULONGLONG, offsetof(test_structmembers, structmembers.ulonglong_member), 0, NULL}, + {"T_CHAR", Py_T_CHAR, offsetof(test_structmembers, structmembers.char_member), 0, NULL}, {NULL} }; @@ -56,7 +58,7 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) "T_BOOL", "T_BYTE", "T_UBYTE", "T_SHORT", "T_USHORT", "T_INT", "T_UINT", "T_LONG", "T_ULONG", "T_PYSSIZET", "T_FLOAT", "T_DOUBLE", "T_STRING_INPLACE", - "T_LONGLONG", "T_ULONGLONG", + "T_LONGLONG", "T_ULONGLONG", "T_CHAR", NULL}; static const char fmt[] = "|bbBhHiIlknfds#LK"; test_structmembers *ob; @@ -82,7 +84,8 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) &ob->structmembers.double_member, &s, &string_len, &ob->structmembers.longlong_member, - &ob->structmembers.ulonglong_member)) + &ob->structmembers.ulonglong_member, + &ob->structmembers.char_member)) { Py_DECREF(ob); return NULL; @@ -132,6 +135,7 @@ static struct PyMemberDef test_members[] = { {"T_STRING_INPLACE", T_STRING_INPLACE, offsetof(test_structmembers, structmembers.inplace_member), 0, NULL}, {"T_LONGLONG", T_LONGLONG, offsetof(test_structmembers, structmembers.longlong_member), 0, NULL}, {"T_ULONGLONG", T_ULONGLONG, offsetof(test_structmembers, structmembers.ulonglong_member), 0, NULL}, + {"T_CHAR", Py_T_CHAR, offsetof(test_structmembers, structmembers.char_member), 0, NULL}, {NULL} }; From f6ba4a33c60add7ed6667f07d950a5aa419daef6 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 15:13:14 +0200 Subject: [PATCH 03/31] signed char races --- Include/cpython/pyatomic.h | 6 ++++++ Include/cpython/pyatomic_gcc.h | 8 ++++++++ Include/cpython/pyatomic_msc.h | 19 +++++++++++++++++++ Include/cpython/pyatomic_std.h | 16 ++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 7 ++++++- Python/structmember.c | 10 +++++----- 6 files changed, 60 insertions(+), 6 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 4ecef4f56edf42..b70d108b32c9eb 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -321,6 +321,9 @@ _Py_atomic_load_ptr(const void *obj); static inline int _Py_atomic_load_int_relaxed(const int *obj); +static inline char +_Py_atomic_load_char_relaxed(const char *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -481,6 +484,9 @@ _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); static inline void _Py_atomic_store_int_release(int *obj, int value); +static inline void +_Py_atomic_store_char_release(char *obj, char value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index ef09954d53ac1d..97a10ba5969464 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -306,6 +306,10 @@ static inline int _Py_atomic_load_int_relaxed(const int *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline char +_Py_atomic_load_char_relaxed(const char *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -508,6 +512,10 @@ static inline void _Py_atomic_store_int_release(int *obj, int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_char_release(char *obj, char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 84da21bdcbff4f..d35ec380ea2a56 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -634,6 +634,12 @@ _Py_atomic_load_int_relaxed(const int *obj) return *(volatile int *)obj; } +static inline int +_Py_atomic_load_char_relaxed(const char *obj) +{ + return *(volatile char *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -964,6 +970,19 @@ _Py_atomic_store_int_release(int *obj, int value) #endif } +static inline void +_Py_atomic_store_char_release(char *obj, char value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(char volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int8); + __stlr8((unsigned __int8 volatile *)obj, (unsigned __int8)value); +#else +# error "no implementation of _Py_atomic_store_char_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 7c71e94c68f8e6..e5a76e1fcedc4f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -515,6 +515,14 @@ _Py_atomic_load_int_relaxed(const int *obj) memory_order_relaxed); } +static inline char +_Py_atomic_load_char_relaxed(const char *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(char)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -895,6 +903,14 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } +static inline void +_Py_atomic_store_char_release(char *obj, char value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(char)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index a1bb383bcd22e9..f78392411b256c 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -61,6 +61,10 @@ extern "C" { _Py_atomic_store_uint16_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ _Py_atomic_store_uint32_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_CHAR_RELEASE(value, new_value) \ + _Py_atomic_store_char_release(&value, new_value) +#define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ + _Py_atomic_load_char_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -68,7 +72,6 @@ extern "C" { #define FT_ATOMIC_LOAD_SSIZE(value) value #define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value -#define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value #define FT_ATOMIC_LOAD_PTR_RELAXED(value) value @@ -85,6 +88,8 @@ extern "C" { #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value +#define FT_ATOMIC_STORE_CHAR_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index d5e7ab83093dc8..71dac80f8bcee2 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -34,10 +34,10 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) const char* addr = obj_addr + l->offset; switch (l->type) { case Py_T_BOOL: - v = PyBool_FromLong(*(char*)addr); + v = PyBool_FromLong(FT_ATOMIC_LOAD_CHAR_RELAXED(*(char*)addr)); break; case Py_T_BYTE: - v = PyLong_FromLong(*(char*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_CHAR_RELAXED(*(char*)addr)); break; case Py_T_UBYTE: v = PyLong_FromUnsignedLong(*(unsigned char*)addr); @@ -169,16 +169,16 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) return -1; } if (v == Py_True) - *(char*)addr = (char) 1; + FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, 1); else - *(char*)addr = (char) 0; + FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, 0); break; } case Py_T_BYTE:{ long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(char*)addr = (char)long_val; + FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, (char)long_val); /* XXX: For compatibility, only warn about truncations for now. */ if ((long_val > CHAR_MAX) || (long_val < CHAR_MIN)) From d9614b743743c1ebbd34f9b145dbdfb40b64d2c3 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 15:22:50 +0200 Subject: [PATCH 04/31] unsigned char race --- Include/cpython/pyatomic.h | 6 ++++++ Include/cpython/pyatomic_gcc.h | 8 ++++++++ Include/cpython/pyatomic_msc.h | 13 +++++++++++++ Include/cpython/pyatomic_std.h | 16 ++++++++++++++++ Include/internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 4 ++-- 6 files changed, 51 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index b70d108b32c9eb..60001c36c58036 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -324,6 +324,9 @@ _Py_atomic_load_int_relaxed(const int *obj); static inline char _Py_atomic_load_char_relaxed(const char *obj); +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -487,6 +490,9 @@ _Py_atomic_store_int_release(int *obj, int value); static inline void _Py_atomic_store_char_release(char *obj, char value); +static inline void +_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 97a10ba5969464..208d87e3df7db4 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -310,6 +310,10 @@ static inline char _Py_atomic_load_char_relaxed(const char *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -516,6 +520,10 @@ static inline void _Py_atomic_store_char_release(char *obj, char value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index d35ec380ea2a56..6ec8ae3546b4c3 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -983,6 +983,19 @@ _Py_atomic_store_char_release(char *obj, char value) #endif } +static inline void +_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(unsigned char volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int8); + __stlr8((unsigned __int8 volatile *)obj, (unsigned __int8)value); +#else +# error "no implementation of _Py_atomic_store_uchar_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index e5a76e1fcedc4f..988df67aa3e438 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -523,6 +523,14 @@ _Py_atomic_load_char_relaxed(const char *obj) memory_order_relaxed); } +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned char)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -911,6 +919,14 @@ _Py_atomic_store_char_release(char *obj, char value) memory_order_release); } +static inline void +_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned char)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index f78392411b256c..a96978767f2916 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -65,6 +65,10 @@ extern "C" { _Py_atomic_store_char_release(&value, new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ _Py_atomic_load_char_relaxed(&value) +#define FT_ATOMIC_STORE_UCHAR_RELEASE(value, new_value) \ + _Py_atomic_store_uchar_release(&value, new_value) +#define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) \ + _Py_atomic_load_uchar_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -90,6 +94,8 @@ extern "C" { #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value #define FT_ATOMIC_STORE_CHAR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value +#define FT_ATOMIC_STORE_UCHAR_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 71dac80f8bcee2..384516c1bd0baf 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -40,7 +40,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromLong(FT_ATOMIC_LOAD_CHAR_RELAXED(*(char*)addr)); break; case Py_T_UBYTE: - v = PyLong_FromUnsignedLong(*(unsigned char*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_UCHAR_RELAXED(*(unsigned char*)addr)); break; case Py_T_SHORT: v = PyLong_FromLong(*(short*)addr); @@ -189,7 +189,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(unsigned char*)addr = (unsigned char)long_val; + FT_ATOMIC_STORE_UCHAR_RELEASE(*(unsigned char*)addr, (unsigned char)long_val); if ((long_val > UCHAR_MAX) || (long_val < 0)) WARN("Truncation of value to unsigned char"); break; From 1659849c09f1b1efe1bb24dafe90fe6c7e3ae845 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 15:31:22 +0200 Subject: [PATCH 05/31] =?UTF-8?q?unsigned=20char=20race=20=F0=9F=91=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Include/cpython/pyatomic_msc.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 6ec8ae3546b4c3..fc24285abb0e07 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -640,6 +640,12 @@ _Py_atomic_load_char_relaxed(const char *obj) return *(volatile char *)obj; } +static inline int +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ + return *(volatile unsigned char *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { From 4cfa619f1dba51339e40da950dbfef37b07513eb Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 15:31:59 +0200 Subject: [PATCH 06/31] =?UTF-8?q?unsigned=20char=20race=20=F0=9F=91=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Include/cpython/pyatomic_msc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index fc24285abb0e07..7d578d3b7c7352 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -640,7 +640,7 @@ _Py_atomic_load_char_relaxed(const char *obj) return *(volatile char *)obj; } -static inline int +static inline unsigned char _Py_atomic_load_uchar_relaxed(const unsigned char *obj) { return *(volatile unsigned char *)obj; From 107a9f5d91f762455ac8f1f9ab90f92ed2d64f56 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 15:37:41 +0200 Subject: [PATCH 07/31] signed short race --- Include/cpython/pyatomic.h | 6 ++++++ Include/cpython/pyatomic_gcc.h | 8 ++++++++ Include/cpython/pyatomic_msc.h | 19 +++++++++++++++++++ Include/cpython/pyatomic_std.h | 16 ++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 4 ++-- 6 files changed, 57 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 60001c36c58036..0d1726d6e6b01c 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -327,6 +327,9 @@ _Py_atomic_load_char_relaxed(const char *obj); static inline unsigned char _Py_atomic_load_uchar_relaxed(const unsigned char *obj); +static inline short +_Py_atomic_load_short_relaxed(const short *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -493,6 +496,9 @@ _Py_atomic_store_char_release(char *obj, char value); static inline void _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value); +static inline void +_Py_atomic_store_short_release(short *obj, short value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 208d87e3df7db4..edec79032b318e 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -314,6 +314,10 @@ static inline unsigned char _Py_atomic_load_uchar_relaxed(const unsigned char *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -524,6 +528,10 @@ static inline void _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_short_release(short *obj, short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 7d578d3b7c7352..0e0b4336ca5c9f 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -646,6 +646,12 @@ _Py_atomic_load_uchar_relaxed(const unsigned char *obj) return *(volatile unsigned char *)obj; } +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ + return *(volatile short *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -1002,6 +1008,19 @@ _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) #endif } +static inline void +_Py_atomic_store_short_release(short *obj, short value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(short volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int16); + __stlr8((unsigned __int16 volatile *)obj, (unsigned __int16)value); +#else +# error "no implementation of _Py_atomic_store_short_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 988df67aa3e438..79275aded376b7 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -531,6 +531,14 @@ _Py_atomic_load_uchar_relaxed(const unsigned char *obj) memory_order_relaxed); } +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(short)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -927,6 +935,14 @@ _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) memory_order_release); } +static inline void +_Py_atomic_store_short_release(short *obj, short value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(short)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index a96978767f2916..fe7e48eaab8e80 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -69,6 +69,10 @@ extern "C" { _Py_atomic_store_uchar_release(&value, new_value) #define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) \ _Py_atomic_load_uchar_relaxed(&value) +#define FT_ATOMIC_STORE_SHORT_RELEASE(value, new_value) \ + _Py_atomic_store_short_release(&value, new_value) +#define FT_ATOMIC_LOAD_SHORT_RELAXED(value) \ + _Py_atomic_load_short_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -96,6 +100,8 @@ extern "C" { #define FT_ATOMIC_STORE_CHAR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value #define FT_ATOMIC_STORE_UCHAR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_SHORT_RELAXED(value) value +#define FT_ATOMIC_STORE_SHORT_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 384516c1bd0baf..ef3cb6aebe74b4 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -43,7 +43,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_UCHAR_RELAXED(*(unsigned char*)addr)); break; case Py_T_SHORT: - v = PyLong_FromLong(*(short*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_SHORT_RELAXED(*(short*)addr)); break; case Py_T_USHORT: v = PyLong_FromUnsignedLong(*(unsigned short*)addr); @@ -198,7 +198,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(short*)addr = (short)long_val; + FT_ATOMIC_STORE_SHORT_RELEASE(*(short*)addr, (short)long_val); if ((long_val > SHRT_MAX) || (long_val < SHRT_MIN)) WARN("Truncation of value to short"); break; From e20b9d158a63bded7663ebe3243920ef45d31c61 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 15:45:55 +0200 Subject: [PATCH 08/31] unsigned short race --- Include/cpython/pyatomic.h | 6 ++++++ Include/cpython/pyatomic_gcc.h | 8 ++++++++ Include/cpython/pyatomic_msc.h | 19 +++++++++++++++++++ Include/cpython/pyatomic_std.h | 16 ++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 4 ++-- 6 files changed, 57 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 0d1726d6e6b01c..530759555447b3 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -330,6 +330,9 @@ _Py_atomic_load_uchar_relaxed(const unsigned char *obj); static inline short _Py_atomic_load_short_relaxed(const short *obj); +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -499,6 +502,9 @@ _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value); static inline void _Py_atomic_store_short_release(short *obj, short value); +static inline void +_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index edec79032b318e..cbe66ed88bdcc4 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -318,6 +318,10 @@ static inline short _Py_atomic_load_short_relaxed(const short *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -532,6 +536,10 @@ static inline void _Py_atomic_store_short_release(short *obj, short value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 0e0b4336ca5c9f..70c61118722c94 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -652,6 +652,12 @@ _Py_atomic_load_short_relaxed(const short *obj) return *(volatile short *)obj; } +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ + return *(volatile unsigned short *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -1021,6 +1027,19 @@ _Py_atomic_store_short_release(short *obj, short value) #endif } +static inline void +_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(short volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int16); + __stlr8((unsigned __int16 volatile *)obj, (unsigned __int16)value); +#else +# error "no implementation of _Py_atomic_store_ushort_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 79275aded376b7..3a0a1917602bea 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -539,6 +539,14 @@ _Py_atomic_load_short_relaxed(const short *obj) memory_order_relaxed); } +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned short)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -943,6 +951,14 @@ _Py_atomic_store_short_release(short *obj, short value) memory_order_release); } +static inline void +_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned short)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index fe7e48eaab8e80..b79f97c9f75e00 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -73,6 +73,10 @@ extern "C" { _Py_atomic_store_short_release(&value, new_value) #define FT_ATOMIC_LOAD_SHORT_RELAXED(value) \ _Py_atomic_load_short_relaxed(&value) +#define FT_ATOMIC_STORE_USHORT_RELEASE(value, new_value) \ + _Py_atomic_store_ushort_release(&value, new_value) +#define FT_ATOMIC_LOAD_USHORT_RELAXED(value) \ + _Py_atomic_load_ushort_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -102,6 +106,8 @@ extern "C" { #define FT_ATOMIC_STORE_UCHAR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_SHORT_RELAXED(value) value #define FT_ATOMIC_STORE_SHORT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_USHORT_RELAXED(value) value +#define FT_ATOMIC_STORE_USHORT_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index ef3cb6aebe74b4..bd35889329d3e8 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -46,7 +46,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromLong(FT_ATOMIC_LOAD_SHORT_RELAXED(*(short*)addr)); break; case Py_T_USHORT: - v = PyLong_FromUnsignedLong(*(unsigned short*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_USHORT_RELAXED(*(unsigned short*)addr)); break; case Py_T_INT: v = PyLong_FromLong(*(int*)addr); @@ -207,7 +207,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(unsigned short*)addr = (unsigned short)long_val; + FT_ATOMIC_STORE_USHORT_RELEASE(*(unsigned short*)addr, (unsigned short)long_val); if ((long_val > USHRT_MAX) || (long_val < 0)) WARN("Truncation of value to unsigned short"); break; From 86a0945b9e18bc5c02bffb2d84013123e73a7c1a Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 15:53:56 +0200 Subject: [PATCH 09/31] signed int race --- Include/internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index b79f97c9f75e00..1bd8f8bd3da117 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -77,6 +77,10 @@ extern "C" { _Py_atomic_store_ushort_release(&value, new_value) #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) \ _Py_atomic_load_ushort_relaxed(&value) +#define FT_ATOMIC_STORE_INT_RELEASE(value, new_value) \ + _Py_atomic_store_int_release(&value, new_value) +#define FT_ATOMIC_LOAD_INT_RELAXED(value) \ + _Py_atomic_load_int_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -108,6 +112,8 @@ extern "C" { #define FT_ATOMIC_STORE_SHORT_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) value #define FT_ATOMIC_STORE_USHORT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_INT_RELAXED(value) value +#define FT_ATOMIC_STORE_INT_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index bd35889329d3e8..5e30325af45ccd 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -49,7 +49,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_USHORT_RELAXED(*(unsigned short*)addr)); break; case Py_T_INT: - v = PyLong_FromLong(*(int*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_INT_RELAXED(*(int*)addr)); break; case Py_T_UINT: v = PyLong_FromUnsignedLong(*(unsigned int*)addr); @@ -216,7 +216,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(int *)addr = (int)long_val; + FT_ATOMIC_STORE_INT_RELEASE(*(int *)addr, (int)long_val); if ((long_val > INT_MAX) || (long_val < INT_MIN)) WARN("Truncation of value to int"); break; From 1a07031cc679418810dfbb50f91e5e52084b59c3 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 16:01:22 +0200 Subject: [PATCH 10/31] who uses MSC anyways? --- Include/cpython/pyatomic_msc.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 70c61118722c94..e064baa3e48837 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1021,7 +1021,7 @@ _Py_atomic_store_short_release(short *obj, short value) *(short volatile *)obj = value; #elif defined(_M_ARM64) _Py_atomic_ASSERT_ARG_TYPE(unsigned __int16); - __stlr8((unsigned __int16 volatile *)obj, (unsigned __int16)value); + __stlr16((unsigned __int16 volatile *)obj, (unsigned __int16)value); #else # error "no implementation of _Py_atomic_store_short_release" #endif @@ -1034,7 +1034,7 @@ _Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) *(short volatile *)obj = value; #elif defined(_M_ARM64) _Py_atomic_ASSERT_ARG_TYPE(unsigned __int16); - __stlr8((unsigned __int16 volatile *)obj, (unsigned __int16)value); + __stlr16((unsigned __int16 volatile *)obj, (unsigned __int16)value); #else # error "no implementation of _Py_atomic_store_ushort_release" #endif From 16f61d8e88fbe2fd3403a5e8661249fb369198c1 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 16:47:38 +0200 Subject: [PATCH 11/31] unsigned int races --- Include/cpython/pyatomic.h | 3 +++ Include/cpython/pyatomic_gcc.h | 4 ++++ Include/cpython/pyatomic_msc.h | 13 +++++++++++++ Include/cpython/pyatomic_std.h | 8 ++++++++ Include/internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 6 +++--- 6 files changed, 37 insertions(+), 3 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 530759555447b3..081239dfbc403b 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -505,6 +505,9 @@ _Py_atomic_store_short_release(short *obj, short value); static inline void _Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value); +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index cbe66ed88bdcc4..ea43d00732f9e3 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -540,6 +540,10 @@ static inline void _Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index e064baa3e48837..85aba5e9ba8204 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1040,6 +1040,19 @@ _Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) #endif } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(short volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_uint_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 3a0a1917602bea..9621cc5b895a98 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -959,6 +959,14 @@ _Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) memory_order_release); } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 1bd8f8bd3da117..cae0ed107dff71 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -81,6 +81,10 @@ extern "C" { _Py_atomic_store_int_release(&value, new_value) #define FT_ATOMIC_LOAD_INT_RELAXED(value) \ _Py_atomic_load_int_relaxed(&value) +#define FT_ATOMIC_STORE_UINT_RELEASE(value, new_value) \ + _Py_atomic_store_uint_release(&value, new_value) +#define FT_ATOMIC_LOAD_UINT_RELAXED(value) \ + _Py_atomic_load_uint_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -114,6 +118,8 @@ extern "C" { #define FT_ATOMIC_STORE_USHORT_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_INT_RELAXED(value) value #define FT_ATOMIC_STORE_INT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UINT_RELAXED(value) value +#define FT_ATOMIC_STORE_UINT_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 5e30325af45ccd..68d8dbdeaada8e 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -52,7 +52,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromLong(FT_ATOMIC_LOAD_INT_RELAXED(*(int*)addr)); break; case Py_T_UINT: - v = PyLong_FromUnsignedLong(*(unsigned int*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_UINT_RELAXED(*(unsigned int*)addr)); break; case Py_T_LONG: v = PyLong_FromLong(*(long*)addr); @@ -234,7 +234,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - *(unsigned int *)addr = (unsigned int)(unsigned long)long_val; + FT_ATOMIC_STORE_UINT_RELEASE(*(unsigned int *)addr, (unsigned int)(unsigned long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -243,7 +243,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; } - *(unsigned int*)addr = (unsigned int)ulong_val; + FT_ATOMIC_STORE_UINT_RELEASE(*(unsigned int *)addr, (unsigned int)ulong_val); if (ulong_val > UINT_MAX) { WARN("Truncation of value to unsigned int"); } From cf25726af36048a7ef91826bd938c695328ffb48 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 16:54:34 +0200 Subject: [PATCH 12/31] =?UTF-8?q?=F0=9F=91=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Include/cpython/pyatomic_msc.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 85aba5e9ba8204..047e4be793adc9 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1031,7 +1031,7 @@ static inline void _Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) { #if defined(_M_X64) || defined(_M_IX86) - *(short volatile *)obj = value; + *(unsigned short volatile *)obj = value; #elif defined(_M_ARM64) _Py_atomic_ASSERT_ARG_TYPE(unsigned __int16); __stlr16((unsigned __int16 volatile *)obj, (unsigned __int16)value); @@ -1044,7 +1044,7 @@ static inline void _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) { #if defined(_M_X64) || defined(_M_IX86) - *(short volatile *)obj = value; + *(unsigned int volatile *)obj = value; #elif defined(_M_ARM64) _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); From d41132c879522d37d6564e3d3b44671b3cde4507 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 16:59:38 +0200 Subject: [PATCH 13/31] signed long race --- Include/cpython/pyatomic.h | 6 ++++++ Include/cpython/pyatomic_gcc.h | 8 ++++++++ Include/cpython/pyatomic_msc.h | 19 +++++++++++++++++++ Include/cpython/pyatomic_std.h | 16 ++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 4 ++-- 6 files changed, 57 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 081239dfbc403b..ccf08fcee66944 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -333,6 +333,9 @@ _Py_atomic_load_short_relaxed(const short *obj); static inline unsigned short _Py_atomic_load_ushort_relaxed(const unsigned short *obj); +static inline long +_Py_atomic_load_long_relaxed(const long *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -508,6 +511,9 @@ _Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value); static inline void _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value); +static inline void +_Py_atomic_store_long_release(long *obj, long value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index ea43d00732f9e3..2602f9feeabf20 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -322,6 +322,10 @@ static inline unsigned short _Py_atomic_load_ushort_relaxed(const unsigned short *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -544,6 +548,10 @@ static inline void _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_long_release(long *obj, long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 047e4be793adc9..5453faf3cdaba4 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -658,6 +658,12 @@ _Py_atomic_load_ushort_relaxed(const unsigned short *obj) return *(volatile unsigned short *)obj; } +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ + return *(volatile long *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -1053,6 +1059,19 @@ _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) #endif } +static inline void +_Py_atomic_store_long_release(long *obj, long value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(long volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_long_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 9621cc5b895a98..24e61688bd1b73 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -547,6 +547,14 @@ _Py_atomic_load_ushort_relaxed(const unsigned short *obj) memory_order_relaxed); } +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(long)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -967,6 +975,14 @@ _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) memory_order_release); } +static inline void +_Py_atomic_store_long_release(long *obj, long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(long)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index cae0ed107dff71..496063d99fb476 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -85,6 +85,10 @@ extern "C" { _Py_atomic_store_uint_release(&value, new_value) #define FT_ATOMIC_LOAD_UINT_RELAXED(value) \ _Py_atomic_load_uint_relaxed(&value) +#define FT_ATOMIC_STORE_LONG_RELEASE(value, new_value) \ + _Py_atomic_store_long_release(&value, new_value) +#define FT_ATOMIC_LOAD_LONG_RELAXED(value) \ + _Py_atomic_load_long_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -120,6 +124,8 @@ extern "C" { #define FT_ATOMIC_STORE_INT_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UINT_RELAXED(value) value #define FT_ATOMIC_STORE_UINT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_LONG_RELAXED(value) value +#define FT_ATOMIC_STORE_LONG_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 68d8dbdeaada8e..2d67d59885dbd5 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -55,7 +55,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_UINT_RELAXED(*(unsigned int*)addr)); break; case Py_T_LONG: - v = PyLong_FromLong(*(long*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_LONG_RELAXED(*(long*)addr)); break; case Py_T_ULONG: v = PyLong_FromUnsignedLong(*(unsigned long*)addr); @@ -251,7 +251,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) break; } case Py_T_LONG:{ - *(long*)addr = PyLong_AsLong(v); + FT_ATOMIC_STORE_LONG_RELEASE(*(long*)addr, PyLong_AsLong(v)); if ((*(long*)addr == -1) && PyErr_Occurred()) return -1; break; From 22126058eed48b6d88e402f1270fce276887052d Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 17:08:30 +0200 Subject: [PATCH 14/31] unsigned long race --- Include/internal/pycore_pyatomic_ft_wrappers.h | 3 +++ Python/structmember.c | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 496063d99fb476..204ce320d3fc10 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -89,6 +89,8 @@ extern "C" { _Py_atomic_store_long_release(&value, new_value) #define FT_ATOMIC_LOAD_LONG_RELAXED(value) \ _Py_atomic_load_long_relaxed(&value) +#define FT_ATOMIC_STORE_ULONG(value, new_value) \ + _Py_atomic_store_ulong(&value, new_value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -126,6 +128,7 @@ extern "C" { #define FT_ATOMIC_STORE_UINT_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_LONG_RELAXED(value) value #define FT_ATOMIC_STORE_LONG_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_ULONG(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 2d67d59885dbd5..827e48596dc3de 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -58,7 +58,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromLong(FT_ATOMIC_LOAD_LONG_RELAXED(*(long*)addr)); break; case Py_T_ULONG: - v = PyLong_FromUnsignedLong(*(unsigned long*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_ULONG_RELAXED(*(unsigned long*)addr)); break; case Py_T_PYSSIZET: v = PyLong_FromSsize_t(*(Py_ssize_t*)addr); @@ -269,7 +269,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - *(unsigned long *)addr = (unsigned long)long_val; + FT_ATOMIC_STORE_ULONG(*(unsigned long *)addr, (unsigned long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -278,7 +278,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; } - *(unsigned long*)addr = ulong_val; + FT_ATOMIC_STORE_ULONG(*(unsigned long *)addr, ulong_val); } break; } From 74616d648d62f257d6a858f19080e8104ed85630 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 17:13:40 +0200 Subject: [PATCH 15/31] ssizet race --- Include/internal/pycore_pyatomic_ft_wrappers.h | 3 +++ Python/structmember.c | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 204ce320d3fc10..3244f9e673cfff 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -91,6 +91,8 @@ extern "C" { _Py_atomic_load_long_relaxed(&value) #define FT_ATOMIC_STORE_ULONG(value, new_value) \ _Py_atomic_store_ulong(&value, new_value) +#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) \ + _Py_atomic_store_ssize_release(&value, new_value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -129,6 +131,7 @@ extern "C" { #define FT_ATOMIC_LOAD_LONG_RELAXED(value) value #define FT_ATOMIC_STORE_LONG_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_ULONG(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 827e48596dc3de..565e63f571bcd0 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -61,7 +61,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_ULONG_RELAXED(*(unsigned long*)addr)); break; case Py_T_PYSSIZET: - v = PyLong_FromSsize_t(*(Py_ssize_t*)addr); + v = PyLong_FromSsize_t(FT_ATOMIC_LOAD_SSIZE_RELAXED(*(Py_ssize_t*)addr)); break; case Py_T_FLOAT: v = PyFloat_FromDouble((double)*(float*)addr); @@ -283,7 +283,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) break; } case Py_T_PYSSIZET:{ - *(Py_ssize_t*)addr = PyLong_AsSsize_t(v); + FT_ATOMIC_STORE_SSIZE_RELEASE(*(Py_ssize_t*)addr, PyLong_AsSsize_t(v)); if ((*(Py_ssize_t*)addr == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; From 23d10c63bb40c227f1b7fa2bd82ff5d02d8728d9 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 17:31:39 +0200 Subject: [PATCH 16/31] float race --- Include/cpython/pyatomic.h | 6 ++++++ Include/cpython/pyatomic_gcc.h | 8 ++++++++ Include/cpython/pyatomic_msc.h | 19 +++++++++++++++++++ Include/cpython/pyatomic_std.h | 16 ++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 4 ++-- 6 files changed, 57 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index ccf08fcee66944..f92b20a08b50dd 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -336,6 +336,9 @@ _Py_atomic_load_ushort_relaxed(const unsigned short *obj); static inline long _Py_atomic_load_long_relaxed(const long *obj); +static inline float +_Py_atomic_load_float_relaxed(const float *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -514,6 +517,9 @@ _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value); static inline void _Py_atomic_store_long_release(long *obj, long value); +static inline void +_Py_atomic_store_float_release(float *obj, float value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 2602f9feeabf20..0132b09477ea90 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -326,6 +326,10 @@ static inline long _Py_atomic_load_long_relaxed(const long *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ float ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -552,6 +556,10 @@ static inline void _Py_atomic_store_long_release(long *obj, long value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_float_release(float *obj, float value) +{ __atomic_store(obj, &value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 5453faf3cdaba4..83c3e5ab106af5 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -664,6 +664,12 @@ _Py_atomic_load_long_relaxed(const long *obj) return *(volatile long *)obj; } +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ + return *(volatile float *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -1072,6 +1078,19 @@ _Py_atomic_store_long_release(long *obj, long value) #endif } +static inline void +_Py_atomic_store_float_release(float *obj, float value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(float volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr64((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_float_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 24e61688bd1b73..656783b7325173 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -555,6 +555,14 @@ _Py_atomic_load_long_relaxed(const long *obj) memory_order_relaxed); } +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(float)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -983,6 +991,14 @@ _Py_atomic_store_long_release(long *obj, long value) memory_order_release); } +static inline void +_Py_atomic_store_float_release(float *obj, float value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(float)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 3244f9e673cfff..489f8025cbac62 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -93,6 +93,10 @@ extern "C" { _Py_atomic_store_ulong(&value, new_value) #define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) \ _Py_atomic_store_ssize_release(&value, new_value) +#define FT_ATOMIC_STORE_FLOAT_RELEASE(value, new_value) \ + _Py_atomic_store_float_release(&value, new_value) +#define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) \ + _Py_atomic_load_float_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -132,6 +136,8 @@ extern "C" { #define FT_ATOMIC_STORE_LONG_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_ULONG(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) value +#define FT_ATOMIC_STORE_FLOAT_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 565e63f571bcd0..b4c509af12885f 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -64,7 +64,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromSsize_t(FT_ATOMIC_LOAD_SSIZE_RELAXED(*(Py_ssize_t*)addr)); break; case Py_T_FLOAT: - v = PyFloat_FromDouble((double)*(float*)addr); + v = PyFloat_FromDouble((double)FT_ATOMIC_LOAD_FLOAT_RELAXED(*(float*)addr)); break; case Py_T_DOUBLE: v = PyFloat_FromDouble(*(double*)addr); @@ -293,7 +293,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) double double_val = PyFloat_AsDouble(v); if ((double_val == -1) && PyErr_Occurred()) return -1; - *(float*)addr = (float)double_val; + FT_ATOMIC_STORE_FLOAT_RELEASE(*(float*)addr, (float)double_val); break; } case Py_T_DOUBLE: From ae37a8e68ca2c77ba1a528ab789a371815b0211e Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 17:38:28 +0200 Subject: [PATCH 17/31] double race --- Include/cpython/pyatomic.h | 7 +++++-- Include/cpython/pyatomic_gcc.h | 8 +++++++ Include/cpython/pyatomic_msc.h | 21 ++++++++++++++++++- Include/cpython/pyatomic_std.h | 16 ++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 4 ++-- 6 files changed, 57 insertions(+), 5 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index f92b20a08b50dd..fbc30624ff3c1b 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -336,8 +336,8 @@ _Py_atomic_load_ushort_relaxed(const unsigned short *obj); static inline long _Py_atomic_load_long_relaxed(const long *obj); -static inline float -_Py_atomic_load_float_relaxed(const float *obj); +static inline double +_Py_atomic_load_double_relaxed(const double *obj); static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -520,6 +520,9 @@ _Py_atomic_store_long_release(long *obj, long value); static inline void _Py_atomic_store_float_release(float *obj, float value); +static inline void +_Py_atomic_store_double_release(double *obj, double value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 0132b09477ea90..744bd5cf811079 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -330,6 +330,10 @@ static inline float _Py_atomic_load_float_relaxed(const float *obj) { float ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; } +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ double ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -560,6 +564,10 @@ static inline void _Py_atomic_store_float_release(float *obj, float value) { __atomic_store(obj, &value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_double_release(double *obj, double value) +{ __atomic_store(obj, &value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 83c3e5ab106af5..93c57117f5f4e0 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -670,6 +670,12 @@ _Py_atomic_load_float_relaxed(const float *obj) return *(volatile float *)obj; } +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ + return *(volatile double *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -1085,12 +1091,25 @@ _Py_atomic_store_float_release(float *obj, float value) *(float volatile *)obj = value; #elif defined(_M_ARM64) _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); - __stlr64((unsigned __int32 volatile *)obj, (unsigned __int32)value); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); #else # error "no implementation of _Py_atomic_store_float_release" #endif } +static inline void +_Py_atomic_store_double_release(double *obj, double value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(double volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); +#else +# error "no implementation of _Py_atomic_store_double_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 656783b7325173..9f66ccfe41882b 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -563,6 +563,14 @@ _Py_atomic_load_float_relaxed(const float *obj) memory_order_relaxed); } +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(double)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -999,6 +1007,14 @@ _Py_atomic_store_float_release(float *obj, float value) memory_order_release); } +static inline void +_Py_atomic_store_double_release(double *obj, double value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(double)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 489f8025cbac62..5b3ea7e1a36885 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -97,6 +97,10 @@ extern "C" { _Py_atomic_store_float_release(&value, new_value) #define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) \ _Py_atomic_load_float_relaxed(&value) +#define FT_ATOMIC_STORE_DOUBLE_RELEASE(value, new_value) \ + _Py_atomic_store_double_release(&value, new_value) +#define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) \ + _Py_atomic_load_double_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -138,6 +142,8 @@ extern "C" { #define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) value #define FT_ATOMIC_STORE_FLOAT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) value +#define FT_ATOMIC_STORE_DOUBLE_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index b4c509af12885f..50558d867b79fb 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -67,7 +67,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyFloat_FromDouble((double)FT_ATOMIC_LOAD_FLOAT_RELAXED(*(float*)addr)); break; case Py_T_DOUBLE: - v = PyFloat_FromDouble(*(double*)addr); + v = PyFloat_FromDouble(FT_ATOMIC_LOAD_DOUBLE_RELAXED(*(double*)addr)); break; case Py_T_STRING: if (*(char**)addr == NULL) { @@ -297,7 +297,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) break; } case Py_T_DOUBLE: - *(double*)addr = PyFloat_AsDouble(v); + FT_ATOMIC_STORE_DOUBLE_RELEASE(*(double*)addr, PyFloat_AsDouble(v)); if ((*(double*)addr == -1) && PyErr_Occurred()) return -1; break; From bce02d0d02b28a533c8cef18a72bf41c5e50669b Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 17:48:09 +0200 Subject: [PATCH 18/31] T_CHAR --- Python/structmember.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Python/structmember.c b/Python/structmember.c index 50558d867b79fb..1b8f1ad140c8df 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -24,6 +24,7 @@ PyObject * PyMember_GetOne(const char *obj_addr, PyMemberDef *l) { PyObject *v; + char value; if (l->flags & Py_RELATIVE_OFFSET) { PyErr_SetString( PyExc_SystemError, @@ -80,7 +81,9 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyUnicode_FromString((char*)addr); break; case Py_T_CHAR: - v = PyUnicode_FromStringAndSize((char*)addr, 1); + // case ...: char value = ... requires a C23-compatible compiler + value = FT_ATOMIC_LOAD_CHAR_RELAXED(*(char *) addr); + v = PyUnicode_FromStringAndSize(&value, 1); break; case _Py_T_OBJECT: v = *(PyObject **)addr; @@ -318,7 +321,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) PyErr_BadArgument(); return -1; } - *(char*)addr = string[0]; + FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, string[0]); break; } case Py_T_STRING: From afe25045275d8fbaf2e5d046a767cce070a28daf Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 18:02:33 +0200 Subject: [PATCH 19/31] signed long long race --- Include/cpython/pyatomic.h | 6 ++++++ Include/cpython/pyatomic_gcc.h | 8 ++++++++ Include/cpython/pyatomic_msc.h | 19 +++++++++++++++++++ Include/cpython/pyatomic_std.h | 16 ++++++++++++++++ .../internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 6 +++--- 6 files changed, 58 insertions(+), 3 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index fbc30624ff3c1b..5f862ef318879a 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -339,6 +339,9 @@ _Py_atomic_load_long_relaxed(const long *obj); static inline double _Py_atomic_load_double_relaxed(const double *obj); +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -523,6 +526,9 @@ _Py_atomic_store_float_release(float *obj, float value); static inline void _Py_atomic_store_double_release(double *obj, double value); +static inline void +_Py_atomic_store_llong_release(long long *obj, long long value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 744bd5cf811079..b856dde6a4f8c7 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -390,6 +390,10 @@ static inline unsigned long long _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + // --- _Py_atomic_store ------------------------------------------------------ @@ -568,6 +572,10 @@ static inline void _Py_atomic_store_double_release(double *obj, double value) { __atomic_store(obj, &value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_llong_release(long long *obj, long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 93c57117f5f4e0..ed87d6faa26cf6 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -760,6 +760,12 @@ _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) return *(volatile unsigned long long *)obj; } +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ + return *(volatile long long *)obj; +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -1110,6 +1116,19 @@ _Py_atomic_store_double_release(double *obj, double value) #endif } +static inline void +_Py_atomic_store_llong_release(long long *obj, long long value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(long long volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int128); + __stlr128((unsigned __int128 volatile *)obj, (unsigned __int128)value); +#else +# error "no implementation of _Py_atomic_store_llong_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 9f66ccfe41882b..8d11633a69335f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -683,6 +683,14 @@ _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) memory_order_relaxed); } +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(long long)*)obj, + memory_order_relaxed); +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -1015,6 +1023,14 @@ _Py_atomic_store_double_release(double *obj, double value) memory_order_release); } +static inline void +_Py_atomic_store_llong_release(long long *obj, long long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(long long)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 5b3ea7e1a36885..55193315ff5145 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -101,6 +101,10 @@ extern "C" { _Py_atomic_store_double_release(&value, new_value) #define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) \ _Py_atomic_load_double_relaxed(&value) +#define FT_ATOMIC_STORE_LLONG_RELEASE(value, new_value) \ + _Py_atomic_store_llong_release(&value, new_value) +#define FT_ATOMIC_LOAD_LLONG_RELAXED(value) \ + _Py_atomic_load_llong_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -144,6 +148,8 @@ extern "C" { #define FT_ATOMIC_STORE_FLOAT_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) value #define FT_ATOMIC_STORE_DOUBLE_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_LLONG_RELAXED(value) value +#define FT_ATOMIC_STORE_LLONG_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 1b8f1ad140c8df..6455f3587dabf2 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -107,7 +107,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) #endif break; case Py_T_LONGLONG: - v = PyLong_FromLongLong(*(long long *)addr); + v = PyLong_FromLongLong(FT_ATOMIC_LOAD_LLONG_RELAXED(*(long long *)addr)); break; case Py_T_ULONGLONG: v = PyLong_FromUnsignedLongLong(*(unsigned long long *)addr); @@ -329,8 +329,8 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) PyErr_SetString(PyExc_TypeError, "readonly attribute"); return -1; case Py_T_LONGLONG:{ - long long value; - *(long long*)addr = value = PyLong_AsLongLong(v); + long long value = PyLong_AsLongLong(v); + FT_ATOMIC_STORE_LLONG_RELEASE(*(long long*)addr, value); if ((value == -1) && PyErr_Occurred()) return -1; break; From 3dfaab8865bae33cb56144466a4a1844febcde51 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 18:09:07 +0200 Subject: [PATCH 20/31] unsigned long long race --- Include/cpython/pyatomic.h | 4 ++++ Include/cpython/pyatomic_gcc.h | 5 +++++ Include/cpython/pyatomic_msc.h | 13 +++++++++++++ Include/cpython/pyatomic_std.h | 9 +++++++++ Include/internal/pycore_pyatomic_ft_wrappers.h | 6 ++++++ Python/structmember.c | 6 +++--- 6 files changed, 40 insertions(+), 3 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 5f862ef318879a..f5ccae1f207556 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -529,6 +529,10 @@ _Py_atomic_store_double_release(double *obj, double value); static inline void _Py_atomic_store_llong_release(long long *obj, long long value); +static inline void +_Py_atomic_store_ullong_release(unsigned long long *obj, + unsigned long long value); + static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index b856dde6a4f8c7..8111611d8657f2 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -576,6 +576,11 @@ static inline void _Py_atomic_store_llong_release(long long *obj, long long value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_ullong_release(unsigned long long *obj, + unsigned long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index ed87d6faa26cf6..fa58a031d65282 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1129,6 +1129,19 @@ _Py_atomic_store_llong_release(long long *obj, long long value) #endif } +static inline void +_Py_atomic_store_ullong_release(unsigned long long *obj, unsigned long long value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(unsigned long long volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int128); + __stlr128((unsigned __int128 volatile *)obj, (unsigned __int128)value); +#else +# error "no implementation of _Py_atomic_store_ullong_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 8d11633a69335f..7e0018c901d62f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -1031,6 +1031,15 @@ _Py_atomic_store_llong_release(long long *obj, long long value) memory_order_release); } +static inline void +_Py_atomic_store_ullong_release(unsigned long long *obj, + unsigned long long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned long long)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 55193315ff5145..d9703cb3274786 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -105,6 +105,10 @@ extern "C" { _Py_atomic_store_llong_release(&value, new_value) #define FT_ATOMIC_LOAD_LLONG_RELAXED(value) \ _Py_atomic_load_llong_relaxed(&value) +#define FT_ATOMIC_STORE_ULLONG_RELEASE(value, new_value) \ + _Py_atomic_store_ullong_release(&value, new_value) +#define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) \ + _Py_atomic_load_ullong_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -150,6 +154,8 @@ extern "C" { #define FT_ATOMIC_STORE_DOUBLE_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_LOAD_LLONG_RELAXED(value) value #define FT_ATOMIC_STORE_LLONG_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value +#define FT_ATOMIC_STORE_ULLONG_RELEASE(value, new_value) value = new_value #endif diff --git a/Python/structmember.c b/Python/structmember.c index 6455f3587dabf2..3484d397d16021 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -110,7 +110,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyLong_FromLongLong(FT_ATOMIC_LOAD_LLONG_RELAXED(*(long long *)addr)); break; case Py_T_ULONGLONG: - v = PyLong_FromUnsignedLongLong(*(unsigned long long *)addr); + v = PyLong_FromUnsignedLongLong(FT_ATOMIC_LOAD_ULLONG_RELAXED(*(unsigned long long *)addr)); break; case _Py_T_NONE: // doesn't require free-threading code path @@ -346,7 +346,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - *(unsigned long long *)addr = (unsigned long long)(long long)long_val; + FT_ATOMIC_STORE_ULLONG_RELEASE(*(unsigned long long *)addr, (unsigned long long)(long long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -355,7 +355,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulonglong_val == (unsigned long long)-1 && PyErr_Occurred()) { return -1; } - *(unsigned long long*)addr = ulonglong_val; + FT_ATOMIC_STORE_ULLONG_RELEASE(*(unsigned long long *)addr, ulonglong_val); } break; } From 50a7bb61adc5c6f00f627514f6729f20289778c7 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 18:18:59 +0200 Subject: [PATCH 21/31] fix return type --- Include/cpython/pyatomic_msc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index fa58a031d65282..fdffcfe75118d1 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -634,7 +634,7 @@ _Py_atomic_load_int_relaxed(const int *obj) return *(volatile int *)obj; } -static inline int +static inline char _Py_atomic_load_char_relaxed(const char *obj) { return *(volatile char *)obj; From 416e0bb8f83d081e36a8a2e5da24968191714ef1 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 18:37:07 +0200 Subject: [PATCH 22/31] arm64 does not have 128-bit integers, I suppose --- Include/cpython/pyatomic_msc.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index fdffcfe75118d1..58dfdaaaf315ec 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1122,8 +1122,8 @@ _Py_atomic_store_llong_release(long long *obj, long long value) #if defined(_M_X64) || defined(_M_IX86) *(long long volatile *)obj = value; #elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int128); - __stlr128((unsigned __int128 volatile *)obj, (unsigned __int128)value); + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); #else # error "no implementation of _Py_atomic_store_llong_release" #endif @@ -1135,8 +1135,8 @@ _Py_atomic_store_ullong_release(unsigned long long *obj, unsigned long long valu #if defined(_M_X64) || defined(_M_IX86) *(unsigned long long volatile *)obj = value; #elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int128); - __stlr128((unsigned __int128 volatile *)obj, (unsigned __int128)value); + _Py_atomic_ASSERT_ARG_TYPE(__int64); + __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); #else # error "no implementation of _Py_atomic_store_ullong_release" #endif From 4bd69275a52bcde0863dd936cc3b14aa0aca2fbb Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 22 Aug 2024 18:43:56 +0200 Subject: [PATCH 23/31] at this point I'm just making guesses --- Include/cpython/pyatomic_msc.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 58dfdaaaf315ec..cf86c59cca9366 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -1083,8 +1083,8 @@ _Py_atomic_store_long_release(long *obj, long value) #if defined(_M_X64) || defined(_M_IX86) *(long volatile *)obj = value; #elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); - __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); #else # error "no implementation of _Py_atomic_store_long_release" #endif From a4c094b1a42a871d49fdd9b2cf8d3ab0afbee096 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 29 Aug 2024 10:54:35 +0200 Subject: [PATCH 24/31] scope --- Python/structmember.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Python/structmember.c b/Python/structmember.c index 3484d397d16021..0f089c3a372caa 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -24,7 +24,6 @@ PyObject * PyMember_GetOne(const char *obj_addr, PyMemberDef *l) { PyObject *v; - char value; if (l->flags & Py_RELATIVE_OFFSET) { PyErr_SetString( PyExc_SystemError, @@ -80,11 +79,11 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) case Py_T_STRING_INPLACE: v = PyUnicode_FromString((char*)addr); break; - case Py_T_CHAR: - // case ...: char value = ... requires a C23-compatible compiler - value = FT_ATOMIC_LOAD_CHAR_RELAXED(*(char *) addr); + case Py_T_CHAR: { + char value; v = PyUnicode_FromStringAndSize(&value, 1); break; + } case _Py_T_OBJECT: v = *(PyObject **)addr; if (v == NULL) From 267f99572935b8a8d215de9d87741de66fa2cf62 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 29 Aug 2024 11:38:24 +0200 Subject: [PATCH 25/31] must 0-initialize --- Python/structmember.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/structmember.c b/Python/structmember.c index 0f089c3a372caa..d6de425cf4978b 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -80,7 +80,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyUnicode_FromString((char*)addr); break; case Py_T_CHAR: { - char value; + char value = 0; v = PyUnicode_FromStringAndSize(&value, 1); break; } From c006e1d465d727d38187dac2f62a1d06ada2b5ad Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 29 Aug 2024 12:28:27 +0200 Subject: [PATCH 26/31] FT_ATOMIC_STORE_CHAR_RELEASE -> FT_ATOMIC_STORE_CHAR_RELAXED --- Include/cpython/pyatomic.h | 6 +++--- Include/cpython/pyatomic_gcc.h | 8 ++++---- Include/cpython/pyatomic_msc.h | 19 ++++++------------- Include/cpython/pyatomic_std.h | 16 ++++++++-------- .../internal/pycore_pyatomic_ft_wrappers.h | 4 ++-- Python/structmember.c | 8 ++++---- 6 files changed, 27 insertions(+), 34 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index f5ccae1f207556..96cc06ce3a61bc 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -479,6 +479,9 @@ static inline void _Py_atomic_store_ullong_relaxed(unsigned long long *obj, unsigned long long value); +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value); + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -502,9 +505,6 @@ _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); static inline void _Py_atomic_store_int_release(int *obj, int value); -static inline void -_Py_atomic_store_char_release(char *obj, char value); - static inline void _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 8111611d8657f2..b11b8a2368d44c 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -517,6 +517,10 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, unsigned long long value) { __atomic_store_n(obj, value, __ATOMIC_RELAXED); } +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -540,10 +544,6 @@ static inline void _Py_atomic_store_int_release(int *obj, int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } -static inline void -_Py_atomic_store_char_release(char *obj, char value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } - static inline void _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index cf86c59cca9366..51708b97fc9479 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -947,6 +947,12 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, *(volatile unsigned long long *)obj = value; } +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ + *(volatile char *)obj = value; +} + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -1012,19 +1018,6 @@ _Py_atomic_store_int_release(int *obj, int value) #endif } -static inline void -_Py_atomic_store_char_release(char *obj, char value) -{ -#if defined(_M_X64) || defined(_M_IX86) - *(char volatile *)obj = value; -#elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int8); - __stlr8((unsigned __int8 volatile *)obj, (unsigned __int8)value); -#else -# error "no implementation of _Py_atomic_store_char_release" -#endif -} - static inline void _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 7e0018c901d62f..c2a411466327ac 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -916,6 +916,14 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, memory_order_relaxed); } +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(char)*)obj, value, + memory_order_relaxed); +} + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ @@ -959,14 +967,6 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } -static inline void -_Py_atomic_store_char_release(char *obj, char value) -{ - _Py_USING_STD; - atomic_store_explicit((_Atomic(char)*)obj, value, - memory_order_release); -} - static inline void _Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) { diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index d9703cb3274786..914dba0b5f6792 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -61,8 +61,8 @@ extern "C" { _Py_atomic_store_uint16_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ _Py_atomic_store_uint32_relaxed(&value, new_value) -#define FT_ATOMIC_STORE_CHAR_RELEASE(value, new_value) \ - _Py_atomic_store_char_release(&value, new_value) +#define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) \ + _Py_atomic_store_char_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ _Py_atomic_load_char_relaxed(&value) #define FT_ATOMIC_STORE_UCHAR_RELEASE(value, new_value) \ diff --git a/Python/structmember.c b/Python/structmember.c index d6de425cf4978b..5fbde435031631 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -171,16 +171,16 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) return -1; } if (v == Py_True) - FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, 1); + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, 1); else - FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, 0); + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, 0); break; } case Py_T_BYTE:{ long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, (char)long_val); + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, (char)long_val); /* XXX: For compatibility, only warn about truncations for now. */ if ((long_val > CHAR_MAX) || (long_val < CHAR_MIN)) @@ -320,7 +320,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) PyErr_BadArgument(); return -1; } - FT_ATOMIC_STORE_CHAR_RELEASE(*(char*)addr, string[0]); + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, string[0]); break; } case Py_T_STRING: From 488dea52797332afd539797cbce8d93c17d3bd08 Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 29 Aug 2024 14:25:07 +0200 Subject: [PATCH 27/31] remaining release -> relaxed --- Include/cpython/pyatomic.h | 47 +++--- Include/cpython/pyatomic_gcc.h | 63 ++++---- Include/cpython/pyatomic_msc.h | 153 +++++------------- Include/cpython/pyatomic_std.h | 93 +++++------ .../internal/pycore_pyatomic_ft_wrappers.h | 48 +++--- Python/structmember.c | 30 ++-- 6 files changed, 170 insertions(+), 264 deletions(-) diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 96cc06ce3a61bc..6d106c1b499c69 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -482,56 +482,49 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, static inline void _Py_atomic_store_char_relaxed(char *obj, char value); - -// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ - -// Loads `*obj` (acquire operation) -static inline void * -_Py_atomic_load_ptr_acquire(const void *obj); - -static inline uintptr_t -_Py_atomic_load_uintptr_acquire(const uintptr_t *obj); - -// Stores `*obj = value` (release operation) static inline void -_Py_atomic_store_ptr_release(void *obj, void *value); +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value); static inline void -_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value); +_Py_atomic_store_short_relaxed(short *obj, short value); static inline void -_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value); static inline void -_Py_atomic_store_int_release(int *obj, int value); +_Py_atomic_store_long_relaxed(long *obj, long value); static inline void -_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value); +_Py_atomic_store_float_relaxed(float *obj, float value); static inline void -_Py_atomic_store_short_release(short *obj, short value); +_Py_atomic_store_double_relaxed(double *obj, double value); static inline void -_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value); +_Py_atomic_store_llong_relaxed(long long *obj, long long value); -static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value); -static inline void -_Py_atomic_store_long_release(long *obj, long value); +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +// Loads `*obj` (acquire operation) +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj); +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj); + +// Stores `*obj = value` (release operation) static inline void -_Py_atomic_store_float_release(float *obj, float value); +_Py_atomic_store_ptr_release(void *obj, void *value); static inline void -_Py_atomic_store_double_release(double *obj, double value); +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value); static inline void -_Py_atomic_store_llong_release(long long *obj, long long value); +_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value); static inline void -_Py_atomic_store_ullong_release(unsigned long long *obj, - unsigned long long value); +_Py_atomic_store_int_release(int *obj, int value); static inline int _Py_atomic_load_int_acquire(const int *obj); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index b11b8a2368d44c..b179e4c9a185a9 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -521,64 +521,55 @@ static inline void _Py_atomic_store_char_relaxed(char *obj, char value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } - -// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ - -static inline void * -_Py_atomic_load_ptr_acquire(const void *obj) -{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_ACQUIRE); } - -static inline uintptr_t -_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) -{ return (uintptr_t)__atomic_load_n(obj, __ATOMIC_ACQUIRE); } - static inline void -_Py_atomic_store_ptr_release(void *obj, void *value) -{ __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); } +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } static inline void -_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +_Py_atomic_store_short_relaxed(short *obj, short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } static inline void -_Py_atomic_store_int_release(int *obj, int value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } static inline void -_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +_Py_atomic_store_long_relaxed(long *obj, long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } static inline void -_Py_atomic_store_short_release(short *obj, short value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +_Py_atomic_store_float_relaxed(float *obj, float value) +{ __atomic_store(obj, &value, __ATOMIC_RELAXED); } static inline void -_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +_Py_atomic_store_double_relaxed(double *obj, double value) +{ __atomic_store(obj, &value, __ATOMIC_RELAXED); } static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +_Py_atomic_store_llong_relaxed(long long *obj, long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } -static inline void -_Py_atomic_store_long_release(long *obj, long value) -{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } -static inline void -_Py_atomic_store_float_release(float *obj, float value) -{ __atomic_store(obj, &value, __ATOMIC_RELEASE); } +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) +{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_ACQUIRE); } + +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +{ return (uintptr_t)__atomic_load_n(obj, __ATOMIC_ACQUIRE); } static inline void -_Py_atomic_store_double_release(double *obj, double value) -{ __atomic_store(obj, &value, __ATOMIC_RELEASE); } +_Py_atomic_store_ptr_release(void *obj, void *value) +{ __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); } static inline void -_Py_atomic_store_llong_release(long long *obj, long long value) +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } static inline void -_Py_atomic_store_ullong_release(unsigned long long *obj, - unsigned long long value) +_Py_atomic_store_int_release(int *obj, int value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } static inline void diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 51708b97fc9479..a7600114a64aef 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -953,185 +953,116 @@ _Py_atomic_store_char_relaxed(char *obj, char value) *(volatile char *)obj = value; } - -// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ - -static inline void * -_Py_atomic_load_ptr_acquire(const void *obj) -{ -#if defined(_M_X64) || defined(_M_IX86) - return *(void * volatile *)obj; -#elif defined(_M_ARM64) - return (void *)__ldar64((unsigned __int64 volatile *)obj); -#else -# error "no implementation of _Py_atomic_load_ptr_acquire" -#endif -} - -static inline uintptr_t -_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) { -#if defined(_M_X64) || defined(_M_IX86) - return *(uintptr_t volatile *)obj; -#elif defined(_M_ARM64) - return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj); -#else -# error "no implementation of _Py_atomic_load_uintptr_acquire" -#endif + *(volatile unsigned char*)obj = value; } static inline void -_Py_atomic_store_ptr_release(void *obj, void *value) +_Py_atomic_store_short_relaxed(short *obj, short value) { -#if defined(_M_X64) || defined(_M_IX86) - *(void * volatile *)obj = value; -#elif defined(_M_ARM64) - __stlr64((unsigned __int64 volatile *)obj, (uintptr_t)value); -#else -# error "no implementation of _Py_atomic_store_ptr_release" -#endif + *(volatile short*)obj = value; } static inline void -_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) { -#if defined(_M_X64) || defined(_M_IX86) - *(uintptr_t volatile *)obj = value; -#elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); - __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); -#else -# error "no implementation of _Py_atomic_store_uintptr_release" -#endif + *(volatile unsigned short*)obj = value; } static inline void -_Py_atomic_store_int_release(int *obj, int value) +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) { -#if defined(_M_X64) || defined(_M_IX86) - *(int volatile *)obj = value; -#elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); - __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); -#else -# error "no implementation of _Py_atomic_store_int_release" -#endif + *(volatile unsigned int*)obj = value; } static inline void -_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) +_Py_atomic_store_long_relaxed(long *obj, long value) { -#if defined(_M_X64) || defined(_M_IX86) - *(unsigned char volatile *)obj = value; -#elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int8); - __stlr8((unsigned __int8 volatile *)obj, (unsigned __int8)value); -#else -# error "no implementation of _Py_atomic_store_uchar_release" -#endif + *(volatile long *)obj = value; } static inline void -_Py_atomic_store_short_release(short *obj, short value) +_Py_atomic_store_float_relaxed(float *obj, float value) { -#if defined(_M_X64) || defined(_M_IX86) - *(short volatile *)obj = value; -#elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int16); - __stlr16((unsigned __int16 volatile *)obj, (unsigned __int16)value); -#else -# error "no implementation of _Py_atomic_store_short_release" -#endif + *(volatile float *)obj = value; } static inline void -_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) +_Py_atomic_store_double_relaxed(double *obj, double value) { -#if defined(_M_X64) || defined(_M_IX86) - *(unsigned short volatile *)obj = value; -#elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int16); - __stlr16((unsigned __int16 volatile *)obj, (unsigned __int16)value); -#else -# error "no implementation of _Py_atomic_store_ushort_release" -#endif + *(volatile double *)obj = value; } static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +_Py_atomic_store_llong_relaxed(long long *obj, long long value) { -#if defined(_M_X64) || defined(_M_IX86) - *(unsigned int volatile *)obj = value; -#elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); - __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); -#else -# error "no implementation of _Py_atomic_store_uint_release" -#endif + *(volatile long long*)obj = value; } -static inline void -_Py_atomic_store_long_release(long *obj, long value) + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) { #if defined(_M_X64) || defined(_M_IX86) - *(long volatile *)obj = value; + return *(void * volatile *)obj; #elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); - __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); + return (void *)__ldar64((unsigned __int64 volatile *)obj); #else -# error "no implementation of _Py_atomic_store_long_release" +# error "no implementation of _Py_atomic_load_ptr_acquire" #endif } -static inline void -_Py_atomic_store_float_release(float *obj, float value) +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) { #if defined(_M_X64) || defined(_M_IX86) - *(float volatile *)obj = value; + return *(uintptr_t volatile *)obj; #elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); - __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); + return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj); #else -# error "no implementation of _Py_atomic_store_float_release" +# error "no implementation of _Py_atomic_load_uintptr_acquire" #endif } static inline void -_Py_atomic_store_double_release(double *obj, double value) +_Py_atomic_store_ptr_release(void *obj, void *value) { #if defined(_M_X64) || defined(_M_IX86) - *(double volatile *)obj = value; + *(void * volatile *)obj = value; #elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); - __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); + __stlr64((unsigned __int64 volatile *)obj, (uintptr_t)value); #else -# error "no implementation of _Py_atomic_store_double_release" +# error "no implementation of _Py_atomic_store_ptr_release" #endif } static inline void -_Py_atomic_store_llong_release(long long *obj, long long value) +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) { #if defined(_M_X64) || defined(_M_IX86) - *(long long volatile *)obj = value; + *(uintptr_t volatile *)obj = value; #elif defined(_M_ARM64) _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64); __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); #else -# error "no implementation of _Py_atomic_store_llong_release" +# error "no implementation of _Py_atomic_store_uintptr_release" #endif } static inline void -_Py_atomic_store_ullong_release(unsigned long long *obj, unsigned long long value) +_Py_atomic_store_int_release(int *obj, int value) { #if defined(_M_X64) || defined(_M_IX86) - *(unsigned long long volatile *)obj = value; + *(int volatile *)obj = value; #elif defined(_M_ARM64) - _Py_atomic_ASSERT_ARG_TYPE(__int64); - __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value); + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); #else -# error "no implementation of _Py_atomic_store_ullong_release" +# error "no implementation of _Py_atomic_store_int_release" #endif } diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index c2a411466327ac..69a8b9e615ea5f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -924,119 +924,110 @@ _Py_atomic_store_char_relaxed(char *obj, char value) memory_order_relaxed); } - -// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ - -static inline void * -_Py_atomic_load_ptr_acquire(const void *obj) -{ - _Py_USING_STD; - return atomic_load_explicit((const _Atomic(void*)*)obj, - memory_order_acquire); -} - -static inline uintptr_t -_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) { _Py_USING_STD; - return atomic_load_explicit((const _Atomic(uintptr_t)*)obj, - memory_order_acquire); + atomic_store_explicit((_Atomic(unsigned char)*)obj, value, + memory_order_relaxed); } static inline void -_Py_atomic_store_ptr_release(void *obj, void *value) +_Py_atomic_store_short_relaxed(short *obj, short value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(void*)*)obj, value, - memory_order_release); + atomic_store_explicit((_Atomic(short)*)obj, value, + memory_order_relaxed); } static inline void -_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(uintptr_t)*)obj, value, - memory_order_release); + atomic_store_explicit((_Atomic(unsigned short)*)obj, value, + memory_order_relaxed); } static inline void -_Py_atomic_store_int_release(int *obj, int value) +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(int)*)obj, value, - memory_order_release); + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_relaxed); } static inline void -_Py_atomic_store_uchar_release(unsigned char *obj, unsigned char value) +_Py_atomic_store_long_relaxed(long *obj, long value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned char)*)obj, value, - memory_order_release); + atomic_store_explicit((_Atomic(long)*)obj, value, + memory_order_relaxed); } static inline void -_Py_atomic_store_short_release(short *obj, short value) +_Py_atomic_store_float_relaxed(float *obj, float value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(short)*)obj, value, - memory_order_release); + atomic_store_explicit((_Atomic(float)*)obj, value, + memory_order_relaxed); } static inline void -_Py_atomic_store_ushort_release(unsigned short *obj, unsigned short value) +_Py_atomic_store_double_relaxed(double *obj, double value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned short)*)obj, value, - memory_order_release); + atomic_store_explicit((_Atomic(double)*)obj, value, + memory_order_relaxed); } static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +_Py_atomic_store_llong_relaxed(long long *obj, long long value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned int)*)obj, value, - memory_order_release); + atomic_store_explicit((_Atomic(long long)*)obj, value, + memory_order_relaxed); } -static inline void -_Py_atomic_store_long_release(long *obj, long value) + +// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ + +static inline void * +_Py_atomic_load_ptr_acquire(const void *obj) { _Py_USING_STD; - atomic_store_explicit((_Atomic(long)*)obj, value, - memory_order_release); + return atomic_load_explicit((const _Atomic(void*)*)obj, + memory_order_acquire); } -static inline void -_Py_atomic_store_float_release(float *obj, float value) +static inline uintptr_t +_Py_atomic_load_uintptr_acquire(const uintptr_t *obj) { _Py_USING_STD; - atomic_store_explicit((_Atomic(float)*)obj, value, - memory_order_release); + return atomic_load_explicit((const _Atomic(uintptr_t)*)obj, + memory_order_acquire); } static inline void -_Py_atomic_store_double_release(double *obj, double value) +_Py_atomic_store_ptr_release(void *obj, void *value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(double)*)obj, value, + atomic_store_explicit((_Atomic(void*)*)obj, value, memory_order_release); } static inline void -_Py_atomic_store_llong_release(long long *obj, long long value) +_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(long long)*)obj, value, + atomic_store_explicit((_Atomic(uintptr_t)*)obj, value, memory_order_release); } static inline void -_Py_atomic_store_ullong_release(unsigned long long *obj, - unsigned long long value) +_Py_atomic_store_int_release(int *obj, int value) { _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned long long)*)obj, value, + atomic_store_explicit((_Atomic(int)*)obj, value, memory_order_release); } diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 914dba0b5f6792..f7b81f757b33e8 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -65,48 +65,48 @@ extern "C" { _Py_atomic_store_char_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ _Py_atomic_load_char_relaxed(&value) -#define FT_ATOMIC_STORE_UCHAR_RELEASE(value, new_value) \ - _Py_atomic_store_uchar_release(&value, new_value) +#define FT_ATOMIC_STORE_UCHAR_RELAXED(value, new_value) \ + _Py_atomic_store_uchar_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) \ _Py_atomic_load_uchar_relaxed(&value) -#define FT_ATOMIC_STORE_SHORT_RELEASE(value, new_value) \ - _Py_atomic_store_short_release(&value, new_value) +#define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) \ + _Py_atomic_store_short_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_SHORT_RELAXED(value) \ _Py_atomic_load_short_relaxed(&value) -#define FT_ATOMIC_STORE_USHORT_RELEASE(value, new_value) \ - _Py_atomic_store_ushort_release(&value, new_value) +#define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) \ + _Py_atomic_store_ushort_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) \ _Py_atomic_load_ushort_relaxed(&value) -#define FT_ATOMIC_STORE_INT_RELEASE(value, new_value) \ - _Py_atomic_store_int_release(&value, new_value) +#define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) \ + _Py_atomic_store_int_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_INT_RELAXED(value) \ _Py_atomic_load_int_relaxed(&value) -#define FT_ATOMIC_STORE_UINT_RELEASE(value, new_value) \ - _Py_atomic_store_uint_release(&value, new_value) +#define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) \ + _Py_atomic_store_uint_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_UINT_RELAXED(value) \ _Py_atomic_load_uint_relaxed(&value) -#define FT_ATOMIC_STORE_LONG_RELEASE(value, new_value) \ - _Py_atomic_store_long_release(&value, new_value) +#define FT_ATOMIC_STORE_LONG_RELAXED(value, new_value) \ + _Py_atomic_store_long_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_LONG_RELAXED(value) \ _Py_atomic_load_long_relaxed(&value) -#define FT_ATOMIC_STORE_ULONG(value, new_value) \ - _Py_atomic_store_ulong(&value, new_value) -#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) \ - _Py_atomic_store_ssize_release(&value, new_value) -#define FT_ATOMIC_STORE_FLOAT_RELEASE(value, new_value) \ - _Py_atomic_store_float_release(&value, new_value) +#define FT_ATOMIC_STORE_ULONG_RELAXED(value, new_value) \ + _Py_atomic_store_ulong_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ + _Py_atomic_store_ssize_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_FLOAT_RELAXED(value, new_value) \ + _Py_atomic_store_float_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) \ _Py_atomic_load_float_relaxed(&value) -#define FT_ATOMIC_STORE_DOUBLE_RELEASE(value, new_value) \ - _Py_atomic_store_double_release(&value, new_value) +#define FT_ATOMIC_STORE_DOUBLE_RELAXED(value, new_value) \ + _Py_atomic_store_double_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) \ _Py_atomic_load_double_relaxed(&value) -#define FT_ATOMIC_STORE_LLONG_RELEASE(value, new_value) \ - _Py_atomic_store_llong_release(&value, new_value) +#define FT_ATOMIC_STORE_LLONG_RELAXED(value, new_value) \ + _Py_atomic_store_llong_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_LLONG_RELAXED(value) \ _Py_atomic_load_llong_relaxed(&value) -#define FT_ATOMIC_STORE_ULLONG_RELEASE(value, new_value) \ - _Py_atomic_store_ullong_release(&value, new_value) +#define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) \ + _Py_atomic_store_ullong_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) \ _Py_atomic_load_ullong_relaxed(&value) diff --git a/Python/structmember.c b/Python/structmember.c index 5fbde435031631..28b1a2597e3f66 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -191,7 +191,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - FT_ATOMIC_STORE_UCHAR_RELEASE(*(unsigned char*)addr, (unsigned char)long_val); + FT_ATOMIC_STORE_UCHAR_RELAXED(*(unsigned char*)addr, (unsigned char)long_val); if ((long_val > UCHAR_MAX) || (long_val < 0)) WARN("Truncation of value to unsigned char"); break; @@ -200,7 +200,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - FT_ATOMIC_STORE_SHORT_RELEASE(*(short*)addr, (short)long_val); + FT_ATOMIC_STORE_SHORT_RELAXED(*(short*)addr, (short)long_val); if ((long_val > SHRT_MAX) || (long_val < SHRT_MIN)) WARN("Truncation of value to short"); break; @@ -209,7 +209,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - FT_ATOMIC_STORE_USHORT_RELEASE(*(unsigned short*)addr, (unsigned short)long_val); + FT_ATOMIC_STORE_USHORT_RELAXED(*(unsigned short*)addr, (unsigned short)long_val); if ((long_val > USHRT_MAX) || (long_val < 0)) WARN("Truncation of value to unsigned short"); break; @@ -218,7 +218,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - FT_ATOMIC_STORE_INT_RELEASE(*(int *)addr, (int)long_val); + FT_ATOMIC_STORE_INT_RELAXED(*(int *)addr, (int)long_val); if ((long_val > INT_MAX) || (long_val < INT_MIN)) WARN("Truncation of value to int"); break; @@ -236,7 +236,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - FT_ATOMIC_STORE_UINT_RELEASE(*(unsigned int *)addr, (unsigned int)(unsigned long)long_val); + FT_ATOMIC_STORE_UINT_RELAXED(*(unsigned int *)addr, (unsigned int)(unsigned long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -245,7 +245,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; } - FT_ATOMIC_STORE_UINT_RELEASE(*(unsigned int *)addr, (unsigned int)ulong_val); + FT_ATOMIC_STORE_UINT_RELAXED(*(unsigned int *)addr, (unsigned int)ulong_val); if (ulong_val > UINT_MAX) { WARN("Truncation of value to unsigned int"); } @@ -253,7 +253,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) break; } case Py_T_LONG:{ - FT_ATOMIC_STORE_LONG_RELEASE(*(long*)addr, PyLong_AsLong(v)); + FT_ATOMIC_STORE_LONG_RELAXED(*(long*)addr, PyLong_AsLong(v)); if ((*(long*)addr == -1) && PyErr_Occurred()) return -1; break; @@ -271,7 +271,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - FT_ATOMIC_STORE_ULONG(*(unsigned long *)addr, (unsigned long)long_val); + FT_ATOMIC_STORE_ULONG_RELAXED(*(unsigned long *)addr, (unsigned long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -280,12 +280,12 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; } - FT_ATOMIC_STORE_ULONG(*(unsigned long *)addr, ulong_val); + FT_ATOMIC_STORE_ULONG_RELAXED(*(unsigned long *)addr, ulong_val); } break; } case Py_T_PYSSIZET:{ - FT_ATOMIC_STORE_SSIZE_RELEASE(*(Py_ssize_t*)addr, PyLong_AsSsize_t(v)); + FT_ATOMIC_STORE_SSIZE_RELAXED(*(Py_ssize_t*)addr, PyLong_AsSsize_t(v)); if ((*(Py_ssize_t*)addr == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; @@ -295,11 +295,11 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) double double_val = PyFloat_AsDouble(v); if ((double_val == -1) && PyErr_Occurred()) return -1; - FT_ATOMIC_STORE_FLOAT_RELEASE(*(float*)addr, (float)double_val); + FT_ATOMIC_STORE_FLOAT_RELAXED(*(float*)addr, (float)double_val); break; } case Py_T_DOUBLE: - FT_ATOMIC_STORE_DOUBLE_RELEASE(*(double*)addr, PyFloat_AsDouble(v)); + FT_ATOMIC_STORE_DOUBLE_RELAXED(*(double*)addr, PyFloat_AsDouble(v)); if ((*(double*)addr == -1) && PyErr_Occurred()) return -1; break; @@ -329,7 +329,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) return -1; case Py_T_LONGLONG:{ long long value = PyLong_AsLongLong(v); - FT_ATOMIC_STORE_LLONG_RELEASE(*(long long*)addr, value); + FT_ATOMIC_STORE_LLONG_RELAXED(*(long long*)addr, value); if ((value == -1) && PyErr_Occurred()) return -1; break; @@ -345,7 +345,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - FT_ATOMIC_STORE_ULLONG_RELEASE(*(unsigned long long *)addr, (unsigned long long)(long long)long_val); + FT_ATOMIC_STORE_ULLONG_RELAXED(*(unsigned long long *)addr, (unsigned long long)(long long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -354,7 +354,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulonglong_val == (unsigned long long)-1 && PyErr_Occurred()) { return -1; } - FT_ATOMIC_STORE_ULLONG_RELEASE(*(unsigned long long *)addr, ulonglong_val); + FT_ATOMIC_STORE_ULLONG_RELAXED(*(unsigned long long *)addr, ulonglong_val); } break; } From b3b07d50280869c998cdea84b5b4c39d9626085e Mon Sep 17 00:00:00 2001 From: dpdani Date: Thu, 29 Aug 2024 14:28:43 +0200 Subject: [PATCH 28/31] default build --- .../internal/pycore_pyatomic_ft_wrappers.h | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index f7b81f757b33e8..d755d03a5fa190 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -133,29 +133,29 @@ extern "C" { #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value -#define FT_ATOMIC_STORE_CHAR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value -#define FT_ATOMIC_STORE_UCHAR_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UCHAR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_SHORT_RELAXED(value) value -#define FT_ATOMIC_STORE_SHORT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) value -#define FT_ATOMIC_STORE_USHORT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_INT_RELAXED(value) value -#define FT_ATOMIC_STORE_INT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UINT_RELAXED(value) value -#define FT_ATOMIC_STORE_UINT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_LONG_RELAXED(value) value -#define FT_ATOMIC_STORE_LONG_RELEASE(value, new_value) value = new_value -#define FT_ATOMIC_STORE_ULONG(value, new_value) value = new_value -#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_LONG_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_ULONG_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) value -#define FT_ATOMIC_STORE_FLOAT_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_FLOAT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) value -#define FT_ATOMIC_STORE_DOUBLE_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_DOUBLE_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_LLONG_RELAXED(value) value -#define FT_ATOMIC_STORE_LLONG_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_LLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value -#define FT_ATOMIC_STORE_ULLONG_RELEASE(value, new_value) value = new_value +#define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #endif From 761257edb63ece49135aaf29880db85d2dbd0827 Mon Sep 17 00:00:00 2001 From: Daniele Parmeggiani <8658291+dpdani@users.noreply.github.com> Date: Sun, 22 Sep 2024 12:02:52 +0200 Subject: [PATCH 29/31] Apply suggestions from code review Co-authored-by: Sam Gross --- Include/cpython/pyatomic_msc.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index a7600114a64aef..d155955df0cddf 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -956,25 +956,25 @@ _Py_atomic_store_char_relaxed(char *obj, char value) static inline void _Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) { - *(volatile unsigned char*)obj = value; + *(volatile unsigned char *)obj = value; } static inline void _Py_atomic_store_short_relaxed(short *obj, short value) { - *(volatile short*)obj = value; + *(volatile short *)obj = value; } static inline void _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) { - *(volatile unsigned short*)obj = value; + *(volatile unsigned short *)obj = value; } static inline void _Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) { - *(volatile unsigned int*)obj = value; + *(volatile unsigned int *)obj = value; } static inline void @@ -998,7 +998,7 @@ _Py_atomic_store_double_relaxed(double *obj, double value) static inline void _Py_atomic_store_llong_relaxed(long long *obj, long long value) { - *(volatile long long*)obj = value; + *(volatile long long *)obj = value; } From d045747320ecc5da41c8f05db8b198e1f1650374 Mon Sep 17 00:00:00 2001 From: dpdani Date: Sun, 22 Sep 2024 12:07:26 +0200 Subject: [PATCH 30/31] missing load --- Python/structmember.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/structmember.c b/Python/structmember.c index 28b1a2597e3f66..7097593e5301b3 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -80,7 +80,7 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyUnicode_FromString((char*)addr); break; case Py_T_CHAR: { - char value = 0; + char value = FT_ATOMIC_LOAD_CHAR_RELAXED(*addr); v = PyUnicode_FromStringAndSize(&value, 1); break; } From c1b4c358838d50a8f407a37c805a8982a5334453 Mon Sep 17 00:00:00 2001 From: dpdani Date: Sat, 2 Nov 2024 18:19:01 +0100 Subject: [PATCH 31/31] fixes --- Lib/test/test_capi/test_structmembers.py | 13 +++++++++ Modules/_testcapi/structmember.c | 4 +-- Python/structmember.c | 35 +++++++++++++----------- 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/Lib/test/test_capi/test_structmembers.py b/Lib/test/test_capi/test_structmembers.py index 6b27dc512a7d15..ae9168fc39243f 100644 --- a/Lib/test/test_capi/test_structmembers.py +++ b/Lib/test/test_capi/test_structmembers.py @@ -37,6 +37,9 @@ def _make_test_object(cls): 9.99999,# T_FLOAT 10.1010101010, # T_DOUBLE "hi", # T_STRING_INPLACE + 12, # T_LONGLONG + 13, # T_ULONGLONG + "c", # T_CHAR ) @@ -162,6 +165,16 @@ def test_inplace_string(self): self.assertRaises(TypeError, setattr, ts, "T_STRING_INPLACE", "s") self.assertRaises(TypeError, delattr, ts, "T_STRING_INPLACE") + def test_char(self): + ts = self.ts + self.assertEqual(ts.T_CHAR, "c") + ts.T_CHAR = "z" + self.assertEqual(ts.T_CHAR, "z") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", "") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", b"a") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", bytearray(b"b")) + self.assertRaises(TypeError, delattr, ts, "T_STRING_INPLACE") + class ReadWriteTests_OldAPI(ReadWriteTests, unittest.TestCase): cls = _test_structmembersType_OldAPI diff --git a/Modules/_testcapi/structmember.c b/Modules/_testcapi/structmember.c index f136a25e316d46..c1861db18c4af2 100644 --- a/Modules/_testcapi/structmember.c +++ b/Modules/_testcapi/structmember.c @@ -60,7 +60,7 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) "T_FLOAT", "T_DOUBLE", "T_STRING_INPLACE", "T_LONGLONG", "T_ULONGLONG", "T_CHAR", NULL}; - static const char fmt[] = "|bbBhHiIlknfds#LK"; + static const char fmt[] = "|bbBhHiIlknfds#LKC"; test_structmembers *ob; const char *s = NULL; Py_ssize_t string_len = 0; @@ -135,7 +135,7 @@ static struct PyMemberDef test_members[] = { {"T_STRING_INPLACE", T_STRING_INPLACE, offsetof(test_structmembers, structmembers.inplace_member), 0, NULL}, {"T_LONGLONG", T_LONGLONG, offsetof(test_structmembers, structmembers.longlong_member), 0, NULL}, {"T_ULONGLONG", T_ULONGLONG, offsetof(test_structmembers, structmembers.ulonglong_member), 0, NULL}, - {"T_CHAR", Py_T_CHAR, offsetof(test_structmembers, structmembers.char_member), 0, NULL}, + {"T_CHAR", T_CHAR, offsetof(test_structmembers, structmembers.char_member), 0, NULL}, {NULL} }; diff --git a/Python/structmember.c b/Python/structmember.c index 7097593e5301b3..d36e049d6b5d20 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -80,8 +80,8 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) v = PyUnicode_FromString((char*)addr); break; case Py_T_CHAR: { - char value = FT_ATOMIC_LOAD_CHAR_RELAXED(*addr); - v = PyUnicode_FromStringAndSize(&value, 1); + char char_val = FT_ATOMIC_LOAD_CHAR_RELAXED(*addr); + v = PyUnicode_FromStringAndSize(&char_val, 1); break; } case _Py_T_OBJECT: @@ -252,12 +252,13 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) } break; } - case Py_T_LONG:{ - FT_ATOMIC_STORE_LONG_RELAXED(*(long*)addr, PyLong_AsLong(v)); - if ((*(long*)addr == -1) && PyErr_Occurred()) + case Py_T_LONG: { + const long long_val = PyLong_AsLong(v); + if ((long_val == -1) && PyErr_Occurred()) return -1; + FT_ATOMIC_STORE_LONG_RELAXED(*(long*)addr, long_val); break; - } + } case Py_T_ULONG: { /* XXX: For compatibility, accept negative int values as well. */ @@ -284,13 +285,13 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) } break; } - case Py_T_PYSSIZET:{ - FT_ATOMIC_STORE_SSIZE_RELAXED(*(Py_ssize_t*)addr, PyLong_AsSsize_t(v)); - if ((*(Py_ssize_t*)addr == (Py_ssize_t)-1) - && PyErr_Occurred()) - return -1; + case Py_T_PYSSIZET: { + const Py_ssize_t ssize_val = PyLong_AsSsize_t(v); + if ((ssize_val == (Py_ssize_t)-1) && PyErr_Occurred()) + return -1; + FT_ATOMIC_STORE_SSIZE_RELAXED(*(Py_ssize_t*)addr, ssize_val); break; - } + } case Py_T_FLOAT:{ double double_val = PyFloat_AsDouble(v); if ((double_val == -1) && PyErr_Occurred()) @@ -298,11 +299,13 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) FT_ATOMIC_STORE_FLOAT_RELAXED(*(float*)addr, (float)double_val); break; } - case Py_T_DOUBLE: - FT_ATOMIC_STORE_DOUBLE_RELAXED(*(double*)addr, PyFloat_AsDouble(v)); - if ((*(double*)addr == -1) && PyErr_Occurred()) + case Py_T_DOUBLE: { + const double double_val = PyFloat_AsDouble(v); + if ((double_val == -1) && PyErr_Occurred()) return -1; + FT_ATOMIC_STORE_DOUBLE_RELAXED(*(double *) addr, double_val); break; + } case _Py_T_OBJECT: case Py_T_OBJECT_EX: Py_BEGIN_CRITICAL_SECTION(obj); @@ -329,9 +332,9 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) return -1; case Py_T_LONGLONG:{ long long value = PyLong_AsLongLong(v); - FT_ATOMIC_STORE_LLONG_RELAXED(*(long long*)addr, value); if ((value == -1) && PyErr_Occurred()) return -1; + FT_ATOMIC_STORE_LLONG_RELAXED(*(long long*)addr, value); break; } case Py_T_ULONGLONG: { pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy