Skip to content

POC: Explicit memory ordering #13974

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed

POC: Explicit memory ordering #13974

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
df0fd63
Work towards macro-based atomic definitions
jhawthorn Jul 19, 2025
6c1f6cb
Implement VALUE and ptr load explicit
jhawthorn Jul 19, 2025
77d3cca
Rename rbimpl_atomic.*_set to _store
jhawthorn Jul 19, 2025
65f1963
More memory ordering
jhawthorn Jul 19, 2025
e76f0c7
More work on atomics
jhawthorn Jul 19, 2025
dd1de89
Make all stores in concurrent_set explicit
jhawthorn Jul 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 14 additions & 14 deletions concurrent_set.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,15 +97,15 @@ static void
concurrent_set_try_resize_without_locking(VALUE old_set_obj, VALUE *set_obj_ptr)
{
// Check if another thread has already resized.
if (RUBY_ATOMIC_VALUE_LOAD(*set_obj_ptr) != old_set_obj) {
if (rbimpl_atomic_value_load_explicit(set_obj_ptr, RUBY_ATOMIC_ACQUIRE) != old_set_obj) {
return;
}

struct concurrent_set *old_set = RTYPEDDATA_GET_DATA(old_set_obj);

// This may overcount by up to the number of threads concurrently attempting to insert
// GC may also happen between now and the set being rebuilt
int expected_size = RUBY_ATOMIC_LOAD(old_set->size) - old_set->deleted_entries;
int expected_size = rbimpl_atomic_load_explicit(&old_set->size, RUBY_ATOMIC_RELAXED) - old_set->deleted_entries;

struct concurrent_set_entry *old_entries = old_set->entries;
int old_capacity = old_set->capacity;
Expand All @@ -123,13 +123,13 @@ concurrent_set_try_resize_without_locking(VALUE old_set_obj, VALUE *set_obj_ptr)

for (int i = 0; i < old_capacity; i++) {
struct concurrent_set_entry *entry = &old_entries[i];
VALUE key = RUBY_ATOMIC_VALUE_EXCHANGE(entry->key, CONCURRENT_SET_MOVED);
VALUE key = rbimpl_atomic_value_exchange_explicit(&entry->key, CONCURRENT_SET_MOVED, RUBY_ATOMIC_ACQUIRE);
RUBY_ASSERT(key != CONCURRENT_SET_MOVED);

if (key < CONCURRENT_SET_SPECIAL_VALUE_COUNT) continue;
if (!RB_SPECIAL_CONST_P(key) && rb_objspace_garbage_object_p(key)) continue;

VALUE hash = RUBY_ATOMIC_VALUE_LOAD(entry->hash);
VALUE hash = rbimpl_atomic_value_load_explicit(&entry->hash, RUBY_ATOMIC_RELAXED);
if (hash == 0) {
// Either in-progress insert or extremely unlikely 0 hash.
// Re-calculate the hash.
Expand Down Expand Up @@ -162,7 +162,7 @@ concurrent_set_try_resize_without_locking(VALUE old_set_obj, VALUE *set_obj_ptr)
}
}

RUBY_ATOMIC_VALUE_SET(*set_obj_ptr, new_set_obj);
rbimpl_atomic_value_store_explicit(set_obj_ptr, new_set_obj, RUBY_ATOMIC_RELEASE);

RB_GC_GUARD(old_set_obj);
}
Expand All @@ -184,7 +184,7 @@ rb_concurrent_set_find(VALUE *set_obj_ptr, VALUE key)
VALUE hash = 0;

retry:
set_obj = RUBY_ATOMIC_VALUE_LOAD(*set_obj_ptr);
set_obj = rbimpl_atomic_value_load_explicit(set_obj_ptr, RUBY_ATOMIC_ACQUIRE);
RUBY_ASSERT(set_obj);
struct concurrent_set *set = RTYPEDDATA_GET_DATA(set_obj);

Expand Down Expand Up @@ -263,7 +263,7 @@ rb_concurrent_set_find_or_insert(VALUE *set_obj_ptr, VALUE key, void *data)

while (true) {
struct concurrent_set_entry *entry = &set->entries[idx];
VALUE curr_key = RUBY_ATOMIC_VALUE_LOAD(entry->key);
VALUE curr_key = rbimpl_atomic_value_load_explicit(&entry->key, RUBY_ATOMIC_ACQUIRE);

switch (curr_key) {
case CONCURRENT_SET_EMPTY: {
Expand All @@ -274,24 +274,24 @@ rb_concurrent_set_find_or_insert(VALUE *set_obj_ptr, VALUE key, void *data)
inserting = true;
}

rb_atomic_t prev_size = RUBY_ATOMIC_FETCH_ADD(set->size, 1);
rb_atomic_t prev_size = rbimpl_atomic_fetch_add_explicit(&set->size, 1, RUBY_ATOMIC_RELAXED);

if (UNLIKELY(prev_size > set->capacity / 2)) {
concurrent_set_try_resize(set_obj, set_obj_ptr);

goto retry;
}

curr_key = RUBY_ATOMIC_VALUE_CAS(entry->key, CONCURRENT_SET_EMPTY, key);
curr_key = rbimpl_atomic_value_cas_explicit(&entry->key, CONCURRENT_SET_EMPTY, key, RUBY_ATOMIC_RELEASE, RUBY_ATOMIC_RELAXED);
if (curr_key == CONCURRENT_SET_EMPTY) {
RUBY_ATOMIC_VALUE_SET(entry->hash, hash);
rbimpl_atomic_value_store_explicit(&entry->hash, hash, RUBY_ATOMIC_RELAXED);

RB_GC_GUARD(set_obj);
return key;
}
else {
// Entry was not inserted.
RUBY_ATOMIC_DEC(set->size);
rbimpl_atomic_sub_explicit(&set->size, 1, RUBY_ATOMIC_RELAXED);

// Another thread won the race, try again at the same location.
continue;
Expand All @@ -305,13 +305,13 @@ rb_concurrent_set_find_or_insert(VALUE *set_obj_ptr, VALUE key, void *data)

goto retry;
default: {
VALUE curr_hash = RUBY_ATOMIC_VALUE_LOAD(entry->hash);
VALUE curr_hash = rbimpl_atomic_value_load_explicit(&entry->hash, RUBY_ATOMIC_RELAXED);
if (curr_hash != 0 && curr_hash != hash) break;

if (UNLIKELY(!RB_SPECIAL_CONST_P(curr_key) && rb_objspace_garbage_object_p(curr_key))) {
// This is a weakref set, so after marking but before sweeping is complete we may find a matching garbage object.
// Skip it and mark it as deleted.
RUBY_ATOMIC_VALUE_CAS(entry->key, curr_key, CONCURRENT_SET_DELETED);
rbimpl_atomic_value_cas_explicit(&entry->key, curr_key, CONCURRENT_SET_DELETED, RUBY_ATOMIC_RELEASE, RUBY_ATOMIC_RELAXED);
break;
}

Expand Down Expand Up @@ -352,7 +352,7 @@ rb_concurrent_set_delete_by_identity(VALUE set_obj, VALUE key)

while (true) {
struct concurrent_set_entry *entry = &set->entries[idx];
VALUE curr_key = RUBY_ATOMIC_VALUE_LOAD(entry->key);
VALUE curr_key = entry->key;

switch (curr_key) {
case CONCURRENT_SET_EMPTY:
Expand Down
18 changes: 9 additions & 9 deletions include/ruby/atomic.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ typedef unsigned int rb_atomic_t;
* @return void
* @post `var` holds `val`.
*/
#define RUBY_ATOMIC_SET(var, val) rbimpl_atomic_set(&(var), (val))
#define RUBY_ATOMIC_SET(var, val) rbimpl_atomic_store(&(var), (val))

/**
* Identical to #RUBY_ATOMIC_FETCH_ADD, except for the return type.
Expand Down Expand Up @@ -327,7 +327,7 @@ typedef unsigned int rb_atomic_t;
* @post `var` holds `val`.
*/
#define RUBY_ATOMIC_PTR_SET(var, val) \
rbimpl_atomic_ptr_set((volatile void **)&(var), (val))
rbimpl_atomic_ptr_store((volatile void **)&(var), (val))

/**
* Identical to #RUBY_ATOMIC_CAS, except it expects its arguments are `void*`.
Expand All @@ -354,7 +354,7 @@ typedef unsigned int rb_atomic_t;
* @post `var` holds `val`.
*/
#define RUBY_ATOMIC_VALUE_SET(var, val) \
rbimpl_atomic_value_set(&(var), (val))
rbimpl_atomic_value_store(&(var), (val))

/**
* Identical to #RUBY_ATOMIC_EXCHANGE, except it expects its arguments are
Expand Down Expand Up @@ -859,7 +859,7 @@ RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_size_set(volatile size_t *ptr, size_t val)
rbimpl_atomic_size_store(volatile size_t *ptr, size_t val)
{
#if 0

Expand Down Expand Up @@ -904,13 +904,13 @@ RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_ptr_set(volatile void **ptr, void *val)
rbimpl_atomic_ptr_store(volatile void **ptr, void *val)
{
RBIMPL_STATIC_ASSERT(sizeof_value, sizeof *ptr == sizeof(size_t));

const size_t sval = RBIMPL_CAST((size_t)val);
volatile size_t *const sptr = RBIMPL_CAST((volatile size_t *)ptr);
rbimpl_atomic_size_set(sptr, sval);
rbimpl_atomic_size_store(sptr, sval);
}

RBIMPL_ATTR_ARTIFICIAL()
Expand All @@ -931,13 +931,13 @@ RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_value_set(volatile VALUE *ptr, VALUE val)
rbimpl_atomic_value_store(volatile VALUE *ptr, VALUE val)
{
RBIMPL_STATIC_ASSERT(sizeof_value, sizeof *ptr == sizeof(size_t));

const size_t sval = RBIMPL_CAST((size_t)val);
volatile size_t *const sptr = RBIMPL_CAST((volatile size_t *)ptr);
rbimpl_atomic_size_set(sptr, sval);
rbimpl_atomic_size_store(sptr, sval);
}

RBIMPL_ATTR_ARTIFICIAL()
Expand All @@ -959,7 +959,7 @@ RBIMPL_ATTR_ARTIFICIAL()
RBIMPL_ATTR_NOALIAS()
RBIMPL_ATTR_NONNULL((1))
static inline void
rbimpl_atomic_set(volatile rb_atomic_t *ptr, rb_atomic_t val)
rbimpl_atomic_store(volatile rb_atomic_t *ptr, rb_atomic_t val)
{
#if 0

Expand Down
125 changes: 119 additions & 6 deletions ruby_atomic.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,28 @@

#include "ruby/atomic.h"

#if defined(HAVE_GCC_ATOMIC_BUILTINS)
#define RUBY_ATOMIC_RELAXED __ATOMIC_RELAXED
#define RUBY_ATOMIC_ACQUIRE __ATOMIC_ACQUIRE
#define RUBY_ATOMIC_RELEASE __ATOMIC_RELEASE
#define RUBY_ATOMIC_ACQ_REL __ATOMIC_ACQ_REL
#define RUBY_ATOMIC_SEQ_CST __ATOMIC_SEQ_CST
#elif defined(HAVE_STDATOMIC_H)
#include <stdatomic.h>
#define RUBY_ATOMIC_RELAXED memory_order_relaxed
#define RUBY_ATOMIC_ACQUIRE memory_order_acquire
#define RUBY_ATOMIC_RELEASE memory_order_release
#define RUBY_ATOMIC_ACQ_REL memory_order_acq_rel
#define RUBY_ATOMIC_SEQ_CST memory_order_seq_cst
#else
/* Dummy values for unsupported platforms */
#define RUBY_ATOMIC_RELAXED 0
#define RUBY_ATOMIC_ACQUIRE 1
#define RUBY_ATOMIC_RELEASE 2
#define RUBY_ATOMIC_ACQ_REL 3
#define RUBY_ATOMIC_SEQ_CST 4
#endif

#define RUBY_ATOMIC_VALUE_LOAD(x) (VALUE)(RUBY_ATOMIC_PTR_LOAD(x))

/* shim macros only */
Expand All @@ -27,16 +49,107 @@
#define ATOMIC_VALUE_CAS(var, oldval, val) RUBY_ATOMIC_VALUE_CAS(var, oldval, val)
#define ATOMIC_VALUE_EXCHANGE(var, val) RUBY_ATOMIC_VALUE_EXCHANGE(var, val)

static inline rb_atomic_t
rbimpl_atomic_load_relaxed(volatile rb_atomic_t *ptr)
{
/**********************************/

/* Platform-specific implementation (or fallback) */
#if defined(HAVE_GCC_ATOMIC_BUILTINS)
return __atomic_load_n(ptr, __ATOMIC_RELAXED);
#define DEFINE_ATOMIC_LOAD_EXPLICIT_BODY(ptr, memory_order, type, type_prefix) \
__atomic_load_n(ptr, memory_order)
#elif defined(HAVE_STDATOMIC_H)
#define DEFINE_ATOMIC_LOAD_EXPLICIT_BODY(ptr, memory_order, type, type_prefix) \
atomic_load_explicit((_Atomic volatile type *)ptr, memory_order)
#else
return *ptr;
#define DEFINE_ATOMIC_LOAD_EXPLICIT_BODY(ptr, memory_order, type, type_prefix) \
((void)memory_order, rbimpl_atomic_##type_prefix##load(ptr))
#endif

/* Single macro definition for load operations with explicit memory ordering */
#define DEFINE_ATOMIC_LOAD_EXPLICIT(type_prefix, type) \
static inline type \
rbimpl_atomic_##type_prefix##load_explicit(type *ptr, int memory_order) \
{ \
return DEFINE_ATOMIC_LOAD_EXPLICIT_BODY(ptr, memory_order, type, type_prefix); \
}
#define ATOMIC_LOAD_RELAXED(var) rbimpl_atomic_load_relaxed(&(var))

/* Generate atomic load function with explicit memory ordering */
DEFINE_ATOMIC_LOAD_EXPLICIT(, rb_atomic_t)
DEFINE_ATOMIC_LOAD_EXPLICIT(value_, VALUE)
DEFINE_ATOMIC_LOAD_EXPLICIT(ptr_, void *)

#undef DEFINE_ATOMIC_LOAD_EXPLICIT
#undef DEFINE_ATOMIC_LOAD_EXPLICIT_BODY

/* Define the body for two-operand atomic operations based on platform */
#if defined(HAVE_GCC_ATOMIC_BUILTINS)
#define ATOMIC_OP_EXPLICIT_BODY(gcc_op, stdatomic_op, ptr, val, memory_order, type, type_prefix) \
__atomic_##gcc_op(ptr, val, memory_order)
#elif defined(HAVE_STDATOMIC_H)
#define ATOMIC_OP_EXPLICIT_BODY(gcc_op, stdatomic_op, ptr, val, memory_order, type, type_prefix) \
atomic_##stdatomic_op##_explicit((_Atomic volatile type *)ptr, val, memory_order)
#else
#define ATOMIC_OP_EXPLICIT_BODY(gcc_op, stdatomic_op, ptr, val, memory_order, type, type_prefix) \
((void)memory_order, rbimpl_atomic_##type_prefix##stdatomic_op(ptr, val))
#endif

/* Generic macro for two-operand atomic operations with explicit memory ordering */
#define DEFINE_ATOMIC_OP_EXPLICIT(gcc_op, stdatomic_op, type_prefix, type) \
static inline type \
rbimpl_atomic_##type_prefix##stdatomic_op##_explicit(volatile type *ptr, type val, int memory_order) \
{ \
return ATOMIC_OP_EXPLICIT_BODY(gcc_op, stdatomic_op, ptr, val, memory_order, type, type_prefix); \
}

/* Special case for store operations that don't return values */
#define DEFINE_ATOMIC_STORE_EXPLICIT(gcc_op, stdatomic_op, type_prefix, type) \
static inline void \
rbimpl_atomic_##type_prefix##stdatomic_op##_explicit(volatile type *ptr, type val, int memory_order) \
{ \
ATOMIC_OP_EXPLICIT_BODY(gcc_op, stdatomic_op, ptr, val, memory_order, type, type_prefix); \
}

/* Generate atomic operations with explicit memory ordering */
DEFINE_ATOMIC_STORE_EXPLICIT(store_n, store, , rb_atomic_t)
DEFINE_ATOMIC_OP_EXPLICIT(add_fetch, add, , rb_atomic_t)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think there's an atomic_add_explicit() in <stdatomic.h> (https://en.cppreference.com/w/c/header/stdatomic.html)

DEFINE_ATOMIC_STORE_EXPLICIT(store_n, store, value_, VALUE)
DEFINE_ATOMIC_OP_EXPLICIT(sub_fetch, sub, , rb_atomic_t)
DEFINE_ATOMIC_OP_EXPLICIT(fetch_add, fetch_add, , rb_atomic_t)
DEFINE_ATOMIC_OP_EXPLICIT(exchange_n, exchange, value_, VALUE)

/* Define the body for CAS operations based on platform */
#if defined(HAVE_GCC_ATOMIC_BUILTINS)
#define ATOMIC_CAS_EXPLICIT_BODY(ptr, oldval, newval, success_memorder, failure_memorder, type, type_prefix) \
type expected = oldval; \
__atomic_compare_exchange_n(ptr, &expected, newval, 0, success_memorder, failure_memorder); \
return expected
#elif defined(HAVE_STDATOMIC_H)
#define ATOMIC_CAS_EXPLICIT_BODY(ptr, oldval, newval, success_memorder, failure_memorder, type, type_prefix) \
type expected = oldval; \
atomic_compare_exchange_strong_explicit((_Atomic volatile type *)ptr, &expected, newval, success_memorder, failure_memorder); \
return expected
#else
#define ATOMIC_CAS_EXPLICIT_BODY(ptr, oldval, newval, success_memorder, failure_memorder, type, type_prefix) \
(void)success_memorder; (void)failure_memorder; \
return rbimpl_atomic_##type_prefix##cas(ptr, oldval, newval)
#endif

/* Generic macro for CAS operations with explicit memory ordering */
#define DEFINE_ATOMIC_CAS_EXPLICIT(type_prefix, type) \
static inline type \
rbimpl_atomic_##type_prefix##cas_explicit(volatile type *ptr, type oldval, type newval, int success_memorder, int failure_memorder) \
{ \
ATOMIC_CAS_EXPLICIT_BODY(ptr, oldval, newval, success_memorder, failure_memorder, type, type_prefix); \
}

/* Generate CAS operations with explicit memory ordering */
DEFINE_ATOMIC_CAS_EXPLICIT(value_, VALUE)

#undef DEFINE_ATOMIC_OP_EXPLICIT
#undef DEFINE_ATOMIC_STORE_EXPLICIT
#undef DEFINE_ATOMIC_CAS_EXPLICIT
#undef ATOMIC_OP_EXPLICIT_BODY
#undef ATOMIC_CAS_EXPLICIT_BODY

#define ATOMIC_LOAD_RELAXED(var) rbimpl_atomic_load_explicit(&(var), RUBY_ATOMIC_RELAXED)

typedef RBIMPL_ALIGNAS(8) uint64_t rbimpl_atomic_uint64_t;

Expand Down
Loading
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy