Skip to content

Commit e8fdbd5

Browse files
anarazelanarazel
committed
Improve 64bit atomics support.
When adding atomics back in b64d92f, I added 64bit support as optional; there wasn't yet a direct user in sight. That turned out to be a bit short-sighted, it'd already have been useful a number of times. Add a fallback implementation of 64bit atomics, just like the one we have for 32bit atomics. Additionally optimize reads/writes to 64bit on a number of platforms where aligned writes of that size are atomic. This can now be tested with PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY. Author: Andres Freund Reviewed-By: Amit Kapila Discussion: https://postgr.es/m/20160330230914.GH13305@awork2.anarazel.de
1 parent 28afad5 commit e8fdbd5

File tree

8 files changed

+136
-17
lines changed

8 files changed

+136
-17
lines changed

‎src/backend/port/atomics.c

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ void
8989
pg_atomic_init_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 val_)
9090
{
9191
StaticAssertStmt(sizeof(ptr->sema) >= sizeof(slock_t),
92-
"size mismatch of atomic_flag vs slock_t");
92+
"size mismatch of atomic_uint32 vs slock_t");
9393

9494
/*
9595
* If we're using semaphore based atomic flags, be careful about nested
@@ -157,3 +157,66 @@ pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_)
157157
}
158158

159159
#endif /* PG_HAVE_ATOMIC_U32_SIMULATION */
160+
161+
162+
#ifdef PG_HAVE_ATOMIC_U64_SIMULATION
163+
164+
void
165+
pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_)
166+
{
167+
StaticAssertStmt(sizeof(ptr->sema) >= sizeof(slock_t),
168+
"size mismatch of atomic_uint64 vs slock_t");
169+
170+
/*
171+
* If we're using semaphore based atomic flags, be careful about nested
172+
* usage of atomics while a spinlock is held.
173+
*/
174+
#ifndef HAVE_SPINLOCKS
175+
s_init_lock_sema((slock_t *) &ptr->sema, true);
176+
#else
177+
SpinLockInit((slock_t *) &ptr->sema);
178+
#endif
179+
ptr->value = val_;
180+
}
181+
182+
bool
183+
pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
184+
uint64 *expected, uint64 newval)
185+
{
186+
bool ret;
187+
188+
/*
189+
* Do atomic op under a spinlock. It might look like we could just skip
190+
* the cmpxchg if the lock isn't available, but that'd just emulate a
191+
* 'weak' compare and swap. I.e. one that allows spurious failures. Since
192+
* several algorithms rely on a strong variant and that is efficiently
193+
* implementable on most major architectures let's emulate it here as
194+
* well.
195+
*/
196+
SpinLockAcquire((slock_t *) &ptr->sema);
197+
198+
/* perform compare/exchange logic */
199+
ret = ptr->value == *expected;
200+
*expected = ptr->value;
201+
if (ret)
202+
ptr->value = newval;
203+
204+
/* and release lock */
205+
SpinLockRelease((slock_t *) &ptr->sema);
206+
207+
return ret;
208+
}
209+
210+
uint64
211+
pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
212+
{
213+
uint64 oldval;
214+
215+
SpinLockAcquire((slock_t *) &ptr->sema);
216+
oldval = ptr->value;
217+
ptr->value += add_;
218+
SpinLockRelease((slock_t *) &ptr->sema);
219+
return oldval;
220+
}
221+
222+
#endif /* PG_HAVE_ATOMIC_U64_SIMULATION */

‎src/include/port/atomics.h

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,14 @@
1212
* * pg_compiler_barrier(), pg_write_barrier(), pg_read_barrier()
1313
* * pg_atomic_compare_exchange_u32(), pg_atomic_fetch_add_u32()
1414
* * pg_atomic_test_set_flag(), pg_atomic_init_flag(), pg_atomic_clear_flag()
15+
* * PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY should be defined if appropriate.
1516
*
1617
* There exist generic, hardware independent, implementations for several
1718
* compilers which might be sufficient, although possibly not optimal, for a
1819
* new platform. If no such generic implementation is available spinlocks (or
1920
* even OS provided semaphores) will be used to implement the API.
2021
*
21-
* Implement the _u64 variants if and only if your platform can use them
22+
* Implement _u64 atomics if and only if your platform can use them
2223
* efficiently (and obviously correctly).
2324
*
2425
* Use higher level functionality (lwlocks, spinlocks, heavyweight locks)
@@ -110,9 +111,9 @@
110111

111112
/*
112113
* Provide a full fallback of the pg_*_barrier(), pg_atomic**_flag and
113-
* pg_atomic_*_u32 APIs for platforms without sufficient spinlock and/or
114-
* atomics support. In the case of spinlock backed atomics the emulation is
115-
* expected to be efficient, although less so than native atomics support.
114+
* pg_atomic_* APIs for platforms without sufficient spinlock and/or atomics
115+
* support. In the case of spinlock backed atomics the emulation is expected
116+
* to be efficient, although less so than native atomics support.
116117
*/
117118
#include "port/atomics/fallback.h"
118119

@@ -421,8 +422,6 @@ pg_atomic_sub_fetch_u32(volatile pg_atomic_uint32 *ptr, int32 sub_)
421422
* documentation.
422423
* ----
423424
*/
424-
#ifdef PG_HAVE_ATOMIC_U64_SUPPORT
425-
426425
static inline void
427426
pg_atomic_init_u64(volatile pg_atomic_uint64 *ptr, uint64 val)
428427
{
@@ -506,8 +505,6 @@ pg_atomic_sub_fetch_u64(volatile pg_atomic_uint64 *ptr, int64 sub_)
506505
return pg_atomic_sub_fetch_u64_impl(ptr, sub_);
507506
}
508507

509-
#endif /* PG_HAVE_64_BIT_ATOMICS */
510-
511508
#undef INSIDE_ATOMICS_H
512509

513510
#endif /* ATOMICS_H */

‎src/include/port/atomics/arch-ia64.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,6 @@
2424
#elif defined(__hpux)
2525
# define pg_memory_barrier_impl() _Asm_mf()
2626
#endif
27+
28+
/* per architecture manual doubleword accesses have single copy atomicity */
29+
#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY

‎src/include/port/atomics/arch-ppc.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,6 @@
2424
#define pg_read_barrier_impl() __asm__ __volatile__ ("lwsync" : : : "memory")
2525
#define pg_write_barrier_impl() __asm__ __volatile__ ("lwsync" : : : "memory")
2626
#endif
27+
28+
/* per architecture manual doubleword accesses have single copy atomicity */
29+
#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY

‎src/include/port/atomics/arch-x86.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,4 +239,14 @@ pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
239239

240240
#endif /* defined(__GNUC__) || defined(__INTEL_COMPILER) */
241241

242+
/*
243+
* 8 byte reads / writes have single-copy atomicity on 32 bit x86 platforms
244+
* since at least the 586. As well as on all x86-64 cpus.
245+
*/
246+
#if defined(__i568__) || defined(__i668__) || /* gcc i586+ */ \
247+
(defined(_M_IX86) && _M_IX86 >= 500) || /* msvc i586+ */ \
248+
defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) /* gcc, sunpro, msvc */
249+
#define PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY
250+
#endif /* 8 byte single-copy atomicity */
251+
242252
#endif /* HAVE_ATOMICS */

‎src/include/port/atomics/fallback.h

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,24 @@ typedef struct pg_atomic_uint32
102102

103103
#endif /* PG_HAVE_ATOMIC_U32_SUPPORT */
104104

105+
#if !defined(PG_HAVE_ATOMIC_U64_SUPPORT)
106+
107+
#define PG_HAVE_ATOMIC_U64_SIMULATION
108+
109+
#define PG_HAVE_ATOMIC_U64_SUPPORT
110+
typedef struct pg_atomic_uint64
111+
{
112+
/* Check pg_atomic_flag's definition above for an explanation */
113+
#if defined(__hppa) || defined(__hppa__) /* HP PA-RISC, GCC and HP compilers */
114+
int sema[4];
115+
#else
116+
int sema;
117+
#endif
118+
volatile uint64 value;
119+
} pg_atomic_uint64;
120+
121+
#endif /* PG_HAVE_ATOMIC_U64_SUPPORT */
122+
105123
#ifdef PG_HAVE_ATOMIC_FLAG_SIMULATION
106124

107125
#define PG_HAVE_ATOMIC_INIT_FLAG
@@ -144,3 +162,18 @@ extern bool pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr,
144162
extern uint32 pg_atomic_fetch_add_u32_impl(volatile pg_atomic_uint32 *ptr, int32 add_);
145163

146164
#endif /* PG_HAVE_ATOMIC_U32_SIMULATION */
165+
166+
167+
#ifdef PG_HAVE_ATOMIC_U64_SIMULATION
168+
169+
#define PG_HAVE_ATOMIC_INIT_U64
170+
extern void pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_);
171+
172+
#define PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64
173+
extern bool pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr,
174+
uint64 *expected, uint64 newval);
175+
176+
#define PG_HAVE_ATOMIC_FETCH_ADD_U64
177+
extern uint64 pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_);
178+
179+
#endif /* PG_HAVE_ATOMIC_U64_SIMULATION */

‎src/include/port/atomics/generic.h

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -255,8 +255,6 @@ pg_atomic_sub_fetch_u32_impl(volatile pg_atomic_uint32 *ptr, int32 sub_)
255255
}
256256
#endif
257257

258-
#ifdef PG_HAVE_ATOMIC_U64_SUPPORT
259-
260258
#if !defined(PG_HAVE_ATOMIC_EXCHANGE_U64) && defined(PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64)
261259
#define PG_HAVE_ATOMIC_EXCHANGE_U64
262260
static inline uint64
@@ -273,6 +271,24 @@ pg_atomic_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 xchg_)
273271
}
274272
#endif
275273

274+
#ifndef PG_HAVE_ATOMIC_READ_U64
275+
#define PG_HAVE_ATOMIC_READ_U64
276+
static inline uint64
277+
pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr)
278+
{
279+
return *(&ptr->value);
280+
}
281+
#endif
282+
283+
#ifndef PG_HAVE_ATOMIC_WRITE_U64
284+
#define PG_HAVE_ATOMIC_WRITE_U64
285+
static inline void
286+
pg_atomic_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val)
287+
{
288+
ptr->value = val;
289+
}
290+
#endif
291+
276292
#ifndef PG_HAVE_ATOMIC_WRITE_U64
277293
#define PG_HAVE_ATOMIC_WRITE_U64
278294
static inline void
@@ -388,5 +404,3 @@ pg_atomic_sub_fetch_u64_impl(volatile pg_atomic_uint64 *ptr, int64 sub_)
388404
return pg_atomic_fetch_sub_u64_impl(ptr, sub_) - sub_;
389405
}
390406
#endif
391-
392-
#endif /* PG_HAVE_ATOMIC_COMPARE_EXCHANGE_U64 */

‎src/test/regress/regress.c

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -997,7 +997,6 @@ test_atomic_uint32(void)
997997
elog(ERROR, "pg_atomic_fetch_and_u32() #3 wrong");
998998
}
999999

1000-
#ifdef PG_HAVE_ATOMIC_U64_SUPPORT
10011000
static void
10021001
test_atomic_uint64(void)
10031002
{
@@ -1073,7 +1072,6 @@ test_atomic_uint64(void)
10731072
if (pg_atomic_fetch_and_u64(&var, ~0) != 0)
10741073
elog(ERROR, "pg_atomic_fetch_and_u64() #3 wrong");
10751074
}
1076-
#endif /* PG_HAVE_ATOMIC_U64_SUPPORT */
10771075

10781076

10791077
PG_FUNCTION_INFO_V1(test_atomic_ops);
@@ -1096,9 +1094,7 @@ test_atomic_ops(PG_FUNCTION_ARGS)
10961094

10971095
test_atomic_uint32();
10981096

1099-
#ifdef PG_HAVE_ATOMIC_U64_SUPPORT
11001097
test_atomic_uint64();
1101-
#endif
11021098

11031099
PG_RETURN_BOOL(true);
11041100
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy