PostgreSQL Source Code git master
atomics.h File Reference
#include <limits.h>
#include "port/atomics/fallback.h"
#include "port/atomics/generic.h"
Include dependency graph for atomics.h:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros

#define INSIDE_ATOMICS_H
 
#define pg_compiler_barrier()   pg_compiler_barrier_impl()
 
#define pg_memory_barrier()   pg_memory_barrier_impl()
 
#define pg_read_barrier()   pg_read_barrier_impl()
 
#define pg_write_barrier()   pg_write_barrier_impl()
 
#define pg_spin_delay()   pg_spin_delay_impl()
 

Functions

static void pg_atomic_init_flag (volatile pg_atomic_flag *ptr)
 
static bool pg_atomic_test_set_flag (volatile pg_atomic_flag *ptr)
 
static bool pg_atomic_unlocked_test_flag (volatile pg_atomic_flag *ptr)
 
static void pg_atomic_clear_flag (volatile pg_atomic_flag *ptr)
 
static void pg_atomic_init_u32 (volatile pg_atomic_uint32 *ptr, uint32 val)
 
static uint32 pg_atomic_read_u32 (volatile pg_atomic_uint32 *ptr)
 
static uint32 pg_atomic_read_membarrier_u32 (volatile pg_atomic_uint32 *ptr)
 
static void pg_atomic_write_u32 (volatile pg_atomic_uint32 *ptr, uint32 val)
 
static void pg_atomic_unlocked_write_u32 (volatile pg_atomic_uint32 *ptr, uint32 val)
 
static void pg_atomic_write_membarrier_u32 (volatile pg_atomic_uint32 *ptr, uint32 val)
 
static uint32 pg_atomic_exchange_u32 (volatile pg_atomic_uint32 *ptr, uint32 newval)
 
static bool pg_atomic_compare_exchange_u32 (volatile pg_atomic_uint32 *ptr, uint32 *expected, uint32 newval)
 
static uint32 pg_atomic_fetch_add_u32 (volatile pg_atomic_uint32 *ptr, int32 add_)
 
static uint32 pg_atomic_fetch_sub_u32 (volatile pg_atomic_uint32 *ptr, int32 sub_)
 
static uint32 pg_atomic_fetch_and_u32 (volatile pg_atomic_uint32 *ptr, uint32 and_)
 
static uint32 pg_atomic_fetch_or_u32 (volatile pg_atomic_uint32 *ptr, uint32 or_)
 
static uint32 pg_atomic_add_fetch_u32 (volatile pg_atomic_uint32 *ptr, int32 add_)
 
static uint32 pg_atomic_sub_fetch_u32 (volatile pg_atomic_uint32 *ptr, int32 sub_)
 
static void pg_atomic_init_u64 (volatile pg_atomic_uint64 *ptr, uint64 val)
 
static uint64 pg_atomic_read_u64 (volatile pg_atomic_uint64 *ptr)
 
static uint64 pg_atomic_read_membarrier_u64 (volatile pg_atomic_uint64 *ptr)
 
static void pg_atomic_write_u64 (volatile pg_atomic_uint64 *ptr, uint64 val)
 
static void pg_atomic_unlocked_write_u64 (volatile pg_atomic_uint64 *ptr, uint64 val)
 
static void pg_atomic_write_membarrier_u64 (volatile pg_atomic_uint64 *ptr, uint64 val)
 
static uint64 pg_atomic_exchange_u64 (volatile pg_atomic_uint64 *ptr, uint64 newval)
 
static bool pg_atomic_compare_exchange_u64 (volatile pg_atomic_uint64 *ptr, uint64 *expected, uint64 newval)
 
static uint64 pg_atomic_fetch_add_u64 (volatile pg_atomic_uint64 *ptr, int64 add_)
 
static uint64 pg_atomic_fetch_sub_u64 (volatile pg_atomic_uint64 *ptr, int64 sub_)
 
static uint64 pg_atomic_fetch_and_u64 (volatile pg_atomic_uint64 *ptr, uint64 and_)
 
static uint64 pg_atomic_fetch_or_u64 (volatile pg_atomic_uint64 *ptr, uint64 or_)
 
static uint64 pg_atomic_add_fetch_u64 (volatile pg_atomic_uint64 *ptr, int64 add_)
 
static uint64 pg_atomic_sub_fetch_u64 (volatile pg_atomic_uint64 *ptr, int64 sub_)
 
static uint64 pg_atomic_monotonic_advance_u64 (volatile pg_atomic_uint64 *ptr, uint64 target)
 

Macro Definition Documentation

◆ INSIDE_ATOMICS_H

#define INSIDE_ATOMICS_H

Definition at line 45 of file atomics.h.

◆ pg_compiler_barrier

#define pg_compiler_barrier ( )    pg_compiler_barrier_impl()

Definition at line 129 of file atomics.h.

◆ pg_memory_barrier

#define pg_memory_barrier ( )    pg_memory_barrier_impl()

Definition at line 141 of file atomics.h.

◆ pg_read_barrier

#define pg_read_barrier ( )    pg_read_barrier_impl()

Definition at line 154 of file atomics.h.

◆ pg_spin_delay

#define pg_spin_delay ( )    pg_spin_delay_impl()

Definition at line 160 of file atomics.h.

◆ pg_write_barrier

#define pg_write_barrier ( )    pg_write_barrier_impl()

Definition at line 155 of file atomics.h.

Function Documentation

◆ pg_atomic_add_fetch_u32()

static uint32 pg_atomic_add_fetch_u32 ( volatile pg_atomic_uint32 ptr,
int32  add_ 
)
inlinestatic

Definition at line 422 of file atomics.h.

423{
425 return pg_atomic_add_fetch_u32_impl(ptr, add_);
426}
#define AssertPointerAlignment(ptr, bndr)
Definition: c.h:896

References AssertPointerAlignment.

Referenced by apply_handle_stream_abort(), apply_handle_stream_start(), compute_parallel_delay(), parallel_vacuum_process_safe_indexes(), parallel_vacuum_process_unsafe_indexes(), tbm_prepare_shared_iterate(), and test_atomic_uint32().

◆ pg_atomic_add_fetch_u64()

static uint64 pg_atomic_add_fetch_u64 ( volatile pg_atomic_uint64 ptr,
int64  add_ 
)
inlinestatic

Definition at line 567 of file atomics.h.

568{
569#ifndef PG_HAVE_ATOMIC_U64_SIMULATION
571#endif
572 return pg_atomic_add_fetch_u64_impl(ptr, add_);
573}

References AssertPointerAlignment.

Referenced by EmitProcSignalBarrier(), and test_atomic_uint64().

◆ pg_atomic_clear_flag()

static void pg_atomic_clear_flag ( volatile pg_atomic_flag *  ptr)
inlinestatic

Definition at line 205 of file atomics.h.

206{
207 pg_atomic_clear_flag_impl(ptr);
208}

Referenced by do_autovacuum(), FreeWorkerInfo(), and test_atomic_flag().

◆ pg_atomic_compare_exchange_u32()

static bool pg_atomic_compare_exchange_u32 ( volatile pg_atomic_uint32 ptr,
uint32 expected,
uint32  newval 
)
inlinestatic

Definition at line 347 of file atomics.h.

349{
351 AssertPointerAlignment(expected, 4);
352
353 return pg_atomic_compare_exchange_u32_impl(ptr, expected, newval);
354}
static bool pg_atomic_compare_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 *expected, uint32 newval)
Definition: arch-ppc.h:80
#define newval

References AssertPointerAlignment, newval, and pg_atomic_compare_exchange_u32_impl().

Referenced by ClockSweepTick(), GetBufferFromRing(), LWLockAttemptLock(), LWLockWakeup(), MarkBufferDirty(), PinBuffer(), ProcArrayGroupClearXid(), StrategyGetBuffer(), test_atomic_uint32(), TransactionGroupUpdateXidStatus(), and UnlockBufHdrExt().

◆ pg_atomic_compare_exchange_u64()

static bool pg_atomic_compare_exchange_u64 ( volatile pg_atomic_uint64 ptr,
uint64 expected,
uint64  newval 
)
inlinestatic

Definition at line 520 of file atomics.h.

522{
523#ifndef PG_HAVE_ATOMIC_U64_SIMULATION
525#endif
526 return pg_atomic_compare_exchange_u64_impl(ptr, expected, newval);
527}
bool pg_atomic_compare_exchange_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 *expected, uint64 newval)
Definition: atomics.c:34

References AssertPointerAlignment, newval, and pg_atomic_compare_exchange_u64_impl().

Referenced by pg_atomic_monotonic_advance_u64(), and test_atomic_uint64().

◆ pg_atomic_exchange_u32()

static uint32 pg_atomic_exchange_u32 ( volatile pg_atomic_uint32 ptr,
uint32  newval 
)
inlinestatic

Definition at line 328 of file atomics.h.

329{
331
333}
static uint32 pg_atomic_exchange_u32_impl(volatile pg_atomic_uint32 *ptr, uint32 newval)
Definition: generic-msvc.h:61

References AssertPointerAlignment, newval, and pg_atomic_exchange_u32_impl().

Referenced by pgarch_readyXlog(), ProcArrayGroupClearXid(), ProcessProcSignalBarrier(), StrategySyncStart(), test_atomic_uint32(), and TransactionGroupUpdateXidStatus().

◆ pg_atomic_exchange_u64()

static uint64 pg_atomic_exchange_u64 ( volatile pg_atomic_uint64 ptr,
uint64  newval 
)
inlinestatic

Definition at line 511 of file atomics.h.

512{
513#ifndef PG_HAVE_ATOMIC_U64_SIMULATION
515#endif
516 return pg_atomic_exchange_u64_impl(ptr, newval);
517}

References AssertPointerAlignment, and newval.

Referenced by LWLockReleaseClearVar(), LWLockUpdateVar(), and test_atomic_uint64().

◆ pg_atomic_fetch_add_u32()

static uint32 pg_atomic_fetch_add_u32 ( volatile pg_atomic_uint32 ptr,
int32  add_ 
)
inlinestatic

◆ pg_atomic_fetch_add_u64()

static uint64 pg_atomic_fetch_add_u64 ( volatile pg_atomic_uint64 ptr,
int64  add_ 
)
inlinestatic

Definition at line 530 of file atomics.h.

531{
532#ifndef PG_HAVE_ATOMIC_U64_SIMULATION
534#endif
535 return pg_atomic_fetch_add_u64_impl(ptr, add_);
536}
uint64 pg_atomic_fetch_add_u64_impl(volatile pg_atomic_uint64 *ptr, int64 add_)
Definition: atomics.c:62

References AssertPointerAlignment, and pg_atomic_fetch_add_u64_impl().

Referenced by GetFakeLSNForUnloggedRel(), pgstat_init_entry(), pgstat_request_entry_refs_gc(), table_block_parallelscan_nextpage(), and test_atomic_uint64().

◆ pg_atomic_fetch_and_u32()

static uint32 pg_atomic_fetch_and_u32 ( volatile pg_atomic_uint32 ptr,
uint32  and_ 
)
inlinestatic

Definition at line 394 of file atomics.h.

395{
397 return pg_atomic_fetch_and_u32_impl(ptr, and_);
398}

References AssertPointerAlignment.

Referenced by ExtendBufferedRelShared(), LWLockDequeueSelf(), LWLockWaitListUnlock(), and test_atomic_uint32().

◆ pg_atomic_fetch_and_u64()

static uint64 pg_atomic_fetch_and_u64 ( volatile pg_atomic_uint64 ptr,
uint64  and_ 
)
inlinestatic

Definition at line 549 of file atomics.h.

550{
551#ifndef PG_HAVE_ATOMIC_U64_SIMULATION
553#endif
554 return pg_atomic_fetch_and_u64_impl(ptr, and_);
555}

References AssertPointerAlignment.

Referenced by test_atomic_uint64().

◆ pg_atomic_fetch_or_u32()

static uint32 pg_atomic_fetch_or_u32 ( volatile pg_atomic_uint32 ptr,
uint32  or_ 
)
inlinestatic

Definition at line 408 of file atomics.h.

409{
411 return pg_atomic_fetch_or_u32_impl(ptr, or_);
412}

References AssertPointerAlignment.

Referenced by EmitProcSignalBarrier(), LockBufHdr(), LWLockAcquire(), LWLockDequeueSelf(), LWLockQueueSelf(), LWLockWaitForVar(), LWLockWaitListLock(), ResetProcSignalBarrierBits(), and test_atomic_uint32().

◆ pg_atomic_fetch_or_u64()

static uint64 pg_atomic_fetch_or_u64 ( volatile pg_atomic_uint64 ptr,
uint64  or_ 
)
inlinestatic

Definition at line 558 of file atomics.h.

559{
560#ifndef PG_HAVE_ATOMIC_U64_SIMULATION
562#endif
563 return pg_atomic_fetch_or_u64_impl(ptr, or_);
564}

References AssertPointerAlignment.

Referenced by test_atomic_uint64().

◆ pg_atomic_fetch_sub_u32()

static uint32 pg_atomic_fetch_sub_u32 ( volatile pg_atomic_uint32 ptr,
int32  sub_ 
)
inlinestatic

Definition at line 379 of file atomics.h.

380{
382 Assert(sub_ != INT_MIN);
383 return pg_atomic_fetch_sub_u32_impl(ptr, sub_);
384}
Assert(PointerIsAligned(start, uint64))

References Assert(), and AssertPointerAlignment.

Referenced by LWLockAcquire(), LWLockAcquireOrWait(), LWLockDequeueSelf(), LWLockWaitForVar(), pgstat_release_entry_ref(), test_atomic_uint32(), UnlockBufHdr(), and UnpinBufferNoOwner().

◆ pg_atomic_fetch_sub_u64()

static uint64 pg_atomic_fetch_sub_u64 ( volatile pg_atomic_uint64 ptr,
int64  sub_ 
)
inlinestatic

Definition at line 539 of file atomics.h.

540{
541#ifndef PG_HAVE_ATOMIC_U64_SIMULATION
543#endif
544 Assert(sub_ != PG_INT64_MIN);
545 return pg_atomic_fetch_sub_u64_impl(ptr, sub_);
546}
#define PG_INT64_MIN
Definition: c.h:599

References Assert(), AssertPointerAlignment, and PG_INT64_MIN.

Referenced by test_atomic_uint64().

◆ pg_atomic_init_flag()

static void pg_atomic_init_flag ( volatile pg_atomic_flag *  ptr)
inlinestatic

Definition at line 168 of file atomics.h.

169{
170 pg_atomic_init_flag_impl(ptr);
171}

Referenced by AutoVacuumShmemInit(), and test_atomic_flag().

◆ pg_atomic_init_u32()

◆ pg_atomic_init_u64()

static void pg_atomic_init_u64 ( volatile pg_atomic_uint64 ptr,
uint64  val 
)
inlinestatic

Definition at line 451 of file atomics.h.

452{
453 /*
454 * Can't necessarily enforce alignment - and don't need it - when using
455 * the spinlock based fallback implementation. Therefore only assert when
456 * not using it.
457 */
458#ifndef PG_HAVE_ATOMIC_U64_SIMULATION
460#endif
462}
void pg_atomic_init_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val_)
Definition: atomics.c:24

References AssertPointerAlignment, pg_atomic_init_u64_impl(), and val.

Referenced by InitProcGlobal(), InjectionPointShmemInit(), MarkAsPreparingGuts(), ProcSignalShmemInit(), shm_mq_create(), SimpleLruInit(), StatsShmemInit(), table_block_parallelscan_initialize(), test_atomic_uint64(), WaitLSNShmemInit(), WalRcvShmemInit(), XLogPrefetchShmemInit(), and XLOGShmemInit().

◆ pg_atomic_monotonic_advance_u64()

static uint64 pg_atomic_monotonic_advance_u64 ( volatile pg_atomic_uint64 ptr,
uint64  target 
)
inlinestatic

Definition at line 593 of file atomics.h.

594{
595 uint64 currval;
596
597#ifndef PG_HAVE_ATOMIC_U64_SIMULATION
599#endif
600
601 currval = pg_atomic_read_u64_impl(ptr);
602 if (currval >= target)
603 {
605 return currval;
606 }
607
608 while (currval < target)
609 {
610 if (pg_atomic_compare_exchange_u64(ptr, &currval, target))
611 return target;
612 }
613
614 return currval;
615}
#define pg_memory_barrier()
Definition: atomics.h:141
static bool pg_atomic_compare_exchange_u64(volatile pg_atomic_uint64 *ptr, uint64 *expected, uint64 newval)
Definition: atomics.h:520
uint64_t uint64
Definition: c.h:542
static uint64 pg_atomic_read_u64_impl(volatile pg_atomic_uint64 *ptr)
Definition: generic.h:328

References AssertPointerAlignment, pg_atomic_compare_exchange_u64(), pg_atomic_read_u64_impl(), and pg_memory_barrier.

Referenced by WaitXLogInsertionsToFinish().

◆ pg_atomic_read_membarrier_u32()

static uint32 pg_atomic_read_membarrier_u32 ( volatile pg_atomic_uint32 ptr)
inlinestatic

Definition at line 256 of file atomics.h.

257{
259
260 return pg_atomic_read_membarrier_u32_impl(ptr);
261}

References AssertPointerAlignment.

◆ pg_atomic_read_membarrier_u64()

static uint64 pg_atomic_read_membarrier_u64 ( volatile pg_atomic_uint64 ptr)
inlinestatic

Definition at line 474 of file atomics.h.

475{
476#ifndef PG_HAVE_ATOMIC_U64_SIMULATION
478#endif
479 return pg_atomic_read_membarrier_u64_impl(ptr);
480}

References AssertPointerAlignment.

Referenced by CreateCheckPoint(), and WaitXLogInsertionsToFinish().

◆ pg_atomic_read_u32()

static uint32 pg_atomic_read_u32 ( volatile pg_atomic_uint32 ptr)
inlinestatic

Definition at line 237 of file atomics.h.

238{
240 return pg_atomic_read_u32_impl(ptr);
241}
static uint32 pg_atomic_read_u32_impl(volatile pg_atomic_uint32 *ptr)
Definition: generic.h:46

References AssertPointerAlignment, and pg_atomic_read_u32_impl().

Referenced by autovac_recalculate_workers_for_balance(), AutoVacuumUpdateCostLimit(), buffer_readv_complete_one(), buffer_stage_common(), BufferIsDirty(), BufferIsPermanent(), BufferSync(), CheckReadBuffersOperation(), CleanupProcSignalState(), compute_parallel_delay(), create_toy_buffer(), DebugPrintBufferRefcount(), DropRelationAllLocalBuffers(), DropRelationLocalBuffers(), EmitProcSignalBarrier(), EvictAllUnpinnedBuffers(), EvictRelUnpinnedBuffers(), EvictUnpinnedBufferInternal(), ExtendBufferedRelLocal(), FlushRelationBuffers(), GetBufferFromRing(), GetLocalVictimBuffer(), GetVictimBuffer(), InitProcess(), InjectionPointAttach(), InjectionPointDetach(), InjectionPointList(), invalidate_rel_block(), InvalidateLocalBuffer(), InvalidateVictimBuffer(), LocalBufferAlloc(), LWLockAttemptLock(), LWLockConflictsWithVar(), LWLockDequeueSelf(), LWLockUpdateVar(), LWLockWaitListLock(), LWLockWakeup(), MarkBufferDirty(), MarkBufferDirtyHint(), MarkDirtyAllUnpinnedBuffers(), MarkDirtyRelUnpinnedBuffers(), MarkDirtyUnpinnedBufferInternal(), MarkLocalBufferDirty(), pa_decr_and_wait_stream_block(), parallel_vacuum_process_all_indexes(), pg_buffercache_summary(), pgstat_acquire_entry_ref(), pgstat_build_snapshot(), pgstat_drop_entry_internal(), pgstat_gc_entry_refs(), pgstat_get_entry_ref_cached(), pgstat_release_entry_ref(), pgstat_write_statsfile(), PinBuffer(), PinBuffer_Locked(), PinLocalBuffer(), ProcArrayGroupClearXid(), ProcSignalInit(), ReadRecentBuffer(), SendCancelRequest(), SendProcSignal(), StartLocalBufferIO(), StartReadBuffersImpl(), StrategyGetBuffer(), StrategySyncStart(), TerminateLocalBufferIO(), test_atomic_uint32(), TransactionGroupUpdateXidStatus(), UnlockBufHdr(), UnpinLocalBufferNoOwner(), WaitBufHdrUnlocked(), and WaitForProcSignalBarrier().

◆ pg_atomic_read_u64()

◆ pg_atomic_sub_fetch_u32()

static uint32 pg_atomic_sub_fetch_u32 ( volatile pg_atomic_uint32 ptr,
int32  sub_ 
)
inlinestatic

◆ pg_atomic_sub_fetch_u64()

static uint64 pg_atomic_sub_fetch_u64 ( volatile pg_atomic_uint64 ptr,
int64  sub_ 
)
inlinestatic

Definition at line 576 of file atomics.h.

577{
578#ifndef PG_HAVE_ATOMIC_U64_SIMULATION
580#endif
581 Assert(sub_ != PG_INT64_MIN);
582 return pg_atomic_sub_fetch_u64_impl(ptr, sub_);
583}

References Assert(), AssertPointerAlignment, and PG_INT64_MIN.

Referenced by pgstat_free_entry(), and test_atomic_uint64().

◆ pg_atomic_test_set_flag()

static bool pg_atomic_test_set_flag ( volatile pg_atomic_flag *  ptr)
inlinestatic

Definition at line 181 of file atomics.h.

182{
183 return pg_atomic_test_set_flag_impl(ptr);
184}

Referenced by do_autovacuum(), and test_atomic_flag().

◆ pg_atomic_unlocked_test_flag()

static bool pg_atomic_unlocked_test_flag ( volatile pg_atomic_flag *  ptr)
inlinestatic

Definition at line 194 of file atomics.h.

195{
196 return pg_atomic_unlocked_test_flag_impl(ptr);
197}

Referenced by autovac_recalculate_workers_for_balance(), AutoVacuumUpdateCostLimit(), test_atomic_flag(), and VacuumUpdateCosts().

◆ pg_atomic_unlocked_write_u32()

static void pg_atomic_unlocked_write_u32 ( volatile pg_atomic_uint32 ptr,
uint32  val 
)
inlinestatic

◆ pg_atomic_unlocked_write_u64()

static void pg_atomic_unlocked_write_u64 ( volatile pg_atomic_uint64 ptr,
uint64  val 
)
inlinestatic

Definition at line 492 of file atomics.h.

493{
494#ifndef PG_HAVE_ATOMIC_U64_SIMULATION
496#endif
497
499}
static void pg_atomic_unlocked_write_u64_impl(volatile pg_atomic_uint64 *ptr, uint64 val)
Definition: generic.h:303

References AssertPointerAlignment, pg_atomic_unlocked_write_u64_impl(), and val.

◆ pg_atomic_write_membarrier_u32()

static void pg_atomic_write_membarrier_u32 ( volatile pg_atomic_uint32 ptr,
uint32  val 
)
inlinestatic

Definition at line 313 of file atomics.h.

314{
316
317 pg_atomic_write_membarrier_u32_impl(ptr, val);
318}

References AssertPointerAlignment, and val.

Referenced by PgArchForceDirScan().

◆ pg_atomic_write_membarrier_u64()

static void pg_atomic_write_membarrier_u64 ( volatile pg_atomic_uint64 ptr,
uint64  val 
)
inlinestatic

Definition at line 502 of file atomics.h.

503{
504#ifndef PG_HAVE_ATOMIC_U64_SIMULATION
506#endif
507 pg_atomic_write_membarrier_u64_impl(ptr, val);
508}

References AssertPointerAlignment, and val.

Referenced by StartupXLOG().

◆ pg_atomic_write_u32()

static void pg_atomic_write_u32 ( volatile pg_atomic_uint32 ptr,
uint32  val 
)
inlinestatic

◆ pg_atomic_write_u64()