#include "pgstat.h"
#include "port/atomics.h"
#include "storage/aio_types.h"
#include "storage/buf.h"
#include "storage/bufmgr.h"
#include "storage/condition_variable.h"
#include "storage/lwlock.h"
#include "storage/procnumber.h"
#include "storage/proclist_types.h"
#include "storage/shmem.h"
#include "storage/smgr.h"
#include "storage/spin.h"
#include "utils/relcache.h"
#include "utils/resowner.h"

Include dependency graph for buf_internals.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures
struct	buftag

struct	BufferDesc

union	BufferDescPadded

struct	PendingWriteback

struct	WritebackContext

struct	CkptSortItem

Macros
#define	BUF_REFCOUNT_BITS 18

#define	BUF_USAGECOUNT_BITS 4

#define	BUF_FLAG_BITS 12

#define	BUF_LOCK_BITS (18+2)

#define	BUF_REFCOUNT_ONE 1

#define	BUF_REFCOUNT_MASK ((UINT64CONST(1) << BUF_REFCOUNT_BITS) - 1)

#define	BUF_USAGECOUNT_SHIFT BUF_REFCOUNT_BITS

#define	BUF_USAGECOUNT_MASK (((UINT64CONST(1) << BUF_USAGECOUNT_BITS) - 1) << (BUF_USAGECOUNT_SHIFT))

#define	BUF_USAGECOUNT_ONE (UINT64CONST(1) << BUF_REFCOUNT_BITS)

#define	BUF_FLAG_SHIFT (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS)

#define	BUF_FLAG_MASK (((UINT64CONST(1) << BUF_FLAG_BITS) - 1) << BUF_FLAG_SHIFT)

#define	BM_LOCK_SHIFT (BUF_FLAG_SHIFT + BUF_FLAG_BITS)

#define	BM_LOCK_VAL_SHARED (UINT64CONST(1) << (BM_LOCK_SHIFT))

#define	BM_LOCK_VAL_SHARE_EXCLUSIVE (UINT64CONST(1) << (BM_LOCK_SHIFT + MAX_BACKENDS_BITS))

#define	BM_LOCK_VAL_EXCLUSIVE (UINT64CONST(1) << (BM_LOCK_SHIFT + MAX_BACKENDS_BITS + 1))

#define	BM_LOCK_MASK ((((uint64) MAX_BACKENDS) << BM_LOCK_SHIFT) \| BM_LOCK_VAL_SHARE_EXCLUSIVE \| BM_LOCK_VAL_EXCLUSIVE)

#define	BUF_STATE_GET_REFCOUNT(state) ((uint32)((state) & BUF_REFCOUNT_MASK))

#define	BUF_STATE_GET_USAGECOUNT(state) ((uint32)(((state) & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT))

#define	BUF_DEFINE_FLAG(flagno) (UINT64CONST(1) << (BUF_FLAG_SHIFT + (flagno)))

#define	BM_LOCKED BUF_DEFINE_FLAG( 0)

#define	BM_DIRTY BUF_DEFINE_FLAG( 1)

#define	BM_VALID BUF_DEFINE_FLAG( 2)

#define	BM_TAG_VALID BUF_DEFINE_FLAG( 3)

#define	BM_IO_IN_PROGRESS BUF_DEFINE_FLAG( 4)

#define	BM_IO_ERROR BUF_DEFINE_FLAG( 5)

#define	BM_PIN_COUNT_WAITER BUF_DEFINE_FLAG( 7)

#define	BM_CHECKPOINT_NEEDED BUF_DEFINE_FLAG( 8)

#define	BM_PERMANENT BUF_DEFINE_FLAG( 9)

#define	BM_LOCK_HAS_WAITERS BUF_DEFINE_FLAG(10)

#define	BM_LOCK_WAKE_IN_PROGRESS BUF_DEFINE_FLAG(11)

#define	BM_MAX_USAGE_COUNT 5

#define	BUFFERDESC_PAD_TO_SIZE (SIZEOF_VOID_P == 8 ? 64 : 1)

Typedefs
typedef struct buftag	BufferTag

typedef struct BufferDesc	BufferDesc

typedef union BufferDescPadded	BufferDescPadded

typedef struct PendingWriteback	PendingWriteback

typedef struct WritebackContext	WritebackContext

typedef struct CkptSortItem	CkptSortItem

typedef enum StartBufferIOResult	StartBufferIOResult

Enumerations
enum	StartBufferIOResult { BUFFER_IO_ALREADY_DONE , BUFFER_IO_IN_PROGRESS , BUFFER_IO_READY_FOR_IO }

Functions
	StaticAssertDecl (BUF_REFCOUNT_BITS+BUF_USAGECOUNT_BITS+BUF_FLAG_BITS+BUF_LOCK_BITS<=64, "parts of buffer state space need to be <= 64")

	StaticAssertDecl (MAX_BACKENDS_BITS<=BUF_REFCOUNT_BITS, "MAX_BACKENDS_BITS needs to be <= BUF_REFCOUNT_BITS")

	StaticAssertDecl (MAX_BACKENDS_BITS<=(BUF_LOCK_BITS - 2), "MAX_BACKENDS_BITS needs to be <= BUF_LOCK_BITS - 2")

	StaticAssertDecl (BM_MAX_USAGE_COUNT<(UINT64CONST(1)<< BUF_USAGECOUNT_BITS), "BM_MAX_USAGE_COUNT doesn't fit in BUF_USAGECOUNT_BITS bits")

static RelFileNumber	BufTagGetRelNumber (const BufferTag *tag)

static ForkNumber	BufTagGetForkNum (const BufferTag *tag)

static void	BufTagSetRelForkDetails (BufferTag *tag, RelFileNumber relnumber, ForkNumber forknum)

static RelFileLocator	BufTagGetRelFileLocator (const BufferTag *tag)

static void	ClearBufferTag (BufferTag *tag)

static void	InitBufferTag (BufferTag tag, const RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockNum)

static bool	BufferTagsEqual (const BufferTag tag1, const BufferTag tag2)

static bool	BufTagMatchesRelFileLocator (const BufferTag tag, const RelFileLocator rlocator)

static uint32	BufTableHashPartition (uint32 hashcode)

static LWLock *	BufMappingPartitionLock (uint32 hashcode)

static LWLock *	BufMappingPartitionLockByIndex (uint32 index)

static BufferDesc *	GetBufferDescriptor (int id)

static BufferDesc *	GetLocalBufferDescriptor (int id)

static Buffer	BufferDescriptorGetBuffer (const BufferDesc *bdesc)

static ConditionVariable *	BufferDescriptorGetIOCV (const BufferDesc *bdesc)

uint64	LockBufHdr (BufferDesc *desc)

static void	UnlockBufHdr (BufferDesc *desc)

static uint64	UnlockBufHdrExt (BufferDesc *desc, uint64 old_buf_state, uint64 set_bits, uint64 unset_bits, int refcount_change)

uint64	WaitBufHdrUnlocked (BufferDesc *buf)

static void	ResourceOwnerRememberBuffer (ResourceOwner owner, Buffer buffer)

static void	ResourceOwnerForgetBuffer (ResourceOwner owner, Buffer buffer)

static void	ResourceOwnerRememberBufferIO (ResourceOwner owner, Buffer buffer)

static void	ResourceOwnerForgetBufferIO (ResourceOwner owner, Buffer buffer)

void	WritebackContextInit (WritebackContext context, int max_pending)

void	IssuePendingWritebacks (WritebackContext *wb_context, IOContext io_context)

void	ScheduleBufferTagForWriteback (WritebackContext wb_context, IOContext io_context, BufferTag tag)

void	TrackNewBufferPin (Buffer buf)

StartBufferIOResult	StartBufferIO (Buffer buffer, bool forInput, bool wait, PgAioWaitRef *io_wref)

StartBufferIOResult	StartSharedBufferIO (BufferDesc buf, bool forInput, bool wait, PgAioWaitRef io_wref)

void	TerminateBufferIO (BufferDesc *buf, bool clear_dirty, uint64 set_flag_bits, bool forget_owner, bool release_aio)

IOContext	IOContextForStrategy (BufferAccessStrategy strategy)

BufferDesc *	StrategyGetBuffer (BufferAccessStrategy strategy, uint64 buf_state, bool from_ring)

bool	StrategyRejectBuffer (BufferAccessStrategy strategy, BufferDesc *buf, bool from_ring)

int	StrategySyncStart (uint32 complete_passes, uint32 num_buf_alloc)

void	StrategyNotifyBgWriter (int bgwprocno)

uint32	BufTableHashCode (BufferTag *tagPtr)

int	BufTableLookup (BufferTag *tagPtr, uint32 hashcode)

int	BufTableInsert (BufferTag *tagPtr, uint32 hashcode, int buf_id)

void	BufTableDelete (BufferTag *tagPtr, uint32 hashcode)

bool	PinLocalBuffer (BufferDesc *buf_hdr, bool adjust_usagecount)

void	UnpinLocalBuffer (Buffer buffer)

void	UnpinLocalBufferNoOwner (Buffer buffer)

PrefetchBufferResult	PrefetchLocalBuffer (SMgrRelation smgr, ForkNumber forkNum, BlockNumber blockNum)

BufferDesc *	LocalBufferAlloc (SMgrRelation smgr, ForkNumber forkNum, BlockNumber blockNum, bool *foundPtr)

BlockNumber	ExtendBufferedRelLocal (BufferManagerRelation bmr, ForkNumber fork, uint32 flags, uint32 extend_by, BlockNumber extend_upto, Buffer buffers, uint32 extended_by)

void	MarkLocalBufferDirty (Buffer buffer)

void	TerminateLocalBufferIO (BufferDesc *bufHdr, bool clear_dirty, uint64 set_flag_bits, bool release_aio)

StartBufferIOResult	StartLocalBufferIO (BufferDesc bufHdr, bool forInput, bool wait, PgAioWaitRef io_wref)

void	FlushLocalBuffer (BufferDesc *bufHdr, SMgrRelation reln)

void	InvalidateLocalBuffer (BufferDesc *bufHdr, bool check_unreferenced)

void	DropRelationLocalBuffers (RelFileLocator rlocator, ForkNumber forkNum, int nforks, BlockNumber firstDelBlock)

void	DropRelationAllLocalBuffers (RelFileLocator rlocator)

void	AtEOXact_LocalBuffers (bool isCommit)

Macro Definition Documentation

◆ BM_CHECKPOINT_NEEDED

#define BM_CHECKPOINT_NEEDED BUF_DEFINE_FLAG( 8)

Definition at line 121 of file buf_internals.h.

◆ BM_DIRTY

#define BM_DIRTY BUF_DEFINE_FLAG( 1)

Definition at line 108 of file buf_internals.h.

◆ BM_IO_ERROR

#define BM_IO_ERROR BUF_DEFINE_FLAG( 5)

Definition at line 116 of file buf_internals.h.

◆ BM_IO_IN_PROGRESS

#define BM_IO_IN_PROGRESS BUF_DEFINE_FLAG( 4)

Definition at line 114 of file buf_internals.h.

◆ BM_LOCK_HAS_WAITERS

#define BM_LOCK_HAS_WAITERS BUF_DEFINE_FLAG(10)

Definition at line 125 of file buf_internals.h.

◆ BM_LOCK_MASK

#define BM_LOCK_MASK ((((uint64) MAX_BACKENDS) << BM_LOCK_SHIFT) | BM_LOCK_VAL_SHARE_EXCLUSIVE | BM_LOCK_VAL_EXCLUSIVE)

Definition at line 85 of file buf_internals.h.

{
    Oid         spcOid;         /* tablespace oid */
    Oid         dbOid;          /* database oid */
    RelFileNumber relNumber;    /* relation file number */
    ForkNumber  forkNum;        /* fork number */
    BlockNumber blockNum;       /* blknum relative to begin of reln */
} BufferTag;
 
static inline RelFileNumber
BufTagGetRelNumber(const BufferTag *tag)
{
    return tag->relNumber;
}
 
static inline ForkNumber
BufTagGetForkNum(const BufferTag *tag)
{
    return tag->forkNum;
}
 
static inline void
BufTagSetRelForkDetails(BufferTag *tag, RelFileNumber relnumber,
                        ForkNumber forknum)
{
    tag->relNumber = relnumber;
    tag->forkNum = forknum;
}
 
static inline RelFileLocator
BufTagGetRelFileLocator(const BufferTag *tag)
{
    RelFileLocator rlocator;
 
    rlocator.spcOid = tag->spcOid;
    rlocator.dbOid = tag->dbOid;
    rlocator.relNumber = BufTagGetRelNumber(tag);
 
    return rlocator;
}
 
static inline void
ClearBufferTag(BufferTag *tag)
{
    tag->spcOid = InvalidOid;
    tag->dbOid = InvalidOid;
    BufTagSetRelForkDetails(tag, InvalidRelFileNumber, InvalidForkNumber);
    tag->blockNum = InvalidBlockNumber;
}
 
static inline void
InitBufferTag(BufferTag *tag, const RelFileLocator *rlocator,
              ForkNumber forkNum, BlockNumber blockNum)
{
    tag->spcOid = rlocator->spcOid;
    tag->dbOid = rlocator->dbOid;
    BufTagSetRelForkDetails(tag, rlocator->relNumber, forkNum);
    tag->blockNum = blockNum;
}
 
static inline bool
BufferTagsEqual(const BufferTag *tag1, const BufferTag *tag2)
{
    return (tag1->spcOid == tag2->spcOid) &&
        (tag1->dbOid == tag2->dbOid) &&
        (tag1->relNumber == tag2->relNumber) &&
        (tag1->blockNum == tag2->blockNum) &&
        (tag1->forkNum == tag2->forkNum);
}
 
static inline bool
BufTagMatchesRelFileLocator(const BufferTag *tag,
                            const RelFileLocator *rlocator)
{
    return (tag->spcOid == rlocator->spcOid) &&
        (tag->dbOid == rlocator->dbOid) &&
        (BufTagGetRelNumber(tag) == rlocator->relNumber);
}
 
 
/*
 * The shared buffer mapping table is partitioned to reduce contention.
 * To determine which partition lock a given tag requires, compute the tag's
 * hash code with BufTableHashCode(), then apply BufMappingPartitionLock().
 * NB: NUM_BUFFER_PARTITIONS must be a power of 2!
 */
static inline uint32
BufTableHashPartition(uint32 hashcode)
{
    return hashcode % NUM_BUFFER_PARTITIONS;
}
 
static inline LWLock *
BufMappingPartitionLock(uint32 hashcode)
{
    return &MainLWLockArray[BUFFER_MAPPING_LWLOCK_OFFSET +
                            BufTableHashPartition(hashcode)].lock;
}
 
static inline LWLock *
BufMappingPartitionLockByIndex(uint32 index)
{
    return &MainLWLockArray[BUFFER_MAPPING_LWLOCK_OFFSET + index].lock;
}
 
/*
 *  BufferDesc -- shared descriptor/state data for a single shared buffer.
 *
 * The state of the buffer is controlled by the, drumroll, state variable. It
 * only may be modified using atomic operations.  The state variable combines
 * various flags, the buffer's refcount and usage count. See comment above
 * BUF_REFCOUNT_BITS for details about the division.  This layout allow us to
 * do some operations in a single atomic operation, without actually acquiring
 * and releasing the spinlock; for instance, increasing or decreasing the
 * refcount.
 *
 * One of the aforementioned flags is BM_LOCKED, used to implement the buffer
 * header lock. See the following paragraphs, as well as the documentation for
 * individual fields, for more details.
 *
 * The identity of the buffer (BufferDesc.tag) can only be changed by the
 * backend holding the buffer header lock.
 *
 * If the lock is held by another backend, neither additional buffer pins may
 * be established (we would like to relax this eventually), nor can flags be
 * set/cleared. These operations either need to acquire the buffer header
 * spinlock, or need to use a CAS loop, waiting for the lock to be released if
 * it is held.  However, existing buffer pins may be released while the buffer
 * header spinlock is held, using an atomic subtraction.
 *
 * If we have the buffer pinned, its tag can't change underneath us, so we can
 * examine the tag without locking the buffer header.  Also, in places we do
 * one-time reads of the flags without bothering to lock the buffer header;
 * this is generally for situations where we don't expect the flag bit being
 * tested to be changing.
 *
 * We can't physically remove items from a disk page if another backend has
 * the buffer pinned.  Hence, a backend may need to wait for all other pins
 * to go away.  This is signaled by storing its own pgprocno into
 * wait_backend_pgprocno and setting flag bit BM_PIN_COUNT_WAITER.  At present,
 * there can be only one such waiter per buffer.
 *
 * The content of buffers is protected via the buffer content lock,
 * implemented as part of the buffer state. Note that the buffer header lock
 * is *not* used to control access to the data in the buffer! We used to use
 * an LWLock to implement the content lock, but having a dedicated
 * implementation of content locks allows us to implement some otherwise hard
 * things (e.g. race-freely checking if AIO is in progress before locking a
 * buffer exclusively) and enables otherwise impossible optimizations
 * (e.g. unlocking and unpinning a buffer in one atomic operation).
 *
 * We use this same struct for local buffer headers, but the locks are not
 * used and not all of the flag bits are useful either. To avoid unnecessary
 * overhead, manipulations of the state field should be done without actual
 * atomic operations (i.e. only pg_atomic_read_u64() and
 * pg_atomic_unlocked_write_u64()).
 *
 * Be careful to avoid increasing the size of the struct when adding or
 * reordering members.  Keeping it below 64 bytes (the most common CPU
 * cache line size) is fairly important for performance.
 *
 * Per-buffer I/O condition variables are currently kept outside this struct in
 * a separate array.  They could be moved in here and still fit within that
 * limit on common systems, but for now that is not done.
 */
typedef struct BufferDesc
{
    /*
     * ID of page contained in buffer. The buffer header spinlock needs to be
     * held to modify this field.
     */
    BufferTag   tag;
 
    /*
     * Buffer's index number (from 0). The field never changes after
     * initialization, so does not need locking.
     */
    int         buf_id;
 
    /*
     * State of the buffer, containing flags, refcount and usagecount. See
     * BUF_* and BM_* defines at the top of this file.
     */
    pg_atomic_uint64 state;
 
    /*
     * Backend of pin-count waiter. The buffer header spinlock needs to be
     * held to modify this field.
     */
    int         wait_backend_pgprocno;
 
    PgAioWaitRef io_wref;       /* set iff AIO is in progress */
 
    /*
     * List of PGPROCs waiting for the buffer content lock. Protected by the
     * buffer header spinlock.
     */
    proclist_head lock_waiters;
} BufferDesc;
 
/*
 * Concurrent access to buffer headers has proven to be more efficient if
 * they're cache line aligned. So we force the start of the BufferDescriptors
 * array to be on a cache line boundary and force the elements to be cache
 * line sized.
 *
 * XXX: As this is primarily matters in highly concurrent workloads which
 * probably all are 64bit these days, and the space wastage would be a bit
 * more noticeable on 32bit systems, we don't force the stride to be cache
 * line sized on those. If somebody does actual performance testing, we can
 * reevaluate.
 *
 * Note that local buffer descriptors aren't forced to be aligned - as there's
 * no concurrent access to those it's unlikely to be beneficial.
 *
 * We use a 64-byte cache line size here, because that's the most common
 * size. Making it bigger would be a waste of memory. Even if running on a
 * platform with either 32 or 128 byte line sizes, it's good to align to
 * boundaries and avoid false sharing.
 */
#define BUFFERDESC_PAD_TO_SIZE  (SIZEOF_VOID_P == 8 ? 64 : 1)
 
typedef union BufferDescPadded
{
    BufferDesc  bufferdesc;
    char        pad[BUFFERDESC_PAD_TO_SIZE];
} BufferDescPadded;
 
/*
 * The PendingWriteback & WritebackContext structure are used to keep
 * information about pending flush requests to be issued to the OS.
 */
typedef struct PendingWriteback
{
    /* could store different types of pending flushes here */
    BufferTag   tag;
} PendingWriteback;
 
/* struct forward declared in bufmgr.h */
typedef struct WritebackContext
{
    /* pointer to the max number of writeback requests to coalesce */
    int        *max_pending;
 
    /* current number of pending writeback requests */
    int         nr_pending;
 
    /* pending requests */
    PendingWriteback pending_writebacks[WRITEBACK_MAX_PENDING_FLUSHES];
} WritebackContext;
 
/* in buf_init.c */
extern PGDLLIMPORT BufferDescPadded *BufferDescriptors;
extern PGDLLIMPORT ConditionVariableMinimallyPadded *BufferIOCVArray;
extern PGDLLIMPORT WritebackContext BackendWritebackContext;
 
/* in localbuf.c */
extern PGDLLIMPORT BufferDesc *LocalBufferDescriptors;
 
 
static inline BufferDesc *
GetBufferDescriptor(int id)
{
    Assert(id >= 0 && id < NBuffers);
 
    return &(BufferDescriptors[id]).bufferdesc;
}
 
static inline BufferDesc *
GetLocalBufferDescriptor(int id)
{
    return &LocalBufferDescriptors[id];
}
 
static inline Buffer
BufferDescriptorGetBuffer(const BufferDesc *bdesc)
{
    return (Buffer) (bdesc->buf_id + 1);
}
 
static inline ConditionVariable *
BufferDescriptorGetIOCV(const BufferDesc *bdesc)
{
    return &(BufferIOCVArray[bdesc->buf_id]).cv;
}
 
/*
 * Functions for acquiring/releasing a shared buffer header's spinlock.  Do
 * not apply these to local buffers!
 */
extern uint64 LockBufHdr(BufferDesc *desc);
 
/*
 * Unlock the buffer header.
 *
 * This can only be used if the caller did not modify BufferDesc.state. To
 * set/unset flag bits or change the refcount use UnlockBufHdrExt().
 */
static inline void
UnlockBufHdr(BufferDesc *desc)
{
    Assert(pg_atomic_read_u64(&desc->state) & BM_LOCKED);
 
    pg_atomic_fetch_sub_u64(&desc->state, BM_LOCKED);
}
 
/*
 * Unlock the buffer header, while atomically unsetting the ones in
 * unset_bits, adding the flags in set_bits, and changing the refcount by
 * refcount_change. If a flag is both cleared and added, it will end up being
 * set.
 *
 * Note that this approach would not trivially work for usagecount, since we
 * need to cap the usagecount at BM_MAX_USAGE_COUNT.
 */
static inline uint64
UnlockBufHdrExt(BufferDesc *desc, uint64 old_buf_state,
                uint64 set_bits, uint64 unset_bits,
                int refcount_change)
{
    for (;;)
    {
        uint64      buf_state = old_buf_state;
 
        Assert(buf_state & BM_LOCKED);
 
        /*
         * Set bits after clearing bits, so that a cleared and re-added flag
         * survives.
         */
        buf_state &= ~unset_bits;
        buf_state |= set_bits;
        buf_state &= ~BM_LOCKED;
 
        if (refcount_change != 0)
            buf_state += BUF_REFCOUNT_ONE * refcount_change;
 
        if (pg_atomic_compare_exchange_u64(&desc->state, &old_buf_state,
                                           buf_state))
        {
            return old_buf_state;
        }
    }
}
 
extern uint64 WaitBufHdrUnlocked(BufferDesc *buf);
 
/* in bufmgr.c */
 
/*
 * Structure to sort buffers per file on checkpoints.
 *
 * This structure is allocated per buffer in shared memory, so it should be
 * kept as small as possible.
 */
typedef struct CkptSortItem
{
    Oid         tsId;
    RelFileNumber relNumber;
    ForkNumber  forkNum;
    BlockNumber blockNum;
    int         buf_id;
} CkptSortItem;
 
extern PGDLLIMPORT CkptSortItem *CkptBufferIds;
 
/* ResourceOwner callbacks to hold buffer I/Os and pins */
extern PGDLLIMPORT const ResourceOwnerDesc buffer_io_resowner_desc;
extern PGDLLIMPORT const ResourceOwnerDesc buffer_resowner_desc;
 
/* Convenience wrappers over ResourceOwnerRemember/Forget */
static inline void
ResourceOwnerRememberBuffer(ResourceOwner owner, Buffer buffer)
{
    ResourceOwnerRemember(owner, Int32GetDatum(buffer), &buffer_resowner_desc);
}
static inline void
ResourceOwnerForgetBuffer(ResourceOwner owner, Buffer buffer)
{
    ResourceOwnerForget(owner, Int32GetDatum(buffer), &buffer_resowner_desc);
}
static inline void
ResourceOwnerRememberBufferIO(ResourceOwner owner, Buffer buffer)
{
    ResourceOwnerRemember(owner, Int32GetDatum(buffer), &buffer_io_resowner_desc);
}
static inline void
ResourceOwnerForgetBufferIO(ResourceOwner owner, Buffer buffer)
{
    ResourceOwnerForget(owner, Int32GetDatum(buffer), &buffer_io_resowner_desc);
}
 
/*
 * Internal buffer management routines
 */
/* bufmgr.c */
extern void WritebackContextInit(WritebackContext *context, int *max_pending);
extern void IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context);
extern void ScheduleBufferTagForWriteback(WritebackContext *wb_context,
                                          IOContext io_context, BufferTag *tag);
 
extern void TrackNewBufferPin(Buffer buf);
 
/*
 * Return value for StartBufferIO / StartSharedBufferIO / StartLocalBufferIO.
 *
 * When preparing a buffer for I/O and setting BM_IO_IN_PROGRESS, the buffer
 * may already have I/O in progress or the I/O may have been done by another
 * backend.  See the documentation of StartSharedBufferIO for more details.
 */
typedef enum StartBufferIOResult
{
    BUFFER_IO_ALREADY_DONE,
    BUFFER_IO_IN_PROGRESS,
    BUFFER_IO_READY_FOR_IO,
} StartBufferIOResult;
 
/* the following are exposed to make it easier to write tests */
extern StartBufferIOResult StartBufferIO(Buffer buffer, bool forInput, bool wait,
                                         PgAioWaitRef *io_wref);
extern StartBufferIOResult StartSharedBufferIO(BufferDesc *buf, bool forInput, bool wait,
                                               PgAioWaitRef *io_wref);
extern void TerminateBufferIO(BufferDesc *buf, bool clear_dirty, uint64 set_flag_bits,
                              bool forget_owner, bool release_aio);
 
 
/* freelist.c */
extern IOContext IOContextForStrategy(BufferAccessStrategy strategy);
extern BufferDesc *StrategyGetBuffer(BufferAccessStrategy strategy,
                                     uint64 *buf_state, bool *from_ring);
extern bool StrategyRejectBuffer(BufferAccessStrategy strategy,
                                 BufferDesc *buf, bool from_ring);
 
extern int  StrategySyncStart(uint32 *complete_passes, uint32 *num_buf_alloc);
extern void StrategyNotifyBgWriter(int bgwprocno);
 
/* buf_table.c */
extern uint32 BufTableHashCode(BufferTag *tagPtr);
extern int  BufTableLookup(BufferTag *tagPtr, uint32 hashcode);
extern int  BufTableInsert(BufferTag *tagPtr, uint32 hashcode, int buf_id);
extern void BufTableDelete(BufferTag *tagPtr, uint32 hashcode);
 
/* localbuf.c */
extern bool PinLocalBuffer(BufferDesc *buf_hdr, bool adjust_usagecount);
extern void UnpinLocalBuffer(Buffer buffer);
extern void UnpinLocalBufferNoOwner(Buffer buffer);
extern PrefetchBufferResult PrefetchLocalBuffer(SMgrRelation smgr,
                                                ForkNumber forkNum,
                                                BlockNumber blockNum);
extern BufferDesc *LocalBufferAlloc(SMgrRelation smgr, ForkNumber forkNum,
                                    BlockNumber blockNum, bool *foundPtr);
extern BlockNumber ExtendBufferedRelLocal(BufferManagerRelation bmr,
                                          ForkNumber fork,
                                          uint32 flags,
                                          uint32 extend_by,
                                          BlockNumber extend_upto,
                                          Buffer *buffers,
                                          uint32 *extended_by);
extern void MarkLocalBufferDirty(Buffer buffer);
extern void TerminateLocalBufferIO(BufferDesc *bufHdr, bool clear_dirty,
                                   uint64 set_flag_bits, bool release_aio);
extern StartBufferIOResult StartLocalBufferIO(BufferDesc *bufHdr, bool forInput,
                                              bool wait, PgAioWaitRef *io_wref);
extern void FlushLocalBuffer(BufferDesc *bufHdr, SMgrRelation reln);
extern void InvalidateLocalBuffer(BufferDesc *bufHdr, bool check_unreferenced);
extern void DropRelationLocalBuffers(RelFileLocator rlocator,
                                     ForkNumber *forkNum, int nforks,
                                     BlockNumber *firstDelBlock);
extern void DropRelationAllLocalBuffers(RelFileLocator rlocator);
extern void AtEOXact_LocalBuffers(bool isCommit);
 
#endif                          /* BUFMGR_INTERNALS_H */

◆ BM_LOCK_SHIFT

#define BM_LOCK_SHIFT (BUF_FLAG_SHIFT + BUF_FLAG_BITS)

Definition at line 77 of file buf_internals.h.

◆ BM_LOCK_VAL_EXCLUSIVE

#define BM_LOCK_VAL_EXCLUSIVE (UINT64CONST(1) << (BM_LOCK_SHIFT + MAX_BACKENDS_BITS + 1))

Definition at line 83 of file buf_internals.h.

◆ BM_LOCK_VAL_SHARE_EXCLUSIVE

#define BM_LOCK_VAL_SHARE_EXCLUSIVE (UINT64CONST(1) << (BM_LOCK_SHIFT + MAX_BACKENDS_BITS))

Definition at line 81 of file buf_internals.h.

◆ BM_LOCK_VAL_SHARED

#define BM_LOCK_VAL_SHARED (UINT64CONST(1) << (BM_LOCK_SHIFT))

Definition at line 79 of file buf_internals.h.

◆ BM_LOCK_WAKE_IN_PROGRESS

#define BM_LOCK_WAKE_IN_PROGRESS BUF_DEFINE_FLAG(11)

Definition at line 127 of file buf_internals.h.

◆ BM_LOCKED

#define BM_LOCKED BUF_DEFINE_FLAG( 0)

Definition at line 106 of file buf_internals.h.

◆ BM_MAX_USAGE_COUNT

#define BM_MAX_USAGE_COUNT 5

Definition at line 144 of file buf_internals.h.

◆ BM_PERMANENT

#define BM_PERMANENT BUF_DEFINE_FLAG( 9)

Definition at line 123 of file buf_internals.h.

◆ BM_PIN_COUNT_WAITER

#define BM_PIN_COUNT_WAITER BUF_DEFINE_FLAG( 7)

Definition at line 119 of file buf_internals.h.

◆ BM_TAG_VALID

#define BM_TAG_VALID BUF_DEFINE_FLAG( 3)

Definition at line 112 of file buf_internals.h.

◆ BM_VALID

#define BM_VALID BUF_DEFINE_FLAG( 2)

Definition at line 110 of file buf_internals.h.

◆ BUF_DEFINE_FLAG

#define BUF_DEFINE_FLAG ( flagno ) (UINT64CONST(1) << (BUF_FLAG_SHIFT + (flagno)))

Definition at line 102 of file buf_internals.h.

◆ BUF_FLAG_BITS

#define BUF_FLAG_BITS 12

Definition at line 51 of file buf_internals.h.

◆ BUF_FLAG_MASK

#define BUF_FLAG_MASK (((UINT64CONST(1) << BUF_FLAG_BITS) - 1) << BUF_FLAG_SHIFT)

Definition at line 73 of file buf_internals.h.

◆ BUF_FLAG_SHIFT

#define BUF_FLAG_SHIFT (BUF_REFCOUNT_BITS + BUF_USAGECOUNT_BITS)

Definition at line 71 of file buf_internals.h.

◆ BUF_LOCK_BITS

#define BUF_LOCK_BITS (18+2)

Definition at line 52 of file buf_internals.h.

◆ BUF_REFCOUNT_BITS

#define BUF_REFCOUNT_BITS 18

Definition at line 49 of file buf_internals.h.

◆ BUF_REFCOUNT_MASK

#define BUF_REFCOUNT_MASK ((UINT64CONST(1) << BUF_REFCOUNT_BITS) - 1)

Definition at line 59 of file buf_internals.h.

◆ BUF_REFCOUNT_ONE

#define BUF_REFCOUNT_ONE 1

Definition at line 58 of file buf_internals.h.

◆ BUF_STATE_GET_REFCOUNT

#define BUF_STATE_GET_REFCOUNT ( state ) ((uint32)((state) & BUF_REFCOUNT_MASK))

Definition at line 90 of file buf_internals.h.

◆ BUF_STATE_GET_USAGECOUNT

#define BUF_STATE_GET_USAGECOUNT ( state ) ((uint32)(((state) & BUF_USAGECOUNT_MASK) >> BUF_USAGECOUNT_SHIFT))

Definition at line 92 of file buf_internals.h.

◆ BUF_USAGECOUNT_BITS

#define BUF_USAGECOUNT_BITS 4

Definition at line 50 of file buf_internals.h.

◆ BUF_USAGECOUNT_MASK

#define BUF_USAGECOUNT_MASK (((UINT64CONST(1) << BUF_USAGECOUNT_BITS) - 1) << (BUF_USAGECOUNT_SHIFT))

Definition at line 65 of file buf_internals.h.

◆ BUF_USAGECOUNT_ONE

#define BUF_USAGECOUNT_ONE (UINT64CONST(1) << BUF_REFCOUNT_BITS)

Definition at line 67 of file buf_internals.h.

◆ BUF_USAGECOUNT_SHIFT

#define BUF_USAGECOUNT_SHIFT BUF_REFCOUNT_BITS

Definition at line 63 of file buf_internals.h.

◆ BUFFERDESC_PAD_TO_SIZE

#define BUFFERDESC_PAD_TO_SIZE (SIZEOF_VOID_P == 8 ? 64 : 1)

Definition at line 381 of file buf_internals.h.

Typedef Documentation

◆ BufferDesc

typedef struct BufferDesc BufferDesc

◆ BufferDescPadded

typedef union BufferDescPadded BufferDescPadded

◆ BufferTag

typedef struct buftag BufferTag

◆ CkptSortItem

typedef struct CkptSortItem CkptSortItem

◆ PendingWriteback

typedef struct PendingWriteback PendingWriteback

◆ StartBufferIOResult

typedef enum StartBufferIOResult StartBufferIOResult

◆ WritebackContext

typedef struct WritebackContext WritebackContext

Enumeration Type Documentation

◆ StartBufferIOResult

enum StartBufferIOResult

Enumerator
BUFFER_IO_ALREADY_DONE
BUFFER_IO_IN_PROGRESS
BUFFER_IO_READY_FOR_IO

Definition at line 571 of file buf_internals.h.

{
    BUFFER_IO_ALREADY_DONE,
    BUFFER_IO_IN_PROGRESS,
    BUFFER_IO_READY_FOR_IO,
} StartBufferIOResult;

Function Documentation

◆ AtEOXact_LocalBuffers()

void AtEOXact_LocalBuffers ( bool isCommit )

extern

Definition at line 1027 of file localbuf.c.

{
    CheckForLocalBufferLeaks();
}

References CheckForLocalBufferLeaks().

Referenced by AtEOXact_Buffers().

◆ BufferDescriptorGetBuffer()

static Buffer BufferDescriptorGetBuffer ( const BufferDesc * bdesc )

inlinestatic

Definition at line 436 of file buf_internals.h.

{
    return (Buffer) (bdesc->buf_id + 1);
}

References fb().

Referenced by AddBufferToRing(), BufferLockHeldByMe(), BufferLockHeldByMeInMode(), ExtendBufferedRelLocal(), ExtendBufferedRelShared(), FlushLocalBuffer(), FlushRelationBuffers(), FlushUnlockedBuffer(), GetBufferFromRing(), GetLocalVictimBuffer(), GetVictimBuffer(), InvalidateBuffer(), InvalidateLocalBuffer(), InvalidateVictimBuffer(), LockBufHdr(), pg_buffercache_os_pages_internal(), pg_buffercache_pages(), PinBuffer(), PinBuffer_Locked(), PinBufferForBlock(), PinLocalBuffer(), StartSharedBufferIO(), StrategyGetBuffer(), StrategyRejectBuffer(), TerminateBufferIO(), UnpinBuffer(), and UnpinBufferNoOwner().

◆ BufferDescriptorGetIOCV()

static ConditionVariable * BufferDescriptorGetIOCV ( const BufferDesc * bdesc )

inlinestatic

Definition at line 442 of file buf_internals.h.

{
    return &(BufferIOCVArray[bdesc->buf_id]).cv;
}

References BufferIOCVArray, and fb().

Referenced by BufferManagerShmemInit(), TerminateBufferIO(), and WaitIO().

◆ BufferTagsEqual()

static bool BufferTagsEqual	(	const BufferTag *	tag1,
		const BufferTag *	tag2
	)

inlinestatic

Definition at line 222 of file buf_internals.h.

{
    return (tag1->spcOid == tag2->spcOid) &&
        (tag1->dbOid == tag2->dbOid) &&
        (tag1->relNumber == tag2->relNumber) &&
        (tag1->blockNum == tag2->blockNum) &&
        (tag1->forkNum == tag2->forkNum);
}

References fb().

Referenced by InvalidateBuffer(), InvalidateVictimBuffer(), LocalBufferAlloc(), and ReadRecentBuffer().

◆ BufMappingPartitionLock()

static LWLock * BufMappingPartitionLock ( uint32 hashcode )

inlinestatic

Definition at line 254 of file buf_internals.h.

{
    return &MainLWLockArray[BUFFER_MAPPING_LWLOCK_OFFSET +
                            BufTableHashPartition(hashcode)].lock;
}

References BUFFER_MAPPING_LWLOCK_OFFSET, BufTableHashPartition(), LWLockPadded::lock, and MainLWLockArray.

Referenced by BufferAlloc(), ExtendBufferedRelShared(), FindAndDropRelationBuffers(), InvalidateBuffer(), InvalidateVictimBuffer(), and PrefetchSharedBuffer().

◆ BufMappingPartitionLockByIndex()

static LWLock * BufMappingPartitionLockByIndex ( uint32 index )

inlinestatic

Definition at line 261 of file buf_internals.h.

{
    return &MainLWLockArray[BUFFER_MAPPING_LWLOCK_OFFSET + index].lock;
}

References BUFFER_MAPPING_LWLOCK_OFFSET, LWLockPadded::lock, and MainLWLockArray.

◆ BufTableDelete()

void BufTableDelete	(	BufferTag *	tagPtr,
		uint32	hashcode
	)

extern

Definition at line 154 of file buf_table.c.

{
    BufferLookupEnt *result;
 
    result = (BufferLookupEnt *)
        hash_search_with_hash_value(SharedBufHash,
                                    tagPtr,
                                    hashcode,
                                    HASH_REMOVE,
                                    NULL);
 
    if (!result)                /* shouldn't happen */
        elog(ERROR, "shared buffer hash table corrupted");
}

References elog, ERROR, fb(), HASH_REMOVE, hash_search_with_hash_value(), result, and SharedBufHash.

Referenced by InvalidateBuffer(), and InvalidateVictimBuffer().

◆ BufTableHashCode()

uint32 BufTableHashCode ( BufferTag * tagPtr )

extern

Definition at line 84 of file buf_table.c.

{
    return get_hash_value(SharedBufHash, tagPtr);
}

References fb(), get_hash_value(), and SharedBufHash.

Referenced by BufferAlloc(), ExtendBufferedRelShared(), FindAndDropRelationBuffers(), InvalidateBuffer(), InvalidateVictimBuffer(), and PrefetchSharedBuffer().

◆ BufTableHashPartition()

static uint32 BufTableHashPartition ( uint32 hashcode )

inlinestatic

Definition at line 248 of file buf_internals.h.

{
    return hashcode % NUM_BUFFER_PARTITIONS;
}

References NUM_BUFFER_PARTITIONS.

Referenced by BufMappingPartitionLock().

◆ BufTableInsert()

int BufTableInsert	(	BufferTag *	tagPtr,
		uint32	hashcode,
		int	buf_id
	)

extern

Definition at line 124 of file buf_table.c.

{
    BufferLookupEnt *result;
    bool        found;
 
    Assert(buf_id >= 0);        /* -1 is reserved for not-in-table */
    Assert(tagPtr->blockNum != P_NEW);  /* invalid tag */
 
    result = (BufferLookupEnt *)
        hash_search_with_hash_value(SharedBufHash,
                                    tagPtr,
                                    hashcode,
                                    HASH_ENTER,
                                    &found);
 
    if (found)                  /* found something already in the table */
        return result->id;
 
    result->id = buf_id;
 
    return -1;
}

References Assert, fb(), HASH_ENTER, hash_search_with_hash_value(), P_NEW, result, and SharedBufHash.

Referenced by BufferAlloc(), and ExtendBufferedRelShared().

◆ BufTableLookup()

int BufTableLookup	(	BufferTag *	tagPtr,
		uint32	hashcode
	)

extern

Definition at line 96 of file buf_table.c.

{
    BufferLookupEnt *result;
 
    result = (BufferLookupEnt *)
        hash_search_with_hash_value(SharedBufHash,
                                    tagPtr,
                                    hashcode,
                                    HASH_FIND,
                                    NULL);
 
    if (!result)
        return -1;
 
    return result->id;
}

References fb(), HASH_FIND, hash_search_with_hash_value(), BufferLookupEnt::id, result, and SharedBufHash.

Referenced by BufferAlloc(), FindAndDropRelationBuffers(), and PrefetchSharedBuffer().

◆ BufTagGetForkNum()

static ForkNumber BufTagGetForkNum ( const BufferTag * tag )

inlinestatic

Definition at line 177 of file buf_internals.h.

{
    return tag->forkNum;
}

References buftag::forkNum.

Referenced by AbortBufferIO(), apw_dump_now(), BufferGetTag(), BufferSync(), buffertag_comparator(), DebugPrintBufferRefcount(), DropRelationBuffers(), DropRelationLocalBuffers(), FindAndDropRelationBuffers(), FlushBuffer(), FlushLocalBuffer(), InvalidateLocalBuffer(), IssuePendingWritebacks(), local_buffer_write_error_callback(), pg_buffercache_pages(), ReleaseAndReadBuffer(), and shared_buffer_write_error_callback().

◆ BufTagGetRelFileLocator()

static RelFileLocator BufTagGetRelFileLocator ( const BufferTag * tag )

inlinestatic

Definition at line 191 of file buf_internals.h.

{
    RelFileLocator rlocator;
 
    rlocator.spcOid = tag->spcOid;
    rlocator.dbOid = tag->dbOid;
    rlocator.relNumber = BufTagGetRelNumber(tag);
 
    return rlocator;
}

References BufTagGetRelNumber(), buftag::dbOid, RelFileLocator::dbOid, RelFileLocator::relNumber, buftag::spcOid, and RelFileLocator::spcOid.

Referenced by AbortBufferIO(), BufferGetTag(), buffertag_comparator(), DebugPrintBufferRefcount(), DropRelationsAllBuffers(), FlushBuffer(), FlushLocalBuffer(), FlushRelationsAllBuffers(), InvalidateLocalBuffer(), IssuePendingWritebacks(), local_buffer_write_error_callback(), MarkSharedBufferDirtyHint(), and shared_buffer_write_error_callback().

◆ BufTagGetRelNumber()

static RelFileNumber BufTagGetRelNumber ( const BufferTag * tag )

inlinestatic

Definition at line 171 of file buf_internals.h.

{
    return tag->relNumber;
}

References buftag::relNumber.

Referenced by apw_dump_now(), BufferSync(), BufTagGetRelFileLocator(), BufTagMatchesRelFileLocator(), and pg_buffercache_pages().

◆ BufTagMatchesRelFileLocator()

static bool BufTagMatchesRelFileLocator	(	const BufferTag *	tag,
		const RelFileLocator *	rlocator
	)

inlinestatic

Definition at line 232 of file buf_internals.h.

{
    return (tag->spcOid == rlocator->spcOid) &&
        (tag->dbOid == rlocator->dbOid) &&
        (BufTagGetRelNumber(tag) == rlocator->relNumber);
}

References BufTagGetRelNumber(), buftag::dbOid, RelFileLocator::dbOid, RelFileLocator::relNumber, buftag::spcOid, and RelFileLocator::spcOid.

Referenced by DropRelationAllLocalBuffers(), DropRelationBuffers(), DropRelationLocalBuffers(), DropRelationsAllBuffers(), EvictRelUnpinnedBuffers(), FindAndDropRelationBuffers(), FlushRelationBuffers(), FlushRelationsAllBuffers(), MarkDirtyRelUnpinnedBuffers(), and ReleaseAndReadBuffer().

◆ BufTagSetRelForkDetails()

static void BufTagSetRelForkDetails	(	BufferTag *	tag,
		RelFileNumber	relnumber,
		ForkNumber	forknum
	)

inlinestatic

Definition at line 183 of file buf_internals.h.

{
    tag->relNumber = relnumber;
    tag->forkNum = forknum;
}

References buftag::forkNum, and buftag::relNumber.

Referenced by ClearBufferTag(), and InitBufferTag().

◆ ClearBufferTag()

static void ClearBufferTag ( BufferTag * tag )

inlinestatic

Definition at line 203 of file buf_internals.h.

{
    tag->spcOid = InvalidOid;
    tag->dbOid = InvalidOid;
    BufTagSetRelForkDetails(tag, InvalidRelFileNumber, InvalidForkNumber);
    tag->blockNum = InvalidBlockNumber;
}

References buftag::blockNum, BufTagSetRelForkDetails(), buftag::dbOid, InvalidBlockNumber, InvalidForkNumber, InvalidOid, InvalidRelFileNumber, and buftag::spcOid.

Referenced by BufferManagerShmemInit(), InvalidateBuffer(), InvalidateLocalBuffer(), and InvalidateVictimBuffer().

◆ DropRelationAllLocalBuffers()

void DropRelationAllLocalBuffers ( RelFileLocator rlocator )

extern

Definition at line 726 of file localbuf.c.

{
    int         i;
 
    for (i = 0; i < NLocBuffer; i++)
    {
        BufferDesc *bufHdr = GetLocalBufferDescriptor(i);
        uint64      buf_state;
 
        buf_state = pg_atomic_read_u64(&bufHdr->state);
 
        if ((buf_state & BM_TAG_VALID) &&
            BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator))
        {
            InvalidateLocalBuffer(bufHdr, true);
        }
    }
}

References BM_TAG_VALID, BufTagMatchesRelFileLocator(), fb(), GetLocalBufferDescriptor(), i, InvalidateLocalBuffer(), NLocBuffer, and pg_atomic_read_u64().

Referenced by DropRelationsAllBuffers().

◆ DropRelationLocalBuffers()

void DropRelationLocalBuffers	(	RelFileLocator	rlocator,
		ForkNumber *	forkNum,
		int	nforks,
		BlockNumber *	firstDelBlock
	)

extern

Definition at line 689 of file localbuf.c.

{
    int         i;
    int         j;
 
    for (i = 0; i < NLocBuffer; i++)
    {
        BufferDesc *bufHdr = GetLocalBufferDescriptor(i);
        uint64      buf_state;
 
        buf_state = pg_atomic_read_u64(&bufHdr->state);
 
        if (!(buf_state & BM_TAG_VALID) ||
            !BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator))
            continue;
 
        for (j = 0; j < nforks; j++)
        {
            if (BufTagGetForkNum(&bufHdr->tag) == forkNum[j] &&
                bufHdr->tag.blockNum >= firstDelBlock[j])
            {
                InvalidateLocalBuffer(bufHdr, true);
                break;
            }
        }
    }
}

References BM_TAG_VALID, BufTagGetForkNum(), BufTagMatchesRelFileLocator(), fb(), GetLocalBufferDescriptor(), i, InvalidateLocalBuffer(), j, NLocBuffer, and pg_atomic_read_u64().

Referenced by DropRelationBuffers().

◆ ExtendBufferedRelLocal()

BlockNumber ExtendBufferedRelLocal	(	BufferManagerRelation	bmr,
		ForkNumber	fork,
		uint32	flags,
		uint32	extend_by,
		BlockNumber	extend_upto,
		Buffer *	buffers,
		uint32 *	extended_by
	)

extern

Definition at line 355 of file localbuf.c.

{
    BlockNumber first_block;
    instr_time  io_start;
 
    /* Initialize local buffers if first request in this session */
    if (LocalBufHash == NULL)
        InitLocalBuffers();
 
    LimitAdditionalLocalPins(&extend_by);
 
    for (uint32 i = 0; i < extend_by; i++)
    {
        BufferDesc *buf_hdr;
        Block       buf_block;
 
        buffers[i] = GetLocalVictimBuffer();
        buf_hdr = GetLocalBufferDescriptor(-buffers[i] - 1);
        buf_block = LocalBufHdrGetBlock(buf_hdr);
 
        /* new buffers are zero-filled */
        MemSet(buf_block, 0, BLCKSZ);
    }
 
    first_block = smgrnblocks(BMR_GET_SMGR(bmr), fork);
 
    if (extend_upto != InvalidBlockNumber)
    {
        /*
         * In contrast to shared relations, nothing could change the relation
         * size concurrently. Thus we shouldn't end up finding that we don't
         * need to do anything.
         */
        Assert(first_block <= extend_upto);
 
        Assert((uint64) first_block + extend_by <= extend_upto);
    }
 
    /* Fail if relation is already at maximum possible length */
    if ((uint64) first_block + extend_by >= MaxBlockNumber)
        ereport(ERROR,
                (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
                 errmsg("cannot extend relation %s beyond %u blocks",
                        relpath(BMR_GET_SMGR(bmr)->smgr_rlocator, fork).str,
                        MaxBlockNumber)));
 
    for (uint32 i = 0; i < extend_by; i++)
    {
        int         victim_buf_id;
        BufferDesc *victim_buf_hdr;
        BufferTag   tag;
        LocalBufferLookupEnt *hresult;
        bool        found;
 
        victim_buf_id = -buffers[i] - 1;
        victim_buf_hdr = GetLocalBufferDescriptor(victim_buf_id);
 
        /* in case we need to pin an existing buffer below */
        ResourceOwnerEnlarge(CurrentResourceOwner);
 
        InitBufferTag(&tag, &BMR_GET_SMGR(bmr)->smgr_rlocator.locator, fork,
                      first_block + i);
 
        hresult = (LocalBufferLookupEnt *)
            hash_search(LocalBufHash, &tag, HASH_ENTER, &found);
        if (found)
        {
            BufferDesc *existing_hdr;
            uint64      buf_state;
 
            UnpinLocalBuffer(BufferDescriptorGetBuffer(victim_buf_hdr));
 
            existing_hdr = GetLocalBufferDescriptor(hresult->id);
            PinLocalBuffer(existing_hdr, false);
            buffers[i] = BufferDescriptorGetBuffer(existing_hdr);
 
            /*
             * Clear the BM_VALID bit, do StartLocalBufferIO() and proceed.
             */
            buf_state = pg_atomic_read_u64(&existing_hdr->state);
            Assert(buf_state & BM_TAG_VALID);
            Assert(!(buf_state & BM_DIRTY));
            buf_state &= ~BM_VALID;
            pg_atomic_unlocked_write_u64(&existing_hdr->state, buf_state);
 
            /* no need to loop for local buffers */
            StartLocalBufferIO(existing_hdr, true, true, NULL);
        }
        else
        {
            uint64      buf_state = pg_atomic_read_u64(&victim_buf_hdr->state);
 
            Assert(!(buf_state & (BM_VALID | BM_TAG_VALID | BM_DIRTY)));
 
            victim_buf_hdr->tag = tag;
 
            buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE;
 
            pg_atomic_unlocked_write_u64(&victim_buf_hdr->state, buf_state);
 
            hresult->id = victim_buf_id;
 
            StartLocalBufferIO(victim_buf_hdr, true, true, NULL);
        }
    }
 
    io_start = pgstat_prepare_io_time(track_io_timing);
 
    /* actually extend relation */
    smgrzeroextend(BMR_GET_SMGR(bmr), fork, first_block, extend_by, false);
 
    pgstat_count_io_op_time(IOOBJECT_TEMP_RELATION, IOCONTEXT_NORMAL, IOOP_EXTEND,
                            io_start, 1, extend_by * BLCKSZ);
 
    for (uint32 i = 0; i < extend_by; i++)
    {
        Buffer      buf = buffers[i];
        BufferDesc *buf_hdr;
        uint64      buf_state;
 
        buf_hdr = GetLocalBufferDescriptor(-buf - 1);
 
        buf_state = pg_atomic_read_u64(&buf_hdr->state);
        buf_state |= BM_VALID;
        pg_atomic_unlocked_write_u64(&buf_hdr->state, buf_state);
    }
 
    *extended_by = extend_by;
 
    pgBufferUsage.local_blks_written += extend_by;
 
    return first_block;
}

References Assert, BM_DIRTY, BM_TAG_VALID, BM_VALID, BMR_GET_SMGR, buf, BUF_USAGECOUNT_ONE, BufferDescriptorGetBuffer(), CurrentResourceOwner, ereport, errcode(), errmsg, ERROR, fb(), GetLocalBufferDescriptor(), GetLocalVictimBuffer(), HASH_ENTER, hash_search(), i, InitBufferTag(), InitLocalBuffers(), InvalidBlockNumber, IOCONTEXT_NORMAL, IOOBJECT_TEMP_RELATION, IOOP_EXTEND, LimitAdditionalLocalPins(), BufferUsage::local_blks_written, LocalBufHash, LocalBufHdrGetBlock, MaxBlockNumber, MemSet, pg_atomic_read_u64(), pg_atomic_unlocked_write_u64(), pgBufferUsage, pgstat_count_io_op_time(), pgstat_prepare_io_time(), PinLocalBuffer(), relpath, ResourceOwnerEnlarge(), smgrnblocks(), smgrzeroextend(), StartLocalBufferIO(), str, track_io_timing, and UnpinLocalBuffer().

Referenced by ExtendBufferedRelCommon().

◆ FlushLocalBuffer()

void FlushLocalBuffer	(	BufferDesc *	bufHdr,
		SMgrRelation	reln
	)

extern

Definition at line 183 of file localbuf.c.

{
    instr_time  io_start;
    Page        localpage = (char *) LocalBufHdrGetBlock(bufHdr);
 
    Assert(LocalRefCount[-BufferDescriptorGetBuffer(bufHdr) - 1] > 0);
 
    /*
     * Try to start an I/O operation.  There currently are no reasons for
     * StartLocalBufferIO to return anything other than
     * BUFFER_IO_READY_FOR_IO, so we raise an error in that case.
     */
    if (StartLocalBufferIO(bufHdr, false, true, NULL) != BUFFER_IO_READY_FOR_IO)
        elog(ERROR, "failed to start write IO on local buffer");
 
    /* Find smgr relation for buffer */
    if (reln == NULL)
        reln = smgropen(BufTagGetRelFileLocator(&bufHdr->tag),
                        MyProcNumber);
 
    PageSetChecksum(localpage, bufHdr->tag.blockNum);
 
    io_start = pgstat_prepare_io_time(track_io_timing);
 
    /* And write... */
    smgrwrite(reln,
              BufTagGetForkNum(&bufHdr->tag),
              bufHdr->tag.blockNum,
              localpage,
              false);
 
    /* Temporary table I/O does not use Buffer Access Strategies */
    pgstat_count_io_op_time(IOOBJECT_TEMP_RELATION, IOCONTEXT_NORMAL,
                            IOOP_WRITE, io_start, 1, BLCKSZ);
 
    /* Mark not-dirty */
    TerminateLocalBufferIO(bufHdr, true, 0, false);
 
    pgBufferUsage.local_blks_written++;
}

References Assert, BUFFER_IO_READY_FOR_IO, BufferDescriptorGetBuffer(), BufTagGetForkNum(), BufTagGetRelFileLocator(), elog, ERROR, fb(), IOCONTEXT_NORMAL, IOOBJECT_TEMP_RELATION, IOOP_WRITE, BufferUsage::local_blks_written, LocalBufHdrGetBlock, LocalRefCount, MyProcNumber, PageSetChecksum(), pgBufferUsage, pgstat_count_io_op_time(), pgstat_prepare_io_time(), smgropen(), smgrwrite(), StartLocalBufferIO(), TerminateLocalBufferIO(), and track_io_timing.

Referenced by FlushRelationBuffers(), GetLocalVictimBuffer(), and invalidate_one_block().

◆ GetBufferDescriptor()

static BufferDesc * GetBufferDescriptor ( int id )

inlinestatic

Definition at line 422 of file buf_internals.h.

{
    Assert(id >= 0 && id < NBuffers);
 
    return &(BufferDescriptors[id]).bufferdesc;
}

References Assert, BufferDescriptors, and NBuffers.

Referenced by AbortBufferIO(), apw_dump_now(), buffer_call_start_io(), buffer_call_terminate_io(), buffer_readv_complete_one(), buffer_stage_common(), BufferAlloc(), BufferBeginSetHintBits(), BufferGetBlockNumber(), BufferGetLSNAtomic(), BufferGetTag(), BufferIsDirty(), BufferIsLockedByMe(), BufferIsLockedByMeInMode(), BufferIsPermanent(), BufferManagerShmemInit(), BufferSetHintBits16(), BufferSync(), CheckReadBuffersOperation(), ConditionalLockBuffer(), ConditionalLockBufferForCleanup(), create_toy_buffer(), DebugPrintBufferRefcount(), DropDatabaseBuffers(), DropRelationBuffers(), DropRelationsAllBuffers(), EvictAllUnpinnedBuffers(), EvictRelUnpinnedBuffers(), EvictUnpinnedBuffer(), ExtendBufferedRelShared(), FindAndDropRelationBuffers(), FlushDatabaseBuffers(), FlushOneBuffer(), FlushRelationBuffers(), FlushRelationsAllBuffers(), GetBufferFromRing(), invalidate_one_block(), IsBufferCleanupOK(), LockBufferForCleanup(), LockBufferInternal(), MarkBufferDirty(), MarkBufferDirtyHint(), MarkDirtyAllUnpinnedBuffers(), MarkDirtyRelUnpinnedBuffers(), MarkDirtyUnpinnedBuffer(), pg_buffercache_os_pages_internal(), pg_buffercache_pages(), pg_buffercache_summary(), pg_buffercache_usage_counts(), read_rel_block_ll(), ReadRecentBuffer(), ReleaseAndReadBuffer(), ReleaseBuffer(), ResOwnerReleaseBuffer(), StartBufferIO(), StartReadBuffersImpl(), StrategyGetBuffer(), SyncOneBuffer(), TrackNewBufferPin(), UnlockBuffer(), UnlockReleaseBuffer(), WaitReadBuffers(), and ZeroAndLockBuffer().

◆ GetLocalBufferDescriptor()

static BufferDesc * GetLocalBufferDescriptor ( int id )

inlinestatic

◆ InitBufferTag()

static void InitBufferTag	(	BufferTag *	tag,
		const RelFileLocator *	rlocator,
		ForkNumber	forkNum,
		BlockNumber	blockNum
	)

inlinestatic

Definition at line 212 of file buf_internals.h.

{
    tag->spcOid = rlocator->spcOid;
    tag->dbOid = rlocator->dbOid;
    BufTagSetRelForkDetails(tag, rlocator->relNumber, forkNum);
    tag->blockNum = blockNum;
}

References buftag::blockNum, BufTagSetRelForkDetails(), buftag::dbOid, RelFileLocator::dbOid, RelFileLocator::relNumber, buftag::spcOid, and RelFileLocator::spcOid.

Referenced by BufferAlloc(), ExtendBufferedRelLocal(), ExtendBufferedRelShared(), FindAndDropRelationBuffers(), LocalBufferAlloc(), PrefetchLocalBuffer(), PrefetchSharedBuffer(), and ReadRecentBuffer().

◆ InvalidateLocalBuffer()

void InvalidateLocalBuffer	(	BufferDesc *	bufHdr,
		bool	check_unreferenced
	)

extern

Definition at line 629 of file localbuf.c.

{
    Buffer      buffer = BufferDescriptorGetBuffer(bufHdr);
    int         bufid = -buffer - 1;
    uint64      buf_state;
    LocalBufferLookupEnt *hresult;
 
    /*
     * It's possible that we started IO on this buffer before e.g. aborting
     * the transaction that created a table. We need to wait for that IO to
     * complete before removing / reusing the buffer.
     */
    if (pgaio_wref_valid(&bufHdr->io_wref))
    {
        PgAioWaitRef iow = bufHdr->io_wref;
 
        pgaio_wref_wait(&iow);
        Assert(!pgaio_wref_valid(&bufHdr->io_wref));
    }
 
    buf_state = pg_atomic_read_u64(&bufHdr->state);
 
    /*
     * We need to test not just LocalRefCount[bufid] but also the BufferDesc
     * itself, as the latter is used to represent a pin by the AIO subsystem.
     * This can happen if AIO is initiated and then the query errors out.
     */
    if (check_unreferenced &&
        (LocalRefCount[bufid] != 0 || BUF_STATE_GET_REFCOUNT(buf_state) != 0))
        elog(ERROR, "block %u of %s is still referenced (local %d)",
             bufHdr->tag.blockNum,
             relpathbackend(BufTagGetRelFileLocator(&bufHdr->tag),
                            MyProcNumber,
                            BufTagGetForkNum(&bufHdr->tag)).str,
             LocalRefCount[bufid]);
 
    /* Remove entry from hashtable */
    hresult = (LocalBufferLookupEnt *)
        hash_search(LocalBufHash, &bufHdr->tag, HASH_REMOVE, NULL);
    if (!hresult)               /* shouldn't happen */
        elog(ERROR, "local buffer hash table corrupted");
    /* Mark buffer invalid */
    ClearBufferTag(&bufHdr->tag);
    buf_state &= ~BUF_FLAG_MASK;
    buf_state &= ~BUF_USAGECOUNT_MASK;
    pg_atomic_unlocked_write_u64(&bufHdr->state, buf_state);
}

References Assert, BUF_STATE_GET_REFCOUNT, BufferDescriptorGetBuffer(), BufTagGetForkNum(), BufTagGetRelFileLocator(), ClearBufferTag(), elog, ERROR, fb(), HASH_REMOVE, hash_search(), LocalBufHash, LocalRefCount, MyProcNumber, pg_atomic_read_u64(), pg_atomic_unlocked_write_u64(), pgaio_wref_valid(), pgaio_wref_wait(), and relpathbackend.

Referenced by DropRelationAllLocalBuffers(), DropRelationLocalBuffers(), GetLocalVictimBuffer(), invalidate_one_block(), and modify_rel_block().

◆ IOContextForStrategy()

IOContext IOContextForStrategy ( BufferAccessStrategy strategy )

extern

Definition at line 712 of file freelist.c.

{
    if (!strategy)
        return IOCONTEXT_NORMAL;
 
    switch (strategy->btype)
    {
        case BAS_NORMAL:
 
            /*
             * Currently, GetAccessStrategy() returns NULL for
             * BufferAccessStrategyType BAS_NORMAL, so this case is
             * unreachable.
             */
            pg_unreachable();
            return IOCONTEXT_NORMAL;
        case BAS_BULKREAD:
            return IOCONTEXT_BULKREAD;
        case BAS_BULKWRITE:
            return IOCONTEXT_BULKWRITE;
        case BAS_VACUUM:
            return IOCONTEXT_VACUUM;
    }
 
    elog(ERROR, "unrecognized BufferAccessStrategyType: %d", strategy->btype);
    pg_unreachable();
}

References BAS_BULKREAD, BAS_BULKWRITE, BAS_NORMAL, BAS_VACUUM, BufferAccessStrategyData::btype, elog, ERROR, IOCONTEXT_BULKREAD, IOCONTEXT_BULKWRITE, IOCONTEXT_NORMAL, IOCONTEXT_VACUUM, and pg_unreachable.

Referenced by AsyncReadBuffers(), ExtendBufferedRelShared(), ReadBuffer_common(), StartReadBuffersImpl(), and WaitReadBuffers().

◆ IssuePendingWritebacks()

void IssuePendingWritebacks	(	WritebackContext *	wb_context,
		IOContext	io_context
	)

extern

Definition at line 7789 of file bufmgr.c.

{
    instr_time  io_start;
    int         i;
 
    if (wb_context->nr_pending == 0)
        return;
 
    /*
     * Executing the writes in-order can make them a lot faster, and allows to
     * merge writeback requests to consecutive blocks into larger writebacks.
     */
    sort_pending_writebacks(wb_context->pending_writebacks,
                            wb_context->nr_pending);
 
    io_start = pgstat_prepare_io_time(track_io_timing);
 
    /*
     * Coalesce neighbouring writes, but nothing else. For that we iterate
     * through the, now sorted, array of pending flushes, and look forward to
     * find all neighbouring (or identical) writes.
     */
    for (i = 0; i < wb_context->nr_pending; i++)
    {
        PendingWriteback *cur;
        PendingWriteback *next;
        SMgrRelation reln;
        int         ahead;
        BufferTag   tag;
        RelFileLocator currlocator;
        Size        nblocks = 1;
 
        cur = &wb_context->pending_writebacks[i];
        tag = cur->tag;
        currlocator = BufTagGetRelFileLocator(&tag);
 
        /*
         * Peek ahead, into following writeback requests, to see if they can
         * be combined with the current one.
         */
        for (ahead = 0; i + ahead + 1 < wb_context->nr_pending; ahead++)
        {
 
            next = &wb_context->pending_writebacks[i + ahead + 1];
 
            /* different file, stop */
            if (!RelFileLocatorEquals(currlocator,
                                      BufTagGetRelFileLocator(&next->tag)) ||
                BufTagGetForkNum(&cur->tag) != BufTagGetForkNum(&next->tag))
                break;
 
            /* ok, block queued twice, skip */
            if (cur->tag.blockNum == next->tag.blockNum)
                continue;
 
            /* only merge consecutive writes */
            if (cur->tag.blockNum + 1 != next->tag.blockNum)
                break;
 
            nblocks++;
            cur = next;
        }
 
        i += ahead;
 
        /* and finally tell the kernel to write the data to storage */
        reln = smgropen(currlocator, INVALID_PROC_NUMBER);
        smgrwriteback(reln, BufTagGetForkNum(&tag), tag.blockNum, nblocks);
    }
 
    /*
     * Assume that writeback requests are only issued for buffers containing
     * blocks of permanent relations.
     */
    pgstat_count_io_op_time(IOOBJECT_RELATION, io_context,
                            IOOP_WRITEBACK, io_start, wb_context->nr_pending, 0);
 
    wb_context->nr_pending = 0;
}

References buftag::blockNum, BufTagGetForkNum(), BufTagGetRelFileLocator(), cur, fb(), i, INVALID_PROC_NUMBER, IOOBJECT_RELATION, IOOP_WRITEBACK, next, pgstat_count_io_op_time(), pgstat_prepare_io_time(), RelFileLocatorEquals, smgropen(), smgrwriteback(), and track_io_timing.

Referenced by BufferSync(), and ScheduleBufferTagForWriteback().

◆ LocalBufferAlloc()

BufferDesc * LocalBufferAlloc	(	SMgrRelation	smgr,
		ForkNumber	forkNum,
		BlockNumber	blockNum,
		bool *	foundPtr
	)

extern

Definition at line 119 of file localbuf.c.

{
    BufferTag   newTag;         /* identity of requested block */
    LocalBufferLookupEnt *hresult;
    BufferDesc *bufHdr;
    Buffer      victim_buffer;
    int         bufid;
    bool        found;
 
    InitBufferTag(&newTag, &smgr->smgr_rlocator.locator, forkNum, blockNum);
 
    /* Initialize local buffers if first request in this session */
    if (LocalBufHash == NULL)
        InitLocalBuffers();
 
    ResourceOwnerEnlarge(CurrentResourceOwner);
 
    /* See if the desired buffer already exists */
    hresult = (LocalBufferLookupEnt *)
        hash_search(LocalBufHash, &newTag, HASH_FIND, NULL);
 
    if (hresult)
    {
        bufid = hresult->id;
        bufHdr = GetLocalBufferDescriptor(bufid);
        Assert(BufferTagsEqual(&bufHdr->tag, &newTag));
 
        *foundPtr = PinLocalBuffer(bufHdr, true);
    }
    else
    {
        uint64      buf_state;
 
        victim_buffer = GetLocalVictimBuffer();
        bufid = -victim_buffer - 1;
        bufHdr = GetLocalBufferDescriptor(bufid);
 
        hresult = (LocalBufferLookupEnt *)
            hash_search(LocalBufHash, &newTag, HASH_ENTER, &found);
        if (found)              /* shouldn't happen */
            elog(ERROR, "local buffer hash table corrupted");
        hresult->id = bufid;
 
        /*
         * it's all ours now.
         */
        bufHdr->tag = newTag;
 
        buf_state = pg_atomic_read_u64(&bufHdr->state);
        buf_state &= ~(BUF_FLAG_MASK | BUF_USAGECOUNT_MASK);
        buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE;
        pg_atomic_unlocked_write_u64(&bufHdr->state, buf_state);
 
        *foundPtr = false;
    }
 
    return bufHdr;
}

References Assert, BM_TAG_VALID, BUF_FLAG_MASK, BUF_USAGECOUNT_MASK, BUF_USAGECOUNT_ONE, BufferTagsEqual(), CurrentResourceOwner, elog, ERROR, fb(), GetLocalBufferDescriptor(), GetLocalVictimBuffer(), HASH_ENTER, HASH_FIND, hash_search(), InitBufferTag(), InitLocalBuffers(), LocalBufHash, RelFileLocatorBackend::locator, pg_atomic_read_u64(), pg_atomic_unlocked_write_u64(), PinLocalBuffer(), ResourceOwnerEnlarge(), and SMgrRelationData::smgr_rlocator.

Referenced by PinBufferForBlock().

◆ LockBufHdr()

uint64 LockBufHdr ( BufferDesc * desc )

extern

Definition at line 7567 of file bufmgr.c.

{
    uint64      old_buf_state;
 
    Assert(!BufferIsLocal(BufferDescriptorGetBuffer(desc)));
 
    while (true)
    {
        /*
         * Always try once to acquire the lock directly, without setting up
         * the spin-delay infrastructure. The work necessary for that shows up
         * in profiles and is rarely necessary.
         */
        old_buf_state = pg_atomic_fetch_or_u64(&desc->state, BM_LOCKED);
        if (likely(!(old_buf_state & BM_LOCKED)))
            break;              /* got lock */
 
        /* and then spin without atomic operations until lock is released */
        {
            SpinDelayStatus delayStatus;
 
            init_local_spin_delay(&delayStatus);
 
            while (old_buf_state & BM_LOCKED)
            {
                perform_spin_delay(&delayStatus);
                old_buf_state = pg_atomic_read_u64(&desc->state);
            }
            finish_spin_delay(&delayStatus);
        }
 
        /*
         * Retry. The lock might obviously already be re-acquired by the time
         * we're attempting to get it again.
         */
    }
 
    return old_buf_state | BM_LOCKED;
}

References Assert, BM_LOCKED, BufferDescriptorGetBuffer(), BufferIsLocal, fb(), finish_spin_delay(), init_local_spin_delay, likely, perform_spin_delay(), pg_atomic_fetch_or_u64(), pg_atomic_read_u64(), and BufferDesc::state.

Referenced by AbortBufferIO(), apw_dump_now(), buffer_stage_common(), BufferAlloc(), BufferGetLSNAtomic(), BufferLockDequeueSelf(), BufferLockQueueSelf(), BufferLockWakeup(), BufferSync(), ConditionalLockBufferForCleanup(), create_toy_buffer(), DropDatabaseBuffers(), DropRelationBuffers(), DropRelationsAllBuffers(), EvictAllUnpinnedBuffers(), EvictRelUnpinnedBuffers(), EvictUnpinnedBuffer(), ExtendBufferedRelShared(), FindAndDropRelationBuffers(), FlushDatabaseBuffers(), FlushRelationBuffers(), FlushRelationsAllBuffers(), InvalidateBuffer(), InvalidateVictimBuffer(), IsBufferCleanupOK(), LockBufferForCleanup(), MarkDirtyAllUnpinnedBuffers(), MarkDirtyRelUnpinnedBuffers(), MarkDirtyUnpinnedBuffer(), MarkSharedBufferDirtyHint(), pg_buffercache_os_pages_internal(), pg_buffercache_pages(), StartSharedBufferIO(), SyncOneBuffer(), TerminateBufferIO(), UnlockBuffers(), WaitIO(), and WakePinCountWaiter().

◆ MarkLocalBufferDirty()

void MarkLocalBufferDirty ( Buffer buffer )

extern

Definition at line 500 of file localbuf.c.

{
    int         bufid;
    BufferDesc *bufHdr;
    uint64      buf_state;
 
    Assert(BufferIsLocal(buffer));
 
#ifdef LBDEBUG
    fprintf(stderr, "LB DIRTY %d\n", buffer);
#endif
 
    bufid = -buffer - 1;
 
    Assert(LocalRefCount[bufid] > 0);
 
    bufHdr = GetLocalBufferDescriptor(bufid);
 
    buf_state = pg_atomic_read_u64(&bufHdr->state);
 
    if (!(buf_state & BM_DIRTY))
        pgBufferUsage.local_blks_dirtied++;
 
    buf_state |= BM_DIRTY;
 
    pg_atomic_unlocked_write_u64(&bufHdr->state, buf_state);
}

References Assert, BM_DIRTY, BufferIsLocal, fb(), fprintf, GetLocalBufferDescriptor(), BufferUsage::local_blks_dirtied, LocalRefCount, pg_atomic_read_u64(), pg_atomic_unlocked_write_u64(), and pgBufferUsage.

Referenced by BufferSetHintBits16(), MarkBufferDirty(), and MarkBufferDirtyHint().

◆ PinLocalBuffer()

bool PinLocalBuffer	(	BufferDesc *	buf_hdr,
		bool	adjust_usagecount
	)

extern

Definition at line 829 of file localbuf.c.

{
    uint64      buf_state;
    Buffer      buffer = BufferDescriptorGetBuffer(buf_hdr);
    int         bufid = -buffer - 1;
 
    buf_state = pg_atomic_read_u64(&buf_hdr->state);
 
    if (LocalRefCount[bufid] == 0)
    {
        NLocalPinnedBuffers++;
        buf_state += BUF_REFCOUNT_ONE;
        if (adjust_usagecount &&
            BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT)
        {
            buf_state += BUF_USAGECOUNT_ONE;
        }
        pg_atomic_unlocked_write_u64(&buf_hdr->state, buf_state);
 
        /*
         * See comment in PinBuffer().
         *
         * If the buffer isn't allocated yet, it'll be marked as defined in
         * GetLocalBufferStorage().
         */
        if (LocalBufHdrGetBlock(buf_hdr) != NULL)
            VALGRIND_MAKE_MEM_DEFINED(LocalBufHdrGetBlock(buf_hdr), BLCKSZ);
    }
    LocalRefCount[bufid]++;
    ResourceOwnerRememberBuffer(CurrentResourceOwner,
                                BufferDescriptorGetBuffer(buf_hdr));
 
    return buf_state & BM_VALID;
}

References BM_MAX_USAGE_COUNT, BM_VALID, BUF_REFCOUNT_ONE, BUF_STATE_GET_USAGECOUNT, BUF_USAGECOUNT_ONE, BufferDescriptorGetBuffer(), CurrentResourceOwner, fb(), LocalBufHdrGetBlock, LocalRefCount, NLocalPinnedBuffers, pg_atomic_read_u64(), pg_atomic_unlocked_write_u64(), ResourceOwnerRememberBuffer(), and VALGRIND_MAKE_MEM_DEFINED.

Referenced by ExtendBufferedRelLocal(), FlushRelationBuffers(), GetLocalVictimBuffer(), LocalBufferAlloc(), and ReadRecentBuffer().

◆ PrefetchLocalBuffer()

PrefetchBufferResult PrefetchLocalBuffer	(	SMgrRelation	smgr,
		ForkNumber	forkNum,
		BlockNumber	blockNum
	)

extern

Definition at line 72 of file localbuf.c.

{
    PrefetchBufferResult result = {InvalidBuffer, false};
    BufferTag   newTag;         /* identity of requested block */
    LocalBufferLookupEnt *hresult;
 
    InitBufferTag(&newTag, &smgr->smgr_rlocator.locator, forkNum, blockNum);
 
    /* Initialize local buffers if first request in this session */
    if (LocalBufHash == NULL)
        InitLocalBuffers();
 
    /* See if the desired buffer already exists */
    hresult = (LocalBufferLookupEnt *)
        hash_search(LocalBufHash, &newTag, HASH_FIND, NULL);
 
    if (hresult)
    {
        /* Yes, so nothing to do */
        result.recent_buffer = -hresult->id - 1;
    }
    else
    {
#ifdef USE_PREFETCH
        /* Not in buffers, so initiate prefetch */
        if ((io_direct_flags & IO_DIRECT_DATA) == 0 &&
            smgrprefetch(smgr, forkNum, blockNum, 1))
        {
            result.initiated_io = true;
        }
#endif                          /* USE_PREFETCH */
    }
 
    return result;
}

References fb(), HASH_FIND, hash_search(), InitBufferTag(), InitLocalBuffers(), InvalidBuffer, IO_DIRECT_DATA, io_direct_flags, LocalBufHash, RelFileLocatorBackend::locator, result, SMgrRelationData::smgr_rlocator, and smgrprefetch().

Referenced by PrefetchBuffer().

◆ ResourceOwnerForgetBuffer()

static void ResourceOwnerForgetBuffer	(	ResourceOwner	owner,
		Buffer	buffer
	)

inlinestatic

Definition at line 538 of file buf_internals.h.

{
    ResourceOwnerForget(owner, Int32GetDatum(buffer), &buffer_resowner_desc);
}

References buffer_resowner_desc, Int32GetDatum(), and ResourceOwnerForget().

Referenced by UnlockReleaseBuffer(), UnpinBuffer(), and UnpinLocalBuffer().

◆ ResourceOwnerForgetBufferIO()

static void ResourceOwnerForgetBufferIO	(	ResourceOwner	owner,
		Buffer	buffer
	)

inlinestatic

Definition at line 548 of file buf_internals.h.

{
    ResourceOwnerForget(owner, Int32GetDatum(buffer), &buffer_io_resowner_desc);
}

References buffer_io_resowner_desc, Int32GetDatum(), and ResourceOwnerForget().

Referenced by buffer_call_start_io(), buffer_stage_common(), and TerminateBufferIO().

◆ ResourceOwnerRememberBuffer()

static void ResourceOwnerRememberBuffer	(	ResourceOwner	owner,
		Buffer	buffer
	)

inlinestatic

Definition at line 533 of file buf_internals.h.

{
    ResourceOwnerRemember(owner, Int32GetDatum(buffer), &buffer_resowner_desc);
}

References buffer_resowner_desc, Int32GetDatum(), and ResourceOwnerRemember().

Referenced by IncrBufferRefCount(), PinBuffer(), PinLocalBuffer(), and TrackNewBufferPin().

◆ ResourceOwnerRememberBufferIO()

static void ResourceOwnerRememberBufferIO	(	ResourceOwner	owner,
		Buffer	buffer
	)

inlinestatic

Definition at line 543 of file buf_internals.h.

{
    ResourceOwnerRemember(owner, Int32GetDatum(buffer), &buffer_io_resowner_desc);
}

References buffer_io_resowner_desc, Int32GetDatum(), and ResourceOwnerRemember().

Referenced by StartSharedBufferIO().

◆ ScheduleBufferTagForWriteback()

void ScheduleBufferTagForWriteback	(	WritebackContext *	wb_context,
		IOContext	io_context,
		BufferTag *	tag
	)

extern

Definition at line 7739 of file bufmgr.c.

{
    PendingWriteback *pending;
 
    /*
     * As pg_flush_data() doesn't do anything with fsync disabled, there's no
     * point in tracking in that case.
     */
    if (io_direct_flags & IO_DIRECT_DATA ||
        !enableFsync)
        return;
 
    /*
     * Add buffer to the pending writeback array, unless writeback control is
     * disabled.
     */
    if (*wb_context->max_pending > 0)
    {
        Assert(*wb_context->max_pending <= WRITEBACK_MAX_PENDING_FLUSHES);
 
        pending = &wb_context->pending_writebacks[wb_context->nr_pending++];
 
        pending->tag = *tag;
    }
 
    /*
     * Perform pending flushes if the writeback limit is exceeded. This
     * includes the case where previously an item has been added, but control
     * is now disabled.
     */
    if (wb_context->nr_pending >= *wb_context->max_pending)
        IssuePendingWritebacks(wb_context, io_context);
}

References Assert, enableFsync, fb(), IO_DIRECT_DATA, io_direct_flags, IssuePendingWritebacks(), PendingWriteback::tag, and WRITEBACK_MAX_PENDING_FLUSHES.

Referenced by GetVictimBuffer(), and SyncOneBuffer().

◆ StartBufferIO()

StartBufferIOResult StartBufferIO	(	Buffer	buffer,
		bool	forInput,
		bool	wait,
		PgAioWaitRef *	io_wref
	)

extern

Definition at line 7370 of file bufmgr.c.

{
    BufferDesc *buf_hdr;
 
    if (BufferIsLocal(buffer))
    {
        buf_hdr = GetLocalBufferDescriptor(-buffer - 1);
 
        return StartLocalBufferIO(buf_hdr, forInput, wait, io_wref);
    }
    else
    {
        buf_hdr = GetBufferDescriptor(buffer - 1);
 
        return StartSharedBufferIO(buf_hdr, forInput, wait, io_wref);
    }
}

References PrivateRefCountEntry::buffer, BufferIsLocal, fb(), GetBufferDescriptor(), GetLocalBufferDescriptor(), StartLocalBufferIO(), and StartSharedBufferIO().

Referenced by AsyncReadBuffers().

◆ StartLocalBufferIO()

StartBufferIOResult StartLocalBufferIO	(	BufferDesc *	bufHdr,
		bool	forInput,
		bool	wait,
		PgAioWaitRef *	io_wref
	)

extern

Definition at line 532 of file localbuf.c.

{
    uint64      buf_state;
 
    /*
     * With AIO the buffer could have IO in progress, e.g. when there are two
     * scans of the same relation.  Either wait for the other IO (if wait =
     * true and io_wref == NULL) or return BUFFER_IO_IN_PROGRESS;
     */
    if (pgaio_wref_valid(&bufHdr->io_wref))
    {
        PgAioWaitRef buf_wref = bufHdr->io_wref;
 
        if (io_wref != NULL)
        {
            /* We've already asynchronously started this IO, so join it */
            *io_wref = buf_wref;
            return BUFFER_IO_IN_PROGRESS;
        }
 
        /*
         * For temp buffers we should never need to wait in
         * StartLocalBufferIO() when called with io_wref == NULL while there
         * are staged IOs, as it's not allowed to call code that is not aware
         * of AIO while in batch mode.
         */
        Assert(!pgaio_have_staged());
 
        if (!wait)
            return BUFFER_IO_IN_PROGRESS;
 
        pgaio_wref_wait(&buf_wref);
    }
 
    /* Once we get here, there is definitely no I/O active on this buffer */
 
    /* Check if someone else already did the I/O */
    buf_state = pg_atomic_read_u64(&bufHdr->state);
    if (forInput ? (buf_state & BM_VALID) : !(buf_state & BM_DIRTY))
    {
        return BUFFER_IO_ALREADY_DONE;
    }
 
    /* BM_IO_IN_PROGRESS isn't currently used for local buffers */
 
    /* local buffers don't track IO using resowners */
 
    return BUFFER_IO_READY_FOR_IO;
}

References Assert, BM_DIRTY, BM_VALID, BUFFER_IO_ALREADY_DONE, BUFFER_IO_IN_PROGRESS, BUFFER_IO_READY_FOR_IO, fb(), pg_atomic_read_u64(), pgaio_have_staged(), pgaio_wref_valid(), and pgaio_wref_wait().

Referenced by buffer_call_start_io(), ExtendBufferedRelLocal(), FlushLocalBuffer(), read_rel_block_ll(), StartBufferIO(), and ZeroAndLockBuffer().

◆ StartSharedBufferIO()

StartBufferIOResult StartSharedBufferIO	(	BufferDesc *	buf,
		bool	forInput,
		bool	wait,
		PgAioWaitRef *	io_wref
	)

extern

Definition at line 7290 of file bufmgr.c.

{
    uint64      buf_state;
 
    ResourceOwnerEnlarge(CurrentResourceOwner);
 
    for (;;)
    {
        buf_state = LockBufHdr(buf);
 
        if (!(buf_state & BM_IO_IN_PROGRESS))
            break;
 
        /* Join the existing IO */
        if (io_wref != NULL && pgaio_wref_valid(&buf->io_wref))
        {
            *io_wref = buf->io_wref;
            UnlockBufHdr(buf);
 
            return BUFFER_IO_IN_PROGRESS;
        }
        else if (!wait)
        {
            UnlockBufHdr(buf);
            return BUFFER_IO_IN_PROGRESS;
        }
        else
        {
            /*
             * With wait = true, we always have to wait if the caller has
             * passed io_wref = NULL.
             *
             * Even with io_wref != NULL, we have to wait if the buffer's wait
             * ref is not valid but the IO is in progress, someone else
             * started IO but hasn't set the wait ref yet. We have no choice
             * but to wait until the IO completes.
             */
            UnlockBufHdr(buf);
 
            /*
             * If this backend currently has staged IO, submit it before
             * waiting for in-progress IO, to avoid potential deadlocks and
             * unnecessary delays.
             */
            pgaio_submit_staged();
 
            WaitIO(buf);
        }
    }
 
    /* Once we get here, there is definitely no I/O active on this buffer */
 
    /* Check if someone else already did the I/O */
    if (forInput ? (buf_state & BM_VALID) : !(buf_state & BM_DIRTY))
    {
        UnlockBufHdr(buf);
        return BUFFER_IO_ALREADY_DONE;
    }
 
    /*
     * No IO in progress and not already done; we will start IO. It's possible
     * that the IO was in progress but we're not done, because the IO errored
     * out. We'll do the IO ourselves.
     */
    UnlockBufHdrExt(buf, buf_state,
                    BM_IO_IN_PROGRESS, 0,
                    0);
 
    ResourceOwnerRememberBufferIO(CurrentResourceOwner,
                                  BufferDescriptorGetBuffer(buf));
 
    return BUFFER_IO_READY_FOR_IO;
}

References BM_DIRTY, BM_IO_IN_PROGRESS, BM_VALID, buf, BUFFER_IO_ALREADY_DONE, BUFFER_IO_IN_PROGRESS, BUFFER_IO_READY_FOR_IO, BufferDescriptorGetBuffer(), CurrentResourceOwner, fb(), LockBufHdr(), pgaio_submit_staged(), pgaio_wref_valid(), ResourceOwnerEnlarge(), ResourceOwnerRememberBufferIO(), UnlockBufHdr(), UnlockBufHdrExt(), and WaitIO().

Referenced by buffer_call_start_io(), ExtendBufferedRelShared(), FlushBuffer(), read_rel_block_ll(), StartBufferIO(), and ZeroAndLockBuffer().

◆ StaticAssertDecl() [1/4]

StaticAssertDecl ( )

◆ StaticAssertDecl() [2/4]

StaticAssertDecl	(	BUF_REFCOUNT_BITS+BUF_USAGECOUNT_BITS+BUF_FLAG_BITS+BUF_LOCK_BITS<=	64,
		"parts of buffer state space need to be <= 64"
	)

◆ StaticAssertDecl() [3/4]

StaticAssertDecl	(	MAX_BACKENDS_BITS<=	BUF_LOCK_BITS - 2,
		"MAX_BACKENDS_BITS needs to be <= BUF_LOCK_BITS - 2"
	)

◆ StaticAssertDecl() [4/4]

StaticAssertDecl	(	MAX_BACKENDS_BITS<=	BUF_REFCOUNT_BITS,
		"MAX_BACKENDS_BITS needs to be <= BUF_REFCOUNT_BITS"
	)

◆ StrategyGetBuffer()

BufferDesc * StrategyGetBuffer	(	BufferAccessStrategy	strategy,
		uint64 *	buf_state,
		bool *	from_ring
	)

extern

Definition at line 184 of file freelist.c.

{
    BufferDesc *buf;
    int         bgwprocno;
    int         trycounter;
 
    *from_ring = false;
 
    /*
     * If given a strategy object, see whether it can select a buffer. We
     * assume strategy objects don't need buffer_strategy_lock.
     */
    if (strategy != NULL)
    {
        buf = GetBufferFromRing(strategy, buf_state);
        if (buf != NULL)
        {
            *from_ring = true;
            return buf;
        }
    }
 
    /*
     * If asked, we need to waken the bgwriter. Since we don't want to rely on
     * a spinlock for this we force a read from shared memory once, and then
     * set the latch based on that value. We need to go through that length
     * because otherwise bgwprocno might be reset while/after we check because
     * the compiler might just reread from memory.
     *
     * This can possibly set the latch of the wrong process if the bgwriter
     * dies in the wrong moment. But since PGPROC->procLatch is never
     * deallocated the worst consequence of that is that we set the latch of
     * some arbitrary process.
     */
    bgwprocno = INT_ACCESS_ONCE(StrategyControl->bgwprocno);
    if (bgwprocno != -1)
    {
        /* reset bgwprocno first, before setting the latch */
        StrategyControl->bgwprocno = -1;
 
        /*
         * Not acquiring ProcArrayLock here which is slightly icky. It's
         * actually fine because procLatch isn't ever freed, so we just can
         * potentially set the wrong process' (or no process') latch.
         */
        SetLatch(&GetPGProcByNumber(bgwprocno)->procLatch);
    }
 
    /*
     * We count buffer allocation requests so that the bgwriter can estimate
     * the rate of buffer consumption.  Note that buffers recycled by a
     * strategy object are intentionally not counted here.
     */
    pg_atomic_fetch_add_u32(&StrategyControl->numBufferAllocs, 1);
 
    /* Use the "clock sweep" algorithm to find a free buffer */
    trycounter = NBuffers;
    for (;;)
    {
        uint64      old_buf_state;
        uint64      local_buf_state;
 
        buf = GetBufferDescriptor(ClockSweepTick());
 
        /*
         * Check whether the buffer can be used and pin it if so. Do this
         * using a CAS loop, to avoid having to lock the buffer header.
         */
        old_buf_state = pg_atomic_read_u64(&buf->state);
        for (;;)
        {
            local_buf_state = old_buf_state;
 
            /*
             * If the buffer is pinned or has a nonzero usage_count, we cannot
             * use it; decrement the usage_count (unless pinned) and keep
             * scanning.
             */
 
            if (BUF_STATE_GET_REFCOUNT(local_buf_state) != 0)
            {
                if (--trycounter == 0)
                {
                    /*
                     * We've scanned all the buffers without making any state
                     * changes, so all the buffers are pinned (or were when we
                     * looked at them). We could hope that someone will free
                     * one eventually, but it's probably better to fail than
                     * to risk getting stuck in an infinite loop.
                     */
                    elog(ERROR, "no unpinned buffers available");
                }
                break;
            }
 
            /* See equivalent code in PinBuffer() */
            if (unlikely(local_buf_state & BM_LOCKED))
            {
                old_buf_state = WaitBufHdrUnlocked(buf);
                continue;
            }
 
            if (BUF_STATE_GET_USAGECOUNT(local_buf_state) != 0)
            {
                local_buf_state -= BUF_USAGECOUNT_ONE;
 
                if (pg_atomic_compare_exchange_u64(&buf->state, &old_buf_state,
                                                   local_buf_state))
                {
                    trycounter = NBuffers;
                    break;
                }
            }
            else
            {
                /* pin the buffer if the CAS succeeds */
                local_buf_state += BUF_REFCOUNT_ONE;
 
                if (pg_atomic_compare_exchange_u64(&buf->state, &old_buf_state,
                                                   local_buf_state))
                {
                    /* Found a usable buffer */
                    if (strategy != NULL)
                        AddBufferToRing(strategy, buf);
                    *buf_state = local_buf_state;
 
                    TrackNewBufferPin(BufferDescriptorGetBuffer(buf));
 
                    return buf;
                }
            }
        }
    }
}

References AddBufferToRing(), BufferStrategyControl::bgwprocno, BM_LOCKED, buf, BUF_REFCOUNT_ONE, BUF_STATE_GET_REFCOUNT, BUF_STATE_GET_USAGECOUNT, BUF_USAGECOUNT_ONE, BufferDescriptorGetBuffer(), ClockSweepTick(), elog, ERROR, fb(), GetBufferDescriptor(), GetBufferFromRing(), GetPGProcByNumber, INT_ACCESS_ONCE, NBuffers, BufferStrategyControl::numBufferAllocs, pg_atomic_compare_exchange_u64(), pg_atomic_fetch_add_u32(), pg_atomic_read_u64(), SetLatch(), StrategyControl, TrackNewBufferPin(), unlikely, and WaitBufHdrUnlocked().

Referenced by GetVictimBuffer().

◆ StrategyNotifyBgWriter()

void StrategyNotifyBgWriter ( int bgwprocno )

extern

Definition at line 368 of file freelist.c.

{
    /*
     * We acquire buffer_strategy_lock just to ensure that the store appears
     * atomic to StrategyGetBuffer.  The bgwriter should call this rather
     * infrequently, so there's no performance penalty from being safe.
     */
    SpinLockAcquire(&StrategyControl->buffer_strategy_lock);
    StrategyControl->bgwprocno = bgwprocno;
    SpinLockRelease(&StrategyControl->buffer_strategy_lock);
}

References BufferStrategyControl::bgwprocno, BufferStrategyControl::buffer_strategy_lock, SpinLockAcquire(), SpinLockRelease(), and StrategyControl.

Referenced by BackgroundWriterMain().

◆ StrategyRejectBuffer()

bool StrategyRejectBuffer	(	BufferAccessStrategy	strategy,
		BufferDesc *	buf,
		bool	from_ring
	)

extern

Definition at line 752 of file freelist.c.

{
    /* We only do this in bulkread mode */
    if (strategy->btype != BAS_BULKREAD)
        return false;
 
    /* Don't muck with behavior of normal buffer-replacement strategy */
    if (!from_ring ||
        strategy->buffers[strategy->current] != BufferDescriptorGetBuffer(buf))
        return false;
 
    /*
     * Remove the dirty buffer from the ring; necessary to prevent infinite
     * loop if all ring members are dirty.
     */
    strategy->buffers[strategy->current] = InvalidBuffer;
 
    return true;
}

References BAS_BULKREAD, BufferAccessStrategyData::btype, buf, BufferDescriptorGetBuffer(), BufferAccessStrategyData::buffers, BufferAccessStrategyData::current, fb(), and InvalidBuffer.

Referenced by GetVictimBuffer().

◆ StrategySyncStart()

int StrategySyncStart	(	uint32 *	complete_passes,
		uint32 *	num_buf_alloc
	)

extern

Definition at line 331 of file freelist.c.

{
    uint32      nextVictimBuffer;
    int         result;
 
    SpinLockAcquire(&StrategyControl->buffer_strategy_lock);
    nextVictimBuffer = pg_atomic_read_u32(&StrategyControl->nextVictimBuffer);
    result = nextVictimBuffer % NBuffers;
 
    if (complete_passes)
    {
        *complete_passes = StrategyControl->completePasses;
 
        /*
         * Additionally add the number of wraparounds that happened before
         * completePasses could be incremented. C.f. ClockSweepTick().
         */
        *complete_passes += nextVictimBuffer / NBuffers;
    }
 
    if (num_buf_alloc)
    {
        *num_buf_alloc = pg_atomic_exchange_u32(&StrategyControl->numBufferAllocs, 0);
    }
    SpinLockRelease(&StrategyControl->buffer_strategy_lock);
    return result;
}

References BufferStrategyControl::buffer_strategy_lock, BufferStrategyControl::completePasses, fb(), NBuffers, BufferStrategyControl::nextVictimBuffer, BufferStrategyControl::numBufferAllocs, pg_atomic_exchange_u32(), pg_atomic_read_u32(), result, SpinLockAcquire(), SpinLockRelease(), and StrategyControl.

Referenced by BgBufferSync().

◆ TerminateBufferIO()

void TerminateBufferIO	(	BufferDesc *	buf,
		bool	clear_dirty,
		uint64	set_flag_bits,
		bool	forget_owner,
		bool	release_aio
	)

extern

Definition at line 7407 of file bufmgr.c.

{
    uint64      buf_state;
    uint64      unset_flag_bits = 0;
    int         refcount_change = 0;
 
    buf_state = LockBufHdr(buf);
 
    Assert(buf_state & BM_IO_IN_PROGRESS);
    unset_flag_bits |= BM_IO_IN_PROGRESS;
 
    /* Clear earlier errors, if this IO failed, it'll be marked again */
    unset_flag_bits |= BM_IO_ERROR;
 
    if (clear_dirty)
        unset_flag_bits |= BM_DIRTY | BM_CHECKPOINT_NEEDED;
 
    if (release_aio)
    {
        /* release ownership by the AIO subsystem */
        Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
        refcount_change = -1;
        pgaio_wref_clear(&buf->io_wref);
    }
 
    buf_state = UnlockBufHdrExt(buf, buf_state,
                                set_flag_bits, unset_flag_bits,
                                refcount_change);
 
    if (forget_owner)
        ResourceOwnerForgetBufferIO(CurrentResourceOwner,
                                    BufferDescriptorGetBuffer(buf));
 
    ConditionVariableBroadcast(BufferDescriptorGetIOCV(buf));
 
    /*
     * Support LockBufferForCleanup()
     *
     * We may have just released the last pin other than the waiter's. In most
     * cases, this backend holds another pin on the buffer. But, if, for
     * example, this backend is completing an IO issued by another backend, it
     * may be time to wake the waiter.
     */
    if (release_aio && (buf_state & BM_PIN_COUNT_WAITER))
        WakePinCountWaiter(buf);
}

References Assert, BM_CHECKPOINT_NEEDED, BM_DIRTY, BM_IO_ERROR, BM_IO_IN_PROGRESS, BM_PIN_COUNT_WAITER, buf, BUF_STATE_GET_REFCOUNT, BufferDescriptorGetBuffer(), BufferDescriptorGetIOCV(), ConditionVariableBroadcast(), CurrentResourceOwner, fb(), LockBufHdr(), pgaio_wref_clear(), ResourceOwnerForgetBufferIO(), UnlockBufHdrExt(), and WakePinCountWaiter().

Referenced by AbortBufferIO(), buffer_call_terminate_io(), buffer_readv_complete_one(), ExtendBufferedRelShared(), FlushBuffer(), and ZeroAndLockBuffer().

◆ TerminateLocalBufferIO()

void TerminateLocalBufferIO	(	BufferDesc *	bufHdr,
		bool	clear_dirty,
		uint64	set_flag_bits,
		bool	release_aio
	)

extern

Definition at line 586 of file localbuf.c.

{
    /* Only need to adjust flags */
    uint64      buf_state = pg_atomic_read_u64(&bufHdr->state);
 
    /* BM_IO_IN_PROGRESS isn't currently used for local buffers */
 
    /* Clear earlier errors, if this IO failed, it'll be marked again */
    buf_state &= ~BM_IO_ERROR;
 
    if (clear_dirty)
        buf_state &= ~BM_DIRTY;
 
    if (release_aio)
    {
        /* release pin held by IO subsystem, see also buffer_stage_common() */
        Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
        buf_state -= BUF_REFCOUNT_ONE;
        pgaio_wref_clear(&bufHdr->io_wref);
    }
 
    buf_state |= set_flag_bits;
    pg_atomic_unlocked_write_u64(&bufHdr->state, buf_state);
 
    /* local buffers don't track IO using resowners */
 
    /* local buffers don't use the IO CV, as no other process can see buffer */
 
    /* local buffers don't use BM_PIN_COUNT_WAITER, so no need to wake */
}

References Assert, BUF_REFCOUNT_ONE, BUF_STATE_GET_REFCOUNT, fb(), pg_atomic_read_u64(), pg_atomic_unlocked_write_u64(), and pgaio_wref_clear().

Referenced by buffer_call_terminate_io(), buffer_readv_complete_one(), FlushLocalBuffer(), and ZeroAndLockBuffer().

◆ TrackNewBufferPin()

void TrackNewBufferPin ( Buffer buf )

externinline

Definition at line 3535 of file bufmgr.c.

{
    PrivateRefCountEntry *ref;
 
    ref = NewPrivateRefCountEntry(buf);
    ref->data.refcount++;
 
    ResourceOwnerRememberBuffer(CurrentResourceOwner, buf);
 
    /*
     * This is the first pin for this page by this backend, mark its page as
     * defined to valgrind. While the page contents might not actually be
     * valid yet, we don't currently guarantee that such pages are marked
     * undefined or non-accessible.
     *
     * It's not necessarily the prettiest to do this here, but otherwise we'd
     * need this block of code in multiple places.
     */
    VALGRIND_MAKE_MEM_DEFINED(BufHdrGetBlock(GetBufferDescriptor(buf - 1)),
                              BLCKSZ);
}

References buf, BufHdrGetBlock, CurrentResourceOwner, fb(), GetBufferDescriptor(), NewPrivateRefCountEntry(), ResourceOwnerRememberBuffer(), and VALGRIND_MAKE_MEM_DEFINED.

Referenced by GetBufferFromRing(), PinBuffer(), PinBuffer_Locked(), and StrategyGetBuffer().

◆ UnlockBufHdr()

static void UnlockBufHdr ( BufferDesc * desc )

inlinestatic

Definition at line 460 of file buf_internals.h.

{
    Assert(pg_atomic_read_u64(&desc->state) & BM_LOCKED);
 
    pg_atomic_fetch_sub_u64(&desc->state, BM_LOCKED);
}

References Assert, BM_LOCKED, pg_atomic_fetch_sub_u64(), pg_atomic_read_u64(), and BufferDesc::state.

Referenced by AbortBufferIO(), apw_dump_now(), BufferGetLSNAtomic(), BufferLockDequeueSelf(), BufferLockQueueSelf(), ConditionalLockBufferForCleanup(), DropDatabaseBuffers(), DropRelationBuffers(), DropRelationsAllBuffers(), EvictRelUnpinnedBuffers(), EvictUnpinnedBufferInternal(), FindAndDropRelationBuffers(), FlushDatabaseBuffers(), FlushRelationBuffers(), FlushRelationsAllBuffers(), InvalidateBuffer(), InvalidateVictimBuffer(), IsBufferCleanupOK(), LockBufferForCleanup(), MarkDirtyRelUnpinnedBuffers(), MarkDirtyUnpinnedBufferInternal(), MarkSharedBufferDirtyHint(), pg_buffercache_os_pages_internal(), pg_buffercache_pages(), StartSharedBufferIO(), SyncOneBuffer(), WaitIO(), and WakePinCountWaiter().

◆ UnlockBufHdrExt()

static uint64 UnlockBufHdrExt	(	BufferDesc *	desc,
		uint64	old_buf_state,
		uint64	set_bits,
		uint64	unset_bits,
		int	refcount_change
	)

inlinestatic

Definition at line 477 of file buf_internals.h.

{
    for (;;)
    {
        uint64      buf_state = old_buf_state;
 
        Assert(buf_state & BM_LOCKED);
 
        /*
         * Set bits after clearing bits, so that a cleared and re-added flag
         * survives.
         */
        buf_state &= ~unset_bits;
        buf_state |= set_bits;
        buf_state &= ~BM_LOCKED;
 
        if (refcount_change != 0)
            buf_state += BUF_REFCOUNT_ONE * refcount_change;
 
        if (pg_atomic_compare_exchange_u64(&desc->state, &old_buf_state,
                                           buf_state))
        {
            return old_buf_state;
        }
    }
}

References Assert, BM_LOCKED, BUF_REFCOUNT_ONE, fb(), pg_atomic_compare_exchange_u64(), and BufferDesc::state.

Referenced by buffer_stage_common(), BufferAlloc(), BufferSync(), create_toy_buffer(), ExtendBufferedRelShared(), InvalidateBuffer(), InvalidateVictimBuffer(), LockBufferForCleanup(), MarkSharedBufferDirtyHint(), PinBuffer_Locked(), StartSharedBufferIO(), TerminateBufferIO(), UnlockBuffers(), and WakePinCountWaiter().

◆ UnpinLocalBuffer()

void UnpinLocalBuffer ( Buffer buffer )

extern

Definition at line 865 of file localbuf.c.

{
    UnpinLocalBufferNoOwner(buffer);
    ResourceOwnerForgetBuffer(CurrentResourceOwner, buffer);
}

References CurrentResourceOwner, ResourceOwnerForgetBuffer(), and UnpinLocalBufferNoOwner().

Referenced by ExtendBufferedRelLocal(), FlushRelationBuffers(), ReleaseAndReadBuffer(), ReleaseBuffer(), and UnlockReleaseBuffer().

◆ UnpinLocalBufferNoOwner()

void UnpinLocalBufferNoOwner ( Buffer buffer )

extern

Definition at line 872 of file localbuf.c.

{
    int         buffid = -buffer - 1;
 
    Assert(BufferIsLocal(buffer));
    Assert(LocalRefCount[buffid] > 0);
    Assert(NLocalPinnedBuffers > 0);
 
    if (--LocalRefCount[buffid] == 0)
    {
        BufferDesc *buf_hdr = GetLocalBufferDescriptor(buffid);
        uint64      buf_state;
 
        NLocalPinnedBuffers--;
 
        buf_state = pg_atomic_read_u64(&buf_hdr->state);
        Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
        buf_state -= BUF_REFCOUNT_ONE;
        pg_atomic_unlocked_write_u64(&buf_hdr->state, buf_state);
 
        /* see comment in UnpinBufferNoOwner */
        VALGRIND_MAKE_MEM_NOACCESS(LocalBufHdrGetBlock(buf_hdr), BLCKSZ);
    }
}

References Assert, BUF_REFCOUNT_ONE, BUF_STATE_GET_REFCOUNT, BufferIsLocal, fb(), GetLocalBufferDescriptor(), LocalBufHdrGetBlock, LocalRefCount, NLocalPinnedBuffers, pg_atomic_read_u64(), pg_atomic_unlocked_write_u64(), and VALGRIND_MAKE_MEM_NOACCESS.

Referenced by ResOwnerReleaseBuffer(), and UnpinLocalBuffer().

◆ WaitBufHdrUnlocked()

uint64 WaitBufHdrUnlocked ( BufferDesc * buf )

extern

Definition at line 7615 of file bufmgr.c.

{
    SpinDelayStatus delayStatus;
    uint64      buf_state;
 
    init_local_spin_delay(&delayStatus);
 
    buf_state = pg_atomic_read_u64(&buf->state);
 
    while (buf_state & BM_LOCKED)
    {
        perform_spin_delay(&delayStatus);
        buf_state = pg_atomic_read_u64(&buf->state);
    }
 
    finish_spin_delay(&delayStatus);
 
    return buf_state;
}

References BM_LOCKED, buf, fb(), finish_spin_delay(), init_local_spin_delay, perform_spin_delay(), and pg_atomic_read_u64().

Referenced by GetBufferFromRing(), MarkBufferDirty(), PinBuffer(), and StrategyGetBuffer().

◆ WritebackContextInit()

void WritebackContextInit	(	WritebackContext *	context,
		int *	max_pending
	)

extern

Definition at line 7727 of file bufmgr.c.

{
    Assert(*max_pending <= WRITEBACK_MAX_PENDING_FLUSHES);
 
    context->max_pending = max_pending;
    context->nr_pending = 0;
}

References Assert, WritebackContext::max_pending, WritebackContext::nr_pending, and WRITEBACK_MAX_PENDING_FLUSHES.

Referenced by BackgroundWriterMain(), BufferManagerShmemAttach(), BufferManagerShmemInit(), and BufferSync().

Variable Documentation

◆ BackendWritebackContext

PGDLLIMPORT WritebackContext BackendWritebackContext

extern

Definition at line 27 of file buf_init.c.

Referenced by BufferManagerShmemAttach(), BufferManagerShmemInit(), and GetVictimBuffer().

◆ buffer_io_resowner_desc

PGDLLIMPORT const ResourceOwnerDesc buffer_io_resowner_desc

extern

Definition at line 285 of file bufmgr.c.

{
    .name = "buffer io",
    .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
    .release_priority = RELEASE_PRIO_BUFFER_IOS,
    .ReleaseResource = ResOwnerReleaseBufferIO,
    .DebugPrint = ResOwnerPrintBufferIO
};

Referenced by ResourceOwnerForgetBufferIO(), and ResourceOwnerRememberBufferIO().

◆ buffer_resowner_desc

PGDLLIMPORT const ResourceOwnerDesc buffer_resowner_desc

extern

Definition at line 294 of file bufmgr.c.

{
    .name = "buffer",
    .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
    .release_priority = RELEASE_PRIO_BUFFER_PINS,
    .ReleaseResource = ResOwnerReleaseBuffer,
    .DebugPrint = ResOwnerPrintBuffer
};

Referenced by ResourceOwnerForgetBuffer(), and ResourceOwnerRememberBuffer().

◆ BufferDescriptors

PGDLLIMPORT BufferDescPadded* BufferDescriptors

extern

Definition at line 24 of file buf_init.c.

Referenced by BufferManagerShmemRequest(), and GetBufferDescriptor().

◆ BufferIOCVArray

PGDLLIMPORT ConditionVariableMinimallyPadded* BufferIOCVArray

extern

Definition at line 26 of file buf_init.c.

Referenced by BufferDescriptorGetIOCV(), and BufferManagerShmemRequest().

◆ CkptBufferIds

PGDLLIMPORT CkptSortItem* CkptBufferIds

extern

Definition at line 28 of file buf_init.c.

Referenced by BufferManagerShmemRequest(), and BufferSync().

◆ LocalBufferDescriptors

PGDLLIMPORT BufferDesc* LocalBufferDescriptors

extern

Definition at line 47 of file localbuf.c.

Referenced by GetLocalBufferDescriptor(), and InitLocalBuffers().

Variables
PGDLLIMPORT BufferDescPadded *	BufferDescriptors

PGDLLIMPORT ConditionVariableMinimallyPadded *	BufferIOCVArray

PGDLLIMPORT WritebackContext	BackendWritebackContext

PGDLLIMPORT BufferDesc *	LocalBufferDescriptors

PGDLLIMPORT CkptSortItem *	CkptBufferIds

PGDLLIMPORT const ResourceOwnerDesc	buffer_io_resowner_desc

PGDLLIMPORT const ResourceOwnerDesc	buffer_resowner_desc

Data Structures

Macros

Typedefs

Enumerations

Functions

Variables

Macro Definition Documentation

◆ BM_CHECKPOINT_NEEDED

◆ BM_DIRTY

◆ BM_IO_ERROR

◆ BM_IO_IN_PROGRESS

◆ BM_LOCK_HAS_WAITERS

◆ BM_LOCK_MASK

◆ BM_LOCK_SHIFT

◆ BM_LOCK_VAL_EXCLUSIVE

◆ BM_LOCK_VAL_SHARE_EXCLUSIVE

◆ BM_LOCK_VAL_SHARED

◆ BM_LOCK_WAKE_IN_PROGRESS

◆ BM_LOCKED

◆ BM_MAX_USAGE_COUNT

◆ BM_PERMANENT

◆ BM_PIN_COUNT_WAITER

◆ BM_TAG_VALID

◆ BM_VALID

◆ BUF_DEFINE_FLAG

◆ BUF_FLAG_BITS

◆ BUF_FLAG_MASK

◆ BUF_FLAG_SHIFT

◆ BUF_LOCK_BITS

◆ BUF_REFCOUNT_BITS

◆ BUF_REFCOUNT_MASK

◆ BUF_REFCOUNT_ONE

◆ BUF_STATE_GET_REFCOUNT

◆ BUF_STATE_GET_USAGECOUNT

◆ BUF_USAGECOUNT_BITS

◆ BUF_USAGECOUNT_MASK

◆ BUF_USAGECOUNT_ONE

◆ BUF_USAGECOUNT_SHIFT

◆ BUFFERDESC_PAD_TO_SIZE

Typedef Documentation

◆ BufferDesc

◆ BufferDescPadded

◆ BufferTag

◆ CkptSortItem

◆ PendingWriteback

◆ StartBufferIOResult

◆ WritebackContext

Enumeration Type Documentation

◆ StartBufferIOResult

Function Documentation

◆ AtEOXact_LocalBuffers()

◆ BufferDescriptorGetBuffer()

◆ BufferDescriptorGetIOCV()

◆ BufferTagsEqual()

◆ BufMappingPartitionLock()

◆ BufMappingPartitionLockByIndex()

◆ BufTableDelete()

◆ BufTableHashCode()

◆ BufTableHashPartition()

◆ BufTableInsert()

◆ BufTableLookup()

◆ BufTagGetForkNum()

◆ BufTagGetRelFileLocator()

◆ BufTagGetRelNumber()

◆ BufTagMatchesRelFileLocator()

◆ BufTagSetRelForkDetails()

◆ ClearBufferTag()

◆ DropRelationAllLocalBuffers()

◆ DropRelationLocalBuffers()

◆ ExtendBufferedRelLocal()

◆ FlushLocalBuffer()

◆ GetBufferDescriptor()

◆ GetLocalBufferDescriptor()

◆ InitBufferTag()

◆ InvalidateLocalBuffer()

◆ IOContextForStrategy()

◆ IssuePendingWritebacks()

◆ LocalBufferAlloc()

◆ LockBufHdr()

◆ MarkLocalBufferDirty()