#include "postgres.h"
#include <sys/file.h>
#include <unistd.h>
#include "access/tableam.h"
#include "access/xloginsert.h"
#include "access/xlogutils.h"
#include "catalog/storage.h"
#include "catalog/storage_xlog.h"
#include "common/hashfn.h"
#include "executor/instrument.h"
#include "lib/binaryheap.h"
#include "miscadmin.h"
#include "pg_trace.h"
#include "pgstat.h"
#include "postmaster/bgwriter.h"
#include "storage/aio.h"
#include "storage/buf_internals.h"
#include "storage/bufmgr.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/lmgr.h"
#include "storage/proc.h"
#include "storage/proclist.h"
#include "storage/procsignal.h"
#include "storage/read_stream.h"
#include "storage/smgr.h"
#include "storage/standby.h"
#include "utils/memdebug.h"
#include "utils/ps_status.h"
#include "utils/rel.h"
#include "utils/resowner.h"
#include "utils/timestamp.h"
#include "utils/wait_event.h"
#include "lib/simplehash.h"
#include "lib/sort_template.h"

Include dependency graph for bufmgr.c:

Data Structures
struct	PrivateRefCountData

struct	PrivateRefCountEntry

struct	CkptTsStatus

struct	SMgrSortArray

Macros
#define	BufHdrGetBlock(bufHdr) ((Block) (BufferBlocks + ((Size) (bufHdr)->buf_id) * BLCKSZ))

#define	BufferGetLSN(bufHdr) (PageGetLSN(BufHdrGetBlock(bufHdr)))

#define	LocalBufHdrGetBlock(bufHdr) LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)]

#define	BUF_WRITTEN 0x01

#define	BUF_REUSABLE 0x02

#define	RELS_BSEARCH_THRESHOLD 20

#define	BUF_DROP_FULL_SCAN_THRESHOLD (uint64) (NBuffers / 32)

#define	SH_PREFIX refcount

#define	SH_ELEMENT_TYPE PrivateRefCountEntry

#define	SH_KEY_TYPE Buffer

#define	SH_KEY buffer

#define	SH_HASH_KEY(tb, key) murmurhash32((uint32) (key))

#define	SH_EQUAL(tb, a, b) ((a) == (b))

#define	SH_SCOPE static inline

#define	SH_DECLARE

#define	SH_DEFINE

#define	REFCOUNT_ARRAY_ENTRIES 8

#define	BufferIsPinned(bufnum)

#define	ST_SORT sort_checkpoint_bufferids

#define	ST_ELEMENT_TYPE CkptSortItem

#define	ST_COMPARE(a, b) ckpt_buforder_comparator(a, b)

#define	ST_SCOPE static

#define	ST_DEFINE

#define	ST_SORT sort_pending_writebacks

#define	ST_ELEMENT_TYPE PendingWriteback

#define	ST_COMPARE(a, b) buffertag_comparator(&a->tag, &b->tag)

#define	ST_SCOPE static

#define	ST_DEFINE

#define	READV_COUNT_BITS 7

#define	READV_COUNT_MASK ((1 << READV_COUNT_BITS) - 1)

Typedefs
typedef struct PrivateRefCountData	PrivateRefCountData

typedef struct PrivateRefCountEntry	PrivateRefCountEntry

typedef struct CkptTsStatus	CkptTsStatus

typedef struct SMgrSortArray	SMgrSortArray

Functions
static void	ReservePrivateRefCountEntry (void)

static PrivateRefCountEntry *	NewPrivateRefCountEntry (Buffer buffer)

static PrivateRefCountEntry *	GetPrivateRefCountEntry (Buffer buffer, bool do_move)

static int32	GetPrivateRefCount (Buffer buffer)

static void	ForgetPrivateRefCountEntry (PrivateRefCountEntry *ref)

static void	ResOwnerReleaseBufferIO (Datum res)

static char *	ResOwnerPrintBufferIO (Datum res)

static void	ResOwnerReleaseBuffer (Datum res)

static char *	ResOwnerPrintBuffer (Datum res)

static pg_noinline PrivateRefCountEntry *	GetPrivateRefCountEntrySlow (Buffer buffer, bool do_move)

static Buffer	ReadBuffer_common (Relation rel, SMgrRelation smgr, char smgr_persistence, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode, BufferAccessStrategy strategy)

static BlockNumber	ExtendBufferedRelCommon (BufferManagerRelation bmr, ForkNumber fork, BufferAccessStrategy strategy, uint32 flags, uint32 extend_by, BlockNumber extend_upto, Buffer buffers, uint32 extended_by)

static BlockNumber	ExtendBufferedRelShared (BufferManagerRelation bmr, ForkNumber fork, BufferAccessStrategy strategy, uint32 flags, uint32 extend_by, BlockNumber extend_upto, Buffer buffers, uint32 extended_by)

static bool	PinBuffer (BufferDesc *buf, BufferAccessStrategy strategy, bool skip_if_not_valid)

static void	PinBuffer_Locked (BufferDesc *buf)

static void	UnpinBuffer (BufferDesc *buf)

static void	UnpinBufferNoOwner (BufferDesc *buf)

static void	BufferSync (int flags)

static int	SyncOneBuffer (int buf_id, bool skip_recently_used, WritebackContext *wb_context)

static void	WaitIO (BufferDesc *buf)

static void	AbortBufferIO (Buffer buffer)

static void	shared_buffer_write_error_callback (void *arg)

static void	local_buffer_write_error_callback (void *arg)

static BufferDesc *	BufferAlloc (SMgrRelation smgr, char relpersistence, ForkNumber forkNum, BlockNumber blockNum, BufferAccessStrategy strategy, bool *foundPtr, IOContext io_context)

static bool	AsyncReadBuffers (ReadBuffersOperation operation, int nblocks_progress)

static void	CheckReadBuffersOperation (ReadBuffersOperation *operation, bool is_complete)

static pg_attribute_always_inline void	TrackBufferHit (IOObject io_object, IOContext io_context, Relation rel, char persistence, SMgrRelation smgr, ForkNumber forknum, BlockNumber blocknum)

static Buffer	GetVictimBuffer (BufferAccessStrategy strategy, IOContext io_context)

static void	FlushUnlockedBuffer (BufferDesc *buf, SMgrRelation reln, IOObject io_object, IOContext io_context)

static void	FlushBuffer (BufferDesc *buf, SMgrRelation reln, IOObject io_object, IOContext io_context)

static void	FindAndDropRelationBuffers (RelFileLocator rlocator, ForkNumber forkNum, BlockNumber nForkBlock, BlockNumber firstDelBlock)

static void	RelationCopyStorageUsingBuffer (RelFileLocator srclocator, RelFileLocator dstlocator, ForkNumber forkNum, bool permanent)

static void	AtProcExit_Buffers (int code, Datum arg)

static void	CheckForBufferLeaks (void)

static int	rlocator_comparator (const void p1, const void p2)

static int	buffertag_comparator (const BufferTag ba, const BufferTag bb)

static int	ckpt_buforder_comparator (const CkptSortItem a, const CkptSortItem b)

static int	ts_ckpt_progress_comparator (Datum a, Datum b, void *arg)

static void	BufferLockAcquire (Buffer buffer, BufferDesc *buf_hdr, BufferLockMode mode)

static void	BufferLockUnlock (Buffer buffer, BufferDesc *buf_hdr)

static bool	BufferLockConditional (Buffer buffer, BufferDesc *buf_hdr, BufferLockMode mode)

static bool	BufferLockHeldByMeInMode (BufferDesc *buf_hdr, BufferLockMode mode)

static bool	BufferLockHeldByMe (BufferDesc *buf_hdr)

static void	BufferLockDisown (Buffer buffer, BufferDesc *buf_hdr)

static int	BufferLockDisownInternal (Buffer buffer, BufferDesc *buf_hdr)

static bool	BufferLockAttempt (BufferDesc *buf_hdr, BufferLockMode mode)

static void	BufferLockQueueSelf (BufferDesc *buf_hdr, BufferLockMode mode)

static void	BufferLockDequeueSelf (BufferDesc *buf_hdr)

static void	BufferLockWakeup (BufferDesc *buf_hdr, bool unlocked)

static void	BufferLockProcessRelease (BufferDesc *buf_hdr, BufferLockMode mode, uint64 lockstate)

static uint64	BufferLockReleaseSub (BufferLockMode mode)

PrefetchBufferResult	PrefetchSharedBuffer (SMgrRelation smgr_reln, ForkNumber forkNum, BlockNumber blockNum)

PrefetchBufferResult	PrefetchBuffer (Relation reln, ForkNumber forkNum, BlockNumber blockNum)

bool	ReadRecentBuffer (RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockNum, Buffer recent_buffer)

Buffer	ReadBuffer (Relation reln, BlockNumber blockNum)

Buffer	ReadBufferExtended (Relation reln, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode, BufferAccessStrategy strategy)

Buffer	ReadBufferWithoutRelcache (RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockNum, ReadBufferMode mode, BufferAccessStrategy strategy, bool permanent)

Buffer	ExtendBufferedRel (BufferManagerRelation bmr, ForkNumber forkNum, BufferAccessStrategy strategy, uint32 flags)

BlockNumber	ExtendBufferedRelBy (BufferManagerRelation bmr, ForkNumber fork, BufferAccessStrategy strategy, uint32 flags, uint32 extend_by, Buffer buffers, uint32 extended_by)

Buffer	ExtendBufferedRelTo (BufferManagerRelation bmr, ForkNumber fork, BufferAccessStrategy strategy, uint32 flags, BlockNumber extend_to, ReadBufferMode mode)

static void	ZeroAndLockBuffer (Buffer buffer, ReadBufferMode mode, bool already_valid)

static pg_attribute_always_inline Buffer	PinBufferForBlock (Relation rel, SMgrRelation smgr, char persistence, ForkNumber forkNum, BlockNumber blockNum, BufferAccessStrategy strategy, IOObject io_object, IOContext io_context, bool *foundPtr)

static pg_attribute_always_inline bool	StartReadBuffersImpl (ReadBuffersOperation operation, Buffer buffers, BlockNumber blockNum, int *nblocks, int flags, bool allow_forwarding)

bool	StartReadBuffers (ReadBuffersOperation operation, Buffer buffers, BlockNumber blockNum, int *nblocks, int flags)

bool	StartReadBuffer (ReadBuffersOperation operation, Buffer buffer, BlockNumber blocknum, int flags)

static void	ProcessReadBuffersResult (ReadBuffersOperation *operation)

bool	WaitReadBuffers (ReadBuffersOperation *operation)

static void	InvalidateBuffer (BufferDesc *buf)

static bool	InvalidateVictimBuffer (BufferDesc *buf_hdr)

uint32	GetPinLimit (void)

uint32	GetAdditionalPinLimit (void)

void	LimitAdditionalPins (uint32 *additional_pins)

bool	BufferIsLockedByMe (Buffer buffer)

bool	BufferIsLockedByMeInMode (Buffer buffer, BufferLockMode mode)

bool	BufferIsDirty (Buffer buffer)

void	MarkBufferDirty (Buffer buffer)

Buffer	ReleaseAndReadBuffer (Buffer buffer, Relation relation, BlockNumber blockNum)

static void	WakePinCountWaiter (BufferDesc *buf)

void	TrackNewBufferPin (Buffer buf)

bool	BgBufferSync (WritebackContext *wb_context)

void	AtEOXact_Buffers (bool isCommit)

void	InitBufferManagerAccess (void)

char *	DebugPrintBufferRefcount (Buffer buffer)

void	CheckPointBuffers (int flags)

BlockNumber	BufferGetBlockNumber (Buffer buffer)

void	BufferGetTag (Buffer buffer, RelFileLocator rlocator, ForkNumber forknum, BlockNumber *blknum)

BlockNumber	RelationGetNumberOfBlocksInFork (Relation relation, ForkNumber forkNum)

bool	BufferIsPermanent (Buffer buffer)

XLogRecPtr	BufferGetLSNAtomic (Buffer buffer)

void	DropRelationBuffers (SMgrRelation smgr_reln, ForkNumber forkNum, int nforks, BlockNumber firstDelBlock)

void	DropRelationsAllBuffers (SMgrRelation *smgr_reln, int nlocators)

void	DropDatabaseBuffers (Oid dbid)

void	FlushRelationBuffers (Relation rel)

void	FlushRelationsAllBuffers (SMgrRelation *smgrs, int nrels)

void	CreateAndCopyRelationData (RelFileLocator src_rlocator, RelFileLocator dst_rlocator, bool permanent)

void	FlushDatabaseBuffers (Oid dbid)

void	FlushOneBuffer (Buffer buffer)

void	ReleaseBuffer (Buffer buffer)

void	UnlockReleaseBuffer (Buffer buffer)

void	IncrBufferRefCount (Buffer buffer)

static void	MarkSharedBufferDirtyHint (Buffer buffer, BufferDesc *bufHdr, uint64 lockstate, bool buffer_std)

void	MarkBufferDirtyHint (Buffer buffer, bool buffer_std)

void	UnlockBuffers (void)

void	UnlockBuffer (Buffer buffer)

void	LockBufferInternal (Buffer buffer, BufferLockMode mode)

bool	ConditionalLockBuffer (Buffer buffer)

void	CheckBufferIsPinnedOnce (Buffer buffer)

void	LockBufferForCleanup (Buffer buffer)

bool	HoldingBufferPinThatDelaysRecovery (void)

bool	ConditionalLockBufferForCleanup (Buffer buffer)

bool	IsBufferCleanupOK (Buffer buffer)

static bool	SharedBufferBeginSetHintBits (Buffer buffer, BufferDesc buf_hdr, uint64 lockstate)

bool	BufferBeginSetHintBits (Buffer buffer)

void	BufferFinishSetHintBits (Buffer buffer, bool mark_dirty, bool buffer_std)

bool	BufferSetHintBits16 (uint16 *ptr, uint16 val, Buffer buffer)

StartBufferIOResult	StartSharedBufferIO (BufferDesc buf, bool forInput, bool wait, PgAioWaitRef io_wref)

StartBufferIOResult	StartBufferIO (Buffer buffer, bool forInput, bool wait, PgAioWaitRef *io_wref)

void	TerminateBufferIO (BufferDesc *buf, bool clear_dirty, uint64 set_flag_bits, bool forget_owner, bool release_aio)

uint64	LockBufHdr (BufferDesc *desc)

pg_noinline uint64	WaitBufHdrUnlocked (BufferDesc *buf)

void	WritebackContextInit (WritebackContext context, int max_pending)

void	ScheduleBufferTagForWriteback (WritebackContext wb_context, IOContext io_context, BufferTag tag)

void	IssuePendingWritebacks (WritebackContext *wb_context, IOContext io_context)

static bool	EvictUnpinnedBufferInternal (BufferDesc desc, bool buffer_flushed)

bool	EvictUnpinnedBuffer (Buffer buf, bool *buffer_flushed)

void	EvictAllUnpinnedBuffers (int32 buffers_evicted, int32 buffers_flushed, int32 *buffers_skipped)

void	EvictRelUnpinnedBuffers (Relation rel, int32 buffers_evicted, int32 buffers_flushed, int32 *buffers_skipped)

static bool	MarkDirtyUnpinnedBufferInternal (Buffer buf, BufferDesc desc, bool buffer_already_dirty)

bool	MarkDirtyUnpinnedBuffer (Buffer buf, bool *buffer_already_dirty)

void	MarkDirtyRelUnpinnedBuffers (Relation rel, int32 buffers_dirtied, int32 buffers_already_dirty, int32 *buffers_skipped)

void	MarkDirtyAllUnpinnedBuffers (int32 buffers_dirtied, int32 buffers_already_dirty, int32 *buffers_skipped)

static pg_attribute_always_inline void	buffer_stage_common (PgAioHandle *ioh, bool is_write, bool is_temp)

static void	buffer_readv_decode_error (PgAioResult result, bool zeroed_any, bool ignored_any, uint8 zeroed_or_error_count, uint8 checkfail_count, uint8 *first_off)

static void	buffer_readv_encode_error (PgAioResult *result, bool is_temp, bool zeroed_any, bool ignored_any, uint8 error_count, uint8 zeroed_count, uint8 checkfail_count, uint8 first_error_off, uint8 first_zeroed_off, uint8 first_ignored_off)

static pg_attribute_always_inline void	buffer_readv_complete_one (PgAioTargetData td, uint8 buf_off, Buffer buffer, uint8 flags, bool failed, bool is_temp, bool buffer_invalid, bool failed_checksum, bool ignored_checksum, bool *zeroed_buffer)

static pg_attribute_always_inline PgAioResult	buffer_readv_complete (PgAioHandle *ioh, PgAioResult prior_result, uint8 cb_data, bool is_temp)

static void	buffer_readv_report (PgAioResult result, const PgAioTargetData *td, int elevel)

static void	shared_buffer_readv_stage (PgAioHandle *ioh, uint8 cb_data)

static PgAioResult	shared_buffer_readv_complete (PgAioHandle *ioh, PgAioResult prior_result, uint8 cb_data)

static PgAioResult	shared_buffer_readv_complete_local (PgAioHandle *ioh, PgAioResult prior_result, uint8 cb_data)

static void	local_buffer_readv_stage (PgAioHandle *ioh, uint8 cb_data)

static PgAioResult	local_buffer_readv_complete (PgAioHandle *ioh, PgAioResult prior_result, uint8 cb_data)

Variables
bool	zero_damaged_pages = false

int	bgwriter_lru_maxpages = 100

double	bgwriter_lru_multiplier = 2.0

bool	track_io_timing = false

int	effective_io_concurrency = DEFAULT_EFFECTIVE_IO_CONCURRENCY

int	maintenance_io_concurrency = DEFAULT_MAINTENANCE_IO_CONCURRENCY

int	io_combine_limit = DEFAULT_IO_COMBINE_LIMIT

int	io_combine_limit_guc = DEFAULT_IO_COMBINE_LIMIT

int	io_max_combine_limit = DEFAULT_IO_COMBINE_LIMIT

int	checkpoint_flush_after = DEFAULT_CHECKPOINT_FLUSH_AFTER

int	bgwriter_flush_after = DEFAULT_BGWRITER_FLUSH_AFTER

int	backend_flush_after = DEFAULT_BACKEND_FLUSH_AFTER

static BufferDesc *	PinCountWaitBuf = NULL

static Buffer	PrivateRefCountArrayKeys [REFCOUNT_ARRAY_ENTRIES]

static struct PrivateRefCountEntry	PrivateRefCountArray [REFCOUNT_ARRAY_ENTRIES]

static refcount_hash *	PrivateRefCountHash = NULL

static int32	PrivateRefCountOverflowed = 0

static uint32	PrivateRefCountClock = 0

static int	ReservedRefCountSlot = -1

static int	PrivateRefCountEntryLast = -1

static uint32	MaxProportionalPins

const ResourceOwnerDesc	buffer_io_resowner_desc

const ResourceOwnerDesc	buffer_resowner_desc

const PgAioHandleCallbacks	aio_shared_buffer_readv_cb

const PgAioHandleCallbacks	aio_local_buffer_readv_cb

Macro Definition Documentation

◆ BUF_DROP_FULL_SCAN_THRESHOLD

#define BUF_DROP_FULL_SCAN_THRESHOLD (uint64) (NBuffers / 32)

Definition at line 95 of file bufmgr.c.

◆ BUF_REUSABLE

#define BUF_REUSABLE 0x02

Definition at line 85 of file bufmgr.c.

◆ BUF_WRITTEN

#define BUF_WRITTEN 0x01

Definition at line 84 of file bufmgr.c.

◆ BufferGetLSN

#define BufferGetLSN ( bufHdr ) (PageGetLSN(BufHdrGetBlock(bufHdr)))

Definition at line 77 of file bufmgr.c.

◆ BufferIsPinned

#define BufferIsPinned ( bufnum )

Value:

( \
    !BufferIsValid(bufnum) ? \
        false \
    : \
		BufferIsLocal(bufnum) ? \
            (LocalRefCount[-(bufnum) - 1] > 0) \
        : \
    (GetPrivateRefCount(bufnum) > 0) \
)

Definition at line 599 of file bufmgr.c.

    : \
		BufferIsLocal(bufnum) ? \
            (LocalRefCount[-(bufnum) - 1] > 0) \
        : \
    (GetPrivateRefCount(bufnum) > 0) \
)

◆ BufHdrGetBlock

#define BufHdrGetBlock ( bufHdr ) ((Block) (BufferBlocks + ((Size) (bufHdr)->buf_id) * BLCKSZ))

Definition at line 76 of file bufmgr.c.

◆ LocalBufHdrGetBlock

#define LocalBufHdrGetBlock ( bufHdr ) LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)]

Definition at line 80 of file bufmgr.c.

◆ READV_COUNT_BITS

#define READV_COUNT_BITS 7

◆ READV_COUNT_MASK

#define READV_COUNT_MASK ((1 << READV_COUNT_BITS) - 1)

◆ REFCOUNT_ARRAY_ENTRIES

#define REFCOUNT_ARRAY_ENTRIES 8

Definition at line 145 of file bufmgr.c.

◆ RELS_BSEARCH_THRESHOLD

#define RELS_BSEARCH_THRESHOLD 20

Definition at line 87 of file bufmgr.c.

◆ SH_DECLARE

#define SH_DECLARE

Definition at line 140 of file bufmgr.c.

◆ SH_DEFINE

#define SH_DEFINE

Definition at line 141 of file bufmgr.c.

◆ SH_ELEMENT_TYPE

#define SH_ELEMENT_TYPE PrivateRefCountEntry

Definition at line 134 of file bufmgr.c.

◆ SH_EQUAL

#define SH_EQUAL	(	tb,
		a,
		b
	)	((a) == (b))

Definition at line 138 of file bufmgr.c.

◆ SH_HASH_KEY

#define SH_HASH_KEY	(	tb,
		key
	)	murmurhash32((uint32) (key))

Definition at line 137 of file bufmgr.c.

◆ SH_KEY

#define SH_KEY buffer

Definition at line 136 of file bufmgr.c.

◆ SH_KEY_TYPE

#define SH_KEY_TYPE Buffer

Definition at line 135 of file bufmgr.c.

◆ SH_PREFIX

#define SH_PREFIX refcount

Definition at line 133 of file bufmgr.c.

◆ SH_SCOPE

#define SH_SCOPE static inline

Definition at line 139 of file bufmgr.c.

◆ ST_COMPARE [1/2]

#define ST_COMPARE	(	a,
		b
	)	ckpt_buforder_comparator(a, b)

Definition at line 3536 of file bufmgr.c.

◆ ST_COMPARE [2/2]

#define ST_COMPARE	(	a,
		b
	)	buffertag_comparator(&a->tag, &b->tag)

Definition at line 3536 of file bufmgr.c.

◆ ST_DEFINE [1/2]

#define ST_DEFINE

Definition at line 3538 of file bufmgr.c.

◆ ST_DEFINE [2/2]

#define ST_DEFINE

Definition at line 3538 of file bufmgr.c.

◆ ST_ELEMENT_TYPE [1/2]

#define ST_ELEMENT_TYPE CkptSortItem

Definition at line 3535 of file bufmgr.c.

◆ ST_ELEMENT_TYPE [2/2]

#define ST_ELEMENT_TYPE PendingWriteback

Definition at line 3535 of file bufmgr.c.

◆ ST_SCOPE [1/2]

#define ST_SCOPE static

Definition at line 3537 of file bufmgr.c.

◆ ST_SCOPE [2/2]

#define ST_SCOPE static

Definition at line 3537 of file bufmgr.c.

◆ ST_SORT [1/2]

#define ST_SORT sort_checkpoint_bufferids

Definition at line 3534 of file bufmgr.c.

◆ ST_SORT [2/2]

#define ST_SORT sort_pending_writebacks

Definition at line 3534 of file bufmgr.c.

Typedef Documentation

◆ CkptTsStatus

typedef struct CkptTsStatus CkptTsStatus

◆ PrivateRefCountData

typedef struct PrivateRefCountData PrivateRefCountData

◆ PrivateRefCountEntry

typedef struct PrivateRefCountEntry PrivateRefCountEntry

◆ SMgrSortArray

typedef struct SMgrSortArray SMgrSortArray

Function Documentation

◆ AbortBufferIO()

static void AbortBufferIO ( Buffer buffer )

static

Definition at line 7420 of file bufmgr.c.

{
    BufferDesc *buf_hdr = GetBufferDescriptor(buffer - 1);
    uint64      buf_state;
 
    buf_state = LockBufHdr(buf_hdr);
    Assert(buf_state & (BM_IO_IN_PROGRESS | BM_TAG_VALID));
 
    if (!(buf_state & BM_VALID))
    {
        Assert(!(buf_state & BM_DIRTY));
        UnlockBufHdr(buf_hdr);
    }
    else
    {
        Assert(buf_state & BM_DIRTY);
        UnlockBufHdr(buf_hdr);
 
        /* Issue notice if this is not the first failure... */
        if (buf_state & BM_IO_ERROR)
        {
            /* Buffer is pinned, so we can read tag without spinlock */
            ereport(WARNING,
                    (errcode(ERRCODE_IO_ERROR),
                     errmsg("could not write block %u of %s",
                            buf_hdr->tag.blockNum,
                            relpathperm(BufTagGetRelFileLocator(&buf_hdr->tag),
                                        BufTagGetForkNum(&buf_hdr->tag)).str),
                     errdetail("Multiple failures --- write error might be permanent.")));
        }
    }
 
    TerminateBufferIO(buf_hdr, false, BM_IO_ERROR, false, false);
}

References Assert, BM_DIRTY, BM_IO_ERROR, BM_IO_IN_PROGRESS, BM_TAG_VALID, BM_VALID, PrivateRefCountEntry::buffer, BufTagGetForkNum(), BufTagGetRelFileLocator(), ereport, errcode(), errdetail(), errmsg, fb(), GetBufferDescriptor(), LockBufHdr(), relpathperm, TerminateBufferIO(), UnlockBufHdr(), and WARNING.

Referenced by ResOwnerReleaseBufferIO().

◆ AsyncReadBuffers()

static bool AsyncReadBuffers	(	ReadBuffersOperation *	operation,
		int *	nblocks_progress
	)

static

Definition at line 1929 of file bufmgr.c.

{
    Buffer     *buffers = &operation->buffers[0];
    int         flags = operation->flags;
    ForkNumber  forknum = operation->forknum;
    char        persistence = operation->persistence;
    int16       nblocks_done = operation->nblocks_done;
    BlockNumber blocknum = operation->blocknum + nblocks_done;
    Buffer     *io_buffers = &operation->buffers[nblocks_done];
    int         io_buffers_len = 0;
    PgAioHandle *ioh;
    uint32      ioh_flags = 0;
    void       *io_pages[MAX_IO_COMBINE_LIMIT];
    IOContext   io_context;
    IOObject    io_object;
    instr_time  io_start;
    StartBufferIOResult status;
 
    if (persistence == RELPERSISTENCE_TEMP)
    {
        io_context = IOCONTEXT_NORMAL;
        io_object = IOOBJECT_TEMP_RELATION;
    }
    else
    {
        io_context = IOContextForStrategy(operation->strategy);
        io_object = IOOBJECT_RELATION;
    }
 
    /*
     * When this IO is executed synchronously, either because the caller will
     * immediately block waiting for the IO or because IOMETHOD_SYNC is used,
     * the AIO subsystem needs to know.
     */
    if (flags & READ_BUFFERS_SYNCHRONOUSLY)
        ioh_flags |= PGAIO_HF_SYNCHRONOUS;
 
    if (persistence == RELPERSISTENCE_TEMP)
        ioh_flags |= PGAIO_HF_REFERENCES_LOCAL;
 
    /*
     * If zero_damaged_pages is enabled, add the READ_BUFFERS_ZERO_ON_ERROR
     * flag. The reason for that is that, hopefully, zero_damaged_pages isn't
     * set globally, but on a per-session basis. The completion callback,
     * which may be run in other processes, e.g. in IO workers, may have a
     * different value of the zero_damaged_pages GUC.
     *
     * XXX: We probably should eventually use a different flag for
     * zero_damaged_pages, so we can report different log levels / error codes
     * for zero_damaged_pages and ZERO_ON_ERROR.
     */
    if (zero_damaged_pages)
        flags |= READ_BUFFERS_ZERO_ON_ERROR;
 
    /*
     * For the same reason as with zero_damaged_pages we need to use this
     * backend's ignore_checksum_failure value.
     */
    if (ignore_checksum_failure)
        flags |= READ_BUFFERS_IGNORE_CHECKSUM_FAILURES;
 
 
    /*
     * To be allowed to report stats in the local completion callback we need
     * to prepare to report stats now. This ensures we can safely report the
     * checksum failure even in a critical section.
     */
    pgstat_prepare_report_checksum_failure(operation->smgr->smgr_rlocator.locator.dbOid);
 
    /*
     * We must get an IO handle before StartBufferIO(), as pgaio_io_acquire()
     * might block, which we don't want after setting IO_IN_PROGRESS. If we
     * don't need to do the IO, we'll release the handle.
     *
     * If we need to wait for IO before we can get a handle, submit
     * already-staged IO first, so that other backends don't need to wait.
     * There wouldn't be a deadlock risk, as pgaio_io_acquire() just needs to
     * wait for already submitted IO, which doesn't require additional locks,
     * but it could still cause undesirable waits.
     *
     * A secondary benefit is that this would allow us to measure the time in
     * pgaio_io_acquire() without causing undue timer overhead in the common,
     * non-blocking, case.  However, currently the pgstats infrastructure
     * doesn't really allow that, as it a) asserts that an operation can't
     * have time without operations b) doesn't have an API to report
     * "accumulated" time.
     */
    ioh = pgaio_io_acquire_nb(CurrentResourceOwner, &operation->io_return);
    if (unlikely(!ioh))
    {
        pgaio_submit_staged();
        ioh = pgaio_io_acquire(CurrentResourceOwner, &operation->io_return);
    }
 
    operation->foreign_io = false;
    pgaio_wref_clear(&operation->io_wref);
 
    /*
     * Try to start IO on the first buffer in a new run of blocks. If AIO is
     * in progress, be it in this backend or another backend, we just
     * associate the wait reference with the operation and wait in
     * WaitReadBuffers(). This turns out to be important for performance in
     * two workloads:
     *
     * 1) A read stream that has to read the same block multiple times within
     * the readahead distance. This can happen e.g. for the table accesses of
     * an index scan.
     *
     * 2) Concurrent scans by multiple backends on the same relation.
     *
     * If we were to synchronously wait for the in-progress IO, we'd not be
     * able to keep enough I/O in flight.
     *
     * If we do find there is ongoing I/O for the buffer, we set up a 1-block
     * ReadBuffersOperation that WaitReadBuffers then can wait on.
     *
     * It's possible that another backend has started IO on the buffer but not
     * yet set its wait reference. In this case, we have no choice but to wait
     * for either the wait reference to be valid or the IO to be done.
     */
    status = StartBufferIO(buffers[nblocks_done], true, true,
                           &operation->io_wref);
    if (status != BUFFER_IO_READY_FOR_IO)
    {
        pgaio_io_release(ioh);
        *nblocks_progress = 1;
        if (status == BUFFER_IO_ALREADY_DONE)
        {
            /*
             * Someone has already completed this block, we're done.
             *
             * When IO is necessary, ->nblocks_done is updated in
             * ProcessReadBuffersResult(), but that is not called if no IO is
             * necessary. Thus update here.
             */
            operation->nblocks_done += 1;
            Assert(operation->nblocks_done <= operation->nblocks);
 
            Assert(!pgaio_wref_valid(&operation->io_wref));
 
            /*
             * Report and track this as a 'hit' for this backend, even though
             * it must have started out as a miss in PinBufferForBlock(). The
             * other backend will track this as a 'read'.
             */
            TrackBufferHit(io_object, io_context,
                           operation->rel, operation->persistence,
                           operation->smgr, operation->forknum,
                           blocknum);
            return false;
        }
 
        /* The IO is already in-progress */
        Assert(status == BUFFER_IO_IN_PROGRESS);
        Assert(pgaio_wref_valid(&operation->io_wref));
        operation->foreign_io = true;
 
        return true;
    }
 
    Assert(io_buffers[0] == buffers[nblocks_done]);
    io_pages[0] = BufferGetBlock(buffers[nblocks_done]);
    io_buffers_len = 1;
 
    /*
     * NB: As little code as possible should be added between the
     * StartBufferIO() above, the further StartBufferIO()s below and the
     * smgrstartreadv(), as some of the buffers are now marked as
     * IO_IN_PROGRESS and will thus cause other backends to wait.
     */
 
    /*
     * How many neighboring-on-disk blocks can we scatter-read into other
     * buffers at the same time?  In this case we don't wait if we see an I/O
     * already in progress (see comment above).
     */
    for (int i = nblocks_done + 1; i < operation->nblocks; i++)
    {
        /* Must be consecutive block numbers. */
        Assert(BufferGetBlockNumber(buffers[i - 1]) ==
               BufferGetBlockNumber(buffers[i]) - 1);
 
        status = StartBufferIO(buffers[i], true, false, NULL);
        if (status != BUFFER_IO_READY_FOR_IO)
            break;
 
        Assert(io_buffers[io_buffers_len] == buffers[i]);
 
        io_pages[io_buffers_len++] = BufferGetBlock(buffers[i]);
    }
 
    /* get a reference to wait for in WaitReadBuffers() */
    pgaio_io_get_wref(ioh, &operation->io_wref);
 
    /* provide the list of buffers to the completion callbacks */
    pgaio_io_set_handle_data_32(ioh, (uint32 *) io_buffers, io_buffers_len);
 
    pgaio_io_register_callbacks(ioh,
                                persistence == RELPERSISTENCE_TEMP ?
                                PGAIO_HCB_LOCAL_BUFFER_READV :
                                PGAIO_HCB_SHARED_BUFFER_READV,
                                flags);
 
    pgaio_io_set_flag(ioh, ioh_flags);
 
    /* ---
     * Even though we're trying to issue IO asynchronously, track the time
     * in smgrstartreadv():
     * - if io_method == IOMETHOD_SYNC, we will always perform the IO
     *   immediately
     * - the io method might not support the IO (e.g. worker IO for a temp
     *   table)
     * ---
     */
    io_start = pgstat_prepare_io_time(track_io_timing);
    smgrstartreadv(ioh, operation->smgr, forknum,
                   blocknum,
                   io_pages, io_buffers_len);
    pgstat_count_io_op_time(io_object, io_context, IOOP_READ,
                            io_start, 1, io_buffers_len * BLCKSZ);
 
    if (persistence == RELPERSISTENCE_TEMP)
        pgBufferUsage.local_blks_read += io_buffers_len;
    else
        pgBufferUsage.shared_blks_read += io_buffers_len;
 
    /*
     * Track vacuum cost when issuing IO, not after waiting for it. Otherwise
     * we could end up issuing a lot of IO in a short timespan, despite a low
     * cost limit.
     */
    if (VacuumCostActive)
        VacuumCostBalance += VacuumCostPageMiss * io_buffers_len;
 
    *nblocks_progress = io_buffers_len;
 
    return true;
}

References Assert, BUFFER_IO_ALREADY_DONE, BUFFER_IO_IN_PROGRESS, BUFFER_IO_READY_FOR_IO, BufferGetBlock(), BufferGetBlockNumber(), CurrentResourceOwner, fb(), i, ignore_checksum_failure, IOCONTEXT_NORMAL, IOContextForStrategy(), IOOBJECT_RELATION, IOOBJECT_TEMP_RELATION, IOOP_READ, BufferUsage::local_blks_read, MAX_IO_COMBINE_LIMIT, operation, PGAIO_HCB_LOCAL_BUFFER_READV, PGAIO_HCB_SHARED_BUFFER_READV, PGAIO_HF_REFERENCES_LOCAL, PGAIO_HF_SYNCHRONOUS, pgaio_io_acquire(), pgaio_io_acquire_nb(), pgaio_io_get_wref(), pgaio_io_register_callbacks(), pgaio_io_release(), pgaio_io_set_flag(), pgaio_io_set_handle_data_32(), pgaio_submit_staged(), pgaio_wref_clear(), pgaio_wref_valid(), pgBufferUsage, pgstat_count_io_op_time(), pgstat_prepare_io_time(), pgstat_prepare_report_checksum_failure(), READ_BUFFERS_IGNORE_CHECKSUM_FAILURES, READ_BUFFERS_SYNCHRONOUSLY, READ_BUFFERS_ZERO_ON_ERROR, BufferUsage::shared_blks_read, smgrstartreadv(), StartBufferIO(), PrivateRefCountEntry::status, track_io_timing, TrackBufferHit(), unlikely, VacuumCostActive, VacuumCostBalance, VacuumCostPageMiss, and zero_damaged_pages.

Referenced by StartReadBuffersImpl(), and WaitReadBuffers().

◆ AtEOXact_Buffers()

void AtEOXact_Buffers ( bool isCommit )

Definition at line 4199 of file bufmgr.c.

{
    CheckForBufferLeaks();
 
    AtEOXact_LocalBuffers(isCommit);
 
    Assert(PrivateRefCountOverflowed == 0);
}

References Assert, AtEOXact_LocalBuffers(), CheckForBufferLeaks(), fb(), and PrivateRefCountOverflowed.

Referenced by AbortTransaction(), AutoVacLauncherMain(), BackgroundWriterMain(), CheckpointerMain(), CommitTransaction(), PrepareTransaction(), and WalWriterMain().

◆ AtProcExit_Buffers()

static void AtProcExit_Buffers	(	int	code,
		Datum	arg
	)

static

Definition at line 4245 of file bufmgr.c.

{
    UnlockBuffers();
 
    CheckForBufferLeaks();
 
    /* localbuf.c needs a chance too */
    AtProcExit_LocalBuffers();
}

References AtProcExit_LocalBuffers(), CheckForBufferLeaks(), and UnlockBuffers().

Referenced by InitBufferManagerAccess().

◆ BgBufferSync()

bool BgBufferSync ( WritebackContext * wb_context )

Definition at line 3831 of file bufmgr.c.

{
    /* info obtained from freelist.c */
    int         strategy_buf_id;
    uint32      strategy_passes;
    uint32      recent_alloc;
 
    /*
     * Information saved between calls so we can determine the strategy
     * point's advance rate and avoid scanning already-cleaned buffers.
     */
    static bool saved_info_valid = false;
    static int  prev_strategy_buf_id;
    static uint32 prev_strategy_passes;
    static int  next_to_clean;
    static uint32 next_passes;
 
    /* Moving averages of allocation rate and clean-buffer density */
    static float smoothed_alloc = 0;
    static float smoothed_density = 10.0;
 
    /* Potentially these could be tunables, but for now, not */
    float       smoothing_samples = 16;
    float       scan_whole_pool_milliseconds = 120000.0;
 
    /* Used to compute how far we scan ahead */
    long        strategy_delta;
    int         bufs_to_lap;
    int         bufs_ahead;
    float       scans_per_alloc;
    int         reusable_buffers_est;
    int         upcoming_alloc_est;
    int         min_scan_buffers;
 
    /* Variables for the scanning loop proper */
    int         num_to_scan;
    int         num_written;
    int         reusable_buffers;
 
    /* Variables for final smoothed_density update */
    long        new_strategy_delta;
    uint32      new_recent_alloc;
 
    /*
     * Find out where the clock-sweep currently is, and how many buffer
     * allocations have happened since our last call.
     */
    strategy_buf_id = StrategySyncStart(&strategy_passes, &recent_alloc);
 
    /* Report buffer alloc counts to pgstat */
    PendingBgWriterStats.buf_alloc += recent_alloc;
 
    /*
     * If we're not running the LRU scan, just stop after doing the stats
     * stuff.  We mark the saved state invalid so that we can recover sanely
     * if LRU scan is turned back on later.
     */
    if (bgwriter_lru_maxpages <= 0)
    {
        saved_info_valid = false;
        return true;
    }
 
    /*
     * Compute strategy_delta = how many buffers have been scanned by the
     * clock-sweep since last time.  If first time through, assume none. Then
     * see if we are still ahead of the clock-sweep, and if so, how many
     * buffers we could scan before we'd catch up with it and "lap" it. Note:
     * weird-looking coding of xxx_passes comparisons are to avoid bogus
     * behavior when the passes counts wrap around.
     */
    if (saved_info_valid)
    {
        int32       passes_delta = strategy_passes - prev_strategy_passes;
 
        strategy_delta = strategy_buf_id - prev_strategy_buf_id;
        strategy_delta += (long) passes_delta * NBuffers;
 
        Assert(strategy_delta >= 0);
 
        if ((int32) (next_passes - strategy_passes) > 0)
        {
            /* we're one pass ahead of the strategy point */
            bufs_to_lap = strategy_buf_id - next_to_clean;
#ifdef BGW_DEBUG
            elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d",
                 next_passes, next_to_clean,
                 strategy_passes, strategy_buf_id,
                 strategy_delta, bufs_to_lap);
#endif
        }
        else if (next_passes == strategy_passes &&
                 next_to_clean >= strategy_buf_id)
        {
            /* on same pass, but ahead or at least not behind */
            bufs_to_lap = NBuffers - (next_to_clean - strategy_buf_id);
#ifdef BGW_DEBUG
            elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d",
                 next_passes, next_to_clean,
                 strategy_passes, strategy_buf_id,
                 strategy_delta, bufs_to_lap);
#endif
        }
        else
        {
            /*
             * We're behind, so skip forward to the strategy point and start
             * cleaning from there.
             */
#ifdef BGW_DEBUG
            elog(DEBUG2, "bgwriter behind: bgw %u-%u strategy %u-%u delta=%ld",
                 next_passes, next_to_clean,
                 strategy_passes, strategy_buf_id,
                 strategy_delta);
#endif
            next_to_clean = strategy_buf_id;
            next_passes = strategy_passes;
            bufs_to_lap = NBuffers;
        }
    }
    else
    {
        /*
         * Initializing at startup or after LRU scanning had been off. Always
         * start at the strategy point.
         */
#ifdef BGW_DEBUG
        elog(DEBUG2, "bgwriter initializing: strategy %u-%u",
             strategy_passes, strategy_buf_id);
#endif
        strategy_delta = 0;
        next_to_clean = strategy_buf_id;
        next_passes = strategy_passes;
        bufs_to_lap = NBuffers;
    }
 
    /* Update saved info for next time */
    prev_strategy_buf_id = strategy_buf_id;
    prev_strategy_passes = strategy_passes;
    saved_info_valid = true;
 
    /*
     * Compute how many buffers had to be scanned for each new allocation, ie,
     * 1/density of reusable buffers, and track a moving average of that.
     *
     * If the strategy point didn't move, we don't update the density estimate
     */
    if (strategy_delta > 0 && recent_alloc > 0)
    {
        scans_per_alloc = (float) strategy_delta / (float) recent_alloc;
        smoothed_density += (scans_per_alloc - smoothed_density) /
            smoothing_samples;
    }
 
    /*
     * Estimate how many reusable buffers there are between the current
     * strategy point and where we've scanned ahead to, based on the smoothed
     * density estimate.
     */
    bufs_ahead = NBuffers - bufs_to_lap;
    reusable_buffers_est = (float) bufs_ahead / smoothed_density;
 
    /*
     * Track a moving average of recent buffer allocations.  Here, rather than
     * a true average we want a fast-attack, slow-decline behavior: we
     * immediately follow any increase.
     */
    if (smoothed_alloc <= (float) recent_alloc)
        smoothed_alloc = recent_alloc;
    else
        smoothed_alloc += ((float) recent_alloc - smoothed_alloc) /
            smoothing_samples;
 
    /* Scale the estimate by a GUC to allow more aggressive tuning. */
    upcoming_alloc_est = (int) (smoothed_alloc * bgwriter_lru_multiplier);
 
    /*
     * If recent_alloc remains at zero for many cycles, smoothed_alloc will
     * eventually underflow to zero, and the underflows produce annoying
     * kernel warnings on some platforms.  Once upcoming_alloc_est has gone to
     * zero, there's no point in tracking smaller and smaller values of
     * smoothed_alloc, so just reset it to exactly zero to avoid this
     * syndrome.  It will pop back up as soon as recent_alloc increases.
     */
    if (upcoming_alloc_est == 0)
        smoothed_alloc = 0;
 
    /*
     * Even in cases where there's been little or no buffer allocation
     * activity, we want to make a small amount of progress through the buffer
     * cache so that as many reusable buffers as possible are clean after an
     * idle period.
     *
     * (scan_whole_pool_milliseconds / BgWriterDelay) computes how many times
     * the BGW will be called during the scan_whole_pool time; slice the
     * buffer pool into that many sections.
     */
    min_scan_buffers = (int) (NBuffers / (scan_whole_pool_milliseconds / BgWriterDelay));
 
    if (upcoming_alloc_est < (min_scan_buffers + reusable_buffers_est))
    {
#ifdef BGW_DEBUG
        elog(DEBUG2, "bgwriter: alloc_est=%d too small, using min=%d + reusable_est=%d",
             upcoming_alloc_est, min_scan_buffers, reusable_buffers_est);
#endif
        upcoming_alloc_est = min_scan_buffers + reusable_buffers_est;
    }
 
    /*
     * Now write out dirty reusable buffers, working forward from the
     * next_to_clean point, until we have lapped the strategy scan, or cleaned
     * enough buffers to match our estimate of the next cycle's allocation
     * requirements, or hit the bgwriter_lru_maxpages limit.
     */
 
    num_to_scan = bufs_to_lap;
    num_written = 0;
    reusable_buffers = reusable_buffers_est;
 
    /* Execute the LRU scan */
    while (num_to_scan > 0 && reusable_buffers < upcoming_alloc_est)
    {
        int         sync_state = SyncOneBuffer(next_to_clean, true,
                                               wb_context);
 
        if (++next_to_clean >= NBuffers)
        {
            next_to_clean = 0;
            next_passes++;
        }
        num_to_scan--;
 
        if (sync_state & BUF_WRITTEN)
        {
            reusable_buffers++;
            if (++num_written >= bgwriter_lru_maxpages)
            {
                PendingBgWriterStats.maxwritten_clean++;
                break;
            }
        }
        else if (sync_state & BUF_REUSABLE)
            reusable_buffers++;
    }
 
    PendingBgWriterStats.buf_written_clean += num_written;
 
#ifdef BGW_DEBUG
    elog(DEBUG1, "bgwriter: recent_alloc=%u smoothed=%.2f delta=%ld ahead=%d density=%.2f reusable_est=%d upcoming_est=%d scanned=%d wrote=%d reusable=%d",
         recent_alloc, smoothed_alloc, strategy_delta, bufs_ahead,
         smoothed_density, reusable_buffers_est, upcoming_alloc_est,
         bufs_to_lap - num_to_scan,
         num_written,
         reusable_buffers - reusable_buffers_est);
#endif
 
    /*
     * Consider the above scan as being like a new allocation scan.
     * Characterize its density and update the smoothed one based on it. This
     * effectively halves the moving average period in cases where both the
     * strategy and the background writer are doing some useful scanning,
     * which is helpful because a long memory isn't as desirable on the
     * density estimates.
     */
    new_strategy_delta = bufs_to_lap - num_to_scan;
    new_recent_alloc = reusable_buffers - reusable_buffers_est;
    if (new_strategy_delta > 0 && new_recent_alloc > 0)
    {
        scans_per_alloc = (float) new_strategy_delta / (float) new_recent_alloc;
        smoothed_density += (scans_per_alloc - smoothed_density) /
            smoothing_samples;
 
#ifdef BGW_DEBUG
        elog(DEBUG2, "bgwriter: cleaner density alloc=%u scan=%ld density=%.2f new smoothed=%.2f",
             new_recent_alloc, new_strategy_delta,
             scans_per_alloc, smoothed_density);
#endif
    }
 
    /* Return true if OK to hibernate */
    return (bufs_to_lap == 0 && recent_alloc == 0);
}

References Assert, bgwriter_lru_maxpages, bgwriter_lru_multiplier, BgWriterDelay, PgStat_BgWriterStats::buf_alloc, BUF_REUSABLE, BUF_WRITTEN, PgStat_BgWriterStats::buf_written_clean, DEBUG1, DEBUG2, elog, fb(), PgStat_BgWriterStats::maxwritten_clean, NBuffers, PendingBgWriterStats, StrategySyncStart(), and SyncOneBuffer().

Referenced by BackgroundWriterMain().

◆ buffer_readv_complete()

static pg_attribute_always_inline PgAioResult buffer_readv_complete	(	PgAioHandle *	ioh,
		PgAioResult	prior_result,
		uint8	cb_data,
		bool	is_temp
	)

static

Definition at line 8676 of file bufmgr.c.

{
    PgAioResult result = prior_result;
    PgAioTargetData *td = pgaio_io_get_target_data(ioh);
    uint8       first_error_off = 0;
    uint8       first_zeroed_off = 0;
    uint8       first_ignored_off = 0;
    uint8       error_count = 0;
    uint8       zeroed_count = 0;
    uint8       ignored_count = 0;
    uint8       checkfail_count = 0;
    uint64     *io_data;
    uint8       handle_data_len;
 
    if (is_temp)
    {
        Assert(td->smgr.is_temp);
        Assert(pgaio_io_get_owner(ioh) == MyProcNumber);
    }
    else
        Assert(!td->smgr.is_temp);
 
    /*
     * Iterate over all the buffers affected by this IO and call the
     * per-buffer completion function for each buffer.
     */
    io_data = pgaio_io_get_handle_data(ioh, &handle_data_len);
    for (uint8 buf_off = 0; buf_off < handle_data_len; buf_off++)
    {
        Buffer      buf = io_data[buf_off];
        bool        failed;
        bool        failed_verification = false;
        bool        failed_checksum = false;
        bool        zeroed_buffer = false;
        bool        ignored_checksum = false;
 
        Assert(BufferIsValid(buf));
 
        /*
         * If the entire I/O failed on a lower-level, each buffer needs to be
         * marked as failed. In case of a partial read, the first few buffers
         * may be ok.
         */
        failed =
            prior_result.status == PGAIO_RS_ERROR
            || prior_result.result <= buf_off;
 
        buffer_readv_complete_one(td, buf_off, buf, cb_data, failed, is_temp,
                                  &failed_verification,
                                  &failed_checksum,
                                  &ignored_checksum,
                                  &zeroed_buffer);
 
        /*
         * Track information about the number of different kinds of error
         * conditions across all pages, as there can be multiple pages failing
         * verification as part of one IO.
         */
        if (failed_verification && !zeroed_buffer && error_count++ == 0)
            first_error_off = buf_off;
        if (zeroed_buffer && zeroed_count++ == 0)
            first_zeroed_off = buf_off;
        if (ignored_checksum && ignored_count++ == 0)
            first_ignored_off = buf_off;
        if (failed_checksum)
            checkfail_count++;
    }
 
    /*
     * If the smgr read succeeded [partially] and page verification failed for
     * some of the pages, adjust the IO's result state appropriately.
     */
    if (prior_result.status != PGAIO_RS_ERROR &&
        (error_count > 0 || ignored_count > 0 || zeroed_count > 0))
    {
        buffer_readv_encode_error(&result, is_temp,
                                  zeroed_count > 0, ignored_count > 0,
                                  error_count, zeroed_count, checkfail_count,
                                  first_error_off, first_zeroed_off,
                                  first_ignored_off);
        pgaio_result_report(result, td, DEBUG1);
    }
 
    /*
     * For shared relations this reporting is done in
     * shared_buffer_readv_complete_local().
     */
    if (is_temp && checkfail_count > 0)
        pgstat_report_checksum_failures_in_db(td->smgr.rlocator.dbOid,
                                              checkfail_count);
 
    return result;
}

References Assert, buf, buffer_readv_complete_one(), buffer_readv_encode_error(), BufferIsValid(), RelFileLocator::dbOid, DEBUG1, fb(), PgAioTargetData::is_temp, MyProcNumber, pgaio_io_get_handle_data(), pgaio_io_get_owner(), pgaio_io_get_target_data(), pgaio_result_report(), PGAIO_RS_ERROR, pgstat_report_checksum_failures_in_db(), result, PgAioTargetData::rlocator, and PgAioTargetData::smgr.

Referenced by local_buffer_readv_complete(), and shared_buffer_readv_complete().

◆ buffer_readv_complete_one()

static pg_attribute_always_inline void buffer_readv_complete_one	(	PgAioTargetData *	td,
		uint8	buf_off,
		Buffer	buffer,
		uint8	flags,
		bool	failed,
		bool	is_temp,
		bool *	buffer_invalid,
		bool *	failed_checksum,
		bool *	ignored_checksum,
		bool *	zeroed_buffer
	)

static

Definition at line 8525 of file bufmgr.c.

{
    BufferDesc *buf_hdr = is_temp ?
        GetLocalBufferDescriptor(-buffer - 1)
        : GetBufferDescriptor(buffer - 1);
    BufferTag   tag = buf_hdr->tag;
    char       *bufdata = BufferGetBlock(buffer);
    uint64      set_flag_bits;
    int         piv_flags;
 
    /* check that the buffer is in the expected state for a read */
#ifdef USE_ASSERT_CHECKING
    {
        uint64      buf_state = pg_atomic_read_u64(&buf_hdr->state);
 
        Assert(buf_state & BM_TAG_VALID);
        Assert(!(buf_state & BM_VALID));
        /* temp buffers don't use BM_IO_IN_PROGRESS */
        if (!is_temp)
            Assert(buf_state & BM_IO_IN_PROGRESS);
        Assert(!(buf_state & BM_DIRTY));
    }
#endif
 
    *buffer_invalid = false;
    *failed_checksum = false;
    *ignored_checksum = false;
    *zeroed_buffer = false;
 
    /*
     * We ask PageIsVerified() to only log the message about checksum errors,
     * as the completion might be run in any backend (or IO workers). We will
     * report checksum errors in buffer_readv_report().
     */
    piv_flags = PIV_LOG_LOG;
 
    /* the local zero_damaged_pages may differ from the definer's */
    if (flags & READ_BUFFERS_IGNORE_CHECKSUM_FAILURES)
        piv_flags |= PIV_IGNORE_CHECKSUM_FAILURE;
 
    /*
     * If the buffers are marked for zero on error, we want to log that in
     * case of a checksum failure.
     */
    if (flags & READ_BUFFERS_ZERO_ON_ERROR)
        piv_flags |= PIV_ZERO_BUFFERS_ON_ERROR;
 
    /* Check for garbage data. */
    if (!failed)
    {
        /*
         * If the buffer is not currently pinned by this backend, e.g. because
         * we're completing this IO after an error, the buffer data will have
         * been marked as inaccessible when the buffer was unpinned. The AIO
         * subsystem holds a pin, but that doesn't prevent the buffer from
         * having been marked as inaccessible. The completion might also be
         * executed in a different process.
         */
#ifdef USE_VALGRIND
        if (!BufferIsPinned(buffer))
            VALGRIND_MAKE_MEM_DEFINED(bufdata, BLCKSZ);
#endif
 
        if (!PageIsVerified((Page) bufdata, tag.blockNum, piv_flags,
                            failed_checksum))
        {
            if (flags & READ_BUFFERS_ZERO_ON_ERROR)
            {
                memset(bufdata, 0, BLCKSZ);
                *zeroed_buffer = true;
            }
            else
            {
                *buffer_invalid = true;
                /* mark buffer as having failed */
                failed = true;
            }
        }
        else if (*failed_checksum)
            *ignored_checksum = true;
 
        /* undo what we did above */
#ifdef USE_VALGRIND
        if (!BufferIsPinned(buffer))
            VALGRIND_MAKE_MEM_NOACCESS(bufdata, BLCKSZ);
#endif
 
        /*
         * Immediately log a message about the invalid page, but only to the
         * server log. The reason to do so immediately is that this may be
         * executed in a different backend than the one that originated the
         * request. The reason to do so immediately is that the originator
         * might not process the query result immediately (because it is busy
         * doing another part of query processing) or at all (e.g. if it was
         * cancelled or errored out due to another IO also failing). The
         * definer of the IO will emit an ERROR or WARNING when processing the
         * IO's results
         *
         * To avoid duplicating the code to emit these log messages, we reuse
         * buffer_readv_report().
         */
        if (*buffer_invalid || *failed_checksum || *zeroed_buffer)
        {
            PgAioResult result_one = {0};
 
            buffer_readv_encode_error(&result_one, is_temp,
                                      *zeroed_buffer,
                                      *ignored_checksum,
                                      *buffer_invalid,
                                      *zeroed_buffer ? 1 : 0,
                                      *failed_checksum ? 1 : 0,
                                      buf_off, buf_off, buf_off);
            pgaio_result_report(result_one, td, LOG_SERVER_ONLY);
        }
    }
 
    /* Terminate I/O and set BM_VALID. */
    set_flag_bits = failed ? BM_IO_ERROR : BM_VALID;
    if (is_temp)
        TerminateLocalBufferIO(buf_hdr, false, set_flag_bits, true);
    else
        TerminateBufferIO(buf_hdr, false, set_flag_bits, false, true);
 
    /*
     * Call the BUFFER_READ_DONE tracepoint in the callback, even though the
     * callback may not be executed in the same backend that called
     * BUFFER_READ_START. The alternative would be to defer calling the
     * tracepoint to a later point (e.g. the local completion callback for
     * shared buffer reads), which seems even less helpful.
     */
    TRACE_POSTGRESQL_BUFFER_READ_DONE(tag.forkNum,
                                      tag.blockNum,
                                      tag.spcOid,
                                      tag.dbOid,
                                      tag.relNumber,
                                      is_temp ? MyProcNumber : INVALID_PROC_NUMBER,
                                      false);
}

References Assert, buftag::blockNum, BM_DIRTY, BM_IO_ERROR, BM_IO_IN_PROGRESS, BM_TAG_VALID, BM_VALID, PrivateRefCountEntry::buffer, buffer_readv_encode_error(), BufferGetBlock(), BufferIsPinned, buftag::dbOid, fb(), buftag::forkNum, GetBufferDescriptor(), GetLocalBufferDescriptor(), INVALID_PROC_NUMBER, LOG_SERVER_ONLY, MyProcNumber, PageIsVerified(), pg_atomic_read_u64(), pgaio_result_report(), PIV_IGNORE_CHECKSUM_FAILURE, PIV_LOG_LOG, PIV_ZERO_BUFFERS_ON_ERROR, READ_BUFFERS_IGNORE_CHECKSUM_FAILURES, READ_BUFFERS_ZERO_ON_ERROR, buftag::relNumber, buftag::spcOid, TerminateBufferIO(), TerminateLocalBufferIO(), VALGRIND_MAKE_MEM_DEFINED, and VALGRIND_MAKE_MEM_NOACCESS.

Referenced by buffer_readv_complete().

◆ buffer_readv_decode_error()

static void buffer_readv_decode_error	(	PgAioResult	result,
		bool *	zeroed_any,
		bool *	ignored_any,
		uint8 *	zeroed_or_error_count,
		uint8 *	checkfail_count,
		uint8 *	first_off
	)

inlinestatic

Definition at line 8388 of file bufmgr.c.

{
    uint32      rem_error = result.error_data;
 
    /* see static asserts in buffer_readv_encode_error */
#define READV_COUNT_BITS    7
#define READV_COUNT_MASK    ((1 << READV_COUNT_BITS) - 1)
 
    *zeroed_any = rem_error & 1;
    rem_error >>= 1;
 
    *ignored_any = rem_error & 1;
    rem_error >>= 1;
 
    *zeroed_or_error_count = rem_error & READV_COUNT_MASK;
    rem_error >>= READV_COUNT_BITS;
 
    *checkfail_count = rem_error & READV_COUNT_MASK;
    rem_error >>= READV_COUNT_BITS;
 
    *first_off = rem_error & READV_COUNT_MASK;
    rem_error >>= READV_COUNT_BITS;
}

References fb(), READV_COUNT_BITS, READV_COUNT_MASK, and result.

Referenced by buffer_readv_encode_error(), buffer_readv_report(), and shared_buffer_readv_complete_local().

◆ buffer_readv_encode_error()

static void buffer_readv_encode_error	(	PgAioResult *	result,
		bool	is_temp,
		bool	zeroed_any,
		bool	ignored_any,
		uint8	error_count,
		uint8	zeroed_count,
		uint8	checkfail_count,
		uint8	first_error_off,
		uint8	first_zeroed_off,
		uint8	first_ignored_off
	)

inlinestatic

Definition at line 8430 of file bufmgr.c.

{
 
    uint8       shift = 0;
    uint8       zeroed_or_error_count =
        error_count > 0 ? error_count : zeroed_count;
    uint8       first_off;
 
    StaticAssertDecl(PG_IOV_MAX <= 1 << READV_COUNT_BITS,
                     "PG_IOV_MAX is bigger than reserved space for error data");
    StaticAssertDecl((1 + 1 + 3 * READV_COUNT_BITS) <= PGAIO_RESULT_ERROR_BITS,
                     "PGAIO_RESULT_ERROR_BITS is insufficient for buffer_readv");
 
    /*
     * We only have space to encode one offset - but luckily that's good
     * enough. If there is an error, the error is the interesting offset, same
     * with a zeroed buffer vs an ignored buffer.
     */
    if (error_count > 0)
        first_off = first_error_off;
    else if (zeroed_count > 0)
        first_off = first_zeroed_off;
    else
        first_off = first_ignored_off;
 
    Assert(!zeroed_any || error_count == 0);
 
    result->error_data = 0;
 
    result->error_data |= zeroed_any << shift;
    shift += 1;
 
    result->error_data |= ignored_any << shift;
    shift += 1;
 
    result->error_data |= ((uint32) zeroed_or_error_count) << shift;
    shift += READV_COUNT_BITS;
 
    result->error_data |= ((uint32) checkfail_count) << shift;
    shift += READV_COUNT_BITS;
 
    result->error_data |= ((uint32) first_off) << shift;
    shift += READV_COUNT_BITS;
 
    result->id = is_temp ? PGAIO_HCB_LOCAL_BUFFER_READV :
        PGAIO_HCB_SHARED_BUFFER_READV;
 
    if (error_count > 0)
        result->status = PGAIO_RS_ERROR;
    else
        result->status = PGAIO_RS_WARNING;
 
    /*
     * The encoding is complicated enough to warrant cross-checking it against
     * the decode function.
     */
#ifdef USE_ASSERT_CHECKING
    {
        bool        zeroed_any_2,
                    ignored_any_2;
        uint8       zeroed_or_error_count_2,
                    checkfail_count_2,
                    first_off_2;
 
        buffer_readv_decode_error(*result,
                                  &zeroed_any_2, &ignored_any_2,
                                  &zeroed_or_error_count_2,
                                  &checkfail_count_2,
                                  &first_off_2);
        Assert(zeroed_any == zeroed_any_2);
        Assert(ignored_any == ignored_any_2);
        Assert(zeroed_or_error_count == zeroed_or_error_count_2);
        Assert(checkfail_count == checkfail_count_2);
        Assert(first_off == first_off_2);
    }
#endif
 
#undef READV_COUNT_BITS
#undef READV_COUNT_MASK
}

References Assert, buffer_readv_decode_error(), fb(), PG_IOV_MAX, PGAIO_HCB_LOCAL_BUFFER_READV, PGAIO_HCB_SHARED_BUFFER_READV, PGAIO_RESULT_ERROR_BITS, PGAIO_RS_ERROR, PGAIO_RS_WARNING, READV_COUNT_BITS, result, and StaticAssertDecl.

Referenced by buffer_readv_complete(), and buffer_readv_complete_one().

◆ buffer_readv_report()

static void buffer_readv_report	(	PgAioResult	result,
		const PgAioTargetData *	td,
		int	elevel
	)

static

Definition at line 8779 of file bufmgr.c.

{
    int         nblocks = td->smgr.nblocks;
    BlockNumber first = td->smgr.blockNum;
    BlockNumber last = first + nblocks - 1;
    ProcNumber  errProc =
        td->smgr.is_temp ? MyProcNumber : INVALID_PROC_NUMBER;
    RelPathStr  rpath =
        relpathbackend(td->smgr.rlocator, errProc, td->smgr.forkNum);
    bool        zeroed_any,
                ignored_any;
    uint8       zeroed_or_error_count,
                checkfail_count,
                first_off;
    uint8       affected_count;
    const char *msg_one,
               *msg_mult,
               *det_mult,
               *hint_mult;
 
    buffer_readv_decode_error(result, &zeroed_any, &ignored_any,
                              &zeroed_or_error_count,
                              &checkfail_count,
                              &first_off);
 
    /*
     * Treat a read that had both zeroed buffers *and* ignored checksums as a
     * special case, it's too irregular to be emitted the same way as the
     * other cases.
     */
    if (zeroed_any && ignored_any)
    {
        Assert(zeroed_any && ignored_any);
        Assert(nblocks > 1);    /* same block can't be both zeroed and ignored */
        Assert(result.status != PGAIO_RS_ERROR);
        affected_count = zeroed_or_error_count;
 
        ereport(elevel,
                errcode(ERRCODE_DATA_CORRUPTED),
                errmsg("zeroing %u page(s) and ignoring %u checksum failure(s) among blocks %u..%u of relation \"%s\"",
                       affected_count, checkfail_count, first, last, rpath.str),
                affected_count > 1 ?
                errdetail("Block %u held the first zeroed page.",
                          first + first_off) : 0,
                errhint_plural("See server log for details about the other %d invalid block.",
                               "See server log for details about the other %d invalid blocks.",
                               affected_count + checkfail_count - 1,
                               affected_count + checkfail_count - 1));
        return;
    }
 
    /*
     * The other messages are highly repetitive. To avoid duplicating a long
     * and complicated ereport(), gather the translated format strings
     * separately and then do one common ereport.
     */
    if (result.status == PGAIO_RS_ERROR)
    {
        Assert(!zeroed_any);    /* can't have invalid pages when zeroing them */
        affected_count = zeroed_or_error_count;
        msg_one = _("invalid page in block %u of relation \"%s\"");
        msg_mult = _("%u invalid pages among blocks %u..%u of relation \"%s\"");
        det_mult = _("Block %u held the first invalid page.");
        hint_mult = _("See server log for the other %u invalid block(s).");
    }
    else if (zeroed_any && !ignored_any)
    {
        affected_count = zeroed_or_error_count;
        msg_one = _("invalid page in block %u of relation \"%s\"; zeroing out page");
        msg_mult = _("zeroing out %u invalid pages among blocks %u..%u of relation \"%s\"");
        det_mult = _("Block %u held the first zeroed page.");
        hint_mult = _("See server log for the other %u zeroed block(s).");
    }
    else if (!zeroed_any && ignored_any)
    {
        affected_count = checkfail_count;
        msg_one = _("ignoring checksum failure in block %u of relation \"%s\"");
        msg_mult = _("ignoring %u checksum failures among blocks %u..%u of relation \"%s\"");
        det_mult = _("Block %u held the first ignored page.");
        hint_mult = _("See server log for the other %u ignored block(s).");
    }
    else
        pg_unreachable();
 
    ereport(elevel,
            errcode(ERRCODE_DATA_CORRUPTED),
            affected_count == 1 ?
            errmsg_internal(msg_one, first + first_off, rpath.str) :
            errmsg_internal(msg_mult, affected_count, first, last, rpath.str),
            affected_count > 1 ? errdetail_internal(det_mult, first + first_off) : 0,
            affected_count > 1 ? errhint_internal(hint_mult, affected_count - 1) : 0);
}

References _, Assert, PgAioTargetData::blockNum, buffer_readv_decode_error(), ereport, errcode(), ERRCODE_DATA_CORRUPTED, errdetail(), errdetail_internal(), errhint_internal(), errhint_plural(), errmsg, errmsg_internal(), fb(), PgAioTargetData::forkNum, INVALID_PROC_NUMBER, PgAioTargetData::is_temp, MyProcNumber, PgAioTargetData::nblocks, pg_unreachable, PGAIO_RS_ERROR, relpathbackend, result, PgAioTargetData::rlocator, PgAioTargetData::smgr, and RelPathStr::str.

◆ buffer_stage_common()

static pg_attribute_always_inline void buffer_stage_common	(	PgAioHandle *	ioh,
		bool	is_write,
		bool	is_temp
	)

static

Definition at line 8281 of file bufmgr.c.

{
    uint64     *io_data;
    uint8       handle_data_len;
    PgAioWaitRef io_ref;
    BufferTag   first PG_USED_FOR_ASSERTS_ONLY = {0};
 
    io_data = pgaio_io_get_handle_data(ioh, &handle_data_len);
 
    pgaio_io_get_wref(ioh, &io_ref);
 
    /* iterate over all buffers affected by the vectored readv/writev */
    for (int i = 0; i < handle_data_len; i++)
    {
        Buffer      buffer = (Buffer) io_data[i];
        BufferDesc *buf_hdr = is_temp ?
            GetLocalBufferDescriptor(-buffer - 1)
            : GetBufferDescriptor(buffer - 1);
        uint64      buf_state;
 
        /*
         * Check that all the buffers are actually ones that could conceivably
         * be done in one IO, i.e. are sequential. This is the last
         * buffer-aware code before IO is actually executed and confusion
         * about which buffers are targeted by IO can be hard to debug, making
         * it worth doing extra-paranoid checks.
         */
        if (i == 0)
            first = buf_hdr->tag;
        else
        {
            Assert(buf_hdr->tag.relNumber == first.relNumber);
            Assert(buf_hdr->tag.blockNum == first.blockNum + i);
        }
 
        if (is_temp)
            buf_state = pg_atomic_read_u64(&buf_hdr->state);
        else
            buf_state = LockBufHdr(buf_hdr);
 
        /* verify the buffer is in the expected state */
        Assert(buf_state & BM_TAG_VALID);
        if (is_write)
        {
            Assert(buf_state & BM_VALID);
            Assert(buf_state & BM_DIRTY);
        }
        else
        {
            Assert(!(buf_state & BM_VALID));
            Assert(!(buf_state & BM_DIRTY));
        }
 
        /* temp buffers don't use BM_IO_IN_PROGRESS */
        if (!is_temp)
            Assert(buf_state & BM_IO_IN_PROGRESS);
 
        Assert(BUF_STATE_GET_REFCOUNT(buf_state) >= 1);
 
        /*
         * Reflect that the buffer is now owned by the AIO subsystem.
         *
         * For local buffers: This can't be done just via LocalRefCount, as
         * one might initially think, as this backend could error out while
         * AIO is still in progress, releasing all the pins by the backend
         * itself.
         *
         * This pin is released again in TerminateBufferIO().
         */
        buf_hdr->io_wref = io_ref;
 
        if (is_temp)
        {
            buf_state += BUF_REFCOUNT_ONE;
            pg_atomic_unlocked_write_u64(&buf_hdr->state, buf_state);
        }
        else
            UnlockBufHdrExt(buf_hdr, buf_state, 0, 0, 1);
 
        /*
         * Ensure the content lock that prevents buffer modifications while
         * the buffer is being written out is not released early due to an
         * error.
         */
        if (is_write && !is_temp)
        {
            Assert(BufferLockHeldByMe(buf_hdr));
 
            /*
             * Lock is now owned by AIO subsystem.
             */
            BufferLockDisown(buffer, buf_hdr);
        }
 
        /*
         * Stop tracking this buffer via the resowner - the AIO system now
         * keeps track.
         */
        if (!is_temp)
            ResourceOwnerForgetBufferIO(CurrentResourceOwner, buffer);
    }
}

References Assert, BM_DIRTY, BM_IO_IN_PROGRESS, BM_TAG_VALID, BM_VALID, BUF_REFCOUNT_ONE, BUF_STATE_GET_REFCOUNT, PrivateRefCountEntry::buffer, BufferLockDisown(), BufferLockHeldByMe(), CurrentResourceOwner, fb(), GetBufferDescriptor(), GetLocalBufferDescriptor(), i, LockBufHdr(), pg_atomic_read_u64(), pg_atomic_unlocked_write_u64(), PG_USED_FOR_ASSERTS_ONLY, pgaio_io_get_handle_data(), pgaio_io_get_wref(), ResourceOwnerForgetBufferIO(), and UnlockBufHdrExt().

Referenced by local_buffer_readv_stage(), and shared_buffer_readv_stage().

◆ BufferAlloc()

static pg_attribute_always_inline BufferDesc * BufferAlloc	(	SMgrRelation	smgr,
		char	relpersistence,
		ForkNumber	forkNum,
		BlockNumber	blockNum,
		BufferAccessStrategy	strategy,
		bool *	foundPtr,
		IOContext	io_context
	)

inlinestatic

Definition at line 2188 of file bufmgr.c.

{
    BufferTag   newTag;         /* identity of requested block */
    uint32      newHash;        /* hash value for newTag */
    LWLock     *newPartitionLock;   /* buffer partition lock for it */
    int         existing_buf_id;
    Buffer      victim_buffer;
    BufferDesc *victim_buf_hdr;
    uint64      victim_buf_state;
    uint64      set_bits = 0;
 
    /* Make sure we will have room to remember the buffer pin */
    ResourceOwnerEnlarge(CurrentResourceOwner);
    ReservePrivateRefCountEntry();
 
    /* create a tag so we can lookup the buffer */
    InitBufferTag(&newTag, &smgr->smgr_rlocator.locator, forkNum, blockNum);
 
    /* determine its hash code and partition lock ID */
    newHash = BufTableHashCode(&newTag);
    newPartitionLock = BufMappingPartitionLock(newHash);
 
    /* see if the block is in the buffer pool already */
    LWLockAcquire(newPartitionLock, LW_SHARED);
    existing_buf_id = BufTableLookup(&newTag, newHash);
    if (existing_buf_id >= 0)
    {
        BufferDesc *buf;
        bool        valid;
 
        /*
         * Found it.  Now, pin the buffer so no one can steal it from the
         * buffer pool, and check to see if the correct data has been loaded
         * into the buffer.
         */
        buf = GetBufferDescriptor(existing_buf_id);
 
        valid = PinBuffer(buf, strategy, false);
 
        /* Can release the mapping lock as soon as we've pinned it */
        LWLockRelease(newPartitionLock);
 
        *foundPtr = true;
 
        if (!valid)
        {
            /*
             * We can only get here if (a) someone else is still reading in
             * the page, (b) a previous read attempt failed, or (c) someone
             * called StartReadBuffers() but not yet WaitReadBuffers().
             */
            *foundPtr = false;
        }
 
        return buf;
    }
 
    /*
     * Didn't find it in the buffer pool.  We'll have to initialize a new
     * buffer.  Remember to unlock the mapping lock while doing the work.
     */
    LWLockRelease(newPartitionLock);
 
    /*
     * Acquire a victim buffer. Somebody else might try to do the same, we
     * don't hold any conflicting locks. If so we'll have to undo our work
     * later.
     */
    victim_buffer = GetVictimBuffer(strategy, io_context);
    victim_buf_hdr = GetBufferDescriptor(victim_buffer - 1);
 
    /*
     * Try to make a hashtable entry for the buffer under its new tag. If
     * somebody else inserted another buffer for the tag, we'll release the
     * victim buffer we acquired and use the already inserted one.
     */
    LWLockAcquire(newPartitionLock, LW_EXCLUSIVE);
    existing_buf_id = BufTableInsert(&newTag, newHash, victim_buf_hdr->buf_id);
    if (existing_buf_id >= 0)
    {
        BufferDesc *existing_buf_hdr;
        bool        valid;
 
        /*
         * Got a collision. Someone has already done what we were about to do.
         * We'll just handle this as if it were found in the buffer pool in
         * the first place.  First, give up the buffer we were planning to
         * use.
         *
         * We could do this after releasing the partition lock, but then we'd
         * have to call ResourceOwnerEnlarge() & ReservePrivateRefCountEntry()
         * before acquiring the lock, for the rare case of such a collision.
         */
        UnpinBuffer(victim_buf_hdr);
 
        /* remaining code should match code at top of routine */
 
        existing_buf_hdr = GetBufferDescriptor(existing_buf_id);
 
        valid = PinBuffer(existing_buf_hdr, strategy, false);
 
        /* Can release the mapping lock as soon as we've pinned it */
        LWLockRelease(newPartitionLock);
 
        *foundPtr = true;
 
        if (!valid)
        {
            /*
             * We can only get here if (a) someone else is still reading in
             * the page, (b) a previous read attempt failed, or (c) someone
             * called StartReadBuffers() but not yet WaitReadBuffers().
             */
            *foundPtr = false;
        }
 
        return existing_buf_hdr;
    }
 
    /*
     * Need to lock the buffer header too in order to change its tag.
     */
    victim_buf_state = LockBufHdr(victim_buf_hdr);
 
    /* some sanity checks while we hold the buffer header lock */
    Assert(BUF_STATE_GET_REFCOUNT(victim_buf_state) == 1);
    Assert(!(victim_buf_state & (BM_TAG_VALID | BM_VALID | BM_DIRTY | BM_IO_IN_PROGRESS)));
 
    victim_buf_hdr->tag = newTag;
 
    /*
     * Make sure BM_PERMANENT is set for buffers that must be written at every
     * checkpoint.  Unlogged buffers only need to be written at shutdown
     * checkpoints, except for their "init" forks, which need to be treated
     * just like permanent relations.
     */
    set_bits |= BM_TAG_VALID | BUF_USAGECOUNT_ONE;
    if (relpersistence == RELPERSISTENCE_PERMANENT || forkNum == INIT_FORKNUM)
        set_bits |= BM_PERMANENT;
 
    UnlockBufHdrExt(victim_buf_hdr, victim_buf_state,
                    set_bits, 0, 0);
 
    LWLockRelease(newPartitionLock);
 
    /*
     * Buffer contents are currently invalid.
     */
    *foundPtr = false;
 
    return victim_buf_hdr;
}

References Assert, BM_DIRTY, BM_IO_IN_PROGRESS, BM_PERMANENT, BM_TAG_VALID, BM_VALID, buf, BUF_STATE_GET_REFCOUNT, BUF_USAGECOUNT_ONE, BufMappingPartitionLock(), BufTableHashCode(), BufTableInsert(), BufTableLookup(), CurrentResourceOwner, fb(), GetBufferDescriptor(), GetVictimBuffer(), INIT_FORKNUM, InitBufferTag(), RelFileLocatorBackend::locator, LockBufHdr(), LW_EXCLUSIVE, LW_SHARED, LWLockAcquire(), LWLockRelease(), PinBuffer(), ReservePrivateRefCountEntry(), ResourceOwnerEnlarge(), SMgrRelationData::smgr_rlocator, UnlockBufHdrExt(), and UnpinBuffer().

Referenced by PinBufferForBlock().

◆ BufferBeginSetHintBits()

bool BufferBeginSetHintBits ( Buffer buffer )

Definition at line 7042 of file bufmgr.c.

{
    BufferDesc *buf_hdr;
    uint64      lockstate;
 
    if (BufferIsLocal(buffer))
    {
        /*
         * NB: Will need to check if there is a write in progress, once it is
         * possible for writes to be done asynchronously.
         */
        return true;
    }
 
    buf_hdr = GetBufferDescriptor(buffer - 1);
 
    return SharedBufferBeginSetHintBits(buffer, buf_hdr, &lockstate);
}

References PrivateRefCountEntry::buffer, BufferIsLocal, fb(), GetBufferDescriptor(), and SharedBufferBeginSetHintBits().

Referenced by _bt_check_unique(), _bt_killitems(), _hash_kill_items(), fsm_search_avail(), fsm_vacuum_page(), gistkillitems(), and SetHintBitsExt().

◆ BufferFinishSetHintBits()

void BufferFinishSetHintBits	(	Buffer	buffer,
		bool	mark_dirty,
		bool	buffer_std
	)

Definition at line 7070 of file bufmgr.c.

{
    if (!BufferIsLocal(buffer))
        Assert(BufferIsLockedByMeInMode(buffer, BUFFER_LOCK_SHARE_EXCLUSIVE) ||
               BufferIsLockedByMeInMode(buffer, BUFFER_LOCK_EXCLUSIVE));
 
    if (mark_dirty)
        MarkBufferDirtyHint(buffer, buffer_std);
}

References Assert, PrivateRefCountEntry::buffer, BUFFER_LOCK_EXCLUSIVE, BUFFER_LOCK_SHARE_EXCLUSIVE, BufferIsLocal, BufferIsLockedByMeInMode(), fb(), and MarkBufferDirtyHint().

Referenced by _bt_check_unique(), _bt_killitems(), _hash_kill_items(), fsm_search_avail(), fsm_vacuum_page(), gistkillitems(), and HeapTupleSatisfiesMVCCBatch().

◆ BufferGetBlockNumber()

BlockNumber BufferGetBlockNumber ( Buffer buffer )

Definition at line 4446 of file bufmgr.c.

{
    BufferDesc *bufHdr;
 
    Assert(BufferIsPinned(buffer));
 
    if (BufferIsLocal(buffer))
        bufHdr = GetLocalBufferDescriptor(-buffer - 1);
    else
        bufHdr = GetBufferDescriptor(buffer - 1);
 
    /* pinned, so OK to read tag without spinlock */
    return bufHdr->tag.blockNum;
}

References Assert, PrivateRefCountEntry::buffer, BufferIsLocal, BufferIsPinned, fb(), GetBufferDescriptor(), and GetLocalBufferDescriptor().

Referenced by _bt_binsrch_insert(), _bt_bottomupdel_pass(), _bt_check_unique(), _bt_checkpage(), _bt_delitems_delete(), _bt_delitems_vacuum(), _bt_doinsert(), _bt_finish_split(), _bt_getroot(), _bt_insert_parent(), _bt_insertonpg(), _bt_mark_page_halfdead(), _bt_moveright(), _bt_newlevel(), _bt_pagedel(), _bt_readpage(), _bt_relandgetbuf(), _bt_restore_meta(), _bt_search(), _bt_simpledel_pass(), _bt_split(), _bt_unlink_halfdead_page(), _hash_addovflpage(), _hash_checkpage(), _hash_doinsert(), _hash_first(), _hash_freeovflpage(), _hash_getnewbuf(), _hash_readnext(), _hash_readpage(), _hash_splitbucket(), allocNewBuffer(), AsyncReadBuffers(), BitmapHeapScanNextBlock(), blinsert(), BloomInitMetapage(), brin_doinsert(), brin_doupdate(), brin_getinsertbuffer(), brin_initialize_empty_new_buffer(), brin_page_cleanup(), brin_xlog_insert_update(), brinbuild(), brinGetTupleForHeapBlock(), btvacuumpage(), check_index_page(), CheckReadBuffersOperation(), collect_corrupt_items(), collectMatchBitmap(), createPostingTree(), dataBeginPlaceToPageLeaf(), dataPrepareDownlink(), doPickSplit(), entryPrepareDownlink(), fill_seq_fork_with_data(), ginDeletePostingPage(), ginEntryInsert(), ginFindParents(), ginFinishSplit(), ginPlaceToPage(), ginRedoDeleteListPages(), ginRedoUpdateMetapage(), gistbufferinginserttuples(), gistbuild(), gistcheckpage(), gistdeletepage(), gistformdownlink(), gistinserttuples(), gistMemorizeAllDownlinks(), gistplacetopage(), gistRelocateBuildBuffersOnSplit(), gistScanPage(), gistvacuumpage(), hash_xlog_add_ovfl_page(), heap_delete(), heap_fetch_next_buffer(), heap_hot_search_buffer(), heap_insert(), heap_multi_insert(), heap_page_prune_opt(), heap_page_would_be_all_visible(), heap_prepare_pagescan(), heap_update(), heap_xlog_confirm(), heap_xlog_lock(), heapam_index_fetch_tuple(), heapam_scan_analyze_next_block(), heapgettup(), heapgettup_pagemode(), index_compute_xid_horizon_for_tuples(), lazy_scan_heap(), lazy_scan_noprune(), lazy_scan_prune(), lazy_vacuum_heap_rel(), makeSublist(), moveLeafs(), moveRightIfItNeeded(), pgstathashindex(), prune_freeze_setup(), read_buffers(), read_stream_start_pending_read(), ReadBufferBI(), RelationAddBlocks(), RelationCopyStorageUsingBuffer(), RelationGetBufferForTuple(), RelationPutHeapTuple(), revmap_get_buffer(), revmap_physical_extend(), ScanSourceDatabasePgClassPage(), spgAddNodeAction(), spgbuild(), spgdoinsert(), SpGistSetLastUsedPage(), spgSplitNodeAction(), spgvacuumpage(), spgWalk(), StartReadBuffersImpl(), startScanEntry(), statapprox_heap(), terminate_brin_buildstate(), vacuumLeafPage(), verify_heapam(), visibilitymap_clear(), visibilitymap_get_status(), visibilitymap_pin(), visibilitymap_pin_ok(), and visibilitymap_set().

◆ BufferGetLSNAtomic()

XLogRecPtr BufferGetLSNAtomic ( Buffer buffer )

Definition at line 4713 of file bufmgr.c.

{
    /* Make sure we've got a real buffer, and that we hold a pin on it. */
    Assert(BufferIsValid(buffer));
    Assert(BufferIsPinned(buffer));
 
#ifdef PG_HAVE_8BYTE_SINGLE_COPY_ATOMICITY
    return PageGetLSN(BufferGetPage(buffer));
#else
    {
        char       *page = BufferGetPage(buffer);
        BufferDesc *bufHdr;
        XLogRecPtr  lsn;
 
        /*
         * If we don't need locking for correctness, fastpath out.
         */
        if (!XLogHintBitIsNeeded() || BufferIsLocal(buffer))
            return PageGetLSN(page);
 
        bufHdr = GetBufferDescriptor(buffer - 1);
        LockBufHdr(bufHdr);
        lsn = PageGetLSN(page);
        UnlockBufHdr(bufHdr);
 
        return lsn;
    }
#endif
}

References Assert, PrivateRefCountEntry::buffer, BufferGetPage(), BufferIsLocal, BufferIsPinned, BufferIsValid(), fb(), GetBufferDescriptor(), LockBufHdr(), PageGetLSN(), UnlockBufHdr(), and XLogHintBitIsNeeded.

Referenced by _bt_drop_lock_and_maybe_pin(), _bt_killitems(), gistdoinsert(), gistFindPath(), gistkillitems(), gistScanPage(), and SetHintBitsExt().

◆ BufferGetTag()

void BufferGetTag	(	Buffer	buffer,
		RelFileLocator *	rlocator,
		ForkNumber *	forknum,
		BlockNumber *	blknum
	)

Definition at line 4467 of file bufmgr.c.

{
    BufferDesc *bufHdr;
 
    /* Do the same checks as BufferGetBlockNumber. */
    Assert(BufferIsPinned(buffer));
 
    if (BufferIsLocal(buffer))
        bufHdr = GetLocalBufferDescriptor(-buffer - 1);
    else
        bufHdr = GetBufferDescriptor(buffer - 1);
 
    /* pinned, so OK to read tag without spinlock */
    *rlocator = BufTagGetRelFileLocator(&bufHdr->tag);
    *forknum = BufTagGetForkNum(&bufHdr->tag);
    *blknum = bufHdr->tag.blockNum;
}

References Assert, PrivateRefCountEntry::buffer, BufferIsLocal, BufferIsPinned, BufTagGetForkNum(), BufTagGetRelFileLocator(), fb(), GetBufferDescriptor(), and GetLocalBufferDescriptor().

Referenced by fsm_search_avail(), ginRedoInsertEntry(), heap_inplace_update_and_unlock(), log_newpage_buffer(), ResolveCminCmaxDuringDecoding(), and XLogRegisterBuffer().

◆ BufferIsDirty()

bool BufferIsDirty ( Buffer buffer )

Definition at line 3114 of file bufmgr.c.

{
    BufferDesc *bufHdr;
 
    Assert(BufferIsPinned(buffer));
 
    if (BufferIsLocal(buffer))
    {
        int         bufid = -buffer - 1;
 
        bufHdr = GetLocalBufferDescriptor(bufid);
        /* Content locks are not maintained for local buffers. */
    }
    else
    {
        bufHdr = GetBufferDescriptor(buffer - 1);
        Assert(BufferIsLockedByMeInMode(buffer, BUFFER_LOCK_SHARE_EXCLUSIVE) ||
               BufferIsLockedByMeInMode(buffer, BUFFER_LOCK_EXCLUSIVE));
    }
 
    return pg_atomic_read_u64(&bufHdr->state) & BM_DIRTY;
}

References Assert, BM_DIRTY, PrivateRefCountEntry::buffer, BUFFER_LOCK_EXCLUSIVE, BUFFER_LOCK_SHARE_EXCLUSIVE, BufferIsLocal, BufferIsLockedByMeInMode(), BufferIsPinned, fb(), GetBufferDescriptor(), GetLocalBufferDescriptor(), and pg_atomic_read_u64().

Referenced by heap_multi_insert(), heap_page_will_set_vm(), heap_xlog_multi_insert(), heap_xlog_prune_freeze(), log_heap_prune_and_freeze(), XLogRegisterBuffer(), and XLogSaveBufferForHint().

◆ BufferIsLockedByMe()

bool BufferIsLockedByMe ( Buffer buffer )

Definition at line 3061 of file bufmgr.c.

{
    BufferDesc *bufHdr;
 
    Assert(BufferIsPinned(buffer));
 
    if (BufferIsLocal(buffer))
    {
        /* Content locks are not maintained for local buffers. */
        return true;
    }
    else
    {
        bufHdr = GetBufferDescriptor(buffer - 1);
        return BufferLockHeldByMe(bufHdr);
    }
}

References Assert, PrivateRefCountEntry::buffer, BufferIsLocal, BufferIsPinned, BufferLockHeldByMe(), fb(), and GetBufferDescriptor().

Referenced by FlushOneBuffer().

◆ BufferIsLockedByMeInMode()

bool BufferIsLockedByMeInMode	(	Buffer	buffer,
		BufferLockMode	mode
	)

Definition at line 3087 of file bufmgr.c.

{
    BufferDesc *bufHdr;
 
    Assert(BufferIsPinned(buffer));
 
    if (BufferIsLocal(buffer))
    {
        /* Content locks are not maintained for local buffers. */
        return true;
    }
    else
    {
        bufHdr = GetBufferDescriptor(buffer - 1);
        return BufferLockHeldByMeInMode(bufHdr, mode);
    }
}

References Assert, PrivateRefCountEntry::buffer, BufferIsLocal, BufferIsPinned, BufferLockHeldByMeInMode(), fb(), GetBufferDescriptor(), and mode.

Referenced by BufferFinishSetHintBits(), BufferIsDirty(), heap_page_fix_vm_corruption(), HeapTupleSetHintBits(), IsBufferCleanupOK(), MarkBufferDirty(), visibilitymap_set(), and XLogRegisterBuffer().

◆ BufferIsPermanent()

bool BufferIsPermanent ( Buffer buffer )

Definition at line 4677 of file bufmgr.c.

{
    BufferDesc *bufHdr;
 
    /* Local buffers are used only for temp relations. */
    if (BufferIsLocal(buffer))
        return false;
 
    /* Make sure we've got a real buffer, and that we hold a pin on it. */
    Assert(BufferIsValid(buffer));
    Assert(BufferIsPinned(buffer));
 
    /*
     * BM_PERMANENT can't be changed while we hold a pin on the buffer, so we
     * need not bother with the buffer header spinlock.  Even if someone else
     * changes the buffer header state while we're doing this, the state is
     * changed atomically, so we'll read the old value or the new value, but
     * not random garbage.
     */
    bufHdr = GetBufferDescriptor(buffer - 1);
    return (pg_atomic_read_u64(&bufHdr->state) & BM_PERMANENT) != 0;
}

References Assert, BM_PERMANENT, PrivateRefCountEntry::buffer, BufferIsLocal, BufferIsPinned, BufferIsValid(), fb(), GetBufferDescriptor(), and pg_atomic_read_u64().

Referenced by SetHintBitsExt().

◆ BufferLockAcquire()

static void BufferLockAcquire	(	Buffer	buffer,
		BufferDesc *	buf_hdr,
		BufferLockMode	mode
	)

inlinestatic

Definition at line 5898 of file bufmgr.c.

{
    PrivateRefCountEntry *entry;
    int         extraWaits = 0;
 
    /*
     * Get reference to the refcount entry before we hold the lock, it seems
     * better to do before holding the lock.
     */
    entry = GetPrivateRefCountEntry(buffer, true);
 
    /*
     * We better not already hold a lock on the buffer.
     */
    Assert(entry->data.lockmode == BUFFER_LOCK_UNLOCK);
 
    /*
     * Lock out cancel/die interrupts until we exit the code section protected
     * by the content lock.  This ensures that interrupts will not interfere
     * with manipulations of data structures in shared memory.
     */
    HOLD_INTERRUPTS();
 
    for (;;)
    {
        uint32      wait_event = 0; /* initialized to avoid compiler warning */
        bool        mustwait;
 
        /*
         * Try to grab the lock the first time, we're not in the waitqueue
         * yet/anymore.
         */
        mustwait = BufferLockAttempt(buf_hdr, mode);
 
        if (likely(!mustwait))
        {
            break;
        }
 
        /*
         * Ok, at this point we couldn't grab the lock on the first try. We
         * cannot simply queue ourselves to the end of the list and wait to be
         * woken up because by now the lock could long have been released.
         * Instead add us to the queue and try to grab the lock again. If we
         * succeed we need to revert the queuing and be happy, otherwise we
         * recheck the lock. If we still couldn't grab it, we know that the
         * other locker will see our queue entries when releasing since they
         * existed before we checked for the lock.
         */
 
        /* add to the queue */
        BufferLockQueueSelf(buf_hdr, mode);
 
        /* we're now guaranteed to be woken up if necessary */
        mustwait = BufferLockAttempt(buf_hdr, mode);
 
        /* ok, grabbed the lock the second time round, need to undo queueing */
        if (!mustwait)
        {
            BufferLockDequeueSelf(buf_hdr);
            break;
        }
 
        switch (mode)
        {
            case BUFFER_LOCK_EXCLUSIVE:
                wait_event = WAIT_EVENT_BUFFER_EXCLUSIVE;
                break;
            case BUFFER_LOCK_SHARE_EXCLUSIVE:
                wait_event = WAIT_EVENT_BUFFER_SHARE_EXCLUSIVE;
                break;
            case BUFFER_LOCK_SHARE:
                wait_event = WAIT_EVENT_BUFFER_SHARED;
                break;
            case BUFFER_LOCK_UNLOCK:
                pg_unreachable();
 
        }
        pgstat_report_wait_start(wait_event);
 
        /*
         * Wait until awakened.
         *
         * It is possible that we get awakened for a reason other than being
         * signaled by BufferLockWakeup().  If so, loop back and wait again.
         * Once we've gotten the lock, re-increment the sema by the number of
         * additional signals received.
         */
        for (;;)
        {
            PGSemaphoreLock(MyProc->sem);
            if (MyProc->lwWaiting == LW_WS_NOT_WAITING)
                break;
            extraWaits++;
        }
 
        pgstat_report_wait_end();
 
        /* Retrying, allow BufferLockRelease to release waiters again. */
        pg_atomic_fetch_and_u64(&buf_hdr->state, ~BM_LOCK_WAKE_IN_PROGRESS);
    }
 
    /* Remember that we now hold this lock */
    entry->data.lockmode = mode;
 
    /*
     * Fix the process wait semaphore's count for any absorbed wakeups.
     */
    while (unlikely(extraWaits-- > 0))
        PGSemaphoreUnlock(MyProc->sem);
}

References Assert, BM_LOCK_WAKE_IN_PROGRESS, PrivateRefCountEntry::buffer, BUFFER_LOCK_EXCLUSIVE, BUFFER_LOCK_SHARE, BUFFER_LOCK_SHARE_EXCLUSIVE, BUFFER_LOCK_UNLOCK, BufferLockAttempt(), BufferLockDequeueSelf(), BufferLockQueueSelf(), PrivateRefCountEntry::data, fb(), GetPrivateRefCountEntry(), HOLD_INTERRUPTS, likely, PrivateRefCountData::lockmode, LW_WS_NOT_WAITING, PGPROC::lwWaiting, mode, MyProc, pg_atomic_fetch_and_u64(), pg_unreachable, PGSemaphoreLock(), PGSemaphoreUnlock(), pgstat_report_wait_end(), pgstat_report_wait_start(), PGPROC::sem, and unlikely.

Referenced by FlushUnlockedBuffer(), LockBufferInternal(), and MarkDirtyUnpinnedBufferInternal().

◆ BufferLockAttempt()

static bool BufferLockAttempt	(	BufferDesc *	buf_hdr,
		BufferLockMode	mode
	)

inlinestatic

Definition at line 6096 of file bufmgr.c.

{
    uint64      old_state;
 
    /*
     * Read once outside the loop, later iterations will get the newer value
     * via compare & exchange.
     */
    old_state = pg_atomic_read_u64(&buf_hdr->state);
 
    /* loop until we've determined whether we could acquire the lock or not */
    while (true)
    {
        uint64      desired_state;
        bool        lock_free;
 
        desired_state = old_state;
 
        if (mode == BUFFER_LOCK_EXCLUSIVE)
        {
            lock_free = (old_state & BM_LOCK_MASK) == 0;
            if (lock_free)
                desired_state += BM_LOCK_VAL_EXCLUSIVE;
        }
        else if (mode == BUFFER_LOCK_SHARE_EXCLUSIVE)
        {
            lock_free = (old_state & (BM_LOCK_VAL_EXCLUSIVE | BM_LOCK_VAL_SHARE_EXCLUSIVE)) == 0;
            if (lock_free)
                desired_state += BM_LOCK_VAL_SHARE_EXCLUSIVE;
        }
        else
        {
            lock_free = (old_state & BM_LOCK_VAL_EXCLUSIVE) == 0;
            if (lock_free)
                desired_state += BM_LOCK_VAL_SHARED;
        }
 
        /*
         * Attempt to swap in the state we are expecting. If we didn't see
         * lock to be free, that's just the old value. If we saw it as free,
         * we'll attempt to mark it acquired. The reason that we always swap
         * in the value is that this doubles as a memory barrier. We could try
         * to be smarter and only swap in values if we saw the lock as free,
         * but benchmark haven't shown it as beneficial so far.
         *
         * Retry if the value changed since we last looked at it.
         */
        if (likely(pg_atomic_compare_exchange_u64(&buf_hdr->state,
                                                  &old_state, desired_state)))
        {
            if (lock_free)
            {
                /* Great! Got the lock. */
                return false;
            }
            else
                return true;    /* somebody else has the lock */
        }
    }
 
    pg_unreachable();
}

References BM_LOCK_MASK, BM_LOCK_VAL_EXCLUSIVE, BM_LOCK_VAL_SHARE_EXCLUSIVE, BM_LOCK_VAL_SHARED, BUFFER_LOCK_EXCLUSIVE, BUFFER_LOCK_SHARE_EXCLUSIVE, fb(), likely, mode, pg_atomic_compare_exchange_u64(), pg_atomic_read_u64(), and pg_unreachable.

Referenced by BufferLockAcquire(), and BufferLockConditional().

◆ BufferLockConditional()

static bool BufferLockConditional	(	Buffer	buffer,
		BufferDesc *	buf_hdr,
		BufferLockMode	mode
	)

static

Definition at line 6050 of file bufmgr.c.

{
    PrivateRefCountEntry *entry = GetPrivateRefCountEntry(buffer, true);
    bool        mustwait;
 
    /*
     * As described above, if we're trying to lock a buffer this backend
     * already has locked, return false, independent of the existing and
     * desired lock level.
     */
    if (entry->data.lockmode != BUFFER_LOCK_UNLOCK)
        return false;
 
    /*
     * Lock out cancel/die interrupts until we exit the code section protected
     * by the content lock.  This ensures that interrupts will not interfere
     * with manipulations of data structures in shared memory.
     */
    HOLD_INTERRUPTS();
 
    /* Check for the lock */
    mustwait = BufferLockAttempt(buf_hdr, mode);
 
    if (mustwait)
    {
        /* Failed to get lock, so release interrupt holdoff */
        RESUME_INTERRUPTS();
    }
    else
    {
        entry->data.lockmode = mode;
    }
 
    return !mustwait;
}

References PrivateRefCountEntry::buffer, BUFFER_LOCK_UNLOCK, BufferLockAttempt(), PrivateRefCountEntry::data, fb(), GetPrivateRefCountEntry(), HOLD_INTERRUPTS, PrivateRefCountData::lockmode, mode, and RESUME_INTERRUPTS.

Referenced by ConditionalLockBuffer(), and GetVictimBuffer().

◆ BufferLockDequeueSelf()

static void BufferLockDequeueSelf ( BufferDesc * buf_hdr )

static

Definition at line 6203 of file bufmgr.c.

{
    bool        on_waitlist;
 
    LockBufHdr(buf_hdr);
 
    on_waitlist = MyProc->lwWaiting == LW_WS_WAITING;
    if (on_waitlist)
        proclist_delete(&buf_hdr->lock_waiters, MyProcNumber, lwWaitLink);
 
    if (proclist_is_empty(&buf_hdr->lock_waiters) &&
        (pg_atomic_read_u64(&buf_hdr->state) & BM_LOCK_HAS_WAITERS) != 0)
    {
        pg_atomic_fetch_and_u64(&buf_hdr->state, ~BM_LOCK_HAS_WAITERS);
    }
 
    /* XXX: combine with fetch_and above? */
    UnlockBufHdr(buf_hdr);
 
    /* clear waiting state again, nice for debugging */
    if (on_waitlist)
        MyProc->lwWaiting = LW_WS_NOT_WAITING;
    else
    {
        int         extraWaits = 0;
 
 
        /*
         * Somebody else dequeued us and has or will wake us up. Deal with the
         * superfluous absorption of a wakeup.
         */
 
        /*
         * Clear BM_LOCK_WAKE_IN_PROGRESS if somebody woke us before we
         * removed ourselves - they'll have set it.
         */
        pg_atomic_fetch_and_u64(&buf_hdr->state, ~BM_LOCK_WAKE_IN_PROGRESS);
 
        /*
         * Now wait for the scheduled wakeup, otherwise our ->lwWaiting would
         * get reset at some inconvenient point later. Most of the time this
         * will immediately return.
         */
        for (;;)
        {
            PGSemaphoreLock(MyProc->sem);
            if (MyProc->lwWaiting == LW_WS_NOT_WAITING)
                break;
            extraWaits++;
        }
 
        /*
         * Fix the process wait semaphore's count for any absorbed wakeups.
         */
        while (extraWaits-- > 0)
            PGSemaphoreUnlock(MyProc->sem);
    }
}

References BM_LOCK_HAS_WAITERS, BM_LOCK_WAKE_IN_PROGRESS, fb(), LockBufHdr(), LW_WS_NOT_WAITING, LW_WS_WAITING, PGPROC::lwWaiting, MyProc, MyProcNumber, pg_atomic_fetch_and_u64(), pg_atomic_read_u64(), PGSemaphoreLock(), PGSemaphoreUnlock(), proclist_delete, proclist_is_empty(), PGPROC::sem, and UnlockBufHdr().

Referenced by BufferLockAcquire().

◆ BufferLockDisown()

static void BufferLockDisown	(	Buffer	buffer,
		BufferDesc *	buf_hdr
	)

inlinestatic

Definition at line 6271 of file bufmgr.c.

{
    BufferLockDisownInternal(buffer, buf_hdr);
    RESUME_INTERRUPTS();
}

References PrivateRefCountEntry::buffer, BufferLockDisownInternal(), fb(), and RESUME_INTERRUPTS.

Referenced by buffer_stage_common().

◆ BufferLockDisownInternal()

static int BufferLockDisownInternal	(	Buffer	buffer,
		BufferDesc *	buf_hdr
	)

inlinestatic

Definition at line 6285 of file bufmgr.c.

{
    BufferLockMode mode;
    PrivateRefCountEntry *ref;
 
    ref = GetPrivateRefCountEntry(buffer, false);
    if (ref == NULL)
        elog(ERROR, "lock %d is not held", buffer);
    mode = ref->data.lockmode;
    ref->data.lockmode = BUFFER_LOCK_UNLOCK;
 
    return mode;
}

References PrivateRefCountEntry::buffer, BUFFER_LOCK_UNLOCK, elog, ERROR, fb(), GetPrivateRefCountEntry(), and mode.

Referenced by BufferLockDisown(), BufferLockUnlock(), and UnlockReleaseBuffer().

◆ BufferLockHeldByMe()

static bool BufferLockHeldByMe ( BufferDesc * buf_hdr )

static

Definition at line 6543 of file bufmgr.c.

{
    PrivateRefCountEntry *entry =
        GetPrivateRefCountEntry(BufferDescriptorGetBuffer(buf_hdr), false);
 
    if (!entry)
        return false;
    else
        return entry->data.lockmode != BUFFER_LOCK_UNLOCK;
}

References BUFFER_LOCK_UNLOCK, BufferDescriptorGetBuffer(), PrivateRefCountEntry::data, fb(), GetPrivateRefCountEntry(), and PrivateRefCountData::lockmode.

Referenced by buffer_stage_common(), BufferIsLockedByMe(), and UnpinBufferNoOwner().

◆ BufferLockHeldByMeInMode()

static bool BufferLockHeldByMeInMode	(	BufferDesc *	buf_hdr,
		BufferLockMode	mode
	)

static

Definition at line 6525 of file bufmgr.c.

{
    PrivateRefCountEntry *entry =
        GetPrivateRefCountEntry(BufferDescriptorGetBuffer(buf_hdr), false);
 
    if (!entry)
        return false;
    else
        return entry->data.lockmode == mode;
}

References BufferDescriptorGetBuffer(), PrivateRefCountEntry::data, fb(), GetPrivateRefCountEntry(), PrivateRefCountData::lockmode, and mode.

Referenced by BufferIsLockedByMeInMode(), FlushBuffer(), and MarkSharedBufferDirtyHint().

◆ BufferLockProcessRelease()

static void BufferLockProcessRelease	(	BufferDesc *	buf_hdr,
		BufferLockMode	mode,
		uint64	lockstate
	)

static

Definition at line 6470 of file bufmgr.c.

{
    bool        check_waiters = false;
    bool        wake_exclusive = false;
 
    /* nobody else can have that kind of lock */
    Assert(!(lockstate & BM_LOCK_VAL_EXCLUSIVE));
 
    /*
     * If we're still waiting for backends to get scheduled, don't wake them
     * up again. Otherwise check if we need to look through the waitqueue to
     * wake other backends.
     */
    if ((lockstate & BM_LOCK_HAS_WAITERS) &&
        !(lockstate & BM_LOCK_WAKE_IN_PROGRESS))
    {
        if ((lockstate & BM_LOCK_MASK) == 0)
        {
            /*
             * We released a lock and the lock was, in that moment, free. We
             * therefore can wake waiters for any kind of lock.
             */
            check_waiters = true;
            wake_exclusive = true;
        }
        else if (mode == BUFFER_LOCK_SHARE_EXCLUSIVE)
        {
            /*
             * We released the lock, but another backend still holds a lock.
             * We can't have released an exclusive lock, as there couldn't
             * have been other lock holders. If we released a share lock, no
             * waiters need to be woken up, as there must be other share
             * lockers. However, if we held a share-exclusive lock, another
             * backend now could acquire a share-exclusive lock.
             */
            check_waiters = true;
            wake_exclusive = false;
        }
    }
 
    /*
     * As waking up waiters requires the spinlock to be acquired, only do so
     * if necessary.
     */
    if (check_waiters)
        BufferLockWakeup(buf_hdr, wake_exclusive);
}

References Assert, BM_LOCK_HAS_WAITERS, BM_LOCK_MASK, BM_LOCK_VAL_EXCLUSIVE, BM_LOCK_WAKE_IN_PROGRESS, BUFFER_LOCK_SHARE_EXCLUSIVE, BufferLockWakeup(), fb(), and mode.

Referenced by BufferLockUnlock(), and UnlockReleaseBuffer().

◆ BufferLockQueueSelf()

static void BufferLockQueueSelf	(	BufferDesc *	buf_hdr,
		BufferLockMode	mode
	)

static

Definition at line 6163 of file bufmgr.c.

{
    /*
     * If we don't have a PGPROC structure, there's no way to wait. This
     * should never occur, since MyProc should only be null during shared
     * memory initialization.
     */
    if (MyProc == NULL)
        elog(PANIC, "cannot wait without a PGPROC structure");
 
    if (MyProc->lwWaiting != LW_WS_NOT_WAITING)
        elog(PANIC, "queueing for lock while waiting on another one");
 
    LockBufHdr(buf_hdr);
 
    /* setting the flag is protected by the spinlock */
    pg_atomic_fetch_or_u64(&buf_hdr->state, BM_LOCK_HAS_WAITERS);
 
    /*
     * These are currently used both for lwlocks and buffer content locks,
     * which is acceptable, although not pretty, because a backend can't wait
     * for both types of locks at the same time.
     */
    MyProc->lwWaiting = LW_WS_WAITING;
    MyProc->lwWaitMode = mode;
 
    proclist_push_tail(&buf_hdr->lock_waiters, MyProcNumber, lwWaitLink);
 
    /* Can release the mutex now */
    UnlockBufHdr(buf_hdr);
}

References BM_LOCK_HAS_WAITERS, elog, fb(), LockBufHdr(), LW_WS_NOT_WAITING, LW_WS_WAITING, PGPROC::lwWaiting, PGPROC::lwWaitMode, mode, MyProc, MyProcNumber, PANIC, pg_atomic_fetch_or_u64(), proclist_push_tail, and UnlockBufHdr().

Referenced by BufferLockAcquire().

◆ BufferLockReleaseSub()

static uint64 BufferLockReleaseSub ( BufferLockMode mode )

inlinestatic

Definition at line 6441 of file bufmgr.c.

{
    /*
     * Turns out that a switch() leads gcc to generate sufficiently worse code
     * for this to show up in profiles...
     */
    if (mode == BUFFER_LOCK_EXCLUSIVE)
        return BM_LOCK_VAL_EXCLUSIVE;
    else if (mode == BUFFER_LOCK_SHARE_EXCLUSIVE)
        return BM_LOCK_VAL_SHARE_EXCLUSIVE;
    else
    {
        Assert(mode == BUFFER_LOCK_SHARE);
        return BM_LOCK_VAL_SHARED;
    }
 
    return 0;                   /* keep compiler quiet */
}

References Assert, BM_LOCK_VAL_EXCLUSIVE, BM_LOCK_VAL_SHARE_EXCLUSIVE, BM_LOCK_VAL_SHARED, BUFFER_LOCK_EXCLUSIVE, BUFFER_LOCK_SHARE, BUFFER_LOCK_SHARE_EXCLUSIVE, and mode.

Referenced by BufferLockUnlock(), and UnlockReleaseBuffer().

◆ BufferLockUnlock()

static void BufferLockUnlock	(	Buffer	buffer,
		BufferDesc *	buf_hdr
	)

static

Definition at line 6014 of file bufmgr.c.

{
    BufferLockMode mode;
    uint64      oldstate;
    uint64      sub;
 
    mode = BufferLockDisownInternal(buffer, buf_hdr);
 
    /*
     * Release my hold on lock, after that it can immediately be acquired by
     * others, even if we still have to wakeup other waiters.
     */
    sub = BufferLockReleaseSub(mode);
 
    oldstate = pg_atomic_sub_fetch_u64(&buf_hdr->state, sub);
 
    BufferLockProcessRelease(buf_hdr, mode, oldstate);
 
    /*
     * Now okay to allow cancel/die interrupts.
     */
    RESUME_INTERRUPTS();
}

References PrivateRefCountEntry::buffer, BufferLockDisownInternal(), BufferLockProcessRelease(), BufferLockReleaseSub(), fb(), mode, pg_atomic_sub_fetch_u64(), and RESUME_INTERRUPTS.

Referenced by FlushUnlockedBuffer(), MarkDirtyUnpinnedBufferInternal(), ResOwnerReleaseBuffer(), and UnlockBuffer().

◆ BufferLockWakeup()

static void BufferLockWakeup	(	BufferDesc *	buf_hdr,
		bool	unlocked
	)

static

Definition at line 6305 of file bufmgr.c.

{
    bool        new_wake_in_progress = false;
    bool        wake_share_exclusive = true;
    proclist_head wakeup;
    proclist_mutable_iter iter;
 
    proclist_init(&wakeup);
 
    /* lock wait list while collecting backends to wake up */
    LockBufHdr(buf_hdr);
 
    proclist_foreach_modify(iter, &buf_hdr->lock_waiters, lwWaitLink)
    {
        PGPROC     *waiter = GetPGProcByNumber(iter.cur);
 
        /*
         * Already woke up a conflicting lock, so skip over this wait list
         * entry.
         */
        if (!wake_exclusive && waiter->lwWaitMode == BUFFER_LOCK_EXCLUSIVE)
            continue;
        if (!wake_share_exclusive && waiter->lwWaitMode == BUFFER_LOCK_SHARE_EXCLUSIVE)
            continue;
 
        proclist_delete(&buf_hdr->lock_waiters, iter.cur, lwWaitLink);
        proclist_push_tail(&wakeup, iter.cur, lwWaitLink);
 
        /*
         * Prevent additional wakeups until retryer gets to run. Backends that
         * are just waiting for the lock to become free don't retry
         * automatically.
         */
        new_wake_in_progress = true;
 
        /*
         * Signal that the process isn't on the wait list anymore. This allows
         * BufferLockDequeueSelf() to remove itself from the waitlist with a
         * proclist_delete(), rather than having to check if it has been
         * removed from the list.
         */
        Assert(waiter->lwWaiting == LW_WS_WAITING);
        waiter->lwWaiting = LW_WS_PENDING_WAKEUP;
 
        /*
         * Don't wakeup further waiters after waking a conflicting waiter.
         */
        if (waiter->lwWaitMode == BUFFER_LOCK_SHARE)
        {
            /*
             * Share locks conflict with exclusive locks.
             */
            wake_exclusive = false;
        }
        else if (waiter->lwWaitMode == BUFFER_LOCK_SHARE_EXCLUSIVE)
        {
            /*
             * Share-exclusive locks conflict with share-exclusive and
             * exclusive locks.
             */
            wake_exclusive = false;
            wake_share_exclusive = false;
        }
        else if (waiter->lwWaitMode == BUFFER_LOCK_EXCLUSIVE)
        {
            /*
             * Exclusive locks conflict with all other locks, there's no point
             * in waking up anybody else.
             */
            break;
        }
    }
 
    Assert(proclist_is_empty(&wakeup) || pg_atomic_read_u64(&buf_hdr->state) & BM_LOCK_HAS_WAITERS);
 
    /* unset required flags, and release lock, in one fell swoop */
    {
        uint64      old_state;
        uint64      desired_state;
 
        old_state = pg_atomic_read_u64(&buf_hdr->state);
        while (true)
        {
            desired_state = old_state;
 
            /* compute desired flags */
 
            if (new_wake_in_progress)
                desired_state |= BM_LOCK_WAKE_IN_PROGRESS;
            else
                desired_state &= ~BM_LOCK_WAKE_IN_PROGRESS;
 
            if (proclist_is_empty(&buf_hdr->lock_waiters))
                desired_state &= ~BM_LOCK_HAS_WAITERS;
 
            desired_state &= ~BM_LOCKED;    /* release lock */
 
            if (pg_atomic_compare_exchange_u64(&buf_hdr->state, &old_state,
                                               desired_state))
                break;
        }
    }
 
    /* Awaken any waiters I removed from the queue. */
    proclist_foreach_modify(iter, &wakeup, lwWaitLink)
    {
        PGPROC     *waiter = GetPGProcByNumber(iter.cur);
 
        proclist_delete(&wakeup, iter.cur, lwWaitLink);
 
        /*
         * Guarantee that lwWaiting being unset only becomes visible once the
         * unlink from the link has completed. Otherwise the target backend
         * could be woken up for other reason and enqueue for a new lock - if
         * that happens before the list unlink happens, the list would end up
         * being corrupted.
         *
         * The barrier pairs with the LockBufHdr() when enqueuing for another
         * lock.
         */
        pg_write_barrier();
        waiter->lwWaiting = LW_WS_NOT_WAITING;
        PGSemaphoreUnlock(waiter->sem);
    }
}

References Assert, BM_LOCK_HAS_WAITERS, BM_LOCK_WAKE_IN_PROGRESS, BUFFER_LOCK_EXCLUSIVE, BUFFER_LOCK_SHARE, BUFFER_LOCK_SHARE_EXCLUSIVE, proclist_mutable_iter::cur, fb(), GetPGProcByNumber, LockBufHdr(), LW_WS_NOT_WAITING, LW_WS_PENDING_WAKEUP, LW_WS_WAITING, PGPROC::lwWaiting, PGPROC::lwWaitMode, pg_atomic_compare_exchange_u64(), pg_atomic_read_u64(), pg_write_barrier, PGSemaphoreUnlock(), proclist_delete, proclist_foreach_modify, proclist_init(), proclist_is_empty(), proclist_push_tail, PGPROC::sem, and wakeup.

Referenced by BufferLockProcessRelease().

◆ BufferSetHintBits16()

bool BufferSetHintBits16	(	uint16 *	ptr,
		uint16	val,
		Buffer	buffer
	)

Definition at line 7093 of file bufmgr.c.

{
    BufferDesc *buf_hdr;
    uint64      lockstate;
#ifdef USE_ASSERT_CHECKING
    char       *page;
 
    /* verify that the address is on the page */
    page = BufferGetPage(buffer);
    Assert((char *) ptr >= page && (char *) ptr < (page + BLCKSZ));
#endif
 
    if (BufferIsLocal(buffer))
    {
        *ptr = val;
 
        MarkLocalBufferDirty(buffer);
 
        return true;
    }
 
    buf_hdr = GetBufferDescriptor(buffer - 1);
 
    if (SharedBufferBeginSetHintBits(buffer, buf_hdr, &lockstate))
    {
        *ptr = val;
 
        MarkSharedBufferDirtyHint(buffer, buf_hdr, lockstate, true);
 
        return true;
    }
 
    return false;
}

References Assert, PrivateRefCountEntry::buffer, BufferGetPage(), BufferIsLocal, fb(), GetBufferDescriptor(), MarkLocalBufferDirty(), MarkSharedBufferDirtyHint(), SharedBufferBeginSetHintBits(), and val.

Referenced by SetHintBitsExt().

◆ BufferSync()

static void BufferSync ( int flags )

static

Definition at line 3552 of file bufmgr.c.

{
    uint64      buf_state;
    int         buf_id;
    int         num_to_scan;
    int         num_spaces;
    int         num_processed;
    int         num_written;
    CkptTsStatus *per_ts_stat = NULL;
    Oid         last_tsid;
    binaryheap *ts_heap;
    int         i;
    uint64      mask = BM_DIRTY;
    WritebackContext wb_context;
 
    /*
     * Unless this is a shutdown checkpoint or we have been explicitly told,
     * we write only permanent, dirty buffers.  But at shutdown or end of
     * recovery, we write all dirty buffers.
     */
    if (!((flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY |
                    CHECKPOINT_FLUSH_UNLOGGED))))
        mask |= BM_PERMANENT;
 
    /*
     * Loop over all buffers, and mark the ones that need to be written with
     * BM_CHECKPOINT_NEEDED.  Count them as we go (num_to_scan), so that we
     * can estimate how much work needs to be done.
     *
     * This allows us to write only those pages that were dirty when the
     * checkpoint began, and not those that get dirtied while it proceeds.
     * Whenever a page with BM_CHECKPOINT_NEEDED is written out, either by us
     * later in this function, or by normal backends or the bgwriter cleaning
     * scan, the flag is cleared.  Any buffer dirtied after this point won't
     * have the flag set.
     *
     * Note that if we fail to write some buffer, we may leave buffers with
     * BM_CHECKPOINT_NEEDED still set.  This is OK since any such buffer would
     * certainly need to be written for the next checkpoint attempt, too.
     */
    num_to_scan = 0;
    for (buf_id = 0; buf_id < NBuffers; buf_id++)
    {
        BufferDesc *bufHdr = GetBufferDescriptor(buf_id);
        uint64      set_bits = 0;
 
        /*
         * Header spinlock is enough to examine BM_DIRTY, see comment in
         * SyncOneBuffer.
         */
        buf_state = LockBufHdr(bufHdr);
 
        if ((buf_state & mask) == mask)
        {
            CkptSortItem *item;
 
            set_bits = BM_CHECKPOINT_NEEDED;
 
            item = &CkptBufferIds[num_to_scan++];
            item->buf_id = buf_id;
            item->tsId = bufHdr->tag.spcOid;
            item->relNumber = BufTagGetRelNumber(&bufHdr->tag);
            item->forkNum = BufTagGetForkNum(&bufHdr->tag);
            item->blockNum = bufHdr->tag.blockNum;
        }
 
        UnlockBufHdrExt(bufHdr, buf_state,
                        set_bits, 0,
                        0);
 
        /* Check for barrier events in case NBuffers is large. */
        if (ProcSignalBarrierPending)
            ProcessProcSignalBarrier();
    }
 
    if (num_to_scan == 0)
        return;                 /* nothing to do */
 
    WritebackContextInit(&wb_context, &checkpoint_flush_after);
 
    TRACE_POSTGRESQL_BUFFER_SYNC_START(NBuffers, num_to_scan);
 
    /*
     * Sort buffers that need to be written to reduce the likelihood of random
     * IO. The sorting is also important for the implementation of balancing
     * writes between tablespaces. Without balancing writes we'd potentially
     * end up writing to the tablespaces one-by-one; possibly overloading the
     * underlying system.
     */
    sort_checkpoint_bufferids(CkptBufferIds, num_to_scan);
 
    num_spaces = 0;
 
    /*
     * Allocate progress status for each tablespace with buffers that need to
     * be flushed. This requires the to-be-flushed array to be sorted.
     */
    last_tsid = InvalidOid;
    for (i = 0; i < num_to_scan; i++)
    {
        CkptTsStatus *s;
        Oid         cur_tsid;
 
        cur_tsid = CkptBufferIds[i].tsId;
 
        /*
         * Grow array of per-tablespace status structs, every time a new
         * tablespace is found.
         */
        if (last_tsid == InvalidOid || last_tsid != cur_tsid)
        {
            Size        sz;
 
            num_spaces++;
 
            /*
             * Not worth adding grow-by-power-of-2 logic here - even with a
             * few hundred tablespaces this should be fine.
             */
            sz = sizeof(CkptTsStatus) * num_spaces;
 
            if (per_ts_stat == NULL)
                per_ts_stat = (CkptTsStatus *) palloc(sz);
            else
                per_ts_stat = (CkptTsStatus *) repalloc(per_ts_stat, sz);
 
            s = &per_ts_stat[num_spaces - 1];
            memset(s, 0, sizeof(*s));
            s->tsId = cur_tsid;
 
            /*
             * The first buffer in this tablespace. As CkptBufferIds is sorted
             * by tablespace all (s->num_to_scan) buffers in this tablespace
             * will follow afterwards.
             */
            s->index = i;
 
            /*
             * progress_slice will be determined once we know how many buffers
             * are in each tablespace, i.e. after this loop.
             */
 
            last_tsid = cur_tsid;
        }
        else
        {
            s = &per_ts_stat[num_spaces - 1];
        }
 
        s->num_to_scan++;
 
        /* Check for barrier events. */
        if (ProcSignalBarrierPending)
            ProcessProcSignalBarrier();
    }
 
    Assert(num_spaces > 0);
 
    /*
     * Build a min-heap over the write-progress in the individual tablespaces,
     * and compute how large a portion of the total progress a single
     * processed buffer is.
     */
    ts_heap = binaryheap_allocate(num_spaces,
                                  ts_ckpt_progress_comparator,
                                  NULL);
 
    for (i = 0; i < num_spaces; i++)
    {
        CkptTsStatus *ts_stat = &per_ts_stat[i];
 
        ts_stat->progress_slice = (float8) num_to_scan / ts_stat->num_to_scan;
 
        binaryheap_add_unordered(ts_heap, PointerGetDatum(ts_stat));
    }
 
    binaryheap_build(ts_heap);
 
    /*
     * Iterate through to-be-checkpointed buffers and write the ones (still)
     * marked with BM_CHECKPOINT_NEEDED. The writes are balanced between
     * tablespaces; otherwise the sorting would lead to only one tablespace
     * receiving writes at a time, making inefficient use of the hardware.
     */
    num_processed = 0;
    num_written = 0;
    while (!binaryheap_empty(ts_heap))
    {
        BufferDesc *bufHdr = NULL;
        CkptTsStatus *ts_stat = (CkptTsStatus *)
            DatumGetPointer(binaryheap_first(ts_heap));
 
        buf_id = CkptBufferIds[ts_stat->index].buf_id;
        Assert(buf_id != -1);
 
        bufHdr = GetBufferDescriptor(buf_id);
 
        num_processed++;
 
        /*
         * We don't need to acquire the lock here, because we're only looking
         * at a single bit. It's possible that someone else writes the buffer
         * and clears the flag right after we check, but that doesn't matter
         * since SyncOneBuffer will then do nothing.  However, there is a
         * further race condition: it's conceivable that between the time we
         * examine the bit here and the time SyncOneBuffer acquires the lock,
         * someone else not only wrote the buffer but replaced it with another
         * page and dirtied it.  In that improbable case, SyncOneBuffer will
         * write the buffer though we didn't need to.  It doesn't seem worth
         * guarding against this, though.
         */
        if (pg_atomic_read_u64(&bufHdr->state) & BM_CHECKPOINT_NEEDED)
        {
            if (SyncOneBuffer(buf_id, false, &wb_context) & BUF_WRITTEN)
            {
                TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id);
                PendingCheckpointerStats.buffers_written++;
                num_written++;
            }
        }
 
        /*
         * Measure progress independent of actually having to flush the buffer
         * - otherwise writing become unbalanced.
         */
        ts_stat->progress += ts_stat->progress_slice;
        ts_stat->num_scanned++;
        ts_stat->index++;
 
        /* Have all the buffers from the tablespace been processed? */
        if (ts_stat->num_scanned == ts_stat->num_to_scan)
        {
            binaryheap_remove_first(ts_heap);
        }
        else
        {
            /* update heap with the new progress */
            binaryheap_replace_first(ts_heap, PointerGetDatum(ts_stat));
        }
 
        /*
         * Sleep to throttle our I/O rate.
         *
         * (This will check for barrier events even if it doesn't sleep.)
         */
        CheckpointWriteDelay(flags, (double) num_processed / num_to_scan);
    }
 
    /*
     * Issue all pending flushes. Only checkpointer calls BufferSync(), so
     * IOContext will always be IOCONTEXT_NORMAL.
     */
    IssuePendingWritebacks(&wb_context, IOCONTEXT_NORMAL);
 
    pfree(per_ts_stat);
    per_ts_stat = NULL;
    binaryheap_free(ts_heap);
 
    /*
     * Update checkpoint statistics. As noted above, this doesn't include
     * buffers written by other backends or bgwriter scan.
     */
    CheckpointStats.ckpt_bufs_written += num_written;
 
    TRACE_POSTGRESQL_BUFFER_SYNC_DONE(NBuffers, num_written, num_to_scan);
}

References Assert, binaryheap_add_unordered(), binaryheap_allocate(), binaryheap_build(), binaryheap_empty, binaryheap_first(), binaryheap_free(), binaryheap_remove_first(), binaryheap_replace_first(), CkptSortItem::blockNum, BM_CHECKPOINT_NEEDED, BM_DIRTY, BM_PERMANENT, CkptSortItem::buf_id, BUF_WRITTEN, PgStat_CheckpointerStats::buffers_written, BufTagGetForkNum(), BufTagGetRelNumber(), CHECKPOINT_END_OF_RECOVERY, checkpoint_flush_after, CHECKPOINT_FLUSH_UNLOGGED, CHECKPOINT_IS_SHUTDOWN, CheckpointStats, CheckpointWriteDelay(), CheckpointStatsData::ckpt_bufs_written, CkptBufferIds, DatumGetPointer(), fb(), CkptSortItem::forkNum, GetBufferDescriptor(), i, CkptTsStatus::index, InvalidOid, IOCONTEXT_NORMAL, IssuePendingWritebacks(), LockBufHdr(), NBuffers, CkptTsStatus::num_to_scan, palloc(), PendingCheckpointerStats, pfree(), pg_atomic_read_u64(), PointerGetDatum(), ProcessProcSignalBarrier(), ProcSignalBarrierPending, CkptTsStatus::progress_slice, CkptSortItem::relNumber, repalloc(), SyncOneBuffer(), ts_ckpt_progress_comparator(), CkptTsStatus::tsId, CkptSortItem::tsId, UnlockBufHdrExt(), and WritebackContextInit().

Referenced by CheckPointBuffers().

◆ buffertag_comparator()

static int buffertag_comparator	(	const BufferTag *	ba,
		const BufferTag *	bb
	)

inlinestatic

Definition at line 7590 of file bufmgr.c.

{
    int         ret;
    RelFileLocator rlocatora;
    RelFileLocator rlocatorb;
 
    rlocatora = BufTagGetRelFileLocator(ba);
    rlocatorb = BufTagGetRelFileLocator(bb);
 
    ret = rlocator_comparator(&rlocatora, &rlocatorb);
 
    if (ret != 0)
        return ret;
 
    if (BufTagGetForkNum(ba) < BufTagGetForkNum(bb))
        return -1;
    if (BufTagGetForkNum(ba) > BufTagGetForkNum(bb))
        return 1;
 
    if (ba->blockNum < bb->blockNum)
        return -1;
    if (ba->blockNum > bb->blockNum)
        return 1;
 
    return 0;
}

References BufTagGetForkNum(), BufTagGetRelFileLocator(), fb(), and rlocator_comparator().

◆ CheckBufferIsPinnedOnce()

void CheckBufferIsPinnedOnce ( Buffer buffer )

Definition at line 6637 of file bufmgr.c.

{
    if (BufferIsLocal(buffer))
    {
        if (LocalRefCount[-buffer - 1] != 1)
            elog(ERROR, "incorrect local pin count: %d",
                 LocalRefCount[-buffer - 1]);
    }
    else
    {
        if (GetPrivateRefCount(buffer) != 1)
            elog(ERROR, "incorrect local pin count: %d",
                 GetPrivateRefCount(buffer));
    }
}

References PrivateRefCountEntry::buffer, BufferIsLocal, elog, ERROR, GetPrivateRefCount(), and LocalRefCount.

Referenced by GetVictimBuffer(), lazy_scan_heap(), and LockBufferForCleanup().

◆ CheckForBufferLeaks()

static void CheckForBufferLeaks ( void )

static

Definition at line 4263 of file bufmgr.c.

{
#ifdef USE_ASSERT_CHECKING
    int         RefCountErrors = 0;
    PrivateRefCountEntry *res;
    int         i;
    char       *s;
 
    /* check the array */
    for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++)
    {
        if (PrivateRefCountArrayKeys[i] != InvalidBuffer)
        {
            res = &PrivateRefCountArray[i];
 
            s = DebugPrintBufferRefcount(res->buffer);
            elog(WARNING, "buffer refcount leak: %s", s);
            pfree(s);
 
            RefCountErrors++;
        }
    }
 
    /* if necessary search the hash */
    if (PrivateRefCountOverflowed)
    {
        refcount_iterator iter;
 
        refcount_start_iterate(PrivateRefCountHash, &iter);
        while ((res = refcount_iterate(PrivateRefCountHash, &iter)) != NULL)
        {
            s = DebugPrintBufferRefcount(res->buffer);
            elog(WARNING, "buffer refcount leak: %s", s);
            pfree(s);
            RefCountErrors++;
        }
    }
 
    Assert(RefCountErrors == 0);
#endif
}

References Assert, PrivateRefCountEntry::buffer, DebugPrintBufferRefcount(), elog, fb(), i, InvalidBuffer, pfree(), PrivateRefCountArray, PrivateRefCountArrayKeys, PrivateRefCountHash, PrivateRefCountOverflowed, REFCOUNT_ARRAY_ENTRIES, and WARNING.

Referenced by AtEOXact_Buffers(), and AtProcExit_Buffers().

◆ CheckPointBuffers()

void CheckPointBuffers ( int flags )

Definition at line 4432 of file bufmgr.c.

{
    BufferSync(flags);
}

References BufferSync().

Referenced by CheckPointGuts().

◆ CheckReadBuffersOperation()

static void CheckReadBuffersOperation	(	ReadBuffersOperation *	operation,
		bool	is_complete
	)

static

Definition at line 1647 of file bufmgr.c.

{
#ifdef USE_ASSERT_CHECKING
    Assert(operation->nblocks_done <= operation->nblocks);
    Assert(!is_complete || operation->nblocks == operation->nblocks_done);
 
    for (int i = 0; i < operation->nblocks; i++)
    {
        Buffer      buffer = operation->buffers[i];
        BufferDesc *buf_hdr = BufferIsLocal(buffer) ?
            GetLocalBufferDescriptor(-buffer - 1) :
            GetBufferDescriptor(buffer - 1);
 
        Assert(BufferGetBlockNumber(buffer) == operation->blocknum + i);
        Assert(pg_atomic_read_u64(&buf_hdr->state) & BM_TAG_VALID);
 
        if (i < operation->nblocks_done)
            Assert(pg_atomic_read_u64(&buf_hdr->state) & BM_VALID);
    }
#endif
}

References Assert, BM_TAG_VALID, BM_VALID, PrivateRefCountEntry::buffer, BufferGetBlockNumber(), BufferIsLocal, fb(), GetBufferDescriptor(), GetLocalBufferDescriptor(), i, operation, and pg_atomic_read_u64().

Referenced by StartReadBuffersImpl(), and WaitReadBuffers().

◆ ckpt_buforder_comparator()

static int ckpt_buforder_comparator	(	const CkptSortItem *	a,
		const CkptSortItem *	b
	)

inlinestatic

Definition at line 7624 of file bufmgr.c.

{
    /* compare tablespace */
    if (a->tsId < b->tsId)
        return -1;
    else if (a->tsId > b->tsId)
        return 1;
    /* compare relation */
    if (a->relNumber < b->relNumber)
        return -1;
    else if (a->relNumber > b->relNumber)
        return 1;
    /* compare fork */
    else if (a->forkNum < b->forkNum)
        return -1;
    else if (a->forkNum > b->forkNum)
        return 1;
    /* compare block number */
    else if (a->blockNum < b->blockNum)
        return -1;
    else if (a->blockNum > b->blockNum)
        return 1;
    /* equal page IDs are unlikely, but not impossible */
    return 0;
}

References a, and b.

◆ ConditionalLockBuffer()

bool ConditionalLockBuffer ( Buffer buffer )

Definition at line 6617 of file bufmgr.c.

{
    BufferDesc *buf;
 
    Assert(BufferIsPinned(buffer));
    if (BufferIsLocal(buffer))
        return true;            /* act as though we got it */
 
    buf = GetBufferDescriptor(buffer - 1);
 
    return BufferLockConditional(buffer, buf, BUFFER_LOCK_EXCLUSIVE);
}

References Assert, buf, PrivateRefCountEntry::buffer, BUFFER_LOCK_EXCLUSIVE, BufferIsLocal, BufferIsPinned, BufferLockConditional(), and GetBufferDescriptor().

Referenced by _bt_conditionallockbuf(), BloomNewBuffer(), ConditionalLockBufferForCleanup(), GinNewBuffer(), gistNewBuffer(), RelationGetBufferForTuple(), spgdoinsert(), SpGistGetBuffer(), SpGistNewBuffer(), and SpGistUpdateMetaPage().

◆ ConditionalLockBufferForCleanup()

bool ConditionalLockBufferForCleanup ( Buffer buffer )

Definition at line 6843 of file bufmgr.c.

{
    BufferDesc *bufHdr;
    uint64      buf_state,
                refcount;
 
    Assert(BufferIsValid(buffer));
 
    /* see AIO related comment in LockBufferForCleanup() */
 
    if (BufferIsLocal(buffer))
    {
        refcount = LocalRefCount[-buffer - 1];
        /* There should be exactly one pin */
        Assert(refcount > 0);
        if (refcount != 1)
            return false;
        /* Nobody else to wait for */
        return true;
    }
 
    /* There should be exactly one local pin */
    refcount = GetPrivateRefCount(buffer);
    Assert(refcount);
    if (refcount != 1)
        return false;
 
    /* Try to acquire lock */
    if (!ConditionalLockBuffer(buffer))
        return false;
 
    bufHdr = GetBufferDescriptor(buffer - 1);
    buf_state = LockBufHdr(bufHdr);
    refcount = BUF_STATE_GET_REFCOUNT(buf_state);
 
    Assert(refcount > 0);
    if (refcount == 1)
    {
        /* Successfully acquired exclusive lock with pincount 1 */
        UnlockBufHdr(bufHdr);
        return true;
    }
 
    /* Failed, so release the lock */
    UnlockBufHdr(bufHdr);
    LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    return false;
}

References Assert, BUF_STATE_GET_REFCOUNT, PrivateRefCountEntry::buffer, BUFFER_LOCK_UNLOCK, BufferIsLocal, BufferIsValid(), ConditionalLockBuffer(), fb(), GetBufferDescriptor(), GetPrivateRefCount(), LocalRefCount, LockBuffer(), LockBufHdr(), and UnlockBufHdr().

Referenced by _hash_finish_split(), _hash_getbuf_with_condlock_cleanup(), heap_page_prune_opt(), and lazy_scan_heap().

◆ CreateAndCopyRelationData()

void CreateAndCopyRelationData	(	RelFileLocator	src_rlocator,
		RelFileLocator	dst_rlocator,
		bool	permanent
	)

Definition at line 5462 of file bufmgr.c.

{
    char        relpersistence;
    SMgrRelation src_rel;
    SMgrRelation dst_rel;
 
    /* Set the relpersistence. */
    relpersistence = permanent ?
        RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED;
 
    src_rel = smgropen(src_rlocator, INVALID_PROC_NUMBER);
    dst_rel = smgropen(dst_rlocator, INVALID_PROC_NUMBER);
 
    /*
     * Create and copy all forks of the relation.  During create database we
     * have a separate cleanup mechanism which deletes complete database
     * directory.  Therefore, each individual relation doesn't need to be
     * registered for cleanup.
     */
    RelationCreateStorage(dst_rlocator, relpersistence, false);
 
    /* copy main fork. */
    RelationCopyStorageUsingBuffer(src_rlocator, dst_rlocator, MAIN_FORKNUM,
                                   permanent);
 
    /* copy those extra forks that exist */
    for (ForkNumber forkNum = MAIN_FORKNUM + 1;
         forkNum <= MAX_FORKNUM; forkNum++)
    {
        if (smgrexists(src_rel, forkNum))
        {
            smgrcreate(dst_rel, forkNum, false);
 
            /*
             * WAL log creation if the relation is persistent, or this is the
             * init fork of an unlogged relation.
             */
            if (permanent || forkNum == INIT_FORKNUM)
                log_smgrcreate(&dst_rlocator, forkNum);
 
            /* Copy a fork's data, block by block. */
            RelationCopyStorageUsingBuffer(src_rlocator, dst_rlocator, forkNum,
                                           permanent);
        }
    }
}

References fb(), INIT_FORKNUM, INVALID_PROC_NUMBER, log_smgrcreate(), MAIN_FORKNUM, MAX_FORKNUM, RelationCopyStorageUsingBuffer(), RelationCreateStorage(), smgrcreate(), smgrexists(), and smgropen().

Referenced by CreateDatabaseUsingWalLog().

◆ DebugPrintBufferRefcount()

char * DebugPrintBufferRefcount ( Buffer buffer )

Definition at line 4389 of file bufmgr.c.

{
    BufferDesc *buf;
    int32       loccount;
    char       *result;
    ProcNumber  backend;
    uint64      buf_state;
 
    Assert(BufferIsValid(buffer));
    if (BufferIsLocal(buffer))
    {
        buf = GetLocalBufferDescriptor(-buffer - 1);
        loccount = LocalRefCount[-buffer - 1];
        backend = MyProcNumber;
    }
    else
    {
        buf = GetBufferDescriptor(buffer - 1);
        loccount = GetPrivateRefCount(buffer);
        backend = INVALID_PROC_NUMBER;
    }
 
    /* theoretically we should lock the bufHdr here */
    buf_state = pg_atomic_read_u64(&buf->state);
 
    result = psprintf("[%03d] (rel=%s, blockNum=%u, flags=0x%" PRIx64 ", refcount=%u %d)",
                      buffer,
                      relpathbackend(BufTagGetRelFileLocator(&buf->tag), backend,
                                     BufTagGetForkNum(&buf->tag)).str,
                      buf->tag.blockNum, buf_state & BUF_FLAG_MASK,
                      BUF_STATE_GET_REFCOUNT(buf_state), loccount);
    return result;
}

References Assert, buf, BUF_FLAG_MASK, BUF_STATE_GET_REFCOUNT, PrivateRefCountEntry::buffer, BufferIsLocal, BufferIsValid(), BufTagGetForkNum(), BufTagGetRelFileLocator(), fb(), GetBufferDescriptor(), GetLocalBufferDescriptor(), GetPrivateRefCount(), INVALID_PROC_NUMBER, LocalRefCount, MyProcNumber, pg_atomic_read_u64(), psprintf(), relpathbackend, and result.

Referenced by buffer_call_start_io(), buffer_call_terminate_io(), CheckForBufferLeaks(), CheckForLocalBufferLeaks(), and ResOwnerPrintBuffer().

◆ DropDatabaseBuffers()

void DropDatabaseBuffers ( Oid dbid )

Definition at line 5115 of file bufmgr.c.

{
    int         i;
 
    /*
     * We needn't consider local buffers, since by assumption the target
     * database isn't our own.
     */
 
    for (i = 0; i < NBuffers; i++)
    {
        BufferDesc *bufHdr = GetBufferDescriptor(i);
 
        /*
         * As in DropRelationBuffers, an unlocked precheck should be safe and
         * saves some cycles.
         */
        if (bufHdr->tag.dbOid != dbid)
            continue;
 
        LockBufHdr(bufHdr);
        if (bufHdr->tag.dbOid == dbid)
            InvalidateBuffer(bufHdr);   /* releases spinlock */
        else
            UnlockBufHdr(bufHdr);
    }
}

References fb(), GetBufferDescriptor(), i, InvalidateBuffer(), LockBufHdr(), NBuffers, and UnlockBufHdr().

Referenced by createdb_failure_callback(), dbase_redo(), dropdb(), and movedb().

◆ DropRelationBuffers()

void DropRelationBuffers	(	SMgrRelation	smgr_reln,
		ForkNumber *	forkNum,
		int	nforks,
		BlockNumber *	firstDelBlock
	)

Definition at line 4765 of file bufmgr.c.

{
    int         i;
    int         j;
    RelFileLocatorBackend rlocator;
    BlockNumber nForkBlock[MAX_FORKNUM];
    uint64      nBlocksToInvalidate = 0;
 
    rlocator = smgr_reln->smgr_rlocator;
 
    /* If it's a local relation, it's localbuf.c's problem. */
    if (RelFileLocatorBackendIsTemp(rlocator))
    {
        if (rlocator.backend == MyProcNumber)
            DropRelationLocalBuffers(rlocator.locator, forkNum, nforks,
                                     firstDelBlock);
 
        return;
    }
 
    /*
     * To remove all the pages of the specified relation forks from the buffer
     * pool, we need to scan the entire buffer pool but we can optimize it by
     * finding the buffers from BufMapping table provided we know the exact
     * size of each fork of the relation. The exact size is required to ensure
     * that we don't leave any buffer for the relation being dropped as
     * otherwise the background writer or checkpointer can lead to a PANIC
     * error while flushing buffers corresponding to files that don't exist.
     *
     * To know the exact size, we rely on the size cached for each fork by us
     * during recovery which limits the optimization to recovery and on
     * standbys but we can easily extend it once we have shared cache for
     * relation size.
     *
     * In recovery, we cache the value returned by the first lseek(SEEK_END)
     * and the future writes keeps the cached value up-to-date. See
     * smgrextend. It is possible that the value of the first lseek is smaller
     * than the actual number of existing blocks in the file due to buggy
     * Linux kernels that might not have accounted for the recent write. But
     * that should be fine because there must not be any buffers after that
     * file size.
     */
    for (i = 0; i < nforks; i++)
    {
        /* Get the number of blocks for a relation's fork */
        nForkBlock[i] = smgrnblocks_cached(smgr_reln, forkNum[i]);
 
        if (nForkBlock[i] == InvalidBlockNumber)
        {
            nBlocksToInvalidate = InvalidBlockNumber;
            break;
        }
 
        /* calculate the number of blocks to be invalidated */
        nBlocksToInvalidate += (nForkBlock[i] - firstDelBlock[i]);
    }
 
    /*
     * We apply the optimization iff the total number of blocks to invalidate
     * is below the BUF_DROP_FULL_SCAN_THRESHOLD.
     */
    if (BlockNumberIsValid(nBlocksToInvalidate) &&
        nBlocksToInvalidate < BUF_DROP_FULL_SCAN_THRESHOLD)
    {
        for (j = 0; j < nforks; j++)
            FindAndDropRelationBuffers(rlocator.locator, forkNum[j],
                                       nForkBlock[j], firstDelBlock[j]);
        return;
    }
 
    for (i = 0; i < NBuffers; i++)
    {
        BufferDesc *bufHdr = GetBufferDescriptor(i);
 
        /*
         * We can make this a tad faster by prechecking the buffer tag before
         * we attempt to lock the buffer; this saves a lot of lock
         * acquisitions in typical cases.  It should be safe because the
         * caller must have AccessExclusiveLock on the relation, or some other
         * reason to be certain that no one is loading new pages of the rel
         * into the buffer pool.  (Otherwise we might well miss such pages
         * entirely.)  Therefore, while the tag might be changing while we
         * look at it, it can't be changing *to* a value we care about, only
         * *away* from such a value.  So false negatives are impossible, and
         * false positives are safe because we'll recheck after getting the
         * buffer lock.
         *
         * We could check forkNum and blockNum as well as the rlocator, but
         * the incremental win from doing so seems small.
         */
        if (!BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator.locator))
            continue;
 
        LockBufHdr(bufHdr);
 
        for (j = 0; j < nforks; j++)
        {
            if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator.locator) &&
                BufTagGetForkNum(&bufHdr->tag) == forkNum[j] &&
                bufHdr->tag.blockNum >= firstDelBlock[j])
            {
                InvalidateBuffer(bufHdr);   /* releases spinlock */
                break;
            }
        }
        if (j >= nforks)
            UnlockBufHdr(bufHdr);
    }
}

References RelFileLocatorBackend::backend, BlockNumberIsValid(), BUF_DROP_FULL_SCAN_THRESHOLD, BufTagGetForkNum(), BufTagMatchesRelFileLocator(), DropRelationLocalBuffers(), fb(), FindAndDropRelationBuffers(), GetBufferDescriptor(), i, InvalidateBuffer(), InvalidBlockNumber, j, RelFileLocatorBackend::locator, LockBufHdr(), MAX_FORKNUM, MyProcNumber, NBuffers, RelFileLocatorBackendIsTemp, smgrnblocks_cached(), and UnlockBufHdr().

Referenced by smgrtruncate().

◆ DropRelationsAllBuffers()

void DropRelationsAllBuffers	(	SMgrRelation *	smgr_reln,
		int	nlocators
	)

Definition at line 4885 of file bufmgr.c.

{
    int         i;
    int         n = 0;
    SMgrRelation *rels;
    BlockNumber (*block)[MAX_FORKNUM + 1];
    uint64      nBlocksToInvalidate = 0;
    RelFileLocator *locators;
    bool        cached = true;
    bool        use_bsearch;
 
    if (nlocators == 0)
        return;
 
    rels = palloc_array(SMgrRelation, nlocators);   /* non-local relations */
 
    /* If it's a local relation, it's localbuf.c's problem. */
    for (i = 0; i < nlocators; i++)
    {
        if (RelFileLocatorBackendIsTemp(smgr_reln[i]->smgr_rlocator))
        {
            if (smgr_reln[i]->smgr_rlocator.backend == MyProcNumber)
                DropRelationAllLocalBuffers(smgr_reln[i]->smgr_rlocator.locator);
        }
        else
            rels[n++] = smgr_reln[i];
    }
 
    /*
     * If there are no non-local relations, then we're done. Release the
     * memory and return.
     */
    if (n == 0)
    {
        pfree(rels);
        return;
    }
 
    /*
     * This is used to remember the number of blocks for all the relations
     * forks.
     */
    block = (BlockNumber (*)[MAX_FORKNUM + 1])
        palloc(sizeof(BlockNumber) * n * (MAX_FORKNUM + 1));
 
    /*
     * We can avoid scanning the entire buffer pool if we know the exact size
     * of each of the given relation forks. See DropRelationBuffers.
     */
    for (i = 0; i < n && cached; i++)
    {
        for (int j = 0; j <= MAX_FORKNUM; j++)
        {
            /* Get the number of blocks for a relation's fork. */
            block[i][j] = smgrnblocks_cached(rels[i], j);
 
            /* We need to only consider the relation forks that exists. */
            if (block[i][j] == InvalidBlockNumber)
            {
                if (!smgrexists(rels[i], j))
                    continue;
                cached = false;
                break;
            }
 
            /* calculate the total number of blocks to be invalidated */
            nBlocksToInvalidate += block[i][j];
        }
    }
 
    /*
     * We apply the optimization iff the total number of blocks to invalidate
     * is below the BUF_DROP_FULL_SCAN_THRESHOLD.
     */
    if (cached && nBlocksToInvalidate < BUF_DROP_FULL_SCAN_THRESHOLD)
    {
        for (i = 0; i < n; i++)
        {
            for (int j = 0; j <= MAX_FORKNUM; j++)
            {
                /* ignore relation forks that doesn't exist */
                if (!BlockNumberIsValid(block[i][j]))
                    continue;
 
                /* drop all the buffers for a particular relation fork */
                FindAndDropRelationBuffers(rels[i]->smgr_rlocator.locator,
                                           j, block[i][j], 0);
            }
        }
 
        pfree(block);
        pfree(rels);
        return;
    }
 
    pfree(block);
    locators = palloc_array(RelFileLocator, n); /* non-local relations */
    for (i = 0; i < n; i++)
        locators[i] = rels[i]->smgr_rlocator.locator;
 
    /*
     * For low number of relations to drop just use a simple walk through, to
     * save the bsearch overhead. The threshold to use is rather a guess than
     * an exactly determined value, as it depends on many factors (CPU and RAM
     * speeds, amount of shared buffers etc.).
     */
    use_bsearch = n > RELS_BSEARCH_THRESHOLD;
 
    /* sort the list of rlocators if necessary */
    if (use_bsearch)
        qsort(locators, n, sizeof(RelFileLocator), rlocator_comparator);
 
    for (i = 0; i < NBuffers; i++)
    {
        RelFileLocator *rlocator = NULL;
        BufferDesc *bufHdr = GetBufferDescriptor(i);
 
        /*
         * As in DropRelationBuffers, an unlocked precheck should be safe and
         * saves some cycles.
         */
 
        if (!use_bsearch)
        {
            int         j;
 
            for (j = 0; j < n; j++)
            {
                if (BufTagMatchesRelFileLocator(&bufHdr->tag, &locators[j]))
                {
                    rlocator = &locators[j];
                    break;
                }
            }
        }
        else
        {
            RelFileLocator locator;
 
            locator = BufTagGetRelFileLocator(&bufHdr->tag);
            rlocator = bsearch(&locator,
                               locators, n, sizeof(RelFileLocator),
                               rlocator_comparator);
        }
 
        /* buffer doesn't belong to any of the given relfilelocators; skip it */
        if (rlocator == NULL)
            continue;
 
        LockBufHdr(bufHdr);
        if (BufTagMatchesRelFileLocator(&bufHdr->tag, rlocator))
            InvalidateBuffer(bufHdr);   /* releases spinlock */
        else
            UnlockBufHdr(bufHdr);
    }
 
    pfree(locators);
    pfree(rels);
}

References BlockNumberIsValid(), BUF_DROP_FULL_SCAN_THRESHOLD, BufTagGetRelFileLocator(), BufTagMatchesRelFileLocator(), DropRelationAllLocalBuffers(), fb(), FindAndDropRelationBuffers(), GetBufferDescriptor(), i, InvalidateBuffer(), InvalidBlockNumber, j, LockBufHdr(), MAX_FORKNUM, MyProcNumber, NBuffers, palloc(), palloc_array, pfree(), qsort, RelFileLocatorBackendIsTemp, RELS_BSEARCH_THRESHOLD, rlocator_comparator(), smgrexists(), smgrnblocks_cached(), and UnlockBufHdr().

Referenced by smgrdounlinkall().

◆ EvictAllUnpinnedBuffers()

void EvictAllUnpinnedBuffers	(	int32 *	buffers_evicted,
		int32 *	buffers_flushed,
		int32 *	buffers_skipped
	)

Definition at line 7982 of file bufmgr.c.

{
    *buffers_evicted = 0;
    *buffers_skipped = 0;
    *buffers_flushed = 0;
 
    for (int buf = 1; buf <= NBuffers; buf++)
    {
        BufferDesc *desc = GetBufferDescriptor(buf - 1);
        uint64      buf_state;
        bool        buffer_flushed;
 
        CHECK_FOR_INTERRUPTS();
 
        buf_state = pg_atomic_read_u64(&desc->state);
        if (!(buf_state & BM_VALID))
            continue;
 
        ResourceOwnerEnlarge(CurrentResourceOwner);
        ReservePrivateRefCountEntry();
 
        LockBufHdr(desc);
 
        if (EvictUnpinnedBufferInternal(desc, &buffer_flushed))
            (*buffers_evicted)++;
        else
            (*buffers_skipped)++;
 
        if (buffer_flushed)
            (*buffers_flushed)++;
    }
}

References BM_VALID, buf, CHECK_FOR_INTERRUPTS, CurrentResourceOwner, EvictUnpinnedBufferInternal(), fb(), GetBufferDescriptor(), LockBufHdr(), NBuffers, pg_atomic_read_u64(), ReservePrivateRefCountEntry(), ResourceOwnerEnlarge(), and BufferDesc::state.

Referenced by pg_buffercache_evict_all().

◆ EvictRelUnpinnedBuffers()

void EvictRelUnpinnedBuffers	(	Relation	rel,
		int32 *	buffers_evicted,
		int32 *	buffers_flushed,
		int32 *	buffers_skipped
	)

Definition at line 8032 of file bufmgr.c.

{
    Assert(!RelationUsesLocalBuffers(rel));
 
    *buffers_skipped = 0;
    *buffers_evicted = 0;
    *buffers_flushed = 0;
 
    for (int buf = 1; buf <= NBuffers; buf++)
    {
        BufferDesc *desc = GetBufferDescriptor(buf - 1);
        uint64      buf_state = pg_atomic_read_u64(&(desc->state));
        bool        buffer_flushed;
 
        CHECK_FOR_INTERRUPTS();
 
        /* An unlocked precheck should be safe and saves some cycles. */
        if ((buf_state & BM_VALID) == 0 ||
            !BufTagMatchesRelFileLocator(&desc->tag, &rel->rd_locator))
            continue;
 
        /* Make sure we can pin the buffer. */
        ResourceOwnerEnlarge(CurrentResourceOwner);
        ReservePrivateRefCountEntry();
 
        buf_state = LockBufHdr(desc);
 
        /* recheck, could have changed without the lock */
        if ((buf_state & BM_VALID) == 0 ||
            !BufTagMatchesRelFileLocator(&desc->tag, &rel->rd_locator))
        {
            UnlockBufHdr(desc);
            continue;
        }
 
        if (EvictUnpinnedBufferInternal(desc, &buffer_flushed))
            (*buffers_evicted)++;
        else
            (*buffers_skipped)++;
 
        if (buffer_flushed)
            (*buffers_flushed)++;
    }
}

References Assert, BM_VALID, buf, BufTagMatchesRelFileLocator(), CHECK_FOR_INTERRUPTS, CurrentResourceOwner, EvictUnpinnedBufferInternal(), fb(), GetBufferDescriptor(), LockBufHdr(), NBuffers, pg_atomic_read_u64(), RelationData::rd_locator, RelationUsesLocalBuffers, ReservePrivateRefCountEntry(), ResourceOwnerEnlarge(), BufferDesc::state, BufferDesc::tag, and UnlockBufHdr().

Referenced by evict_rel(), and pg_buffercache_evict_relation().

◆ EvictUnpinnedBuffer()

bool EvictUnpinnedBuffer	(	Buffer	buf,
		bool *	buffer_flushed
	)

Definition at line 7953 of file bufmgr.c.

{
    BufferDesc *desc;
 
    Assert(BufferIsValid(buf) && !BufferIsLocal(buf));
 
    /* Make sure we can pin the buffer. */
    ResourceOwnerEnlarge(CurrentResourceOwner);
    ReservePrivateRefCountEntry();
 
    desc = GetBufferDescriptor(buf - 1);
    LockBufHdr(desc);
 
    return EvictUnpinnedBufferInternal(desc, buffer_flushed);
}

References Assert, buf, BufferIsLocal, BufferIsValid(), CurrentResourceOwner, EvictUnpinnedBufferInternal(), fb(), GetBufferDescriptor(), LockBufHdr(), ReservePrivateRefCountEntry(), and ResourceOwnerEnlarge().

Referenced by invalidate_one_block(), modify_rel_block(), and pg_buffercache_evict().

◆ EvictUnpinnedBufferInternal()

static bool EvictUnpinnedBufferInternal	(	BufferDesc *	desc,
		bool *	buffer_flushed
	)

static

Definition at line 7891 of file bufmgr.c.

{
    uint64      buf_state;
    bool        result;
 
    *buffer_flushed = false;
 
    buf_state = pg_atomic_read_u64(&(desc->state));
    Assert(buf_state & BM_LOCKED);
 
    if ((buf_state & BM_VALID) == 0)
    {
        UnlockBufHdr(desc);
        return false;
    }
 
    /* Check that it's not pinned already. */
    if (BUF_STATE_GET_REFCOUNT(buf_state) > 0)
    {
        UnlockBufHdr(desc);
        return false;
    }
 
    PinBuffer_Locked(desc);     /* releases spinlock */
 
    /* If it was dirty, try to clean it once. */
    if (buf_state & BM_DIRTY)
    {
        FlushUnlockedBuffer(desc, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
        *buffer_flushed = true;
    }
 
    /* This will return false if it becomes dirty or someone else pins it. */
    result = InvalidateVictimBuffer(desc);
 
    UnpinBuffer(desc);
 
    return result;
}

References Assert, BM_DIRTY, BM_LOCKED, BM_VALID, BUF_STATE_GET_REFCOUNT, fb(), FlushUnlockedBuffer(), InvalidateVictimBuffer(), IOCONTEXT_NORMAL, IOOBJECT_RELATION, pg_atomic_read_u64(), PinBuffer_Locked(), result, BufferDesc::state, UnlockBufHdr(), and UnpinBuffer().

Referenced by EvictAllUnpinnedBuffers(), EvictRelUnpinnedBuffers(), and EvictUnpinnedBuffer().

◆ ExtendBufferedRel()

Buffer ExtendBufferedRel	(	BufferManagerRelation	bmr,
		ForkNumber	forkNum,
		BufferAccessStrategy	strategy,
		uint32	flags
	)

Definition at line 979 of file bufmgr.c.

{
    Buffer      buf;
    uint32      extend_by = 1;
 
    ExtendBufferedRelBy(bmr, forkNum, strategy, flags, extend_by,
                        &buf, &extend_by);
 
    return buf;
}

References buf, ExtendBufferedRelBy(), and fb().

Referenced by _bt_allocbuf(), _hash_getnewbuf(), BloomNewBuffer(), brinbuild(), brinbuildempty(), fill_seq_fork_with_data(), ginbuildempty(), GinNewBuffer(), gistbuildempty(), gistNewBuffer(), ReadBuffer_common(), revmap_physical_extend(), and SpGistNewBuffer().

◆ ExtendBufferedRelBy()

BlockNumber ExtendBufferedRelBy	(	BufferManagerRelation	bmr,
		ForkNumber	fork,
		BufferAccessStrategy	strategy,
		uint32	flags,
		uint32	extend_by,
		Buffer *	buffers,
		uint32 *	extended_by
	)

Definition at line 1011 of file bufmgr.c.

{
    Assert((bmr.rel != NULL) != (bmr.smgr != NULL));
    Assert(bmr.smgr == NULL || bmr.relpersistence != '\0');
    Assert(extend_by > 0);
 
    if (bmr.relpersistence == '\0')
        bmr.relpersistence = bmr.rel->rd_rel->relpersistence;
 
    return ExtendBufferedRelCommon(bmr, fork, strategy, flags,
                                   extend_by, InvalidBlockNumber,
                                   buffers, extended_by);
}

References Assert, ExtendBufferedRelCommon(), fb(), and InvalidBlockNumber.

Referenced by ExtendBufferedRel(), grow_rel(), and RelationAddBlocks().

◆ ExtendBufferedRelCommon()

static BlockNumber ExtendBufferedRelCommon	(	BufferManagerRelation	bmr,
		ForkNumber	fork,
		BufferAccessStrategy	strategy,
		uint32	flags,
		uint32	extend_by,
		BlockNumber	extend_upto,
		Buffer *	buffers,
		uint32 *	extended_by
	)

static

Definition at line 2742 of file bufmgr.c.

{
    BlockNumber first_block;
 
    TRACE_POSTGRESQL_BUFFER_EXTEND_START(fork,
                                         BMR_GET_SMGR(bmr)->smgr_rlocator.locator.spcOid,
                                         BMR_GET_SMGR(bmr)->smgr_rlocator.locator.dbOid,
                                         BMR_GET_SMGR(bmr)->smgr_rlocator.locator.relNumber,
                                         BMR_GET_SMGR(bmr)->smgr_rlocator.backend,
                                         extend_by);
 
    if (bmr.relpersistence == RELPERSISTENCE_TEMP)
        first_block = ExtendBufferedRelLocal(bmr, fork, flags,
                                             extend_by, extend_upto,
                                             buffers, &extend_by);
    else
        first_block = ExtendBufferedRelShared(bmr, fork, strategy, flags,
                                              extend_by, extend_upto,
                                              buffers, &extend_by);
    *extended_by = extend_by;
 
    TRACE_POSTGRESQL_BUFFER_EXTEND_DONE(fork,
                                        BMR_GET_SMGR(bmr)->smgr_rlocator.locator.spcOid,
                                        BMR_GET_SMGR(bmr)->smgr_rlocator.locator.dbOid,
                                        BMR_GET_SMGR(bmr)->smgr_rlocator.locator.relNumber,
                                        BMR_GET_SMGR(bmr)->smgr_rlocator.backend,
                                        *extended_by,
                                        first_block);
 
    return first_block;
}

References BMR_GET_SMGR, ExtendBufferedRelLocal(), ExtendBufferedRelShared(), and fb().

Referenced by ExtendBufferedRelBy(), and ExtendBufferedRelTo().

◆ ExtendBufferedRelShared()

static BlockNumber ExtendBufferedRelShared	(	BufferManagerRelation	bmr,
		ForkNumber	fork,
		BufferAccessStrategy	strategy,
		uint32	flags,
		uint32	extend_by,
		BlockNumber	extend_upto,
		Buffer *	buffers,
		uint32 *	extended_by
	)

static

Definition at line 2786 of file bufmgr.c.

{
    BlockNumber first_block;
    IOContext   io_context = IOContextForStrategy(strategy);
    instr_time  io_start;
 
    LimitAdditionalPins(&extend_by);
 
    /*
     * Acquire victim buffers for extension without holding extension lock.
     * Writing out victim buffers is the most expensive part of extending the
     * relation, particularly when doing so requires WAL flushes. Zeroing out
     * the buffers is also quite expensive, so do that before holding the
     * extension lock as well.
     *
     * These pages are pinned by us and not valid. While we hold the pin they
     * can't be acquired as victim buffers by another backend.
     */
    for (uint32 i = 0; i < extend_by; i++)
    {
        Block       buf_block;
 
        buffers[i] = GetVictimBuffer(strategy, io_context);
        buf_block = BufHdrGetBlock(GetBufferDescriptor(buffers[i] - 1));
 
        /* new buffers are zero-filled */
        MemSet(buf_block, 0, BLCKSZ);
    }
 
    /*
     * Lock relation against concurrent extensions, unless requested not to.
     *
     * We use the same extension lock for all forks. That's unnecessarily
     * restrictive, but currently extensions for forks don't happen often
     * enough to make it worth locking more granularly.
     *
     * Note that another backend might have extended the relation by the time
     * we get the lock.
     */
    if (!(flags & EB_SKIP_EXTENSION_LOCK))
        LockRelationForExtension(bmr.rel, ExclusiveLock);
 
    /*
     * If requested, invalidate size cache, so that smgrnblocks asks the
     * kernel.
     */
    if (flags & EB_CLEAR_SIZE_CACHE)
        BMR_GET_SMGR(bmr)->smgr_cached_nblocks[fork] = InvalidBlockNumber;
 
    first_block = smgrnblocks(BMR_GET_SMGR(bmr), fork);
 
    /*
     * Now that we have the accurate relation size, check if the caller wants
     * us to extend to only up to a specific size. If there were concurrent
     * extensions, we might have acquired too many buffers and need to release
     * them.
     */
    if (extend_upto != InvalidBlockNumber)
    {
        uint32      orig_extend_by = extend_by;
 
        if (first_block > extend_upto)
            extend_by = 0;
        else if ((uint64) first_block + extend_by > extend_upto)
            extend_by = extend_upto - first_block;
 
        for (uint32 i = extend_by; i < orig_extend_by; i++)
        {
            BufferDesc *buf_hdr = GetBufferDescriptor(buffers[i] - 1);
 
            UnpinBuffer(buf_hdr);
        }
 
        if (extend_by == 0)
        {
            if (!(flags & EB_SKIP_EXTENSION_LOCK))
                UnlockRelationForExtension(bmr.rel, ExclusiveLock);
            *extended_by = extend_by;
            return first_block;
        }
    }
 
    /* Fail if relation is already at maximum possible length */
    if ((uint64) first_block + extend_by >= MaxBlockNumber)
        ereport(ERROR,
                (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
                 errmsg("cannot extend relation %s beyond %u blocks",
                        relpath(BMR_GET_SMGR(bmr)->smgr_rlocator, fork).str,
                        MaxBlockNumber)));
 
    /*
     * Insert buffers into buffer table, mark as IO_IN_PROGRESS.
     *
     * This needs to happen before we extend the relation, because as soon as
     * we do, other backends can start to read in those pages.
     */
    for (uint32 i = 0; i < extend_by; i++)
    {
        Buffer      victim_buf = buffers[i];
        BufferDesc *victim_buf_hdr = GetBufferDescriptor(victim_buf - 1);
        BufferTag   tag;
        uint32      hash;
        LWLock     *partition_lock;
        int         existing_id;
 
        /* in case we need to pin an existing buffer below */
        ResourceOwnerEnlarge(CurrentResourceOwner);
        ReservePrivateRefCountEntry();
 
        InitBufferTag(&tag, &BMR_GET_SMGR(bmr)->smgr_rlocator.locator, fork,
                      first_block + i);
        hash = BufTableHashCode(&tag);
        partition_lock = BufMappingPartitionLock(hash);
 
        LWLockAcquire(partition_lock, LW_EXCLUSIVE);
 
        existing_id = BufTableInsert(&tag, hash, victim_buf_hdr->buf_id);
 
        /*
         * We get here only in the corner case where we are trying to extend
         * the relation but we found a pre-existing buffer. This can happen
         * because a prior attempt at extending the relation failed, and
         * because mdread doesn't complain about reads beyond EOF (when
         * zero_damaged_pages is ON) and so a previous attempt to read a block
         * beyond EOF could have left a "valid" zero-filled buffer.
         *
         * This has also been observed when relation was overwritten by
         * external process. Since the legitimate cases should always have
         * left a zero-filled buffer, complain if not PageIsNew.
         */
        if (existing_id >= 0)
        {
            BufferDesc *existing_hdr = GetBufferDescriptor(existing_id);
            Block       buf_block;
            bool        valid;
 
            /*
             * Pin the existing buffer before releasing the partition lock,
             * preventing it from being evicted.
             */
            valid = PinBuffer(existing_hdr, strategy, false);
 
            LWLockRelease(partition_lock);
            UnpinBuffer(victim_buf_hdr);
 
            buffers[i] = BufferDescriptorGetBuffer(existing_hdr);
            buf_block = BufHdrGetBlock(existing_hdr);
 
            if (valid && !PageIsNew((Page) buf_block))
                ereport(ERROR,
                        (errmsg("unexpected data beyond EOF in block %u of relation \"%s\"",
                                existing_hdr->tag.blockNum,
                                relpath(BMR_GET_SMGR(bmr)->smgr_rlocator, fork).str)));
 
            /*
             * We *must* do smgr[zero]extend before succeeding, else the page
             * will not be reserved by the kernel, and the next P_NEW call
             * will decide to return the same page.  Clear the BM_VALID bit,
             * do StartSharedBufferIO() and proceed.
             *
             * Loop to handle the very small possibility that someone re-sets
             * BM_VALID between our clearing it and StartSharedBufferIO
             * inspecting it.
             */
            while (true)
            {
                StartBufferIOResult sbres;
 
                pg_atomic_fetch_and_u64(&existing_hdr->state, ~BM_VALID);
 
                sbres = StartSharedBufferIO(existing_hdr, true, true, NULL);
 
                if (sbres != BUFFER_IO_ALREADY_DONE)
                    break;
            }
        }
        else
        {
            uint64      buf_state;
            uint64      set_bits = 0;
 
            buf_state = LockBufHdr(victim_buf_hdr);
 
            /* some sanity checks while we hold the buffer header lock */
            Assert(!(buf_state & (BM_VALID | BM_TAG_VALID | BM_DIRTY)));
            Assert(BUF_STATE_GET_REFCOUNT(buf_state) == 1);
 
            victim_buf_hdr->tag = tag;
 
            set_bits |= BM_TAG_VALID | BUF_USAGECOUNT_ONE;
            if (bmr.relpersistence == RELPERSISTENCE_PERMANENT || fork == INIT_FORKNUM)
                set_bits |= BM_PERMANENT;
 
            UnlockBufHdrExt(victim_buf_hdr, buf_state,
                            set_bits, 0,
                            0);
 
            LWLockRelease(partition_lock);
 
            /* XXX: could combine the locked operations in it with the above */
            StartSharedBufferIO(victim_buf_hdr, true, true, NULL);
        }
    }
 
    io_start = pgstat_prepare_io_time(track_io_timing);
 
    /*
     * Note: if smgrzeroextend fails, we will end up with buffers that are
     * allocated but not marked BM_VALID.  The next relation extension will
     * still select the same block number (because the relation didn't get any
     * longer on disk) and so future attempts to extend the relation will find
     * the same buffers (if they have not been recycled) but come right back
     * here to try smgrzeroextend again.
     *
     * We don't need to set checksum for all-zero pages.
     */
    smgrzeroextend(BMR_GET_SMGR(bmr), fork, first_block, extend_by, false);
 
    /*
     * Release the file-extension lock; it's now OK for someone else to extend
     * the relation some more.
     *
     * We remove IO_IN_PROGRESS after this, as waking up waiting backends can
     * take noticeable time.
     */
    if (!(flags & EB_SKIP_EXTENSION_LOCK))
        UnlockRelationForExtension(bmr.rel, ExclusiveLock);
 
    pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, IOOP_EXTEND,
                            io_start, 1, extend_by * BLCKSZ);
 
    /* Set BM_VALID, terminate IO, and wake up any waiters */
    for (uint32 i = 0; i < extend_by; i++)
    {
        Buffer      buf = buffers[i];
        BufferDesc *buf_hdr = GetBufferDescriptor(buf - 1);
        bool        lock = false;
 
        if (flags & EB_LOCK_FIRST && i == 0)
            lock = true;
        else if (flags & EB_LOCK_TARGET)
        {
            Assert(extend_upto != InvalidBlockNumber);
            if (first_block + i + 1 == extend_upto)
                lock = true;
        }
 
        if (lock)
            LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
 
        TerminateBufferIO(buf_hdr, false, BM_VALID, true, false);
    }
 
    pgBufferUsage.shared_blks_written += extend_by;
 
    *extended_by = extend_by;
 
    return first_block;
}

References Assert, BM_DIRTY, BM_PERMANENT, BM_TAG_VALID, BM_VALID, BMR_GET_SMGR, buf, BUF_STATE_GET_REFCOUNT, BUF_USAGECOUNT_ONE, BUFFER_IO_ALREADY_DONE, BUFFER_LOCK_EXCLUSIVE, BufferDescriptorGetBuffer(), BufHdrGetBlock, BufMappingPartitionLock(), BufTableHashCode(), BufTableInsert(), CurrentResourceOwner, EB_CLEAR_SIZE_CACHE, EB_LOCK_FIRST, EB_LOCK_TARGET, EB_SKIP_EXTENSION_LOCK, ereport, errcode(), errmsg, ERROR, ExclusiveLock, fb(), GetBufferDescriptor(), GetVictimBuffer(), hash(), i, INIT_FORKNUM, InitBufferTag(), InvalidBlockNumber, IOContextForStrategy(), IOOBJECT_RELATION, IOOP_EXTEND, LimitAdditionalPins(), LockBuffer(), LockBufHdr(), LockRelationForExtension(), LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), MaxBlockNumber, MemSet, PageIsNew(), pg_atomic_fetch_and_u64(), pgBufferUsage, pgstat_count_io_op_time(), pgstat_prepare_io_time(), PinBuffer(), relpath, ReservePrivateRefCountEntry(), ResourceOwnerEnlarge(), BufferUsage::shared_blks_written, smgrnblocks(), smgrzeroextend(), StartSharedBufferIO(), str, TerminateBufferIO(), track_io_timing, UnlockBufHdrExt(), UnlockRelationForExtension(), and UnpinBuffer().

Referenced by ExtendBufferedRelCommon().

◆ ExtendBufferedRelTo()

Buffer ExtendBufferedRelTo	(	BufferManagerRelation	bmr,
		ForkNumber	fork,
		BufferAccessStrategy	strategy,
		uint32	flags,
		BlockNumber	extend_to,
		ReadBufferMode	mode
	)

Definition at line 1040 of file bufmgr.c.

{
    BlockNumber current_size;
    uint32      extended_by = 0;
    Buffer      buffer = InvalidBuffer;
    Buffer      buffers[64];
 
    Assert((bmr.rel != NULL) != (bmr.smgr != NULL));
    Assert(bmr.smgr == NULL || bmr.relpersistence != '\0');
    Assert(extend_to != InvalidBlockNumber && extend_to > 0);
 
    if (bmr.relpersistence == '\0')
        bmr.relpersistence = bmr.rel->rd_rel->relpersistence;
 
    /*
     * If desired, create the file if it doesn't exist.  If
     * smgr_cached_nblocks[fork] is positive then it must exist, no need for
     * an smgrexists call.
     */
    if ((flags & EB_CREATE_FORK_IF_NEEDED) &&
        (BMR_GET_SMGR(bmr)->smgr_cached_nblocks[fork] == 0 ||
         BMR_GET_SMGR(bmr)->smgr_cached_nblocks[fork] == InvalidBlockNumber) &&
        !smgrexists(BMR_GET_SMGR(bmr), fork))
    {
        LockRelationForExtension(bmr.rel, ExclusiveLock);
 
        /* recheck, fork might have been created concurrently */
        if (!smgrexists(BMR_GET_SMGR(bmr), fork))
            smgrcreate(BMR_GET_SMGR(bmr), fork, flags & EB_PERFORMING_RECOVERY);
 
        UnlockRelationForExtension(bmr.rel, ExclusiveLock);
    }
 
    /*
     * If requested, invalidate size cache, so that smgrnblocks asks the
     * kernel.
     */
    if (flags & EB_CLEAR_SIZE_CACHE)
        BMR_GET_SMGR(bmr)->smgr_cached_nblocks[fork] = InvalidBlockNumber;
 
    /*
     * Estimate how many pages we'll need to extend by. This avoids acquiring
     * unnecessarily many victim buffers.
     */
    current_size = smgrnblocks(BMR_GET_SMGR(bmr), fork);
 
    /*
     * Since no-one else can be looking at the page contents yet, there is no
     * difference between an exclusive lock and a cleanup-strength lock. Note
     * that we pass the original mode to ReadBuffer_common() below, when
     * falling back to reading the buffer to a concurrent relation extension.
     */
    if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK)
        flags |= EB_LOCK_TARGET;
 
    while (current_size < extend_to)
    {
        uint32      num_pages = lengthof(buffers);
        BlockNumber first_block;
 
        if ((uint64) current_size + num_pages > extend_to)
            num_pages = extend_to - current_size;
 
        first_block = ExtendBufferedRelCommon(bmr, fork, strategy, flags,
                                              num_pages, extend_to,
                                              buffers, &extended_by);
 
        current_size = first_block + extended_by;
        Assert(num_pages != 0 || current_size >= extend_to);
 
        for (uint32 i = 0; i < extended_by; i++)
        {
            if (first_block + i != extend_to - 1)
                ReleaseBuffer(buffers[i]);
            else
                buffer = buffers[i];
        }
    }
 
    /*
     * It's possible that another backend concurrently extended the relation.
     * In that case read the buffer.
     *
     * XXX: Should we control this via a flag?
     */
    if (buffer == InvalidBuffer)
    {
        Assert(extended_by == 0);
        buffer = ReadBuffer_common(bmr.rel, BMR_GET_SMGR(bmr), bmr.relpersistence,
                                   fork, extend_to - 1, mode, strategy);
    }
 
    return buffer;
}

References Assert, BMR_GET_SMGR, PrivateRefCountEntry::buffer, current_size, EB_CLEAR_SIZE_CACHE, EB_CREATE_FORK_IF_NEEDED, EB_LOCK_TARGET, EB_PERFORMING_RECOVERY, ExclusiveLock, ExtendBufferedRelCommon(), fb(), i, InvalidBlockNumber, InvalidBuffer, lengthof, LockRelationForExtension(), mode, RBM_ZERO_AND_CLEANUP_LOCK, RBM_ZERO_AND_LOCK, ReadBuffer_common(), ReleaseBuffer(), smgrcreate(), smgrexists(), smgrnblocks(), and UnlockRelationForExtension().

Referenced by fsm_extend(), vm_extend(), and XLogReadBufferExtended().

◆ FindAndDropRelationBuffers()

static void FindAndDropRelationBuffers	(	RelFileLocator	rlocator,
		ForkNumber	forkNum,
		BlockNumber	nForkBlock,
		BlockNumber	firstDelBlock
	)

static

Definition at line 5055 of file bufmgr.c.

{
    BlockNumber curBlock;
 
    for (curBlock = firstDelBlock; curBlock < nForkBlock; curBlock++)
    {
        uint32      bufHash;    /* hash value for tag */
        BufferTag   bufTag;     /* identity of requested block */
        LWLock     *bufPartitionLock;   /* buffer partition lock for it */
        int         buf_id;
        BufferDesc *bufHdr;
 
        /* create a tag so we can lookup the buffer */
        InitBufferTag(&bufTag, &rlocator, forkNum, curBlock);
 
        /* determine its hash code and partition lock ID */
        bufHash = BufTableHashCode(&bufTag);
        bufPartitionLock = BufMappingPartitionLock(bufHash);
 
        /* Check that it is in the buffer pool. If not, do nothing. */
        LWLockAcquire(bufPartitionLock, LW_SHARED);
        buf_id = BufTableLookup(&bufTag, bufHash);
        LWLockRelease(bufPartitionLock);
 
        if (buf_id < 0)
            continue;
 
        bufHdr = GetBufferDescriptor(buf_id);
 
        /*
         * We need to lock the buffer header and recheck if the buffer is
         * still associated with the same block because the buffer could be
         * evicted by some other backend loading blocks for a different
         * relation after we release lock on the BufMapping table.
         */
        LockBufHdr(bufHdr);
 
        if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator) &&
            BufTagGetForkNum(&bufHdr->tag) == forkNum &&
            bufHdr->tag.blockNum >= firstDelBlock)
            InvalidateBuffer(bufHdr);   /* releases spinlock */
        else
            UnlockBufHdr(bufHdr);
    }
}

References BufMappingPartitionLock(), BufTableHashCode(), BufTableLookup(), BufTagGetForkNum(), BufTagMatchesRelFileLocator(), fb(), GetBufferDescriptor(), InitBufferTag(), InvalidateBuffer(), LockBufHdr(), LW_SHARED, LWLockAcquire(), LWLockRelease(), and UnlockBufHdr().

Referenced by DropRelationBuffers(), and DropRelationsAllBuffers().

◆ FlushBuffer()

static void FlushBuffer	(	BufferDesc *	buf,
		SMgrRelation	reln,
		IOObject	io_object,
		IOContext	io_context
	)

static

Definition at line 4503 of file bufmgr.c.

{
    XLogRecPtr  recptr;
    ErrorContextCallback errcallback;
    instr_time  io_start;
    Block       bufBlock;
 
    Assert(BufferLockHeldByMeInMode(buf, BUFFER_LOCK_EXCLUSIVE) ||
           BufferLockHeldByMeInMode(buf, BUFFER_LOCK_SHARE_EXCLUSIVE));
 
    /*
     * Try to start an I/O operation.  If StartBufferIO returns false, then
     * someone else flushed the buffer before we could, so we need not do
     * anything.
     */
    if (StartSharedBufferIO(buf, false, true, NULL) == BUFFER_IO_ALREADY_DONE)
        return;
 
    /* Setup error traceback support for ereport() */
    errcallback.callback = shared_buffer_write_error_callback;
    errcallback.arg = buf;
    errcallback.previous = error_context_stack;
    error_context_stack = &errcallback;
 
    /* Find smgr relation for buffer */
    if (reln == NULL)
        reln = smgropen(BufTagGetRelFileLocator(&buf->tag), INVALID_PROC_NUMBER);
 
    TRACE_POSTGRESQL_BUFFER_FLUSH_START(BufTagGetForkNum(&buf->tag),
                                        buf->tag.blockNum,
                                        reln->smgr_rlocator.locator.spcOid,
                                        reln->smgr_rlocator.locator.dbOid,
                                        reln->smgr_rlocator.locator.relNumber);
 
    /*
     * As we hold at least a share-exclusive lock on the buffer, the LSN
     * cannot change during the flush (and thus can't be torn).
     */
    recptr = BufferGetLSN(buf);
 
    /*
     * Force XLOG flush up to buffer's LSN.  This implements the basic WAL
     * rule that log updates must hit disk before any of the data-file changes
     * they describe do.
     *
     * However, this rule does not apply to unlogged relations, which will be
     * lost after a crash anyway.  Most unlogged relation pages do not bear
     * LSNs since we never emit WAL records for them, and therefore flushing
     * up through the buffer LSN would be useless, but harmless.  However,
     * some index AMs use LSNs internally to detect concurrent page
     * modifications, and therefore unlogged index pages bear "fake" LSNs
     * generated by XLogGetFakeLSN.  It is unlikely but possible that the fake
     * LSN counter could advance past the WAL insertion point; and if it did
     * happen, attempting to flush WAL through that location would fail, with
     * disastrous system-wide consequences.  To make sure that can't happen,
     * skip the flush if the buffer isn't permanent.
     */
    if (pg_atomic_read_u64(&buf->state) & BM_PERMANENT)
        XLogFlush(recptr);
 
    /*
     * Now it's safe to write the buffer to disk. Note that no one else should
     * have been able to write it, while we were busy with log flushing,
     * because we got the exclusive right to perform I/O by setting the
     * BM_IO_IN_PROGRESS bit.
     */
    bufBlock = BufHdrGetBlock(buf);
 
    /* Update page checksum if desired. */
    PageSetChecksum((Page) bufBlock, buf->tag.blockNum);
 
    io_start = pgstat_prepare_io_time(track_io_timing);
 
    smgrwrite(reln,
              BufTagGetForkNum(&buf->tag),
              buf->tag.blockNum,
              bufBlock,
              false);
 
    /*
     * When a strategy is in use, only flushes of dirty buffers already in the
     * strategy ring are counted as strategy writes (IOCONTEXT
     * [BULKREAD|BULKWRITE|VACUUM] IOOP_WRITE) for the purpose of IO
     * statistics tracking.
     *
     * If a shared buffer initially added to the ring must be flushed before
     * being used, this is counted as an IOCONTEXT_NORMAL IOOP_WRITE.
     *
     * If a shared buffer which was added to the ring later because the
     * current strategy buffer is pinned or in use or because all strategy
     * buffers were dirty and rejected (for BAS_BULKREAD operations only)
     * requires flushing, this is counted as an IOCONTEXT_NORMAL IOOP_WRITE
     * (from_ring will be false).
     *
     * When a strategy is not in use, the write can only be a "regular" write
     * of a dirty shared buffer (IOCONTEXT_NORMAL IOOP_WRITE).
     */
    pgstat_count_io_op_time(IOOBJECT_RELATION, io_context,
                            IOOP_WRITE, io_start, 1, BLCKSZ);
 
    pgBufferUsage.shared_blks_written++;
 
    /*
     * Mark the buffer as clean and end the BM_IO_IN_PROGRESS state.
     */
    TerminateBufferIO(buf, true, 0, true, false);
 
    TRACE_POSTGRESQL_BUFFER_FLUSH_DONE(BufTagGetForkNum(&buf->tag),
                                       buf->tag.blockNum,
                                       reln->smgr_rlocator.locator.spcOid,
                                       reln->smgr_rlocator.locator.dbOid,
                                       reln->smgr_rlocator.locator.relNumber);
 
    /* Pop the error context stack */
    error_context_stack = errcallback.previous;
}

References ErrorContextCallback::arg, Assert, BM_PERMANENT, buf, BUFFER_IO_ALREADY_DONE, BUFFER_LOCK_EXCLUSIVE, BUFFER_LOCK_SHARE_EXCLUSIVE, BufferGetLSN, BufferLockHeldByMeInMode(), BufHdrGetBlock, BufTagGetForkNum(), BufTagGetRelFileLocator(), ErrorContextCallback::callback, error_context_stack, fb(), INVALID_PROC_NUMBER, IOOBJECT_RELATION, IOOP_WRITE, PageSetChecksum(), pg_atomic_read_u64(), pgBufferUsage, pgstat_count_io_op_time(), pgstat_prepare_io_time(), ErrorContextCallback::previous, BufferUsage::shared_blks_written, shared_buffer_write_error_callback(), smgropen(), smgrwrite(), StartSharedBufferIO(), TerminateBufferIO(), track_io_timing, and XLogFlush().

Referenced by FlushOneBuffer(), FlushUnlockedBuffer(), and GetVictimBuffer().

◆ FlushDatabaseBuffers()

void FlushDatabaseBuffers ( Oid dbid )

Definition at line 5526 of file bufmgr.c.

{
    int         i;
    BufferDesc *bufHdr;
 
    for (i = 0; i < NBuffers; i++)
    {
        uint64      buf_state;
 
        bufHdr = GetBufferDescriptor(i);
 
        /*
         * As in DropRelationBuffers, an unlocked precheck should be safe and
         * saves some cycles.
         */
        if (bufHdr->tag.dbOid != dbid)
            continue;
 
        /* Make sure we can handle the pin */
        ReservePrivateRefCountEntry();
        ResourceOwnerEnlarge(CurrentResourceOwner);
 
        buf_state = LockBufHdr(bufHdr);
        if (bufHdr->tag.dbOid == dbid &&
            (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
        {
            PinBuffer_Locked(bufHdr);
            FlushUnlockedBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
            UnpinBuffer(bufHdr);
        }
        else
            UnlockBufHdr(bufHdr);
    }
}

References BM_DIRTY, BM_VALID, CurrentResourceOwner, fb(), FlushUnlockedBuffer(), GetBufferDescriptor(), i, IOCONTEXT_NORMAL, IOOBJECT_RELATION, LockBufHdr(), NBuffers, PinBuffer_Locked(), ReservePrivateRefCountEntry(), ResourceOwnerEnlarge(), UnlockBufHdr(), and UnpinBuffer().

Referenced by dbase_redo().

◆ FlushOneBuffer()

void FlushOneBuffer ( Buffer buffer )

Definition at line 5566 of file bufmgr.c.

{
    BufferDesc *bufHdr;
 
    /* currently not needed, but no fundamental reason not to support */
    Assert(!BufferIsLocal(buffer));
 
    Assert(BufferIsPinned(buffer));
 
    bufHdr = GetBufferDescriptor(buffer - 1);
 
    Assert(BufferIsLockedByMe(buffer));
 
    FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
}

References Assert, PrivateRefCountEntry::buffer, BufferIsLocal, BufferIsLockedByMe(), BufferIsPinned, fb(), FlushBuffer(), GetBufferDescriptor(), IOCONTEXT_NORMAL, and IOOBJECT_RELATION.

Referenced by hash_xlog_init_bitmap_page(), hash_xlog_init_meta_page(), invalidate_one_block(), and XLogReadBufferForRedoExtended().

◆ FlushRelationBuffers()

void FlushRelationBuffers ( Relation rel )

Definition at line 5162 of file bufmgr.c.

{
    int         i;
    BufferDesc *bufHdr;
    SMgrRelation srel = RelationGetSmgr(rel);
 
    if (RelationUsesLocalBuffers(rel))
    {
        for (i = 0; i < NLocBuffer; i++)
        {
            uint64      buf_state;
 
            bufHdr = GetLocalBufferDescriptor(i);
            if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator) &&
                ((buf_state = pg_atomic_read_u64(&bufHdr->state)) &
                 (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
            {
                ErrorContextCallback errcallback;
 
                /* Setup error traceback support for ereport() */
                errcallback.callback = local_buffer_write_error_callback;
                errcallback.arg = bufHdr;
                errcallback.previous = error_context_stack;
                error_context_stack = &errcallback;
 
                /* Make sure we can handle the pin */
                ReservePrivateRefCountEntry();
                ResourceOwnerEnlarge(CurrentResourceOwner);
 
                /*
                 * Pin/unpin mostly to make valgrind work, but it also seems
                 * like the right thing to do.
                 */
                PinLocalBuffer(bufHdr, false);
 
 
                FlushLocalBuffer(bufHdr, srel);
 
                UnpinLocalBuffer(BufferDescriptorGetBuffer(bufHdr));
 
                /* Pop the error context stack */
                error_context_stack = errcallback.previous;
            }
        }
 
        return;
    }
 
    for (i = 0; i < NBuffers; i++)
    {
        uint64      buf_state;
 
        bufHdr = GetBufferDescriptor(i);
 
        /*
         * As in DropRelationBuffers, an unlocked precheck should be safe and
         * saves some cycles.
         */
        if (!BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator))
            continue;
 
        /* Make sure we can handle the pin */
        ReservePrivateRefCountEntry();
        ResourceOwnerEnlarge(CurrentResourceOwner);
 
        buf_state = LockBufHdr(bufHdr);
        if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator) &&
            (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
        {
            PinBuffer_Locked(bufHdr);
            FlushUnlockedBuffer(bufHdr, srel, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
            UnpinBuffer(bufHdr);
        }
        else
            UnlockBufHdr(bufHdr);
    }
}

References ErrorContextCallback::arg, BM_DIRTY, BM_VALID, BufferDescriptorGetBuffer(), BufTagMatchesRelFileLocator(), ErrorContextCallback::callback, CurrentResourceOwner, error_context_stack, fb(), FlushLocalBuffer(), FlushUnlockedBuffer(), GetBufferDescriptor(), GetLocalBufferDescriptor(), i, IOCONTEXT_NORMAL, IOOBJECT_RELATION, local_buffer_write_error_callback(), LockBufHdr(), NBuffers, NLocBuffer, pg_atomic_read_u64(), PinBuffer_Locked(), PinLocalBuffer(), ErrorContextCallback::previous, RelationData::rd_locator, RelationGetSmgr(), RelationUsesLocalBuffers, ReservePrivateRefCountEntry(), ResourceOwnerEnlarge(), UnlockBufHdr(), UnpinBuffer(), and UnpinLocalBuffer().

Referenced by fill_seq_with_data(), heapam_relation_copy_data(), and index_copy_data().

◆ FlushRelationsAllBuffers()

void FlushRelationsAllBuffers	(	SMgrRelation *	smgrs,
		int	nrels
	)

Definition at line 5250 of file bufmgr.c.

{
    int         i;
    SMgrSortArray *srels;
    bool        use_bsearch;
 
    if (nrels == 0)
        return;
 
    /* fill-in array for qsort */
    srels = palloc_array(SMgrSortArray, nrels);
 
    for (i = 0; i < nrels; i++)
    {
        Assert(!RelFileLocatorBackendIsTemp(smgrs[i]->smgr_rlocator));
 
        srels[i].rlocator = smgrs[i]->smgr_rlocator.locator;
        srels[i].srel = smgrs[i];
    }
 
    /*
     * Save the bsearch overhead for low number of relations to sync. See
     * DropRelationsAllBuffers for details.
     */
    use_bsearch = nrels > RELS_BSEARCH_THRESHOLD;
 
    /* sort the list of SMgrRelations if necessary */
    if (use_bsearch)
        qsort(srels, nrels, sizeof(SMgrSortArray), rlocator_comparator);
 
    for (i = 0; i < NBuffers; i++)
    {
        SMgrSortArray *srelent = NULL;
        BufferDesc *bufHdr = GetBufferDescriptor(i);
        uint64      buf_state;
 
        /*
         * As in DropRelationBuffers, an unlocked precheck should be safe and
         * saves some cycles.
         */
 
        if (!use_bsearch)
        {
            int         j;
 
            for (j = 0; j < nrels; j++)
            {
                if (BufTagMatchesRelFileLocator(&bufHdr->tag, &srels[j].rlocator))
                {
                    srelent = &srels[j];
                    break;
                }
            }
        }
        else
        {
            RelFileLocator rlocator;
 
            rlocator = BufTagGetRelFileLocator(&bufHdr->tag);
            srelent = bsearch(&rlocator,
                              srels, nrels, sizeof(SMgrSortArray),
                              rlocator_comparator);
        }
 
        /* buffer doesn't belong to any of the given relfilelocators; skip it */
        if (srelent == NULL)
            continue;
 
        /* Make sure we can handle the pin */
        ReservePrivateRefCountEntry();
        ResourceOwnerEnlarge(CurrentResourceOwner);
 
        buf_state = LockBufHdr(bufHdr);
        if (BufTagMatchesRelFileLocator(&bufHdr->tag, &srelent->rlocator) &&
            (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
        {
            PinBuffer_Locked(bufHdr);
            FlushUnlockedBuffer(bufHdr, srelent->srel, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
            UnpinBuffer(bufHdr);
        }
        else
            UnlockBufHdr(bufHdr);
    }
 
    pfree(srels);
}

References Assert, BM_DIRTY, BM_VALID, BufTagGetRelFileLocator(), BufTagMatchesRelFileLocator(), CurrentResourceOwner, fb(), FlushUnlockedBuffer(), GetBufferDescriptor(), i, IOCONTEXT_NORMAL, IOOBJECT_RELATION, j, LockBufHdr(), NBuffers, palloc_array, pfree(), PinBuffer_Locked(), qsort, RelFileLocatorBackendIsTemp, RELS_BSEARCH_THRESHOLD, ReservePrivateRefCountEntry(), ResourceOwnerEnlarge(), rlocator_comparator(), UnlockBufHdr(), and UnpinBuffer().

Referenced by smgrdosyncall().

◆ FlushUnlockedBuffer()

static void FlushUnlockedBuffer	(	BufferDesc *	buf,
		SMgrRelation	reln,
		IOObject	io_object,
		IOContext	io_context
	)

static

Definition at line 4626 of file bufmgr.c.

{
    Buffer      buffer = BufferDescriptorGetBuffer(buf);
 
    BufferLockAcquire(buffer, buf, BUFFER_LOCK_SHARE_EXCLUSIVE);
    FlushBuffer(buf, reln, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
    BufferLockUnlock(buffer, buf);
}

References buf, PrivateRefCountEntry::buffer, BUFFER_LOCK_SHARE_EXCLUSIVE, BufferDescriptorGetBuffer(), BufferLockAcquire(), BufferLockUnlock(), fb(), FlushBuffer(), IOCONTEXT_NORMAL, and IOOBJECT_RELATION.

Referenced by EvictUnpinnedBufferInternal(), FlushDatabaseBuffers(), FlushRelationBuffers(), FlushRelationsAllBuffers(), and SyncOneBuffer().

◆ ForgetPrivateRefCountEntry()

static void ForgetPrivateRefCountEntry ( PrivateRefCountEntry * ref )

static

Definition at line 565 of file bufmgr.c.

{
    Assert(ref->data.refcount == 0);
    Assert(ref->data.lockmode == BUFFER_LOCK_UNLOCK);
 
    if (ref >= &PrivateRefCountArray[0] &&
        ref < &PrivateRefCountArray[REFCOUNT_ARRAY_ENTRIES])
    {
        ref->buffer = InvalidBuffer;
        PrivateRefCountArrayKeys[ref - PrivateRefCountArray] = InvalidBuffer;
 
 
        /*
         * Mark the just used entry as reserved - in many scenarios that
         * allows us to avoid ever having to search the array/hash for free
         * entries.
         */
        ReservedRefCountSlot = ref - PrivateRefCountArray;
    }
    else
    {
        refcount_delete_item(PrivateRefCountHash, ref);
        Assert(PrivateRefCountOverflowed > 0);
        PrivateRefCountOverflowed--;
    }
}

References Assert, BUFFER_LOCK_UNLOCK, fb(), InvalidBuffer, PrivateRefCountArray, PrivateRefCountArrayKeys, PrivateRefCountHash, PrivateRefCountOverflowed, REFCOUNT_ARRAY_ENTRIES, and ReservedRefCountSlot.

Referenced by UnlockReleaseBuffer(), and UnpinBufferNoOwner().

◆ GetAdditionalPinLimit()

uint32 GetAdditionalPinLimit ( void )

Definition at line 2698 of file bufmgr.c.

{
    uint32      estimated_pins_held;
 
    /*
     * We get the number of "overflowed" pins for free, but don't know the
     * number of pins in PrivateRefCountArray.  The cost of calculating that
     * exactly doesn't seem worth it, so just assume the max.
     */
    estimated_pins_held = PrivateRefCountOverflowed + REFCOUNT_ARRAY_ENTRIES;
 
    /* Is this backend already holding more than its fair share? */
    if (estimated_pins_held > MaxProportionalPins)
        return 0;
 
    return MaxProportionalPins - estimated_pins_held;
}

References fb(), MaxProportionalPins, PrivateRefCountOverflowed, and REFCOUNT_ARRAY_ENTRIES.

Referenced by LimitAdditionalPins(), and read_stream_start_pending_read().

◆ GetPinLimit()

uint32 GetPinLimit ( void )

Definition at line 2686 of file bufmgr.c.

{
    return MaxProportionalPins;
}

References MaxProportionalPins.

Referenced by GetAccessStrategy(), and read_stream_begin_impl().

◆ GetPrivateRefCount()

static int32 GetPrivateRefCount ( Buffer buffer )

inlinestatic

Definition at line 542 of file bufmgr.c.

{
    PrivateRefCountEntry *ref;
 
    Assert(BufferIsValid(buffer));
    Assert(!BufferIsLocal(buffer));
 
    /*
     * Not moving the entry - that's ok for the current users, but we might
     * want to change this one day.
     */
    ref = GetPrivateRefCountEntry(buffer, false);
 
    if (ref == NULL)
        return 0;
    return ref->data.refcount;
}

References Assert, PrivateRefCountEntry::buffer, BufferIsLocal, BufferIsValid(), fb(), and GetPrivateRefCountEntry().

Referenced by CheckBufferIsPinnedOnce(), ConditionalLockBufferForCleanup(), DebugPrintBufferRefcount(), HoldingBufferPinThatDelaysRecovery(), InvalidateBuffer(), InvalidateVictimBuffer(), IsBufferCleanupOK(), and MarkSharedBufferDirtyHint().

◆ GetPrivateRefCountEntry()

static PrivateRefCountEntry * GetPrivateRefCountEntry	(	Buffer	buffer,
		bool	do_move
	)

inlinestatic

Definition at line 507 of file bufmgr.c.

{
    Assert(BufferIsValid(buffer));
    Assert(!BufferIsLocal(buffer));
 
    /*
     * It's very common to look up the same buffer repeatedly. To make that
     * fast, we have a one-entry cache.
     *
     * In contrast to the loop in GetPrivateRefCountEntrySlow(), here it
     * faster to check PrivateRefCountArray[].buffer, as in the case of a hit
     * fewer addresses are computed and fewer cachelines are accessed. Whereas
     * in GetPrivateRefCountEntrySlow()'s case, checking
     * PrivateRefCountArrayKeys saves a lot of memory accesses.
     */
    if (likely(PrivateRefCountEntryLast != -1) &&
        likely(PrivateRefCountArray[PrivateRefCountEntryLast].buffer == buffer))
    {
        return &PrivateRefCountArray[PrivateRefCountEntryLast];
    }
 
    /*
     * The code for the cached lookup is small enough to be worth inlining
     * into the caller. In the miss case however, that empirically doesn't
     * seem worth it.
     */
    return GetPrivateRefCountEntrySlow(buffer, do_move);
}

References Assert, PrivateRefCountEntry::buffer, BufferIsLocal, BufferIsValid(), fb(), GetPrivateRefCountEntrySlow(), likely, PrivateRefCountArray, and PrivateRefCountEntryLast.

Referenced by BufferLockAcquire(), BufferLockConditional(), BufferLockDisownInternal(), BufferLockHeldByMe(), BufferLockHeldByMeInMode(), GetPrivateRefCount(), IncrBufferRefCount(), PinBuffer(), PinBuffer_Locked(), ResOwnerReleaseBuffer(), SharedBufferBeginSetHintBits(), UnlockReleaseBuffer(), and UnpinBufferNoOwner().

◆ GetPrivateRefCountEntrySlow()

static pg_noinline PrivateRefCountEntry * GetPrivateRefCountEntrySlow	(	Buffer	buffer,
		bool	do_move
	)

static

Definition at line 419 of file bufmgr.c.

{
    PrivateRefCountEntry *res;
    int         match = -1;
    int         i;
 
    /*
     * First search for references in the array, that'll be sufficient in the
     * majority of cases.
     */
    for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++)
    {
        if (PrivateRefCountArrayKeys[i] == buffer)
        {
            match = i;
            /* see ReservePrivateRefCountEntry() for why we don't return */
        }
    }
 
    if (likely(match != -1))
    {
        /* update cache for the next lookup */
        PrivateRefCountEntryLast = match;
 
        return &PrivateRefCountArray[match];
    }
 
    /*
     * By here we know that the buffer, if already pinned, isn't residing in
     * the array.
     *
     * Only look up the buffer in the hashtable if we've previously overflowed
     * into it.
     */
    if (PrivateRefCountOverflowed == 0)
        return NULL;
 
    res = refcount_lookup(PrivateRefCountHash, buffer);
 
    if (res == NULL)
        return NULL;
    else if (!do_move)
    {
        /* caller doesn't want us to move the hash entry into the array */
        return res;
    }
    else
    {
        /* move buffer from hashtable into the free array slot */
        PrivateRefCountEntry *free;
        PrivateRefCountData data;
 
        /* Save data and delete from hashtable while res is still valid */
        data = res->data;
        refcount_delete_item(PrivateRefCountHash, res);
        Assert(PrivateRefCountOverflowed > 0);
        PrivateRefCountOverflowed--;
 
        /* Ensure there's a free array slot */
        ReservePrivateRefCountEntry();
 
        /* Use up the reserved slot */
        Assert(ReservedRefCountSlot != -1);
        free = &PrivateRefCountArray[ReservedRefCountSlot];
        Assert(PrivateRefCountArrayKeys[ReservedRefCountSlot] == free->buffer);
        Assert(free->buffer == InvalidBuffer);
 
        /* and fill it */
        free->buffer = buffer;
        free->data = data;
        PrivateRefCountArrayKeys[ReservedRefCountSlot] = buffer;
        /* update cache for the next lookup */
        PrivateRefCountEntryLast = ReservedRefCountSlot;
 
        ReservedRefCountSlot = -1;
 
        return free;
    }
}

References Assert, PrivateRefCountEntry::buffer, PrivateRefCountEntry::data, data, fb(), free, i, InvalidBuffer, likely, PrivateRefCountArray, PrivateRefCountArrayKeys, PrivateRefCountEntryLast, PrivateRefCountHash, PrivateRefCountOverflowed, REFCOUNT_ARRAY_ENTRIES, ReservedRefCountSlot, and ReservePrivateRefCountEntry().

Referenced by GetPrivateRefCountEntry().

◆ GetVictimBuffer()

static Buffer GetVictimBuffer	(	BufferAccessStrategy	strategy,
		IOContext	io_context
	)

static

Definition at line 2539 of file bufmgr.c.

{
    BufferDesc *buf_hdr;
    Buffer      buf;
    uint64      buf_state;
    bool        from_ring;
 
    /*
     * Ensure, before we pin a victim buffer, that there's a free refcount
     * entry and resource owner slot for the pin.
     */
    ReservePrivateRefCountEntry();
    ResourceOwnerEnlarge(CurrentResourceOwner);
 
    /* we return here if a prospective victim buffer gets used concurrently */
again:
 
    /*
     * Select a victim buffer.  The buffer is returned pinned and owned by
     * this backend.
     */
    buf_hdr = StrategyGetBuffer(strategy, &buf_state, &from_ring);
    buf = BufferDescriptorGetBuffer(buf_hdr);
 
    /*
     * We shouldn't have any other pins for this buffer.
     */
    CheckBufferIsPinnedOnce(buf);
 
    /*
     * If the buffer was dirty, try to write it out.  There is a race
     * condition here, another backend could dirty the buffer between
     * StrategyGetBuffer() checking that it is not in use and invalidating the
     * buffer below. That's addressed by InvalidateVictimBuffer() verifying
     * that the buffer is not dirty.
     */
    if (buf_state & BM_DIRTY)
    {
        Assert(buf_state & BM_TAG_VALID);
        Assert(buf_state & BM_VALID);
 
        /*
         * We need a share-exclusive lock on the buffer contents to write it
         * out (else we might write invalid data, eg because someone else is
         * compacting the page contents while we write).  We must use a
         * conditional lock acquisition here to avoid deadlock.  Even though
         * the buffer was not pinned (and therefore surely not locked) when
         * StrategyGetBuffer returned it, someone else could have pinned and
         * (share-)exclusive-locked it by the time we get here. If we try to
         * get the lock unconditionally, we'd block waiting for them; if they
         * later block waiting for us, deadlock ensues. (This has been
         * observed to happen when two backends are both trying to split btree
         * index pages, and the second one just happens to be trying to split
         * the page the first one got from StrategyGetBuffer.)
         */
        if (!BufferLockConditional(buf, buf_hdr, BUFFER_LOCK_SHARE_EXCLUSIVE))
        {
            /*
             * Someone else has locked the buffer, so give it up and loop back
             * to get another one.
             */
            UnpinBuffer(buf_hdr);
            goto again;
        }
 
        /*
         * If using a nondefault strategy, and this victim came from the
         * strategy ring, let the strategy decide whether to reject it when
         * reusing it would require a WAL flush.  This only applies to
         * permanent buffers; unlogged buffers can have fake LSNs, so
         * XLogNeedsFlush() is not meaningful for them.
         *
         * We need to hold the content lock in at least share-exclusive mode
         * to safely inspect the page LSN, so this couldn't have been done
         * inside StrategyGetBuffer().
         */
        if (strategy && from_ring &&
            buf_state & BM_PERMANENT &&
            XLogNeedsFlush(BufferGetLSN(buf_hdr)) &&
            StrategyRejectBuffer(strategy, buf_hdr, from_ring))
        {
            UnlockReleaseBuffer(buf);
            goto again;
        }
 
        /* OK, do the I/O */
        FlushBuffer(buf_hdr, NULL, IOOBJECT_RELATION, io_context);
        LockBuffer(buf, BUFFER_LOCK_UNLOCK);
 
        ScheduleBufferTagForWriteback(&BackendWritebackContext, io_context,
                                      &buf_hdr->tag);
    }
 
 
    if (buf_state & BM_VALID)
    {
        /*
         * When a BufferAccessStrategy is in use, blocks evicted from shared
         * buffers are counted as IOOP_EVICT in the corresponding context
         * (e.g. IOCONTEXT_BULKWRITE). Shared buffers are evicted by a
         * strategy in two cases: 1) while initially claiming buffers for the
         * strategy ring 2) to replace an existing strategy ring buffer
         * because it is pinned or in use and cannot be reused.
         *
         * Blocks evicted from buffers already in the strategy ring are
         * counted as IOOP_REUSE in the corresponding strategy context.
         *
         * At this point, we can accurately count evictions and reuses,
         * because we have successfully claimed the valid buffer. Previously,
         * we may have been forced to release the buffer due to concurrent
         * pinners or erroring out.
         */
        pgstat_count_io_op(IOOBJECT_RELATION, io_context,
                           from_ring ? IOOP_REUSE : IOOP_EVICT, 1, 0);
    }
 
    /*
     * If the buffer has an entry in the buffer mapping table, delete it. This
     * can fail because another backend could have pinned or dirtied the
     * buffer.
     */
    if ((buf_state & BM_TAG_VALID) && !InvalidateVictimBuffer(buf_hdr))
    {
        UnpinBuffer(buf_hdr);
        goto again;
    }
 
    /* a final set of sanity checks */
#ifdef USE_ASSERT_CHECKING
    buf_state = pg_atomic_read_u64(&buf_hdr->state);
 
    Assert(BUF_STATE_GET_REFCOUNT(buf_state) == 1);
    Assert(!(buf_state & (BM_TAG_VALID | BM_VALID | BM_DIRTY)));
 
    CheckBufferIsPinnedOnce(buf);
#endif
 
    return buf;
}

References Assert, BackendWritebackContext, BM_DIRTY, BM_PERMANENT, BM_TAG_VALID, BM_VALID, buf, BUF_STATE_GET_REFCOUNT, BUFFER_LOCK_SHARE_EXCLUSIVE, BUFFER_LOCK_UNLOCK, BufferDescriptorGetBuffer(), BufferGetLSN, BufferLockConditional(), CheckBufferIsPinnedOnce(), CurrentResourceOwner, fb(), FlushBuffer(), InvalidateVictimBuffer(), IOOBJECT_RELATION, IOOP_EVICT, IOOP_REUSE, LockBuffer(), pg_atomic_read_u64(), pgstat_count_io_op(), ReservePrivateRefCountEntry(), ResourceOwnerEnlarge(), ScheduleBufferTagForWriteback(), StrategyGetBuffer(), StrategyRejectBuffer(), UnlockReleaseBuffer(), UnpinBuffer(), and XLogNeedsFlush().

Referenced by BufferAlloc(), and ExtendBufferedRelShared().

◆ HoldingBufferPinThatDelaysRecovery()

bool HoldingBufferPinThatDelaysRecovery ( void )

Definition at line 6817 of file bufmgr.c.

{
    int         bufid = GetStartupBufferPinWaitBufId();
 
    /*
     * If we get woken slowly then it's possible that the Startup process was
     * already woken by other backends before we got here. Also possible that
     * we get here by multiple interrupts or interrupts at inappropriate
     * times, so make sure we do nothing if the bufid is not set.
     */
    if (bufid < 0)
        return false;
 
    if (GetPrivateRefCount(bufid + 1) > 0)
        return true;
 
    return false;
}

References fb(), GetPrivateRefCount(), and GetStartupBufferPinWaitBufId().

Referenced by CheckRecoveryConflictDeadlock(), and ProcessRecoveryConflictInterrupt().

◆ IncrBufferRefCount()

void IncrBufferRefCount ( Buffer buffer )

Definition at line 5670 of file bufmgr.c.

{
    Assert(BufferIsPinned(buffer));
    ResourceOwnerEnlarge(CurrentResourceOwner);
    if (BufferIsLocal(buffer))
        LocalRefCount[-buffer - 1]++;
    else
    {
        PrivateRefCountEntry *ref;
 
        ref = GetPrivateRefCountEntry(buffer, true);
        Assert(ref != NULL);
        ref->data.refcount++;
    }
    ResourceOwnerRememberBuffer(CurrentResourceOwner, buffer);
}

References Assert, PrivateRefCountEntry::buffer, BufferIsLocal, BufferIsPinned, CurrentResourceOwner, fb(), GetPrivateRefCountEntry(), LocalRefCount, ResourceOwnerEnlarge(), and ResourceOwnerRememberBuffer().

Referenced by _bt_steppage(), btrestrpos(), entryLoadMoreItems(), ReadBufferBI(), RelationAddBlocks(), scanPostingTree(), startScanEntry(), and tts_buffer_heap_store_tuple().

◆ InitBufferManagerAccess()

void InitBufferManagerAccess ( void )

Definition at line 4216 of file bufmgr.c.

{
    /*
     * An advisory limit on the number of pins each backend should hold, based
     * on shared_buffers and the maximum number of connections possible.
     * That's very pessimistic, but outside toy-sized shared_buffers it should
     * allow plenty of pins.  LimitAdditionalPins() and
     * GetAdditionalPinLimit() can be used to check the remaining balance.
     */
    MaxProportionalPins = NBuffers / (MaxBackends + NUM_AUXILIARY_PROCS);
 
    memset(&PrivateRefCountArray, 0, sizeof(PrivateRefCountArray));
    memset(&PrivateRefCountArrayKeys, 0, sizeof(PrivateRefCountArrayKeys));
 
    PrivateRefCountHash = refcount_create(CurrentMemoryContext, 100, NULL);
 
    /*
     * AtProcExit_Buffers needs LWLock access, and thereby has to be called at
     * the corresponding phase of backend shutdown.
     */
    Assert(MyProc != NULL);
    on_shmem_exit(AtProcExit_Buffers, 0);
}

References Assert, AtProcExit_Buffers(), CurrentMemoryContext, fb(), MaxBackends, MaxProportionalPins, MyProc, NBuffers, NUM_AUXILIARY_PROCS, on_shmem_exit(), PrivateRefCountArray, PrivateRefCountArrayKeys, and PrivateRefCountHash.

Referenced by BaseInit().

◆ InvalidateBuffer()

static void InvalidateBuffer ( BufferDesc * buf )

static

Definition at line 2361 of file bufmgr.c.

{
    BufferTag   oldTag;
    uint32      oldHash;        /* hash value for oldTag */
    LWLock     *oldPartitionLock;   /* buffer partition lock for it */
    uint32      oldFlags;
    uint64      buf_state;
 
    /* Save the original buffer tag before dropping the spinlock */
    oldTag = buf->tag;
 
    UnlockBufHdr(buf);
 
    /*
     * Need to compute the old tag's hashcode and partition lock ID. XXX is it
     * worth storing the hashcode in BufferDesc so we need not recompute it
     * here?  Probably not.
     */
    oldHash = BufTableHashCode(&oldTag);
    oldPartitionLock = BufMappingPartitionLock(oldHash);
 
retry:
 
    /*
     * Acquire exclusive mapping lock in preparation for changing the buffer's
     * association.
     */
    LWLockAcquire(oldPartitionLock, LW_EXCLUSIVE);
 
    /* Re-lock the buffer header */
    buf_state = LockBufHdr(buf);
 
    /* If it's changed while we were waiting for lock, do nothing */
    if (!BufferTagsEqual(&buf->tag, &oldTag))
    {
        UnlockBufHdr(buf);
        LWLockRelease(oldPartitionLock);
        return;
    }
 
    /*
     * We assume the reason for it to be pinned is that either we were
     * asynchronously reading the page in before erroring out or someone else
     * is flushing the page out.  Wait for the IO to finish.  (This could be
     * an infinite loop if the refcount is messed up... it would be nice to
     * time out after awhile, but there seems no way to be sure how many loops
     * may be needed.  Note that if the other guy has pinned the buffer but
     * not yet done StartBufferIO, WaitIO will fall through and we'll
     * effectively be busy-looping here.)
     */
    if (BUF_STATE_GET_REFCOUNT(buf_state) != 0)
    {
        UnlockBufHdr(buf);
        LWLockRelease(oldPartitionLock);
        /* safety check: should definitely not be our *own* pin */
        if (GetPrivateRefCount(BufferDescriptorGetBuffer(buf)) > 0)
            elog(ERROR, "buffer is pinned in InvalidateBuffer");
        WaitIO(buf);
        goto retry;
    }
 
    /*
     * An invalidated buffer should not have any backends waiting to lock the
     * buffer, therefore BM_LOCK_WAKE_IN_PROGRESS should not be set.
     */
    Assert(!(buf_state & BM_LOCK_WAKE_IN_PROGRESS));
 
    /*
     * Clear out the buffer's tag and flags.  We must do this to ensure that
     * linear scans of the buffer array don't think the buffer is valid.
     */
    oldFlags = buf_state & BUF_FLAG_MASK;
    ClearBufferTag(&buf->tag);
 
    UnlockBufHdrExt(buf, buf_state,
                    0,
                    BUF_FLAG_MASK | BUF_USAGECOUNT_MASK,
                    0);
 
    /*
     * Remove the buffer from the lookup hashtable, if it was in there.
     */
    if (oldFlags & BM_TAG_VALID)
        BufTableDelete(&oldTag, oldHash);
 
    /*
     * Done with mapping lock.
     */
    LWLockRelease(oldPartitionLock);
}

References Assert, BM_LOCK_WAKE_IN_PROGRESS, BM_TAG_VALID, buf, BUF_FLAG_MASK, BUF_STATE_GET_REFCOUNT, BUF_USAGECOUNT_MASK, BufferDescriptorGetBuffer(), BufferTagsEqual(), BufMappingPartitionLock(), BufTableDelete(), BufTableHashCode(), ClearBufferTag(), elog, ERROR, fb(), GetPrivateRefCount(), LockBufHdr(), LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), UnlockBufHdr(), UnlockBufHdrExt(), and WaitIO().

Referenced by DropDatabaseBuffers(), DropRelationBuffers(), DropRelationsAllBuffers(), and FindAndDropRelationBuffers().

◆ InvalidateVictimBuffer()

static bool InvalidateVictimBuffer ( BufferDesc * buf_hdr )

static

Definition at line 2462 of file bufmgr.c.

{
    uint64      buf_state;
    uint32      hash;
    LWLock     *partition_lock;
    BufferTag   tag;
 
    Assert(GetPrivateRefCount(BufferDescriptorGetBuffer(buf_hdr)) == 1);
 
    /* have buffer pinned, so it's safe to read tag without lock */
    tag = buf_hdr->tag;
 
    hash = BufTableHashCode(&tag);
    partition_lock = BufMappingPartitionLock(hash);
 
    LWLockAcquire(partition_lock, LW_EXCLUSIVE);
 
    /* lock the buffer header */
    buf_state = LockBufHdr(buf_hdr);
 
    /*
     * We have the buffer pinned nobody else should have been able to unset
     * this concurrently.
     */
    Assert(buf_state & BM_TAG_VALID);
    Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
    Assert(BufferTagsEqual(&buf_hdr->tag, &tag));
 
    /*
     * If somebody else pinned the buffer since, or even worse, dirtied it,
     * give up on this buffer: It's clearly in use.
     */
    if (BUF_STATE_GET_REFCOUNT(buf_state) != 1 || (buf_state & BM_DIRTY))
    {
        Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
 
        UnlockBufHdr(buf_hdr);
        LWLockRelease(partition_lock);
 
        return false;
    }
 
    /*
     * An invalidated buffer should not have any backends waiting to lock the
     * buffer, therefore BM_LOCK_WAKE_IN_PROGRESS should not be set.
     */
    Assert(!(buf_state & BM_LOCK_WAKE_IN_PROGRESS));
 
    /*
     * Clear out the buffer's tag and flags and usagecount.  This is not
     * strictly required, as BM_TAG_VALID/BM_VALID needs to be checked before
     * doing anything with the buffer. But currently it's beneficial, as the
     * cheaper pre-check for several linear scans of shared buffers use the
     * tag (see e.g. FlushDatabaseBuffers()).
     */
    ClearBufferTag(&buf_hdr->tag);
    UnlockBufHdrExt(buf_hdr, buf_state,
                    0,
                    BUF_FLAG_MASK | BUF_USAGECOUNT_MASK,
                    0);
 
    Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
 
    /* finally delete buffer from the buffer mapping table */
    BufTableDelete(&tag, hash);
 
    LWLockRelease(partition_lock);
 
    buf_state = pg_atomic_read_u64(&buf_hdr->state);
    Assert(!(buf_state & (BM_DIRTY | BM_VALID | BM_TAG_VALID)));
    Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
    Assert(BUF_STATE_GET_REFCOUNT(pg_atomic_read_u64(&buf_hdr->state)) > 0);
 
    return true;
}

References Assert, BM_DIRTY, BM_LOCK_WAKE_IN_PROGRESS, BM_TAG_VALID, BM_VALID, BUF_FLAG_MASK, BUF_STATE_GET_REFCOUNT, BUF_USAGECOUNT_MASK, BufferDescriptorGetBuffer(), BufferTagsEqual(), BufMappingPartitionLock(), BufTableDelete(), BufTableHashCode(), ClearBufferTag(), fb(), GetPrivateRefCount(), hash(), LockBufHdr(), LW_EXCLUSIVE, LWLockAcquire(), LWLockRelease(), pg_atomic_read_u64(), UnlockBufHdr(), and UnlockBufHdrExt().

Referenced by EvictUnpinnedBufferInternal(), and GetVictimBuffer().

◆ IsBufferCleanupOK()

bool IsBufferCleanupOK ( Buffer buffer )

Definition at line 6901 of file bufmgr.c.

{
    BufferDesc *bufHdr;
    uint64      buf_state;
 
    Assert(BufferIsValid(buffer));
 
    /* see AIO related comment in LockBufferForCleanup() */
 
    if (BufferIsLocal(buffer))
    {
        /* There should be exactly one pin */
        if (LocalRefCount[-buffer - 1] != 1)
            return false;
        /* Nobody else to wait for */
        return true;
    }
 
    /* There should be exactly one local pin */
    if (GetPrivateRefCount(buffer) != 1)
        return false;
 
    bufHdr = GetBufferDescriptor(buffer - 1);
 
    /* caller must hold exclusive lock on buffer */
    Assert(BufferIsLockedByMeInMode(buffer, BUFFER_LOCK_EXCLUSIVE));
 
    buf_state = LockBufHdr(bufHdr);
 
    Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
    if (BUF_STATE_GET_REFCOUNT(buf_state) == 1)
    {
        /* pincount is OK. */
        UnlockBufHdr(bufHdr);
        return true;
    }
 
    UnlockBufHdr(bufHdr);
    return false;
}

References Assert, BUF_STATE_GET_REFCOUNT, PrivateRefCountEntry::buffer, BUFFER_LOCK_EXCLUSIVE, BufferIsLocal, BufferIsLockedByMeInMode(), BufferIsValid(), fb(), GetBufferDescriptor(), GetPrivateRefCount(), LocalRefCount, LockBufHdr(), and UnlockBufHdr().

Referenced by _hash_doinsert(), _hash_expandtable(), _hash_splitbucket(), and hashbucketcleanup().

◆ IssuePendingWritebacks()

void IssuePendingWritebacks	(	WritebackContext *	wb_context,
		IOContext	io_context
	)

Definition at line 7740 of file bufmgr.c.

{
    instr_time  io_start;
    int         i;
 
    if (wb_context->nr_pending == 0)
        return;
 
    /*
     * Executing the writes in-order can make them a lot faster, and allows to
     * merge writeback requests to consecutive blocks into larger writebacks.
     */
    sort_pending_writebacks(wb_context->pending_writebacks,
                            wb_context->nr_pending);
 
    io_start = pgstat_prepare_io_time(track_io_timing);
 
    /*
     * Coalesce neighbouring writes, but nothing else. For that we iterate
     * through the, now sorted, array of pending flushes, and look forward to
     * find all neighbouring (or identical) writes.
     */
    for (i = 0; i < wb_context->nr_pending; i++)
    {
        PendingWriteback *cur;
        PendingWriteback *next;
        SMgrRelation reln;
        int         ahead;
        BufferTag   tag;
        RelFileLocator currlocator;
        Size        nblocks = 1;
 
        cur = &wb_context->pending_writebacks[i];
        tag = cur->tag;
        currlocator = BufTagGetRelFileLocator(&tag);
 
        /*
         * Peek ahead, into following writeback requests, to see if they can
         * be combined with the current one.
         */
        for (ahead = 0; i + ahead + 1 < wb_context->nr_pending; ahead++)
        {
 
            next = &wb_context->pending_writebacks[i + ahead + 1];
 
            /* different file, stop */
            if (!RelFileLocatorEquals(currlocator,
                                      BufTagGetRelFileLocator(&next->tag)) ||
                BufTagGetForkNum(&cur->tag) != BufTagGetForkNum(&next->tag))
                break;
 
            /* ok, block queued twice, skip */
            if (cur->tag.blockNum == next->tag.blockNum)
                continue;
 
            /* only merge consecutive writes */
            if (cur->tag.blockNum + 1 != next->tag.blockNum)
                break;
 
            nblocks++;
            cur = next;
        }
 
        i += ahead;
 
        /* and finally tell the kernel to write the data to storage */
        reln = smgropen(currlocator, INVALID_PROC_NUMBER);
        smgrwriteback(reln, BufTagGetForkNum(&tag), tag.blockNum, nblocks);
    }
 
    /*
     * Assume that writeback requests are only issued for buffers containing
     * blocks of permanent relations.
     */
    pgstat_count_io_op_time(IOOBJECT_RELATION, io_context,
                            IOOP_WRITEBACK, io_start, wb_context->nr_pending, 0);
 
    wb_context->nr_pending = 0;
}

References buftag::blockNum, BufTagGetForkNum(), BufTagGetRelFileLocator(), cur, fb(), i, INVALID_PROC_NUMBER, IOOBJECT_RELATION, IOOP_WRITEBACK, next, pgstat_count_io_op_time(), pgstat_prepare_io_time(), RelFileLocatorEquals, smgropen(), smgrwriteback(), and track_io_timing.

Referenced by BufferSync(), and ScheduleBufferTagForWriteback().

◆ LimitAdditionalPins()

void LimitAdditionalPins ( uint32 * additional_pins )

Definition at line 2724 of file bufmgr.c.

{
    uint32      limit;
 
    if (*additional_pins <= 1)
        return;
 
    limit = GetAdditionalPinLimit();
    limit = Max(limit, 1);
    if (limit < *additional_pins)
        *additional_pins = limit;
}

References fb(), GetAdditionalPinLimit(), and Max.

Referenced by ExtendBufferedRelShared().

◆ local_buffer_readv_complete()

static PgAioResult local_buffer_readv_complete	(	PgAioHandle *	ioh,
		PgAioResult	prior_result,
		uint8	cb_data
	)

static

Definition at line 8931 of file bufmgr.c.

{
    return buffer_readv_complete(ioh, prior_result, cb_data, true);
}

References buffer_readv_complete(), and fb().

◆ local_buffer_readv_stage()

static void local_buffer_readv_stage	(	PgAioHandle *	ioh,
		uint8	cb_data
	)

static

Definition at line 8925 of file bufmgr.c.

{
    buffer_stage_common(ioh, false, true);
}

References buffer_stage_common(), and fb().

◆ local_buffer_write_error_callback()

static void local_buffer_write_error_callback ( void * arg )

static

Definition at line 7475 of file bufmgr.c.

{
    BufferDesc *bufHdr = (BufferDesc *) arg;
 
    if (bufHdr != NULL)
        errcontext("writing block %u of relation \"%s\"",
                   bufHdr->tag.blockNum,
                   relpathbackend(BufTagGetRelFileLocator(&bufHdr->tag),
                                  MyProcNumber,
                                  BufTagGetForkNum(&bufHdr->tag)).str);
}

References arg, BufTagGetForkNum(), BufTagGetRelFileLocator(), errcontext, fb(), MyProcNumber, and relpathbackend.

Referenced by FlushRelationBuffers().

◆ LockBufferForCleanup()

void LockBufferForCleanup ( Buffer buffer )

Definition at line 6670 of file bufmgr.c.

{
    BufferDesc *bufHdr;
    TimestampTz waitStart = 0;
    bool        waiting = false;
    bool        logged_recovery_conflict = false;
 
    Assert(BufferIsPinned(buffer));
    Assert(PinCountWaitBuf == NULL);
 
    CheckBufferIsPinnedOnce(buffer);
 
    /*
     * We do not yet need to be worried about in-progress AIOs holding a pin,
     * as we, so far, only support doing reads via AIO and this function can
     * only be called once the buffer is valid (i.e. no read can be in
     * flight).
     */
 
    /* Nobody else to wait for */
    if (BufferIsLocal(buffer))
        return;
 
    bufHdr = GetBufferDescriptor(buffer - 1);
 
    for (;;)
    {
        uint64      buf_state;
        uint64      unset_bits = 0;
 
        /* Try to acquire lock */
        LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
        buf_state = LockBufHdr(bufHdr);
 
        Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
        if (BUF_STATE_GET_REFCOUNT(buf_state) == 1)
        {
            /* Successfully acquired exclusive lock with pincount 1 */
            UnlockBufHdr(bufHdr);
 
            /*
             * Emit the log message if recovery conflict on buffer pin was
             * resolved but the startup process waited longer than
             * deadlock_timeout for it.
             */
            if (logged_recovery_conflict)
                LogRecoveryConflict(RECOVERY_CONFLICT_BUFFERPIN,
                                    waitStart, GetCurrentTimestamp(),
                                    NULL, false);
 
            if (waiting)
            {
                /* reset ps display to remove the suffix if we added one */
                set_ps_display_remove_suffix();
                waiting = false;
            }
            return;
        }
        /* Failed, so mark myself as waiting for pincount 1 */
        if (buf_state & BM_PIN_COUNT_WAITER)
        {
            UnlockBufHdr(bufHdr);
            LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
            elog(ERROR, "multiple backends attempting to wait for pincount 1");
        }
        bufHdr->wait_backend_pgprocno = MyProcNumber;
        PinCountWaitBuf = bufHdr;
        UnlockBufHdrExt(bufHdr, buf_state,
                        BM_PIN_COUNT_WAITER, 0,
                        0);
        LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
 
        /* Wait to be signaled by UnpinBuffer() */
        if (InHotStandby)
        {
            if (!waiting)
            {
                /* adjust the process title to indicate that it's waiting */
                set_ps_display_suffix("waiting");
                waiting = true;
            }
 
            /*
             * Emit the log message if the startup process is waiting longer
             * than deadlock_timeout for recovery conflict on buffer pin.
             *
             * Skip this if first time through because the startup process has
             * not started waiting yet in this case. So, the wait start
             * timestamp is set after this logic.
             */
            if (waitStart != 0 && !logged_recovery_conflict)
            {
                TimestampTz now = GetCurrentTimestamp();
 
                if (TimestampDifferenceExceeds(waitStart, now,
                                               DeadlockTimeout))
                {
                    LogRecoveryConflict(RECOVERY_CONFLICT_BUFFERPIN,
                                        waitStart, now, NULL, true);
                    logged_recovery_conflict = true;
                }
            }
 
            /*
             * Set the wait start timestamp if logging is enabled and first
             * time through.
             */
            if (log_recovery_conflict_waits && waitStart == 0)
                waitStart = GetCurrentTimestamp();
 
            /* Publish the bufid that Startup process waits on */
            SetStartupBufferPinWaitBufId(buffer - 1);
            /* Set alarm and then wait to be signaled by UnpinBuffer() */
            ResolveRecoveryConflictWithBufferPin();
            /* Reset the published bufid */
            SetStartupBufferPinWaitBufId(-1);
        }
        else
            ProcWaitForSignal(WAIT_EVENT_BUFFER_CLEANUP);
 
        /*
         * Remove flag marking us as waiter. Normally this will not be set
         * anymore, but ProcWaitForSignal() can return for other signals as
         * well.  We take care to only reset the flag if we're the waiter, as
         * theoretically another backend could have started waiting. That's
         * impossible with the current usages due to table level locking, but
         * better be safe.
         */
        buf_state = LockBufHdr(bufHdr);
        if ((buf_state & BM_PIN_COUNT_WAITER) != 0 &&
            bufHdr->wait_backend_pgprocno == MyProcNumber)
            unset_bits |= BM_PIN_COUNT_WAITER;
 
        UnlockBufHdrExt(bufHdr, buf_state,
                        0, unset_bits,
                        0);
 
        PinCountWaitBuf = NULL;
        /* Loop back and try again */
    }
}

References Assert, BM_PIN_COUNT_WAITER, BUF_STATE_GET_REFCOUNT, PrivateRefCountEntry::buffer, BUFFER_LOCK_EXCLUSIVE, BUFFER_LOCK_UNLOCK, BufferIsLocal, BufferIsPinned, CheckBufferIsPinnedOnce(), DeadlockTimeout, elog, ERROR, fb(), GetBufferDescriptor(), GetCurrentTimestamp(), InHotStandby, LockBuffer(), LockBufHdr(), log_recovery_conflict_waits, LogRecoveryConflict(), MyProcNumber, now(), PinCountWaitBuf, ProcWaitForSignal(), RECOVERY_CONFLICT_BUFFERPIN, ResolveRecoveryConflictWithBufferPin(), set_ps_display_remove_suffix(), set_ps_display_suffix(), SetStartupBufferPinWaitBufId(), TimestampDifferenceExceeds(), UnlockBufHdr(), UnlockBufHdrExt(), and waiting.

Referenced by _bt_upgradelockbufcleanup(), ginVacuumPostingTree(), hashbulkdelete(), heap_force_common(), lazy_scan_heap(), XLogReadBufferForRedoExtended(), and ZeroAndLockBuffer().

◆ LockBufferInternal()

void LockBufferInternal	(	Buffer	buffer,
		BufferLockMode	mode
	)

Definition at line 6574 of file bufmgr.c.

{
    BufferDesc *buf_hdr;
 
    /*
     * We can't wait if we haven't got a PGPROC.  This should only occur
     * during bootstrap or shared memory initialization.  Put an Assert here
     * to catch unsafe coding practices.
     */
    Assert(!(MyProc == NULL && IsUnderPostmaster));
 
    /* handled in LockBuffer() wrapper */
    Assert(mode != BUFFER_LOCK_UNLOCK);
 
    Assert(BufferIsPinned(buffer));
    if (BufferIsLocal(buffer))
        return;                 /* local buffers need no lock */
 
    buf_hdr = GetBufferDescriptor(buffer - 1);
 
    /*
     * Test the most frequent lock modes first. While a switch (mode) would be
     * nice, at least gcc generates considerably worse code for it.
     *
     * Call BufferLockAcquire() with a constant argument for mode, to generate
     * more efficient code for the different lock modes.
     */
    if (mode == BUFFER_LOCK_SHARE)
        BufferLockAcquire(buffer, buf_hdr, BUFFER_LOCK_SHARE);
    else if (mode == BUFFER_LOCK_EXCLUSIVE)
        BufferLockAcquire(buffer, buf_hdr, BUFFER_LOCK_EXCLUSIVE);
    else if (mode == BUFFER_LOCK_SHARE_EXCLUSIVE)
        BufferLockAcquire(buffer, buf_hdr, BUFFER_LOCK_SHARE_EXCLUSIVE);
    else
        elog(ERROR, "unrecognized buffer lock mode: %d", mode);
}

References Assert, PrivateRefCountEntry::buffer, BUFFER_LOCK_EXCLUSIVE, BUFFER_LOCK_SHARE, BUFFER_LOCK_SHARE_EXCLUSIVE, BUFFER_LOCK_UNLOCK, BufferIsLocal, BufferIsPinned, BufferLockAcquire(), elog, ERROR, fb(), GetBufferDescriptor(), IsUnderPostmaster, mode, and MyProc.

Referenced by LockBuffer().

◆ LockBufHdr()

uint64 LockBufHdr ( BufferDesc * desc )

Definition at line 7518 of file bufmgr.c.

{
    uint64      old_buf_state;
 
    Assert(!BufferIsLocal(BufferDescriptorGetBuffer(desc)));
 
    while (true)
    {
        /*
         * Always try once to acquire the lock directly, without setting up
         * the spin-delay infrastructure. The work necessary for that shows up
         * in profiles and is rarely necessary.
         */
        old_buf_state = pg_atomic_fetch_or_u64(&desc->state, BM_LOCKED);
        if (likely(!(old_buf_state & BM_LOCKED)))
            break;              /* got lock */
 
        /* and then spin without atomic operations until lock is released */
        {
            SpinDelayStatus delayStatus;
 
            init_local_spin_delay(&delayStatus);
 
            while (old_buf_state & BM_LOCKED)
            {
                perform_spin_delay(&delayStatus);
                old_buf_state = pg_atomic_read_u64(&desc->state);
            }
            finish_spin_delay(&delayStatus);
        }
 
        /*
         * Retry. The lock might obviously already be re-acquired by the time
         * we're attempting to get it again.
         */
    }
 
    return old_buf_state | BM_LOCKED;
}

References Assert, BM_LOCKED, BufferDescriptorGetBuffer(), BufferIsLocal, fb(), finish_spin_delay(), init_local_spin_delay, likely, perform_spin_delay(), pg_atomic_fetch_or_u64(), pg_atomic_read_u64(), and BufferDesc::state.

Referenced by AbortBufferIO(), apw_dump_now(), buffer_stage_common(), BufferAlloc(), BufferGetLSNAtomic(), BufferLockDequeueSelf(), BufferLockQueueSelf(), BufferLockWakeup(), BufferSync(), ConditionalLockBufferForCleanup(), create_toy_buffer(), DropDatabaseBuffers(), DropRelationBuffers(), DropRelationsAllBuffers(), EvictAllUnpinnedBuffers(), EvictRelUnpinnedBuffers(), EvictUnpinnedBuffer(), ExtendBufferedRelShared(), FindAndDropRelationBuffers(), FlushDatabaseBuffers(), FlushRelationBuffers(), FlushRelationsAllBuffers(), InvalidateBuffer(), InvalidateVictimBuffer(), IsBufferCleanupOK(), LockBufferForCleanup(), MarkDirtyAllUnpinnedBuffers(), MarkDirtyRelUnpinnedBuffers(), MarkDirtyUnpinnedBuffer(), MarkSharedBufferDirtyHint(), pg_buffercache_os_pages_internal(), pg_buffercache_pages(), StartSharedBufferIO(), SyncOneBuffer(), TerminateBufferIO(), UnlockBuffers(), WaitIO(), and WakePinCountWaiter().

◆ MarkBufferDirty()

void MarkBufferDirty ( Buffer buffer )

Definition at line 3147 of file bufmgr.c.

{
    BufferDesc *bufHdr;
    uint64      buf_state;
    uint64      old_buf_state;
 
    if (!BufferIsValid(buffer))
        elog(ERROR, "bad buffer ID: %d", buffer);
 
    if (BufferIsLocal(buffer))
    {
        MarkLocalBufferDirty(buffer);
        return;
    }
 
    bufHdr = GetBufferDescriptor(buffer - 1);
 
    Assert(BufferIsPinned(buffer));
    Assert(BufferIsLockedByMeInMode(buffer, BUFFER_LOCK_EXCLUSIVE));
 
    /*
     * NB: We have to wait for the buffer header spinlock to be not held, as
     * TerminateBufferIO() relies on the spinlock.
     */
    old_buf_state = pg_atomic_read_u64(&bufHdr->state);
    for (;;)
    {
        if (old_buf_state & BM_LOCKED)
            old_buf_state = WaitBufHdrUnlocked(bufHdr);
 
        buf_state = old_buf_state;
 
        Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
        buf_state |= BM_DIRTY;
 
        if (pg_atomic_compare_exchange_u64(&bufHdr->state, &old_buf_state,
                                           buf_state))
            break;
    }
 
    /*
     * If the buffer was not dirty already, do vacuum accounting.
     */
    if (!(old_buf_state & BM_DIRTY))
    {
        pgBufferUsage.shared_blks_dirtied++;
        if (VacuumCostActive)
            VacuumCostBalance += VacuumCostPageDirty;
    }
}

References Assert, BM_DIRTY, BM_LOCKED, BUF_STATE_GET_REFCOUNT, PrivateRefCountEntry::buffer, BUFFER_LOCK_EXCLUSIVE, BufferIsLocal, BufferIsLockedByMeInMode(), BufferIsPinned, BufferIsValid(), elog, ERROR, fb(), GetBufferDescriptor(), MarkLocalBufferDirty(), pg_atomic_compare_exchange_u64(), pg_atomic_read_u64(), pgBufferUsage, BufferUsage::shared_blks_dirtied, VacuumCostActive, VacuumCostBalance, VacuumCostPageDirty, and WaitBufHdrUnlocked().

Referenced by _bt_clear_incomplete_split(), _bt_dedup_pass(), _bt_delitems_delete(), _bt_delitems_vacuum(), _bt_getroot(), _bt_insertonpg(), _bt_mark_page_halfdead(), _bt_newlevel(), _bt_restore_meta(), _bt_set_cleanup_info(), _bt_split(), _bt_unlink_halfdead_page(), _hash_addovflpage(), _hash_doinsert(), _hash_expandtable(), _hash_freeovflpage(), _hash_init(), _hash_splitbucket(), _hash_squeezebucket(), _hash_vacuum_one_page(), addLeafTuple(), brin_doinsert(), brin_doupdate(), brin_initialize_empty_new_buffer(), brin_xlog_createidx(), brin_xlog_desummarize_page(), brin_xlog_insert_update(), brin_xlog_revmap_extend(), brin_xlog_samepage_update(), brin_xlog_update(), brinbuild(), brinbuildempty(), brinRevmapDesummarizeRange(), btree_xlog_dedup(), btree_xlog_delete(), btree_xlog_insert(), btree_xlog_mark_page_halfdead(), btree_xlog_newroot(), btree_xlog_split(), btree_xlog_unlink_page(), btree_xlog_vacuum(), createPostingTree(), dataExecPlaceToPageInternal(), dataExecPlaceToPageLeaf(), doPickSplit(), entryExecPlaceToPage(), fill_seq_fork_with_data(), FreeSpaceMapPrepareTruncateRel(), generic_redo(), GenericXLogFinish(), ginbuild(), ginbuildempty(), ginbulkdelete(), ginDeletePostingPage(), ginHeapTupleFastInsert(), ginPlaceToPage(), ginRedoClearIncompleteSplit(), ginRedoCreatePTree(), ginRedoDeleteListPages(), ginRedoDeletePage(), ginRedoInsert(), ginRedoInsertListPage(), ginRedoUpdateMetapage(), ginRedoVacuumDataLeafPage(), ginUpdateStats(), ginVacuumPostingTreeLeaf(), gistbuild(), gistbuildempty(), gistdeletepage(), gistplacetopage(), gistprunepage(), gistRedoClearFollowRight(), gistRedoDeleteRecord(), gistRedoPageDelete(), gistRedoPageSplitRecord(), gistRedoPageUpdateRecord(), gistvacuumpage(), hash_xlog_add_ovfl_page(), hash_xlog_delete(), hash_xlog_init_bitmap_page(), hash_xlog_init_meta_page(), hash_xlog_insert(), hash_xlog_move_page_contents(), hash_xlog_split_allocate_page(), hash_xlog_split_cleanup(), hash_xlog_split_complete(), hash_xlog_squeeze_page(), hash_xlog_update_meta_page(), hash_xlog_vacuum_one_page(), hashbucketcleanup(), hashbulkdelete(), heap_abort_speculative(), heap_delete(), heap_finish_speculative(), heap_force_common(), heap_inplace_update_and_unlock(), heap_insert(), heap_lock_tuple(), heap_lock_updated_tuple_rec(), heap_multi_insert(), heap_page_prune_and_freeze(), heap_update(), heap_xlog_confirm(), heap_xlog_delete(), heap_xlog_inplace(), heap_xlog_insert(), heap_xlog_lock(), heap_xlog_lock_updated(), heap_xlog_multi_insert(), heap_xlog_prune_freeze(), heap_xlog_update(), lazy_scan_new_or_empty(), lazy_vacuum_heap_page(), log_newpage_range(), MarkDirtyUnpinnedBufferInternal(), moveLeafs(), nextval_internal(), ProcessSingleRelationFork(), RelationAddBlocks(), RelationCopyStorageUsingBuffer(), RelationGetBufferForTuple(), revmap_physical_extend(), saveNodeLink(), seq_redo(), SetSequence(), shiftList(), spgAddNodeAction(), spgbuild(), SpGistUpdateMetaPage(), spgRedoAddLeaf(), spgRedoAddNode(), spgRedoMoveLeafs(), spgRedoPickSplit(), spgRedoSplitTuple(), spgRedoVacuumLeaf(), spgRedoVacuumRedirect(), spgRedoVacuumRoot(), spgSplitNodeAction(), vacuumLeafPage(), vacuumLeafRoot(), vacuumRedirectAndPlaceholder(), visibilitymap_clear(), visibilitymap_prepare_truncate(), visibilitymap_set(), writeListPage(), and XLogReadBufferForRedoExtended().

◆ MarkBufferDirtyHint()

void MarkBufferDirtyHint	(	Buffer	buffer,
		bool	buffer_std
	)

inline

Definition at line 5821 of file bufmgr.c.

{
    BufferDesc *bufHdr;
 
    bufHdr = GetBufferDescriptor(buffer - 1);
 
    if (!BufferIsValid(buffer))
        elog(ERROR, "bad buffer ID: %d", buffer);
 
    if (BufferIsLocal(buffer))
    {
        MarkLocalBufferDirty(buffer);
        return;
    }
 
    MarkSharedBufferDirtyHint(buffer, bufHdr,
                              pg_atomic_read_u64(&bufHdr->state),
                              buffer_std);
}

References PrivateRefCountEntry::buffer, BufferIsLocal, BufferIsValid(), elog, ERROR, fb(), GetBufferDescriptor(), MarkLocalBufferDirty(), MarkSharedBufferDirtyHint(), and pg_atomic_read_u64().

Referenced by brin_start_evacuating_page(), btvacuumpage(), BufferFinishSetHintBits(), fsm_search(), fsm_search_avail(), fsm_set_and_search(), fsm_vacuum_page(), heap_page_fix_vm_corruption(), heap_page_prune_and_freeze(), prune_freeze_fast_path(), read_seq_tuple(), and XLogRecordPageWithFreeSpace().

◆ MarkDirtyAllUnpinnedBuffers()

void MarkDirtyAllUnpinnedBuffers	(	int32 *	buffers_dirtied,
		int32 *	buffers_already_dirty,
		int32 *	buffers_skipped
	)

Definition at line 8232 of file bufmgr.c.

{
    *buffers_dirtied = 0;
    *buffers_already_dirty = 0;
    *buffers_skipped = 0;
 
    for (int buf = 1; buf <= NBuffers; buf++)
    {
        BufferDesc *desc = GetBufferDescriptor(buf - 1);
        uint64      buf_state;
        bool        buffer_already_dirty;
 
        CHECK_FOR_INTERRUPTS();
 
        buf_state = pg_atomic_read_u64(&desc->state);
        if (!(buf_state & BM_VALID))
            continue;
 
        ResourceOwnerEnlarge(CurrentResourceOwner);
        ReservePrivateRefCountEntry();
 
        LockBufHdr(desc);
 
        if (MarkDirtyUnpinnedBufferInternal(buf, desc, &buffer_already_dirty))
            (*buffers_dirtied)++;
        else if (buffer_already_dirty)
            (*buffers_already_dirty)++;
        else
            (*buffers_skipped)++;
    }
}

References BM_VALID, buf, CHECK_FOR_INTERRUPTS, CurrentResourceOwner, fb(), GetBufferDescriptor(), LockBufHdr(), MarkDirtyUnpinnedBufferInternal(), NBuffers, pg_atomic_read_u64(), ReservePrivateRefCountEntry(), ResourceOwnerEnlarge(), and BufferDesc::state.

Referenced by pg_buffercache_mark_dirty_all().

◆ MarkDirtyRelUnpinnedBuffers()

void MarkDirtyRelUnpinnedBuffers	(	Relation	rel,
		int32 *	buffers_dirtied,
		int32 *	buffers_already_dirty,
		int32 *	buffers_skipped
	)

Definition at line 8175 of file bufmgr.c.

{
    Assert(!RelationUsesLocalBuffers(rel));
 
    *buffers_dirtied = 0;
    *buffers_already_dirty = 0;
    *buffers_skipped = 0;
 
    for (int buf = 1; buf <= NBuffers; buf++)
    {
        BufferDesc *desc = GetBufferDescriptor(buf - 1);
        uint64      buf_state = pg_atomic_read_u64(&(desc->state));
        bool        buffer_already_dirty;
 
        CHECK_FOR_INTERRUPTS();
 
        /* An unlocked precheck should be safe and saves some cycles. */
        if ((buf_state & BM_VALID) == 0 ||
            !BufTagMatchesRelFileLocator(&desc->tag, &rel->rd_locator))
            continue;
 
        /* Make sure we can pin the buffer. */
        ResourceOwnerEnlarge(CurrentResourceOwner);
        ReservePrivateRefCountEntry();
 
        buf_state = LockBufHdr(desc);
 
        /* recheck, could have changed without the lock */
        if ((buf_state & BM_VALID) == 0 ||
            !BufTagMatchesRelFileLocator(&desc->tag, &rel->rd_locator))
        {
            UnlockBufHdr(desc);
            continue;
        }
 
        if (MarkDirtyUnpinnedBufferInternal(buf, desc, &buffer_already_dirty))
            (*buffers_dirtied)++;
        else if (buffer_already_dirty)
            (*buffers_already_dirty)++;
        else
            (*buffers_skipped)++;
    }
}

References Assert, BM_VALID, buf, BufTagMatchesRelFileLocator(), CHECK_FOR_INTERRUPTS, CurrentResourceOwner, fb(), GetBufferDescriptor(), LockBufHdr(), MarkDirtyUnpinnedBufferInternal(), NBuffers, pg_atomic_read_u64(), RelationData::rd_locator, RelationUsesLocalBuffers, ReservePrivateRefCountEntry(), ResourceOwnerEnlarge(), BufferDesc::state, BufferDesc::tag, and UnlockBufHdr().

Referenced by pg_buffercache_mark_dirty_relation().

◆ MarkDirtyUnpinnedBuffer()

bool MarkDirtyUnpinnedBuffer	(	Buffer	buf,
		bool *	buffer_already_dirty
	)

Definition at line 8139 of file bufmgr.c.

{
    BufferDesc *desc;
    bool        buffer_dirtied = false;
 
    Assert(!BufferIsLocal(buf));
 
    /* Make sure we can pin the buffer. */
    ResourceOwnerEnlarge(CurrentResourceOwner);
    ReservePrivateRefCountEntry();
 
    desc = GetBufferDescriptor(buf - 1);
    LockBufHdr(desc);
 
    buffer_dirtied = MarkDirtyUnpinnedBufferInternal(buf, desc, buffer_already_dirty);
    /* Both can not be true at the same time */
    Assert(!(buffer_dirtied && *buffer_already_dirty));
 
    return buffer_dirtied;
}

References Assert, buf, BufferIsLocal, CurrentResourceOwner, fb(), GetBufferDescriptor(), LockBufHdr(), MarkDirtyUnpinnedBufferInternal(), ReservePrivateRefCountEntry(), and ResourceOwnerEnlarge().

Referenced by pg_buffercache_mark_dirty().

◆ MarkDirtyUnpinnedBufferInternal()

static bool MarkDirtyUnpinnedBufferInternal	(	Buffer	buf,
		BufferDesc *	desc,
		bool *	buffer_already_dirty
	)

static

Definition at line 8083 of file bufmgr.c.

{
    uint64      buf_state;
    bool        result = false;
 
    *buffer_already_dirty = false;
 
    buf_state = pg_atomic_read_u64(&(desc->state));
    Assert(buf_state & BM_LOCKED);
 
    if ((buf_state & BM_VALID) == 0)
    {
        UnlockBufHdr(desc);
        return false;
    }
 
    /* Check that it's not pinned already. */
    if (BUF_STATE_GET_REFCOUNT(buf_state) > 0)
    {
        UnlockBufHdr(desc);
        return false;
    }
 
    /* Pin the buffer and then release the buffer spinlock */
    PinBuffer_Locked(desc);
 
    /* If it was not already dirty, mark it as dirty. */
    if (!(buf_state & BM_DIRTY))
    {
        BufferLockAcquire(buf, desc, BUFFER_LOCK_EXCLUSIVE);
        MarkBufferDirty(buf);
        result = true;
        BufferLockUnlock(buf, desc);
    }
    else
        *buffer_already_dirty = true;
 
    UnpinBuffer(desc);
 
    return result;
}

References Assert, BM_DIRTY, BM_LOCKED, BM_VALID, buf, BUF_STATE_GET_REFCOUNT, BUFFER_LOCK_EXCLUSIVE, BufferLockAcquire(), BufferLockUnlock(), fb(), MarkBufferDirty(), pg_atomic_read_u64(), PinBuffer_Locked(), result, BufferDesc::state, UnlockBufHdr(), and UnpinBuffer().

Referenced by MarkDirtyAllUnpinnedBuffers(), MarkDirtyRelUnpinnedBuffers(), and MarkDirtyUnpinnedBuffer().

◆ MarkSharedBufferDirtyHint()

static void MarkSharedBufferDirtyHint	(	Buffer	buffer,
		BufferDesc *	bufHdr,
		uint64	lockstate,
		bool	buffer_std
	)

inlinestatic

Definition at line 5696 of file bufmgr.c.

{
    Page        page = BufferGetPage(buffer);
 
    Assert(GetPrivateRefCount(buffer) > 0);
 
    /* here, either share-exclusive or exclusive lock is OK */
    Assert(BufferLockHeldByMeInMode(bufHdr, BUFFER_LOCK_EXCLUSIVE) ||
           BufferLockHeldByMeInMode(bufHdr, BUFFER_LOCK_SHARE_EXCLUSIVE));
 
    /*
     * This routine might get called many times on the same page, if we are
     * making the first scan after commit of an xact that added/deleted many
     * tuples. So, be as quick as we can if the buffer is already dirty.
     *
     * As we are holding (at least) a share-exclusive lock, nobody could have
     * cleaned or dirtied the page concurrently, so we can just rely on the
     * previously fetched value here without any danger of races.
     */
    if (unlikely(!(lockstate & BM_DIRTY)))
    {
        XLogRecPtr  lsn = InvalidXLogRecPtr;
        bool        wal_log = false;
        uint64      buf_state;
 
        /*
         * If we need to protect hint bit updates from torn writes, WAL-log a
         * full page image of the page. This full page image is only necessary
         * if the hint bit update is the first change to the page since the
         * last checkpoint.
         *
         * We don't check full_page_writes here because that logic is included
         * when we call XLogInsert() since the value changes dynamically.
         */
        if (XLogHintBitIsNeeded() && (lockstate & BM_PERMANENT))
        {
            /*
             * If we must not write WAL, due to a relfilelocator-specific
             * condition or being in recovery, don't dirty the page.  We can
             * set the hint, just not dirty the page as a result so the hint
             * is lost when we evict the page or shutdown.
             *
             * See src/backend/storage/page/README for longer discussion.
             */
            if (RecoveryInProgress() ||
                RelFileLocatorSkippingWAL(BufTagGetRelFileLocator(&bufHdr->tag)))
                return;
 
            wal_log = true;
        }
 
        /*
         * We must mark the page dirty before we emit the WAL record, as per
         * the usual rules, to ensure that BufferSync()/SyncOneBuffer() try to
         * flush the buffer, even if we haven't inserted the WAL record yet.
         * As we hold at least a share-exclusive lock, checkpoints will wait
         * for this backend to be done with the buffer before continuing. If
         * we did it the other way round, a checkpoint could start between
         * writing the WAL record and marking the buffer dirty.
         */
        buf_state = LockBufHdr(bufHdr);
 
        /*
         * It should not be possible for the buffer to already be dirty, see
         * comment above.
         */
        Assert(!(buf_state & BM_DIRTY));
        Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
        UnlockBufHdrExt(bufHdr, buf_state,
                        BM_DIRTY,
                        0, 0);
 
        /*
         * If the block is already dirty because we either made a change or
         * set a hint already, then we don't need to write a full page image.
         * Note that aggressive cleaning of blocks dirtied by hint bit setting
         * would increase the call rate. Bulk setting of hint bits would
         * reduce the call rate...
         */
        if (wal_log)
            lsn = XLogSaveBufferForHint(buffer, buffer_std);
 
        if (XLogRecPtrIsValid(lsn))
        {
            /*
             * Set the page LSN if we wrote a backup block. To allow backends
             * that only hold a share lock on the buffer to read the LSN in a
             * tear-free manner, we set the page LSN while holding the buffer
             * header lock. This allows any reader of an LSN who holds only a
             * share lock to also obtain a buffer header lock before using
             * PageGetLSN() to read the LSN in a tear free way. This is done
             * in BufferGetLSNAtomic().
             *
             * If checksums are enabled, you might think we should reset the
             * checksum here. That will happen when the page is written
             * sometime later in this checkpoint cycle.
             */
            buf_state = LockBufHdr(bufHdr);
            PageSetLSN(page, lsn);
            UnlockBufHdr(bufHdr);
        }
 
        pgBufferUsage.shared_blks_dirtied++;
        if (VacuumCostActive)
            VacuumCostBalance += VacuumCostPageDirty;
    }
}

References Assert, BM_DIRTY, BM_PERMANENT, BUF_STATE_GET_REFCOUNT, PrivateRefCountEntry::buffer, BUFFER_LOCK_EXCLUSIVE, BUFFER_LOCK_SHARE_EXCLUSIVE, BufferGetPage(), BufferLockHeldByMeInMode(), BufTagGetRelFileLocator(), fb(), GetPrivateRefCount(), InvalidXLogRecPtr, LockBufHdr(), PageSetLSN(), pgBufferUsage, RecoveryInProgress(), RelFileLocatorSkippingWAL(), BufferUsage::shared_blks_dirtied, unlikely, UnlockBufHdr(), UnlockBufHdrExt(), VacuumCostActive, VacuumCostBalance, VacuumCostPageDirty, XLogHintBitIsNeeded, XLogRecPtrIsValid, and XLogSaveBufferForHint().

Referenced by BufferSetHintBits16(), and MarkBufferDirtyHint().

◆ NewPrivateRefCountEntry()

static PrivateRefCountEntry * NewPrivateRefCountEntry ( Buffer buffer )

static

Definition at line 388 of file bufmgr.c.

{
    PrivateRefCountEntry *res;
 
    /* only allowed to be called when a reservation has been made */
    Assert(ReservedRefCountSlot != -1);
 
    /* use up the reserved entry */
    res = &PrivateRefCountArray[ReservedRefCountSlot];
 
    /* and fill it */
    PrivateRefCountArrayKeys[ReservedRefCountSlot] = buffer;
    res->buffer = buffer;
    res->data.refcount = 0;
    res->data.lockmode = BUFFER_LOCK_UNLOCK;
 
    /* update cache for the next lookup */
    PrivateRefCountEntryLast = ReservedRefCountSlot;
 
    ReservedRefCountSlot = -1;
 
    return res;
}

References Assert, PrivateRefCountEntry::buffer, BUFFER_LOCK_UNLOCK, PrivateRefCountEntry::data, PrivateRefCountData::lockmode, PrivateRefCountArray, PrivateRefCountArrayKeys, PrivateRefCountEntryLast, PrivateRefCountData::refcount, and ReservedRefCountSlot.

Referenced by TrackNewBufferPin().

◆ PinBuffer()

static bool PinBuffer	(	BufferDesc *	buf,
		BufferAccessStrategy	strategy,
		bool	skip_if_not_valid
	)

static

Definition at line 3272 of file bufmgr.c.

{
    Buffer      b = BufferDescriptorGetBuffer(buf);
    bool        result;
    PrivateRefCountEntry *ref;
 
    Assert(!BufferIsLocal(b));
    Assert(ReservedRefCountSlot != -1);
 
    ref = GetPrivateRefCountEntry(b, true);
 
    if (ref == NULL)
    {
        uint64      buf_state;
        uint64      old_buf_state;
 
        old_buf_state = pg_atomic_read_u64(&buf->state);
        for (;;)
        {
            if (unlikely(skip_if_not_valid && !(old_buf_state & BM_VALID)))
                return false;
 
            /*
             * We're not allowed to increase the refcount while the buffer
             * header spinlock is held. Wait for the lock to be released.
             */
            if (unlikely(old_buf_state & BM_LOCKED))
            {
                old_buf_state = WaitBufHdrUnlocked(buf);
 
                /* perform checks at the top of the loop again */
                continue;
            }
 
            buf_state = old_buf_state;
 
            /* increase refcount */
            buf_state += BUF_REFCOUNT_ONE;
 
            if (strategy == NULL)
            {
                /* Default case: increase usagecount unless already max. */
                if (BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT)
                    buf_state += BUF_USAGECOUNT_ONE;
            }
            else
            {
                /*
                 * Ring buffers shouldn't evict others from pool.  Thus we
                 * don't make usagecount more than 1.
                 */
                if (BUF_STATE_GET_USAGECOUNT(buf_state) == 0)
                    buf_state += BUF_USAGECOUNT_ONE;
            }
 
            if (pg_atomic_compare_exchange_u64(&buf->state, &old_buf_state,
                                               buf_state))
            {
                result = (buf_state & BM_VALID) != 0;
 
                TrackNewBufferPin(b);
                break;
            }
        }
    }
    else
    {
        /*
         * If we previously pinned the buffer, it is likely to be valid, but
         * it may not be if StartReadBuffers() was called and
         * WaitReadBuffers() hasn't been called yet.  We'll check by loading
         * the flags without locking.  This is racy, but it's OK to return
         * false spuriously: when WaitReadBuffers() calls StartBufferIO(),
         * it'll see that it's now valid.
         *
         * Note: We deliberately avoid a Valgrind client request here.
         * Individual access methods can optionally superimpose buffer page
         * client requests on top of our client requests to enforce that
         * buffers are only accessed while locked (and pinned).  It's possible
         * that the buffer page is legitimately non-accessible here.  We
         * cannot meddle with that.
         */
        result = (pg_atomic_read_u64(&buf->state) & BM_VALID) != 0;
 
        Assert(ref->data.refcount > 0);
        ref->data.refcount++;
        ResourceOwnerRememberBuffer(CurrentResourceOwner, b);
    }
 
    return result;
}

References Assert, b, BM_LOCKED, BM_MAX_USAGE_COUNT, BM_VALID, buf, BUF_REFCOUNT_ONE, BUF_STATE_GET_USAGECOUNT, BUF_USAGECOUNT_ONE, BufferDescriptorGetBuffer(), BufferIsLocal, CurrentResourceOwner, fb(), GetPrivateRefCountEntry(), pg_atomic_compare_exchange_u64(), pg_atomic_read_u64(), ReservedRefCountSlot, ResourceOwnerRememberBuffer(), result, TrackNewBufferPin(), unlikely, and WaitBufHdrUnlocked().

Referenced by BufferAlloc(), ExtendBufferedRelShared(), and ReadRecentBuffer().

◆ PinBuffer_Locked()

static void PinBuffer_Locked ( BufferDesc * buf )

static

Definition at line 3388 of file bufmgr.c.

{
    uint64      old_buf_state;
 
    /*
     * As explained, We don't expect any preexisting pins. That allows us to
     * manipulate the PrivateRefCount after releasing the spinlock
     */
    Assert(GetPrivateRefCountEntry(BufferDescriptorGetBuffer(buf), false) == NULL);
 
    /*
     * Since we hold the buffer spinlock, we can update the buffer state and
     * release the lock in one operation.
     */
    old_buf_state = pg_atomic_read_u64(&buf->state);
 
    UnlockBufHdrExt(buf, old_buf_state,
                    0, 0, 1);
 
    TrackNewBufferPin(BufferDescriptorGetBuffer(buf));
}

References Assert, buf, BufferDescriptorGetBuffer(), fb(), GetPrivateRefCountEntry(), pg_atomic_read_u64(), TrackNewBufferPin(), and UnlockBufHdrExt().

Referenced by EvictUnpinnedBufferInternal(), FlushDatabaseBuffers(), FlushRelationBuffers(), FlushRelationsAllBuffers(), MarkDirtyUnpinnedBufferInternal(), and SyncOneBuffer().

◆ PinBufferForBlock()

static pg_attribute_always_inline Buffer PinBufferForBlock	(	Relation	rel,
		SMgrRelation	smgr,
		char	persistence,
		ForkNumber	forkNum,
		BlockNumber	blockNum,
		BufferAccessStrategy	strategy,
		IOObject	io_object,
		IOContext	io_context,
		bool *	foundPtr
	)

static

Definition at line 1232 of file bufmgr.c.

{
    BufferDesc *bufHdr;
 
    Assert(blockNum != P_NEW);
 
    /* Persistence should be set before */
    Assert((persistence == RELPERSISTENCE_TEMP ||
            persistence == RELPERSISTENCE_PERMANENT ||
            persistence == RELPERSISTENCE_UNLOGGED));
 
    TRACE_POSTGRESQL_BUFFER_READ_START(forkNum, blockNum,
                                       smgr->smgr_rlocator.locator.spcOid,
                                       smgr->smgr_rlocator.locator.dbOid,
                                       smgr->smgr_rlocator.locator.relNumber,
                                       smgr->smgr_rlocator.backend);
 
    if (persistence == RELPERSISTENCE_TEMP)
        bufHdr = LocalBufferAlloc(smgr, forkNum, blockNum, foundPtr);
    else
        bufHdr = BufferAlloc(smgr, persistence, forkNum, blockNum,
                             strategy, foundPtr, io_context);
 
    if (*foundPtr)
        TrackBufferHit(io_object, io_context, rel, persistence, smgr, forkNum, blockNum);
 
    if (rel)
    {
        /*
         * While pgBufferUsage's "read" counter isn't bumped unless we reach
         * WaitReadBuffers() (so, not for hits, and not for buffers that are
         * zeroed instead), the per-relation stats always count them.
         */
        pgstat_count_buffer_read(rel);
    }
 
    return BufferDescriptorGetBuffer(bufHdr);
}

References Assert, RelFileLocatorBackend::backend, BufferAlloc(), BufferDescriptorGetBuffer(), RelFileLocator::dbOid, fb(), LocalBufferAlloc(), RelFileLocatorBackend::locator, P_NEW, pgstat_count_buffer_read, RelFileLocator::relNumber, SMgrRelationData::smgr_rlocator, RelFileLocator::spcOid, and TrackBufferHit().

Referenced by ReadBuffer_common(), and StartReadBuffersImpl().

◆ PrefetchBuffer()

PrefetchBufferResult PrefetchBuffer	(	Relation	reln,
		ForkNumber	forkNum,
		BlockNumber	blockNum
	)

Definition at line 787 of file bufmgr.c.

{
    Assert(RelationIsValid(reln));
    Assert(BlockNumberIsValid(blockNum));
 
    if (RelationUsesLocalBuffers(reln))
    {
        /* see comments in ReadBufferExtended */
        if (RELATION_IS_OTHER_TEMP(reln))
            ereport(ERROR,
                    (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
                     errmsg("cannot access temporary tables of other sessions")));
 
        /* pass it off to localbuf.c */
        return PrefetchLocalBuffer(RelationGetSmgr(reln), forkNum, blockNum);
    }
    else
    {
        /* pass it to the shared buffer version */
        return PrefetchSharedBuffer(RelationGetSmgr(reln), forkNum, blockNum);
    }
}

References Assert, BlockNumberIsValid(), ereport, errcode(), errmsg, ERROR, fb(), PrefetchLocalBuffer(), PrefetchSharedBuffer(), RELATION_IS_OTHER_TEMP, RelationGetSmgr(), RelationIsValid, and RelationUsesLocalBuffers.

Referenced by count_nondeletable_pages(), invalidate_one_block(), and pg_prewarm().

◆ PrefetchSharedBuffer()

PrefetchBufferResult PrefetchSharedBuffer	(	SMgrRelation	smgr_reln,
		ForkNumber	forkNum,
		BlockNumber	blockNum
	)

Definition at line 697 of file bufmgr.c.

{
    PrefetchBufferResult result = {InvalidBuffer, false};
    BufferTag   newTag;         /* identity of requested block */
    uint32      newHash;        /* hash value for newTag */
    LWLock     *newPartitionLock;   /* buffer partition lock for it */
    int         buf_id;
 
    Assert(BlockNumberIsValid(blockNum));
 
    /* create a tag so we can lookup the buffer */
    InitBufferTag(&newTag, &smgr_reln->smgr_rlocator.locator,
                  forkNum, blockNum);
 
    /* determine its hash code and partition lock ID */
    newHash = BufTableHashCode(&newTag);
    newPartitionLock = BufMappingPartitionLock(newHash);
 
    /* see if the block is in the buffer pool already */
    LWLockAcquire(newPartitionLock, LW_SHARED);
    buf_id = BufTableLookup(&newTag, newHash);
    LWLockRelease(newPartitionLock);
 
    /* If not in buffers, initiate prefetch */
    if (buf_id < 0)
    {
#ifdef USE_PREFETCH
        /*
         * Try to initiate an asynchronous read.  This returns false in
         * recovery if the relation file doesn't exist.
         */
        if ((io_direct_flags & IO_DIRECT_DATA) == 0 &&
            smgrprefetch(smgr_reln, forkNum, blockNum, 1))
        {
            result.initiated_io = true;
        }
#endif                          /* USE_PREFETCH */
    }
    else
    {
        /*
         * Report the buffer it was in at that time.  The caller may be able
         * to avoid a buffer table lookup, but it's not pinned and it must be
         * rechecked!
         */
        result.recent_buffer = buf_id + 1;
    }
 
    /*
     * If the block *is* in buffers, we do nothing.  This is not really ideal:
     * the block might be just about to be evicted, which would be stupid
     * since we know we are going to need it soon.  But the only easy answer
     * is to bump the usage_count, which does not seem like a great solution:
     * when the caller does ultimately touch the block, usage_count would get
     * bumped again, resulting in too much favoritism for blocks that are
     * involved in a prefetch sequence. A real fix would involve some
     * additional per-buffer state, and it's not clear that there's enough of
     * a problem to justify that.
     */
 
    return result;
}

References Assert, BlockNumberIsValid(), BufMappingPartitionLock(), BufTableHashCode(), BufTableLookup(), fb(), InitBufferTag(), InvalidBuffer, IO_DIRECT_DATA, io_direct_flags, LW_SHARED, LWLockAcquire(), LWLockRelease(), result, and smgrprefetch().

Referenced by PrefetchBuffer(), and XLogPrefetcherNextBlock().

◆ ProcessReadBuffersResult()

static void ProcessReadBuffersResult ( ReadBuffersOperation * operation )

static

Definition at line 1705 of file bufmgr.c.

{
    PgAioReturn *aio_ret = &operation->io_return;
    PgAioResultStatus rs = aio_ret->result.status;
    int         newly_read_blocks = 0;
 
    Assert(pgaio_wref_valid(&operation->io_wref));
    Assert(aio_ret->result.status != PGAIO_RS_UNKNOWN);
 
    /*
     * SMGR reports the number of blocks successfully read as the result of
     * the IO operation. Thus we can simply add that to ->nblocks_done.
     */
 
    if (likely(rs != PGAIO_RS_ERROR))
        newly_read_blocks = aio_ret->result.result;
 
    if (rs == PGAIO_RS_ERROR || rs == PGAIO_RS_WARNING)
        pgaio_result_report(aio_ret->result, &aio_ret->target_data,
                            rs == PGAIO_RS_ERROR ? ERROR : WARNING);
    else if (aio_ret->result.status == PGAIO_RS_PARTIAL)
    {
        /*
         * We'll retry, so we just emit a debug message to the server log (or
         * not even that in prod scenarios).
         */
        pgaio_result_report(aio_ret->result, &aio_ret->target_data, DEBUG1);
        elog(DEBUG3, "partial read, will retry");
    }
 
    Assert(newly_read_blocks > 0);
    Assert(newly_read_blocks <= MAX_IO_COMBINE_LIMIT);
 
    operation->nblocks_done += newly_read_blocks;
 
    Assert(operation->nblocks_done <= operation->nblocks);
}

References Assert, DEBUG1, DEBUG3, elog, ERROR, fb(), likely, MAX_IO_COMBINE_LIMIT, operation, pgaio_result_report(), PGAIO_RS_ERROR, PGAIO_RS_PARTIAL, PGAIO_RS_UNKNOWN, PGAIO_RS_WARNING, pgaio_wref_valid(), PgAioReturn::result, PgAioResult::status, and WARNING.

Referenced by WaitReadBuffers().

◆ ReadBuffer()

Buffer ReadBuffer	(	Relation	reln,
		BlockNumber	blockNum
	)

Definition at line 879 of file bufmgr.c.

{
    return ReadBufferExtended(reln, MAIN_FORKNUM, blockNum, RBM_NORMAL, NULL);
}

References fb(), MAIN_FORKNUM, RBM_NORMAL, and ReadBufferExtended().

Referenced by _bt_allocbuf(), _bt_getbuf(), _bt_relandgetbuf(), _bt_search_insert(), _hash_getbuf(), _hash_getbuf_with_condlock_cleanup(), blbulkdelete(), blinsert(), BloomNewBuffer(), brin_getinsertbuffer(), brinGetStats(), brinGetTupleForHeapBlock(), brinRevmapDesummarizeRange(), brinRevmapInitialize(), bt_metap(), bt_multi_page_stats(), bt_page_items_internal(), bt_page_stats_internal(), ginFindLeafPage(), ginFindParents(), ginGetStats(), ginHeapTupleFastInsert(), ginInsertCleanup(), GinNewBuffer(), ginStepRight(), ginUpdateStats(), gistBufferingFindCorrectParent(), gistbufferinginserttuples(), gistdoinsert(), gistFindCorrectParent(), gistFindPath(), gistfixsplit(), gistGetMaxLevel(), gistkillitems(), gistNewBuffer(), gistProcessItup(), gistScanPage(), heap_abort_speculative(), heap_delete(), heap_fetch(), heap_finish_speculative(), heap_force_common(), heap_get_latest_tid(), heap_index_delete_tuples(), heap_lock_tuple(), heap_update(), heapam_index_fetch_tuple(), initBloomState(), pg_visibility(), pgstatginindex_internal(), read_seq_tuple(), RelationGetBufferForTuple(), ReleaseAndReadBuffer(), revmap_get_buffer(), revmap_physical_extend(), scanGetCandidate(), scanPendingInsert(), shiftList(), spgdoinsert(), spgGetCache(), SpGistGetBuffer(), SpGistNewBuffer(), SpGistUpdateMetaPage(), and spgWalk().

◆ ReadBuffer_common()

static pg_attribute_always_inline Buffer ReadBuffer_common	(	Relation	rel,
		SMgrRelation	smgr,
		char	smgr_persistence,
		ForkNumber	forkNum,
		BlockNumber	blockNum,
		ReadBufferMode	mode,
		BufferAccessStrategy	strategy
	)

static

Definition at line 1285 of file bufmgr.c.

{
    ReadBuffersOperation operation;
    Buffer      buffer;
    int         flags;
    char        persistence;
 
    /*
     * Backward compatibility path, most code should use ExtendBufferedRel()
     * instead, as acquiring the extension lock inside ExtendBufferedRel()
     * scales a lot better.
     */
    if (unlikely(blockNum == P_NEW))
    {
        uint32      flags = EB_SKIP_EXTENSION_LOCK;
 
        /*
         * Since no-one else can be looking at the page contents yet, there is
         * no difference between an exclusive lock and a cleanup-strength
         * lock.
         */
        if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK)
            flags |= EB_LOCK_FIRST;
 
        return ExtendBufferedRel(BMR_REL(rel), forkNum, strategy, flags);
    }
 
    if (rel)
        persistence = rel->rd_rel->relpersistence;
    else
        persistence = smgr_persistence;
 
    if (unlikely(mode == RBM_ZERO_AND_CLEANUP_LOCK ||
                 mode == RBM_ZERO_AND_LOCK))
    {
        bool        found;
        IOContext   io_context;
        IOObject    io_object;
 
        if (persistence == RELPERSISTENCE_TEMP)
        {
            io_context = IOCONTEXT_NORMAL;
            io_object = IOOBJECT_TEMP_RELATION;
        }
        else
        {
            io_context = IOContextForStrategy(strategy);
            io_object = IOOBJECT_RELATION;
        }
 
        buffer = PinBufferForBlock(rel, smgr, persistence,
                                   forkNum, blockNum, strategy,
                                   io_object, io_context, &found);
        ZeroAndLockBuffer(buffer, mode, found);
        return buffer;
    }
 
    /*
     * Signal that we are going to immediately wait. If we're immediately
     * waiting, there is no benefit in actually executing the IO
     * asynchronously, it would just add dispatch overhead.
     */
    flags = READ_BUFFERS_SYNCHRONOUSLY;
    if (mode == RBM_ZERO_ON_ERROR)
        flags |= READ_BUFFERS_ZERO_ON_ERROR;
    operation.smgr = smgr;
    operation.rel = rel;
    operation.persistence = persistence;
    operation.forknum = forkNum;
    operation.strategy = strategy;
    if (StartReadBuffer(&operation,
                        &buffer,
                        blockNum,
                        flags))
        WaitReadBuffers(&operation);
 
    return buffer;
}

References BMR_REL, PrivateRefCountEntry::buffer, EB_LOCK_FIRST, EB_SKIP_EXTENSION_LOCK, ExtendBufferedRel(), fb(), IOCONTEXT_NORMAL, IOContextForStrategy(), IOOBJECT_RELATION, IOOBJECT_TEMP_RELATION, mode, operation, P_NEW, PinBufferForBlock(), RBM_ZERO_AND_CLEANUP_LOCK, RBM_ZERO_AND_LOCK, RBM_ZERO_ON_ERROR, RelationData::rd_rel, READ_BUFFERS_SYNCHRONOUSLY, READ_BUFFERS_ZERO_ON_ERROR, StartReadBuffer(), unlikely, WaitReadBuffers(), and ZeroAndLockBuffer().

Referenced by ExtendBufferedRelTo(), ReadBufferExtended(), and ReadBufferWithoutRelcache().

◆ ReadBufferExtended()

Buffer ReadBufferExtended	(	Relation	reln,
		ForkNumber	forkNum,
		BlockNumber	blockNum,
		ReadBufferMode	mode,
		BufferAccessStrategy	strategy
	)

inline

Definition at line 926 of file bufmgr.c.

{
    Buffer      buf;
 
    /*
     * Reject attempts to read non-local temporary relations; we would be
     * likely to get wrong data since we have no visibility into the owning
     * session's local buffers.
     */
    if (RELATION_IS_OTHER_TEMP(reln))
        ereport(ERROR,
                (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
                 errmsg("cannot access temporary tables of other sessions")));
 
    /*
     * Read the buffer, and update pgstat counters to reflect a cache hit or
     * miss.
     */
    buf = ReadBuffer_common(reln, RelationGetSmgr(reln), 0,
                            forkNum, blockNum, mode, strategy);
 
    return buf;
}

References buf, ereport, errcode(), errmsg, ERROR, fb(), mode, ReadBuffer_common(), RELATION_IS_OTHER_TEMP, and RelationGetSmgr().

Referenced by _hash_getbuf_with_strategy(), _hash_getinitbuf(), _hash_getnewbuf(), BloomInitMetapage(), bt_recheck_sibling_links(), btvacuumpage(), count_nondeletable_pages(), create_toy_buffer(), fsm_readbuf(), get_raw_page_internal(), gin_check_parent_keys_consistency(), gin_check_posting_tree_parent_keys_consistency(), gin_refind_parent(), ginbulkdelete(), ginScanPostingTreeToDelete(), ginVacuumPostingTree(), ginVacuumPostingTreeLeaves(), gistvacuum_delete_empty_pages(), gistvacuumpage(), heapam_scan_sample_next_block(), log_newpage_range(), modify_rel_block(), palloc_btree_page(), pgstat_btree_page(), pgstat_gist_page(), pgstat_hash_page(), pgstat_heap(), pgstatindex_impl(), ProcessSingleRelationFork(), ReadBuffer(), ReadBufferBI(), spgprocesspending(), and vm_readbuf().

◆ ReadBufferWithoutRelcache()

Buffer ReadBufferWithoutRelcache	(	RelFileLocator	rlocator,
		ForkNumber	forkNum,
		BlockNumber	blockNum,
		ReadBufferMode	mode,
		BufferAccessStrategy	strategy,
		bool	permanent
	)

Definition at line 963 of file bufmgr.c.

{
    SMgrRelation smgr = smgropen(rlocator, INVALID_PROC_NUMBER);
 
    return ReadBuffer_common(NULL, smgr,
                             permanent ? RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED,
                             forkNum, blockNum,
                             mode, strategy);
}

References fb(), INVALID_PROC_NUMBER, mode, ReadBuffer_common(), and smgropen().

Referenced by RelationCopyStorageUsingBuffer(), ScanSourceDatabasePgClass(), and XLogReadBufferExtended().

◆ ReadRecentBuffer()

bool ReadRecentBuffer	(	RelFileLocator	rlocator,
		ForkNumber	forkNum,
		BlockNumber	blockNum,
		Buffer	recent_buffer
	)

Definition at line 818 of file bufmgr.c.

{
    BufferDesc *bufHdr;
    BufferTag   tag;
    uint64      buf_state;
 
    Assert(BufferIsValid(recent_buffer));
 
    ResourceOwnerEnlarge(CurrentResourceOwner);
    ReservePrivateRefCountEntry();
    InitBufferTag(&tag, &rlocator, forkNum, blockNum);
 
    if (BufferIsLocal(recent_buffer))
    {
        int         b = -recent_buffer - 1;
 
        bufHdr = GetLocalBufferDescriptor(b);
        buf_state = pg_atomic_read_u64(&bufHdr->state);
 
        /* Is it still valid and holding the right tag? */
        if ((buf_state & BM_VALID) && BufferTagsEqual(&tag, &bufHdr->tag))
        {
            PinLocalBuffer(bufHdr, true);
 
            pgBufferUsage.local_blks_hit++;
 
            return true;
        }
    }
    else
    {
        bufHdr = GetBufferDescriptor(recent_buffer - 1);
 
        /*
         * Is it still valid and holding the right tag?  We do an unlocked tag
         * comparison first, to make it unlikely that we'll increment the
         * usage counter of the wrong buffer, if someone calls us with a very
         * out of date recent_buffer.  Then we'll check it again if we get the
         * pin.
         */
        if (BufferTagsEqual(&tag, &bufHdr->tag) &&
            PinBuffer(bufHdr, NULL, true))
        {
            if (BufferTagsEqual(&tag, &bufHdr->tag))
            {
                pgBufferUsage.shared_blks_hit++;
                return true;
            }
            UnpinBuffer(bufHdr);
        }
    }
 
    return false;
}

References Assert, b, BM_VALID, BufferIsLocal, BufferIsValid(), BufferTagsEqual(), CurrentResourceOwner, fb(), GetBufferDescriptor(), GetLocalBufferDescriptor(), InitBufferTag(), BufferUsage::local_blks_hit, pg_atomic_read_u64(), pgBufferUsage, PinBuffer(), PinLocalBuffer(), ReservePrivateRefCountEntry(), ResourceOwnerEnlarge(), BufferUsage::shared_blks_hit, and UnpinBuffer().

Referenced by invalidate_one_block(), and XLogReadBufferExtended().

◆ RelationCopyStorageUsingBuffer()

static void RelationCopyStorageUsingBuffer	(	RelFileLocator	srclocator,
		RelFileLocator	dstlocator,
		ForkNumber	forkNum,
		bool	permanent
	)

static

Definition at line 5348 of file bufmgr.c.

{
    Buffer      srcBuf;
    Buffer      dstBuf;
    Page        srcPage;
    Page        dstPage;
    bool        use_wal;
    BlockNumber nblocks;
    BlockNumber blkno;
    PGIOAlignedBlock buf;
    BufferAccessStrategy bstrategy_src;
    BufferAccessStrategy bstrategy_dst;
    BlockRangeReadStreamPrivate p;
    ReadStream *src_stream;
    SMgrRelation src_smgr;
 
    /*
     * In general, we want to write WAL whenever wal_level > 'minimal', but we
     * can skip it when copying any fork of an unlogged relation other than
     * the init fork.
     */
    use_wal = XLogIsNeeded() && (permanent || forkNum == INIT_FORKNUM);
 
    /* Get number of blocks in the source relation. */
    nblocks = smgrnblocks(smgropen(srclocator, INVALID_PROC_NUMBER),
                          forkNum);
 
    /* Nothing to copy; just return. */
    if (nblocks == 0)
        return;
 
    /*
     * Bulk extend the destination relation of the same size as the source
     * relation before starting to copy block by block.
     */
    memset(buf.data, 0, BLCKSZ);
    smgrextend(smgropen(dstlocator, INVALID_PROC_NUMBER), forkNum, nblocks - 1,
               buf.data, true);
 
    /* This is a bulk operation, so use buffer access strategies. */
    bstrategy_src = GetAccessStrategy(BAS_BULKREAD);
    bstrategy_dst = GetAccessStrategy(BAS_BULKWRITE);
 
    /* Initialize streaming read */
    p.current_blocknum = 0;
    p.last_exclusive = nblocks;
    src_smgr = smgropen(srclocator, INVALID_PROC_NUMBER);
 
    /*
     * It is safe to use batchmode as block_range_read_stream_cb takes no
     * locks.
     */
    src_stream = read_stream_begin_smgr_relation(READ_STREAM_FULL |
                                                 READ_STREAM_USE_BATCHING,
                                                 bstrategy_src,
                                                 src_smgr,
                                                 permanent ? RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED,
                                                 forkNum,
                                                 block_range_read_stream_cb,
                                                 &p,
                                                 0);
 
    /* Iterate over each block of the source relation file. */
    for (blkno = 0; blkno < nblocks; blkno++)
    {
        CHECK_FOR_INTERRUPTS();
 
        /* Read block from source relation. */
        srcBuf = read_stream_next_buffer(src_stream, NULL);
        LockBuffer(srcBuf, BUFFER_LOCK_SHARE);
        srcPage = BufferGetPage(srcBuf);
 
        dstBuf = ReadBufferWithoutRelcache(dstlocator, forkNum,
                                           BufferGetBlockNumber(srcBuf),
                                           RBM_ZERO_AND_LOCK, bstrategy_dst,
                                           permanent);
        dstPage = BufferGetPage(dstBuf);
 
        START_CRIT_SECTION();
 
        /* Copy page data from the source to the destination. */
        memcpy(dstPage, srcPage, BLCKSZ);
        MarkBufferDirty(dstBuf);
 
        /* WAL-log the copied page. */
        if (use_wal)
            log_newpage_buffer(dstBuf, true);
 
        END_CRIT_SECTION();
 
        UnlockReleaseBuffer(dstBuf);
        UnlockReleaseBuffer(srcBuf);
    }
    Assert(read_stream_next_buffer(src_stream, NULL) == InvalidBuffer);
    read_stream_end(src_stream);
 
    FreeAccessStrategy(bstrategy_src);
    FreeAccessStrategy(bstrategy_dst);
}

References Assert, BAS_BULKREAD, BAS_BULKWRITE, block_range_read_stream_cb(), buf, BUFFER_LOCK_SHARE, BufferGetBlockNumber(), BufferGetPage(), CHECK_FOR_INTERRUPTS, BlockRangeReadStreamPrivate::current_blocknum, END_CRIT_SECTION, fb(), FreeAccessStrategy(), GetAccessStrategy(), INIT_FORKNUM, INVALID_PROC_NUMBER, InvalidBuffer, BlockRangeReadStreamPrivate::last_exclusive, LockBuffer(), log_newpage_buffer(), MarkBufferDirty(), memcpy(), RBM_ZERO_AND_LOCK, read_stream_begin_smgr_relation(), read_stream_end(), READ_STREAM_FULL, read_stream_next_buffer(), READ_STREAM_USE_BATCHING, ReadBufferWithoutRelcache(), smgrextend(), smgrnblocks(), smgropen(), START_CRIT_SECTION, UnlockReleaseBuffer(), and XLogIsNeeded.

Referenced by CreateAndCopyRelationData().

◆ RelationGetNumberOfBlocksInFork()

BlockNumber RelationGetNumberOfBlocksInFork	(	Relation	relation,
		ForkNumber	forkNum
	)

Definition at line 4645 of file bufmgr.c.

{
    if (RELKIND_HAS_TABLE_AM(relation->rd_rel->relkind))
    {
        /*
         * Not every table AM uses BLCKSZ wide fixed size blocks. Therefore
         * tableam returns the size in bytes - but for the purpose of this
         * routine, we want the number of blocks. Therefore divide, rounding
         * up.
         */
        uint64      szbytes;
 
        szbytes = table_relation_size(relation, forkNum);
 
        return (szbytes + (BLCKSZ - 1)) / BLCKSZ;
    }
    else if (RELKIND_HAS_STORAGE(relation->rd_rel->relkind))
    {
        return smgrnblocks(RelationGetSmgr(relation), forkNum);
    }
    else
        Assert(false);
 
    return 0;                   /* keep compiler quiet */
}

References Assert, fb(), RelationData::rd_rel, RelationGetSmgr(), smgrnblocks(), and table_relation_size().

Referenced by _hash_getnewbuf(), _hash_init(), autoprewarm_database_main(), get_raw_page_internal(), pg_prewarm(), and ProcessSingleRelationFork().

◆ ReleaseAndReadBuffer()

Buffer ReleaseAndReadBuffer	(	Buffer	buffer,
		Relation	relation,
		BlockNumber	blockNum
	)

Definition at line 3212 of file bufmgr.c.

{
    ForkNumber  forkNum = MAIN_FORKNUM;
    BufferDesc *bufHdr;
 
    if (BufferIsValid(buffer))
    {
        Assert(BufferIsPinned(buffer));
        if (BufferIsLocal(buffer))
        {
            bufHdr = GetLocalBufferDescriptor(-buffer - 1);
            if (bufHdr->tag.blockNum == blockNum &&
                BufTagMatchesRelFileLocator(&bufHdr->tag, &relation->rd_locator) &&
                BufTagGetForkNum(&bufHdr->tag) == forkNum)
                return buffer;
            UnpinLocalBuffer(buffer);
        }
        else
        {
            bufHdr = GetBufferDescriptor(buffer - 1);
            /* we have pin, so it's ok to examine tag without spinlock */
            if (bufHdr->tag.blockNum == blockNum &&
                BufTagMatchesRelFileLocator(&bufHdr->tag, &relation->rd_locator) &&
                BufTagGetForkNum(&bufHdr->tag) == forkNum)
                return buffer;
            UnpinBuffer(bufHdr);
        }
    }
 
    return ReadBuffer(relation, blockNum);
}

References Assert, PrivateRefCountEntry::buffer, BufferIsLocal, BufferIsPinned, BufferIsValid(), BufTagGetForkNum(), BufTagMatchesRelFileLocator(), fb(), GetBufferDescriptor(), GetLocalBufferDescriptor(), MAIN_FORKNUM, RelationData::rd_locator, ReadBuffer(), UnpinBuffer(), and UnpinLocalBuffer().

Referenced by ginFindLeafPage().

◆ ReleaseBuffer()

void ReleaseBuffer ( Buffer buffer )

Definition at line 5586 of file bufmgr.c.

{
    if (!BufferIsValid(buffer))
        elog(ERROR, "bad buffer ID: %d", buffer);
 
    if (BufferIsLocal(buffer))
        UnpinLocalBuffer(buffer);
    else
        UnpinBuffer(GetBufferDescriptor(buffer - 1));
}

References PrivateRefCountEntry::buffer, BufferIsLocal, BufferIsValid(), elog, ERROR, GetBufferDescriptor(), UnpinBuffer(), and UnpinLocalBuffer().

Referenced by _bt_allocbuf(), _bt_pagedel(), _bt_search_insert(), _bt_unlink_halfdead_page(), _hash_dropbuf(), _hash_getbuf_with_condlock_cleanup(), autoprewarm_database_main(), BitmapHeapScanNextBlock(), blinsert(), BloomNewBuffer(), brin_getinsertbuffer(), brin_vacuum_scan(), bringetbitmap(), brinGetTupleForHeapBlock(), brininsert(), brinRevmapTerminate(), brinsummarize(), buffer_create_toy(), collect_corrupt_items(), collect_visibility_data(), entryLoadMoreItems(), ExecEndIndexOnlyScan(), ExtendBufferedRelTo(), FreeBulkInsertState(), freeGinBtreeStack(), fsm_search(), get_actual_variable_endpoint(), GetRecordedFreeSpace(), ginFindParents(), ginFinishSplit(), ginFreeScanKeys(), ginInsertCleanup(), GinNewBuffer(), gistdoinsert(), gistFindCorrectParent(), gistNewBuffer(), gistvacuum_delete_empty_pages(), grow_rel(), heap_abort_speculative(), heap_delete(), heap_endscan(), heap_fetch(), heap_fetch_next_buffer(), heap_force_common(), heap_insert(), heap_lock_tuple(), heap_lock_updated_tuple_rec(), heap_multi_insert(), heap_rescan(), heap_update(), heap_vac_scan_next_block(), heap_xlog_delete(), heap_xlog_insert(), heap_xlog_lock(), heap_xlog_lock_updated(), heap_xlog_multi_insert(), heap_xlog_update(), heapam_index_fetch_end(), heapam_index_fetch_tuple(), heapam_scan_sample_next_block(), heapam_tuple_lock(), heapgettup(), heapgettup_pagemode(), lazy_scan_heap(), lazy_vacuum_heap_rel(), pg_prewarm(), pg_visibility(), pg_visibility_map(), pgstatindex_impl(), read_buffers(), read_rel_block_ll(), read_stream_for_blocks(), read_stream_reset(), ReadBufferBI(), RelationAddBlocks(), ReleaseBulkInsertStatePin(), revmap_get_buffer(), spgdoinsert(), SpGistGetBuffer(), SpGistNewBuffer(), SpGistUpdateMetaPage(), statapprox_heap(), summarize_range(), terminate_brin_buildstate(), tts_buffer_heap_clear(), tts_buffer_heap_materialize(), tts_buffer_heap_store_tuple(), verify_heapam(), visibilitymap_count(), visibilitymap_get_status(), visibilitymap_pin(), and XLogReadBufferExtended().

◆ ReservePrivateRefCountEntry()

static void ReservePrivateRefCountEntry ( void )

static

Definition at line 309 of file bufmgr.c.

{
    /* Already reserved (or freed), nothing to do */
    if (ReservedRefCountSlot != -1)
        return;
 
    /*
     * First search for a free entry the array, that'll be sufficient in the
     * majority of cases.
     */
    {
        int         i;
 
        for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++)
        {
            if (PrivateRefCountArrayKeys[i] == InvalidBuffer)
            {
                ReservedRefCountSlot = i;
 
                /*
                 * We could return immediately, but iterating till the end of
                 * the array allows compiler-autovectorization.
                 */
            }
        }
 
        if (ReservedRefCountSlot != -1)
            return;
    }
 
    /*
     * No luck. All array entries are full. Move one array entry into the hash
     * table.
     */
    {
        /*
         * Move entry from the current clock position in the array into the
         * hashtable. Use that slot.
         */
        int         victim_slot;
        PrivateRefCountEntry *victim_entry;
        PrivateRefCountEntry *hashent;
        bool        found;
 
        /* select victim slot */
        victim_slot = PrivateRefCountClock++ % REFCOUNT_ARRAY_ENTRIES;
        victim_entry = &PrivateRefCountArray[victim_slot];
        ReservedRefCountSlot = victim_slot;
 
        /* Better be used, otherwise we shouldn't get here. */
        Assert(PrivateRefCountArrayKeys[victim_slot] != InvalidBuffer);
        Assert(PrivateRefCountArray[victim_slot].buffer != InvalidBuffer);
        Assert(PrivateRefCountArrayKeys[victim_slot] == PrivateRefCountArray[victim_slot].buffer);
 
        /* enter victim array entry into hashtable */
        hashent = refcount_insert(PrivateRefCountHash,
                                  PrivateRefCountArrayKeys[victim_slot],
                                  &found);
        Assert(!found);
        /* move data from the entry in the array to the hash entry */
        hashent->data = victim_entry->data;
 
        /* clear the now free array slot */
        PrivateRefCountArrayKeys[victim_slot] = InvalidBuffer;
        victim_entry->buffer = InvalidBuffer;
 
        /* clear the whole data member, just for future proofing */
        memset(&victim_entry->data, 0, sizeof(victim_entry->data));
        victim_entry->data.refcount = 0;
        victim_entry->data.lockmode = BUFFER_LOCK_UNLOCK;
 
        PrivateRefCountOverflowed++;
    }
}

References Assert, PrivateRefCountEntry::buffer, BUFFER_LOCK_UNLOCK, fb(), i, InvalidBuffer, PrivateRefCountArray, PrivateRefCountArrayKeys, PrivateRefCountClock, PrivateRefCountHash, PrivateRefCountOverflowed, REFCOUNT_ARRAY_ENTRIES, and ReservedRefCountSlot.

Referenced by BufferAlloc(), EvictAllUnpinnedBuffers(), EvictRelUnpinnedBuffers(), EvictUnpinnedBuffer(), ExtendBufferedRelShared(), FlushDatabaseBuffers(), FlushRelationBuffers(), FlushRelationsAllBuffers(), GetPrivateRefCountEntrySlow(), GetVictimBuffer(), MarkDirtyAllUnpinnedBuffers(), MarkDirtyRelUnpinnedBuffers(), MarkDirtyUnpinnedBuffer(), ReadRecentBuffer(), and SyncOneBuffer().

◆ ResOwnerPrintBuffer()

static char * ResOwnerPrintBuffer ( Datum res )

static

Definition at line 7881 of file bufmgr.c.

{
    return DebugPrintBufferRefcount(DatumGetInt32(res));
}

References DatumGetInt32(), and DebugPrintBufferRefcount().

◆ ResOwnerPrintBufferIO()

static char * ResOwnerPrintBufferIO ( Datum res )

static

Definition at line 7831 of file bufmgr.c.

{
    Buffer      buffer = DatumGetInt32(res);
 
    return psprintf("lost track of buffer IO on buffer %d", buffer);
}

References PrivateRefCountEntry::buffer, DatumGetInt32(), and psprintf().

◆ ResOwnerReleaseBuffer()

static void ResOwnerReleaseBuffer ( Datum res )

static

Definition at line 7845 of file bufmgr.c.

{
    Buffer      buffer = DatumGetInt32(res);
 
    /* Like ReleaseBuffer, but don't call ResourceOwnerForgetBuffer */
    if (!BufferIsValid(buffer))
        elog(ERROR, "bad buffer ID: %d", buffer);
 
    if (BufferIsLocal(buffer))
        UnpinLocalBufferNoOwner(buffer);
    else
    {
        PrivateRefCountEntry *ref;
 
        ref = GetPrivateRefCountEntry(buffer, false);
 
        /* not having a private refcount would imply resowner corruption */
        Assert(ref != NULL);
 
        /*
         * If the buffer was locked at the time of the resowner release,
         * release the lock now. This should only happen after errors.
         */
        if (ref->data.lockmode != BUFFER_LOCK_UNLOCK)
        {
            BufferDesc *buf = GetBufferDescriptor(buffer - 1);
 
            HOLD_INTERRUPTS();  /* match the upcoming RESUME_INTERRUPTS */
            BufferLockUnlock(buffer, buf);
        }
 
        UnpinBufferNoOwner(GetBufferDescriptor(buffer - 1));
    }
}

References Assert, buf, PrivateRefCountEntry::buffer, BUFFER_LOCK_UNLOCK, BufferIsLocal, BufferIsValid(), BufferLockUnlock(), DatumGetInt32(), elog, ERROR, fb(), GetBufferDescriptor(), GetPrivateRefCountEntry(), HOLD_INTERRUPTS, UnpinBufferNoOwner(), and UnpinLocalBufferNoOwner().

◆ ResOwnerReleaseBufferIO()

static void ResOwnerReleaseBufferIO ( Datum res )

static

Definition at line 7823 of file bufmgr.c.

{
    Buffer      buffer = DatumGetInt32(res);
 
    AbortBufferIO(buffer);
}

References AbortBufferIO(), PrivateRefCountEntry::buffer, and DatumGetInt32().

◆ rlocator_comparator()

static int rlocator_comparator	(	const void *	p1,
		const void *	p2
	)

static

Definition at line 7491 of file bufmgr.c.

{
    RelFileLocator n1 = *(const RelFileLocator *) p1;
    RelFileLocator n2 = *(const RelFileLocator *) p2;
 
    if (n1.relNumber < n2.relNumber)
        return -1;
    else if (n1.relNumber > n2.relNumber)
        return 1;
 
    if (n1.dbOid < n2.dbOid)
        return -1;
    else if (n1.dbOid > n2.dbOid)
        return 1;
 
    if (n1.spcOid < n2.spcOid)
        return -1;
    else if (n1.spcOid > n2.spcOid)
        return 1;
    else
        return 0;
}

References fb().

Referenced by buffertag_comparator(), DropRelationsAllBuffers(), and FlushRelationsAllBuffers().

◆ ScheduleBufferTagForWriteback()

void ScheduleBufferTagForWriteback	(	WritebackContext *	wb_context,
		IOContext	io_context,
		BufferTag *	tag
	)

Definition at line 7690 of file bufmgr.c.

{
    PendingWriteback *pending;
 
    /*
     * As pg_flush_data() doesn't do anything with fsync disabled, there's no
     * point in tracking in that case.
     */
    if (io_direct_flags & IO_DIRECT_DATA ||
        !enableFsync)
        return;
 
    /*
     * Add buffer to the pending writeback array, unless writeback control is
     * disabled.
     */
    if (*wb_context->max_pending > 0)
    {
        Assert(*wb_context->max_pending <= WRITEBACK_MAX_PENDING_FLUSHES);
 
        pending = &wb_context->pending_writebacks[wb_context->nr_pending++];
 
        pending->tag = *tag;
    }
 
    /*
     * Perform pending flushes if the writeback limit is exceeded. This
     * includes the case where previously an item has been added, but control
     * is now disabled.
     */
    if (wb_context->nr_pending >= *wb_context->max_pending)
        IssuePendingWritebacks(wb_context, io_context);
}

References Assert, enableFsync, fb(), IO_DIRECT_DATA, io_direct_flags, IssuePendingWritebacks(), PendingWriteback::tag, and WRITEBACK_MAX_PENDING_FLUSHES.

Referenced by GetVictimBuffer(), and SyncOneBuffer().

◆ shared_buffer_readv_complete()

static PgAioResult shared_buffer_readv_complete	(	PgAioHandle *	ioh,
		PgAioResult	prior_result,
		uint8	cb_data
	)

static

Definition at line 8880 of file bufmgr.c.

{
    return buffer_readv_complete(ioh, prior_result, cb_data, false);
}

References buffer_readv_complete(), and fb().

◆ shared_buffer_readv_complete_local()

static PgAioResult shared_buffer_readv_complete_local	(	PgAioHandle *	ioh,
		PgAioResult	prior_result,
		uint8	cb_data
	)

static

Definition at line 8894 of file bufmgr.c.

{
    bool        zeroed_any,
                ignored_any;
    uint8       zeroed_or_error_count,
                checkfail_count,
                first_off;
 
    if (prior_result.status == PGAIO_RS_OK)
        return prior_result;
 
    buffer_readv_decode_error(prior_result,
                              &zeroed_any,
                              &ignored_any,
                              &zeroed_or_error_count,
                              &checkfail_count,
                              &first_off);
 
    if (checkfail_count)
    {
        PgAioTargetData *td = pgaio_io_get_target_data(ioh);
 
        pgstat_report_checksum_failures_in_db(td->smgr.rlocator.dbOid,
                                              checkfail_count);
    }
 
    return prior_result;
}

References buffer_readv_decode_error(), RelFileLocator::dbOid, fb(), pgaio_io_get_target_data(), PGAIO_RS_OK, pgstat_report_checksum_failures_in_db(), PgAioTargetData::rlocator, and PgAioTargetData::smgr.

◆ shared_buffer_readv_stage()

static void shared_buffer_readv_stage	(	PgAioHandle *	ioh,
		uint8	cb_data
	)

static

Definition at line 8874 of file bufmgr.c.

{
    buffer_stage_common(ioh, false, false);
}

References buffer_stage_common(), and fb().

◆ shared_buffer_write_error_callback()

static void shared_buffer_write_error_callback ( void * arg )

static

Definition at line 7459 of file bufmgr.c.

{
    BufferDesc *bufHdr = (BufferDesc *) arg;
 
    /* Buffer is pinned, so we can read the tag without locking the spinlock */
    if (bufHdr != NULL)
        errcontext("writing block %u of relation \"%s\"",
                   bufHdr->tag.blockNum,
                   relpathperm(BufTagGetRelFileLocator(&bufHdr->tag),
                               BufTagGetForkNum(&bufHdr->tag)).str);
}

References arg, BufTagGetForkNum(), BufTagGetRelFileLocator(), errcontext, fb(), and relpathperm.

Referenced by FlushBuffer().

◆ SharedBufferBeginSetHintBits()

static bool SharedBufferBeginSetHintBits	(	Buffer	buffer,
		BufferDesc *	buf_hdr,
		uint64 *	lockstate
	)

inlinestatic

Definition at line 6951 of file bufmgr.c.

{
    uint64      old_state;
    PrivateRefCountEntry *ref;
    BufferLockMode mode;
 
    ref = GetPrivateRefCountEntry(buffer, true);
 
    if (ref == NULL)
        elog(ERROR, "buffer is not pinned");
 
    mode = ref->data.lockmode;
    if (mode == BUFFER_LOCK_UNLOCK)
        elog(ERROR, "buffer is not locked");
 
    /* we're done if we are already holding a sufficient lock level */
    if (mode == BUFFER_LOCK_EXCLUSIVE || mode == BUFFER_LOCK_SHARE_EXCLUSIVE)
    {
        *lockstate = pg_atomic_read_u64(&buf_hdr->state);
        return true;
    }
 
    /*
     * We are only holding a share lock right now, try to upgrade it to
     * SHARE_EXCLUSIVE.
     */
    Assert(mode == BUFFER_LOCK_SHARE);
 
    old_state = pg_atomic_read_u64(&buf_hdr->state);
    while (true)
    {
        uint64      desired_state;
 
        desired_state = old_state;
 
        /*
         * Can't upgrade if somebody else holds the lock in exclusive or
         * share-exclusive mode.
         */
        if (unlikely((old_state & (BM_LOCK_VAL_EXCLUSIVE | BM_LOCK_VAL_SHARE_EXCLUSIVE)) != 0))
        {
            return false;
        }
 
        /* currently held lock state */
        desired_state -= BM_LOCK_VAL_SHARED;
 
        /* new lock level */
        desired_state += BM_LOCK_VAL_SHARE_EXCLUSIVE;
 
        if (likely(pg_atomic_compare_exchange_u64(&buf_hdr->state,
                                                  &old_state, desired_state)))
        {
            ref->data.lockmode = BUFFER_LOCK_SHARE_EXCLUSIVE;
            *lockstate = desired_state;
 
            return true;
        }
    }
}

References Assert, BM_LOCK_VAL_EXCLUSIVE, BM_LOCK_VAL_SHARE_EXCLUSIVE, BM_LOCK_VAL_SHARED, PrivateRefCountEntry::buffer, BUFFER_LOCK_EXCLUSIVE, BUFFER_LOCK_SHARE, BUFFER_LOCK_SHARE_EXCLUSIVE, BUFFER_LOCK_UNLOCK, elog, ERROR, fb(), GetPrivateRefCountEntry(), likely, mode, pg_atomic_compare_exchange_u64(), pg_atomic_read_u64(), and unlikely.

Referenced by BufferBeginSetHintBits(), and BufferSetHintBits16().

◆ StartBufferIO()

StartBufferIOResult StartBufferIO	(	Buffer	buffer,
		bool	forInput,
		bool	wait,
		PgAioWaitRef *	io_wref
	)

Definition at line 7321 of file bufmgr.c.

{
    BufferDesc *buf_hdr;
 
    if (BufferIsLocal(buffer))
    {
        buf_hdr = GetLocalBufferDescriptor(-buffer - 1);
 
        return StartLocalBufferIO(buf_hdr, forInput, wait, io_wref);
    }
    else
    {
        buf_hdr = GetBufferDescriptor(buffer - 1);
 
        return StartSharedBufferIO(buf_hdr, forInput, wait, io_wref);
    }
}

References PrivateRefCountEntry::buffer, BufferIsLocal, fb(), GetBufferDescriptor(), GetLocalBufferDescriptor(), StartLocalBufferIO(), and StartSharedBufferIO().

Referenced by AsyncReadBuffers().

◆ StartReadBuffer()

bool StartReadBuffer	(	ReadBuffersOperation *	operation,
		Buffer *	buffer,
		BlockNumber	blocknum,
		int	flags
	)

Definition at line 1628 of file bufmgr.c.

{
    int         nblocks = 1;
    bool        result;
 
    result = StartReadBuffersImpl(operation, buffer, blocknum, &nblocks, flags,
                                  false /* single block, no forwarding */ );
    Assert(nblocks == 1);       /* single block can't be short */
 
    return result;
}

References Assert, PrivateRefCountEntry::buffer, operation, result, and StartReadBuffersImpl().

Referenced by read_stream_next_buffer(), and ReadBuffer_common().

◆ StartReadBuffers()

bool StartReadBuffers	(	ReadBuffersOperation *	operation,
		Buffer *	buffers,
		BlockNumber	blockNum,
		int *	nblocks,
		int	flags
	)

Definition at line 1609 of file bufmgr.c.

{
    return StartReadBuffersImpl(operation, buffers, blockNum, nblocks, flags,
                                true /* expect forwarded buffers */ );
}

References operation, and StartReadBuffersImpl().

Referenced by read_buffers(), and read_stream_start_pending_read().

◆ StartReadBuffersImpl()

static pg_attribute_always_inline bool StartReadBuffersImpl	(	ReadBuffersOperation *	operation,
		Buffer *	buffers,
		BlockNumber	blockNum,
		int *	nblocks,
		int	flags,
		bool	allow_forwarding
	)

static

Definition at line 1368 of file bufmgr.c.

{
    int         actual_nblocks = *nblocks;
    int         maxcombine = 0;
    bool        did_start_io;
    IOContext   io_context;
    IOObject    io_object;
 
    Assert(*nblocks == 1 || allow_forwarding);
    Assert(*nblocks > 0);
    Assert(*nblocks <= MAX_IO_COMBINE_LIMIT);
 
    if (operation->persistence == RELPERSISTENCE_TEMP)
    {
        io_context = IOCONTEXT_NORMAL;
        io_object = IOOBJECT_TEMP_RELATION;
    }
    else
    {
        io_context = IOContextForStrategy(operation->strategy);
        io_object = IOOBJECT_RELATION;
    }
 
    for (int i = 0; i < actual_nblocks; ++i)
    {
        bool        found;
 
        if (allow_forwarding && buffers[i] != InvalidBuffer)
        {
            BufferDesc *bufHdr;
 
            /*
             * This is a buffer that was pinned by an earlier call to
             * StartReadBuffers(), but couldn't be handled in one operation at
             * that time.  The operation was split, and the caller has passed
             * an already pinned buffer back to us to handle the rest of the
             * operation.  It must continue at the expected block number.
             */
            Assert(BufferGetBlockNumber(buffers[i]) == blockNum + i);
 
            /*
             * It might be an already valid buffer (a hit) that followed the
             * final contiguous block of an earlier I/O (a miss) marking the
             * end of it, or a buffer that some other backend has since made
             * valid by performing the I/O for us, in which case we can handle
             * it as a hit now.  It is safe to check for a BM_VALID flag with
             * a relaxed load, because we got a fresh view of it while pinning
             * it in the previous call.
             *
             * On the other hand if we don't see BM_VALID yet, it must be an
             * I/O that was split by the previous call and we need to try to
             * start a new I/O from this block.  We're also racing against any
             * other backend that might start the I/O or even manage to mark
             * it BM_VALID after this check, but StartBufferIO() will handle
             * those cases.
             */
            if (BufferIsLocal(buffers[i]))
                bufHdr = GetLocalBufferDescriptor(-buffers[i] - 1);
            else
                bufHdr = GetBufferDescriptor(buffers[i] - 1);
            Assert(pg_atomic_read_u64(&bufHdr->state) & BM_TAG_VALID);
            found = pg_atomic_read_u64(&bufHdr->state) & BM_VALID;
        }
        else
        {
            buffers[i] = PinBufferForBlock(operation->rel,
                                           operation->smgr,
                                           operation->persistence,
                                           operation->forknum,
                                           blockNum + i,
                                           operation->strategy,
                                           io_object, io_context,
                                           &found);
        }
 
        if (found)
        {
            /*
             * We have a hit.  If it's the first block in the requested range,
             * we can return it immediately and report that WaitReadBuffers()
             * does not need to be called.  If the initial value of *nblocks
             * was larger, the caller will have to call again for the rest.
             */
            if (i == 0)
            {
                *nblocks = 1;
 
#ifdef USE_ASSERT_CHECKING
 
                /*
                 * Initialize enough of ReadBuffersOperation to make
                 * CheckReadBuffersOperation() work. Outside of assertions
                 * that's not necessary when no IO is issued.
                 */
                operation->buffers = buffers;
                operation->blocknum = blockNum;
                operation->nblocks = 1;
                operation->nblocks_done = 1;
                CheckReadBuffersOperation(operation, true);
#endif
                return false;
            }
 
            /*
             * Otherwise we already have an I/O to perform, but this block
             * can't be included as it is already valid.  Split the I/O here.
             * There may or may not be more blocks requiring I/O after this
             * one, we haven't checked, but they can't be contiguous with this
             * one in the way.  We'll leave this buffer pinned, forwarding it
             * to the next call, avoiding the need to unpin it here and re-pin
             * it in the next call.
             */
            actual_nblocks = i;
            break;
        }
        else
        {
            /*
             * Check how many blocks we can cover with the same IO. The smgr
             * implementation might e.g. be limited due to a segment boundary.
             */
            if (i == 0 && actual_nblocks > 1)
            {
                maxcombine = smgrmaxcombine(operation->smgr,
                                            operation->forknum,
                                            blockNum);
                if (unlikely(maxcombine < actual_nblocks))
                {
                    elog(DEBUG2, "limiting nblocks at %u from %u to %u",
                         blockNum, actual_nblocks, maxcombine);
                    actual_nblocks = maxcombine;
                }
            }
        }
    }
    *nblocks = actual_nblocks;
 
    /* Populate information needed for I/O. */
    operation->buffers = buffers;
    operation->blocknum = blockNum;
    operation->flags = flags;
    operation->nblocks = actual_nblocks;
    operation->nblocks_done = 0;
    pgaio_wref_clear(&operation->io_wref);
 
    /*
     * When using AIO, start the IO in the background. If not, issue prefetch
     * requests if desired by the caller.
     *
     * The reason we have a dedicated path for IOMETHOD_SYNC here is to
     * de-risk the introduction of AIO somewhat. It's a large architectural
     * change, with lots of chances for unanticipated performance effects.
     *
     * Use of IOMETHOD_SYNC already leads to not actually performing IO
     * asynchronously, but without the check here we'd execute IO earlier than
     * we used to. Eventually this IOMETHOD_SYNC specific path should go away.
     */
    if (io_method != IOMETHOD_SYNC)
    {
        /*
         * Try to start IO asynchronously. It's possible that no IO needs to
         * be started, if another backend already performed the IO.
         *
         * Note that if an IO is started, it might not cover the entire
         * requested range, e.g. because an intermediary block has been read
         * in by another backend.  In that case any "trailing" buffers we
         * already pinned above will be "forwarded" by read_stream.c to the
         * next call to StartReadBuffers().
         *
         * This is signalled to the caller by decrementing *nblocks *and*
         * reducing operation->nblocks. The latter is done here, but not below
         * WaitReadBuffers(), as in WaitReadBuffers() we can't "shorten" the
         * overall read size anymore, we need to retry until done in its
         * entirety or until failed.
         */
        did_start_io = AsyncReadBuffers(operation, nblocks);
 
        operation->nblocks = *nblocks;
    }
    else
    {
        operation->flags |= READ_BUFFERS_SYNCHRONOUSLY;
 
        if (flags & READ_BUFFERS_ISSUE_ADVICE)
        {
            /*
             * In theory we should only do this if PinBufferForBlock() had to
             * allocate new buffers above.  That way, if two calls to
             * StartReadBuffers() were made for the same blocks before
             * WaitReadBuffers(), only the first would issue the advice.
             * That'd be a better simulation of true asynchronous I/O, which
             * would only start the I/O once, but isn't done here for
             * simplicity.
             */
            smgrprefetch(operation->smgr,
                         operation->forknum,
                         blockNum,
                         actual_nblocks);
        }
 
        /*
         * Indicate that WaitReadBuffers() should be called. WaitReadBuffers()
         * will initiate the necessary IO.
         */
        did_start_io = true;
    }
 
    CheckReadBuffersOperation(operation, !did_start_io);
 
    return did_start_io;
}

References Assert, AsyncReadBuffers(), BM_TAG_VALID, BM_VALID, BufferGetBlockNumber(), BufferIsLocal, CheckReadBuffersOperation(), DEBUG2, elog, fb(), GetBufferDescriptor(), GetLocalBufferDescriptor(), i, InvalidBuffer, io_method, IOCONTEXT_NORMAL, IOContextForStrategy(), IOMETHOD_SYNC, IOOBJECT_RELATION, IOOBJECT_TEMP_RELATION, MAX_IO_COMBINE_LIMIT, operation, pg_atomic_read_u64(), pgaio_wref_clear(), PinBufferForBlock(), READ_BUFFERS_ISSUE_ADVICE, READ_BUFFERS_SYNCHRONOUSLY, smgrmaxcombine(), smgrprefetch(), and unlikely.

Referenced by StartReadBuffer(), and StartReadBuffers().

◆ StartSharedBufferIO()

StartBufferIOResult StartSharedBufferIO	(	BufferDesc *	buf,
		bool	forInput,
		bool	wait,
		PgAioWaitRef *	io_wref
	)

Definition at line 7241 of file bufmgr.c.

{
    uint64      buf_state;
 
    ResourceOwnerEnlarge(CurrentResourceOwner);
 
    for (;;)
    {
        buf_state = LockBufHdr(buf);
 
        if (!(buf_state & BM_IO_IN_PROGRESS))
            break;
 
        /* Join the existing IO */
        if (io_wref != NULL && pgaio_wref_valid(&buf->io_wref))
        {
            *io_wref = buf->io_wref;
            UnlockBufHdr(buf);
 
            return BUFFER_IO_IN_PROGRESS;
        }
        else if (!wait)
        {
            UnlockBufHdr(buf);
            return BUFFER_IO_IN_PROGRESS;
        }
        else
        {
            /*
             * With wait = true, we always have to wait if the caller has
             * passed io_wref = NULL.
             *
             * Even with io_wref != NULL, we have to wait if the buffer's wait
             * ref is not valid but the IO is in progress, someone else
             * started IO but hasn't set the wait ref yet. We have no choice
             * but to wait until the IO completes.
             */
            UnlockBufHdr(buf);
 
            /*
             * If this backend currently has staged IO, submit it before
             * waiting for in-progress IO, to avoid potential deadlocks and
             * unnecessary delays.
             */
            pgaio_submit_staged();
 
            WaitIO(buf);
        }
    }
 
    /* Once we get here, there is definitely no I/O active on this buffer */
 
    /* Check if someone else already did the I/O */
    if (forInput ? (buf_state & BM_VALID) : !(buf_state & BM_DIRTY))
    {
        UnlockBufHdr(buf);
        return BUFFER_IO_ALREADY_DONE;
    }
 
    /*
     * No IO in progress and not already done; we will start IO. It's possible
     * that the IO was in progress but we're not done, because the IO errored
     * out. We'll do the IO ourselves.
     */
    UnlockBufHdrExt(buf, buf_state,
                    BM_IO_IN_PROGRESS, 0,
                    0);
 
    ResourceOwnerRememberBufferIO(CurrentResourceOwner,
                                  BufferDescriptorGetBuffer(buf));
 
    return BUFFER_IO_READY_FOR_IO;
}

References BM_DIRTY, BM_IO_IN_PROGRESS, BM_VALID, buf, BUFFER_IO_ALREADY_DONE, BUFFER_IO_IN_PROGRESS, BUFFER_IO_READY_FOR_IO, BufferDescriptorGetBuffer(), CurrentResourceOwner, fb(), LockBufHdr(), pgaio_submit_staged(), pgaio_wref_valid(), ResourceOwnerEnlarge(), ResourceOwnerRememberBufferIO(), UnlockBufHdr(), UnlockBufHdrExt(), and WaitIO().

Referenced by buffer_call_start_io(), ExtendBufferedRelShared(), FlushBuffer(), read_rel_block_ll(), StartBufferIO(), and ZeroAndLockBuffer().

◆ SyncOneBuffer()

static int SyncOneBuffer	(	int	buf_id,
		bool	skip_recently_used,
		WritebackContext *	wb_context
	)

static

Definition at line 4129 of file bufmgr.c.

{
    BufferDesc *bufHdr = GetBufferDescriptor(buf_id);
    int         result = 0;
    uint64      buf_state;
    BufferTag   tag;
 
    /* Make sure we can handle the pin */
    ReservePrivateRefCountEntry();
    ResourceOwnerEnlarge(CurrentResourceOwner);
 
    /*
     * Check whether buffer needs writing.
     *
     * We can make this check without taking the buffer content lock so long
     * as we mark pages dirty in access methods *before* logging changes with
     * XLogInsert(): if someone marks the buffer dirty just after our check we
     * don't worry because our checkpoint.redo points before log record for
     * upcoming changes and so we are not required to write such dirty buffer.
     */
    buf_state = LockBufHdr(bufHdr);
 
    if (BUF_STATE_GET_REFCOUNT(buf_state) == 0 &&
        BUF_STATE_GET_USAGECOUNT(buf_state) == 0)
    {
        result |= BUF_REUSABLE;
    }
    else if (skip_recently_used)
    {
        /* Caller told us not to write recently-used buffers */
        UnlockBufHdr(bufHdr);
        return result;
    }
 
    if (!(buf_state & BM_VALID) || !(buf_state & BM_DIRTY))
    {
        /* It's clean, so nothing to do */
        UnlockBufHdr(bufHdr);
        return result;
    }
 
    /*
     * Pin it, share-exclusive-lock it, write it.  (FlushBuffer will do
     * nothing if the buffer is clean by the time we've locked it.)
     */
    PinBuffer_Locked(bufHdr);
 
    FlushUnlockedBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
 
    tag = bufHdr->tag;
 
    UnpinBuffer(bufHdr);
 
    /*
     * SyncOneBuffer() is only called by checkpointer and bgwriter, so
     * IOContext will always be IOCONTEXT_NORMAL.
     */
    ScheduleBufferTagForWriteback(wb_context, IOCONTEXT_NORMAL, &tag);
 
    return result | BUF_WRITTEN;
}

References BM_DIRTY, BM_VALID, BUF_REUSABLE, BUF_STATE_GET_REFCOUNT, BUF_STATE_GET_USAGECOUNT, BUF_WRITTEN, CurrentResourceOwner, fb(), FlushUnlockedBuffer(), GetBufferDescriptor(), IOCONTEXT_NORMAL, IOOBJECT_RELATION, LockBufHdr(), PinBuffer_Locked(), ReservePrivateRefCountEntry(), ResourceOwnerEnlarge(), result, ScheduleBufferTagForWriteback(), UnlockBufHdr(), and UnpinBuffer().

Referenced by BgBufferSync(), and BufferSync().

◆ TerminateBufferIO()

void TerminateBufferIO	(	BufferDesc *	buf,
		bool	clear_dirty,
		uint64	set_flag_bits,
		bool	forget_owner,
		bool	release_aio
	)

Definition at line 7358 of file bufmgr.c.

{
    uint64      buf_state;
    uint64      unset_flag_bits = 0;
    int         refcount_change = 0;
 
    buf_state = LockBufHdr(buf);
 
    Assert(buf_state & BM_IO_IN_PROGRESS);
    unset_flag_bits |= BM_IO_IN_PROGRESS;
 
    /* Clear earlier errors, if this IO failed, it'll be marked again */
    unset_flag_bits |= BM_IO_ERROR;
 
    if (clear_dirty)
        unset_flag_bits |= BM_DIRTY | BM_CHECKPOINT_NEEDED;
 
    if (release_aio)
    {
        /* release ownership by the AIO subsystem */
        Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
        refcount_change = -1;
        pgaio_wref_clear(&buf->io_wref);
    }
 
    buf_state = UnlockBufHdrExt(buf, buf_state,
                                set_flag_bits, unset_flag_bits,
                                refcount_change);
 
    if (forget_owner)
        ResourceOwnerForgetBufferIO(CurrentResourceOwner,
                                    BufferDescriptorGetBuffer(buf));
 
    ConditionVariableBroadcast(BufferDescriptorGetIOCV(buf));
 
    /*
     * Support LockBufferForCleanup()
     *
     * We may have just released the last pin other than the waiter's. In most
     * cases, this backend holds another pin on the buffer. But, if, for
     * example, this backend is completing an IO issued by another backend, it
     * may be time to wake the waiter.
     */
    if (release_aio && (buf_state & BM_PIN_COUNT_WAITER))
        WakePinCountWaiter(buf);
}

References Assert, BM_CHECKPOINT_NEEDED, BM_DIRTY, BM_IO_ERROR, BM_IO_IN_PROGRESS, BM_PIN_COUNT_WAITER, buf, BUF_STATE_GET_REFCOUNT, BufferDescriptorGetBuffer(), BufferDescriptorGetIOCV(), ConditionVariableBroadcast(), CurrentResourceOwner, fb(), LockBufHdr(), pgaio_wref_clear(), ResourceOwnerForgetBufferIO(), UnlockBufHdrExt(), and WakePinCountWaiter().

Referenced by AbortBufferIO(), buffer_call_terminate_io(), buffer_readv_complete_one(), ExtendBufferedRelShared(), FlushBuffer(), and ZeroAndLockBuffer().

◆ TrackBufferHit()

static pg_attribute_always_inline void TrackBufferHit	(	IOObject	io_object,
		IOContext	io_context,
		Relation	rel,
		char	persistence,
		SMgrRelation	smgr,
		ForkNumber	forknum,
		BlockNumber	blocknum
	)

static

Definition at line 1674 of file bufmgr.c.

{
    TRACE_POSTGRESQL_BUFFER_READ_DONE(forknum,
                                      blocknum,
                                      smgr->smgr_rlocator.locator.spcOid,
                                      smgr->smgr_rlocator.locator.dbOid,
                                      smgr->smgr_rlocator.locator.relNumber,
                                      smgr->smgr_rlocator.backend,
                                      true);
 
    if (persistence == RELPERSISTENCE_TEMP)
        pgBufferUsage.local_blks_hit += 1;
    else
        pgBufferUsage.shared_blks_hit += 1;
 
    pgstat_count_io_op(io_object, io_context, IOOP_HIT, 1, 0);
 
    if (VacuumCostActive)
        VacuumCostBalance += VacuumCostPageHit;
 
    if (rel)
        pgstat_count_buffer_hit(rel);
}

References RelFileLocatorBackend::backend, RelFileLocator::dbOid, fb(), IOOP_HIT, BufferUsage::local_blks_hit, RelFileLocatorBackend::locator, pgBufferUsage, pgstat_count_buffer_hit, pgstat_count_io_op(), RelFileLocator::relNumber, BufferUsage::shared_blks_hit, SMgrRelationData::smgr_rlocator, RelFileLocator::spcOid, VacuumCostActive, VacuumCostBalance, and VacuumCostPageHit.

Referenced by AsyncReadBuffers(), PinBufferForBlock(), and WaitReadBuffers().

◆ TrackNewBufferPin()

void TrackNewBufferPin ( Buffer buf )

inline

Definition at line 3512 of file bufmgr.c.

{
    PrivateRefCountEntry *ref;
 
    ref = NewPrivateRefCountEntry(buf);
    ref->data.refcount++;
 
    ResourceOwnerRememberBuffer(CurrentResourceOwner, buf);
 
    /*
     * This is the first pin for this page by this backend, mark its page as
     * defined to valgrind. While the page contents might not actually be
     * valid yet, we don't currently guarantee that such pages are marked
     * undefined or non-accessible.
     *
     * It's not necessarily the prettiest to do this here, but otherwise we'd
     * need this block of code in multiple places.
     */
    VALGRIND_MAKE_MEM_DEFINED(BufHdrGetBlock(GetBufferDescriptor(buf - 1)),
                              BLCKSZ);
}

References buf, BufHdrGetBlock, CurrentResourceOwner, fb(), GetBufferDescriptor(), NewPrivateRefCountEntry(), ResourceOwnerRememberBuffer(), and VALGRIND_MAKE_MEM_DEFINED.

Referenced by GetBufferFromRing(), PinBuffer(), PinBuffer_Locked(), and StrategyGetBuffer().

◆ ts_ckpt_progress_comparator()

static int ts_ckpt_progress_comparator	(	Datum	a,
		Datum	b,
		void *	arg
	)

static

Definition at line 7655 of file bufmgr.c.

{
    CkptTsStatus *sa = (CkptTsStatus *) DatumGetPointer(a);
    CkptTsStatus *sb = (CkptTsStatus *) DatumGetPointer(b);
 
    /* we want a min-heap, so return 1 for the a < b */
    if (sa->progress < sb->progress)
        return 1;
    else if (sa->progress == sb->progress)
        return 0;
    else
        return -1;
}

References a, b, DatumGetPointer(), and fb().

Referenced by BufferSync().

◆ UnlockBuffer()

void UnlockBuffer ( Buffer buffer )

Definition at line 6558 of file bufmgr.c.

{
    BufferDesc *buf_hdr;
 
    Assert(BufferIsPinned(buffer));
    if (BufferIsLocal(buffer))
        return;                 /* local buffers need no lock */
 
    buf_hdr = GetBufferDescriptor(buffer - 1);
    BufferLockUnlock(buffer, buf_hdr);
}

References Assert, PrivateRefCountEntry::buffer, BufferIsLocal, BufferIsPinned, BufferLockUnlock(), fb(), and GetBufferDescriptor().

Referenced by LockBuffer().

◆ UnlockBuffers()

void UnlockBuffers ( void )

Definition at line 5852 of file bufmgr.c.

{
    BufferDesc *buf = PinCountWaitBuf;
 
    if (buf)
    {
        uint64      buf_state;
        uint64      unset_bits = 0;
 
        buf_state = LockBufHdr(buf);
 
        /*
         * Don't complain if flag bit not set; it could have been reset but we
         * got a cancel/die interrupt before getting the signal.
         */
        if ((buf_state & BM_PIN_COUNT_WAITER) != 0 &&
            buf->wait_backend_pgprocno == MyProcNumber)
            unset_bits = BM_PIN_COUNT_WAITER;
 
        UnlockBufHdrExt(buf, buf_state,
                        0, unset_bits,
                        0);
 
        PinCountWaitBuf = NULL;
    }
}

References BM_PIN_COUNT_WAITER, buf, fb(), LockBufHdr(), MyProcNumber, PinCountWaitBuf, and UnlockBufHdrExt().

Referenced by AbortSubTransaction(), AbortTransaction(), AtProcExit_Buffers(), AutoVacLauncherMain(), BackgroundWriterMain(), CheckpointerMain(), and WalWriterMain().

◆ UnlockReleaseBuffer()

void UnlockReleaseBuffer ( Buffer buffer )

Definition at line 5603 of file bufmgr.c.

{
    int         mode;
    BufferDesc *buf;
    PrivateRefCountEntry *ref;
    uint64      sub;
    uint64      lockstate;
 
    Assert(BufferIsPinned(buffer));
 
    if (BufferIsLocal(buffer))
    {
        UnpinLocalBuffer(buffer);
        return;
    }
 
    ResourceOwnerForgetBuffer(CurrentResourceOwner, buffer);
 
    buf = GetBufferDescriptor(buffer - 1);
 
    mode = BufferLockDisownInternal(buffer, buf);
 
    /* compute state modification for lock release */
    sub = BufferLockReleaseSub(mode);
 
    /* compute state modification for pin release */
    ref = GetPrivateRefCountEntry(buffer, false);
    Assert(ref != NULL);
    Assert(ref->data.refcount > 0);
    ref->data.refcount--;
 
    /* no more backend local pins, reduce shared pin count */
    if (likely(ref->data.refcount == 0))
    {
        /* See comment in UnpinBufferNoOwner() */
        VALGRIND_MAKE_MEM_NOACCESS(BufHdrGetBlock(buf), BLCKSZ);
 
        sub |= BUF_REFCOUNT_ONE;
        ForgetPrivateRefCountEntry(ref);
    }
 
    /* perform the lock and pin release in one atomic op */
    lockstate = pg_atomic_sub_fetch_u64(&buf->state, sub);
 
    /* wake up waiters for the lock */
    BufferLockProcessRelease(buf, mode, lockstate);
 
    /* wake up waiter for the pin release */
    if (lockstate & BM_PIN_COUNT_WAITER)
        WakePinCountWaiter(buf);
 
    /*
     * Now okay to allow cancel/die interrupts again, which were held when the
     * lock was acquired.
     */
    RESUME_INTERRUPTS();
}

References Assert, BM_PIN_COUNT_WAITER, buf, BUF_REFCOUNT_ONE, PrivateRefCountEntry::buffer, BufferIsLocal, BufferIsPinned, BufferLockDisownInternal(), BufferLockProcessRelease(), BufferLockReleaseSub(), BufHdrGetBlock, CurrentResourceOwner, fb(), ForgetPrivateRefCountEntry(), GetBufferDescriptor(), GetPrivateRefCountEntry(), likely, mode, pg_atomic_sub_fetch_u64(), ResourceOwnerForgetBuffer(), RESUME_INTERRUPTS, UnpinLocalBuffer(), VALGRIND_MAKE_MEM_NOACCESS, and WakePinCountWaiter().

Referenced by _bt_clear_incomplete_split(), _bt_relbuf(), _bt_restore_meta(), _hash_relbuf(), allocNewBuffer(), AlterSequence(), blbulkdelete(), blgetbitmap(), blinsert(), BloomInitMetapage(), blvacuumcleanup(), brin_doinsert(), brin_doupdate(), brin_evacuate_page(), brin_getinsertbuffer(), brin_xlog_createidx(), brin_xlog_desummarize_page(), brin_xlog_insert_update(), brin_xlog_revmap_extend(), brin_xlog_samepage_update(), brin_xlog_update(), brinbuild(), brinbuildempty(), brinGetStats(), brinRevmapDesummarizeRange(), bt_metap(), bt_multi_page_stats(), bt_page_items_internal(), bt_page_stats_internal(), bt_recheck_sibling_links(), btree_xlog_dedup(), btree_xlog_delete(), btree_xlog_insert(), btree_xlog_mark_page_halfdead(), btree_xlog_newroot(), btree_xlog_split(), btree_xlog_unlink_page(), btree_xlog_vacuum(), collect_corrupt_items(), collect_visibility_data(), count_nondeletable_pages(), createPostingTree(), doPickSplit(), entryLoadMoreItems(), fill_seq_fork_with_data(), flushCachedPage(), FreeSpaceMapPrepareTruncateRel(), fsm_search(), fsm_set_and_search(), fsm_vacuum_page(), generic_redo(), get_raw_page_internal(), GetVictimBuffer(), gin_check_parent_keys_consistency(), gin_check_posting_tree_parent_keys_consistency(), gin_refind_parent(), ginbuild(), ginbuildempty(), ginbulkdelete(), ginGetStats(), ginHeapTupleFastInsert(), ginInsertCleanup(), ginPlaceToPage(), ginRedoClearIncompleteSplit(), ginRedoCreatePTree(), ginRedoDeleteListPages(), ginRedoDeletePage(), ginRedoInsert(), ginRedoInsertListPage(), ginRedoSplit(), ginRedoUpdateMetapage(), ginRedoVacuumDataLeafPage(), ginRedoVacuumPage(), ginScanPostingTreeToDelete(), ginStepRight(), ginUpdateStats(), ginvacuumcleanup(), ginVacuumPostingTree(), ginVacuumPostingTreeLeaves(), gistbufferinginserttuples(), gistbuild(), gistbuildempty(), gistdoinsert(), gistFindCorrectParent(), gistFindPath(), gistGetMaxLevel(), gistinserttuples(), gistkillitems(), gistplacetopage(), gistProcessItup(), gistRedoClearFollowRight(), gistRedoDeleteRecord(), gistRedoPageDelete(), gistRedoPageSplitRecord(), gistRedoPageUpdateRecord(), gistScanPage(), gistvacuum_delete_empty_pages(), gistvacuumpage(), hash_xlog_add_ovfl_page(), hash_xlog_delete(), hash_xlog_init_bitmap_page(), hash_xlog_init_meta_page(), hash_xlog_insert(), hash_xlog_move_page_contents(), hash_xlog_split_allocate_page(), hash_xlog_split_cleanup(), hash_xlog_split_complete(), hash_xlog_split_page(), hash_xlog_squeeze_page(), hash_xlog_update_meta_page(), hash_xlog_vacuum_one_page(), heap_delete(), heap_fetch(), heap_finish_speculative(), heap_force_common(), heap_get_latest_tid(), heap_index_delete_tuples(), heap_insert(), heap_lock_updated_tuple_rec(), heap_multi_insert(), heap_update(), heap_xlog_confirm(), heap_xlog_delete(), heap_xlog_inplace(), heap_xlog_insert(), heap_xlog_lock(), heap_xlog_lock_updated(), heap_xlog_multi_insert(), heap_xlog_prune_freeze(), heap_xlog_update(), heapam_scan_analyze_next_tuple(), initBloomState(), invalidate_one_block(), lazy_scan_heap(), lazy_scan_new_or_empty(), lazy_vacuum_heap_rel(), log_newpage_range(), modify_rel_block(), moveLeafs(), nextval_internal(), palloc_btree_page(), pg_get_sequence_data(), pg_sequence_last_value(), pg_visibility(), pgstat_gist_page(), pgstat_heap(), pgstatginindex_internal(), pgstathashindex(), pgstatindex_impl(), ProcessSingleRelationFork(), RelationCopyStorageUsingBuffer(), RelationGetBufferForTuple(), ResetSequence(), revmap_physical_extend(), scanGetCandidate(), scanPendingInsert(), scanPostingTree(), ScanSourceDatabasePgClass(), seq_redo(), SequenceChangePersistence(), SetSequence(), shiftList(), spgAddNodeAction(), spgbuild(), spgdoinsert(), spgGetCache(), SpGistGetBuffer(), SpGistUpdateMetaPage(), spgMatchNodeAction(), spgprocesspending(), spgRedoAddLeaf(), spgRedoAddNode(), spgRedoMoveLeafs(), spgRedoPickSplit(), spgRedoSplitTuple(), spgRedoVacuumLeaf(), spgRedoVacuumRedirect(), spgRedoVacuumRoot(), spgSplitNodeAction(), spgvacuumpage(), spgWalk(), statapprox_heap(), verify_heapam(), verifyBackupPageConsistency(), visibilitymap_prepare_truncate(), writeListPage(), xlog_redo(), and XLogRecordPageWithFreeSpace().

◆ UnpinBuffer()

static void UnpinBuffer ( BufferDesc * buf )

static

Definition at line 3456 of file bufmgr.c.

{
    Buffer      b = BufferDescriptorGetBuffer(buf);
 
    ResourceOwnerForgetBuffer(CurrentResourceOwner, b);
    UnpinBufferNoOwner(buf);
}

References b, buf, BufferDescriptorGetBuffer(), CurrentResourceOwner, ResourceOwnerForgetBuffer(), and UnpinBufferNoOwner().

Referenced by BufferAlloc(), EvictUnpinnedBufferInternal(), ExtendBufferedRelShared(), FlushDatabaseBuffers(), FlushRelationBuffers(), FlushRelationsAllBuffers(), GetVictimBuffer(), MarkDirtyUnpinnedBufferInternal(), ReadRecentBuffer(), ReleaseAndReadBuffer(), ReleaseBuffer(), and SyncOneBuffer().

◆ UnpinBufferNoOwner()

static void UnpinBufferNoOwner ( BufferDesc * buf )

static

Definition at line 3465 of file bufmgr.c.

{
    PrivateRefCountEntry *ref;
    Buffer      b = BufferDescriptorGetBuffer(buf);
 
    Assert(!BufferIsLocal(b));
 
    /* not moving as we're likely deleting it soon anyway */
    ref = GetPrivateRefCountEntry(b, false);
    Assert(ref != NULL);
    Assert(ref->data.refcount > 0);
    ref->data.refcount--;
    if (ref->data.refcount == 0)
    {
        uint64      old_buf_state;
 
        /*
         * Mark buffer non-accessible to Valgrind.
         *
         * Note that the buffer may have already been marked non-accessible
         * within access method code that enforces that buffers are only
         * accessed while a buffer lock is held.
         */
        VALGRIND_MAKE_MEM_NOACCESS(BufHdrGetBlock(buf), BLCKSZ);
 
        /*
         * I'd better not still hold the buffer content lock. Can't use
         * BufferIsLockedByMe(), as that asserts the buffer is pinned.
         */
        Assert(!BufferLockHeldByMe(buf));
 
        /* decrement the shared reference count */
        old_buf_state = pg_atomic_fetch_sub_u64(&buf->state, BUF_REFCOUNT_ONE);
 
        /* Support LockBufferForCleanup() */
        if (old_buf_state & BM_PIN_COUNT_WAITER)
            WakePinCountWaiter(buf);
 
        ForgetPrivateRefCountEntry(ref);
    }
}

References Assert, b, BM_PIN_COUNT_WAITER, buf, BUF_REFCOUNT_ONE, BufferDescriptorGetBuffer(), BufferIsLocal, BufferLockHeldByMe(), BufHdrGetBlock, fb(), ForgetPrivateRefCountEntry(), GetPrivateRefCountEntry(), pg_atomic_fetch_sub_u64(), VALGRIND_MAKE_MEM_NOACCESS, and WakePinCountWaiter().

Referenced by ResOwnerReleaseBuffer(), and UnpinBuffer().

◆ WaitBufHdrUnlocked()

pg_noinline uint64 WaitBufHdrUnlocked ( BufferDesc * buf )

Definition at line 7566 of file bufmgr.c.

{
    SpinDelayStatus delayStatus;
    uint64      buf_state;
 
    init_local_spin_delay(&delayStatus);
 
    buf_state = pg_atomic_read_u64(&buf->state);
 
    while (buf_state & BM_LOCKED)
    {
        perform_spin_delay(&delayStatus);
        buf_state = pg_atomic_read_u64(&buf->state);
    }
 
    finish_spin_delay(&delayStatus);
 
    return buf_state;
}

References BM_LOCKED, buf, fb(), finish_spin_delay(), init_local_spin_delay, perform_spin_delay(), and pg_atomic_read_u64().

Referenced by GetBufferFromRing(), MarkBufferDirty(), PinBuffer(), and StrategyGetBuffer().

◆ WaitIO()

static void WaitIO ( BufferDesc * buf )

static

Definition at line 7139 of file bufmgr.c.

{
    ConditionVariable *cv = BufferDescriptorGetIOCV(buf);
 
    /*
     * Should never end up here with unsubmitted IO, as no AIO unaware code
     * may be used while in batch mode and AIO aware code needs to have
     * submitted all staged IO to avoid deadlocks & slowness.
     */
    Assert(!pgaio_have_staged());
 
    ConditionVariablePrepareToSleep(cv);
    for (;;)
    {
        uint64      buf_state;
        PgAioWaitRef iow;
 
        /*
         * It may not be necessary to acquire the spinlock to check the flag
         * here, but since this test is essential for correctness, we'd better
         * play it safe.
         */
        buf_state = LockBufHdr(buf);
 
        /*
         * Copy the wait reference while holding the spinlock. This protects
         * against a concurrent TerminateBufferIO() in another backend from
         * clearing the wref while it's being read.
         */
        iow = buf->io_wref;
        UnlockBufHdr(buf);
 
        /* no IO in progress, we don't need to wait */
        if (!(buf_state & BM_IO_IN_PROGRESS))
            break;
 
        /*
         * The buffer has asynchronous IO in progress, wait for it to
         * complete.
         */
        if (pgaio_wref_valid(&iow))
        {
            pgaio_wref_wait(&iow);
 
            /*
             * The AIO subsystem internally uses condition variables and thus
             * might remove this backend from the BufferDesc's CV. While that
             * wouldn't cause a correctness issue (the first CV sleep just
             * immediately returns if not already registered), it seems worth
             * avoiding unnecessary loop iterations, given that we take care
             * to do so at the start of the function.
             */
            ConditionVariablePrepareToSleep(cv);
            continue;
        }
 
        /* wait on BufferDesc->cv, e.g. for concurrent synchronous IO */
        ConditionVariableSleep(cv, WAIT_EVENT_BUFFER_IO);
    }
    ConditionVariableCancelSleep();
}

References Assert, BM_IO_IN_PROGRESS, buf, BufferDescriptorGetIOCV(), ConditionVariableCancelSleep(), ConditionVariablePrepareToSleep(), ConditionVariableSleep(), fb(), LockBufHdr(), pgaio_have_staged(), pgaio_wref_valid(), pgaio_wref_wait(), and UnlockBufHdr().

Referenced by InvalidateBuffer(), and StartSharedBufferIO().

◆ WaitReadBuffers()

bool WaitReadBuffers ( ReadBuffersOperation * operation )

Definition at line 1750 of file bufmgr.c.

{
    PgAioReturn *aio_ret = &operation->io_return;
    IOContext   io_context;
    IOObject    io_object;
    bool        needed_wait = false;
 
    if (operation->persistence == RELPERSISTENCE_TEMP)
    {
        io_context = IOCONTEXT_NORMAL;
        io_object = IOOBJECT_TEMP_RELATION;
    }
    else
    {
        io_context = IOContextForStrategy(operation->strategy);
        io_object = IOOBJECT_RELATION;
    }
 
    /*
     * If we get here without an IO operation having been issued, the
     * io_method == IOMETHOD_SYNC path must have been used. Otherwise the
     * caller should not have called WaitReadBuffers().
     *
     * In the case of IOMETHOD_SYNC, we start - as we used to before the
     * introducing of AIO - the IO in WaitReadBuffers(). This is done as part
     * of the retry logic below, no extra code is required.
     *
     * This path is expected to eventually go away.
     */
    if (!pgaio_wref_valid(&operation->io_wref) && io_method != IOMETHOD_SYNC)
        elog(ERROR, "waiting for read operation that didn't read");
 
    /*
     * To handle partial reads, and IOMETHOD_SYNC, we re-issue IO until we're
     * done. We may need multiple retries, not just because we could get
     * multiple partial reads, but also because some of the remaining
     * to-be-read buffers may have been read in by other backends, limiting
     * the IO size.
     */
    while (true)
    {
        int         ignored_nblocks_progress;
 
        CheckReadBuffersOperation(operation, false);
 
        /*
         * If there is an IO associated with the operation, we may need to
         * wait for it.
         */
        if (pgaio_wref_valid(&operation->io_wref))
        {
            /*
             * Track the time spent waiting for the IO to complete. As
             * tracking a wait even if we don't actually need to wait
             *
             * a) is not cheap, due to the timestamping overhead
             *
             * b) reports some time as waiting, even if we never waited
             *
             * we first check if we already know the IO is complete.
             *
             * Note that operation->io_return is uninitialized for foreign IO,
             * so we cannot use the cheaper PGAIO_RS_UNKNOWN pre-check.
             */
            if ((operation->foreign_io || aio_ret->result.status == PGAIO_RS_UNKNOWN) &&
                !pgaio_wref_check_done(&operation->io_wref))
            {
                instr_time  io_start = pgstat_prepare_io_time(track_io_timing);
 
                pgaio_wref_wait(&operation->io_wref);
                needed_wait = true;
 
                /*
                 * The IO operation itself was already counted earlier, in
                 * AsyncReadBuffers(), this just accounts for the wait time.
                 */
                pgstat_count_io_op_time(io_object, io_context, IOOP_READ,
                                        io_start, 0, 0);
            }
            else
            {
                Assert(pgaio_wref_check_done(&operation->io_wref));
            }
 
            if (unlikely(operation->foreign_io))
            {
                Buffer      buffer = operation->buffers[operation->nblocks_done];
                BufferDesc *desc = BufferIsLocal(buffer) ?
                    GetLocalBufferDescriptor(-buffer - 1) :
                    GetBufferDescriptor(buffer - 1);
                uint64      buf_state = pg_atomic_read_u64(&desc->state);
 
                if (buf_state & BM_VALID)
                {
                    BlockNumber blocknum = operation->blocknum + operation->nblocks_done;
 
                    operation->nblocks_done += 1;
                    Assert(operation->nblocks_done <= operation->nblocks);
 
                    /*
                     * Track this as a 'hit' for this backend. The backend
                     * performing the IO will track it as a 'read'.
                     */
                    TrackBufferHit(io_object, io_context,
                                   operation->rel, operation->persistence,
                                   operation->smgr, operation->forknum,
                                   blocknum);
                }
 
                /*
                 * If the foreign IO failed and left the buffer invalid,
                 * nblocks_done is not incremented. The retry loop below will
                 * call AsyncReadBuffers() which will attempt the IO itself.
                 */
            }
            else
            {
                /*
                 * We now are sure the IO completed. Check the results. This
                 * includes reporting on errors if there were any.
                 */
                ProcessReadBuffersResult(operation);
            }
        }
 
        /*
         * Most of the time, the one IO we already started, will read in
         * everything.  But we need to deal with partial reads and buffers not
         * needing IO anymore.
         */
        if (operation->nblocks_done == operation->nblocks)
            break;
 
        CHECK_FOR_INTERRUPTS();
 
        /*
         * If the IO completed only partially, we need to perform additional
         * work, consider that a form of having had to wait.
         */
        needed_wait = true;
 
        /*
         * This may only complete the IO partially, either because some
         * buffers were already valid, or because of a partial read.
         *
         * NB: In contrast to after the AsyncReadBuffers() call in
         * StartReadBuffers(), we do *not* reduce
         * ReadBuffersOperation->nblocks here, callers expect the full
         * operation to be completed at this point (as more operations may
         * have been queued).
         */
        AsyncReadBuffers(operation, &ignored_nblocks_progress);
    }
 
    CheckReadBuffersOperation(operation, true);
 
    /* NB: READ_DONE tracepoint was already executed in completion callback */
    return needed_wait;
}

References Assert, AsyncReadBuffers(), BM_VALID, PrivateRefCountEntry::buffer, BufferIsLocal, CHECK_FOR_INTERRUPTS, CheckReadBuffersOperation(), elog, ERROR, fb(), GetBufferDescriptor(), GetLocalBufferDescriptor(), io_method, IOCONTEXT_NORMAL, IOContextForStrategy(), IOMETHOD_SYNC, IOOBJECT_RELATION, IOOBJECT_TEMP_RELATION, IOOP_READ, operation, pg_atomic_read_u64(), PGAIO_RS_UNKNOWN, pgaio_wref_check_done(), pgaio_wref_valid(), pgaio_wref_wait(), pgstat_count_io_op_time(), pgstat_prepare_io_time(), ProcessReadBuffersResult(), BufferDesc::state, track_io_timing, TrackBufferHit(), and unlikely.

Referenced by read_buffers(), read_stream_next_buffer(), and ReadBuffer_common().

◆ WakePinCountWaiter()

static void WakePinCountWaiter ( BufferDesc * buf )

static

Definition at line 3420 of file bufmgr.c.

{
    /*
     * Acquire the buffer header lock, re-check that there's a waiter. Another
     * backend could have unpinned this buffer, and already woken up the
     * waiter.
     *
     * There's no danger of the buffer being replaced after we unpinned it
     * above, as it's pinned by the waiter. The waiter removes
     * BM_PIN_COUNT_WAITER if it stops waiting for a reason other than this
     * backend waking it up.
     */
    uint64      buf_state = LockBufHdr(buf);
 
    if ((buf_state & BM_PIN_COUNT_WAITER) &&
        BUF_STATE_GET_REFCOUNT(buf_state) == 1)
    {
        /* we just released the last pin other than the waiter's */
        int         wait_backend_pgprocno = buf->wait_backend_pgprocno;
 
        UnlockBufHdrExt(buf, buf_state,
                        0, BM_PIN_COUNT_WAITER,
                        0);
        ProcSendSignal(wait_backend_pgprocno);
    }
    else
        UnlockBufHdr(buf);
}

References BM_PIN_COUNT_WAITER, buf, BUF_STATE_GET_REFCOUNT, fb(), LockBufHdr(), ProcSendSignal(), UnlockBufHdr(), and UnlockBufHdrExt().

Referenced by TerminateBufferIO(), UnlockReleaseBuffer(), and UnpinBufferNoOwner().

◆ WritebackContextInit()

void WritebackContextInit	(	WritebackContext *	context,
		int *	max_pending
	)

Definition at line 7678 of file bufmgr.c.

{
    Assert(*max_pending <= WRITEBACK_MAX_PENDING_FLUSHES);
 
    context->max_pending = max_pending;
    context->nr_pending = 0;
}

References Assert, WritebackContext::max_pending, WritebackContext::nr_pending, and WRITEBACK_MAX_PENDING_FLUSHES.

Referenced by BackgroundWriterMain(), BufferManagerShmemAttach(), BufferManagerShmemInit(), and BufferSync().

◆ ZeroAndLockBuffer()

static void ZeroAndLockBuffer	(	Buffer	buffer,
		ReadBufferMode	mode,
		bool	already_valid
	)

static

Definition at line 1146 of file bufmgr.c.

{
    BufferDesc *bufHdr;
    bool        need_to_zero;
    bool        isLocalBuf = BufferIsLocal(buffer);
    StartBufferIOResult sbres;
 
    Assert(mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK);
 
    if (already_valid)
    {
        /*
         * If the caller already knew the buffer was valid, we can skip some
         * header interaction.  The caller just wants to lock the buffer.
         */
        need_to_zero = false;
    }
    else
    {
        if (isLocalBuf)
        {
            /* Simple case for non-shared buffers. */
            bufHdr = GetLocalBufferDescriptor(-buffer - 1);
            sbres = StartLocalBufferIO(bufHdr, true, true, NULL);
        }
        else
        {
            /*
             * Take BM_IO_IN_PROGRESS, or discover that BM_VALID has been set
             * concurrently.  Even though we aren't doing I/O, that ensures
             * that we don't zero a page that someone else has pinned.  An
             * exclusive content lock wouldn't be enough, because readers are
             * allowed to drop the content lock after determining that a tuple
             * is visible (see buffer access rules in README).
             */
            bufHdr = GetBufferDescriptor(buffer - 1);
            sbres = StartSharedBufferIO(bufHdr, true, true, NULL);
        }
 
        Assert(sbres != BUFFER_IO_IN_PROGRESS);
        need_to_zero = sbres == BUFFER_IO_READY_FOR_IO;
    }
 
    if (need_to_zero)
    {
        memset(BufferGetPage(buffer), 0, BLCKSZ);
 
        /*
         * Grab the buffer content lock before marking the page as valid, to
         * make sure that no other backend sees the zeroed page before the
         * caller has had a chance to initialize it.
         *
         * Since no-one else can be looking at the page contents yet, there is
         * no difference between an exclusive lock and a cleanup-strength
         * lock. (Note that we cannot use LockBuffer() or
         * LockBufferForCleanup() here, because they assert that the buffer is
         * already valid.)
         */
        if (!isLocalBuf)
            LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
 
        /* Set BM_VALID, terminate IO, and wake up any waiters */
        if (isLocalBuf)
            TerminateLocalBufferIO(bufHdr, false, BM_VALID, false);
        else
            TerminateBufferIO(bufHdr, false, BM_VALID, true, false);
    }
    else if (!isLocalBuf)
    {
        /*
         * The buffer is valid, so we can't zero it.  The caller still expects
         * the page to be locked on return.
         */
        if (mode == RBM_ZERO_AND_LOCK)
            LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
        else
            LockBufferForCleanup(buffer);
    }
}

References Assert, BM_VALID, PrivateRefCountEntry::buffer, BUFFER_IO_IN_PROGRESS, BUFFER_IO_READY_FOR_IO, BUFFER_LOCK_EXCLUSIVE, BufferGetPage(), BufferIsLocal, fb(), GetBufferDescriptor(), GetLocalBufferDescriptor(), LockBuffer(), LockBufferForCleanup(), mode, RBM_ZERO_AND_CLEANUP_LOCK, RBM_ZERO_AND_LOCK, StartLocalBufferIO(), StartSharedBufferIO(), TerminateBufferIO(), and TerminateLocalBufferIO().

Referenced by ReadBuffer_common().

Variable Documentation

◆ aio_local_buffer_readv_cb

const PgAioHandleCallbacks aio_local_buffer_readv_cb

Initial value:

= {
    .stage = local_buffer_readv_stage,
 
    
    .complete_local = local_buffer_readv_complete,
    .report = buffer_readv_report,
}

Definition at line 8947 of file bufmgr.c.

                                                       {
    .stage = local_buffer_readv_stage,
 
    /*
     * Note that this, in contrast to the shared_buffers case, uses
     * complete_local, as only the issuing backend has access to the required
     * datastructures. This is important in case the IO completion may be
     * consumed incidentally by another backend.
     */
    .complete_local = local_buffer_readv_complete,
    .report = buffer_readv_report,
};

◆ aio_shared_buffer_readv_cb

const PgAioHandleCallbacks aio_shared_buffer_readv_cb

Initial value:

= {
    .stage = shared_buffer_readv_stage,
    .complete_shared = shared_buffer_readv_complete,
    
    .complete_local = shared_buffer_readv_complete_local,
    .report = buffer_readv_report,
}

Definition at line 8938 of file bufmgr.c.

                                                        {
    .stage = shared_buffer_readv_stage,
    .complete_shared = shared_buffer_readv_complete,
    /* need a local callback to report checksum failures */
    .complete_local = shared_buffer_readv_complete_local,
    .report = buffer_readv_report,
};

◆ backend_flush_after

int backend_flush_after = DEFAULT_BACKEND_FLUSH_AFTER

Definition at line 225 of file bufmgr.c.

Referenced by BufferManagerShmemAttach(), and BufferManagerShmemInit().

◆ bgwriter_flush_after

int bgwriter_flush_after = DEFAULT_BGWRITER_FLUSH_AFTER

Definition at line 224 of file bufmgr.c.

Referenced by BackgroundWriterMain().

◆ bgwriter_lru_maxpages

int bgwriter_lru_maxpages = 100

Definition at line 190 of file bufmgr.c.

Referenced by BgBufferSync().

◆ bgwriter_lru_multiplier

double bgwriter_lru_multiplier = 2.0

Definition at line 191 of file bufmgr.c.

Referenced by BgBufferSync().

◆ buffer_io_resowner_desc

const ResourceOwnerDesc buffer_io_resowner_desc

Initial value:

=
{
    .name = "buffer io",
    .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
    .release_priority = RELEASE_PRIO_BUFFER_IOS,
    .ReleaseResource = ResOwnerReleaseBufferIO,
    .DebugPrint = ResOwnerPrintBufferIO
}

Definition at line 285 of file bufmgr.c.

{
    .name = "buffer io",
    .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
    .release_priority = RELEASE_PRIO_BUFFER_IOS,
    .ReleaseResource = ResOwnerReleaseBufferIO,
    .DebugPrint = ResOwnerPrintBufferIO
};

Referenced by ResourceOwnerForgetBufferIO(), and ResourceOwnerRememberBufferIO().

◆ buffer_resowner_desc

const ResourceOwnerDesc buffer_resowner_desc

Initial value:

=
{
    .name = "buffer",
    .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
    .release_priority = RELEASE_PRIO_BUFFER_PINS,
    .ReleaseResource = ResOwnerReleaseBuffer,
    .DebugPrint = ResOwnerPrintBuffer
}

Definition at line 294 of file bufmgr.c.

{
    .name = "buffer",
    .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
    .release_priority = RELEASE_PRIO_BUFFER_PINS,
    .ReleaseResource = ResOwnerReleaseBuffer,
    .DebugPrint = ResOwnerPrintBuffer
};