storage.c File Reference

#include "postgres.h"
#include "access/visibilitymap.h"
#include "access/xact.h"
#include "access/xlog.h"
#include "access/xloginsert.h"
#include "access/xlogutils.h"
#include "catalog/storage.h"
#include "catalog/storage_xlog.h"
#include "miscadmin.h"
#include "pgstat.h"
#include "storage/bulk_write.h"
#include "storage/freespace.h"
#include "storage/proc.h"
#include "storage/smgr.h"
#include "utils/hsearch.h"
#include "utils/memutils.h"
#include "utils/rel.h"

Include dependency graph for storage.c:

Go to the source code of this file.

Data Structures
struct	PendingRelDelete
struct	PendingRelSync

Typedefs
typedef struct PendingRelDelete	PendingRelDelete
typedef struct PendingRelSync	PendingRelSync

Functions
static void	AddPendingSync (const RelFileLocator *rlocator)
SMgrRelation	RelationCreateStorage (RelFileLocator rlocator, char relpersistence, bool register_delete)
void	log_smgrcreate (const RelFileLocator *rlocator, ForkNumber forkNum)
void	RelationDropStorage (Relation rel)
void	RelationPreserveStorage (RelFileLocator rlocator, bool atCommit)
void	RelationTruncate (Relation rel, BlockNumber nblocks)
void	RelationPreTruncate (Relation rel)
void	RelationCopyStorage (SMgrRelation src, SMgrRelation dst, ForkNumber forkNum, char relpersistence)
bool	RelFileLocatorSkippingWAL (RelFileLocator rlocator)
Size	EstimatePendingSyncsSpace (void)
void	SerializePendingSyncs (Size maxSize, char *startAddress)
void	RestorePendingSyncs (char *startAddress)
void	smgrDoPendingDeletes (bool isCommit)
void	smgrDoPendingSyncs (bool isCommit, bool isParallelWorker)
int	smgrGetPendingDeletes (bool forCommit, RelFileLocator **ptr)
void	PostPrepare_smgr (void)
void	AtSubCommit_smgr (void)
void	AtSubAbort_smgr (void)
void	smgr_redo (XLogReaderState *record)

Variables
int	wal_skip_threshold = 2048
static PendingRelDelete *	pendingDeletes = NULL
static HTAB *	pendingSyncHash = NULL

Typedef Documentation

PendingRelDelete

typedef struct PendingRelDelete PendingRelDelete

PendingRelSync

typedef struct PendingRelSync PendingRelSync

Function Documentation

AddPendingSync()

static void AddPendingSync ( const RelFileLocator *  rlocator )

static

Definition at line 86 of file storage.c.

{
    PendingRelSync *pending;
    bool        found;
 
    /* create the hash if not yet */
    if (!pendingSyncHash)
    {
        HASHCTL     ctl;
 
        ctl.keysize = sizeof(RelFileLocator);
        ctl.entrysize = sizeof(PendingRelSync);
        ctl.hcxt = TopTransactionContext;
        pendingSyncHash = hash_create("pending sync hash", 16, &ctl,
                                      HASH_ELEM | HASH_BLOBS | HASH_CONTEXT);
    }
 
    pending = hash_search(pendingSyncHash, rlocator, HASH_ENTER, &found);
    Assert(!found);
    pending->is_truncated = false;
}

References Assert(), ctl, HASH_BLOBS, HASH_CONTEXT, hash_create(), HASH_ELEM, HASH_ENTER, hash_search(), PendingRelSync::is_truncated, pendingSyncHash, and TopTransactionContext.

Referenced by RelationCreateStorage(), and RestorePendingSyncs().

AtSubAbort_smgr()

void AtSubAbort_smgr

(

void

)

Definition at line 975 of file storage.c.

{
    smgrDoPendingDeletes(false);
}

References smgrDoPendingDeletes().

Referenced by AbortSubTransaction().

AtSubCommit_smgr()

void AtSubCommit_smgr

(

void

)

Definition at line 955 of file storage.c.

{
    int         nestLevel = GetCurrentTransactionNestLevel();
    PendingRelDelete *pending;
 
    for (pending = pendingDeletes; pending != NULL; pending = pending->next)
    {
        if (pending->nestLevel >= nestLevel)
            pending->nestLevel = nestLevel - 1;
    }
}

References GetCurrentTransactionNestLevel(), PendingRelDelete::nestLevel, PendingRelDelete::next, and pendingDeletes.

Referenced by CommitSubTransaction().

EstimatePendingSyncsSpace()

Size EstimatePendingSyncsSpace

(

void

)

Definition at line 587 of file storage.c.

{
    long        entries;
 
    entries = pendingSyncHash ? hash_get_num_entries(pendingSyncHash) : 0;
    return mul_size(1 + entries, sizeof(RelFileLocator));
}

References hash_get_num_entries(), mul_size(), and pendingSyncHash.

Referenced by InitializeParallelDSM().

log_smgrcreate()

void log_smgrcreate	(	const RelFileLocator *	rlocator,
		ForkNumber	forkNum
	)

Definition at line 187 of file storage.c.

{
    xl_smgr_create xlrec;
 
    /*
     * Make an XLOG entry reporting the file creation.
     */
    xlrec.rlocator = *rlocator;
    xlrec.forkNum = forkNum;
 
    XLogBeginInsert();
    XLogRegisterData(&xlrec, sizeof(xlrec));
    XLogInsert(RM_SMGR_ID, XLOG_SMGR_CREATE | XLR_SPECIAL_REL_UPDATE);
}

References xl_smgr_create::forkNum, xl_smgr_create::rlocator, XLOG_SMGR_CREATE, XLogBeginInsert(), XLogInsert(), XLogRegisterData(), and XLR_SPECIAL_REL_UPDATE.

Referenced by CreateAndCopyRelationData(), fill_seq_with_data(), heapam_relation_copy_data(), heapam_relation_set_new_filelocator(), index_build(), index_copy_data(), and RelationCreateStorage().

PostPrepare_smgr()

void PostPrepare_smgr

(

void

)

Definition at line 934 of file storage.c.

{
    PendingRelDelete *pending;
    PendingRelDelete *next;
 
    for (pending = pendingDeletes; pending != NULL; pending = next)
    {
        next = pending->next;
        pendingDeletes = next;
        /* must explicitly free the list entry */
        pfree(pending);
    }
}

References next, PendingRelDelete::next, pendingDeletes, and pfree().

Referenced by PrepareTransaction().

RelationCopyStorage()

void RelationCopyStorage	(	SMgrRelation	src,
		SMgrRelation	dst,
		ForkNumber	forkNum,
		char	relpersistence
	)

Definition at line 478 of file storage.c.

{
    bool        use_wal;
    bool        copying_initfork;
    BlockNumber nblocks;
    BlockNumber blkno;
    BulkWriteState *bulkstate;
 
    /*
     * The init fork for an unlogged relation in many respects has to be
     * treated the same as normal relation, changes need to be WAL logged and
     * it needs to be synced to disk.
     */
    copying_initfork = relpersistence == RELPERSISTENCE_UNLOGGED &&
        forkNum == INIT_FORKNUM;
 
    /*
     * We need to log the copied data in WAL iff WAL archiving/streaming is
     * enabled AND it's a permanent relation.  This gives the same answer as
     * "RelationNeedsWAL(rel) || copying_initfork", because we know the
     * current operation created new relation storage.
     */
    use_wal = XLogIsNeeded() &&
        (relpersistence == RELPERSISTENCE_PERMANENT || copying_initfork);
 
    bulkstate = smgr_bulk_start_smgr(dst, forkNum, use_wal);
 
    nblocks = smgrnblocks(src, forkNum);
 
    for (blkno = 0; blkno < nblocks; blkno++)
    {
        BulkWriteBuffer buf;
        int         piv_flags;
        bool        checksum_failure;
        bool        verified;
 
        /* If we got a cancel signal during the copy of the data, quit */
        CHECK_FOR_INTERRUPTS();
 
        buf = smgr_bulk_get_buf(bulkstate);
        smgrread(src, forkNum, blkno, (Page) buf);
 
        piv_flags = PIV_LOG_WARNING;
        if (ignore_checksum_failure)
            piv_flags |= PIV_IGNORE_CHECKSUM_FAILURE;
        verified = PageIsVerified((Page) buf, blkno, piv_flags,
                                  &checksum_failure);
        if (checksum_failure)
        {
            RelFileLocatorBackend rloc = src->smgr_rlocator;
 
            pgstat_prepare_report_checksum_failure(rloc.locator.dbOid);
            pgstat_report_checksum_failures_in_db(rloc.locator.dbOid, 1);
        }
 
        if (!verified)
        {
            /*
             * For paranoia's sake, capture the file path before invoking the
             * ereport machinery.  This guards against the possibility of a
             * relcache flush caused by, e.g., an errcontext callback.
             * (errcontext callbacks shouldn't be risking any such thing, but
             * people have been known to forget that rule.)
             */
            RelPathStr  relpath = relpathbackend(src->smgr_rlocator.locator,
                                                 src->smgr_rlocator.backend,
                                                 forkNum);
 
            ereport(ERROR,
                    (errcode(ERRCODE_DATA_CORRUPTED),
                     errmsg("invalid page in block %u of relation %s",
                            blkno, relpath.str)));
        }
 
        /*
         * Queue the page for WAL-logging and writing out.  Unfortunately we
         * don't know what kind of a page this is, so we have to log the full
         * page including any unused space.
         */
        smgr_bulk_write(bulkstate, blkno, buf, false);
    }
    smgr_bulk_finish(bulkstate);
}

References RelFileLocatorBackend::backend, buf, CHECK_FOR_INTERRUPTS, checksum_failure, RelFileLocator::dbOid, ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg(), ERROR, ignore_checksum_failure, INIT_FORKNUM, RelFileLocatorBackend::locator, PageIsVerified(), pgstat_prepare_report_checksum_failure(), pgstat_report_checksum_failures_in_db(), PIV_IGNORE_CHECKSUM_FAILURE, PIV_LOG_WARNING, relpath, relpathbackend, smgr_bulk_finish(), smgr_bulk_get_buf(), smgr_bulk_start_smgr(), smgr_bulk_write(), SMgrRelationData::smgr_rlocator, smgrnblocks(), smgrread(), and XLogIsNeeded.

Referenced by heapam_relation_copy_data(), and index_copy_data().

RelationCreateStorage()

SMgrRelation RelationCreateStorage	(	RelFileLocator	rlocator,
		char	relpersistence,
		bool	register_delete
	)

Definition at line 122 of file storage.c.

{
    SMgrRelation srel;
    ProcNumber  procNumber;
    bool        needs_wal;
 
    Assert(!IsInParallelMode());    /* couldn't update pendingSyncHash */
 
    switch (relpersistence)
    {
        case RELPERSISTENCE_TEMP:
            procNumber = ProcNumberForTempRelations();
            needs_wal = false;
            break;
        case RELPERSISTENCE_UNLOGGED:
            procNumber = INVALID_PROC_NUMBER;
            needs_wal = false;
            break;
        case RELPERSISTENCE_PERMANENT:
            procNumber = INVALID_PROC_NUMBER;
            needs_wal = true;
            break;
        default:
            elog(ERROR, "invalid relpersistence: %c", relpersistence);
            return NULL;        /* placate compiler */
    }
 
    srel = smgropen(rlocator, procNumber);
    smgrcreate(srel, MAIN_FORKNUM, false);
 
    if (needs_wal)
        log_smgrcreate(&srel->smgr_rlocator.locator, MAIN_FORKNUM);
 
    /*
     * Add the relation to the list of stuff to delete at abort, if we are
     * asked to do so.
     */
    if (register_delete)
    {
        PendingRelDelete *pending;
 
        pending = (PendingRelDelete *)
            MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
        pending->rlocator = rlocator;
        pending->procNumber = procNumber;
        pending->atCommit = false;  /* delete if abort */
        pending->nestLevel = GetCurrentTransactionNestLevel();
        pending->next = pendingDeletes;
        pendingDeletes = pending;
    }
 
    if (relpersistence == RELPERSISTENCE_PERMANENT && !XLogIsNeeded())
    {
        Assert(procNumber == INVALID_PROC_NUMBER);
        AddPendingSync(&rlocator);
    }
 
    return srel;
}

References AddPendingSync(), Assert(), PendingRelDelete::atCommit, elog, ERROR, GetCurrentTransactionNestLevel(), INVALID_PROC_NUMBER, IsInParallelMode(), RelFileLocatorBackend::locator, log_smgrcreate(), MAIN_FORKNUM, MemoryContextAlloc(), PendingRelDelete::nestLevel, PendingRelDelete::next, pendingDeletes, PendingRelDelete::procNumber, ProcNumberForTempRelations, PendingRelDelete::rlocator, SMgrRelationData::smgr_rlocator, smgrcreate(), smgropen(), TopMemoryContext, and XLogIsNeeded.

Referenced by CreateAndCopyRelationData(), heap_create(), heapam_relation_copy_data(), heapam_relation_set_new_filelocator(), index_copy_data(), and RelationSetNewRelfilenumber().

RelationDropStorage()

void RelationDropStorage

(

Relation

rel

)

Definition at line 207 of file storage.c.

{
    PendingRelDelete *pending;
 
    /* Add the relation to the list of stuff to delete at commit */
    pending = (PendingRelDelete *)
        MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
    pending->rlocator = rel->rd_locator;
    pending->procNumber = rel->rd_backend;
    pending->atCommit = true;   /* delete if commit */
    pending->nestLevel = GetCurrentTransactionNestLevel();
    pending->next = pendingDeletes;
    pendingDeletes = pending;
 
    /*
     * NOTE: if the relation was created in this transaction, it will now be
     * present in the pending-delete list twice, once with atCommit true and
     * once with atCommit false.  Hence, it will be physically deleted at end
     * of xact in either case (and the other entry will be ignored by
     * smgrDoPendingDeletes, so no error will occur).  We could instead remove
     * the existing list entry and delete the physical file immediately, but
     * for now I'll keep the logic simple.
     */
 
    RelationCloseSmgr(rel);
}

References PendingRelDelete::atCommit, GetCurrentTransactionNestLevel(), MemoryContextAlloc(), PendingRelDelete::nestLevel, PendingRelDelete::next, pendingDeletes, PendingRelDelete::procNumber, RelationData::rd_backend, RelationData::rd_locator, RelationCloseSmgr(), PendingRelDelete::rlocator, and TopMemoryContext.

Referenced by heap_drop_with_catalog(), heapam_relation_copy_data(), index_copy_data(), index_drop(), reindex_index(), and RelationSetNewRelfilenumber().

RelationPreserveStorage()

void RelationPreserveStorage	(	RelFileLocator	rlocator,
		bool	atCommit
	)

Definition at line 252 of file storage.c.

{
    PendingRelDelete *pending;
    PendingRelDelete *prev;
    PendingRelDelete *next;
 
    prev = NULL;
    for (pending = pendingDeletes; pending != NULL; pending = next)
    {
        next = pending->next;
        if (RelFileLocatorEquals(rlocator, pending->rlocator)
            && pending->atCommit == atCommit)
        {
            /* unlink and delete list entry */
            if (prev)
                prev->next = next;
            else
                pendingDeletes = next;
            pfree(pending);
            /* prev does not change */
        }
        else
        {
            /* unrelated entry, don't touch it */
            prev = pending;
        }
    }
}

References PendingRelDelete::atCommit, next, PendingRelDelete::next, pendingDeletes, pfree(), RelFileLocatorEquals, and PendingRelDelete::rlocator.

Referenced by ATExecAddIndex(), and write_relmap_file().

RelationPreTruncate()

void RelationPreTruncate

(

Relation

rel

)

Definition at line 450 of file storage.c.

{
    PendingRelSync *pending;
 
    if (!pendingSyncHash)
        return;
 
    pending = hash_search(pendingSyncHash,
                          &(RelationGetSmgr(rel)->smgr_rlocator.locator),
                          HASH_FIND, NULL);
    if (pending)
        pending->is_truncated = true;
}

References HASH_FIND, hash_search(), PendingRelSync::is_truncated, pendingSyncHash, and RelationGetSmgr().

Referenced by RelationTruncate().

RelationTruncate()

void RelationTruncate	(	Relation	rel,
		BlockNumber	nblocks
	)

Definition at line 289 of file storage.c.

{
    bool        fsm;
    bool        vm;
    bool        need_fsm_vacuum = false;
    ForkNumber  forks[MAX_FORKNUM];
    BlockNumber old_blocks[MAX_FORKNUM];
    BlockNumber blocks[MAX_FORKNUM];
    int         nforks = 0;
    SMgrRelation reln;
 
    /*
     * Make sure smgr_targblock etc aren't pointing somewhere past new end.
     * (Note: don't rely on this reln pointer below this loop.)
     */
    reln = RelationGetSmgr(rel);
    reln->smgr_targblock = InvalidBlockNumber;
    for (int i = 0; i <= MAX_FORKNUM; ++i)
        reln->smgr_cached_nblocks[i] = InvalidBlockNumber;
 
    /* Prepare for truncation of MAIN fork of the relation */
    forks[nforks] = MAIN_FORKNUM;
    old_blocks[nforks] = smgrnblocks(reln, MAIN_FORKNUM);
    blocks[nforks] = nblocks;
    nforks++;
 
    /* Prepare for truncation of the FSM if it exists */
    fsm = smgrexists(RelationGetSmgr(rel), FSM_FORKNUM);
    if (fsm)
    {
        blocks[nforks] = FreeSpaceMapPrepareTruncateRel(rel, nblocks);
        if (BlockNumberIsValid(blocks[nforks]))
        {
            forks[nforks] = FSM_FORKNUM;
            old_blocks[nforks] = smgrnblocks(reln, FSM_FORKNUM);
            nforks++;
            need_fsm_vacuum = true;
        }
    }
 
    /* Prepare for truncation of the visibility map too if it exists */
    vm = smgrexists(RelationGetSmgr(rel), VISIBILITYMAP_FORKNUM);
    if (vm)
    {
        blocks[nforks] = visibilitymap_prepare_truncate(rel, nblocks);
        if (BlockNumberIsValid(blocks[nforks]))
        {
            forks[nforks] = VISIBILITYMAP_FORKNUM;
            old_blocks[nforks] = smgrnblocks(reln, VISIBILITYMAP_FORKNUM);
            nforks++;
        }
    }
 
    RelationPreTruncate(rel);
 
    /*
     * The code which follows can interact with concurrent checkpoints in two
     * separate ways.
     *
     * First, the truncation operation might drop buffers that the checkpoint
     * otherwise would have flushed. If it does, then it's essential that the
     * files actually get truncated on disk before the checkpoint record is
     * written. Otherwise, if reply begins from that checkpoint, the
     * to-be-truncated blocks might still exist on disk but have older
     * contents than expected, which can cause replay to fail. It's OK for the
     * blocks to not exist on disk at all, but not for them to have the wrong
     * contents. For this reason, we need to set DELAY_CHKPT_COMPLETE while
     * this code executes.
     *
     * Second, the call to smgrtruncate() below will in turn call
     * RegisterSyncRequest(). We need the sync request created by that call to
     * be processed before the checkpoint completes. CheckPointGuts() will
     * call ProcessSyncRequests(), but if we register our sync request after
     * that happens, then the WAL record for the truncation could end up
     * preceding the checkpoint record, while the actual sync doesn't happen
     * until the next checkpoint. To prevent that, we need to set
     * DELAY_CHKPT_START here. That way, if the XLOG_SMGR_TRUNCATE precedes
     * the redo pointer of a concurrent checkpoint, we're guaranteed that the
     * corresponding sync request will be processed before the checkpoint
     * completes.
     */
    Assert((MyProc->delayChkptFlags & (DELAY_CHKPT_START | DELAY_CHKPT_COMPLETE)) == 0);
    MyProc->delayChkptFlags |= DELAY_CHKPT_START | DELAY_CHKPT_COMPLETE;
 
    /*
     * We WAL-log the truncation first and then truncate in a critical
     * section. Truncation drops buffers, even if dirty, and then truncates
     * disk files. All of that work needs to complete before the lock is
     * released, or else old versions of pages on disk that are missing recent
     * changes would become accessible again.  We'll try the whole operation
     * again in crash recovery if we panic, but even then we can't give up
     * because we don't want standbys' relation sizes to diverge and break
     * replay or visibility invariants downstream.  The critical section also
     * suppresses interrupts.
     *
     * (See also visibilitymap.c if changing this code.)
     */
    START_CRIT_SECTION();
 
    if (RelationNeedsWAL(rel))
    {
        /*
         * Make an XLOG entry reporting the file truncation.
         */
        XLogRecPtr  lsn;
        xl_smgr_truncate xlrec;
 
        xlrec.blkno = nblocks;
        xlrec.rlocator = rel->rd_locator;
        xlrec.flags = SMGR_TRUNCATE_ALL;
 
        XLogBeginInsert();
        XLogRegisterData(&xlrec, sizeof(xlrec));
 
        lsn = XLogInsert(RM_SMGR_ID,
                         XLOG_SMGR_TRUNCATE | XLR_SPECIAL_REL_UPDATE);
 
        /*
         * Flush, because otherwise the truncation of the main relation might
         * hit the disk before the WAL record, and the truncation of the FSM
         * or visibility map. If we crashed during that window, we'd be left
         * with a truncated heap, but the FSM or visibility map would still
         * contain entries for the non-existent heap pages, and standbys would
         * also never replay the truncation.
         */
        XLogFlush(lsn);
    }
 
    /*
     * This will first remove any buffers from the buffer pool that should no
     * longer exist after truncation is complete, and then truncate the
     * corresponding files on disk.
     */
    smgrtruncate(RelationGetSmgr(rel), forks, nforks, old_blocks, blocks);
 
    END_CRIT_SECTION();
 
    /* We've done all the critical work, so checkpoints are OK now. */
    MyProc->delayChkptFlags &= ~(DELAY_CHKPT_START | DELAY_CHKPT_COMPLETE);
 
    /*
     * Update upper-level FSM pages to account for the truncation. This is
     * important because the just-truncated pages were likely marked as
     * all-free, and would be preferentially selected.
     *
     * NB: There's no point in delaying checkpoints until this is done.
     * Because the FSM is not WAL-logged, we have to be prepared for the
     * possibility of corruption after a crash anyway.
     */
    if (need_fsm_vacuum)
        FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber);
}

References Assert(), xl_smgr_truncate::blkno, BlockNumberIsValid(), DELAY_CHKPT_COMPLETE, DELAY_CHKPT_START, PGPROC::delayChkptFlags, END_CRIT_SECTION, xl_smgr_truncate::flags, FreeSpaceMapPrepareTruncateRel(), FreeSpaceMapVacuumRange(), FSM_FORKNUM, i, InvalidBlockNumber, MAIN_FORKNUM, MAX_FORKNUM, MyProc, RelationData::rd_locator, RelationGetSmgr(), RelationNeedsWAL, RelationPreTruncate(), xl_smgr_truncate::rlocator, SMgrRelationData::smgr_cached_nblocks, SMgrRelationData::smgr_targblock, SMGR_TRUNCATE_ALL, smgrexists(), smgrnblocks(), smgrtruncate(), START_CRIT_SECTION, VISIBILITYMAP_FORKNUM, visibilitymap_prepare_truncate(), XLOG_SMGR_TRUNCATE, XLogBeginInsert(), XLogFlush(), XLogInsert(), XLogRegisterData(), and XLR_SPECIAL_REL_UPDATE.

Referenced by heapam_relation_nontransactional_truncate(), lazy_truncate_heap(), RelationTruncateIndexes(), and spgvacuumscan().

RelFileLocatorSkippingWAL()

bool RelFileLocatorSkippingWAL

(

RelFileLocator

rlocator

)

Definition at line 573 of file storage.c.

{
    if (!pendingSyncHash ||
        hash_search(pendingSyncHash, &rlocator, HASH_FIND, NULL) == NULL)
        return false;
 
    return true;
}

References HASH_FIND, hash_search(), and pendingSyncHash.

Referenced by MarkBufferDirtyHint(), and RelationInitPhysicalAddr().

RestorePendingSyncs()

void RestorePendingSyncs

(

char *

startAddress

)

Definition at line 651 of file storage.c.

{
    RelFileLocator *rlocator;
 
    Assert(pendingSyncHash == NULL);
    for (rlocator = (RelFileLocator *) startAddress; rlocator->relNumber != 0;
         rlocator++)
        AddPendingSync(rlocator);
}

References AddPendingSync(), Assert(), pendingSyncHash, and RelFileLocator::relNumber.

Referenced by ParallelWorkerMain().

SerializePendingSyncs()

void SerializePendingSyncs	(	Size	maxSize,
		char *	startAddress
	)

Definition at line 600 of file storage.c.

{
    HTAB       *tmphash;
    HASHCTL     ctl;
    HASH_SEQ_STATUS scan;
    PendingRelSync *sync;
    PendingRelDelete *delete;
    RelFileLocator *src;
    RelFileLocator *dest = (RelFileLocator *) startAddress;
 
    if (!pendingSyncHash)
        goto terminate;
 
    /* Create temporary hash to collect active relfilelocators */
    ctl.keysize = sizeof(RelFileLocator);
    ctl.entrysize = sizeof(RelFileLocator);
    ctl.hcxt = CurrentMemoryContext;
    tmphash = hash_create("tmp relfilelocators",
                          hash_get_num_entries(pendingSyncHash), &ctl,
                          HASH_ELEM | HASH_BLOBS | HASH_CONTEXT);
 
    /* collect all rlocator from pending syncs */
    hash_seq_init(&scan, pendingSyncHash);
    while ((sync = (PendingRelSync *) hash_seq_search(&scan)))
        (void) hash_search(tmphash, &sync->rlocator, HASH_ENTER, NULL);
 
    /* remove deleted rnodes */
    for (delete = pendingDeletes; delete != NULL; delete = delete->next)
        if (delete->atCommit)
            (void) hash_search(tmphash, &delete->rlocator,
                               HASH_REMOVE, NULL);
 
    hash_seq_init(&scan, tmphash);
    while ((src = (RelFileLocator *) hash_seq_search(&scan)))
        *dest++ = *src;
 
    hash_destroy(tmphash);
 
terminate:
    MemSet(dest, 0, sizeof(RelFileLocator));
}

References ctl, CurrentMemoryContext, generate_unaccent_rules::dest, HASH_BLOBS, HASH_CONTEXT, hash_create(), hash_destroy(), HASH_ELEM, HASH_ENTER, hash_get_num_entries(), HASH_REMOVE, hash_search(), hash_seq_init(), hash_seq_search(), MemSet, PendingRelDelete::next, pendingDeletes, pendingSyncHash, and PendingRelSync::rlocator.

Referenced by InitializeParallelDSM().

smgr_redo()

void smgr_redo

(

XLogReaderState *

record

)

Definition at line 981 of file storage.c.

{
    XLogRecPtr  lsn = record->EndRecPtr;
    uint8       info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
 
    /* Backup blocks are not used in smgr records */
    Assert(!XLogRecHasAnyBlockRefs(record));
 
    if (info == XLOG_SMGR_CREATE)
    {
        xl_smgr_create *xlrec = (xl_smgr_create *) XLogRecGetData(record);
        SMgrRelation reln;
 
        reln = smgropen(xlrec->rlocator, INVALID_PROC_NUMBER);
        smgrcreate(reln, xlrec->forkNum, true);
    }
    else if (info == XLOG_SMGR_TRUNCATE)
    {
        xl_smgr_truncate *xlrec = (xl_smgr_truncate *) XLogRecGetData(record);
        SMgrRelation reln;
        Relation    rel;
        ForkNumber  forks[MAX_FORKNUM];
        BlockNumber blocks[MAX_FORKNUM];
        BlockNumber old_blocks[MAX_FORKNUM];
        int         nforks = 0;
        bool        need_fsm_vacuum = false;
 
        reln = smgropen(xlrec->rlocator, INVALID_PROC_NUMBER);
 
        /*
         * Forcibly create relation if it doesn't exist (which suggests that
         * it was dropped somewhere later in the WAL sequence).  As in
         * XLogReadBufferForRedo, we prefer to recreate the rel and replay the
         * log as best we can until the drop is seen.
         */
        smgrcreate(reln, MAIN_FORKNUM, true);
 
        /*
         * Before we perform the truncation, update minimum recovery point to
         * cover this WAL record. Once the relation is truncated, there's no
         * going back. The buffer manager enforces the WAL-first rule for
         * normal updates to relation files, so that the minimum recovery
         * point is always updated before the corresponding change in the data
         * file is flushed to disk. We have to do the same manually here.
         *
         * Doing this before the truncation means that if the truncation fails
         * for some reason, you cannot start up the system even after restart,
         * until you fix the underlying situation so that the truncation will
         * succeed. Alternatively, we could update the minimum recovery point
         * after truncation, but that would leave a small window where the
         * WAL-first rule could be violated.
         */
        XLogFlush(lsn);
 
        /* Prepare for truncation of MAIN fork */
        if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
        {
            forks[nforks] = MAIN_FORKNUM;
            old_blocks[nforks] = smgrnblocks(reln, MAIN_FORKNUM);
            blocks[nforks] = xlrec->blkno;
            nforks++;
 
            /* Also tell xlogutils.c about it */
            XLogTruncateRelation(xlrec->rlocator, MAIN_FORKNUM, xlrec->blkno);
        }
 
        /* Prepare for truncation of FSM and VM too */
        rel = CreateFakeRelcacheEntry(xlrec->rlocator);
 
        if ((xlrec->flags & SMGR_TRUNCATE_FSM) != 0 &&
            smgrexists(reln, FSM_FORKNUM))
        {
            blocks[nforks] = FreeSpaceMapPrepareTruncateRel(rel, xlrec->blkno);
            if (BlockNumberIsValid(blocks[nforks]))
            {
                forks[nforks] = FSM_FORKNUM;
                old_blocks[nforks] = smgrnblocks(reln, FSM_FORKNUM);
                nforks++;
                need_fsm_vacuum = true;
            }
        }
        if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0 &&
            smgrexists(reln, VISIBILITYMAP_FORKNUM))
        {
            blocks[nforks] = visibilitymap_prepare_truncate(rel, xlrec->blkno);
            if (BlockNumberIsValid(blocks[nforks]))
            {
                forks[nforks] = VISIBILITYMAP_FORKNUM;
                old_blocks[nforks] = smgrnblocks(reln, VISIBILITYMAP_FORKNUM);
                nforks++;
            }
        }
 
        /* Do the real work to truncate relation forks */
        if (nforks > 0)
        {
            START_CRIT_SECTION();
            smgrtruncate(reln, forks, nforks, old_blocks, blocks);
            END_CRIT_SECTION();
        }
 
        /*
         * Update upper-level FSM pages to account for the truncation. This is
         * important because the just-truncated pages were likely marked as
         * all-free, and would be preferentially selected.
         */
        if (need_fsm_vacuum)
            FreeSpaceMapVacuumRange(rel, xlrec->blkno,
                                    InvalidBlockNumber);
 
        FreeFakeRelcacheEntry(rel);
    }
    else
        elog(PANIC, "smgr_redo: unknown op code %u", info);
}

References Assert(), xl_smgr_truncate::blkno, BlockNumberIsValid(), CreateFakeRelcacheEntry(), elog, END_CRIT_SECTION, XLogReaderState::EndRecPtr, xl_smgr_truncate::flags, xl_smgr_create::forkNum, FreeFakeRelcacheEntry(), FreeSpaceMapPrepareTruncateRel(), FreeSpaceMapVacuumRange(), FSM_FORKNUM, INVALID_PROC_NUMBER, InvalidBlockNumber, MAIN_FORKNUM, MAX_FORKNUM, PANIC, xl_smgr_create::rlocator, xl_smgr_truncate::rlocator, SMGR_TRUNCATE_FSM, SMGR_TRUNCATE_HEAP, SMGR_TRUNCATE_VM, smgrcreate(), smgrexists(), smgrnblocks(), smgropen(), smgrtruncate(), START_CRIT_SECTION, VISIBILITYMAP_FORKNUM, visibilitymap_prepare_truncate(), XLOG_SMGR_CREATE, XLOG_SMGR_TRUNCATE, XLogFlush(), XLogRecGetData, XLogRecGetInfo, XLogRecHasAnyBlockRefs, and XLogTruncateRelation().

smgrDoPendingDeletes()

void smgrDoPendingDeletes

(

bool

isCommit

)

Definition at line 673 of file storage.c.

{
    int         nestLevel = GetCurrentTransactionNestLevel();
    PendingRelDelete *pending;
    PendingRelDelete *prev;
    PendingRelDelete *next;
    int         nrels = 0,
                maxrels = 0;
    SMgrRelation *srels = NULL;
 
    prev = NULL;
    for (pending = pendingDeletes; pending != NULL; pending = next)
    {
        next = pending->next;
        if (pending->nestLevel < nestLevel)
        {
            /* outer-level entries should not be processed yet */
            prev = pending;
        }
        else
        {
            /* unlink list entry first, so we don't retry on failure */
            if (prev)
                prev->next = next;
            else
                pendingDeletes = next;
            /* do deletion if called for */
            if (pending->atCommit == isCommit)
            {
                SMgrRelation srel;
 
                srel = smgropen(pending->rlocator, pending->procNumber);
 
                /* allocate the initial array, or extend it, if needed */
                if (maxrels == 0)
                {
                    maxrels = 8;
                    srels = palloc(sizeof(SMgrRelation) * maxrels);
                }
                else if (maxrels <= nrels)
                {
                    maxrels *= 2;
                    srels = repalloc(srels, sizeof(SMgrRelation) * maxrels);
                }
 
                srels[nrels++] = srel;
            }
            /* must explicitly free the list entry */
            pfree(pending);
            /* prev does not change */
        }
    }
 
    if (nrels > 0)
    {
        smgrdounlinkall(srels, nrels, false);
 
        for (int i = 0; i < nrels; i++)
            smgrclose(srels[i]);
 
        pfree(srels);
    }
}

References PendingRelDelete::atCommit, GetCurrentTransactionNestLevel(), i, PendingRelDelete::nestLevel, next, PendingRelDelete::next, palloc(), pendingDeletes, pfree(), PendingRelDelete::procNumber, repalloc(), PendingRelDelete::rlocator, smgrclose(), smgrdounlinkall(), and smgropen().

Referenced by AbortTransaction(), AtSubAbort_smgr(), and CommitTransaction().

smgrDoPendingSyncs()

void smgrDoPendingSyncs	(	bool	isCommit,
		bool	isParallelWorker
	)

Definition at line 741 of file storage.c.

{
    PendingRelDelete *pending;
    int         nrels = 0,
                maxrels = 0;
    SMgrRelation *srels = NULL;
    HASH_SEQ_STATUS scan;
    PendingRelSync *pendingsync;
 
    Assert(GetCurrentTransactionNestLevel() == 1);
 
    if (!pendingSyncHash)
        return;                 /* no relation needs sync */
 
    /* Abort -- just throw away all pending syncs */
    if (!isCommit)
    {
        pendingSyncHash = NULL;
        return;
    }
 
    AssertPendingSyncs_RelationCache();
 
    /* Parallel worker -- just throw away all pending syncs */
    if (isParallelWorker)
    {
        pendingSyncHash = NULL;
        return;
    }
 
    /* Skip syncing nodes that smgrDoPendingDeletes() will delete. */
    for (pending = pendingDeletes; pending != NULL; pending = pending->next)
        if (pending->atCommit)
            (void) hash_search(pendingSyncHash, &pending->rlocator,
                               HASH_REMOVE, NULL);
 
    hash_seq_init(&scan, pendingSyncHash);
    while ((pendingsync = (PendingRelSync *) hash_seq_search(&scan)))
    {
        ForkNumber  fork;
        BlockNumber nblocks[MAX_FORKNUM + 1];
        uint64      total_blocks = 0;
        SMgrRelation srel;
 
        srel = smgropen(pendingsync->rlocator, INVALID_PROC_NUMBER);
 
        /*
         * We emit newpage WAL records for smaller relations.
         *
         * Small WAL records have a chance to be flushed along with other
         * backends' WAL records.  We emit WAL records instead of syncing for
         * files that are smaller than a certain threshold, expecting faster
         * commit.  The threshold is defined by the GUC wal_skip_threshold.
         */
        if (!pendingsync->is_truncated)
        {
            for (fork = 0; fork <= MAX_FORKNUM; fork++)
            {
                if (smgrexists(srel, fork))
                {
                    BlockNumber n = smgrnblocks(srel, fork);
 
                    /* we shouldn't come here for unlogged relations */
                    Assert(fork != INIT_FORKNUM);
                    nblocks[fork] = n;
                    total_blocks += n;
                }
                else
                    nblocks[fork] = InvalidBlockNumber;
            }
        }
 
        /*
         * Sync file or emit WAL records for its contents.
         *
         * Although we emit WAL record if the file is small enough, do file
         * sync regardless of the size if the file has experienced a
         * truncation. It is because the file would be followed by trailing
         * garbage blocks after a crash recovery if, while a past longer file
         * had been flushed out, we omitted syncing-out of the file and
         * emitted WAL instead.  You might think that we could choose WAL if
         * the current main fork is longer than ever, but there's a case where
         * main fork is longer than ever but FSM fork gets shorter.
         */
        if (pendingsync->is_truncated ||
            total_blocks >= wal_skip_threshold * (uint64) 1024 / BLCKSZ)
        {
            /* allocate the initial array, or extend it, if needed */
            if (maxrels == 0)
            {
                maxrels = 8;
                srels = palloc(sizeof(SMgrRelation) * maxrels);
            }
            else if (maxrels <= nrels)
            {
                maxrels *= 2;
                srels = repalloc(srels, sizeof(SMgrRelation) * maxrels);
            }
 
            srels[nrels++] = srel;
        }
        else
        {
            /* Emit WAL records for all blocks.  The file is small enough. */
            for (fork = 0; fork <= MAX_FORKNUM; fork++)
            {
                int         n = nblocks[fork];
                Relation    rel;
 
                if (!BlockNumberIsValid(n))
                    continue;
 
                /*
                 * Emit WAL for the whole file.  Unfortunately we don't know
                 * what kind of a page this is, so we have to log the full
                 * page including any unused space.  ReadBufferExtended()
                 * counts some pgstat events; unfortunately, we discard them.
                 */
                rel = CreateFakeRelcacheEntry(srel->smgr_rlocator.locator);
                log_newpage_range(rel, fork, 0, n, false);
                FreeFakeRelcacheEntry(rel);
            }
        }
    }
 
    pendingSyncHash = NULL;
 
    if (nrels > 0)
    {
        smgrdosyncall(srels, nrels);
        pfree(srels);
    }
}

References Assert(), AssertPendingSyncs_RelationCache, PendingRelDelete::atCommit, BlockNumberIsValid(), CreateFakeRelcacheEntry(), FreeFakeRelcacheEntry(), GetCurrentTransactionNestLevel(), HASH_REMOVE, hash_search(), hash_seq_init(), hash_seq_search(), INIT_FORKNUM, INVALID_PROC_NUMBER, InvalidBlockNumber, PendingRelSync::is_truncated, RelFileLocatorBackend::locator, log_newpage_range(), MAX_FORKNUM, PendingRelDelete::next, palloc(), pendingDeletes, pendingSyncHash, pfree(), repalloc(), PendingRelDelete::rlocator, PendingRelSync::rlocator, SMgrRelationData::smgr_rlocator, smgrdosyncall(), smgrexists(), smgrnblocks(), smgropen(), and wal_skip_threshold.

Referenced by AbortTransaction(), CommitTransaction(), and PrepareTransaction().

smgrGetPendingDeletes()

int smgrGetPendingDeletes	(	bool	forCommit,
		RelFileLocator **	ptr
	)

Definition at line 893 of file storage.c.

{
    int         nestLevel = GetCurrentTransactionNestLevel();
    int         nrels;
    RelFileLocator *rptr;
    PendingRelDelete *pending;
 
    nrels = 0;
    for (pending = pendingDeletes; pending != NULL; pending = pending->next)
    {
        if (pending->nestLevel >= nestLevel && pending->atCommit == forCommit
            && pending->procNumber == INVALID_PROC_NUMBER)
            nrels++;
    }
    if (nrels == 0)
    {
        *ptr = NULL;
        return 0;
    }
    rptr = (RelFileLocator *) palloc(nrels * sizeof(RelFileLocator));
    *ptr = rptr;
    for (pending = pendingDeletes; pending != NULL; pending = pending->next)
    {
        if (pending->nestLevel >= nestLevel && pending->atCommit == forCommit
            && pending->procNumber == INVALID_PROC_NUMBER)
        {
            *rptr = pending->rlocator;
            rptr++;
        }
    }
    return nrels;
}

References PendingRelDelete::atCommit, GetCurrentTransactionNestLevel(), INVALID_PROC_NUMBER, PendingRelDelete::nestLevel, PendingRelDelete::next, palloc(), pendingDeletes, PendingRelDelete::procNumber, and PendingRelDelete::rlocator.

Referenced by RecordTransactionAbort(), RecordTransactionCommit(), and StartPrepare().

Variable Documentation

pendingDeletes

PendingRelDelete* pendingDeletes = NULL

static

Definition at line 77 of file storage.c.

Referenced by AtSubCommit_smgr(), PostPrepare_smgr(), RelationCreateStorage(), RelationDropStorage(), RelationPreserveStorage(), SerializePendingSyncs(), smgrDoPendingDeletes(), smgrDoPendingSyncs(), and smgrGetPendingDeletes().

pendingSyncHash

HTAB* pendingSyncHash = NULL

static

Definition at line 78 of file storage.c.

Referenced by AddPendingSync(), EstimatePendingSyncsSpace(), RelationPreTruncate(), RelFileLocatorSkippingWAL(), RestorePendingSyncs(), SerializePendingSyncs(), and smgrDoPendingSyncs().

wal_skip_threshold

int wal_skip_threshold = 2048

Definition at line 40 of file storage.c.

Referenced by smgrDoPendingSyncs().