storage.c File Reference

#include "postgres.h"
#include "access/parallel.h"
#include "access/visibilitymap.h"
#include "access/xact.h"
#include "access/xlog.h"
#include "access/xloginsert.h"
#include "access/xlogutils.h"
#include "catalog/storage.h"
#include "catalog/storage_xlog.h"
#include "miscadmin.h"
#include "storage/freespace.h"
#include "storage/smgr.h"
#include "utils/hsearch.h"
#include "utils/memutils.h"
#include "utils/rel.h"

Include dependency graph for storage.c:

Go to the source code of this file.

Data Structures
struct	PendingRelDelete
struct	PendingRelSync

Typedefs
typedef struct PendingRelDelete	PendingRelDelete
typedef struct PendingRelSync	PendingRelSync

Functions
static void	AddPendingSync (const RelFileLocator *rlocator)
SMgrRelation	RelationCreateStorage (RelFileLocator rlocator, char relpersistence, bool register_delete)
void	log_smgrcreate (const RelFileLocator *rlocator, ForkNumber forkNum)
void	RelationDropStorage (Relation rel)
void	RelationPreserveStorage (RelFileLocator rlocator, bool atCommit)
void	RelationTruncate (Relation rel, BlockNumber nblocks)
void	RelationPreTruncate (Relation rel)
void	RelationCopyStorage (SMgrRelation src, SMgrRelation dst, ForkNumber forkNum, char relpersistence)
bool	RelFileLocatorSkippingWAL (RelFileLocator rlocator)
Size	EstimatePendingSyncsSpace (void)
void	SerializePendingSyncs (Size maxSize, char *startAddress)
void	RestorePendingSyncs (char *startAddress)
void	smgrDoPendingDeletes (bool isCommit)
void	smgrDoPendingSyncs (bool isCommit, bool isParallelWorker)
int	smgrGetPendingDeletes (bool forCommit, RelFileLocator **ptr)
void	PostPrepare_smgr (void)
void	AtSubCommit_smgr (void)
void	AtSubAbort_smgr (void)
void	smgr_redo (XLogReaderState *record)

Variables
int	wal_skip_threshold = 2048
static PendingRelDelete *	pendingDeletes = NULL
static HTAB *	pendingSyncHash = NULL

Typedef Documentation

PendingRelDelete

typedef struct PendingRelDelete PendingRelDelete

PendingRelSync

typedef struct PendingRelSync PendingRelSync

Function Documentation

AddPendingSync()

static void AddPendingSync ( const RelFileLocator *  rlocator )

static

Definition at line 84 of file storage.c.

{
    PendingRelSync *pending;
    bool        found;
 
    /* create the hash if not yet */
    if (!pendingSyncHash)
    {
        HASHCTL     ctl;
 
        ctl.keysize = sizeof(RelFileLocator);
        ctl.entrysize = sizeof(PendingRelSync);
        ctl.hcxt = TopTransactionContext;
        pendingSyncHash = hash_create("pending sync hash", 16, &ctl,
                                      HASH_ELEM | HASH_BLOBS | HASH_CONTEXT);
    }
 
    pending = hash_search(pendingSyncHash, rlocator, HASH_ENTER, &found);
    Assert(!found);
    pending->is_truncated = false;
}

References Assert(), HASHCTL::entrysize, HASH_BLOBS, HASH_CONTEXT, hash_create(), HASH_ELEM, HASH_ENTER, hash_search(), HASHCTL::hcxt, PendingRelSync::is_truncated, HASHCTL::keysize, pendingSyncHash, and TopTransactionContext.

Referenced by RelationCreateStorage(), and RestorePendingSyncs().

AtSubAbort_smgr()

void AtSubAbort_smgr

(

void

)

Definition at line 952 of file storage.c.

{
    smgrDoPendingDeletes(false);
}

References smgrDoPendingDeletes().

Referenced by AbortSubTransaction().

AtSubCommit_smgr()

void AtSubCommit_smgr

(

void

)

Definition at line 932 of file storage.c.

{
    int         nestLevel = GetCurrentTransactionNestLevel();
    PendingRelDelete *pending;
 
    for (pending = pendingDeletes; pending != NULL; pending = pending->next)
    {
        if (pending->nestLevel >= nestLevel)
            pending->nestLevel = nestLevel - 1;
    }
}

References GetCurrentTransactionNestLevel(), PendingRelDelete::nestLevel, PendingRelDelete::next, and pendingDeletes.

Referenced by CommitSubTransaction().

EstimatePendingSyncsSpace()

Size EstimatePendingSyncsSpace

(

void

)

Definition at line 564 of file storage.c.

{
    long        entries;
 
    entries = pendingSyncHash ? hash_get_num_entries(pendingSyncHash) : 0;
    return mul_size(1 + entries, sizeof(RelFileLocator));
}

References hash_get_num_entries(), mul_size(), and pendingSyncHash.

Referenced by InitializeParallelDSM().

log_smgrcreate()

void log_smgrcreate	(	const RelFileLocator *	rlocator,
		ForkNumber	forkNum
	)

Definition at line 185 of file storage.c.

{
    xl_smgr_create xlrec;
 
    /*
     * Make an XLOG entry reporting the file creation.
     */
    xlrec.rlocator = *rlocator;
    xlrec.forkNum = forkNum;
 
    XLogBeginInsert();
    XLogRegisterData((char *) &xlrec, sizeof(xlrec));
    XLogInsert(RM_SMGR_ID, XLOG_SMGR_CREATE | XLR_SPECIAL_REL_UPDATE);
}

References xl_smgr_create::forkNum, xl_smgr_create::rlocator, XLOG_SMGR_CREATE, XLogBeginInsert(), XLogInsert(), XLogRegisterData(), and XLR_SPECIAL_REL_UPDATE.

Referenced by CreateAndCopyRelationData(), fill_seq_with_data(), heapam_relation_copy_data(), heapam_relation_set_new_filelocator(), index_build(), index_copy_data(), and RelationCreateStorage().

PostPrepare_smgr()

void PostPrepare_smgr

(

void

)

Definition at line 911 of file storage.c.

{
    PendingRelDelete *pending;
    PendingRelDelete *next;
 
    for (pending = pendingDeletes; pending != NULL; pending = next)
    {
        next = pending->next;
        pendingDeletes = next;
        /* must explicitly free the list entry */
        pfree(pending);
    }
}

References next, PendingRelDelete::next, pendingDeletes, and pfree().

Referenced by PrepareTransaction().

RelationCopyStorage()

void RelationCopyStorage	(	SMgrRelation	src,
		SMgrRelation	dst,
		ForkNumber	forkNum,
		char	relpersistence
	)

Definition at line 451 of file storage.c.

{
    PGIOAlignedBlock buf;
    Page        page;
    bool        use_wal;
    bool        copying_initfork;
    BlockNumber nblocks;
    BlockNumber blkno;
 
    page = (Page) buf.data;
 
    /*
     * The init fork for an unlogged relation in many respects has to be
     * treated the same as normal relation, changes need to be WAL logged and
     * it needs to be synced to disk.
     */
    copying_initfork = relpersistence == RELPERSISTENCE_UNLOGGED &&
        forkNum == INIT_FORKNUM;
 
    /*
     * We need to log the copied data in WAL iff WAL archiving/streaming is
     * enabled AND it's a permanent relation.  This gives the same answer as
     * "RelationNeedsWAL(rel) || copying_initfork", because we know the
     * current operation created new relation storage.
     */
    use_wal = XLogIsNeeded() &&
        (relpersistence == RELPERSISTENCE_PERMANENT || copying_initfork);
 
    nblocks = smgrnblocks(src, forkNum);
 
    for (blkno = 0; blkno < nblocks; blkno++)
    {
        /* If we got a cancel signal during the copy of the data, quit */
        CHECK_FOR_INTERRUPTS();
 
        smgrread(src, forkNum, blkno, buf.data);
 
        if (!PageIsVerifiedExtended(page, blkno,
                                    PIV_LOG_WARNING | PIV_REPORT_STAT))
        {
            /*
             * For paranoia's sake, capture the file path before invoking the
             * ereport machinery.  This guards against the possibility of a
             * relcache flush caused by, e.g., an errcontext callback.
             * (errcontext callbacks shouldn't be risking any such thing, but
             * people have been known to forget that rule.)
             */
            char       *relpath = relpathbackend(src->smgr_rlocator.locator,
                                                 src->smgr_rlocator.backend,
                                                 forkNum);
 
            ereport(ERROR,
                    (errcode(ERRCODE_DATA_CORRUPTED),
                     errmsg("invalid page in block %u of relation %s",
                            blkno, relpath)));
        }
 
        /*
         * WAL-log the copied page. Unfortunately we don't know what kind of a
         * page this is, so we have to log the full page including any unused
         * space.
         */
        if (use_wal)
            log_newpage(&dst->smgr_rlocator.locator, forkNum, blkno, page, false);
 
        PageSetChecksumInplace(page, blkno);
 
        /*
         * Now write the page.  We say skipFsync = true because there's no
         * need for smgr to schedule an fsync for this write; we'll do it
         * ourselves below.
         */
        smgrextend(dst, forkNum, blkno, buf.data, true);
    }
 
    /*
     * When we WAL-logged rel pages, we must nonetheless fsync them.  The
     * reason is that since we're copying outside shared buffers, a CHECKPOINT
     * occurring during the copy has no way to flush the previously written
     * data to disk (indeed it won't know the new rel even exists).  A crash
     * later on would replay WAL from the checkpoint, therefore it wouldn't
     * replay our earlier WAL entries. If we do not fsync those pages here,
     * they might still not be on disk when the crash occurs.
     */
    if (use_wal || copying_initfork)
        smgrimmedsync(dst, forkNum);
}

References RelFileLocatorBackend::backend, buf, CHECK_FOR_INTERRUPTS, ereport, errcode(), ERRCODE_DATA_CORRUPTED, errmsg(), ERROR, INIT_FORKNUM, RelFileLocatorBackend::locator, log_newpage(), PageIsVerifiedExtended(), PageSetChecksumInplace(), PIV_LOG_WARNING, PIV_REPORT_STAT, relpath, relpathbackend, SMgrRelationData::smgr_rlocator, smgrextend(), smgrimmedsync(), smgrnblocks(), smgrread(), and XLogIsNeeded.

Referenced by heapam_relation_copy_data(), and index_copy_data().

RelationCreateStorage()

SMgrRelation RelationCreateStorage	(	RelFileLocator	rlocator,
		char	relpersistence,
		bool	register_delete
	)

Definition at line 120 of file storage.c.

{
    SMgrRelation srel;
    BackendId   backend;
    bool        needs_wal;
 
    Assert(!IsInParallelMode());    /* couldn't update pendingSyncHash */
 
    switch (relpersistence)
    {
        case RELPERSISTENCE_TEMP:
            backend = BackendIdForTempRelations();
            needs_wal = false;
            break;
        case RELPERSISTENCE_UNLOGGED:
            backend = InvalidBackendId;
            needs_wal = false;
            break;
        case RELPERSISTENCE_PERMANENT:
            backend = InvalidBackendId;
            needs_wal = true;
            break;
        default:
            elog(ERROR, "invalid relpersistence: %c", relpersistence);
            return NULL;        /* placate compiler */
    }
 
    srel = smgropen(rlocator, backend);
    smgrcreate(srel, MAIN_FORKNUM, false);
 
    if (needs_wal)
        log_smgrcreate(&srel->smgr_rlocator.locator, MAIN_FORKNUM);
 
    /*
     * Add the relation to the list of stuff to delete at abort, if we are
     * asked to do so.
     */
    if (register_delete)
    {
        PendingRelDelete *pending;
 
        pending = (PendingRelDelete *)
            MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
        pending->rlocator = rlocator;
        pending->backend = backend;
        pending->atCommit = false;  /* delete if abort */
        pending->nestLevel = GetCurrentTransactionNestLevel();
        pending->next = pendingDeletes;
        pendingDeletes = pending;
    }
 
    if (relpersistence == RELPERSISTENCE_PERMANENT && !XLogIsNeeded())
    {
        Assert(backend == InvalidBackendId);
        AddPendingSync(&rlocator);
    }
 
    return srel;
}

References AddPendingSync(), Assert(), PendingRelDelete::atCommit, PendingRelDelete::backend, BackendIdForTempRelations, elog(), ERROR, GetCurrentTransactionNestLevel(), InvalidBackendId, IsInParallelMode(), RelFileLocatorBackend::locator, log_smgrcreate(), MAIN_FORKNUM, MemoryContextAlloc(), PendingRelDelete::nestLevel, PendingRelDelete::next, pendingDeletes, PendingRelDelete::rlocator, SMgrRelationData::smgr_rlocator, smgrcreate(), smgropen(), TopMemoryContext, and XLogIsNeeded.

Referenced by CreateAndCopyRelationData(), heap_create(), heapam_relation_copy_data(), heapam_relation_set_new_filelocator(), index_copy_data(), and RelationSetNewRelfilenumber().

RelationDropStorage()

void RelationDropStorage

(

Relation

rel

)

Definition at line 205 of file storage.c.

{
    PendingRelDelete *pending;
 
    /* Add the relation to the list of stuff to delete at commit */
    pending = (PendingRelDelete *)
        MemoryContextAlloc(TopMemoryContext, sizeof(PendingRelDelete));
    pending->rlocator = rel->rd_locator;
    pending->backend = rel->rd_backend;
    pending->atCommit = true;   /* delete if commit */
    pending->nestLevel = GetCurrentTransactionNestLevel();
    pending->next = pendingDeletes;
    pendingDeletes = pending;
 
    /*
     * NOTE: if the relation was created in this transaction, it will now be
     * present in the pending-delete list twice, once with atCommit true and
     * once with atCommit false.  Hence, it will be physically deleted at end
     * of xact in either case (and the other entry will be ignored by
     * smgrDoPendingDeletes, so no error will occur).  We could instead remove
     * the existing list entry and delete the physical file immediately, but
     * for now I'll keep the logic simple.
     */
 
    RelationCloseSmgr(rel);
}

References PendingRelDelete::atCommit, PendingRelDelete::backend, GetCurrentTransactionNestLevel(), MemoryContextAlloc(), PendingRelDelete::nestLevel, PendingRelDelete::next, pendingDeletes, RelationData::rd_backend, RelationData::rd_locator, RelationCloseSmgr(), PendingRelDelete::rlocator, and TopMemoryContext.

Referenced by heap_drop_with_catalog(), heapam_relation_copy_data(), index_copy_data(), index_drop(), reindex_index(), and RelationSetNewRelfilenumber().

RelationPreserveStorage()

void RelationPreserveStorage	(	RelFileLocator	rlocator,
		bool	atCommit
	)

Definition at line 250 of file storage.c.

{
    PendingRelDelete *pending;
    PendingRelDelete *prev;
    PendingRelDelete *next;
 
    prev = NULL;
    for (pending = pendingDeletes; pending != NULL; pending = next)
    {
        next = pending->next;
        if (RelFileLocatorEquals(rlocator, pending->rlocator)
            && pending->atCommit == atCommit)
        {
            /* unlink and delete list entry */
            if (prev)
                prev->next = next;
            else
                pendingDeletes = next;
            pfree(pending);
            /* prev does not change */
        }
        else
        {
            /* unrelated entry, don't touch it */
            prev = pending;
        }
    }
}

References PendingRelDelete::atCommit, next, PendingRelDelete::next, pendingDeletes, pfree(), RelFileLocatorEquals, and PendingRelDelete::rlocator.

Referenced by ATExecAddIndex(), and write_relmap_file().

RelationPreTruncate()

void RelationPreTruncate

(

Relation

rel

)

Definition at line 423 of file storage.c.

{
    PendingRelSync *pending;
 
    if (!pendingSyncHash)
        return;
 
    pending = hash_search(pendingSyncHash,
                          &(RelationGetSmgr(rel)->smgr_rlocator.locator),
                          HASH_FIND, NULL);
    if (pending)
        pending->is_truncated = true;
}

References HASH_FIND, hash_search(), PendingRelSync::is_truncated, pendingSyncHash, and RelationGetSmgr().

Referenced by RelationTruncate().

RelationTruncate()

void RelationTruncate	(	Relation	rel,
		BlockNumber	nblocks
	)

Definition at line 287 of file storage.c.

{
    bool        fsm;
    bool        vm;
    bool        need_fsm_vacuum = false;
    ForkNumber  forks[MAX_FORKNUM];
    BlockNumber blocks[MAX_FORKNUM];
    int         nforks = 0;
    SMgrRelation reln;
 
    /*
     * Make sure smgr_targblock etc aren't pointing somewhere past new end.
     * (Note: don't rely on this reln pointer below this loop.)
     */
    reln = RelationGetSmgr(rel);
    reln->smgr_targblock = InvalidBlockNumber;
    for (int i = 0; i <= MAX_FORKNUM; ++i)
        reln->smgr_cached_nblocks[i] = InvalidBlockNumber;
 
    /* Prepare for truncation of MAIN fork of the relation */
    forks[nforks] = MAIN_FORKNUM;
    blocks[nforks] = nblocks;
    nforks++;
 
    /* Prepare for truncation of the FSM if it exists */
    fsm = smgrexists(RelationGetSmgr(rel), FSM_FORKNUM);
    if (fsm)
    {
        blocks[nforks] = FreeSpaceMapPrepareTruncateRel(rel, nblocks);
        if (BlockNumberIsValid(blocks[nforks]))
        {
            forks[nforks] = FSM_FORKNUM;
            nforks++;
            need_fsm_vacuum = true;
        }
    }
 
    /* Prepare for truncation of the visibility map too if it exists */
    vm = smgrexists(RelationGetSmgr(rel), VISIBILITYMAP_FORKNUM);
    if (vm)
    {
        blocks[nforks] = visibilitymap_prepare_truncate(rel, nblocks);
        if (BlockNumberIsValid(blocks[nforks]))
        {
            forks[nforks] = VISIBILITYMAP_FORKNUM;
            nforks++;
        }
    }
 
    RelationPreTruncate(rel);
 
    /*
     * Make sure that a concurrent checkpoint can't complete while truncation
     * is in progress.
     *
     * The truncation operation might drop buffers that the checkpoint
     * otherwise would have flushed. If it does, then it's essential that the
     * files actually get truncated on disk before the checkpoint record is
     * written. Otherwise, if reply begins from that checkpoint, the
     * to-be-truncated blocks might still exist on disk but have older
     * contents than expected, which can cause replay to fail. It's OK for the
     * blocks to not exist on disk at all, but not for them to have the wrong
     * contents.
     */
    Assert((MyProc->delayChkptFlags & DELAY_CHKPT_COMPLETE) == 0);
    MyProc->delayChkptFlags |= DELAY_CHKPT_COMPLETE;
 
    /*
     * We WAL-log the truncation before actually truncating, which means
     * trouble if the truncation fails. If we then crash, the WAL replay
     * likely isn't going to succeed in the truncation either, and cause a
     * PANIC. It's tempting to put a critical section here, but that cure
     * would be worse than the disease. It would turn a usually harmless
     * failure to truncate, that might spell trouble at WAL replay, into a
     * certain PANIC.
     */
    if (RelationNeedsWAL(rel))
    {
        /*
         * Make an XLOG entry reporting the file truncation.
         */
        XLogRecPtr  lsn;
        xl_smgr_truncate xlrec;
 
        xlrec.blkno = nblocks;
        xlrec.rlocator = rel->rd_locator;
        xlrec.flags = SMGR_TRUNCATE_ALL;
 
        XLogBeginInsert();
        XLogRegisterData((char *) &xlrec, sizeof(xlrec));
 
        lsn = XLogInsert(RM_SMGR_ID,
                         XLOG_SMGR_TRUNCATE | XLR_SPECIAL_REL_UPDATE);
 
        /*
         * Flush, because otherwise the truncation of the main relation might
         * hit the disk before the WAL record, and the truncation of the FSM
         * or visibility map. If we crashed during that window, we'd be left
         * with a truncated heap, but the FSM or visibility map would still
         * contain entries for the non-existent heap pages.
         */
        if (fsm || vm)
            XLogFlush(lsn);
    }
 
    /*
     * This will first remove any buffers from the buffer pool that should no
     * longer exist after truncation is complete, and then truncate the
     * corresponding files on disk.
     */
    smgrtruncate(RelationGetSmgr(rel), forks, nforks, blocks);
 
    /* We've done all the critical work, so checkpoints are OK now. */
    MyProc->delayChkptFlags &= ~DELAY_CHKPT_COMPLETE;
 
    /*
     * Update upper-level FSM pages to account for the truncation. This is
     * important because the just-truncated pages were likely marked as
     * all-free, and would be preferentially selected.
     *
     * NB: There's no point in delaying checkpoints until this is done.
     * Because the FSM is not WAL-logged, we have to be prepared for the
     * possibility of corruption after a crash anyway.
     */
    if (need_fsm_vacuum)
        FreeSpaceMapVacuumRange(rel, nblocks, InvalidBlockNumber);
}

References Assert(), xl_smgr_truncate::blkno, BlockNumberIsValid(), DELAY_CHKPT_COMPLETE, PGPROC::delayChkptFlags, xl_smgr_truncate::flags, FreeSpaceMapPrepareTruncateRel(), FreeSpaceMapVacuumRange(), FSM_FORKNUM, i, InvalidBlockNumber, MAIN_FORKNUM, MAX_FORKNUM, MyProc, RelationData::rd_locator, RelationGetSmgr(), RelationNeedsWAL, RelationPreTruncate(), xl_smgr_truncate::rlocator, SMgrRelationData::smgr_cached_nblocks, SMgrRelationData::smgr_targblock, SMGR_TRUNCATE_ALL, smgrexists(), smgrtruncate(), VISIBILITYMAP_FORKNUM, visibilitymap_prepare_truncate(), XLOG_SMGR_TRUNCATE, XLogBeginInsert(), XLogFlush(), XLogInsert(), XLogRegisterData(), and XLR_SPECIAL_REL_UPDATE.

Referenced by heapam_relation_nontransactional_truncate(), lazy_truncate_heap(), RelationTruncateIndexes(), and spgvacuumscan().

RelFileLocatorSkippingWAL()

bool RelFileLocatorSkippingWAL

(

RelFileLocator

rlocator

)

Definition at line 550 of file storage.c.

{
    if (!pendingSyncHash ||
        hash_search(pendingSyncHash, &rlocator, HASH_FIND, NULL) == NULL)
        return false;
 
    return true;
}

References HASH_FIND, hash_search(), and pendingSyncHash.

Referenced by MarkBufferDirtyHint(), and RelationInitPhysicalAddr().

RestorePendingSyncs()

void RestorePendingSyncs

(

char *

startAddress

)

Definition at line 628 of file storage.c.

{
    RelFileLocator *rlocator;
 
    Assert(pendingSyncHash == NULL);
    for (rlocator = (RelFileLocator *) startAddress; rlocator->relNumber != 0;
         rlocator++)
        AddPendingSync(rlocator);
}

References AddPendingSync(), Assert(), pendingSyncHash, and RelFileLocator::relNumber.

Referenced by ParallelWorkerMain().

SerializePendingSyncs()

void SerializePendingSyncs	(	Size	maxSize,
		char *	startAddress
	)

Definition at line 577 of file storage.c.

{
    HTAB       *tmphash;
    HASHCTL     ctl;
    HASH_SEQ_STATUS scan;
    PendingRelSync *sync;
    PendingRelDelete *delete;
    RelFileLocator *src;
    RelFileLocator *dest = (RelFileLocator *) startAddress;
 
    if (!pendingSyncHash)
        goto terminate;
 
    /* Create temporary hash to collect active relfilelocators */
    ctl.keysize = sizeof(RelFileLocator);
    ctl.entrysize = sizeof(RelFileLocator);
    ctl.hcxt = CurrentMemoryContext;
    tmphash = hash_create("tmp relfilelocators",
                          hash_get_num_entries(pendingSyncHash), &ctl,
                          HASH_ELEM | HASH_BLOBS | HASH_CONTEXT);
 
    /* collect all rlocator from pending syncs */
    hash_seq_init(&scan, pendingSyncHash);
    while ((sync = (PendingRelSync *) hash_seq_search(&scan)))
        (void) hash_search(tmphash, &sync->rlocator, HASH_ENTER, NULL);
 
    /* remove deleted rnodes */
    for (delete = pendingDeletes; delete != NULL; delete = delete->next)
        if (delete->atCommit)
            (void) hash_search(tmphash, &delete->rlocator,
                               HASH_REMOVE, NULL);
 
    hash_seq_init(&scan, tmphash);
    while ((src = (RelFileLocator *) hash_seq_search(&scan)))
        *dest++ = *src;
 
    hash_destroy(tmphash);
 
terminate:
    MemSet(dest, 0, sizeof(RelFileLocator));
}

References CurrentMemoryContext, generate_unaccent_rules::dest, HASHCTL::entrysize, HASH_BLOBS, HASH_CONTEXT, hash_create(), hash_destroy(), HASH_ELEM, HASH_ENTER, hash_get_num_entries(), HASH_REMOVE, hash_search(), hash_seq_init(), hash_seq_search(), HASHCTL::hcxt, HASHCTL::keysize, MemSet, PendingRelDelete::next, pendingDeletes, pendingSyncHash, and PendingRelSync::rlocator.

Referenced by InitializeParallelDSM().

smgr_redo()

void smgr_redo

(

XLogReaderState *

record

)

Definition at line 958 of file storage.c.

{
    XLogRecPtr  lsn = record->EndRecPtr;
    uint8       info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
 
    /* Backup blocks are not used in smgr records */
    Assert(!XLogRecHasAnyBlockRefs(record));
 
    if (info == XLOG_SMGR_CREATE)
    {
        xl_smgr_create *xlrec = (xl_smgr_create *) XLogRecGetData(record);
        SMgrRelation reln;
 
        reln = smgropen(xlrec->rlocator, InvalidBackendId);
        smgrcreate(reln, xlrec->forkNum, true);
    }
    else if (info == XLOG_SMGR_TRUNCATE)
    {
        xl_smgr_truncate *xlrec = (xl_smgr_truncate *) XLogRecGetData(record);
        SMgrRelation reln;
        Relation    rel;
        ForkNumber  forks[MAX_FORKNUM];
        BlockNumber blocks[MAX_FORKNUM];
        int         nforks = 0;
        bool        need_fsm_vacuum = false;
 
        reln = smgropen(xlrec->rlocator, InvalidBackendId);
 
        /*
         * Forcibly create relation if it doesn't exist (which suggests that
         * it was dropped somewhere later in the WAL sequence).  As in
         * XLogReadBufferForRedo, we prefer to recreate the rel and replay the
         * log as best we can until the drop is seen.
         */
        smgrcreate(reln, MAIN_FORKNUM, true);
 
        /*
         * Before we perform the truncation, update minimum recovery point to
         * cover this WAL record. Once the relation is truncated, there's no
         * going back. The buffer manager enforces the WAL-first rule for
         * normal updates to relation files, so that the minimum recovery
         * point is always updated before the corresponding change in the data
         * file is flushed to disk. We have to do the same manually here.
         *
         * Doing this before the truncation means that if the truncation fails
         * for some reason, you cannot start up the system even after restart,
         * until you fix the underlying situation so that the truncation will
         * succeed. Alternatively, we could update the minimum recovery point
         * after truncation, but that would leave a small window where the
         * WAL-first rule could be violated.
         */
        XLogFlush(lsn);
 
        /* Prepare for truncation of MAIN fork */
        if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
        {
            forks[nforks] = MAIN_FORKNUM;
            blocks[nforks] = xlrec->blkno;
            nforks++;
 
            /* Also tell xlogutils.c about it */
            XLogTruncateRelation(xlrec->rlocator, MAIN_FORKNUM, xlrec->blkno);
        }
 
        /* Prepare for truncation of FSM and VM too */
        rel = CreateFakeRelcacheEntry(xlrec->rlocator);
 
        if ((xlrec->flags & SMGR_TRUNCATE_FSM) != 0 &&
            smgrexists(reln, FSM_FORKNUM))
        {
            blocks[nforks] = FreeSpaceMapPrepareTruncateRel(rel, xlrec->blkno);
            if (BlockNumberIsValid(blocks[nforks]))
            {
                forks[nforks] = FSM_FORKNUM;
                nforks++;
                need_fsm_vacuum = true;
            }
        }
        if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0 &&
            smgrexists(reln, VISIBILITYMAP_FORKNUM))
        {
            blocks[nforks] = visibilitymap_prepare_truncate(rel, xlrec->blkno);
            if (BlockNumberIsValid(blocks[nforks]))
            {
                forks[nforks] = VISIBILITYMAP_FORKNUM;
                nforks++;
            }
        }
 
        /* Do the real work to truncate relation forks */
        if (nforks > 0)
            smgrtruncate(reln, forks, nforks, blocks);
 
        /*
         * Update upper-level FSM pages to account for the truncation. This is
         * important because the just-truncated pages were likely marked as
         * all-free, and would be preferentially selected.
         */
        if (need_fsm_vacuum)
            FreeSpaceMapVacuumRange(rel, xlrec->blkno,
                                    InvalidBlockNumber);
 
        FreeFakeRelcacheEntry(rel);
    }
    else
        elog(PANIC, "smgr_redo: unknown op code %u", info);
}

References Assert(), xl_smgr_truncate::blkno, BlockNumberIsValid(), CreateFakeRelcacheEntry(), elog(), XLogReaderState::EndRecPtr, xl_smgr_truncate::flags, xl_smgr_create::forkNum, FreeFakeRelcacheEntry(), FreeSpaceMapPrepareTruncateRel(), FreeSpaceMapVacuumRange(), FSM_FORKNUM, InvalidBackendId, InvalidBlockNumber, MAIN_FORKNUM, MAX_FORKNUM, PANIC, xl_smgr_create::rlocator, xl_smgr_truncate::rlocator, SMGR_TRUNCATE_FSM, SMGR_TRUNCATE_HEAP, SMGR_TRUNCATE_VM, smgrcreate(), smgrexists(), smgropen(), smgrtruncate(), VISIBILITYMAP_FORKNUM, visibilitymap_prepare_truncate(), XLOG_SMGR_CREATE, XLOG_SMGR_TRUNCATE, XLogFlush(), XLogRecGetData, XLogRecGetInfo, XLogRecHasAnyBlockRefs, XLogTruncateRelation(), and XLR_INFO_MASK.

smgrDoPendingDeletes()

void smgrDoPendingDeletes

(

bool

isCommit

)

Definition at line 650 of file storage.c.

{
    int         nestLevel = GetCurrentTransactionNestLevel();
    PendingRelDelete *pending;
    PendingRelDelete *prev;
    PendingRelDelete *next;
    int         nrels = 0,
                maxrels = 0;
    SMgrRelation *srels = NULL;
 
    prev = NULL;
    for (pending = pendingDeletes; pending != NULL; pending = next)
    {
        next = pending->next;
        if (pending->nestLevel < nestLevel)
        {
            /* outer-level entries should not be processed yet */
            prev = pending;
        }
        else
        {
            /* unlink list entry first, so we don't retry on failure */
            if (prev)
                prev->next = next;
            else
                pendingDeletes = next;
            /* do deletion if called for */
            if (pending->atCommit == isCommit)
            {
                SMgrRelation srel;
 
                srel = smgropen(pending->rlocator, pending->backend);
 
                /* allocate the initial array, or extend it, if needed */
                if (maxrels == 0)
                {
                    maxrels = 8;
                    srels = palloc(sizeof(SMgrRelation) * maxrels);
                }
                else if (maxrels <= nrels)
                {
                    maxrels *= 2;
                    srels = repalloc(srels, sizeof(SMgrRelation) * maxrels);
                }
 
                srels[nrels++] = srel;
            }
            /* must explicitly free the list entry */
            pfree(pending);
            /* prev does not change */
        }
    }
 
    if (nrels > 0)
    {
        smgrdounlinkall(srels, nrels, false);
 
        for (int i = 0; i < nrels; i++)
            smgrclose(srels[i]);
 
        pfree(srels);
    }
}

References PendingRelDelete::atCommit, PendingRelDelete::backend, GetCurrentTransactionNestLevel(), i, PendingRelDelete::nestLevel, next, PendingRelDelete::next, palloc(), pendingDeletes, pfree(), repalloc(), PendingRelDelete::rlocator, smgrclose(), smgrdounlinkall(), and smgropen().

Referenced by AbortTransaction(), AtSubAbort_smgr(), and CommitTransaction().

smgrDoPendingSyncs()

void smgrDoPendingSyncs	(	bool	isCommit,
		bool	isParallelWorker
	)

Definition at line 718 of file storage.c.

{
    PendingRelDelete *pending;
    int         nrels = 0,
                maxrels = 0;
    SMgrRelation *srels = NULL;
    HASH_SEQ_STATUS scan;
    PendingRelSync *pendingsync;
 
    Assert(GetCurrentTransactionNestLevel() == 1);
 
    if (!pendingSyncHash)
        return;                 /* no relation needs sync */
 
    /* Abort -- just throw away all pending syncs */
    if (!isCommit)
    {
        pendingSyncHash = NULL;
        return;
    }
 
    AssertPendingSyncs_RelationCache();
 
    /* Parallel worker -- just throw away all pending syncs */
    if (isParallelWorker)
    {
        pendingSyncHash = NULL;
        return;
    }
 
    /* Skip syncing nodes that smgrDoPendingDeletes() will delete. */
    for (pending = pendingDeletes; pending != NULL; pending = pending->next)
        if (pending->atCommit)
            (void) hash_search(pendingSyncHash, &pending->rlocator,
                               HASH_REMOVE, NULL);
 
    hash_seq_init(&scan, pendingSyncHash);
    while ((pendingsync = (PendingRelSync *) hash_seq_search(&scan)))
    {
        ForkNumber  fork;
        BlockNumber nblocks[MAX_FORKNUM + 1];
        BlockNumber total_blocks = 0;
        SMgrRelation srel;
 
        srel = smgropen(pendingsync->rlocator, InvalidBackendId);
 
        /*
         * We emit newpage WAL records for smaller relations.
         *
         * Small WAL records have a chance to be flushed along with other
         * backends' WAL records.  We emit WAL records instead of syncing for
         * files that are smaller than a certain threshold, expecting faster
         * commit.  The threshold is defined by the GUC wal_skip_threshold.
         */
        if (!pendingsync->is_truncated)
        {
            for (fork = 0; fork <= MAX_FORKNUM; fork++)
            {
                if (smgrexists(srel, fork))
                {
                    BlockNumber n = smgrnblocks(srel, fork);
 
                    /* we shouldn't come here for unlogged relations */
                    Assert(fork != INIT_FORKNUM);
                    nblocks[fork] = n;
                    total_blocks += n;
                }
                else
                    nblocks[fork] = InvalidBlockNumber;
            }
        }
 
        /*
         * Sync file or emit WAL records for its contents.
         *
         * Although we emit WAL record if the file is small enough, do file
         * sync regardless of the size if the file has experienced a
         * truncation. It is because the file would be followed by trailing
         * garbage blocks after a crash recovery if, while a past longer file
         * had been flushed out, we omitted syncing-out of the file and
         * emitted WAL instead.  You might think that we could choose WAL if
         * the current main fork is longer than ever, but there's a case where
         * main fork is longer than ever but FSM fork gets shorter.
         */
        if (pendingsync->is_truncated ||
            total_blocks * BLCKSZ / 1024 >= wal_skip_threshold)
        {
            /* allocate the initial array, or extend it, if needed */
            if (maxrels == 0)
            {
                maxrels = 8;
                srels = palloc(sizeof(SMgrRelation) * maxrels);
            }
            else if (maxrels <= nrels)
            {
                maxrels *= 2;
                srels = repalloc(srels, sizeof(SMgrRelation) * maxrels);
            }
 
            srels[nrels++] = srel;
        }
        else
        {
            /* Emit WAL records for all blocks.  The file is small enough. */
            for (fork = 0; fork <= MAX_FORKNUM; fork++)
            {
                int         n = nblocks[fork];
                Relation    rel;
 
                if (!BlockNumberIsValid(n))
                    continue;
 
                /*
                 * Emit WAL for the whole file.  Unfortunately we don't know
                 * what kind of a page this is, so we have to log the full
                 * page including any unused space.  ReadBufferExtended()
                 * counts some pgstat events; unfortunately, we discard them.
                 */
                rel = CreateFakeRelcacheEntry(srel->smgr_rlocator.locator);
                log_newpage_range(rel, fork, 0, n, false);
                FreeFakeRelcacheEntry(rel);
            }
        }
    }
 
    pendingSyncHash = NULL;
 
    if (nrels > 0)
    {
        smgrdosyncall(srels, nrels);
        pfree(srels);
    }
}

References Assert(), AssertPendingSyncs_RelationCache, PendingRelDelete::atCommit, BlockNumberIsValid(), CreateFakeRelcacheEntry(), FreeFakeRelcacheEntry(), GetCurrentTransactionNestLevel(), HASH_REMOVE, hash_search(), hash_seq_init(), hash_seq_search(), INIT_FORKNUM, InvalidBackendId, InvalidBlockNumber, PendingRelSync::is_truncated, RelFileLocatorBackend::locator, log_newpage_range(), MAX_FORKNUM, PendingRelDelete::next, palloc(), pendingDeletes, pendingSyncHash, pfree(), repalloc(), PendingRelDelete::rlocator, PendingRelSync::rlocator, SMgrRelationData::smgr_rlocator, smgrdosyncall(), smgrexists(), smgrnblocks(), smgropen(), and wal_skip_threshold.

Referenced by AbortTransaction(), CommitTransaction(), and PrepareTransaction().

smgrGetPendingDeletes()

int smgrGetPendingDeletes	(	bool	forCommit,
		RelFileLocator **	ptr
	)

Definition at line 870 of file storage.c.

{
    int         nestLevel = GetCurrentTransactionNestLevel();
    int         nrels;
    RelFileLocator *rptr;
    PendingRelDelete *pending;
 
    nrels = 0;
    for (pending = pendingDeletes; pending != NULL; pending = pending->next)
    {
        if (pending->nestLevel >= nestLevel && pending->atCommit == forCommit
            && pending->backend == InvalidBackendId)
            nrels++;
    }
    if (nrels == 0)
    {
        *ptr = NULL;
        return 0;
    }
    rptr = (RelFileLocator *) palloc(nrels * sizeof(RelFileLocator));
    *ptr = rptr;
    for (pending = pendingDeletes; pending != NULL; pending = pending->next)
    {
        if (pending->nestLevel >= nestLevel && pending->atCommit == forCommit
            && pending->backend == InvalidBackendId)
        {
            *rptr = pending->rlocator;
            rptr++;
        }
    }
    return nrels;
}

References PendingRelDelete::atCommit, PendingRelDelete::backend, GetCurrentTransactionNestLevel(), InvalidBackendId, PendingRelDelete::nestLevel, PendingRelDelete::next, palloc(), pendingDeletes, and PendingRelDelete::rlocator.

Referenced by RecordTransactionAbort(), RecordTransactionCommit(), and StartPrepare().

Variable Documentation

pendingDeletes

PendingRelDelete* pendingDeletes = NULL

static

Definition at line 75 of file storage.c.

Referenced by AtSubCommit_smgr(), PostPrepare_smgr(), RelationCreateStorage(), RelationDropStorage(), RelationPreserveStorage(), SerializePendingSyncs(), smgrDoPendingDeletes(), smgrDoPendingSyncs(), and smgrGetPendingDeletes().

pendingSyncHash

HTAB* pendingSyncHash = NULL

static

Definition at line 76 of file storage.c.

Referenced by AddPendingSync(), EstimatePendingSyncsSpace(), RelationPreTruncate(), RelFileLocatorSkippingWAL(), RestorePendingSyncs(), SerializePendingSyncs(), and smgrDoPendingSyncs().

wal_skip_threshold

int wal_skip_threshold = 2048

Definition at line 38 of file storage.c.

Referenced by smgrDoPendingSyncs().