rewriteheap.h File Reference

#include "access/htup.h"
#include "storage/itemptr.h"
#include "storage/relfilenode.h"
#include "utils/relcache.h"

Include dependency graph for rewriteheap.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures
struct	LogicalRewriteMappingData

Macros
#define	LOGICAL_REWRITE_FORMAT   "map-%x-%x-%X_%X-%x-%x"

Typedefs
typedef struct RewriteStateData *	RewriteState
typedef struct LogicalRewriteMappingData	LogicalRewriteMappingData

Functions
RewriteState	begin_heap_rewrite (Relation OldHeap, Relation NewHeap, TransactionId OldestXmin, TransactionId FreezeXid, MultiXactId MultiXactCutoff, bool use_wal)
void	end_heap_rewrite (RewriteState state)
void	rewrite_heap_tuple (RewriteState state, HeapTuple oldTuple, HeapTuple newTuple)
bool	rewrite_heap_dead_tuple (RewriteState state, HeapTuple oldTuple)
void	CheckPointLogicalRewriteHeap (void)

Macro Definition Documentation

LOGICAL_REWRITE_FORMAT

#define LOGICAL_REWRITE_FORMAT   "map-%x-%x-%X_%X-%x-%x"

Definition at line 54 of file rewriteheap.h.

Referenced by CheckPointLogicalRewriteHeap(), heap_xlog_logical_rewrite(), logical_rewrite_log_mapping(), and UpdateLogicalMappings().

Typedef Documentation

LogicalRewriteMappingData

typedef struct LogicalRewriteMappingData LogicalRewriteMappingData

RewriteState

typedef struct RewriteStateData* RewriteState

Definition at line 22 of file rewriteheap.h.

Function Documentation

begin_heap_rewrite()

RewriteState begin_heap_rewrite	(	Relation	OldHeap,
		Relation	NewHeap,
		TransactionId	OldestXmin,
		TransactionId	FreezeXid,
		MultiXactId	MultiXactCutoff,
		bool	use_wal
	)

Definition at line 239 of file rewriteheap.c.

References ALLOCSET_DEFAULT_SIZES, AllocSetContextCreate, CurrentMemoryContext, HASHCTL::entrysize, HASH_BLOBS, HASH_CONTEXT, hash_create(), HASH_ELEM, HASHCTL::hcxt, HASHCTL::keysize, logical_begin_heap_rewrite(), MemoryContextSwitchTo(), palloc(), palloc0(), RelationGetNumberOfBlocks, RewriteStateData::rs_blockno, RewriteStateData::rs_buffer, RewriteStateData::rs_buffer_valid, RewriteStateData::rs_cutoff_multi, RewriteStateData::rs_cxt, RewriteStateData::rs_freeze_xid, RewriteStateData::rs_new_rel, RewriteStateData::rs_old_new_tid_map, RewriteStateData::rs_old_rel, RewriteStateData::rs_oldest_xmin, RewriteStateData::rs_unresolved_tups, and RewriteStateData::rs_use_wal.

Referenced by heapam_relation_copy_for_cluster().

{
    RewriteState state;
    MemoryContext rw_cxt;
    MemoryContext old_cxt;
    HASHCTL     hash_ctl;

    /*
     * To ease cleanup, make a separate context that will contain the
     * RewriteState struct itself plus all subsidiary data.
     */
    rw_cxt = AllocSetContextCreate(CurrentMemoryContext,
                                   "Table rewrite",
                                   ALLOCSET_DEFAULT_SIZES);
    old_cxt = MemoryContextSwitchTo(rw_cxt);

    /* Create and fill in the state struct */
    state = palloc0(sizeof(RewriteStateData));

    state->rs_old_rel = old_heap;
    state->rs_new_rel = new_heap;
    state->rs_buffer = (Page) palloc(BLCKSZ);
    /* new_heap needn't be empty, just locked */
    state->rs_blockno = RelationGetNumberOfBlocks(new_heap);
    state->rs_buffer_valid = false;
    state->rs_use_wal = use_wal;
    state->rs_oldest_xmin = oldest_xmin;
    state->rs_freeze_xid = freeze_xid;
    state->rs_cutoff_multi = cutoff_multi;
    state->rs_cxt = rw_cxt;

    /* Initialize hash tables used to track update chains */
    memset(&hash_ctl, 0, sizeof(hash_ctl));
    hash_ctl.keysize = sizeof(TidHashKey);
    hash_ctl.entrysize = sizeof(UnresolvedTupData);
    hash_ctl.hcxt = state->rs_cxt;

    state->rs_unresolved_tups =
        hash_create("Rewrite / Unresolved ctids",
                    128,        /* arbitrary initial size */
                    &hash_ctl,
                    HASH_ELEM | HASH_BLOBS | HASH_CONTEXT);

    hash_ctl.entrysize = sizeof(OldToNewMappingData);

    state->rs_old_new_tid_map =
        hash_create("Rewrite / Old to new tid map",
                    128,        /* arbitrary initial size */
                    &hash_ctl,
                    HASH_ELEM | HASH_BLOBS | HASH_CONTEXT);

    MemoryContextSwitchTo(old_cxt);

    logical_begin_heap_rewrite(state);

    return state;
}

CheckPointLogicalRewriteHeap()

void CheckPointLogicalRewriteHeap

(

void

)

Definition at line 1213 of file rewriteheap.c.

References AllocateDir(), CloseTransientFile(), dirent::d_name, data_sync_elevel(), DEBUG1, elog, ereport, errcode_for_file_access(), errmsg(), ERROR, fd(), FreeDir(), GetRedoRecPtr(), InvalidXLogRecPtr, LOGICAL_REWRITE_FORMAT, lstat, MAXPGPATH, OpenTransientFile(), PG_BINARY, pg_fsync(), pgstat_report_wait_end(), pgstat_report_wait_start(), ReadDir(), ReplicationSlotsComputeLogicalRestartLSN(), S_ISREG, snprintf, stat, and WAIT_EVENT_LOGICAL_REWRITE_CHECKPOINT_SYNC.

Referenced by CheckPointGuts().

{
    XLogRecPtr  cutoff;
    XLogRecPtr  redo;
    DIR        *mappings_dir;
    struct dirent *mapping_de;
    char        path[MAXPGPATH + 20];

    /*
     * We start of with a minimum of the last redo pointer. No new decoding
     * slot will start before that, so that's a safe upper bound for removal.
     */
    redo = GetRedoRecPtr();

    /* now check for the restart ptrs from existing slots */
    cutoff = ReplicationSlotsComputeLogicalRestartLSN();

    /* don't start earlier than the restart lsn */
    if (cutoff != InvalidXLogRecPtr && redo < cutoff)
        cutoff = redo;

    mappings_dir = AllocateDir("pg_logical/mappings");
    while ((mapping_de = ReadDir(mappings_dir, "pg_logical/mappings")) != NULL)
    {
        struct stat statbuf;
        Oid         dboid;
        Oid         relid;
        XLogRecPtr  lsn;
        TransactionId rewrite_xid;
        TransactionId create_xid;
        uint32      hi,
                    lo;

        if (strcmp(mapping_de->d_name, ".") == 0 ||
            strcmp(mapping_de->d_name, "..") == 0)
            continue;

        snprintf(path, sizeof(path), "pg_logical/mappings/%s", mapping_de->d_name);
        if (lstat(path, &statbuf) == 0 && !S_ISREG(statbuf.st_mode))
            continue;

        /* Skip over files that cannot be ours. */
        if (strncmp(mapping_de->d_name, "map-", 4) != 0)
            continue;

        if (sscanf(mapping_de->d_name, LOGICAL_REWRITE_FORMAT,
                   &dboid, &relid, &hi, &lo, &rewrite_xid, &create_xid) != 6)
            elog(ERROR, "could not parse filename \"%s\"", mapping_de->d_name);

        lsn = ((uint64) hi) << 32 | lo;

        if (lsn < cutoff || cutoff == InvalidXLogRecPtr)
        {
            elog(DEBUG1, "removing logical rewrite file \"%s\"", path);
            if (unlink(path) < 0)
                ereport(ERROR,
                        (errcode_for_file_access(),
                         errmsg("could not remove file \"%s\": %m", path)));
        }
        else
        {
            /* on some operating systems fsyncing a file requires O_RDWR */
            int         fd = OpenTransientFile(path, O_RDWR | PG_BINARY);

            /*
             * The file cannot vanish due to concurrency since this function
             * is the only one removing logical mappings and it's run while
             * CheckpointLock is held exclusively.
             */
            if (fd < 0)
                ereport(ERROR,
                        (errcode_for_file_access(),
                         errmsg("could not open file \"%s\": %m", path)));

            /*
             * We could try to avoid fsyncing files that either haven't
             * changed or have only been created since the checkpoint's start,
             * but it's currently not deemed worth the effort.
             */
            pgstat_report_wait_start(WAIT_EVENT_LOGICAL_REWRITE_CHECKPOINT_SYNC);
            if (pg_fsync(fd) != 0)
                ereport(data_sync_elevel(ERROR),
                        (errcode_for_file_access(),
                         errmsg("could not fsync file \"%s\": %m", path)));
            pgstat_report_wait_end();

            if (CloseTransientFile(fd) != 0)
                ereport(ERROR,
                        (errcode_for_file_access(),
                         errmsg("could not close file \"%s\": %m", path)));
        }
    }
    FreeDir(mappings_dir);
}

end_heap_rewrite()

void end_heap_rewrite

(

RewriteState

state

)

Definition at line 305 of file rewriteheap.c.

References hash_seq_init(), hash_seq_search(), heap_sync(), ItemPointerSetInvalid, log_newpage(), logical_end_heap_rewrite(), MAIN_FORKNUM, MemoryContextDelete(), PageSetChecksumInplace(), raw_heap_insert(), RelationData::rd_node, RelationData::rd_smgr, RelationNeedsWAL, RelationOpenSmgr, RewriteStateData::rs_blockno, RewriteStateData::rs_buffer, RewriteStateData::rs_buffer_valid, RewriteStateData::rs_cxt, RewriteStateData::rs_new_rel, RewriteStateData::rs_unresolved_tups, RewriteStateData::rs_use_wal, smgrextend(), HeapTupleHeaderData::t_ctid, HeapTupleData::t_data, and UnresolvedTupData::tuple.

Referenced by heapam_relation_copy_for_cluster().

{
    HASH_SEQ_STATUS seq_status;
    UnresolvedTup unresolved;

    /*
     * Write any remaining tuples in the UnresolvedTups table. If we have any
     * left, they should in fact be dead, but let's err on the safe side.
     */
    hash_seq_init(&seq_status, state->rs_unresolved_tups);

    while ((unresolved = hash_seq_search(&seq_status)) != NULL)
    {
        ItemPointerSetInvalid(&unresolved->tuple->t_data->t_ctid);
        raw_heap_insert(state, unresolved->tuple);
    }

    /* Write the last page, if any */
    if (state->rs_buffer_valid)
    {
        if (state->rs_use_wal)
            log_newpage(&state->rs_new_rel->rd_node,
                        MAIN_FORKNUM,
                        state->rs_blockno,
                        state->rs_buffer,
                        true);
        RelationOpenSmgr(state->rs_new_rel);

        PageSetChecksumInplace(state->rs_buffer, state->rs_blockno);

        smgrextend(state->rs_new_rel->rd_smgr, MAIN_FORKNUM, state->rs_blockno,
                   (char *) state->rs_buffer, true);
    }

    /*
     * If the rel is WAL-logged, must fsync before commit.  We use heap_sync
     * to ensure that the toast table gets fsync'd too.
     *
     * It's obvious that we must do this when not WAL-logging. It's less
     * obvious that we have to do it even if we did WAL-log the pages. The
     * reason is the same as in storage.c's RelationCopyStorage(): we're
     * writing data that's not in shared buffers, and so a CHECKPOINT
     * occurring during the rewriteheap operation won't have fsync'd data we
     * wrote before the checkpoint.
     */
    if (RelationNeedsWAL(state->rs_new_rel))
        heap_sync(state->rs_new_rel);

    logical_end_heap_rewrite(state);

    /* Deleting the context frees everything */
    MemoryContextDelete(state->rs_cxt);
}

rewrite_heap_dead_tuple()

bool rewrite_heap_dead_tuple	(	RewriteState	state,
		HeapTuple	oldTuple
	)

Definition at line 573 of file rewriteheap.c.

References Assert, HASH_FIND, HASH_REMOVE, hash_search(), heap_freetuple(), HeapTupleHeaderGetXmin, RewriteStateData::rs_unresolved_tups, HeapTupleData::t_data, HeapTupleData::t_self, TidHashKey::tid, UnresolvedTupData::tuple, and TidHashKey::xmin.

Referenced by heapam_relation_copy_for_cluster().

{
    /*
     * If we have already seen an earlier tuple in the update chain that
     * points to this tuple, let's forget about that earlier tuple. It's in
     * fact dead as well, our simple xmax < OldestXmin test in
     * HeapTupleSatisfiesVacuum just wasn't enough to detect it. It happens
     * when xmin of a tuple is greater than xmax, which sounds
     * counter-intuitive but is perfectly valid.
     *
     * We don't bother to try to detect the situation the other way round,
     * when we encounter the dead tuple first and then the recently dead one
     * that points to it. If that happens, we'll have some unmatched entries
     * in the UnresolvedTups hash table at the end. That can happen anyway,
     * because a vacuum might have removed the dead tuple in the chain before
     * us.
     */
    UnresolvedTup unresolved;
    TidHashKey  hashkey;
    bool        found;

    memset(&hashkey, 0, sizeof(hashkey));
    hashkey.xmin = HeapTupleHeaderGetXmin(old_tuple->t_data);
    hashkey.tid = old_tuple->t_self;

    unresolved = hash_search(state->rs_unresolved_tups, &hashkey,
                             HASH_FIND, NULL);

    if (unresolved != NULL)
    {
        /* Need to free the contained tuple as well as the hashtable entry */
        heap_freetuple(unresolved->tuple);
        hash_search(state->rs_unresolved_tups, &hashkey,
                    HASH_REMOVE, &found);
        Assert(found);
        return true;
    }

    return false;
}

rewrite_heap_tuple()

void rewrite_heap_tuple	(	RewriteState	state,
		HeapTuple	oldTuple,
		HeapTuple	newTuple
	)

Definition at line 371 of file rewriteheap.c.

References Assert, HASH_ENTER, HASH_FIND, HASH_REMOVE, hash_search(), HEAP2_XACT_MASK, heap_copytuple(), heap_freetuple(), heap_freeze_tuple(), HEAP_UPDATED, HEAP_XACT_MASK, HEAP_XMAX_INVALID, HeapTupleHeaderGetUpdateXid, HeapTupleHeaderGetXmin, HeapTupleHeaderIndicatesMovedPartitions, HeapTupleHeaderIsOnlyLocked(), ItemPointerEquals(), ItemPointerSetInvalid, logical_rewrite_heap_tuple(), MemoryContextSwitchTo(), OldToNewMappingData::new_tid, UnresolvedTupData::old_tid, raw_heap_insert(), RelationData::rd_rel, RewriteStateData::rs_cutoff_multi, RewriteStateData::rs_cxt, RewriteStateData::rs_freeze_xid, RewriteStateData::rs_old_new_tid_map, RewriteStateData::rs_old_rel, RewriteStateData::rs_oldest_xmin, RewriteStateData::rs_unresolved_tups, HeapTupleHeaderData::t_choice, HeapTupleHeaderData::t_ctid, HeapTupleData::t_data, HeapTupleHeaderData::t_heap, HeapTupleHeaderData::t_infomask, HeapTupleHeaderData::t_infomask2, HeapTupleData::t_self, TidHashKey::tid, TransactionIdPrecedes(), UnresolvedTupData::tuple, and TidHashKey::xmin.

Referenced by reform_and_rewrite_tuple().

{
    MemoryContext old_cxt;
    ItemPointerData old_tid;
    TidHashKey  hashkey;
    bool        found;
    bool        free_new;

    old_cxt = MemoryContextSwitchTo(state->rs_cxt);

    /*
     * Copy the original tuple's visibility information into new_tuple.
     *
     * XXX we might later need to copy some t_infomask2 bits, too? Right now,
     * we intentionally clear the HOT status bits.
     */
    memcpy(&new_tuple->t_data->t_choice.t_heap,
           &old_tuple->t_data->t_choice.t_heap,
           sizeof(HeapTupleFields));

    new_tuple->t_data->t_infomask &= ~HEAP_XACT_MASK;
    new_tuple->t_data->t_infomask2 &= ~HEAP2_XACT_MASK;
    new_tuple->t_data->t_infomask |=
        old_tuple->t_data->t_infomask & HEAP_XACT_MASK;

    /*
     * While we have our hands on the tuple, we may as well freeze any
     * eligible xmin or xmax, so that future VACUUM effort can be saved.
     */
    heap_freeze_tuple(new_tuple->t_data,
                      state->rs_old_rel->rd_rel->relfrozenxid,
                      state->rs_old_rel->rd_rel->relminmxid,
                      state->rs_freeze_xid,
                      state->rs_cutoff_multi);

    /*
     * Invalid ctid means that ctid should point to the tuple itself. We'll
     * override it later if the tuple is part of an update chain.
     */
    ItemPointerSetInvalid(&new_tuple->t_data->t_ctid);

    /*
     * If the tuple has been updated, check the old-to-new mapping hash table.
     */
    if (!((old_tuple->t_data->t_infomask & HEAP_XMAX_INVALID) ||
          HeapTupleHeaderIsOnlyLocked(old_tuple->t_data)) &&
        !HeapTupleHeaderIndicatesMovedPartitions(old_tuple->t_data) &&
        !(ItemPointerEquals(&(old_tuple->t_self),
                            &(old_tuple->t_data->t_ctid))))
    {
        OldToNewMapping mapping;

        memset(&hashkey, 0, sizeof(hashkey));
        hashkey.xmin = HeapTupleHeaderGetUpdateXid(old_tuple->t_data);
        hashkey.tid = old_tuple->t_data->t_ctid;

        mapping = (OldToNewMapping)
            hash_search(state->rs_old_new_tid_map, &hashkey,
                        HASH_FIND, NULL);

        if (mapping != NULL)
        {
            /*
             * We've already copied the tuple that t_ctid points to, so we can
             * set the ctid of this tuple to point to the new location, and
             * insert it right away.
             */
            new_tuple->t_data->t_ctid = mapping->new_tid;

            /* We don't need the mapping entry anymore */
            hash_search(state->rs_old_new_tid_map, &hashkey,
                        HASH_REMOVE, &found);
            Assert(found);
        }
        else
        {
            /*
             * We haven't seen the tuple t_ctid points to yet. Stash this
             * tuple into unresolved_tups to be written later.
             */
            UnresolvedTup unresolved;

            unresolved = hash_search(state->rs_unresolved_tups, &hashkey,
                                     HASH_ENTER, &found);
            Assert(!found);

            unresolved->old_tid = old_tuple->t_self;
            unresolved->tuple = heap_copytuple(new_tuple);

            /*
             * We can't do anything more now, since we don't know where the
             * tuple will be written.
             */
            MemoryContextSwitchTo(old_cxt);
            return;
        }
    }

    /*
     * Now we will write the tuple, and then check to see if it is the B tuple
     * in any new or known pair.  When we resolve a known pair, we will be
     * able to write that pair's A tuple, and then we have to check if it
     * resolves some other pair.  Hence, we need a loop here.
     */
    old_tid = old_tuple->t_self;
    free_new = false;

    for (;;)
    {
        ItemPointerData new_tid;

        /* Insert the tuple and find out where it's put in new_heap */
        raw_heap_insert(state, new_tuple);
        new_tid = new_tuple->t_self;

        logical_rewrite_heap_tuple(state, old_tid, new_tuple);

        /*
         * If the tuple is the updated version of a row, and the prior version
         * wouldn't be DEAD yet, then we need to either resolve the prior
         * version (if it's waiting in rs_unresolved_tups), or make an entry
         * in rs_old_new_tid_map (so we can resolve it when we do see it). The
         * previous tuple's xmax would equal this one's xmin, so it's
         * RECENTLY_DEAD if and only if the xmin is not before OldestXmin.
         */
        if ((new_tuple->t_data->t_infomask & HEAP_UPDATED) &&
            !TransactionIdPrecedes(HeapTupleHeaderGetXmin(new_tuple->t_data),
                                   state->rs_oldest_xmin))
        {
            /*
             * Okay, this is B in an update pair.  See if we've seen A.
             */
            UnresolvedTup unresolved;

            memset(&hashkey, 0, sizeof(hashkey));
            hashkey.xmin = HeapTupleHeaderGetXmin(new_tuple->t_data);
            hashkey.tid = old_tid;

            unresolved = hash_search(state->rs_unresolved_tups, &hashkey,
                                     HASH_FIND, NULL);

            if (unresolved != NULL)
            {
                /*
                 * We have seen and memorized the previous tuple already. Now
                 * that we know where we inserted the tuple its t_ctid points
                 * to, fix its t_ctid and insert it to the new heap.
                 */
                if (free_new)
                    heap_freetuple(new_tuple);
                new_tuple = unresolved->tuple;
                free_new = true;
                old_tid = unresolved->old_tid;
                new_tuple->t_data->t_ctid = new_tid;

                /*
                 * We don't need the hash entry anymore, but don't free its
                 * tuple just yet.
                 */
                hash_search(state->rs_unresolved_tups, &hashkey,
                            HASH_REMOVE, &found);
                Assert(found);

                /* loop back to insert the previous tuple in the chain */
                continue;
            }
            else
            {
                /*
                 * Remember the new tid of this tuple. We'll use it to set the
                 * ctid when we find the previous tuple in the chain.
                 */
                OldToNewMapping mapping;

                mapping = hash_search(state->rs_old_new_tid_map, &hashkey,
                                      HASH_ENTER, &found);
                Assert(!found);

                mapping->new_tid = new_tid;
            }
        }

        /* Done with this (chain of) tuples, for now */
        if (free_new)
            heap_freetuple(new_tuple);
        break;
    }

    MemoryContextSwitchTo(old_cxt);
}