#include "postgres_fe.h"
#include <unistd.h>
#include "access/rmgr.h"
#include "access/xact.h"
#include "access/xlog_internal.h"
#include "access/xlogreader.h"
#include "catalog/pg_control.h"
#include "catalog/storage_xlog.h"
#include "commands/dbcommands_xlog.h"
#include "fe_utils/archive.h"
#include "filemap.h"
#include "pg_rewind.h"
#include "access/rmgrlist.h"

Include dependency graph for parsexlog.c:

Data Structures
struct	XLogPageReadPrivate

Macros
#define	PG_RMGR(symname, name, redo, desc, identify, startup, cleanup, mask, decode) name,

#define	RmgrName(rmid)

Typedefs
typedef struct XLogPageReadPrivate	XLogPageReadPrivate

Functions
static void	extractPageInfo (XLogReaderState *record)

static int	SimpleXLogPageRead (XLogReaderState xlogreader, XLogRecPtr targetPagePtr, int reqLen, XLogRecPtr targetRecPtr, char readBuf)

void	extractPageMap (const char datadir, XLogRecPtr startpoint, int tliIndex, XLogRecPtr endpoint, const char restoreCommand)

XLogRecPtr	readOneRecord (const char datadir, XLogRecPtr ptr, int tliIndex, const char restoreCommand)

void	findLastCheckpoint (const char datadir, XLogRecPtr forkptr, int tliIndex, XLogRecPtr lastchkptrec, TimeLineID lastchkpttli, XLogRecPtr lastchkptredo, const char *restoreCommand)

Variables
static const char *const	RmgrNames [RM_MAX_ID+1]

static int	xlogreadfd = -1

static XLogSegNo	xlogreadsegno = 0

static char	xlogfpath [MAXPGPATH]

Macro Definition Documentation

◆ PG_RMGR

#define PG_RMGR	(	symname,
		name,
		redo,
		desc,
		identify,
		startup,
		cleanup,
		mask,
		decode
	)	name,

Definition at line 31 of file parsexlog.c.

◆ RmgrName

#define RmgrName ( rmid )

Value:

(((rmid) <= RM_MAX_BUILTIN_ID) ? \

RmgrNames[rmid] : "custom")

RM_MAX_BUILTIN_ID

#define RM_MAX_BUILTIN_ID

Definition: rmgr.h:34

Definition at line 38 of file parsexlog.c.

Typedef Documentation

◆ XLogPageReadPrivate

typedef struct XLogPageReadPrivate XLogPageReadPrivate

Function Documentation

◆ extractPageInfo()

static void extractPageInfo ( XLogReaderState * record )

static

Definition at line 389 of file parsexlog.c.

{
    int         block_id;
    RmgrId      rmid = XLogRecGetRmid(record);
    uint8       info = XLogRecGetInfo(record);
    uint8       rminfo = info & ~XLR_INFO_MASK;
 
    /* Is this a special record type that I recognize? */
 
    if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE_FILE_COPY)
    {
        /*
         * New databases can be safely ignored. It won't be present in the
         * source system, so it will be deleted. There's one corner-case,
         * though: if a new, different, database is also created in the source
         * system, we'll see that the files already exist and not copy them.
         * That's OK, though; WAL replay of creating the new database, from
         * the source systems's WAL, will re-copy the new database,
         * overwriting the database created in the target system.
         */
    }
    else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE_WAL_LOG)
    {
        /*
         * New databases can be safely ignored. It won't be present in the
         * source system, so it will be deleted.
         */
    }
    else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP)
    {
        /*
         * An existing database was dropped. We'll see that the files don't
         * exist in the target data dir, and copy them in toto from the source
         * system. No need to do anything special here.
         */
    }
    else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE)
    {
        /*
         * We can safely ignore these. The file will be removed from the
         * target, if it doesn't exist in source system. If a file with same
         * name is created in source system, too, there will be WAL records
         * for all the blocks in it.
         */
    }
    else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE)
    {
        /*
         * We can safely ignore these. When we compare the sizes later on,
         * we'll notice that they differ, and copy the missing tail from
         * source system.
         */
    }
    else if (rmid == RM_XACT_ID &&
             ((rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT ||
              (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT_PREPARED ||
              (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_ABORT ||
              (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_ABORT_PREPARED))
    {
        /*
         * These records can include "dropped rels". We can safely ignore
         * them, we will see that they are missing and copy them from the
         * source.
         */
    }
    else if (info & XLR_SPECIAL_REL_UPDATE)
    {
        /*
         * This record type modifies a relation file in some special way, but
         * we don't recognize the type. That's bad - we don't know how to
         * track that change.
         */
        pg_fatal("WAL record modifies a relation, but record type is not recognized: "
                 "lsn: %X/%X, rmid: %d, rmgr: %s, info: %02X",
                 LSN_FORMAT_ARGS(record->ReadRecPtr),
                 rmid, RmgrName(rmid), info);
    }
 
    for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++)
    {
        RelFileLocator rlocator;
        ForkNumber  forknum;
        BlockNumber blkno;
 
        if (!XLogRecGetBlockTagExtended(record, block_id,
                                        &rlocator, &forknum, &blkno, NULL))
            continue;
 
        /* We only care about the main fork; others are copied in toto */
        if (forknum != MAIN_FORKNUM)
            continue;
 
        process_target_wal_block_change(forknum, rlocator, blkno);
    }
}

References LSN_FORMAT_ARGS, MAIN_FORKNUM, pg_fatal, process_target_wal_block_change(), XLogReaderState::ReadRecPtr, RmgrName, XLOG_DBASE_CREATE_FILE_COPY, XLOG_DBASE_CREATE_WAL_LOG, XLOG_DBASE_DROP, XLOG_SMGR_CREATE, XLOG_SMGR_TRUNCATE, XLOG_XACT_ABORT, XLOG_XACT_ABORT_PREPARED, XLOG_XACT_COMMIT, XLOG_XACT_COMMIT_PREPARED, XLOG_XACT_OPMASK, XLogRecGetBlockTagExtended(), XLogRecGetInfo, XLogRecGetRmid, XLogRecMaxBlockId, and XLR_SPECIAL_REL_UPDATE.

Referenced by extractPageMap().

◆ extractPageMap()

void extractPageMap	(	const char *	datadir,
		XLogRecPtr	startpoint,
		int	tliIndex,
		XLogRecPtr	endpoint,
		const char *	restoreCommand
	)

Definition at line 66 of file parsexlog.c.

{
    XLogRecord *record;
    XLogReaderState *xlogreader;
    char       *errormsg;
    XLogPageReadPrivate private;
 
    private.tliIndex = tliIndex;
    private.restoreCommand = restoreCommand;
    xlogreader = XLogReaderAllocate(WalSegSz, datadir,
                                    XL_ROUTINE(.page_read = &SimpleXLogPageRead),
                                    &private);
    if (xlogreader == NULL)
        pg_fatal("out of memory while allocating a WAL reading processor");
 
    XLogBeginRead(xlogreader, startpoint);
    do
    {
        record = XLogReadRecord(xlogreader, &errormsg);
 
        if (record == NULL)
        {
            XLogRecPtr  errptr = xlogreader->EndRecPtr;
 
            if (errormsg)
                pg_fatal("could not read WAL record at %X/%X: %s",
                         LSN_FORMAT_ARGS(errptr),
                         errormsg);
            else
                pg_fatal("could not read WAL record at %X/%X",
                         LSN_FORMAT_ARGS(errptr));
        }
 
        extractPageInfo(xlogreader);
    } while (xlogreader->EndRecPtr < endpoint);
 
    /*
     * If 'endpoint' didn't point exactly at a record boundary, the caller
     * messed up.
     */
    if (xlogreader->EndRecPtr != endpoint)
        pg_fatal("end pointer %X/%X is not a valid end point; expected %X/%X",
                 LSN_FORMAT_ARGS(endpoint), LSN_FORMAT_ARGS(xlogreader->EndRecPtr));
 
    XLogReaderFree(xlogreader);
    if (xlogreadfd != -1)
    {
        close(xlogreadfd);
        xlogreadfd = -1;
    }
}

References close, datadir, XLogReaderState::EndRecPtr, extractPageInfo(), LSN_FORMAT_ARGS, pg_fatal, SimpleXLogPageRead(), XLogPageReadPrivate::tliIndex, WalSegSz, XL_ROUTINE, XLogBeginRead(), xlogreader, XLogReaderAllocate(), XLogReaderFree(), xlogreadfd, and XLogReadRecord().

Referenced by main().

◆ findLastCheckpoint()

void findLastCheckpoint	(	const char *	datadir,
		XLogRecPtr	forkptr,
		int	tliIndex,
		XLogRecPtr *	lastchkptrec,
		TimeLineID *	lastchkpttli,
		XLogRecPtr *	lastchkptredo,
		const char *	restoreCommand
	)

Definition at line 168 of file parsexlog.c.

{
    /* Walk backwards, starting from the given record */
    XLogRecord *record;
    XLogRecPtr  searchptr;
    XLogReaderState *xlogreader;
    char       *errormsg;
    XLogPageReadPrivate private;
    XLogSegNo   current_segno = 0;
    TimeLineID  current_tli = 0;
 
    /*
     * The given fork pointer points to the end of the last common record,
     * which is not necessarily the beginning of the next record, if the
     * previous record happens to end at a page boundary. Skip over the page
     * header in that case to find the next record.
     */
    if (forkptr % XLOG_BLCKSZ == 0)
    {
        if (XLogSegmentOffset(forkptr, WalSegSz) == 0)
            forkptr += SizeOfXLogLongPHD;
        else
            forkptr += SizeOfXLogShortPHD;
    }
 
    private.tliIndex = tliIndex;
    private.restoreCommand = restoreCommand;
    xlogreader = XLogReaderAllocate(WalSegSz, datadir,
                                    XL_ROUTINE(.page_read = &SimpleXLogPageRead),
                                    &private);
    if (xlogreader == NULL)
        pg_fatal("out of memory while allocating a WAL reading processor");
 
    searchptr = forkptr;
    for (;;)
    {
        uint8       info;
 
        XLogBeginRead(xlogreader, searchptr);
        record = XLogReadRecord(xlogreader, &errormsg);
 
        if (record == NULL)
        {
            if (errormsg)
                pg_fatal("could not find previous WAL record at %X/%X: %s",
                         LSN_FORMAT_ARGS(searchptr),
                         errormsg);
            else
                pg_fatal("could not find previous WAL record at %X/%X",
                         LSN_FORMAT_ARGS(searchptr));
        }
 
        /* Detect if a new WAL file has been opened */
        if (xlogreader->seg.ws_tli != current_tli ||
            xlogreader->seg.ws_segno != current_segno)
        {
            char        xlogfname[MAXFNAMELEN];
 
            snprintf(xlogfname, MAXFNAMELEN, XLOGDIR "/");
 
            /* update current values */
            current_tli = xlogreader->seg.ws_tli;
            current_segno = xlogreader->seg.ws_segno;
 
            XLogFileName(xlogfname + sizeof(XLOGDIR),
                         current_tli, current_segno, WalSegSz);
 
            /* Track this filename as one to not remove */
            keepwal_add_entry(xlogfname);
        }
 
        /*
         * Check if it is a checkpoint record. This checkpoint record needs to
         * be the latest checkpoint before WAL forked and not the checkpoint
         * where the primary has been stopped to be rewound.
         */
        info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
        if (searchptr < forkptr &&
            XLogRecGetRmid(xlogreader) == RM_XLOG_ID &&
            (info == XLOG_CHECKPOINT_SHUTDOWN ||
             info == XLOG_CHECKPOINT_ONLINE))
        {
            CheckPoint  checkPoint;
 
            memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
            *lastchkptrec = searchptr;
            *lastchkpttli = checkPoint.ThisTimeLineID;
            *lastchkptredo = checkPoint.redo;
            break;
        }
 
        /* Walk backwards to previous record. */
        searchptr = record->xl_prev;
    }
 
    XLogReaderFree(xlogreader);
    if (xlogreadfd != -1)
    {
        close(xlogreadfd);
        xlogreadfd = -1;
    }
}

References close, datadir, keepwal_add_entry(), LSN_FORMAT_ARGS, MAXFNAMELEN, pg_fatal, CheckPoint::redo, XLogReaderState::seg, SimpleXLogPageRead(), SizeOfXLogLongPHD, SizeOfXLogShortPHD, snprintf, CheckPoint::ThisTimeLineID, WalSegSz, WALOpenSegment::ws_segno, WALOpenSegment::ws_tli, XLogRecord::xl_prev, XL_ROUTINE, XLOG_CHECKPOINT_ONLINE, XLOG_CHECKPOINT_SHUTDOWN, XLogBeginRead(), XLOGDIR, XLogFileName(), xlogreader, XLogReaderAllocate(), XLogReaderFree(), xlogreadfd, XLogReadRecord(), XLogRecGetData, XLogRecGetInfo, XLogRecGetRmid, and XLogSegmentOffset.

Referenced by main().

◆ readOneRecord()

XLogRecPtr readOneRecord	(	const char *	datadir,
		XLogRecPtr	ptr,
		int	tliIndex,
		const char *	restoreCommand
	)

Definition at line 124 of file parsexlog.c.

{
    XLogRecord *record;
    XLogReaderState *xlogreader;
    char       *errormsg;
    XLogPageReadPrivate private;
    XLogRecPtr  endptr;
 
    private.tliIndex = tliIndex;
    private.restoreCommand = restoreCommand;
    xlogreader = XLogReaderAllocate(WalSegSz, datadir,
                                    XL_ROUTINE(.page_read = &SimpleXLogPageRead),
                                    &private);
    if (xlogreader == NULL)
        pg_fatal("out of memory while allocating a WAL reading processor");
 
    XLogBeginRead(xlogreader, ptr);
    record = XLogReadRecord(xlogreader, &errormsg);
    if (record == NULL)
    {
        if (errormsg)
            pg_fatal("could not read WAL record at %X/%X: %s",
                     LSN_FORMAT_ARGS(ptr), errormsg);
        else
            pg_fatal("could not read WAL record at %X/%X",
                     LSN_FORMAT_ARGS(ptr));
    }
    endptr = xlogreader->EndRecPtr;
 
    XLogReaderFree(xlogreader);
    if (xlogreadfd != -1)
    {
        close(xlogreadfd);
        xlogreadfd = -1;
    }
 
    return endptr;
}

References close, datadir, XLogReaderState::EndRecPtr, LSN_FORMAT_ARGS, pg_fatal, SimpleXLogPageRead(), XLogPageReadPrivate::tliIndex, WalSegSz, XL_ROUTINE, XLogBeginRead(), xlogreader, XLogReaderAllocate(), XLogReaderFree(), xlogreadfd, and XLogReadRecord().

Referenced by main().

◆ SimpleXLogPageRead()

static int SimpleXLogPageRead	(	XLogReaderState *	xlogreader,
		XLogRecPtr	targetPagePtr,
		int	reqLen,
		XLogRecPtr	targetRecPtr,
		char *	readBuf
	)

static

Definition at line 275 of file parsexlog.c.

{
    XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
    uint32      targetPageOff;
    XLogRecPtr  targetSegEnd;
    XLogSegNo   targetSegNo;
    int         r;
 
    XLByteToSeg(targetPagePtr, targetSegNo, WalSegSz);
    XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, WalSegSz, targetSegEnd);
    targetPageOff = XLogSegmentOffset(targetPagePtr, WalSegSz);
 
    /*
     * See if we need to switch to a new segment because the requested record
     * is not in the currently open one.
     */
    if (xlogreadfd >= 0 &&
        !XLByteInSeg(targetPagePtr, xlogreadsegno, WalSegSz))
    {
        close(xlogreadfd);
        xlogreadfd = -1;
    }
 
    XLByteToSeg(targetPagePtr, xlogreadsegno, WalSegSz);
 
    if (xlogreadfd < 0)
    {
        char        xlogfname[MAXFNAMELEN];
 
        /*
         * Since incomplete segments are copied into next timelines, switch to
         * the timeline holding the required segment. Assuming this scan can
         * be done both forward and backward, consider also switching timeline
         * accordingly.
         */
        while (private->tliIndex < targetNentries - 1 &&
               targetHistory[private->tliIndex].end < targetSegEnd)
            private->tliIndex++;
        while (private->tliIndex > 0 &&
               targetHistory[private->tliIndex].begin >= targetSegEnd)
            private->tliIndex--;
 
        XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
                     xlogreadsegno, WalSegSz);
 
        snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
                 xlogreader->segcxt.ws_dir, xlogfname);
 
        xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
 
        if (xlogreadfd < 0)
        {
            /*
             * If we have no restore_command to execute, then exit.
             */
            if (private->restoreCommand == NULL)
            {
                pg_log_error("could not open file \"%s\": %m", xlogfpath);
                return -1;
            }
 
            /*
             * Since we have restore_command, then try to retrieve missing WAL
             * file from the archive.
             */
            xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
                                             xlogfname,
                                             WalSegSz,
                                             private->restoreCommand);
 
            if (xlogreadfd < 0)
                return -1;
            else
                pg_log_debug("using file \"%s\" restored from archive",
                             xlogfpath);
        }
    }
 
    /*
     * At this point, we have the right segment open.
     */
    Assert(xlogreadfd != -1);
 
    /* Read the requested page */
    if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
    {
        pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
        return -1;
    }
 
 
    r = read(xlogreadfd, readBuf, XLOG_BLCKSZ);
    if (r != XLOG_BLCKSZ)
    {
        if (r < 0)
            pg_log_error("could not read file \"%s\": %m", xlogfpath);
        else
            pg_log_error("could not read file \"%s\": read %d of %zu",
                         xlogfpath, r, (Size) XLOG_BLCKSZ);
 
        return -1;
    }
 
    Assert(targetSegNo == xlogreadsegno);
 
    xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
    return XLOG_BLCKSZ;
}

References Assert(), TimeLineHistoryEntry::begin, close, TimeLineHistoryEntry::end, MAXFNAMELEN, MAXPGPATH, PG_BINARY, pg_log_debug, pg_log_error, XLogReaderState::private_data, read, RestoreArchivedFile(), XLogReaderState::seg, XLogReaderState::segcxt, snprintf, targetHistory, targetNentries, TimeLineHistoryEntry::tli, WalSegSz, WALSegmentContext::ws_dir, WALOpenSegment::ws_tli, XLByteInSeg, XLByteToSeg, XLOGDIR, XLogFileName(), xlogfpath, xlogreader, xlogreadfd, xlogreadsegno, XLogSegmentOffset, and XLogSegNoOffsetToRecPtr.

Referenced by extractPageMap(), findLastCheckpoint(), and readOneRecord().

Variable Documentation

◆ RmgrNames

const char* const RmgrNames[RM_MAX_ID+1]

static

Initial value:

= {

}

Definition at line 34 of file parsexlog.c.

◆ xlogfpath

char xlogfpath[MAXPGPATH]

static

Definition at line 45 of file parsexlog.c.

Referenced by KeepFileRestoredFromArchive(), and SimpleXLogPageRead().

◆ xlogreadfd

int xlogreadfd = -1

static

Definition at line 43 of file parsexlog.c.

Referenced by extractPageMap(), findLastCheckpoint(), readOneRecord(), and SimpleXLogPageRead().

◆ xlogreadsegno

XLogSegNo xlogreadsegno = 0

static

Definition at line 44 of file parsexlog.c.

Referenced by SimpleXLogPageRead().

Data Structures

Macros

Typedefs

Functions

Variables

Macro Definition Documentation

◆ PG_RMGR

◆ RmgrName

Typedef Documentation

◆ XLogPageReadPrivate

Function Documentation

◆ extractPageInfo()

◆ extractPageMap()

◆ findLastCheckpoint()

◆ readOneRecord()

◆ SimpleXLogPageRead()

Variable Documentation

◆ RmgrNames

◆ xlogfpath

◆ xlogreadfd

◆ xlogreadsegno