basebackup_incremental.c File Reference

#include "postgres.h"
#include "access/timeline.h"
#include "access/xlog.h"
#include "backup/basebackup_incremental.h"
#include "backup/walsummary.h"
#include "common/blkreftable.h"
#include "common/hashfn.h"
#include "common/int.h"
#include "common/parse_manifest.h"
#include "datatype/timestamp.h"
#include "postmaster/walsummarizer.h"
#include "utils/timestamp.h"
#include "lib/simplehash.h"

Include dependency graph for basebackup_incremental.c:

Go to the source code of this file.

Data Structures
struct	backup_wal_range
struct	backup_file_entry
struct	IncrementalBackupInfo

Macros
#define	BLOCKS_PER_READ   512
#define	MIN_CHUNK   1024
#define	MAX_CHUNK   (128 * 1024)
#define	SH_PREFIX   backup_file
#define	SH_ELEMENT_TYPE   backup_file_entry
#define	SH_KEY_TYPE   const char *
#define	SH_KEY   path
#define	SH_HASH_KEY(tb, key)   hash_string_pointer(key)
#define	SH_EQUAL(tb, a, b)   (strcmp(a, b) == 0)
#define	SH_SCOPE   static inline
#define	SH_DECLARE
#define	SH_DEFINE

Functions
static uint32	hash_string_pointer (const char *s)
static void	manifest_process_version (JsonManifestParseContext *context, int manifest_version)
static void	manifest_process_system_identifier (JsonManifestParseContext *context, uint64 manifest_system_identifier)
static void	manifest_process_file (JsonManifestParseContext *context, const char *pathname, size_t size, pg_checksum_type checksum_type, int checksum_length, uint8 *checksum_payload)
static void	manifest_process_wal_range (JsonManifestParseContext *context, TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
static void	manifest_report_error (JsonManifestParseContext *context, const char *fmt,...) pg_attribute_printf(2
static void	pg_attribute_noreturn ()
static int	compare_block_numbers (const void *a, const void *b)
IncrementalBackupInfo *	CreateIncrementalBackupInfo (MemoryContext mcxt)
void	AppendIncrementalManifestData (IncrementalBackupInfo *ib, const char *data, int len)
void	FinalizeIncrementalManifest (IncrementalBackupInfo *ib)
void	PrepareForIncrementalBackup (IncrementalBackupInfo *ib, BackupState *backup_state)
char *	GetIncrementalFilePath (Oid dboid, Oid spcoid, RelFileNumber relfilenumber, ForkNumber forknum, unsigned segno)
FileBackupMethod	GetFileBackupMethod (IncrementalBackupInfo *ib, const char *path, Oid dboid, Oid spcoid, RelFileNumber relfilenumber, ForkNumber forknum, unsigned segno, size_t size, unsigned *num_blocks_required, BlockNumber *relative_block_numbers, unsigned *truncation_block_length)
size_t	GetIncrementalHeaderSize (unsigned num_blocks_required)
size_t	GetIncrementalFileSize (unsigned num_blocks_required)

Macro Definition Documentation

BLOCKS_PER_READ

#define BLOCKS_PER_READ   512

Definition at line 34 of file basebackup_incremental.c.

MAX_CHUNK

#define MAX_CHUNK   (128 * 1024)

Definition at line 42 of file basebackup_incremental.c.

MIN_CHUNK

#define MIN_CHUNK   1024

Definition at line 41 of file basebackup_incremental.c.

SH_DECLARE

#define SH_DECLARE

Definition at line 72 of file basebackup_incremental.c.

SH_DEFINE

#define SH_DEFINE

Definition at line 73 of file basebackup_incremental.c.

SH_ELEMENT_TYPE

#define SH_ELEMENT_TYPE   backup_file_entry

Definition at line 66 of file basebackup_incremental.c.

SH_EQUAL

#define SH_EQUAL	(		tb,
			a,
			b
	)		(strcmp(a, b) == 0)

Definition at line 70 of file basebackup_incremental.c.

SH_HASH_KEY

#define SH_HASH_KEY	(		tb,
			key
	)		hash_string_pointer(key)

Definition at line 69 of file basebackup_incremental.c.

SH_KEY

#define SH_KEY   path

Definition at line 68 of file basebackup_incremental.c.

SH_KEY_TYPE

#define SH_KEY_TYPE   const char *

Definition at line 67 of file basebackup_incremental.c.

SH_PREFIX

#define SH_PREFIX   backup_file

Definition at line 65 of file basebackup_incremental.c.

SH_SCOPE

#define SH_SCOPE   static inline

Definition at line 71 of file basebackup_incremental.c.

Function Documentation

AppendIncrementalManifestData()

void AppendIncrementalManifestData	(	IncrementalBackupInfo *	ib,
		const char *	data,
		int	len
	)

Definition at line 196 of file basebackup_incremental.c.

{
    MemoryContext oldcontext;
 
    /* Switch to our memory context. */
    oldcontext = MemoryContextSwitchTo(ib->mcxt);
 
    if (ib->buf.len > MIN_CHUNK && ib->buf.len + len > MAX_CHUNK)
    {
        /*
         * time for an incremental parse. We'll do all but the last MIN_CHUNK
         * so that we have enough left for the final piece.
         */
        json_parse_manifest_incremental_chunk(
                                              ib->inc_state, ib->buf.data, ib->buf.len - MIN_CHUNK, false);
        /* now remove what we just parsed  */
        memmove(ib->buf.data, ib->buf.data + (ib->buf.len - MIN_CHUNK),
                MIN_CHUNK + 1);
        ib->buf.len = MIN_CHUNK;
    }
 
    appendBinaryStringInfo(&ib->buf, data, len);
 
    /* Switch back to previous memory context. */
    MemoryContextSwitchTo(oldcontext);
}

References appendBinaryStringInfo(), IncrementalBackupInfo::buf, StringInfoData::data, data, IncrementalBackupInfo::inc_state, json_parse_manifest_incremental_chunk(), StringInfoData::len, len, MAX_CHUNK, IncrementalBackupInfo::mcxt, MemoryContextSwitchTo(), and MIN_CHUNK.

Referenced by HandleUploadManifestPacket().

compare_block_numbers()

static int compare_block_numbers ( const void *  a, const void *  b  )

static

Definition at line 1118 of file basebackup_incremental.c.

{
    BlockNumber aa = *(BlockNumber *) a;
    BlockNumber bb = *(BlockNumber *) b;
 
    return pg_cmp_u32(aa, bb);
}

References a, b, and pg_cmp_u32().

Referenced by GetFileBackupMethod().

CreateIncrementalBackupInfo()

IncrementalBackupInfo* CreateIncrementalBackupInfo

(

MemoryContext

mcxt

)

Definition at line 154 of file basebackup_incremental.c.

{
    IncrementalBackupInfo *ib;
    MemoryContext oldcontext;
    JsonManifestParseContext *context;
 
    oldcontext = MemoryContextSwitchTo(mcxt);
 
    ib = palloc0(sizeof(IncrementalBackupInfo));
    ib->mcxt = mcxt;
    initStringInfo(&ib->buf);
 
    /*
     * It's hard to guess how many files a "typical" installation will have in
     * the data directory, but a fresh initdb creates almost 1000 files as of
     * this writing, so it seems to make sense for our estimate to
     * substantially higher.
     */
    ib->manifest_files = backup_file_create(mcxt, 10000, NULL);
 
    context = palloc0(sizeof(JsonManifestParseContext));
    /* Parse the manifest. */
    context->private_data = ib;
    context->version_cb = manifest_process_version;
    context->system_identifier_cb = manifest_process_system_identifier;
    context->per_file_cb = manifest_process_file;
    context->per_wal_range_cb = manifest_process_wal_range;
    context->error_cb = manifest_report_error;
 
    ib->inc_state = json_parse_manifest_incremental_init(context);
 
    MemoryContextSwitchTo(oldcontext);
 
    return ib;
}

References IncrementalBackupInfo::buf, context, IncrementalBackupInfo::inc_state, initStringInfo(), json_parse_manifest_incremental_init(), IncrementalBackupInfo::manifest_files, manifest_process_file(), manifest_process_system_identifier(), manifest_process_version(), manifest_process_wal_range(), manifest_report_error(), IncrementalBackupInfo::mcxt, MemoryContextSwitchTo(), and palloc0().

Referenced by UploadManifest().

FinalizeIncrementalManifest()

void FinalizeIncrementalManifest

(

IncrementalBackupInfo *

ib

)

Definition at line 229 of file basebackup_incremental.c.

{
    MemoryContext oldcontext;
 
    /* Switch to our memory context. */
    oldcontext = MemoryContextSwitchTo(ib->mcxt);
 
    /* Parse the last chunk of the manifest */
    json_parse_manifest_incremental_chunk(
                                          ib->inc_state, ib->buf.data, ib->buf.len, true);
 
    /* Done with the buffer, so release memory. */
    pfree(ib->buf.data);
    ib->buf.data = NULL;
 
    /* Done with inc_state, so release that memory too */
    json_parse_manifest_incremental_shutdown(ib->inc_state);
 
    /* Switch back to previous memory context. */
    MemoryContextSwitchTo(oldcontext);
}

References IncrementalBackupInfo::buf, StringInfoData::data, IncrementalBackupInfo::inc_state, json_parse_manifest_incremental_chunk(), json_parse_manifest_incremental_shutdown(), StringInfoData::len, IncrementalBackupInfo::mcxt, MemoryContextSwitchTo(), and pfree().

Referenced by UploadManifest().

GetFileBackupMethod()

FileBackupMethod GetFileBackupMethod	(	IncrementalBackupInfo *	ib,
		const char *	path,
		Oid	dboid,
		Oid	spcoid,
		RelFileNumber	relfilenumber,
		ForkNumber	forknum,
		unsigned	segno,
		size_t	size,
		unsigned *	num_blocks_required,
		BlockNumber *	relative_block_numbers,
		unsigned *	truncation_block_length
	)

Definition at line 745 of file basebackup_incremental.c.

{
    BlockNumber limit_block;
    BlockNumber start_blkno;
    BlockNumber stop_blkno;
    RelFileLocator rlocator;
    BlockRefTableEntry *brtentry;
    unsigned    i;
    unsigned    nblocks;
 
    /* Should only be called after PrepareForIncrementalBackup. */
    Assert(ib->buf.data == NULL);
 
    /*
     * dboid could be InvalidOid if shared rel, but spcoid and relfilenumber
     * should have legal values.
     */
    Assert(OidIsValid(spcoid));
    Assert(RelFileNumberIsValid(relfilenumber));
 
    /*
     * If the file size is too large or not a multiple of BLCKSZ, then
     * something weird is happening, so give up and send the whole file.
     */
    if ((size % BLCKSZ) != 0 || size / BLCKSZ > RELSEG_SIZE)
        return BACK_UP_FILE_FULLY;
 
    /*
     * The free-space map fork is not properly WAL-logged, so we need to
     * backup the entire file every time.
     */
    if (forknum == FSM_FORKNUM)
        return BACK_UP_FILE_FULLY;
 
    /*
     * If this file was not part of the prior backup, back it up fully.
     *
     * If this file was created after the prior backup and before the start of
     * the current backup, then the WAL summary information will tell us to
     * back up the whole file. However, if this file was created after the
     * start of the current backup, then the WAL summary won't know anything
     * about it. Without this logic, we would erroneously conclude that it was
     * OK to send it incrementally.
     *
     * Note that the file could have existed at the time of the prior backup,
     * gotten deleted, and then a new file with the same name could have been
     * created.  In that case, this logic won't prevent the file from being
     * backed up incrementally. But, if the deletion happened before the start
     * of the current backup, the limit block will be 0, inducing a full
     * backup. If the deletion happened after the start of the current backup,
     * reconstruction will erroneously combine blocks from the current
     * lifespan of the file with blocks from the previous lifespan -- but in
     * this type of case, WAL replay to reach backup consistency should remove
     * and recreate the file anyway, so the initial bogus contents should not
     * matter.
     */
    if (backup_file_lookup(ib->manifest_files, path) == NULL)
    {
        char       *ipath;
 
        ipath = GetIncrementalFilePath(dboid, spcoid, relfilenumber,
                                       forknum, segno);
        if (backup_file_lookup(ib->manifest_files, ipath) == NULL)
            return BACK_UP_FILE_FULLY;
    }
 
    /*
     * Look up the special block reference table entry for the database as a
     * whole.
     */
    rlocator.spcOid = spcoid;
    rlocator.dbOid = dboid;
    rlocator.relNumber = 0;
    if (BlockRefTableGetEntry(ib->brtab, &rlocator, MAIN_FORKNUM,
                              &limit_block) != NULL)
    {
        /*
         * According to the WAL summary, this database OID/tablespace OID
         * pairing has been created since the previous backup. So, everything
         * in it must be backed up fully.
         */
        return BACK_UP_FILE_FULLY;
    }
 
    /* Look up the block reference table entry for this relfilenode. */
    rlocator.relNumber = relfilenumber;
    brtentry = BlockRefTableGetEntry(ib->brtab, &rlocator, forknum,
                                     &limit_block);
 
    /*
     * If there is no entry, then there have been no WAL-logged changes to the
     * relation since the predecessor backup was taken, so we can back it up
     * incrementally and need not include any modified blocks.
     *
     * However, if the file is zero-length, we should do a full backup,
     * because an incremental file is always more than zero length, and it's
     * silly to take an incremental backup when a full backup would be
     * smaller.
     */
    if (brtentry == NULL)
    {
        if (size == 0)
            return BACK_UP_FILE_FULLY;
        *num_blocks_required = 0;
        *truncation_block_length = size / BLCKSZ;
        return BACK_UP_FILE_INCREMENTALLY;
    }
 
    /*
     * If the limit_block is less than or equal to the point where this
     * segment starts, send the whole file.
     */
    if (limit_block <= segno * RELSEG_SIZE)
        return BACK_UP_FILE_FULLY;
 
    /*
     * Get relevant entries from the block reference table entry.
     *
     * We shouldn't overflow computing the start or stop block numbers, but if
     * it manages to happen somehow, detect it and throw an error.
     */
    start_blkno = segno * RELSEG_SIZE;
    stop_blkno = start_blkno + (size / BLCKSZ);
    if (start_blkno / RELSEG_SIZE != segno || stop_blkno < start_blkno)
        ereport(ERROR,
                errcode(ERRCODE_INTERNAL_ERROR),
                errmsg_internal("overflow computing block number bounds for segment %u with size %zu",
                                segno, size));
 
    /*
     * This will write *absolute* block numbers into the output array, but
     * we'll transpose them below.
     */
    nblocks = BlockRefTableEntryGetBlocks(brtentry, start_blkno, stop_blkno,
                                          relative_block_numbers, RELSEG_SIZE);
    Assert(nblocks <= RELSEG_SIZE);
 
    /*
     * If we're going to have to send nearly all of the blocks, then just send
     * the whole file, because that won't require much extra storage or
     * transfer and will speed up and simplify backup restoration. It's not
     * clear what threshold is most appropriate here and perhaps it ought to
     * be configurable, but for now we're just going to say that if we'd need
     * to send 90% of the blocks anyway, give up and send the whole file.
     *
     * NB: If you change the threshold here, at least make sure to back up the
     * file fully when every single block must be sent, because there's
     * nothing good about sending an incremental file in that case.
     */
    if (nblocks * BLCKSZ > size * 0.9)
        return BACK_UP_FILE_FULLY;
 
    /*
     * Looks like we can send an incremental file, so sort the block numbers
     * and then transpose them from absolute block numbers to relative block
     * numbers if necessary.
     *
     * NB: If the block reference table was using the bitmap representation
     * for a given chunk, the block numbers in that chunk will already be
     * sorted, but when the array-of-offsets representation is used, we can
     * receive block numbers here out of order.
     */
    qsort(relative_block_numbers, nblocks, sizeof(BlockNumber),
          compare_block_numbers);
    if (start_blkno != 0)
    {
        for (i = 0; i < nblocks; ++i)
            relative_block_numbers[i] -= start_blkno;
    }
    *num_blocks_required = nblocks;
 
    /*
     * The truncation block length is the minimum length of the reconstructed
     * file. Any block numbers below this threshold that are not present in
     * the backup need to be fetched from the prior backup. At or above this
     * threshold, blocks should only be included in the result if they are
     * present in the backup. (This may require inserting zero blocks if the
     * blocks included in the backup are non-consecutive.)
     */
    *truncation_block_length = size / BLCKSZ;
    if (BlockNumberIsValid(limit_block))
    {
        unsigned    relative_limit = limit_block - segno * RELSEG_SIZE;
 
        if (*truncation_block_length < relative_limit)
            *truncation_block_length = relative_limit;
    }
 
    /* Send it incrementally. */
    return BACK_UP_FILE_INCREMENTALLY;
}

References Assert, BACK_UP_FILE_FULLY, BACK_UP_FILE_INCREMENTALLY, BlockNumberIsValid(), BlockRefTableEntryGetBlocks(), BlockRefTableGetEntry(), IncrementalBackupInfo::brtab, IncrementalBackupInfo::buf, compare_block_numbers(), StringInfoData::data, RelFileLocator::dbOid, ereport, errcode(), errmsg_internal(), ERROR, FSM_FORKNUM, GetIncrementalFilePath(), i, MAIN_FORKNUM, IncrementalBackupInfo::manifest_files, OidIsValid, qsort, RelFileNumberIsValid, RelFileLocator::relNumber, size, and RelFileLocator::spcOid.

Referenced by sendDir().

GetIncrementalFilePath()

char* GetIncrementalFilePath	(	Oid	dboid,
		Oid	spcoid,
		RelFileNumber	relfilenumber,
		ForkNumber	forknum,
		unsigned	segno
	)

Definition at line 705 of file basebackup_incremental.c.

{
    char       *path;
    char       *lastslash;
    char       *ipath;
 
    path = GetRelationPath(dboid, spcoid, relfilenumber, INVALID_PROC_NUMBER,
                           forknum);
 
    lastslash = strrchr(path, '/');
    Assert(lastslash != NULL);
    *lastslash = '\0';
 
    if (segno > 0)
        ipath = psprintf("%s/INCREMENTAL.%s.%u", path, lastslash + 1, segno);
    else
        ipath = psprintf("%s/INCREMENTAL.%s", path, lastslash + 1);
 
    pfree(path);
 
    return ipath;
}

References Assert, GetRelationPath(), INVALID_PROC_NUMBER, pfree(), and psprintf().

Referenced by GetFileBackupMethod().

GetIncrementalFileSize()

size_t GetIncrementalFileSize

(

unsigned

num_blocks_required

)

Definition at line 977 of file basebackup_incremental.c.

{
    size_t      result;
 
    /* Make sure we're not going to overflow. */
    Assert(num_blocks_required <= RELSEG_SIZE);
 
    /*
     * Header with three four byte quantities (magic number, truncation block
     * length, block count) followed by block numbers, rounded to a multiple
     * of BLCKSZ (for files with block data), followed by block contents.
     */
    result = GetIncrementalHeaderSize(num_blocks_required);
    result += BLCKSZ * num_blocks_required;
 
    return result;
}

References Assert, and GetIncrementalHeaderSize().

Referenced by sendDir().

GetIncrementalHeaderSize()

size_t GetIncrementalHeaderSize

(

unsigned

num_blocks_required

)

Definition at line 949 of file basebackup_incremental.c.

{
    size_t      result;
 
    /* Make sure we're not going to overflow. */
    Assert(num_blocks_required <= RELSEG_SIZE);
 
    /*
     * Three four byte quantities (magic number, truncation block length,
     * block count) followed by block numbers.
     */
    result = 3 * sizeof(uint32) + (sizeof(BlockNumber) * num_blocks_required);
 
    /*
     * Round the header size to a multiple of BLCKSZ - when not a multiple of
     * BLCKSZ, add the missing fraction of a block. But do this only if the
     * file will store data for some blocks, otherwise keep it small.
     */
    if ((num_blocks_required > 0) && (result % BLCKSZ != 0))
        result += BLCKSZ - (result % BLCKSZ);
 
    return result;
}

References Assert.

Referenced by GetIncrementalFileSize().

hash_string_pointer()

static uint32 hash_string_pointer ( const char *  s )

static

Definition at line 999 of file basebackup_incremental.c.

{
    unsigned char *ss = (unsigned char *) s;
 
    return hash_bytes(ss, strlen(s));
}

References hash_bytes().

manifest_process_file()

static void manifest_process_file ( JsonManifestParseContext *  context, const char *  pathname, size_t  size, pg_checksum_type  checksum_type, int  checksum_length, uint8 *  checksum_payload  )

static

Definition at line 1046 of file basebackup_incremental.c.

{
    IncrementalBackupInfo *ib = context->private_data;
    backup_file_entry *entry;
    bool        found;
 
    entry = backup_file_insert(ib->manifest_files, pathname, &found);
    if (!found)
    {
        entry->path = MemoryContextStrdup(ib->manifest_files->ctx,
                                          pathname);
        entry->size = size;
    }
}

References context, IncrementalBackupInfo::manifest_files, MemoryContextStrdup(), backup_file_entry::path, backup_file_entry::size, and size.

Referenced by CreateIncrementalBackupInfo().

manifest_process_system_identifier()

static void manifest_process_system_identifier ( JsonManifestParseContext *  context, uint64  manifest_system_identifier  )

static

Definition at line 1024 of file basebackup_incremental.c.

{
    uint64      system_identifier;
 
    /* Get system identifier of current system */
    system_identifier = GetSystemIdentifier();
 
    if (manifest_system_identifier != system_identifier)
        context->error_cb(context,
                          "manifest system identifier is %llu, but database system identifier is %llu",
                          (unsigned long long) manifest_system_identifier,
                          (unsigned long long) system_identifier);
}

References context, and GetSystemIdentifier().

Referenced by CreateIncrementalBackupInfo().

manifest_process_version()

static void manifest_process_version ( JsonManifestParseContext *  context, int  manifest_version  )

static

Definition at line 1010 of file basebackup_incremental.c.

{
    /* Incremental backups don't work with manifest version 1 */
    if (manifest_version == 1)
        context->error_cb(context,
                          "backup manifest version 1 does not support incremental backup");
}

References context.

Referenced by CreateIncrementalBackupInfo().

manifest_process_wal_range()

static void manifest_process_wal_range ( JsonManifestParseContext *  context, TimeLineID  tli, XLogRecPtr  start_lsn, XLogRecPtr  end_lsn  )

static

Definition at line 1073 of file basebackup_incremental.c.

{
    IncrementalBackupInfo *ib = context->private_data;
    backup_wal_range *range = palloc(sizeof(backup_wal_range));
 
    range->tli = tli;
    range->start_lsn = start_lsn;
    range->end_lsn = end_lsn;
    ib->manifest_wal_ranges = lappend(ib->manifest_wal_ranges, range);
}

References context, lappend(), IncrementalBackupInfo::manifest_wal_ranges, palloc(), and range().

Referenced by CreateIncrementalBackupInfo().

manifest_report_error()

static void manifest_report_error ( JsonManifestParseContext *  context, const char *  fmt,   ...  )

static

Definition at line 1091 of file basebackup_incremental.c.

{
    StringInfoData errbuf;
 
    initStringInfo(&errbuf);
 
    for (;;)
    {
        va_list     ap;
        int         needed;
 
        va_start(ap, fmt);
        needed = appendStringInfoVA(&errbuf, fmt, ap);
        va_end(ap);
        if (needed == 0)
            break;
        enlargeStringInfo(&errbuf, needed);
    }
 
    ereport(ERROR,
            errmsg_internal("%s", errbuf.data));
}

References appendStringInfoVA(), StringInfoData::data, enlargeStringInfo(), ereport, errmsg_internal(), ERROR, fmt, initStringInfo(), va_end(), and va_start().

Referenced by CreateIncrementalBackupInfo().

pg_attribute_noreturn()

static void pg_attribute_noreturn

(

)

PrepareForIncrementalBackup()

void PrepareForIncrementalBackup	(	IncrementalBackupInfo *	ib,
		BackupState *	backup_state
	)

Definition at line 265 of file basebackup_incremental.c.

{
    MemoryContext oldcontext;
    List       *expectedTLEs;
    List       *all_wslist,
               *required_wslist = NIL;
    ListCell   *lc;
    TimeLineHistoryEntry **tlep;
    int         num_wal_ranges;
    int         i;
    bool        found_backup_start_tli = false;
    TimeLineID  earliest_wal_range_tli = 0;
    XLogRecPtr  earliest_wal_range_start_lsn = InvalidXLogRecPtr;
    TimeLineID  latest_wal_range_tli = 0;
    XLogRecPtr  summarized_lsn;
    XLogRecPtr  pending_lsn;
    XLogRecPtr  prior_pending_lsn = InvalidXLogRecPtr;
    int         deadcycles = 0;
    TimestampTz initial_time,
                current_time;
 
    Assert(ib->buf.data == NULL);
 
    /* Switch to our memory context. */
    oldcontext = MemoryContextSwitchTo(ib->mcxt);
 
    /*
     * A valid backup manifest must always contain at least one WAL range
     * (usually exactly one, unless the backup spanned a timeline switch).
     */
    num_wal_ranges = list_length(ib->manifest_wal_ranges);
    if (num_wal_ranges == 0)
        ereport(ERROR,
                (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                 errmsg("manifest contains no required WAL ranges")));
 
    /*
     * Match up the TLIs that appear in the WAL ranges of the backup manifest
     * with those that appear in this server's timeline history. We expect
     * every backup_wal_range to match to a TimeLineHistoryEntry; if it does
     * not, that's an error.
     *
     * This loop also decides which of the WAL ranges is the manifest is most
     * ancient and which one is the newest, according to the timeline history
     * of this server, and stores TLIs of those WAL ranges into
     * earliest_wal_range_tli and latest_wal_range_tli. It also updates
     * earliest_wal_range_start_lsn to the start LSN of the WAL range for
     * earliest_wal_range_tli.
     *
     * Note that the return value of readTimeLineHistory puts the latest
     * timeline at the beginning of the list, not the end. Hence, the earliest
     * TLI is the one that occurs nearest the end of the list returned by
     * readTimeLineHistory, and the latest TLI is the one that occurs closest
     * to the beginning.
     */
    expectedTLEs = readTimeLineHistory(backup_state->starttli);
    tlep = palloc0(num_wal_ranges * sizeof(TimeLineHistoryEntry *));
    for (i = 0; i < num_wal_ranges; ++i)
    {
        backup_wal_range *range = list_nth(ib->manifest_wal_ranges, i);
        bool        saw_earliest_wal_range_tli = false;
        bool        saw_latest_wal_range_tli = false;
 
        /* Search this server's history for this WAL range's TLI. */
        foreach(lc, expectedTLEs)
        {
            TimeLineHistoryEntry *tle = lfirst(lc);
 
            if (tle->tli == range->tli)
            {
                tlep[i] = tle;
                break;
            }
 
            if (tle->tli == earliest_wal_range_tli)
                saw_earliest_wal_range_tli = true;
            if (tle->tli == latest_wal_range_tli)
                saw_latest_wal_range_tli = true;
        }
 
        /*
         * An incremental backup can only be taken relative to a backup that
         * represents a previous state of this server. If the backup requires
         * WAL from a timeline that's not in our history, that definitely
         * isn't the case.
         */
        if (tlep[i] == NULL)
            ereport(ERROR,
                    (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                     errmsg("timeline %u found in manifest, but not in this server's history",
                            range->tli)));
 
        /*
         * If we found this TLI in the server's history before encountering
         * the latest TLI seen so far in the server's history, then this TLI
         * is the latest one seen so far.
         *
         * If on the other hand we saw the earliest TLI seen so far before
         * finding this TLI, this TLI is earlier than the earliest one seen so
         * far. And if this is the first TLI for which we've searched, it's
         * also the earliest one seen so far.
         *
         * On the first loop iteration, both things should necessarily be
         * true.
         */
        if (!saw_latest_wal_range_tli)
            latest_wal_range_tli = range->tli;
        if (earliest_wal_range_tli == 0 || saw_earliest_wal_range_tli)
        {
            earliest_wal_range_tli = range->tli;
            earliest_wal_range_start_lsn = range->start_lsn;
        }
    }
 
    /*
     * Propagate information about the prior backup into the backup_label that
     * will be generated for this backup.
     */
    backup_state->istartpoint = earliest_wal_range_start_lsn;
    backup_state->istarttli = earliest_wal_range_tli;
 
    /*
     * Sanity check start and end LSNs for the WAL ranges in the manifest.
     *
     * Commonly, there won't be any timeline switches during the prior backup
     * at all, but if there are, they should happen at the same LSNs that this
     * server switched timelines.
     *
     * Whether there are any timeline switches during the prior backup or not,
     * the prior backup shouldn't require any WAL from a timeline prior to the
     * start of that timeline. It also shouldn't require any WAL from later
     * than the start of this backup.
     *
     * If any of these sanity checks fail, one possible explanation is that
     * the user has generated WAL on the same timeline with the same LSNs more
     * than once. For instance, if two standbys running on timeline 1 were
     * both promoted and (due to a broken archiving setup) both selected new
     * timeline ID 2, then it's possible that one of these checks might trip.
     *
     * Note that there are lots of ways for the user to do something very bad
     * without tripping any of these checks, and they are not intended to be
     * comprehensive. It's pretty hard to see how we could be certain of
     * anything here. However, if there's a problem staring us right in the
     * face, it's best to report it, so we do.
     */
    for (i = 0; i < num_wal_ranges; ++i)
    {
        backup_wal_range *range = list_nth(ib->manifest_wal_ranges, i);
 
        if (range->tli == earliest_wal_range_tli)
        {
            if (range->start_lsn < tlep[i]->begin)
                ereport(ERROR,
                        (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                         errmsg("manifest requires WAL from initial timeline %u starting at %X/%X, but that timeline begins at %X/%X",
                                range->tli,
                                LSN_FORMAT_ARGS(range->start_lsn),
                                LSN_FORMAT_ARGS(tlep[i]->begin))));
        }
        else
        {
            if (range->start_lsn != tlep[i]->begin)
                ereport(ERROR,
                        (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                         errmsg("manifest requires WAL from continuation timeline %u starting at %X/%X, but that timeline begins at %X/%X",
                                range->tli,
                                LSN_FORMAT_ARGS(range->start_lsn),
                                LSN_FORMAT_ARGS(tlep[i]->begin))));
        }
 
        if (range->tli == latest_wal_range_tli)
        {
            if (range->end_lsn > backup_state->startpoint)
                ereport(ERROR,
                        (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                         errmsg("manifest requires WAL from final timeline %u ending at %X/%X, but this backup starts at %X/%X",
                                range->tli,
                                LSN_FORMAT_ARGS(range->end_lsn),
                                LSN_FORMAT_ARGS(backup_state->startpoint)),
                         errhint("This can happen for incremental backups on a standby if there was little activity since the previous backup.")));
        }
        else
        {
            if (range->end_lsn != tlep[i]->end)
                ereport(ERROR,
                        (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                         errmsg("manifest requires WAL from non-final timeline %u ending at %X/%X, but this server switched timelines at %X/%X",
                                range->tli,
                                LSN_FORMAT_ARGS(range->end_lsn),
                                LSN_FORMAT_ARGS(tlep[i]->end))));
        }
 
    }
 
    /*
     * Wait for WAL summarization to catch up to the backup start LSN (but
     * time out if it doesn't do so quickly enough).
     */
    initial_time = current_time = GetCurrentTimestamp();
    while (1)
    {
        long        timeout_in_ms = 10000;
        long        elapsed_seconds;
 
        /*
         * Align the wait time to prevent drift. This doesn't really matter,
         * but we'd like the warnings about how long we've been waiting to say
         * 10 seconds, 20 seconds, 30 seconds, 40 seconds ... without ever
         * drifting to something that is not a multiple of ten.
         */
        timeout_in_ms -=
            TimestampDifferenceMilliseconds(initial_time, current_time) %
            timeout_in_ms;
 
        /* Wait for up to 10 seconds. */
        summarized_lsn = WaitForWalSummarization(backup_state->startpoint,
                                                 timeout_in_ms, &pending_lsn);
 
        /* If WAL summarization has progressed sufficiently, stop waiting. */
        if (summarized_lsn >= backup_state->startpoint)
            break;
 
        /*
         * Keep track of the number of cycles during which there has been no
         * progression of pending_lsn. If pending_lsn is not advancing, that
         * means that not only are no new files appearing on disk, but we're
         * not even incorporating new records into the in-memory state.
         */
        if (pending_lsn > prior_pending_lsn)
        {
            prior_pending_lsn = pending_lsn;
            deadcycles = 0;
        }
        else
            ++deadcycles;
 
        /*
         * If we've managed to wait for an entire minute without the WAL
         * summarizer absorbing a single WAL record, error out; probably
         * something is wrong.
         *
         * We could consider also erroring out if the summarizer is taking too
         * long to catch up, but it's not clear what rate of progress would be
         * acceptable and what would be too slow. So instead, we just try to
         * error out in the case where there's no progress at all. That seems
         * likely to catch a reasonable number of the things that can go wrong
         * in practice (e.g. the summarizer process is completely hung, say
         * because somebody hooked up a debugger to it or something) without
         * giving up too quickly when the system is just slow.
         */
        if (deadcycles >= 6)
            ereport(ERROR,
                    (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                     errmsg("WAL summarization is not progressing"),
                     errdetail("Summarization is needed through %X/%X, but is stuck at %X/%X on disk and %X/%X in memory.",
                               LSN_FORMAT_ARGS(backup_state->startpoint),
                               LSN_FORMAT_ARGS(summarized_lsn),
                               LSN_FORMAT_ARGS(pending_lsn))));
 
        /*
         * Otherwise, just let the user know what's happening.
         */
        current_time = GetCurrentTimestamp();
        elapsed_seconds =
            TimestampDifferenceMilliseconds(initial_time, current_time) / 1000;
        ereport(WARNING,
                (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                 errmsg("still waiting for WAL summarization through %X/%X after %ld seconds",
                        LSN_FORMAT_ARGS(backup_state->startpoint),
                        elapsed_seconds),
                 errdetail("Summarization has reached %X/%X on disk and %X/%X in memory.",
                           LSN_FORMAT_ARGS(summarized_lsn),
                           LSN_FORMAT_ARGS(pending_lsn))));
    }
 
    /*
     * Retrieve a list of all WAL summaries on any timeline that overlap with
     * the LSN range of interest. We could instead call GetWalSummaries() once
     * per timeline in the loop that follows, but that would involve reading
     * the directory multiple times. It should be mildly faster - and perhaps
     * a bit safer - to do it just once.
     */
    all_wslist = GetWalSummaries(0, earliest_wal_range_start_lsn,
                                 backup_state->startpoint);
 
    /*
     * We need WAL summaries for everything that happened during the prior
     * backup and everything that happened afterward up until the point where
     * the current backup started.
     */
    foreach(lc, expectedTLEs)
    {
        TimeLineHistoryEntry *tle = lfirst(lc);
        XLogRecPtr  tli_start_lsn = tle->begin;
        XLogRecPtr  tli_end_lsn = tle->end;
        XLogRecPtr  tli_missing_lsn = InvalidXLogRecPtr;
        List       *tli_wslist;
 
        /*
         * Working through the history of this server from the current
         * timeline backwards, we skip everything until we find the timeline
         * where this backup started. Most of the time, this means we won't
         * skip anything at all, as it's unlikely that the timeline has
         * changed since the beginning of the backup moments ago.
         */
        if (tle->tli == backup_state->starttli)
        {
            found_backup_start_tli = true;
            tli_end_lsn = backup_state->startpoint;
        }
        else if (!found_backup_start_tli)
            continue;
 
        /*
         * Find the summaries that overlap the LSN range of interest for this
         * timeline. If this is the earliest timeline involved, the range of
         * interest begins with the start LSN of the prior backup; otherwise,
         * it begins at the LSN at which this timeline came into existence. If
         * this is the latest TLI involved, the range of interest ends at the
         * start LSN of the current backup; otherwise, it ends at the point
         * where we switched from this timeline to the next one.
         */
        if (tle->tli == earliest_wal_range_tli)
            tli_start_lsn = earliest_wal_range_start_lsn;
        tli_wslist = FilterWalSummaries(all_wslist, tle->tli,
                                        tli_start_lsn, tli_end_lsn);
 
        /*
         * There is no guarantee that the WAL summaries we found cover the
         * entire range of LSNs for which summaries are required, or indeed
         * that we found any WAL summaries at all. Check whether we have a
         * problem of that sort.
         */
        if (!WalSummariesAreComplete(tli_wslist, tli_start_lsn, tli_end_lsn,
                                     &tli_missing_lsn))
        {
            if (XLogRecPtrIsInvalid(tli_missing_lsn))
                ereport(ERROR,
                        (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                         errmsg("WAL summaries are required on timeline %u from %X/%X to %X/%X, but no summaries for that timeline and LSN range exist",
                                tle->tli,
                                LSN_FORMAT_ARGS(tli_start_lsn),
                                LSN_FORMAT_ARGS(tli_end_lsn))));
            else
                ereport(ERROR,
                        (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
                         errmsg("WAL summaries are required on timeline %u from %X/%X to %X/%X, but the summaries for that timeline and LSN range are incomplete",
                                tle->tli,
                                LSN_FORMAT_ARGS(tli_start_lsn),
                                LSN_FORMAT_ARGS(tli_end_lsn)),
                         errdetail("The first unsummarized LSN in this range is %X/%X.",
                                   LSN_FORMAT_ARGS(tli_missing_lsn))));
        }
 
        /*
         * Remember that we need to read these summaries.
         *
         * Technically, it's possible that this could read more files than
         * required, since tli_wslist in theory could contain redundant
         * summaries. For instance, if we have a summary from 0/10000000 to
         * 0/20000000 and also one from 0/00000000 to 0/30000000, then the
         * latter subsumes the former and the former could be ignored.
         *
         * We ignore this possibility because the WAL summarizer only tries to
         * generate summaries that do not overlap. If somehow they exist,
         * we'll do a bit of extra work but the results should still be
         * correct.
         */
        required_wslist = list_concat(required_wslist, tli_wslist);
 
        /*
         * Timelines earlier than the one in which the prior backup began are
         * not relevant.
         */
        if (tle->tli == earliest_wal_range_tli)
            break;
    }
 
    /*
     * Read all of the required block reference table files and merge all of
     * the data into a single in-memory block reference table.
     *
     * See the comments for struct IncrementalBackupInfo for some thoughts on
     * memory usage.
     */
    ib->brtab = CreateEmptyBlockRefTable();
    foreach(lc, required_wslist)
    {
        WalSummaryFile *ws = lfirst(lc);
        WalSummaryIO wsio;
        BlockRefTableReader *reader;
        RelFileLocator rlocator;
        ForkNumber  forknum;
        BlockNumber limit_block;
        BlockNumber blocks[BLOCKS_PER_READ];
 
        wsio.file = OpenWalSummaryFile(ws, false);
        wsio.filepos = 0;
        ereport(DEBUG1,
                (errmsg_internal("reading WAL summary file \"%s\"",
                                 FilePathName(wsio.file))));
        reader = CreateBlockRefTableReader(ReadWalSummary, &wsio,
                                           FilePathName(wsio.file),
                                           ReportWalSummaryError, NULL);
        while (BlockRefTableReaderNextRelation(reader, &rlocator, &forknum,
                                               &limit_block))
        {
            BlockRefTableSetLimitBlock(ib->brtab, &rlocator,
                                       forknum, limit_block);
 
            while (1)
            {
                unsigned    nblocks;
                unsigned    i;
 
                nblocks = BlockRefTableReaderGetBlocks(reader, blocks,
                                                       BLOCKS_PER_READ);
                if (nblocks == 0)
                    break;
 
                for (i = 0; i < nblocks; ++i)
                    BlockRefTableMarkBlockModified(ib->brtab, &rlocator,
                                                   forknum, blocks[i]);
            }
        }
        DestroyBlockRefTableReader(reader);
        FileClose(wsio.file);
    }
 
    /* Switch back to previous memory context. */
    MemoryContextSwitchTo(oldcontext);
}

References Assert, backup_state, TimeLineHistoryEntry::begin, BlockRefTableMarkBlockModified(), BlockRefTableReaderGetBlocks(), BlockRefTableReaderNextRelation(), BlockRefTableSetLimitBlock(), BLOCKS_PER_READ, IncrementalBackupInfo::brtab, IncrementalBackupInfo::buf, CreateBlockRefTableReader(), StringInfoData::data, DEBUG1, DestroyBlockRefTableReader(), TimeLineHistoryEntry::end, ereport, errcode(), errdetail(), errhint(), errmsg(), errmsg_internal(), ERROR, expectedTLEs, WalSummaryIO::file, FileClose(), FilePathName(), WalSummaryIO::filepos, FilterWalSummaries(), GetCurrentTimestamp(), GetWalSummaries(), i, InvalidXLogRecPtr, BackupState::istartpoint, BackupState::istarttli, lfirst, list_concat(), list_length(), list_nth(), LSN_FORMAT_ARGS, IncrementalBackupInfo::manifest_wal_ranges, IncrementalBackupInfo::mcxt, MemoryContextSwitchTo(), NIL, OpenWalSummaryFile(), palloc0(), range(), readTimeLineHistory(), ReadWalSummary(), ReportWalSummaryError(), BackupState::startpoint, BackupState::starttli, TimestampDifferenceMilliseconds(), TimeLineHistoryEntry::tli, WaitForWalSummarization(), WalSummariesAreComplete(), WARNING, and XLogRecPtrIsInvalid.

Referenced by perform_base_backup().