34 #define BLOCKS_PER_READ 512
41 #define MIN_CHUNK 1024
42 #define MAX_CHUNK (128 * 1024)
65 #define SH_PREFIX backup_file
66 #define SH_ELEMENT_TYPE backup_file_entry
67 #define SH_KEY_TYPE const char *
69 #define SH_HASH_KEY(tb, key) hash_string_pointer(key)
70 #define SH_EQUAL(tb, a, b) (strcmp(a, b) == 0)
71 #define SH_SCOPE static inline
131 int manifest_version);
133 uint64 manifest_system_identifier);
135 const char *pathname,
139 uint8 *checksum_payload);
271 *required_wslist =
NIL;
276 bool found_backup_start_tli =
false;
297 if (num_wal_ranges == 0)
299 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
300 errmsg(
"manifest contains no required WAL ranges")));
323 for (
i = 0;
i < num_wal_ranges; ++
i)
326 bool saw_earliest_wal_range_tli =
false;
327 bool saw_latest_wal_range_tli =
false;
340 if (tle->
tli == earliest_wal_range_tli)
341 saw_earliest_wal_range_tli =
true;
342 if (tle->
tli == latest_wal_range_tli)
343 saw_latest_wal_range_tli =
true;
354 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
355 errmsg(
"timeline %u found in manifest, but not in this server's history",
371 if (!saw_latest_wal_range_tli)
372 latest_wal_range_tli =
range->tli;
373 if (earliest_wal_range_tli == 0 || saw_earliest_wal_range_tli)
375 earliest_wal_range_tli =
range->tli;
376 earliest_wal_range_start_lsn =
range->start_lsn;
411 for (
i = 0;
i < num_wal_ranges; ++
i)
415 if (
range->tli == earliest_wal_range_tli)
419 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
420 errmsg(
"manifest requires WAL from initial timeline %u starting at %X/%X, but that timeline begins at %X/%X",
429 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
430 errmsg(
"manifest requires WAL from continuation timeline %u starting at %X/%X, but that timeline begins at %X/%X",
436 if (
range->tli == latest_wal_range_tli)
440 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
441 errmsg(
"manifest requires WAL from final timeline %u ending at %X/%X, but this backup starts at %X/%X",
445 errhint(
"This can happen for incremental backups on a standby if there was little activity since the previous backup.")));
451 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
452 errmsg(
"manifest requires WAL from non-final timeline %u ending at %X/%X, but this server switched timelines at %X/%X",
467 long timeout_in_ms = 10000;
468 long elapsed_seconds;
482 timeout_in_ms, &pending_lsn);
494 if (pending_lsn > prior_pending_lsn)
496 prior_pending_lsn = pending_lsn;
518 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
519 errmsg(
"WAL summarization is not progressing"),
520 errdetail(
"Summarization is needed through %X/%X, but is stuck at %X/%X on disk and %X/%X in memory.",
532 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
533 errmsg(
"still waiting for WAL summarization through %X/%X after %ld seconds",
536 errdetail(
"Summarization has reached %X/%X on disk and %X/%X in memory.",
573 found_backup_start_tli =
true;
576 else if (!found_backup_start_tli)
588 if (tle->
tli == earliest_wal_range_tli)
589 tli_start_lsn = earliest_wal_range_start_lsn;
591 tli_start_lsn, tli_end_lsn);
604 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
605 errmsg(
"WAL summaries are required on timeline %u from %X/%X to %X/%X, but no summaries for that timeline and LSN range exist",
611 (
errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
612 errmsg(
"WAL summaries are required on timeline %u from %X/%X to %X/%X, but the summaries for that timeline and LSN range are incomplete",
616 errdetail(
"The first unsummarized LSN in this range is %X/%X.",
634 required_wslist =
list_concat(required_wslist, tli_wslist);
640 if (tle->
tli == earliest_wal_range_tli)
652 foreach(lc, required_wslist)
674 forknum, limit_block);
686 for (
i = 0;
i < nblocks; ++
i)
715 lastslash = strrchr(path,
'/');
716 Assert(lastslash != NULL);
720 ipath =
psprintf(
"%s/INCREMENTAL.%s.%u", path, lastslash + 1, segno);
722 ipath =
psprintf(
"%s/INCREMENTAL.%s", path, lastslash + 1);
748 unsigned segno,
size_t size,
749 unsigned *num_blocks_required,
751 unsigned *truncation_block_length)
775 if ((
size % BLCKSZ) != 0 ||
size / BLCKSZ > RELSEG_SIZE)
822 rlocator.
dbOid = dboid;
825 &limit_block) != NULL)
850 if (brtentry == NULL)
854 *num_blocks_required = 0;
855 *truncation_block_length =
size / BLCKSZ;
863 if (limit_block <= segno * RELSEG_SIZE)
872 start_blkno = segno * RELSEG_SIZE;
873 stop_blkno = start_blkno + (
size / BLCKSZ);
874 if (start_blkno / RELSEG_SIZE != segno || stop_blkno < start_blkno)
876 errcode(ERRCODE_INTERNAL_ERROR),
877 errmsg_internal(
"overflow computing block number bounds for segment %u with size %zu",
885 relative_block_numbers, RELSEG_SIZE);
886 Assert(nblocks <= RELSEG_SIZE);
900 if (nblocks * BLCKSZ >
size * 0.9)
915 if (start_blkno != 0)
917 for (
i = 0;
i < nblocks; ++
i)
918 relative_block_numbers[
i] -= start_blkno;
920 *num_blocks_required = nblocks;
930 *truncation_block_length =
size / BLCKSZ;
933 unsigned relative_limit = limit_block - segno * RELSEG_SIZE;
935 if (*truncation_block_length < relative_limit)
936 *truncation_block_length = relative_limit;
954 Assert(num_blocks_required <= RELSEG_SIZE);
967 if ((num_blocks_required > 0) && (result % BLCKSZ != 0))
968 result += BLCKSZ - (result % BLCKSZ);
982 Assert(num_blocks_required <= RELSEG_SIZE);
990 result += BLCKSZ * num_blocks_required;
1001 unsigned char *ss = (
unsigned char *) s;
1011 int manifest_version)
1014 if (manifest_version == 1)
1016 "backup manifest version 1 does not support incremental backup");
1025 uint64 manifest_system_identifier)
1027 uint64 system_identifier;
1032 if (manifest_system_identifier != system_identifier)
1034 "manifest system identifier is %llu, but database system identifier is %llu",
1035 (
unsigned long long) manifest_system_identifier,
1036 (
unsigned long long) system_identifier);
1047 const char *pathname,
size_t size,
1049 int checksum_length,
1050 uint8 *checksum_payload)
1056 entry = backup_file_insert(ib->
manifest_files, pathname, &found);
1081 range->start_lsn = start_lsn;
1082 range->end_lsn = end_lsn;
List * readTimeLineHistory(TimeLineID targetTLI)
long TimestampDifferenceMilliseconds(TimestampTz start_time, TimestampTz stop_time)
TimestampTz GetCurrentTimestamp(void)
void AppendIncrementalManifestData(IncrementalBackupInfo *ib, const char *data, int len)
static void manifest_process_version(JsonManifestParseContext *context, int manifest_version)
static int compare_block_numbers(const void *a, const void *b)
static uint32 hash_string_pointer(const char *s)
size_t GetIncrementalHeaderSize(unsigned num_blocks_required)
char * GetIncrementalFilePath(Oid dboid, Oid spcoid, RelFileNumber relfilenumber, ForkNumber forknum, unsigned segno)
static void manifest_process_system_identifier(JsonManifestParseContext *context, uint64 manifest_system_identifier)
size_t GetIncrementalFileSize(unsigned num_blocks_required)
static void manifest_process_file(JsonManifestParseContext *context, const char *pathname, size_t size, pg_checksum_type checksum_type, int checksum_length, uint8 *checksum_payload)
FileBackupMethod GetFileBackupMethod(IncrementalBackupInfo *ib, const char *path, Oid dboid, Oid spcoid, RelFileNumber relfilenumber, ForkNumber forknum, unsigned segno, size_t size, unsigned *num_blocks_required, BlockNumber *relative_block_numbers, unsigned *truncation_block_length)
static void pg_attribute_noreturn()
static void manifest_report_error(JsonManifestParseContext *context, const char *fmt,...) pg_attribute_printf(2
static void manifest_process_wal_range(JsonManifestParseContext *context, TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
IncrementalBackupInfo * CreateIncrementalBackupInfo(MemoryContext mcxt)
void PrepareForIncrementalBackup(IncrementalBackupInfo *ib, BackupState *backup_state)
void FinalizeIncrementalManifest(IncrementalBackupInfo *ib)
@ BACK_UP_FILE_INCREMENTALLY
BlockRefTableReader * CreateBlockRefTableReader(io_callback_fn read_callback, void *read_callback_arg, char *error_filename, report_error_fn error_callback, void *error_callback_arg)
bool BlockRefTableReaderNextRelation(BlockRefTableReader *reader, RelFileLocator *rlocator, ForkNumber *forknum, BlockNumber *limit_block)
int BlockRefTableEntryGetBlocks(BlockRefTableEntry *entry, BlockNumber start_blkno, BlockNumber stop_blkno, BlockNumber *blocks, int nblocks)
void BlockRefTableMarkBlockModified(BlockRefTable *brtab, const RelFileLocator *rlocator, ForkNumber forknum, BlockNumber blknum)
BlockRefTableEntry * BlockRefTableGetEntry(BlockRefTable *brtab, const RelFileLocator *rlocator, ForkNumber forknum, BlockNumber *limit_block)
unsigned BlockRefTableReaderGetBlocks(BlockRefTableReader *reader, BlockNumber *blocks, int nblocks)
void BlockRefTableSetLimitBlock(BlockRefTable *brtab, const RelFileLocator *rlocator, ForkNumber forknum, BlockNumber limit_block)
void DestroyBlockRefTableReader(BlockRefTableReader *reader)
void(*) BlockRefTable CreateEmptyBlockRefTable)(void)
static bool BlockNumberIsValid(BlockNumber blockNumber)
#define Assert(condition)
#define pg_attribute_printf(f, a)
#define OidIsValid(objectId)
int errmsg_internal(const char *fmt,...)
int errdetail(const char *fmt,...)
int errhint(const char *fmt,...)
int errcode(int sqlerrcode)
int errmsg(const char *fmt,...)
#define ereport(elevel,...)
void FileClose(File file)
char * FilePathName(File file)
uint32 hash_bytes(const unsigned char *k, int keylen)
static int pg_cmp_u32(uint32 a, uint32 b)
static void const char * fmt
List * lappend(List *list, void *datum)
List * list_concat(List *list1, const List *list2)
void pfree(void *pointer)
void * palloc0(Size size)
char * MemoryContextStrdup(MemoryContext context, const char *string)
JsonManifestParseIncrementalState * json_parse_manifest_incremental_init(JsonManifestParseContext *context)
void json_parse_manifest_incremental_shutdown(JsonManifestParseIncrementalState *incstate)
void json_parse_manifest_incremental_chunk(JsonManifestParseIncrementalState *incstate, const char *chunk, size_t size, bool is_last)
static int list_length(const List *l)
static void * list_nth(const List *list, int n)
#define qsort(a, b, c, d)
#define INVALID_PROC_NUMBER
char * psprintf(const char *fmt,...)
MemoryContextSwitchTo(old_ctx)
static struct cvec * range(struct vars *v, chr a, chr b, int cases)
char * GetRelationPath(Oid dbOid, Oid spcOid, RelFileNumber relNumber, int procNumber, ForkNumber forkNumber)
#define RelFileNumberIsValid(relnumber)
static pg_noinline void Size size
int appendStringInfoVA(StringInfo str, const char *fmt, va_list args)
void enlargeStringInfo(StringInfo str, int needed)
void appendBinaryStringInfo(StringInfo str, const void *data, int datalen)
void initStringInfo(StringInfo str)
List * manifest_wal_ranges
backup_file_hash * manifest_files
JsonManifestParseIncrementalState * inc_state
XLogRecPtr WaitForWalSummarization(XLogRecPtr lsn, long timeout, XLogRecPtr *pending_lsn)
File OpenWalSummaryFile(WalSummaryFile *ws, bool missing_ok)
List * GetWalSummaries(TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
bool WalSummariesAreComplete(List *wslist, XLogRecPtr start_lsn, XLogRecPtr end_lsn, XLogRecPtr *missing_lsn)
int ReadWalSummary(void *wal_summary_io, void *data, int length)
List * FilterWalSummaries(List *wslist, TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
void ReportWalSummaryError(void *callback_arg, char *fmt,...)
uint64 GetSystemIdentifier(void)
#define LSN_FORMAT_ARGS(lsn)
#define XLogRecPtrIsInvalid(r)
#define InvalidXLogRecPtr
static BackupState * backup_state
static List * expectedTLEs