PostgreSQL Source Code git master
All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
basebackup.c
Go to the documentation of this file.
1/*-------------------------------------------------------------------------
2 *
3 * basebackup.c
4 * code for taking a base backup and streaming it to a standby
5 *
6 * Portions Copyright (c) 2010-2024, PostgreSQL Global Development Group
7 *
8 * IDENTIFICATION
9 * src/backend/backup/basebackup.c
10 *
11 *-------------------------------------------------------------------------
12 */
13#include "postgres.h"
14
15#include <sys/stat.h>
16#include <unistd.h>
17#include <time.h>
18
20#include "access/xlogbackup.h"
22#include "backup/basebackup.h"
26#include "catalog/pg_tablespace_d.h"
27#include "commands/defrem.h"
28#include "common/compression.h"
29#include "common/file_perm.h"
30#include "common/file_utils.h"
31#include "lib/stringinfo.h"
32#include "miscadmin.h"
33#include "nodes/pg_list.h"
34#include "pgstat.h"
35#include "pgtar.h"
38#include "replication/slot.h"
41#include "storage/bufpage.h"
42#include "storage/checksum.h"
43#include "storage/dsm_impl.h"
44#include "storage/ipc.h"
45#include "storage/reinit.h"
46#include "utils/builtins.h"
47#include "utils/guc.h"
48#include "utils/ps_status.h"
49#include "utils/relcache.h"
50#include "utils/resowner.h"
51
52/*
53 * How much data do we want to send in one CopyData message? Note that
54 * this may also result in reading the underlying files in chunks of this
55 * size.
56 *
57 * NB: The buffer size is required to be a multiple of the system block
58 * size, so use that value instead if it's bigger than our preference.
59 */
60#define SINK_BUFFER_LENGTH Max(32768, BLCKSZ)
61
62typedef struct
63{
64 const char *label;
67 bool nowait;
80
81static int64 sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly,
84static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
85 List *tablespaces, bool sendtblspclinks,
88static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
89 struct stat *statbuf, bool missing_ok,
90 Oid dboid, Oid spcoid, RelFileNumber relfilenumber,
91 unsigned segno,
93 unsigned num_incremental_blocks,
94 BlockNumber *incremental_blocks,
95 unsigned truncation_block_length);
96static off_t read_file_data_into_buffer(bbsink *sink,
97 const char *readfilename, int fd,
98 off_t offset, size_t length,
99 BlockNumber blkno,
100 bool verify_checksum,
101 int *checksum_failures);
102static void push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx,
103 size_t *bytes_done, void *data, size_t length);
104static bool verify_page_checksum(Page page, XLogRecPtr start_lsn,
105 BlockNumber blkno,
106 uint16 *expected_checksum);
107static void sendFileWithContent(bbsink *sink, const char *filename,
108 const char *content, int len,
110static int64 _tarWriteHeader(bbsink *sink, const char *filename,
111 const char *linktarget, struct stat *statbuf,
112 bool sizeonly);
113static void _tarWritePadding(bbsink *sink, int len);
114static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf);
115static void perform_base_backup(basebackup_options *opt, bbsink *sink,
118static int compareWalFileNames(const ListCell *a, const ListCell *b);
119static ssize_t basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
120 const char *filename, bool partial_read_ok);
121
122/* Was the backup currently in-progress initiated in recovery mode? */
123static bool backup_started_in_recovery = false;
124
125/* Total number of checksum failures during base backup. */
126static long long int total_checksum_failures;
127
128/* Do not verify checksums. */
129static bool noverify_checksums = false;
130
131/*
132 * Definition of one element part of an exclusion list, used for paths part
133 * of checksum validation or base backups. "name" is the name of the file
134 * or path to check for exclusion. If "match_prefix" is true, any items
135 * matching the name as prefix are excluded.
136 */
138{
139 const char *name;
141};
142
143/*
144 * The contents of these directories are removed or recreated during server
145 * start so they are not included in backups. The directories themselves are
146 * kept and included as empty to preserve access permissions.
147 *
148 * Note: this list should be kept in sync with the filter lists in pg_rewind's
149 * filemap.c.
150 */
151static const char *const excludeDirContents[] =
152{
153 /*
154 * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped
155 * because extensions like pg_stat_statements store data there.
156 */
158
159 /*
160 * It is generally not useful to backup the contents of this directory
161 * even if the intention is to restore to another primary. See backup.sgml
162 * for a more detailed description.
163 */
165
166 /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
168
169 /* Contents removed on startup, see AsyncShmemInit(). */
170 "pg_notify",
171
172 /*
173 * Old contents are loaded for possible debugging but are not required for
174 * normal operation, see SerialInit().
175 */
176 "pg_serial",
177
178 /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
179 "pg_snapshots",
180
181 /* Contents zeroed on startup, see StartupSUBTRANS(). */
182 "pg_subtrans",
183
184 /* end of list */
185 NULL
186};
187
188/*
189 * List of files excluded from backups.
190 */
191static const struct exclude_list_item excludeFiles[] =
192{
193 /* Skip auto conf temporary file. */
194 {PG_AUTOCONF_FILENAME ".tmp", false},
195
196 /* Skip current log file temporary file */
198
199 /*
200 * Skip relation cache because it is rebuilt on startup. This includes
201 * temporary files.
202 */
204
205 /*
206 * backup_label and tablespace_map should not exist in a running cluster
207 * capable of doing an online backup, but exclude them just in case.
208 */
209 {BACKUP_LABEL_FILE, false},
210 {TABLESPACE_MAP, false},
211
212 /*
213 * If there's a backup_manifest, it belongs to a backup that was used to
214 * start this server. It is *not* correct for this backup. Our
215 * backup_manifest is injected into the backup separately if users want
216 * it.
217 */
218 {"backup_manifest", false},
219
220 {"postmaster.pid", false},
221 {"postmaster.opts", false},
222
223 /* end of list */
224 {NULL, false}
225};
226
227/*
228 * Actually do a base backup for the specified tablespaces.
229 *
230 * This is split out mainly to avoid complaints about "variable might be
231 * clobbered by longjmp" from stupider versions of gcc.
232 */
233static void
236{
238 XLogRecPtr endptr;
239 TimeLineID endtli;
243
244 /* Initial backup state, insofar as we know it now. */
245 state.tablespaces = NIL;
246 state.tablespace_num = 0;
247 state.bytes_done = 0;
248 state.bytes_total = 0;
249 state.bytes_total_is_valid = false;
250
251 /* we're going to use a BufFile, so we need a ResourceOwner */
254 CurrentResourceOwner == NULL);
256
258
261
263
264 /* Allocate backup related variables. */
267
269 do_pg_backup_start(opt->label, opt->fastcheckpoint, &state.tablespaces,
271
272 state.startptr = backup_state->startpoint;
273 state.starttli = backup_state->starttli;
274
275 /*
276 * Once do_pg_backup_start has been called, ensure that any failure causes
277 * us to abort the backup so we don't "leak" a backup counter. For this
278 * reason, *all* functionality between do_pg_backup_start() and the end of
279 * do_pg_backup_stop() should be inside the error cleanup block!
280 */
281
283 {
284 ListCell *lc;
285 tablespaceinfo *newti;
286
287 /* If this is an incremental backup, execute preparatory steps. */
288 if (ib != NULL)
290
291 /* Add a node for the base directory at the end */
292 newti = palloc0(sizeof(tablespaceinfo));
293 newti->size = -1;
294 state.tablespaces = lappend(state.tablespaces, newti);
295
296 /*
297 * Calculate the total backup size by summing up the size of each
298 * tablespace
299 */
300 if (opt->progress)
301 {
303
304 foreach(lc, state.tablespaces)
305 {
306 tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
307
308 if (tmp->path == NULL)
309 tmp->size = sendDir(sink, ".", 1, true, state.tablespaces,
310 true, NULL, InvalidOid, NULL);
311 else
312 tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true,
313 NULL, NULL);
314 state.bytes_total += tmp->size;
315 }
316 state.bytes_total_is_valid = true;
317 }
318
319 /* notify basebackup sink about start of backup */
321
322 /* Send off our tablespaces one by one */
323 foreach(lc, state.tablespaces)
324 {
326
327 if (ti->path == NULL)
328 {
329 struct stat statbuf;
330 bool sendtblspclinks = true;
331 char *backup_label;
332
333 bbsink_begin_archive(sink, "base.tar");
334
335 /* In the main tar, include the backup_label first... */
336 backup_label = build_backup_content(backup_state, false);
338 backup_label, -1, &manifest);
339 pfree(backup_label);
340
341 /* Then the tablespace_map file, if required... */
342 if (opt->sendtblspcmapfile)
343 {
346 sendtblspclinks = false;
347 }
348
349 /* Then the bulk of the files... */
350 sendDir(sink, ".", 1, false, state.tablespaces,
351 sendtblspclinks, &manifest, InvalidOid, ib);
352
353 /* ... and pg_control after everything else. */
354 if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
357 errmsg("could not stat file \"%s\": %m",
360 false, InvalidOid, InvalidOid,
361 InvalidRelFileNumber, 0, &manifest, 0, NULL, 0);
362 }
363 else
364 {
365 char *archive_name = psprintf("%u.tar", ti->oid);
366
367 bbsink_begin_archive(sink, archive_name);
368
369 sendTablespace(sink, ti->path, ti->oid, false, &manifest, ib);
370 }
371
372 /*
373 * If we're including WAL, and this is the main data directory we
374 * don't treat this as the end of the tablespace. Instead, we will
375 * include the xlog files below and stop afterwards. This is safe
376 * since the main data directory is always sent *last*.
377 */
378 if (opt->includewal && ti->path == NULL)
379 {
380 Assert(lnext(state.tablespaces, lc) == NULL);
381 }
382 else
383 {
384 /* Properly terminate the tarfile. */
385 StaticAssertDecl(2 * TAR_BLOCK_SIZE <= BLCKSZ,
386 "BLCKSZ too small for 2 tar blocks");
387 memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
389
390 /* OK, that's the end of the archive. */
391 bbsink_end_archive(sink);
392 }
393 }
394
397
398 endptr = backup_state->stoppoint;
399 endtli = backup_state->stoptli;
400
401 /* Deallocate backup-related variables. */
404 }
406
407
408 if (opt->includewal)
409 {
410 /*
411 * We've left the last tar file "open", so we can now append the
412 * required WAL files to it.
413 */
414 char pathbuf[MAXPGPATH];
415 XLogSegNo segno;
416 XLogSegNo startsegno;
417 XLogSegNo endsegno;
418 struct stat statbuf;
419 List *historyFileList = NIL;
420 List *walFileList = NIL;
421 char firstoff[MAXFNAMELEN];
422 char lastoff[MAXFNAMELEN];
423 DIR *dir;
424 struct dirent *de;
425 ListCell *lc;
426 TimeLineID tli;
427
429
430 /*
431 * I'd rather not worry about timelines here, so scan pg_wal and
432 * include all WAL files in the range between 'startptr' and 'endptr',
433 * regardless of the timeline the file is stamped with. If there are
434 * some spurious WAL files belonging to timelines that don't belong in
435 * this server's history, they will be included too. Normally there
436 * shouldn't be such files, but if there are, there's little harm in
437 * including them.
438 */
439 XLByteToSeg(state.startptr, startsegno, wal_segment_size);
440 XLogFileName(firstoff, state.starttli, startsegno, wal_segment_size);
441 XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
442 XLogFileName(lastoff, endtli, endsegno, wal_segment_size);
443
444 dir = AllocateDir("pg_wal");
445 while ((de = ReadDir(dir, "pg_wal")) != NULL)
446 {
447 /* Does it look like a WAL segment, and is it in the range? */
448 if (IsXLogFileName(de->d_name) &&
449 strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
450 strcmp(de->d_name + 8, lastoff + 8) <= 0)
451 {
452 walFileList = lappend(walFileList, pstrdup(de->d_name));
453 }
454 /* Does it look like a timeline history file? */
455 else if (IsTLHistoryFileName(de->d_name))
456 {
457 historyFileList = lappend(historyFileList, pstrdup(de->d_name));
458 }
459 }
460 FreeDir(dir);
461
462 /*
463 * Before we go any further, check that none of the WAL segments we
464 * need were removed.
465 */
466 CheckXLogRemoved(startsegno, state.starttli);
467
468 /*
469 * Sort the WAL filenames. We want to send the files in order from
470 * oldest to newest, to reduce the chance that a file is recycled
471 * before we get a chance to send it over.
472 */
473 list_sort(walFileList, compareWalFileNames);
474
475 /*
476 * There must be at least one xlog file in the pg_wal directory, since
477 * we are doing backup-including-xlog.
478 */
479 if (walFileList == NIL)
481 (errmsg("could not find any WAL files")));
482
483 /*
484 * Sanity check: the first and last segment should cover startptr and
485 * endptr, with no gaps in between.
486 */
487 XLogFromFileName((char *) linitial(walFileList),
488 &tli, &segno, wal_segment_size);
489 if (segno != startsegno)
490 {
491 char startfname[MAXFNAMELEN];
492
493 XLogFileName(startfname, state.starttli, startsegno,
496 (errmsg("could not find WAL file \"%s\"", startfname)));
497 }
498 foreach(lc, walFileList)
499 {
500 char *walFileName = (char *) lfirst(lc);
501 XLogSegNo currsegno = segno;
502 XLogSegNo nextsegno = segno + 1;
503
504 XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
505 if (!(nextsegno == segno || currsegno == segno))
506 {
507 char nextfname[MAXFNAMELEN];
508
509 XLogFileName(nextfname, tli, nextsegno, wal_segment_size);
511 (errmsg("could not find WAL file \"%s\"", nextfname)));
512 }
513 }
514 if (segno != endsegno)
515 {
516 char endfname[MAXFNAMELEN];
517
518 XLogFileName(endfname, endtli, endsegno, wal_segment_size);
520 (errmsg("could not find WAL file \"%s\"", endfname)));
521 }
522
523 /* Ok, we have everything we need. Send the WAL files. */
524 foreach(lc, walFileList)
525 {
526 char *walFileName = (char *) lfirst(lc);
527 int fd;
528 ssize_t cnt;
529 pgoff_t len = 0;
530
531 snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
532 XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
533
534 fd = OpenTransientFile(pathbuf, O_RDONLY | PG_BINARY);
535 if (fd < 0)
536 {
537 int save_errno = errno;
538
539 /*
540 * Most likely reason for this is that the file was already
541 * removed by a checkpoint, so check for that to get a better
542 * error message.
543 */
544 CheckXLogRemoved(segno, tli);
545
546 errno = save_errno;
549 errmsg("could not open file \"%s\": %m", pathbuf)));
550 }
551
552 if (fstat(fd, &statbuf) != 0)
555 errmsg("could not stat file \"%s\": %m",
556 pathbuf)));
557 if (statbuf.st_size != wal_segment_size)
558 {
559 CheckXLogRemoved(segno, tli);
562 errmsg("unexpected WAL file size \"%s\"", walFileName)));
563 }
564
565 /* send the WAL file itself */
566 _tarWriteHeader(sink, pathbuf, NULL, &statbuf, false);
567
568 while ((cnt = basebackup_read_file(fd, sink->bbs_buffer,
571 len, pathbuf, true)) > 0)
572 {
573 CheckXLogRemoved(segno, tli);
574 bbsink_archive_contents(sink, cnt);
575
576 len += cnt;
577
578 if (len == wal_segment_size)
579 break;
580 }
581
582 if (len != wal_segment_size)
583 {
584 CheckXLogRemoved(segno, tli);
587 errmsg("unexpected WAL file size \"%s\"", walFileName)));
588 }
589
590 /*
591 * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need
592 * for padding.
593 */
595
597
598 /*
599 * Mark file as archived, otherwise files can get archived again
600 * after promotion of a new node. This is in line with
601 * walreceiver.c always doing an XLogArchiveForceDone() after a
602 * complete segment.
603 */
604 StatusFilePath(pathbuf, walFileName, ".done");
605 sendFileWithContent(sink, pathbuf, "", -1, &manifest);
606 }
607
608 /*
609 * Send timeline history files too. Only the latest timeline history
610 * file is required for recovery, and even that only if there happens
611 * to be a timeline switch in the first WAL segment that contains the
612 * checkpoint record, or if we're taking a base backup from a standby
613 * server and the target timeline changes while the backup is taken.
614 * But they are small and highly useful for debugging purposes, so
615 * better include them all, always.
616 */
617 foreach(lc, historyFileList)
618 {
619 char *fname = lfirst(lc);
620
621 snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
622
623 if (lstat(pathbuf, &statbuf) != 0)
626 errmsg("could not stat file \"%s\": %m", pathbuf)));
627
628 sendFile(sink, pathbuf, pathbuf, &statbuf, false,
630 &manifest, 0, NULL, 0);
631
632 /* unconditionally mark file as archived */
633 StatusFilePath(pathbuf, fname, ".done");
634 sendFileWithContent(sink, pathbuf, "", -1, &manifest);
635 }
636
637 /* Properly terminate the tar file. */
638 StaticAssertStmt(2 * TAR_BLOCK_SIZE <= BLCKSZ,
639 "BLCKSZ too small for 2 tar blocks");
640 memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
642
643 /* OK, that's the end of the archive. */
644 bbsink_end_archive(sink);
645 }
646
647 AddWALInfoToBackupManifest(&manifest, state.startptr, state.starttli,
648 endptr, endtli);
649
651
652 bbsink_end_backup(sink, endptr, endtli);
653
655 {
658 (errmsg_plural("%lld total checksum verification failure",
659 "%lld total checksum verification failures",
662
665 errmsg("checksum verification failure during base backup")));
666 }
667
668 /*
669 * Make sure to free the manifest before the resource owners as manifests
670 * use cryptohash contexts that may depend on resource owners (like
671 * OpenSSL).
672 */
674
675 /* clean up the resource owner we created */
677
679}
680
681/*
682 * list_sort comparison function, to compare log/seg portion of WAL segment
683 * filenames, ignoring the timeline portion.
684 */
685static int
687{
688 char *fna = (char *) lfirst(a);
689 char *fnb = (char *) lfirst(b);
690
691 return strcmp(fna + 8, fnb + 8);
692}
693
694/*
695 * Parse the base backup options passed down by the parser
696 */
697static void
699{
700 ListCell *lopt;
701 bool o_label = false;
702 bool o_progress = false;
703 bool o_checkpoint = false;
704 bool o_nowait = false;
705 bool o_wal = false;
706 bool o_incremental = false;
707 bool o_maxrate = false;
708 bool o_tablespace_map = false;
709 bool o_noverify_checksums = false;
710 bool o_manifest = false;
711 bool o_manifest_checksums = false;
712 bool o_target = false;
713 bool o_target_detail = false;
714 char *target_str = NULL;
715 char *target_detail_str = NULL;
716 bool o_compression = false;
717 bool o_compression_detail = false;
718 char *compression_detail_str = NULL;
719
720 MemSet(opt, 0, sizeof(*opt));
725
726 foreach(lopt, options)
727 {
728 DefElem *defel = (DefElem *) lfirst(lopt);
729
730 if (strcmp(defel->defname, "label") == 0)
731 {
732 if (o_label)
734 (errcode(ERRCODE_SYNTAX_ERROR),
735 errmsg("duplicate option \"%s\"", defel->defname)));
736 opt->label = defGetString(defel);
737 o_label = true;
738 }
739 else if (strcmp(defel->defname, "progress") == 0)
740 {
741 if (o_progress)
743 (errcode(ERRCODE_SYNTAX_ERROR),
744 errmsg("duplicate option \"%s\"", defel->defname)));
745 opt->progress = defGetBoolean(defel);
746 o_progress = true;
747 }
748 else if (strcmp(defel->defname, "checkpoint") == 0)
749 {
750 char *optval = defGetString(defel);
751
752 if (o_checkpoint)
754 (errcode(ERRCODE_SYNTAX_ERROR),
755 errmsg("duplicate option \"%s\"", defel->defname)));
756 if (pg_strcasecmp(optval, "fast") == 0)
757 opt->fastcheckpoint = true;
758 else if (pg_strcasecmp(optval, "spread") == 0)
759 opt->fastcheckpoint = false;
760 else
762 (errcode(ERRCODE_SYNTAX_ERROR),
763 errmsg("unrecognized checkpoint type: \"%s\"",
764 optval)));
765 o_checkpoint = true;
766 }
767 else if (strcmp(defel->defname, "wait") == 0)
768 {
769 if (o_nowait)
771 (errcode(ERRCODE_SYNTAX_ERROR),
772 errmsg("duplicate option \"%s\"", defel->defname)));
773 opt->nowait = !defGetBoolean(defel);
774 o_nowait = true;
775 }
776 else if (strcmp(defel->defname, "wal") == 0)
777 {
778 if (o_wal)
780 (errcode(ERRCODE_SYNTAX_ERROR),
781 errmsg("duplicate option \"%s\"", defel->defname)));
782 opt->includewal = defGetBoolean(defel);
783 o_wal = true;
784 }
785 else if (strcmp(defel->defname, "incremental") == 0)
786 {
787 if (o_incremental)
789 (errcode(ERRCODE_SYNTAX_ERROR),
790 errmsg("duplicate option \"%s\"", defel->defname)));
791 opt->incremental = defGetBoolean(defel);
792 if (opt->incremental && !summarize_wal)
794 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
795 errmsg("incremental backups cannot be taken unless WAL summarization is enabled")));
796 o_incremental = true;
797 }
798 else if (strcmp(defel->defname, "max_rate") == 0)
799 {
801
802 if (o_maxrate)
804 (errcode(ERRCODE_SYNTAX_ERROR),
805 errmsg("duplicate option \"%s\"", defel->defname)));
806
807 maxrate = defGetInt64(defel);
808 if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
810 (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
811 errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
812 (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
813
814 opt->maxrate = (uint32) maxrate;
815 o_maxrate = true;
816 }
817 else if (strcmp(defel->defname, "tablespace_map") == 0)
818 {
819 if (o_tablespace_map)
821 (errcode(ERRCODE_SYNTAX_ERROR),
822 errmsg("duplicate option \"%s\"", defel->defname)));
823 opt->sendtblspcmapfile = defGetBoolean(defel);
824 o_tablespace_map = true;
825 }
826 else if (strcmp(defel->defname, "verify_checksums") == 0)
827 {
828 if (o_noverify_checksums)
830 (errcode(ERRCODE_SYNTAX_ERROR),
831 errmsg("duplicate option \"%s\"", defel->defname)));
833 o_noverify_checksums = true;
834 }
835 else if (strcmp(defel->defname, "manifest") == 0)
836 {
837 char *optval = defGetString(defel);
838 bool manifest_bool;
839
840 if (o_manifest)
842 (errcode(ERRCODE_SYNTAX_ERROR),
843 errmsg("duplicate option \"%s\"", defel->defname)));
844 if (parse_bool(optval, &manifest_bool))
845 {
846 if (manifest_bool)
848 else
850 }
851 else if (pg_strcasecmp(optval, "force-encode") == 0)
853 else
855 (errcode(ERRCODE_SYNTAX_ERROR),
856 errmsg("unrecognized manifest option: \"%s\"",
857 optval)));
858 o_manifest = true;
859 }
860 else if (strcmp(defel->defname, "manifest_checksums") == 0)
861 {
862 char *optval = defGetString(defel);
863
864 if (o_manifest_checksums)
866 (errcode(ERRCODE_SYNTAX_ERROR),
867 errmsg("duplicate option \"%s\"", defel->defname)));
868 if (!pg_checksum_parse_type(optval,
871 (errcode(ERRCODE_SYNTAX_ERROR),
872 errmsg("unrecognized checksum algorithm: \"%s\"",
873 optval)));
874 o_manifest_checksums = true;
875 }
876 else if (strcmp(defel->defname, "target") == 0)
877 {
878 if (o_target)
880 (errcode(ERRCODE_SYNTAX_ERROR),
881 errmsg("duplicate option \"%s\"", defel->defname)));
882 target_str = defGetString(defel);
883 o_target = true;
884 }
885 else if (strcmp(defel->defname, "target_detail") == 0)
886 {
887 char *optval = defGetString(defel);
888
889 if (o_target_detail)
891 (errcode(ERRCODE_SYNTAX_ERROR),
892 errmsg("duplicate option \"%s\"", defel->defname)));
893 target_detail_str = optval;
894 o_target_detail = true;
895 }
896 else if (strcmp(defel->defname, "compression") == 0)
897 {
898 char *optval = defGetString(defel);
899
900 if (o_compression)
902 (errcode(ERRCODE_SYNTAX_ERROR),
903 errmsg("duplicate option \"%s\"", defel->defname)));
904 if (!parse_compress_algorithm(optval, &opt->compression))
906 (errcode(ERRCODE_SYNTAX_ERROR),
907 errmsg("unrecognized compression algorithm: \"%s\"",
908 optval)));
909 o_compression = true;
910 }
911 else if (strcmp(defel->defname, "compression_detail") == 0)
912 {
913 if (o_compression_detail)
915 (errcode(ERRCODE_SYNTAX_ERROR),
916 errmsg("duplicate option \"%s\"", defel->defname)));
917 compression_detail_str = defGetString(defel);
918 o_compression_detail = true;
919 }
920 else
922 (errcode(ERRCODE_SYNTAX_ERROR),
923 errmsg("unrecognized base backup option: \"%s\"",
924 defel->defname)));
925 }
926
927 if (opt->label == NULL)
928 opt->label = "base backup";
929 if (opt->manifest == MANIFEST_OPTION_NO)
930 {
931 if (o_manifest_checksums)
933 (errcode(ERRCODE_SYNTAX_ERROR),
934 errmsg("manifest checksums require a backup manifest")));
936 }
937
938 if (target_str == NULL)
939 {
940 if (target_detail_str != NULL)
942 (errcode(ERRCODE_SYNTAX_ERROR),
943 errmsg("target detail cannot be used without target")));
944 opt->use_copytblspc = true;
945 opt->send_to_client = true;
946 }
947 else if (strcmp(target_str, "client") == 0)
948 {
949 if (target_detail_str != NULL)
951 (errcode(ERRCODE_SYNTAX_ERROR),
952 errmsg("target \"%s\" does not accept a target detail",
953 target_str)));
954 opt->send_to_client = true;
955 }
956 else
957 opt->target_handle =
958 BaseBackupGetTargetHandle(target_str, target_detail_str);
959
960 if (o_compression_detail && !o_compression)
962 (errcode(ERRCODE_SYNTAX_ERROR),
963 errmsg("compression detail cannot be specified unless compression is enabled")));
964
965 if (o_compression)
966 {
967 char *error_detail;
968
969 parse_compress_specification(opt->compression, compression_detail_str,
971 error_detail =
973 if (error_detail != NULL)
975 errcode(ERRCODE_SYNTAX_ERROR),
976 errmsg("invalid compression specification: %s",
977 error_detail));
978 }
979}
980
981
982/*
983 * SendBaseBackup() - send a complete base backup.
984 *
985 * The function will put the system into backup mode like pg_backup_start()
986 * does, so that the backup is consistent even though we read directly from
987 * the filesystem, bypassing the buffer cache.
988 */
989void
991{
993 bbsink *sink;
995
996 if (status == SESSION_BACKUP_RUNNING)
998 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
999 errmsg("a backup is already in progress in this session")));
1000
1002
1004
1006 {
1007 char activitymsg[50];
1008
1009 snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
1010 opt.label);
1011 set_ps_display(activitymsg);
1012 }
1013
1014 /*
1015 * If we're asked to perform an incremental backup and the user has not
1016 * supplied a manifest, that's an ERROR.
1017 *
1018 * If we're asked to perform a full backup and the user did supply a
1019 * manifest, just ignore it.
1020 */
1021 if (!opt.incremental)
1022 ib = NULL;
1023 else if (ib == NULL)
1024 ereport(ERROR,
1025 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1026 errmsg("must UPLOAD_MANIFEST before performing an incremental BASE_BACKUP")));
1027
1028 /*
1029 * If the target is specifically 'client' then set up to stream the backup
1030 * to the client; otherwise, it's being sent someplace else and should not
1031 * be sent to the client. BaseBackupGetSink has the job of setting up a
1032 * sink to send the backup data wherever it needs to go.
1033 */
1035 if (opt.target_handle != NULL)
1036 sink = BaseBackupGetSink(opt.target_handle, sink);
1037
1038 /* Set up network throttling, if client requested it */
1039 if (opt.maxrate > 0)
1040 sink = bbsink_throttle_new(sink, opt.maxrate);
1041
1042 /* Set up server-side compression, if client requested it */
1044 sink = bbsink_gzip_new(sink, &opt.compression_specification);
1045 else if (opt.compression == PG_COMPRESSION_LZ4)
1046 sink = bbsink_lz4_new(sink, &opt.compression_specification);
1047 else if (opt.compression == PG_COMPRESSION_ZSTD)
1048 sink = bbsink_zstd_new(sink, &opt.compression_specification);
1049
1050 /* Set up progress reporting. */
1051 sink = bbsink_progress_new(sink, opt.progress);
1052
1053 /*
1054 * Perform the base backup, but make sure we clean up the bbsink even if
1055 * an error occurs.
1056 */
1057 PG_TRY();
1058 {
1059 perform_base_backup(&opt, sink, ib);
1060 }
1061 PG_FINALLY();
1062 {
1063 bbsink_cleanup(sink);
1064 }
1065 PG_END_TRY();
1066}
1067
1068/*
1069 * Inject a file with given name and content in the output tar stream.
1070 *
1071 * "len" can optionally be set to an arbitrary length of data sent. If set
1072 * to -1, the content sent is treated as a string with strlen() as length.
1073 */
1074static void
1075sendFileWithContent(bbsink *sink, const char *filename, const char *content,
1077{
1078 struct stat statbuf;
1079 int bytes_done = 0;
1080 pg_checksum_context checksum_ctx;
1081
1082 if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1083 elog(ERROR, "could not initialize checksum of file \"%s\"",
1084 filename);
1085
1086 if (len < 0)
1087 len = strlen(content);
1088
1089 /*
1090 * Construct a stat struct for the file we're injecting in the tar.
1091 */
1092
1093 /* Windows doesn't have the concept of uid and gid */
1094#ifdef WIN32
1095 statbuf.st_uid = 0;
1096 statbuf.st_gid = 0;
1097#else
1098 statbuf.st_uid = geteuid();
1099 statbuf.st_gid = getegid();
1100#endif
1101 statbuf.st_mtime = time(NULL);
1102 statbuf.st_mode = pg_file_create_mode;
1103 statbuf.st_size = len;
1104
1105 _tarWriteHeader(sink, filename, NULL, &statbuf, false);
1106
1107 if (pg_checksum_update(&checksum_ctx, (uint8 *) content, len) < 0)
1108 elog(ERROR, "could not update checksum of file \"%s\"",
1109 filename);
1110
1111 while (bytes_done < len)
1112 {
1113 size_t remaining = len - bytes_done;
1114 size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1115
1116 memcpy(sink->bbs_buffer, content, nbytes);
1117 bbsink_archive_contents(sink, nbytes);
1118 bytes_done += nbytes;
1119 content += nbytes;
1120 }
1121
1122 _tarWritePadding(sink, len);
1123
1125 (pg_time_t) statbuf.st_mtime, &checksum_ctx);
1126}
1127
1128/*
1129 * Include the tablespace directory pointed to by 'path' in the output tar
1130 * stream. If 'sizeonly' is true, we just calculate a total length and return
1131 * it, without actually sending anything.
1132 *
1133 * Only used to send auxiliary tablespaces, not PGDATA.
1134 */
1135static int64
1136sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly,
1138{
1139 int64 size;
1140 char pathbuf[MAXPGPATH];
1141 struct stat statbuf;
1142
1143 /*
1144 * 'path' points to the tablespace location, but we only want to include
1145 * the version directory in it that belongs to us.
1146 */
1147 snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1149
1150 /*
1151 * Store a directory entry in the tar file so we get the permissions
1152 * right.
1153 */
1154 if (lstat(pathbuf, &statbuf) != 0)
1155 {
1156 if (errno != ENOENT)
1157 ereport(ERROR,
1159 errmsg("could not stat file or directory \"%s\": %m",
1160 pathbuf)));
1161
1162 /* If the tablespace went away while scanning, it's no error. */
1163 return 0;
1164 }
1165
1166 size = _tarWriteHeader(sink, TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1167 sizeonly);
1168
1169 /* Send all the files in the tablespace version directory */
1170 size += sendDir(sink, pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1171 spcoid, ib);
1172
1173 return size;
1174}
1175
1176/*
1177 * Include all files from the given directory in the output tar stream. If
1178 * 'sizeonly' is true, we just calculate a total length and return it, without
1179 * actually sending anything.
1180 *
1181 * Omit any directory in the tablespaces list, to avoid backing up
1182 * tablespaces twice when they were created inside PGDATA.
1183 *
1184 * If sendtblspclinks is true, we need to include symlink
1185 * information in the tar file. If not, we can skip that
1186 * as it will be sent separately in the tablespace_map file.
1187 */
1188static int64
1189sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
1190 List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest,
1191 Oid spcoid, IncrementalBackupInfo *ib)
1192{
1193 DIR *dir;
1194 struct dirent *de;
1195 char pathbuf[MAXPGPATH * 2];
1196 struct stat statbuf;
1197 int64 size = 0;
1198 const char *lastDir; /* Split last dir from parent path. */
1199 bool isRelationDir = false; /* Does directory contain relations? */
1200 bool isGlobalDir = false;
1201 Oid dboid = InvalidOid;
1202 BlockNumber *relative_block_numbers = NULL;
1203
1204 /*
1205 * Since this array is relatively large, avoid putting it on the stack.
1206 * But we don't need it at all if this is not an incremental backup.
1207 */
1208 if (ib != NULL)
1209 relative_block_numbers = palloc(sizeof(BlockNumber) * RELSEG_SIZE);
1210
1211 /*
1212 * Determine if the current path is a database directory that can contain
1213 * relations.
1214 *
1215 * Start by finding the location of the delimiter between the parent path
1216 * and the current path.
1217 */
1218 lastDir = last_dir_separator(path);
1219
1220 /* Does this path look like a database path (i.e. all digits)? */
1221 if (lastDir != NULL &&
1222 strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1223 {
1224 /* Part of path that contains the parent directory. */
1225 int parentPathLen = lastDir - path;
1226
1227 /*
1228 * Mark path as a database directory if the parent path is either
1229 * $PGDATA/base or a tablespace version path.
1230 */
1231 if (strncmp(path, "./base", parentPathLen) == 0 ||
1232 (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1233 strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1235 sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1236 {
1237 isRelationDir = true;
1238 dboid = atooid(lastDir + 1);
1239 }
1240 }
1241 else if (strcmp(path, "./global") == 0)
1242 {
1243 isRelationDir = true;
1244 isGlobalDir = true;
1245 }
1246
1247 dir = AllocateDir(path);
1248 while ((de = ReadDir(dir, path)) != NULL)
1249 {
1250 int excludeIdx;
1251 bool excludeFound;
1252 RelFileNumber relfilenumber = InvalidRelFileNumber;
1253 ForkNumber relForkNum = InvalidForkNumber;
1254 unsigned segno = 0;
1255 bool isRelationFile = false;
1256
1257 /* Skip special stuff */
1258 if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1259 continue;
1260
1261 /* Skip temporary files */
1262 if (strncmp(de->d_name,
1264 strlen(PG_TEMP_FILE_PREFIX)) == 0)
1265 continue;
1266
1267 /* Skip macOS system files */
1268 if (strcmp(de->d_name, ".DS_Store") == 0)
1269 continue;
1270
1271 /*
1272 * Check if the postmaster has signaled us to exit, and abort with an
1273 * error in that case. The error handler further up will call
1274 * do_pg_abort_backup() for us. Also check that if the backup was
1275 * started while still in recovery, the server wasn't promoted.
1276 * do_pg_backup_stop() will check that too, but it's better to stop
1277 * the backup early than continue to the end and fail there.
1278 */
1281 ereport(ERROR,
1282 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1283 errmsg("the standby was promoted during online backup"),
1284 errhint("This means that the backup being taken is corrupt "
1285 "and should not be used. "
1286 "Try taking another online backup.")));
1287
1288 /* Scan for files that should be excluded */
1289 excludeFound = false;
1290 for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1291 {
1292 int cmplen = strlen(excludeFiles[excludeIdx].name);
1293
1294 if (!excludeFiles[excludeIdx].match_prefix)
1295 cmplen++;
1296 if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
1297 {
1298 elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1299 excludeFound = true;
1300 break;
1301 }
1302 }
1303
1304 if (excludeFound)
1305 continue;
1306
1307 /*
1308 * If there could be non-temporary relation files in this directory,
1309 * try to parse the filename.
1310 */
1311 if (isRelationDir)
1312 isRelationFile =
1314 &relfilenumber,
1315 &relForkNum, &segno);
1316
1317 /* Exclude all forks for unlogged tables except the init fork */
1318 if (isRelationFile && relForkNum != INIT_FORKNUM)
1319 {
1320 char initForkFile[MAXPGPATH];
1321
1322 /*
1323 * If any other type of fork, check if there is an init fork with
1324 * the same RelFileNumber. If so, the file can be excluded.
1325 */
1326 snprintf(initForkFile, sizeof(initForkFile), "%s/%u_init",
1327 path, relfilenumber);
1328
1329 if (lstat(initForkFile, &statbuf) == 0)
1330 {
1331 elog(DEBUG2,
1332 "unlogged relation file \"%s\" excluded from backup",
1333 de->d_name);
1334
1335 continue;
1336 }
1337 }
1338
1339 /* Exclude temporary relations */
1340 if (OidIsValid(dboid) && looks_like_temp_rel_name(de->d_name))
1341 {
1342 elog(DEBUG2,
1343 "temporary relation file \"%s\" excluded from backup",
1344 de->d_name);
1345
1346 continue;
1347 }
1348
1349 snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1350
1351 /* Skip pg_control here to back up it last */
1352 if (strcmp(pathbuf, "./global/pg_control") == 0)
1353 continue;
1354
1355 if (lstat(pathbuf, &statbuf) != 0)
1356 {
1357 if (errno != ENOENT)
1358 ereport(ERROR,
1360 errmsg("could not stat file or directory \"%s\": %m",
1361 pathbuf)));
1362
1363 /* If the file went away while scanning, it's not an error. */
1364 continue;
1365 }
1366
1367 /* Scan for directories whose contents should be excluded */
1368 excludeFound = false;
1369 for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1370 {
1371 if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1372 {
1373 elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1374 convert_link_to_directory(pathbuf, &statbuf);
1375 size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1376 &statbuf, sizeonly);
1377 excludeFound = true;
1378 break;
1379 }
1380 }
1381
1382 if (excludeFound)
1383 continue;
1384
1385 /*
1386 * We can skip pg_wal, the WAL segments need to be fetched from the
1387 * WAL archive anyway. But include it as an empty directory anyway, so
1388 * we get permissions right.
1389 */
1390 if (strcmp(pathbuf, "./pg_wal") == 0)
1391 {
1392 /* If pg_wal is a symlink, write it as a directory anyway */
1393 convert_link_to_directory(pathbuf, &statbuf);
1394 size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1395 &statbuf, sizeonly);
1396
1397 /*
1398 * Also send archive_status and summaries directories (by
1399 * hackishly reusing statbuf from above ...).
1400 */
1401 size += _tarWriteHeader(sink, "./pg_wal/archive_status", NULL,
1402 &statbuf, sizeonly);
1403 size += _tarWriteHeader(sink, "./pg_wal/summaries", NULL,
1404 &statbuf, sizeonly);
1405
1406 continue; /* don't recurse into pg_wal */
1407 }
1408
1409 /* Allow symbolic links in pg_tblspc only */
1410 if (strcmp(path, "./pg_tblspc") == 0 && S_ISLNK(statbuf.st_mode))
1411 {
1412 char linkpath[MAXPGPATH];
1413 int rllen;
1414
1415 rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1416 if (rllen < 0)
1417 ereport(ERROR,
1419 errmsg("could not read symbolic link \"%s\": %m",
1420 pathbuf)));
1421 if (rllen >= sizeof(linkpath))
1422 ereport(ERROR,
1423 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1424 errmsg("symbolic link \"%s\" target is too long",
1425 pathbuf)));
1426 linkpath[rllen] = '\0';
1427
1428 size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, linkpath,
1429 &statbuf, sizeonly);
1430 }
1431 else if (S_ISDIR(statbuf.st_mode))
1432 {
1433 bool skip_this_dir = false;
1434 ListCell *lc;
1435
1436 /*
1437 * Store a directory entry in the tar file so we can get the
1438 * permissions right.
1439 */
1440 size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, &statbuf,
1441 sizeonly);
1442
1443 /*
1444 * Call ourselves recursively for a directory, unless it happens
1445 * to be a separate tablespace located within PGDATA.
1446 */
1447 foreach(lc, tablespaces)
1448 {
1449 tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1450
1451 /*
1452 * ti->rpath is the tablespace relative path within PGDATA, or
1453 * NULL if the tablespace has been properly located somewhere
1454 * else.
1455 *
1456 * Skip past the leading "./" in pathbuf when comparing.
1457 */
1458 if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1459 {
1460 skip_this_dir = true;
1461 break;
1462 }
1463 }
1464
1465 /*
1466 * skip sending directories inside pg_tblspc, if not required.
1467 */
1468 if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1469 skip_this_dir = true;
1470
1471 if (!skip_this_dir)
1472 size += sendDir(sink, pathbuf, basepathlen, sizeonly, tablespaces,
1473 sendtblspclinks, manifest, spcoid, ib);
1474 }
1475 else if (S_ISREG(statbuf.st_mode))
1476 {
1477 bool sent = false;
1478 unsigned num_blocks_required = 0;
1479 unsigned truncation_block_length = 0;
1480 char tarfilenamebuf[MAXPGPATH * 2];
1481 char *tarfilename = pathbuf + basepathlen + 1;
1483
1484 if (ib != NULL && isRelationFile)
1485 {
1486 Oid relspcoid;
1487 char *lookup_path;
1488
1489 if (OidIsValid(spcoid))
1490 {
1491 relspcoid = spcoid;
1492 lookup_path = psprintf("%s/%u/%s", PG_TBLSPC_DIR, spcoid,
1493 tarfilename);
1494 }
1495 else
1496 {
1497 if (isGlobalDir)
1498 relspcoid = GLOBALTABLESPACE_OID;
1499 else
1500 relspcoid = DEFAULTTABLESPACE_OID;
1501 lookup_path = pstrdup(tarfilename);
1502 }
1503
1504 method = GetFileBackupMethod(ib, lookup_path, dboid, relspcoid,
1505 relfilenumber, relForkNum,
1506 segno, statbuf.st_size,
1507 &num_blocks_required,
1508 relative_block_numbers,
1509 &truncation_block_length);
1510 if (method == BACK_UP_FILE_INCREMENTALLY)
1511 {
1512 statbuf.st_size =
1513 GetIncrementalFileSize(num_blocks_required);
1514 snprintf(tarfilenamebuf, sizeof(tarfilenamebuf),
1515 "%s/INCREMENTAL.%s",
1516 path + basepathlen + 1,
1517 de->d_name);
1518 tarfilename = tarfilenamebuf;
1519 }
1520
1521 pfree(lookup_path);
1522 }
1523
1524 if (!sizeonly)
1525 sent = sendFile(sink, pathbuf, tarfilename, &statbuf,
1526 true, dboid, spcoid,
1527 relfilenumber, segno, manifest,
1528 num_blocks_required,
1529 method == BACK_UP_FILE_INCREMENTALLY ? relative_block_numbers : NULL,
1530 truncation_block_length);
1531
1532 if (sent || sizeonly)
1533 {
1534 /* Add size. */
1535 size += statbuf.st_size;
1536
1537 /* Pad to a multiple of the tar block size. */
1539
1540 /* Size of the header for the file. */
1542 }
1543 }
1544 else
1546 (errmsg("skipping special file \"%s\"", pathbuf)));
1547 }
1548
1549 if (relative_block_numbers != NULL)
1550 pfree(relative_block_numbers);
1551
1552 FreeDir(dir);
1553 return size;
1554}
1555
1556/*
1557 * Given the member, write the TAR header & send the file.
1558 *
1559 * If 'missing_ok' is true, will not throw an error if the file is not found.
1560 *
1561 * If dboid is anything other than InvalidOid then any checksum failures
1562 * detected will get reported to the cumulative stats system.
1563 *
1564 * If the file is to be sent incrementally, then num_incremental_blocks
1565 * should be the number of blocks to be sent, and incremental_blocks
1566 * an array of block numbers relative to the start of the current segment.
1567 * If the whole file is to be sent, then incremental_blocks should be NULL,
1568 * and num_incremental_blocks can have any value, as it will be ignored.
1569 *
1570 * Returns true if the file was successfully sent, false if 'missing_ok',
1571 * and the file did not exist.
1572 */
1573static bool
1574sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
1575 struct stat *statbuf, bool missing_ok, Oid dboid, Oid spcoid,
1576 RelFileNumber relfilenumber, unsigned segno,
1577 backup_manifest_info *manifest, unsigned num_incremental_blocks,
1578 BlockNumber *incremental_blocks, unsigned truncation_block_length)
1579{
1580 int fd;
1581 BlockNumber blkno = 0;
1582 int checksum_failures = 0;
1583 off_t cnt;
1584 pgoff_t bytes_done = 0;
1585 bool verify_checksum = false;
1586 pg_checksum_context checksum_ctx;
1587 int ibindex = 0;
1588
1589 if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1590 elog(ERROR, "could not initialize checksum of file \"%s\"",
1591 readfilename);
1592
1593 fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY);
1594 if (fd < 0)
1595 {
1596 if (errno == ENOENT && missing_ok)
1597 return false;
1598 ereport(ERROR,
1600 errmsg("could not open file \"%s\": %m", readfilename)));
1601 }
1602
1603 _tarWriteHeader(sink, tarfilename, NULL, statbuf, false);
1604
1605 /*
1606 * Checksums are verified in multiples of BLCKSZ, so the buffer length
1607 * should be a multiple of the block size as well.
1608 */
1609 Assert((sink->bbs_buffer_length % BLCKSZ) == 0);
1610
1611 /*
1612 * If we weren't told not to verify checksums, and if checksums are
1613 * enabled for this cluster, and if this is a relation file, then verify
1614 * the checksum.
1615 */
1617 RelFileNumberIsValid(relfilenumber))
1618 verify_checksum = true;
1619
1620 /*
1621 * If we're sending an incremental file, write the file header.
1622 */
1623 if (incremental_blocks != NULL)
1624 {
1625 unsigned magic = INCREMENTAL_MAGIC;
1626 size_t header_bytes_done = 0;
1627 char padding[BLCKSZ];
1628 size_t paddinglen;
1629
1630 /* Emit header data. */
1631 push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1632 &magic, sizeof(magic));
1633 push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1634 &num_incremental_blocks, sizeof(num_incremental_blocks));
1635 push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1636 &truncation_block_length, sizeof(truncation_block_length));
1637 push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1638 incremental_blocks,
1639 sizeof(BlockNumber) * num_incremental_blocks);
1640
1641 /*
1642 * Add padding to align header to a multiple of BLCKSZ, but only if
1643 * the incremental file has some blocks, and the alignment is actually
1644 * needed (i.e. header is not already a multiple of BLCKSZ). If there
1645 * are no blocks we don't want to make the file unnecessarily large,
1646 * as that might make some filesystem optimizations impossible.
1647 */
1648 if ((num_incremental_blocks > 0) && (header_bytes_done % BLCKSZ != 0))
1649 {
1650 paddinglen = (BLCKSZ - (header_bytes_done % BLCKSZ));
1651
1652 memset(padding, 0, paddinglen);
1653 bytes_done += paddinglen;
1654
1655 push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1656 padding, paddinglen);
1657 }
1658
1659 /* Flush out any data still in the buffer so it's again empty. */
1660 if (header_bytes_done > 0)
1661 {
1662 bbsink_archive_contents(sink, header_bytes_done);
1663 if (pg_checksum_update(&checksum_ctx,
1664 (uint8 *) sink->bbs_buffer,
1665 header_bytes_done) < 0)
1666 elog(ERROR, "could not update checksum of base backup");
1667 }
1668
1669 /* Update our notion of file position. */
1670 bytes_done += sizeof(magic);
1671 bytes_done += sizeof(num_incremental_blocks);
1672 bytes_done += sizeof(truncation_block_length);
1673 bytes_done += sizeof(BlockNumber) * num_incremental_blocks;
1674 }
1675
1676 /*
1677 * Loop until we read the amount of data the caller told us to expect. The
1678 * file could be longer, if it was extended while we were sending it, but
1679 * for a base backup we can ignore such extended data. It will be restored
1680 * from WAL.
1681 */
1682 while (1)
1683 {
1684 /*
1685 * Determine whether we've read all the data that we need, and if not,
1686 * read some more.
1687 */
1688 if (incremental_blocks == NULL)
1689 {
1690 size_t remaining = statbuf->st_size - bytes_done;
1691
1692 /*
1693 * If we've read the required number of bytes, then it's time to
1694 * stop.
1695 */
1696 if (bytes_done >= statbuf->st_size)
1697 break;
1698
1699 /*
1700 * Read as many bytes as will fit in the buffer, or however many
1701 * are left to read, whichever is less.
1702 */
1703 cnt = read_file_data_into_buffer(sink, readfilename, fd,
1704 bytes_done, remaining,
1705 blkno + segno * RELSEG_SIZE,
1706 verify_checksum,
1707 &checksum_failures);
1708 }
1709 else
1710 {
1711 BlockNumber relative_blkno;
1712
1713 /*
1714 * If we've read all the blocks, then it's time to stop.
1715 */
1716 if (ibindex >= num_incremental_blocks)
1717 break;
1718
1719 /*
1720 * Read just one block, whichever one is the next that we're
1721 * supposed to include.
1722 */
1723 relative_blkno = incremental_blocks[ibindex++];
1724 cnt = read_file_data_into_buffer(sink, readfilename, fd,
1725 relative_blkno * BLCKSZ,
1726 BLCKSZ,
1727 relative_blkno + segno * RELSEG_SIZE,
1728 verify_checksum,
1729 &checksum_failures);
1730
1731 /*
1732 * If we get a partial read, that must mean that the relation is
1733 * being truncated. Ultimately, it should be truncated to a
1734 * multiple of BLCKSZ, since this path should only be reached for
1735 * relation files, but we might transiently observe an
1736 * intermediate value.
1737 *
1738 * It should be fine to treat this just as if the entire block had
1739 * been truncated away - i.e. fill this and all later blocks with
1740 * zeroes. WAL replay will fix things up.
1741 */
1742 if (cnt < BLCKSZ)
1743 break;
1744 }
1745
1746 /*
1747 * If the amount of data we were able to read was not a multiple of
1748 * BLCKSZ, we cannot verify checksums, which are block-level.
1749 */
1750 if (verify_checksum && (cnt % BLCKSZ != 0))
1751 {
1753 (errmsg("could not verify checksum in file \"%s\", block "
1754 "%u: read buffer size %d and page size %d "
1755 "differ",
1756 readfilename, blkno, (int) cnt, BLCKSZ)));
1757 verify_checksum = false;
1758 }
1759
1760 /*
1761 * If we hit end-of-file, a concurrent truncation must have occurred.
1762 * That's not an error condition, because WAL replay will fix things
1763 * up.
1764 */
1765 if (cnt == 0)
1766 break;
1767
1768 /* Update block number and # of bytes done for next loop iteration. */
1769 blkno += cnt / BLCKSZ;
1770 bytes_done += cnt;
1771
1772 /*
1773 * Make sure incremental files with block data are properly aligned
1774 * (header is a multiple of BLCKSZ, blocks are BLCKSZ too).
1775 */
1776 Assert(!((incremental_blocks != NULL && num_incremental_blocks > 0) &&
1777 (bytes_done % BLCKSZ != 0)));
1778
1779 /* Archive the data we just read. */
1780 bbsink_archive_contents(sink, cnt);
1781
1782 /* Also feed it to the checksum machinery. */
1783 if (pg_checksum_update(&checksum_ctx,
1784 (uint8 *) sink->bbs_buffer, cnt) < 0)
1785 elog(ERROR, "could not update checksum of base backup");
1786 }
1787
1788 /* If the file was truncated while we were sending it, pad it with zeros */
1789 while (bytes_done < statbuf->st_size)
1790 {
1791 size_t remaining = statbuf->st_size - bytes_done;
1792 size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1793
1794 MemSet(sink->bbs_buffer, 0, nbytes);
1795 if (pg_checksum_update(&checksum_ctx,
1796 (uint8 *) sink->bbs_buffer,
1797 nbytes) < 0)
1798 elog(ERROR, "could not update checksum of base backup");
1799 bbsink_archive_contents(sink, nbytes);
1800 bytes_done += nbytes;
1801 }
1802
1803 /*
1804 * Pad to a block boundary, per tar format requirements. (This small piece
1805 * of data is probably not worth throttling, and is not checksummed
1806 * because it's not actually part of the file.)
1807 */
1808 _tarWritePadding(sink, bytes_done);
1809
1811
1812 if (checksum_failures > 1)
1813 {
1815 (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1816 "file \"%s\" has a total of %d checksum verification failures",
1817 checksum_failures,
1818 readfilename, checksum_failures)));
1819
1820 pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1821 }
1822
1823 total_checksum_failures += checksum_failures;
1824
1825 AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
1826 (pg_time_t) statbuf->st_mtime, &checksum_ctx);
1827
1828 return true;
1829}
1830
1831/*
1832 * Read some more data from the file into the bbsink's buffer, verifying
1833 * checksums as required.
1834 *
1835 * 'offset' is the file offset from which we should begin to read, and
1836 * 'length' is the amount of data that should be read. The actual amount
1837 * of data read will be less than the requested amount if the bbsink's
1838 * buffer isn't big enough to hold it all, or if the underlying file has
1839 * been truncated. The return value is the number of bytes actually read.
1840 *
1841 * 'blkno' is the block number of the first page in the bbsink's buffer
1842 * relative to the start of the relation.
1843 *
1844 * 'verify_checksum' indicates whether we should try to verify checksums
1845 * for the blocks we read. If we do this, we'll update *checksum_failures
1846 * and issue warnings as appropriate.
1847 */
1848static off_t
1849read_file_data_into_buffer(bbsink *sink, const char *readfilename, int fd,
1850 off_t offset, size_t length, BlockNumber blkno,
1851 bool verify_checksum, int *checksum_failures)
1852{
1853 off_t cnt;
1854 int i;
1855 char *page;
1856
1857 /* Try to read some more data. */
1858 cnt = basebackup_read_file(fd, sink->bbs_buffer,
1859 Min(sink->bbs_buffer_length, length),
1860 offset, readfilename, true);
1861
1862 /* Can't verify checksums if read length is not a multiple of BLCKSZ. */
1863 if (!verify_checksum || (cnt % BLCKSZ) != 0)
1864 return cnt;
1865
1866 /* Verify checksum for each block. */
1867 for (i = 0; i < cnt / BLCKSZ; i++)
1868 {
1869 int reread_cnt;
1870 uint16 expected_checksum;
1871
1872 page = sink->bbs_buffer + BLCKSZ * i;
1873
1874 /* If the page is OK, go on to the next one. */
1875 if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i,
1876 &expected_checksum))
1877 continue;
1878
1879 /*
1880 * Retry the block on the first failure. It's possible that we read
1881 * the first 4K page of the block just before postgres updated the
1882 * entire block so it ends up looking torn to us. If, before we retry
1883 * the read, the concurrent write of the block finishes, the page LSN
1884 * will be updated and we'll realize that we should ignore this block.
1885 *
1886 * There's no guarantee that this will actually happen, though: the
1887 * torn write could take an arbitrarily long time to complete.
1888 * Retrying multiple times wouldn't fix this problem, either, though
1889 * it would reduce the chances of it happening in practice. The only
1890 * real fix here seems to be to have some kind of interlock that
1891 * allows us to wait until we can be certain that no write to the
1892 * block is in progress. Since we don't have any such thing right now,
1893 * we just do this and hope for the best.
1894 */
1895 reread_cnt =
1896 basebackup_read_file(fd, sink->bbs_buffer + BLCKSZ * i,
1897 BLCKSZ, offset + BLCKSZ * i,
1898 readfilename, false);
1899 if (reread_cnt == 0)
1900 {
1901 /*
1902 * If we hit end-of-file, a concurrent truncation must have
1903 * occurred, so reduce cnt to reflect only the blocks already
1904 * processed and break out of this loop.
1905 */
1906 cnt = BLCKSZ * i;
1907 break;
1908 }
1909
1910 /* If the page now looks OK, go on to the next one. */
1911 if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i,
1912 &expected_checksum))
1913 continue;
1914
1915 /* Handle checksum failure. */
1916 (*checksum_failures)++;
1917 if (*checksum_failures <= 5)
1919 (errmsg("checksum verification failed in "
1920 "file \"%s\", block %u: calculated "
1921 "%X but expected %X",
1922 readfilename, blkno + i, expected_checksum,
1923 ((PageHeader) page)->pd_checksum)));
1924 if (*checksum_failures == 5)
1926 (errmsg("further checksum verification "
1927 "failures in file \"%s\" will not "
1928 "be reported", readfilename)));
1929 }
1930
1931 return cnt;
1932}
1933
1934/*
1935 * Push data into a bbsink.
1936 *
1937 * It's better, when possible, to read data directly into the bbsink's buffer,
1938 * rather than using this function to copy it into the buffer; this function is
1939 * for cases where that approach is not practical.
1940 *
1941 * bytes_done should point to a count of the number of bytes that are
1942 * currently used in the bbsink's buffer. Upon return, the bytes identified by
1943 * data and length will have been copied into the bbsink's buffer, flushing
1944 * as required, and *bytes_done will have been updated accordingly. If the
1945 * buffer was flushed, the previous contents will also have been fed to
1946 * checksum_ctx.
1947 *
1948 * Note that after one or more calls to this function it is the caller's
1949 * responsibility to perform any required final flush.
1950 */
1951static void
1953 size_t *bytes_done, void *data, size_t length)
1954{
1955 while (length > 0)
1956 {
1957 size_t bytes_to_copy;
1958
1959 /*
1960 * We use < here rather than <= so that if the data exactly fills the
1961 * remaining buffer space, we trigger a flush now.
1962 */
1963 if (length < sink->bbs_buffer_length - *bytes_done)
1964 {
1965 /* Append remaining data to buffer. */
1966 memcpy(sink->bbs_buffer + *bytes_done, data, length);
1967 *bytes_done += length;
1968 return;
1969 }
1970
1971 /* Copy until buffer is full and flush it. */
1972 bytes_to_copy = sink->bbs_buffer_length - *bytes_done;
1973 memcpy(sink->bbs_buffer + *bytes_done, data, bytes_to_copy);
1974 data = ((char *) data) + bytes_to_copy;
1975 length -= bytes_to_copy;
1977 if (pg_checksum_update(checksum_ctx, (uint8 *) sink->bbs_buffer,
1978 sink->bbs_buffer_length) < 0)
1979 elog(ERROR, "could not update checksum");
1980 *bytes_done = 0;
1981 }
1982}
1983
1984/*
1985 * Try to verify the checksum for the provided page, if it seems appropriate
1986 * to do so.
1987 *
1988 * Returns true if verification succeeds or if we decide not to check it,
1989 * and false if verification fails. When return false, it also sets
1990 * *expected_checksum to the computed value.
1991 */
1992static bool
1994 uint16 *expected_checksum)
1995{
1996 PageHeader phdr;
1997 uint16 checksum;
1998
1999 /*
2000 * Only check pages which have not been modified since the start of the
2001 * base backup. Otherwise, they might have been written only halfway and
2002 * the checksum would not be valid. However, replaying WAL would
2003 * reinstate the correct page in this case. We also skip completely new
2004 * pages, since they don't have a checksum yet.
2005 */
2006 if (PageIsNew(page) || PageGetLSN(page) >= start_lsn)
2007 return true;
2008
2009 /* Perform the actual checksum calculation. */
2010 checksum = pg_checksum_page(page, blkno);
2011
2012 /* See whether it matches the value from the page. */
2013 phdr = (PageHeader) page;
2014 if (phdr->pd_checksum == checksum)
2015 return true;
2016 *expected_checksum = checksum;
2017 return false;
2018}
2019
2020static int64
2021_tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget,
2022 struct stat *statbuf, bool sizeonly)
2023{
2024 enum tarError rc;
2025
2026 if (!sizeonly)
2027 {
2028 /*
2029 * As of this writing, the smallest supported block size is 1kB, which
2030 * is twice TAR_BLOCK_SIZE. Since the buffer size is required to be a
2031 * multiple of BLCKSZ, it should be safe to assume that the buffer is
2032 * large enough to fit an entire tar block. We double-check by means
2033 * of these assertions.
2034 */
2036 "BLCKSZ too small for tar block");
2038
2039 rc = tarCreateHeader(sink->bbs_buffer, filename, linktarget,
2040 statbuf->st_size, statbuf->st_mode,
2041 statbuf->st_uid, statbuf->st_gid,
2042 statbuf->st_mtime);
2043
2044 switch (rc)
2045 {
2046 case TAR_OK:
2047 break;
2048 case TAR_NAME_TOO_LONG:
2049 ereport(ERROR,
2050 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
2051 errmsg("file name too long for tar format: \"%s\"",
2052 filename)));
2053 break;
2055 ereport(ERROR,
2056 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
2057 errmsg("symbolic link target too long for tar format: "
2058 "file name \"%s\", target \"%s\"",
2059 filename, linktarget)));
2060 break;
2061 default:
2062 elog(ERROR, "unrecognized tar error: %d", rc);
2063 }
2064
2066 }
2067
2068 return TAR_BLOCK_SIZE;
2069}
2070
2071/*
2072 * Pad with zero bytes out to a multiple of TAR_BLOCK_SIZE.
2073 */
2074static void
2076{
2077 int pad = tarPaddingBytesRequired(len);
2078
2079 /*
2080 * As in _tarWriteHeader, it should be safe to assume that the buffer is
2081 * large enough that we don't need to do this in multiple chunks.
2082 */
2084 Assert(pad <= TAR_BLOCK_SIZE);
2085
2086 if (pad > 0)
2087 {
2088 MemSet(sink->bbs_buffer, 0, pad);
2089 bbsink_archive_contents(sink, pad);
2090 }
2091}
2092
2093/*
2094 * If the entry in statbuf is a link, then adjust statbuf to make it look like a
2095 * directory, so that it will be written that way.
2096 */
2097static void
2098convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
2099{
2100 /* If symlink, write it as a directory anyway */
2101 if (S_ISLNK(statbuf->st_mode))
2102 statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
2103}
2104
2105/*
2106 * Read some data from a file, setting a wait event and reporting any error
2107 * encountered.
2108 *
2109 * If partial_read_ok is false, also report an error if the number of bytes
2110 * read is not equal to the number of bytes requested.
2111 *
2112 * Returns the number of bytes read.
2113 */
2114static ssize_t
2115basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
2116 const char *filename, bool partial_read_ok)
2117{
2118 ssize_t rc;
2119
2120 pgstat_report_wait_start(WAIT_EVENT_BASEBACKUP_READ);
2121 rc = pg_pread(fd, buf, nbytes, offset);
2123
2124 if (rc < 0)
2125 ereport(ERROR,
2127 errmsg("could not read file \"%s\": %m", filename)));
2128 if (!partial_read_ok && rc > 0 && rc != nbytes)
2129 ereport(ERROR,
2131 errmsg("could not read file \"%s\": read %zd of %zu",
2132 filename, rc, nbytes)));
2133
2134 return rc;
2135}
void InitializeBackupManifest(backup_manifest_info *manifest, backup_manifest_option want_manifest, pg_checksum_type manifest_checksum_type)
void AddFileToBackupManifest(backup_manifest_info *manifest, Oid spcoid, const char *pathname, size_t size, pg_time_t mtime, pg_checksum_context *checksum_ctx)
void AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr, TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)
void SendBackupManifest(backup_manifest_info *manifest, bbsink *sink)
void FreeBackupManifest(backup_manifest_info *manifest)
@ MANIFEST_OPTION_NO
@ MANIFEST_OPTION_FORCE_ENCODE
@ MANIFEST_OPTION_YES
enum manifest_option backup_manifest_option
static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, Oid spcoid, RelFileNumber relfilenumber, unsigned segno, backup_manifest_info *manifest, unsigned num_incremental_blocks, BlockNumber *incremental_blocks, unsigned truncation_block_length)
Definition: basebackup.c:1574
static const struct exclude_list_item excludeFiles[]
Definition: basebackup.c:191
static int compareWalFileNames(const ListCell *a, const ListCell *b)
Definition: basebackup.c:686
static void sendFileWithContent(bbsink *sink, const char *filename, const char *content, int len, backup_manifest_info *manifest)
Definition: basebackup.c:1075
static off_t read_file_data_into_buffer(bbsink *sink, const char *readfilename, int fd, off_t offset, size_t length, BlockNumber blkno, bool verify_checksum, int *checksum_failures)
Definition: basebackup.c:1849
static void push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx, size_t *bytes_done, void *data, size_t length)
Definition: basebackup.c:1952
static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, Oid spcoid, IncrementalBackupInfo *ib)
Definition: basebackup.c:1189
#define SINK_BUFFER_LENGTH
Definition: basebackup.c:60
static void parse_basebackup_options(List *options, basebackup_options *opt)
Definition: basebackup.c:698
static const char *const excludeDirContents[]
Definition: basebackup.c:151
static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
Definition: basebackup.c:2098
void SendBaseBackup(BaseBackupCmd *cmd, IncrementalBackupInfo *ib)
Definition: basebackup.c:990
static bool backup_started_in_recovery
Definition: basebackup.c:123
static int64 _tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:2021
static void perform_base_backup(basebackup_options *opt, bbsink *sink, IncrementalBackupInfo *ib)
Definition: basebackup.c:234
static int64 sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly, struct backup_manifest_info *manifest, IncrementalBackupInfo *ib)
Definition: basebackup.c:1136
static bool noverify_checksums
Definition: basebackup.c:129
static void _tarWritePadding(bbsink *sink, int len)
Definition: basebackup.c:2075
static bool verify_page_checksum(Page page, XLogRecPtr start_lsn, BlockNumber blkno, uint16 *expected_checksum)
Definition: basebackup.c:1993
static ssize_t basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset, const char *filename, bool partial_read_ok)
Definition: basebackup.c:2115
static long long int total_checksum_failures
Definition: basebackup.c:126
#define MAX_RATE_UPPER
Definition: basebackup.h:21
#define MAX_RATE_LOWER
Definition: basebackup.h:20
bbsink * bbsink_copystream_new(bool send_to_client)
bbsink * bbsink_gzip_new(bbsink *next, pg_compress_specification *compress)
size_t GetIncrementalFileSize(unsigned num_blocks_required)
FileBackupMethod GetFileBackupMethod(IncrementalBackupInfo *ib, const char *path, Oid dboid, Oid spcoid, RelFileNumber relfilenumber, ForkNumber forknum, unsigned segno, size_t size, unsigned *num_blocks_required, BlockNumber *relative_block_numbers, unsigned *truncation_block_length)
void PrepareForIncrementalBackup(IncrementalBackupInfo *ib, BackupState *backup_state)
#define INCREMENTAL_MAGIC
@ BACK_UP_FILE_INCREMENTALLY
@ BACK_UP_FILE_FULLY
bbsink * bbsink_lz4_new(bbsink *next, pg_compress_specification *compress)
void basebackup_progress_wait_checkpoint(void)
bbsink * bbsink_progress_new(bbsink *next, bool estimate_backup_size)
void basebackup_progress_wait_wal_archive(bbsink_state *state)
void basebackup_progress_done(void)
void basebackup_progress_transfer_wal(void)
void basebackup_progress_estimate_backup_size(void)
static void bbsink_begin_backup(bbsink *sink, bbsink_state *state, int buffer_length)
static void bbsink_begin_archive(bbsink *sink, const char *archive_name)
static void bbsink_end_archive(bbsink *sink)
static void bbsink_end_backup(bbsink *sink, XLogRecPtr endptr, TimeLineID endtli)
static void bbsink_cleanup(bbsink *sink)
static void bbsink_archive_contents(bbsink *sink, size_t len)
BaseBackupTargetHandle * BaseBackupGetTargetHandle(char *target, char *target_detail)
bbsink * BaseBackupGetSink(BaseBackupTargetHandle *handle, bbsink *next_sink)
bbsink * bbsink_throttle_new(bbsink *next, uint32 maxrate)
bbsink * bbsink_zstd_new(bbsink *next, pg_compress_specification *compress)
uint32 BlockNumber
Definition: block.h:31
bool parse_bool(const char *value, bool *result)
Definition: bool.c:31
PageHeaderData * PageHeader
Definition: bufpage.h:173
Pointer Page
Definition: bufpage.h:81
static bool PageIsNew(Page page)
Definition: bufpage.h:233
static XLogRecPtr PageGetLSN(const char *page)
Definition: bufpage.h:386
#define Min(x, y)
Definition: c.h:958
uint8_t uint8
Definition: c.h:483
#define Assert(condition)
Definition: c.h:812
int64_t int64
Definition: c.h:482
#define PG_BINARY
Definition: c.h:1227
uint16_t uint16
Definition: c.h:484
uint32_t uint32
Definition: c.h:485
#define MemSet(start, val, len)
Definition: c.h:974
#define StaticAssertDecl(condition, errmessage)
Definition: c.h:890
#define StaticAssertStmt(condition, errmessage)
Definition: c.h:892
#define OidIsValid(objectId)
Definition: c.h:729
uint16 pg_checksum_page(char *page, BlockNumber blkno)
bool pg_checksum_parse_type(char *name, pg_checksum_type *type)
int pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
int pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
pg_checksum_type
@ CHECKSUM_TYPE_NONE
@ CHECKSUM_TYPE_CRC32C
char * validate_compress_specification(pg_compress_specification *spec)
Definition: compression.c:344
bool parse_compress_algorithm(char *name, pg_compress_algorithm *algorithm)
Definition: compression.c:49
void parse_compress_specification(pg_compress_algorithm algorithm, char *specification, pg_compress_specification *result)
Definition: compression.c:107
pg_compress_algorithm
Definition: compression.h:22
@ PG_COMPRESSION_GZIP
Definition: compression.h:24
@ PG_COMPRESSION_LZ4
Definition: compression.h:25
@ PG_COMPRESSION_NONE
Definition: compression.h:23
@ PG_COMPRESSION_ZSTD
Definition: compression.h:26
char * defGetString(DefElem *def)
Definition: define.c:35
bool defGetBoolean(DefElem *def)
Definition: define.c:94
int64 defGetInt64(DefElem *def)
Definition: define.c:173
#define PG_DYNSHMEM_DIR
Definition: dsm_impl.h:51
int errmsg_plural(const char *fmt_singular, const char *fmt_plural, unsigned long n,...)
Definition: elog.c:1180
int errcode_for_file_access(void)
Definition: elog.c:876
int errhint(const char *fmt,...)
Definition: elog.c:1317
int errcode(int sqlerrcode)
Definition: elog.c:853
int errmsg(const char *fmt,...)
Definition: elog.c:1070
#define PG_TRY(...)
Definition: elog.h:371
#define WARNING
Definition: elog.h:36
#define DEBUG2
Definition: elog.h:29
#define PG_END_TRY(...)
Definition: elog.h:396
#define DEBUG1
Definition: elog.h:30
#define ERROR
Definition: elog.h:39
#define elog(elevel,...)
Definition: elog.h:225
#define PG_FINALLY(...)
Definition: elog.h:388
#define ereport(elevel,...)
Definition: elog.h:149
int FreeDir(DIR *dir)
Definition: fd.c:2983
int CloseTransientFile(int fd)
Definition: fd.c:2831
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2865
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2931
bool looks_like_temp_rel_name(const char *name)
Definition: fd.c:3472
int OpenTransientFile(const char *fileName, int fileFlags)
Definition: fd.c:2655
int pg_file_create_mode
Definition: file_perm.c:19
int pg_dir_create_mode
Definition: file_perm.c:18
#define PG_TEMP_FILE_PREFIX
Definition: file_utils.h:63
#define PG_AUTOCONF_FILENAME
Definition: guc.h:33
int remaining
Definition: informix.c:692
#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:47
#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:52
int b
Definition: isn.c:69
int a
Definition: isn.c:68
int i
Definition: isn.c:72
List * lappend(List *list, void *datum)
Definition: list.c:339
void list_sort(List *list, list_sort_comparator cmp)
Definition: list.c:1674
char * pstrdup(const char *in)
Definition: mcxt.c:1696
void pfree(void *pointer)
Definition: mcxt.c:1521
void * palloc0(Size size)
Definition: mcxt.c:1347
void * palloc(Size size)
Definition: mcxt.c:1317
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:122
static bool manifest
static int32 maxrate
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:41
#define MAXPGPATH
const void size_t len
const void * data
static char * filename
Definition: pg_dumpall.c:119
#define lfirst(lc)
Definition: pg_list.h:172
#define NIL
Definition: pg_list.h:68
#define linitial(l)
Definition: pg_list.h:178
static ListCell * lnext(const List *l, const ListCell *c)
Definition: pg_list.h:343
static char * buf
Definition: pg_test_fsync.c:72
#define PG_STAT_TMP_DIR
Definition: pgstat.h:34
void pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount)
static size_t tarPaddingBytesRequired(size_t len)
Definition: pgtar.h:79
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
tarError
Definition: pgtar.h:20
@ TAR_SYMLINK_TOO_LONG
Definition: pgtar.h:23
@ TAR_OK
Definition: pgtar.h:21
@ TAR_NAME_TOO_LONG
Definition: pgtar.h:22
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
int64 pg_time_t
Definition: pgtime.h:23
char * last_dir_separator(const char *filename)
Definition: path.c:140
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
#define pg_pread
Definition: port.h:225
#define snprintf
Definition: port.h:238
#define pgoff_t
Definition: port.h:373
static Datum BoolGetDatum(bool X)
Definition: postgres.h:102
#define InvalidOid
Definition: postgres_ext.h:36
unsigned int Oid
Definition: postgres_ext.h:31
#define atooid(x)
Definition: postgres_ext.h:42
static int fd(const char *x, int i)
Definition: preproc-init.c:105
bool update_process_title
Definition: ps_status.c:31
static void set_ps_display(const char *activity)
Definition: ps_status.h:40
char * psprintf(const char *fmt,...)
Definition: psprintf.c:43
bool parse_filename_for_nontemp_relation(const char *name, RelFileNumber *relnumber, ForkNumber *fork, unsigned *segno)
Definition: reinit.c:380
#define RELCACHE_INIT_FILENAME
Definition: relcache.h:25
Oid RelFileNumber
Definition: relpath.h:25
ForkNumber
Definition: relpath.h:56
@ InvalidForkNumber
Definition: relpath.h:57
@ INIT_FORKNUM
Definition: relpath.h:61
#define InvalidRelFileNumber
Definition: relpath.h:26
#define PG_TBLSPC_DIR
Definition: relpath.h:41
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:33
#define RelFileNumberIsValid(relnumber)
Definition: relpath.h:27
void ReleaseAuxProcessResources(bool isCommit)
Definition: resowner.c:1002
ResourceOwner CurrentResourceOwner
Definition: resowner.c:165
ResourceOwner AuxProcessResourceOwner
Definition: resowner.c:168
static pg_noinline void Size size
Definition: slab.c:607
#define PG_REPLSLOT_DIR
Definition: slot.h:21
void destroyStringInfo(StringInfo str)
Definition: stringinfo.c:358
StringInfo makeStringInfo(void)
Definition: stringinfo.c:38
TimeLineID starttli
Definition: xlogbackup.h:27
XLogRecPtr startpoint
Definition: xlogbackup.h:26
TimeLineID stoptli
Definition: xlogbackup.h:36
XLogRecPtr stoppoint
Definition: xlogbackup.h:35
List * options
Definition: replnodes.h:44
Definition: dirent.c:26
char * defname
Definition: parsenodes.h:817
Definition: pg_list.h:54
uint16 pd_checksum
Definition: bufpage.h:163
pg_compress_specification compression_specification
Definition: basebackup.c:77
pg_checksum_type manifest_checksum_type
Definition: basebackup.c:78
const char * label
Definition: basebackup.c:64
pg_compress_algorithm compression
Definition: basebackup.c:76
backup_manifest_option manifest
Definition: basebackup.c:75
BaseBackupTargetHandle * target_handle
Definition: basebackup.c:74
XLogRecPtr startptr
bbsink_state * bbs_state
char * bbs_buffer
size_t bbs_buffer_length
Definition: dirent.h:10
char d_name[MAX_PATH]
Definition: dirent.h:15
const char * name
Definition: basebackup.c:139
pg_compress_algorithm algorithm
Definition: compression.h:34
__time64_t st_mtime
Definition: win32_port.h:275
__int64 st_size
Definition: win32_port.h:273
short st_gid
Definition: win32_port.h:271
unsigned short st_mode
Definition: win32_port.h:268
short st_uid
Definition: win32_port.h:270
Definition: regguts.h:323
char * rpath
Definition: basebackup.h:32
#define LOG_METAINFO_DATAFILE_TMP
Definition: syslogger.h:103
static void pgstat_report_wait_start(uint32 wait_event_info)
Definition: wait_event.h:85
static void pgstat_report_wait_end(void)
Definition: wait_event.h:101
const char * name
void WalSndSetState(WalSndState state)
Definition: walsender.c:3782
@ WALSNDSTATE_BACKUP
bool summarize_wal
#define lstat(path, sb)
Definition: win32_port.h:285
#define S_ISDIR(m)
Definition: win32_port.h:325
#define S_ISLNK(m)
Definition: win32_port.h:344
#define fstat
Definition: win32_port.h:283
#define readlink(path, buf, size)
Definition: win32_port.h:236
#define S_ISREG(m)
Definition: win32_port.h:328
bool RecoveryInProgress(void)
Definition: xlog.c:6334
void do_pg_abort_backup(int code, Datum arg)
Definition: xlog.c:9394
SessionBackupState get_backup_status(void)
Definition: xlog.c:9101
void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)
Definition: xlog.c:3727
int wal_segment_size
Definition: xlog.c:143
bool DataChecksumsEnabled(void)
Definition: xlog.c:4588
void do_pg_backup_start(const char *backupidstr, bool fast, List **tablespaces, BackupState *state, StringInfo tblspcmapfile)
Definition: xlog.c:8792
void do_pg_backup_stop(BackupState *state, bool waitforarchive)
Definition: xlog.c:9120
#define TABLESPACE_MAP
Definition: xlog.h:305
#define BACKUP_LABEL_FILE
Definition: xlog.h:302
SessionBackupState
Definition: xlog.h:286
@ SESSION_BACKUP_RUNNING
Definition: xlog.h:288
#define XLOG_CONTROL_FILE
static bool IsXLogFileName(const char *fname)
static void XLogFromFileName(const char *fname, TimeLineID *tli, XLogSegNo *logSegNo, int wal_segsz_bytes)
#define XLByteToPrevSeg(xlrp, logSegNo, wal_segsz_bytes)
static bool IsTLHistoryFileName(const char *fname)
#define MAXFNAMELEN
#define XLOGDIR
static void StatusFilePath(char *path, const char *xlog, const char *suffix)
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)
static void XLogFileName(char *fname, TimeLineID tli, XLogSegNo logSegNo, int wal_segsz_bytes)
char * build_backup_content(BackupState *state, bool ishistoryfile)
Definition: xlogbackup.c:29
uint64 XLogRecPtr
Definition: xlogdefs.h:21
uint32 TimeLineID
Definition: xlogdefs.h:59
uint64 XLogSegNo
Definition: xlogdefs.h:48
static BackupState * backup_state
Definition: xlogfuncs.c:41
static StringInfo tablespace_map
Definition: xlogfuncs.c:42