PostgreSQL Source Code  git master
basebackup.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * basebackup.c
4  * code for taking a base backup and streaming it to a standby
5  *
6  * Portions Copyright (c) 2010-2024, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * src/backend/backup/basebackup.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 #include "postgres.h"
14 
15 #include <sys/stat.h>
16 #include <unistd.h>
17 #include <time.h>
18 
19 #include "access/xlog_internal.h"
20 #include "access/xlogbackup.h"
21 #include "backup/backup_manifest.h"
22 #include "backup/basebackup.h"
24 #include "backup/basebackup_sink.h"
26 #include "catalog/pg_tablespace_d.h"
27 #include "commands/defrem.h"
28 #include "common/compression.h"
29 #include "common/file_perm.h"
30 #include "common/file_utils.h"
31 #include "lib/stringinfo.h"
32 #include "miscadmin.h"
33 #include "nodes/pg_list.h"
34 #include "pgstat.h"
35 #include "pgtar.h"
36 #include "port.h"
37 #include "postmaster/syslogger.h"
39 #include "replication/walsender.h"
41 #include "storage/bufpage.h"
42 #include "storage/checksum.h"
43 #include "storage/dsm_impl.h"
44 #include "storage/ipc.h"
45 #include "storage/reinit.h"
46 #include "utils/builtins.h"
47 #include "utils/guc.h"
48 #include "utils/ps_status.h"
49 #include "utils/relcache.h"
50 #include "utils/resowner.h"
51 
52 /*
53  * How much data do we want to send in one CopyData message? Note that
54  * this may also result in reading the underlying files in chunks of this
55  * size.
56  *
57  * NB: The buffer size is required to be a multiple of the system block
58  * size, so use that value instead if it's bigger than our preference.
59  */
60 #define SINK_BUFFER_LENGTH Max(32768, BLCKSZ)
61 
62 typedef struct
63 {
64  const char *label;
65  bool progress;
67  bool nowait;
68  bool includewal;
80 
81 static int64 sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly,
84 static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
85  List *tablespaces, bool sendtblspclinks,
88 static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
89  struct stat *statbuf, bool missing_ok,
90  Oid dboid, Oid spcoid, RelFileNumber relfilenumber,
91  unsigned segno,
93  unsigned num_incremental_blocks,
94  BlockNumber *incremental_blocks,
95  unsigned truncation_block_length);
96 static off_t read_file_data_into_buffer(bbsink *sink,
97  const char *readfilename, int fd,
98  off_t offset, size_t length,
99  BlockNumber blkno,
100  bool verify_checksum,
101  int *checksum_failures);
102 static void push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx,
103  size_t *bytes_done, void *data, size_t length);
104 static bool verify_page_checksum(Page page, XLogRecPtr start_lsn,
105  BlockNumber blkno,
106  uint16 *expected_checksum);
107 static void sendFileWithContent(bbsink *sink, const char *filename,
108  const char *content, int len,
110 static int64 _tarWriteHeader(bbsink *sink, const char *filename,
111  const char *linktarget, struct stat *statbuf,
112  bool sizeonly);
113 static void _tarWritePadding(bbsink *sink, int len);
114 static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf);
115 static void perform_base_backup(basebackup_options *opt, bbsink *sink,
118 static int compareWalFileNames(const ListCell *a, const ListCell *b);
119 static ssize_t basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
120  const char *filename, bool partial_read_ok);
121 
122 /* Was the backup currently in-progress initiated in recovery mode? */
123 static bool backup_started_in_recovery = false;
124 
125 /* Total number of checksum failures during base backup. */
126 static long long int total_checksum_failures;
127 
128 /* Do not verify checksums. */
129 static bool noverify_checksums = false;
130 
131 /*
132  * Definition of one element part of an exclusion list, used for paths part
133  * of checksum validation or base backups. "name" is the name of the file
134  * or path to check for exclusion. If "match_prefix" is true, any items
135  * matching the name as prefix are excluded.
136  */
138 {
139  const char *name;
141 };
142 
143 /*
144  * The contents of these directories are removed or recreated during server
145  * start so they are not included in backups. The directories themselves are
146  * kept and included as empty to preserve access permissions.
147  *
148  * Note: this list should be kept in sync with the filter lists in pg_rewind's
149  * filemap.c.
150  */
151 static const char *const excludeDirContents[] =
152 {
153  /*
154  * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped
155  * because extensions like pg_stat_statements store data there.
156  */
158 
159  /*
160  * It is generally not useful to backup the contents of this directory
161  * even if the intention is to restore to another primary. See backup.sgml
162  * for a more detailed description.
163  */
164  "pg_replslot",
165 
166  /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
168 
169  /* Contents removed on startup, see AsyncShmemInit(). */
170  "pg_notify",
171 
172  /*
173  * Old contents are loaded for possible debugging but are not required for
174  * normal operation, see SerialInit().
175  */
176  "pg_serial",
177 
178  /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
179  "pg_snapshots",
180 
181  /* Contents zeroed on startup, see StartupSUBTRANS(). */
182  "pg_subtrans",
183 
184  /* end of list */
185  NULL
186 };
187 
188 /*
189  * List of files excluded from backups.
190  */
191 static const struct exclude_list_item excludeFiles[] =
192 {
193  /* Skip auto conf temporary file. */
194  {PG_AUTOCONF_FILENAME ".tmp", false},
195 
196  /* Skip current log file temporary file */
197  {LOG_METAINFO_DATAFILE_TMP, false},
198 
199  /*
200  * Skip relation cache because it is rebuilt on startup. This includes
201  * temporary files.
202  */
203  {RELCACHE_INIT_FILENAME, true},
204 
205  /*
206  * backup_label and tablespace_map should not exist in a running cluster
207  * capable of doing an online backup, but exclude them just in case.
208  */
209  {BACKUP_LABEL_FILE, false},
210  {TABLESPACE_MAP, false},
211 
212  /*
213  * If there's a backup_manifest, it belongs to a backup that was used to
214  * start this server. It is *not* correct for this backup. Our
215  * backup_manifest is injected into the backup separately if users want
216  * it.
217  */
218  {"backup_manifest", false},
219 
220  {"postmaster.pid", false},
221  {"postmaster.opts", false},
222 
223  /* end of list */
224  {NULL, false}
225 };
226 
227 /*
228  * Actually do a base backup for the specified tablespaces.
229  *
230  * This is split out mainly to avoid complaints about "variable might be
231  * clobbered by longjmp" from stupider versions of gcc.
232  */
233 static void
236 {
238  XLogRecPtr endptr;
239  TimeLineID endtli;
243 
244  /* Initial backup state, insofar as we know it now. */
245  state.tablespaces = NIL;
246  state.tablespace_num = 0;
247  state.bytes_done = 0;
248  state.bytes_total = 0;
249  state.bytes_total_is_valid = false;
250 
251  /* we're going to use a BufFile, so we need a ResourceOwner */
252  Assert(CurrentResourceOwner == NULL);
253  CurrentResourceOwner = ResourceOwnerCreate(NULL, "base backup");
254 
256 
259 
261 
262  /* Allocate backup related variables. */
265 
267  do_pg_backup_start(opt->label, opt->fastcheckpoint, &state.tablespaces,
269 
270  state.startptr = backup_state->startpoint;
271  state.starttli = backup_state->starttli;
272 
273  /*
274  * Once do_pg_backup_start has been called, ensure that any failure causes
275  * us to abort the backup so we don't "leak" a backup counter. For this
276  * reason, *all* functionality between do_pg_backup_start() and the end of
277  * do_pg_backup_stop() should be inside the error cleanup block!
278  */
279 
281  {
282  ListCell *lc;
283  tablespaceinfo *newti;
284 
285  /* If this is an incremental backup, execute preparatory steps. */
286  if (ib != NULL)
288 
289  /* Add a node for the base directory at the end */
290  newti = palloc0(sizeof(tablespaceinfo));
291  newti->size = -1;
292  state.tablespaces = lappend(state.tablespaces, newti);
293 
294  /*
295  * Calculate the total backup size by summing up the size of each
296  * tablespace
297  */
298  if (opt->progress)
299  {
301 
302  foreach(lc, state.tablespaces)
303  {
304  tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
305 
306  if (tmp->path == NULL)
307  tmp->size = sendDir(sink, ".", 1, true, state.tablespaces,
308  true, NULL, InvalidOid, NULL);
309  else
310  tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true,
311  NULL, NULL);
312  state.bytes_total += tmp->size;
313  }
314  state.bytes_total_is_valid = true;
315  }
316 
317  /* notify basebackup sink about start of backup */
319 
320  /* Send off our tablespaces one by one */
321  foreach(lc, state.tablespaces)
322  {
323  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
324 
325  if (ti->path == NULL)
326  {
327  struct stat statbuf;
328  bool sendtblspclinks = true;
329  char *backup_label;
330 
331  bbsink_begin_archive(sink, "base.tar");
332 
333  /* In the main tar, include the backup_label first... */
334  backup_label = build_backup_content(backup_state, false);
336  backup_label, -1, &manifest);
337  pfree(backup_label);
338 
339  /* Then the tablespace_map file, if required... */
340  if (opt->sendtblspcmapfile)
341  {
343  tablespace_map->data, -1, &manifest);
344  sendtblspclinks = false;
345  }
346 
347  /* Then the bulk of the files... */
348  sendDir(sink, ".", 1, false, state.tablespaces,
349  sendtblspclinks, &manifest, InvalidOid, ib);
350 
351  /* ... and pg_control after everything else. */
352  if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
353  ereport(ERROR,
355  errmsg("could not stat file \"%s\": %m",
358  false, InvalidOid, InvalidOid,
359  InvalidRelFileNumber, 0, &manifest, 0, NULL, 0);
360  }
361  else
362  {
363  char *archive_name = psprintf("%u.tar", ti->oid);
364 
365  bbsink_begin_archive(sink, archive_name);
366 
367  sendTablespace(sink, ti->path, ti->oid, false, &manifest, ib);
368  }
369 
370  /*
371  * If we're including WAL, and this is the main data directory we
372  * don't treat this as the end of the tablespace. Instead, we will
373  * include the xlog files below and stop afterwards. This is safe
374  * since the main data directory is always sent *last*.
375  */
376  if (opt->includewal && ti->path == NULL)
377  {
378  Assert(lnext(state.tablespaces, lc) == NULL);
379  }
380  else
381  {
382  /* Properly terminate the tarfile. */
383  StaticAssertDecl(2 * TAR_BLOCK_SIZE <= BLCKSZ,
384  "BLCKSZ too small for 2 tar blocks");
385  memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
387 
388  /* OK, that's the end of the archive. */
389  bbsink_end_archive(sink);
390  }
391  }
392 
395 
396  endptr = backup_state->stoppoint;
397  endtli = backup_state->stoptli;
398 
399  /* Deallocate backup-related variables. */
402  }
404 
405 
406  if (opt->includewal)
407  {
408  /*
409  * We've left the last tar file "open", so we can now append the
410  * required WAL files to it.
411  */
412  char pathbuf[MAXPGPATH];
413  XLogSegNo segno;
414  XLogSegNo startsegno;
415  XLogSegNo endsegno;
416  struct stat statbuf;
417  List *historyFileList = NIL;
418  List *walFileList = NIL;
419  char firstoff[MAXFNAMELEN];
420  char lastoff[MAXFNAMELEN];
421  DIR *dir;
422  struct dirent *de;
423  ListCell *lc;
424  TimeLineID tli;
425 
427 
428  /*
429  * I'd rather not worry about timelines here, so scan pg_wal and
430  * include all WAL files in the range between 'startptr' and 'endptr',
431  * regardless of the timeline the file is stamped with. If there are
432  * some spurious WAL files belonging to timelines that don't belong in
433  * this server's history, they will be included too. Normally there
434  * shouldn't be such files, but if there are, there's little harm in
435  * including them.
436  */
437  XLByteToSeg(state.startptr, startsegno, wal_segment_size);
438  XLogFileName(firstoff, state.starttli, startsegno, wal_segment_size);
439  XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
440  XLogFileName(lastoff, endtli, endsegno, wal_segment_size);
441 
442  dir = AllocateDir("pg_wal");
443  while ((de = ReadDir(dir, "pg_wal")) != NULL)
444  {
445  /* Does it look like a WAL segment, and is it in the range? */
446  if (IsXLogFileName(de->d_name) &&
447  strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
448  strcmp(de->d_name + 8, lastoff + 8) <= 0)
449  {
450  walFileList = lappend(walFileList, pstrdup(de->d_name));
451  }
452  /* Does it look like a timeline history file? */
453  else if (IsTLHistoryFileName(de->d_name))
454  {
455  historyFileList = lappend(historyFileList, pstrdup(de->d_name));
456  }
457  }
458  FreeDir(dir);
459 
460  /*
461  * Before we go any further, check that none of the WAL segments we
462  * need were removed.
463  */
464  CheckXLogRemoved(startsegno, state.starttli);
465 
466  /*
467  * Sort the WAL filenames. We want to send the files in order from
468  * oldest to newest, to reduce the chance that a file is recycled
469  * before we get a chance to send it over.
470  */
471  list_sort(walFileList, compareWalFileNames);
472 
473  /*
474  * There must be at least one xlog file in the pg_wal directory, since
475  * we are doing backup-including-xlog.
476  */
477  if (walFileList == NIL)
478  ereport(ERROR,
479  (errmsg("could not find any WAL files")));
480 
481  /*
482  * Sanity check: the first and last segment should cover startptr and
483  * endptr, with no gaps in between.
484  */
485  XLogFromFileName((char *) linitial(walFileList),
486  &tli, &segno, wal_segment_size);
487  if (segno != startsegno)
488  {
489  char startfname[MAXFNAMELEN];
490 
491  XLogFileName(startfname, state.starttli, startsegno,
493  ereport(ERROR,
494  (errmsg("could not find WAL file \"%s\"", startfname)));
495  }
496  foreach(lc, walFileList)
497  {
498  char *walFileName = (char *) lfirst(lc);
499  XLogSegNo currsegno = segno;
500  XLogSegNo nextsegno = segno + 1;
501 
502  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
503  if (!(nextsegno == segno || currsegno == segno))
504  {
505  char nextfname[MAXFNAMELEN];
506 
507  XLogFileName(nextfname, tli, nextsegno, wal_segment_size);
508  ereport(ERROR,
509  (errmsg("could not find WAL file \"%s\"", nextfname)));
510  }
511  }
512  if (segno != endsegno)
513  {
514  char endfname[MAXFNAMELEN];
515 
516  XLogFileName(endfname, endtli, endsegno, wal_segment_size);
517  ereport(ERROR,
518  (errmsg("could not find WAL file \"%s\"", endfname)));
519  }
520 
521  /* Ok, we have everything we need. Send the WAL files. */
522  foreach(lc, walFileList)
523  {
524  char *walFileName = (char *) lfirst(lc);
525  int fd;
526  ssize_t cnt;
527  pgoff_t len = 0;
528 
529  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
530  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
531 
532  fd = OpenTransientFile(pathbuf, O_RDONLY | PG_BINARY);
533  if (fd < 0)
534  {
535  int save_errno = errno;
536 
537  /*
538  * Most likely reason for this is that the file was already
539  * removed by a checkpoint, so check for that to get a better
540  * error message.
541  */
542  CheckXLogRemoved(segno, tli);
543 
544  errno = save_errno;
545  ereport(ERROR,
547  errmsg("could not open file \"%s\": %m", pathbuf)));
548  }
549 
550  if (fstat(fd, &statbuf) != 0)
551  ereport(ERROR,
553  errmsg("could not stat file \"%s\": %m",
554  pathbuf)));
555  if (statbuf.st_size != wal_segment_size)
556  {
557  CheckXLogRemoved(segno, tli);
558  ereport(ERROR,
560  errmsg("unexpected WAL file size \"%s\"", walFileName)));
561  }
562 
563  /* send the WAL file itself */
564  _tarWriteHeader(sink, pathbuf, NULL, &statbuf, false);
565 
566  while ((cnt = basebackup_read_file(fd, sink->bbs_buffer,
567  Min(sink->bbs_buffer_length,
569  len, pathbuf, true)) > 0)
570  {
571  CheckXLogRemoved(segno, tli);
572  bbsink_archive_contents(sink, cnt);
573 
574  len += cnt;
575 
576  if (len == wal_segment_size)
577  break;
578  }
579 
580  if (len != wal_segment_size)
581  {
582  CheckXLogRemoved(segno, tli);
583  ereport(ERROR,
585  errmsg("unexpected WAL file size \"%s\"", walFileName)));
586  }
587 
588  /*
589  * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need
590  * for padding.
591  */
593 
595 
596  /*
597  * Mark file as archived, otherwise files can get archived again
598  * after promotion of a new node. This is in line with
599  * walreceiver.c always doing an XLogArchiveForceDone() after a
600  * complete segment.
601  */
602  StatusFilePath(pathbuf, walFileName, ".done");
603  sendFileWithContent(sink, pathbuf, "", -1, &manifest);
604  }
605 
606  /*
607  * Send timeline history files too. Only the latest timeline history
608  * file is required for recovery, and even that only if there happens
609  * to be a timeline switch in the first WAL segment that contains the
610  * checkpoint record, or if we're taking a base backup from a standby
611  * server and the target timeline changes while the backup is taken.
612  * But they are small and highly useful for debugging purposes, so
613  * better include them all, always.
614  */
615  foreach(lc, historyFileList)
616  {
617  char *fname = lfirst(lc);
618 
619  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
620 
621  if (lstat(pathbuf, &statbuf) != 0)
622  ereport(ERROR,
624  errmsg("could not stat file \"%s\": %m", pathbuf)));
625 
626  sendFile(sink, pathbuf, pathbuf, &statbuf, false,
628  &manifest, 0, NULL, 0);
629 
630  /* unconditionally mark file as archived */
631  StatusFilePath(pathbuf, fname, ".done");
632  sendFileWithContent(sink, pathbuf, "", -1, &manifest);
633  }
634 
635  /* Properly terminate the tar file. */
636  StaticAssertStmt(2 * TAR_BLOCK_SIZE <= BLCKSZ,
637  "BLCKSZ too small for 2 tar blocks");
638  memset(sink->bbs_buffer, 0, 2 * TAR_BLOCK_SIZE);
640 
641  /* OK, that's the end of the archive. */
642  bbsink_end_archive(sink);
643  }
644 
645  AddWALInfoToBackupManifest(&manifest, state.startptr, state.starttli,
646  endptr, endtli);
647 
649 
650  bbsink_end_backup(sink, endptr, endtli);
651 
653  {
654  if (total_checksum_failures > 1)
656  (errmsg_plural("%lld total checksum verification failure",
657  "%lld total checksum verification failures",
660 
661  ereport(ERROR,
663  errmsg("checksum verification failure during base backup")));
664  }
665 
666  /*
667  * Make sure to free the manifest before the resource owners as manifests
668  * use cryptohash contexts that may depend on resource owners (like
669  * OpenSSL).
670  */
672 
673  /* clean up the resource owner we created */
674  WalSndResourceCleanup(true);
675 
677 }
678 
679 /*
680  * list_sort comparison function, to compare log/seg portion of WAL segment
681  * filenames, ignoring the timeline portion.
682  */
683 static int
685 {
686  char *fna = (char *) lfirst(a);
687  char *fnb = (char *) lfirst(b);
688 
689  return strcmp(fna + 8, fnb + 8);
690 }
691 
692 /*
693  * Parse the base backup options passed down by the parser
694  */
695 static void
697 {
698  ListCell *lopt;
699  bool o_label = false;
700  bool o_progress = false;
701  bool o_checkpoint = false;
702  bool o_nowait = false;
703  bool o_wal = false;
704  bool o_incremental = false;
705  bool o_maxrate = false;
706  bool o_tablespace_map = false;
707  bool o_noverify_checksums = false;
708  bool o_manifest = false;
709  bool o_manifest_checksums = false;
710  bool o_target = false;
711  bool o_target_detail = false;
712  char *target_str = NULL;
713  char *target_detail_str = NULL;
714  bool o_compression = false;
715  bool o_compression_detail = false;
716  char *compression_detail_str = NULL;
717 
718  MemSet(opt, 0, sizeof(*opt));
723 
724  foreach(lopt, options)
725  {
726  DefElem *defel = (DefElem *) lfirst(lopt);
727 
728  if (strcmp(defel->defname, "label") == 0)
729  {
730  if (o_label)
731  ereport(ERROR,
732  (errcode(ERRCODE_SYNTAX_ERROR),
733  errmsg("duplicate option \"%s\"", defel->defname)));
734  opt->label = defGetString(defel);
735  o_label = true;
736  }
737  else if (strcmp(defel->defname, "progress") == 0)
738  {
739  if (o_progress)
740  ereport(ERROR,
741  (errcode(ERRCODE_SYNTAX_ERROR),
742  errmsg("duplicate option \"%s\"", defel->defname)));
743  opt->progress = defGetBoolean(defel);
744  o_progress = true;
745  }
746  else if (strcmp(defel->defname, "checkpoint") == 0)
747  {
748  char *optval = defGetString(defel);
749 
750  if (o_checkpoint)
751  ereport(ERROR,
752  (errcode(ERRCODE_SYNTAX_ERROR),
753  errmsg("duplicate option \"%s\"", defel->defname)));
754  if (pg_strcasecmp(optval, "fast") == 0)
755  opt->fastcheckpoint = true;
756  else if (pg_strcasecmp(optval, "spread") == 0)
757  opt->fastcheckpoint = false;
758  else
759  ereport(ERROR,
760  (errcode(ERRCODE_SYNTAX_ERROR),
761  errmsg("unrecognized checkpoint type: \"%s\"",
762  optval)));
763  o_checkpoint = true;
764  }
765  else if (strcmp(defel->defname, "wait") == 0)
766  {
767  if (o_nowait)
768  ereport(ERROR,
769  (errcode(ERRCODE_SYNTAX_ERROR),
770  errmsg("duplicate option \"%s\"", defel->defname)));
771  opt->nowait = !defGetBoolean(defel);
772  o_nowait = true;
773  }
774  else if (strcmp(defel->defname, "wal") == 0)
775  {
776  if (o_wal)
777  ereport(ERROR,
778  (errcode(ERRCODE_SYNTAX_ERROR),
779  errmsg("duplicate option \"%s\"", defel->defname)));
780  opt->includewal = defGetBoolean(defel);
781  o_wal = true;
782  }
783  else if (strcmp(defel->defname, "incremental") == 0)
784  {
785  if (o_incremental)
786  ereport(ERROR,
787  (errcode(ERRCODE_SYNTAX_ERROR),
788  errmsg("duplicate option \"%s\"", defel->defname)));
789  opt->incremental = defGetBoolean(defel);
790  if (opt->incremental && !summarize_wal)
791  ereport(ERROR,
792  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
793  errmsg("incremental backups cannot be taken unless WAL summarization is enabled")));
794  o_incremental = true;
795  }
796  else if (strcmp(defel->defname, "max_rate") == 0)
797  {
798  int64 maxrate;
799 
800  if (o_maxrate)
801  ereport(ERROR,
802  (errcode(ERRCODE_SYNTAX_ERROR),
803  errmsg("duplicate option \"%s\"", defel->defname)));
804 
805  maxrate = defGetInt64(defel);
806  if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
807  ereport(ERROR,
808  (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
809  errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
810  (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
811 
812  opt->maxrate = (uint32) maxrate;
813  o_maxrate = true;
814  }
815  else if (strcmp(defel->defname, "tablespace_map") == 0)
816  {
817  if (o_tablespace_map)
818  ereport(ERROR,
819  (errcode(ERRCODE_SYNTAX_ERROR),
820  errmsg("duplicate option \"%s\"", defel->defname)));
821  opt->sendtblspcmapfile = defGetBoolean(defel);
822  o_tablespace_map = true;
823  }
824  else if (strcmp(defel->defname, "verify_checksums") == 0)
825  {
826  if (o_noverify_checksums)
827  ereport(ERROR,
828  (errcode(ERRCODE_SYNTAX_ERROR),
829  errmsg("duplicate option \"%s\"", defel->defname)));
831  o_noverify_checksums = true;
832  }
833  else if (strcmp(defel->defname, "manifest") == 0)
834  {
835  char *optval = defGetString(defel);
836  bool manifest_bool;
837 
838  if (o_manifest)
839  ereport(ERROR,
840  (errcode(ERRCODE_SYNTAX_ERROR),
841  errmsg("duplicate option \"%s\"", defel->defname)));
842  if (parse_bool(optval, &manifest_bool))
843  {
844  if (manifest_bool)
846  else
848  }
849  else if (pg_strcasecmp(optval, "force-encode") == 0)
851  else
852  ereport(ERROR,
853  (errcode(ERRCODE_SYNTAX_ERROR),
854  errmsg("unrecognized manifest option: \"%s\"",
855  optval)));
856  o_manifest = true;
857  }
858  else if (strcmp(defel->defname, "manifest_checksums") == 0)
859  {
860  char *optval = defGetString(defel);
861 
862  if (o_manifest_checksums)
863  ereport(ERROR,
864  (errcode(ERRCODE_SYNTAX_ERROR),
865  errmsg("duplicate option \"%s\"", defel->defname)));
866  if (!pg_checksum_parse_type(optval,
867  &opt->manifest_checksum_type))
868  ereport(ERROR,
869  (errcode(ERRCODE_SYNTAX_ERROR),
870  errmsg("unrecognized checksum algorithm: \"%s\"",
871  optval)));
872  o_manifest_checksums = true;
873  }
874  else if (strcmp(defel->defname, "target") == 0)
875  {
876  if (o_target)
877  ereport(ERROR,
878  (errcode(ERRCODE_SYNTAX_ERROR),
879  errmsg("duplicate option \"%s\"", defel->defname)));
880  target_str = defGetString(defel);
881  o_target = true;
882  }
883  else if (strcmp(defel->defname, "target_detail") == 0)
884  {
885  char *optval = defGetString(defel);
886 
887  if (o_target_detail)
888  ereport(ERROR,
889  (errcode(ERRCODE_SYNTAX_ERROR),
890  errmsg("duplicate option \"%s\"", defel->defname)));
891  target_detail_str = optval;
892  o_target_detail = true;
893  }
894  else if (strcmp(defel->defname, "compression") == 0)
895  {
896  char *optval = defGetString(defel);
897 
898  if (o_compression)
899  ereport(ERROR,
900  (errcode(ERRCODE_SYNTAX_ERROR),
901  errmsg("duplicate option \"%s\"", defel->defname)));
902  if (!parse_compress_algorithm(optval, &opt->compression))
903  ereport(ERROR,
904  (errcode(ERRCODE_SYNTAX_ERROR),
905  errmsg("unrecognized compression algorithm: \"%s\"",
906  optval)));
907  o_compression = true;
908  }
909  else if (strcmp(defel->defname, "compression_detail") == 0)
910  {
911  if (o_compression_detail)
912  ereport(ERROR,
913  (errcode(ERRCODE_SYNTAX_ERROR),
914  errmsg("duplicate option \"%s\"", defel->defname)));
915  compression_detail_str = defGetString(defel);
916  o_compression_detail = true;
917  }
918  else
919  ereport(ERROR,
920  (errcode(ERRCODE_SYNTAX_ERROR),
921  errmsg("unrecognized base backup option: \"%s\"",
922  defel->defname)));
923  }
924 
925  if (opt->label == NULL)
926  opt->label = "base backup";
927  if (opt->manifest == MANIFEST_OPTION_NO)
928  {
929  if (o_manifest_checksums)
930  ereport(ERROR,
931  (errcode(ERRCODE_SYNTAX_ERROR),
932  errmsg("manifest checksums require a backup manifest")));
934  }
935 
936  if (target_str == NULL)
937  {
938  if (target_detail_str != NULL)
939  ereport(ERROR,
940  (errcode(ERRCODE_SYNTAX_ERROR),
941  errmsg("target detail cannot be used without target")));
942  opt->use_copytblspc = true;
943  opt->send_to_client = true;
944  }
945  else if (strcmp(target_str, "client") == 0)
946  {
947  if (target_detail_str != NULL)
948  ereport(ERROR,
949  (errcode(ERRCODE_SYNTAX_ERROR),
950  errmsg("target \"%s\" does not accept a target detail",
951  target_str)));
952  opt->send_to_client = true;
953  }
954  else
955  opt->target_handle =
956  BaseBackupGetTargetHandle(target_str, target_detail_str);
957 
958  if (o_compression_detail && !o_compression)
959  ereport(ERROR,
960  (errcode(ERRCODE_SYNTAX_ERROR),
961  errmsg("compression detail cannot be specified unless compression is enabled")));
962 
963  if (o_compression)
964  {
965  char *error_detail;
966 
967  parse_compress_specification(opt->compression, compression_detail_str,
969  error_detail =
971  if (error_detail != NULL)
972  ereport(ERROR,
973  errcode(ERRCODE_SYNTAX_ERROR),
974  errmsg("invalid compression specification: %s",
975  error_detail));
976  }
977 }
978 
979 
980 /*
981  * SendBaseBackup() - send a complete base backup.
982  *
983  * The function will put the system into backup mode like pg_backup_start()
984  * does, so that the backup is consistent even though we read directly from
985  * the filesystem, bypassing the buffer cache.
986  */
987 void
989 {
990  basebackup_options opt;
991  bbsink *sink;
993 
994  if (status == SESSION_BACKUP_RUNNING)
995  ereport(ERROR,
996  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
997  errmsg("a backup is already in progress in this session")));
998 
999  parse_basebackup_options(cmd->options, &opt);
1000 
1002 
1004  {
1005  char activitymsg[50];
1006 
1007  snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
1008  opt.label);
1009  set_ps_display(activitymsg);
1010  }
1011 
1012  /*
1013  * If we're asked to perform an incremental backup and the user has not
1014  * supplied a manifest, that's an ERROR.
1015  *
1016  * If we're asked to perform a full backup and the user did supply a
1017  * manifest, just ignore it.
1018  */
1019  if (!opt.incremental)
1020  ib = NULL;
1021  else if (ib == NULL)
1022  ereport(ERROR,
1023  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1024  errmsg("must UPLOAD_MANIFEST before performing an incremental BASE_BACKUP")));
1025 
1026  /*
1027  * If the target is specifically 'client' then set up to stream the backup
1028  * to the client; otherwise, it's being sent someplace else and should not
1029  * be sent to the client. BaseBackupGetSink has the job of setting up a
1030  * sink to send the backup data wherever it needs to go.
1031  */
1033  if (opt.target_handle != NULL)
1034  sink = BaseBackupGetSink(opt.target_handle, sink);
1035 
1036  /* Set up network throttling, if client requested it */
1037  if (opt.maxrate > 0)
1038  sink = bbsink_throttle_new(sink, opt.maxrate);
1039 
1040  /* Set up server-side compression, if client requested it */
1041  if (opt.compression == PG_COMPRESSION_GZIP)
1042  sink = bbsink_gzip_new(sink, &opt.compression_specification);
1043  else if (opt.compression == PG_COMPRESSION_LZ4)
1044  sink = bbsink_lz4_new(sink, &opt.compression_specification);
1045  else if (opt.compression == PG_COMPRESSION_ZSTD)
1046  sink = bbsink_zstd_new(sink, &opt.compression_specification);
1047 
1048  /* Set up progress reporting. */
1049  sink = bbsink_progress_new(sink, opt.progress);
1050 
1051  /*
1052  * Perform the base backup, but make sure we clean up the bbsink even if
1053  * an error occurs.
1054  */
1055  PG_TRY();
1056  {
1057  perform_base_backup(&opt, sink, ib);
1058  }
1059  PG_FINALLY();
1060  {
1061  bbsink_cleanup(sink);
1062  }
1063  PG_END_TRY();
1064 }
1065 
1066 /*
1067  * Inject a file with given name and content in the output tar stream.
1068  *
1069  * "len" can optionally be set to an arbitrary length of data sent. If set
1070  * to -1, the content sent is treated as a string with strlen() as length.
1071  */
1072 static void
1073 sendFileWithContent(bbsink *sink, const char *filename, const char *content,
1075 {
1076  struct stat statbuf;
1077  int bytes_done = 0;
1078  pg_checksum_context checksum_ctx;
1079 
1080  if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1081  elog(ERROR, "could not initialize checksum of file \"%s\"",
1082  filename);
1083 
1084  if (len < 0)
1085  len = strlen(content);
1086 
1087  /*
1088  * Construct a stat struct for the file we're injecting in the tar.
1089  */
1090 
1091  /* Windows doesn't have the concept of uid and gid */
1092 #ifdef WIN32
1093  statbuf.st_uid = 0;
1094  statbuf.st_gid = 0;
1095 #else
1096  statbuf.st_uid = geteuid();
1097  statbuf.st_gid = getegid();
1098 #endif
1099  statbuf.st_mtime = time(NULL);
1100  statbuf.st_mode = pg_file_create_mode;
1101  statbuf.st_size = len;
1102 
1103  _tarWriteHeader(sink, filename, NULL, &statbuf, false);
1104 
1105  if (pg_checksum_update(&checksum_ctx, (uint8 *) content, len) < 0)
1106  elog(ERROR, "could not update checksum of file \"%s\"",
1107  filename);
1108 
1109  while (bytes_done < len)
1110  {
1111  size_t remaining = len - bytes_done;
1112  size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1113 
1114  memcpy(sink->bbs_buffer, content, nbytes);
1115  bbsink_archive_contents(sink, nbytes);
1116  bytes_done += nbytes;
1117  content += nbytes;
1118  }
1119 
1120  _tarWritePadding(sink, len);
1121 
1123  (pg_time_t) statbuf.st_mtime, &checksum_ctx);
1124 }
1125 
1126 /*
1127  * Include the tablespace directory pointed to by 'path' in the output tar
1128  * stream. If 'sizeonly' is true, we just calculate a total length and return
1129  * it, without actually sending anything.
1130  *
1131  * Only used to send auxiliary tablespaces, not PGDATA.
1132  */
1133 static int64
1134 sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly,
1136 {
1137  int64 size;
1138  char pathbuf[MAXPGPATH];
1139  struct stat statbuf;
1140 
1141  /*
1142  * 'path' points to the tablespace location, but we only want to include
1143  * the version directory in it that belongs to us.
1144  */
1145  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1147 
1148  /*
1149  * Store a directory entry in the tar file so we get the permissions
1150  * right.
1151  */
1152  if (lstat(pathbuf, &statbuf) != 0)
1153  {
1154  if (errno != ENOENT)
1155  ereport(ERROR,
1157  errmsg("could not stat file or directory \"%s\": %m",
1158  pathbuf)));
1159 
1160  /* If the tablespace went away while scanning, it's no error. */
1161  return 0;
1162  }
1163 
1164  size = _tarWriteHeader(sink, TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1165  sizeonly);
1166 
1167  /* Send all the files in the tablespace version directory */
1168  size += sendDir(sink, pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1169  spcoid, ib);
1170 
1171  return size;
1172 }
1173 
1174 /*
1175  * Include all files from the given directory in the output tar stream. If
1176  * 'sizeonly' is true, we just calculate a total length and return it, without
1177  * actually sending anything.
1178  *
1179  * Omit any directory in the tablespaces list, to avoid backing up
1180  * tablespaces twice when they were created inside PGDATA.
1181  *
1182  * If sendtblspclinks is true, we need to include symlink
1183  * information in the tar file. If not, we can skip that
1184  * as it will be sent separately in the tablespace_map file.
1185  */
1186 static int64
1187 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
1188  List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest,
1189  Oid spcoid, IncrementalBackupInfo *ib)
1190 {
1191  DIR *dir;
1192  struct dirent *de;
1193  char pathbuf[MAXPGPATH * 2];
1194  struct stat statbuf;
1195  int64 size = 0;
1196  const char *lastDir; /* Split last dir from parent path. */
1197  bool isRelationDir = false; /* Does directory contain relations? */
1198  bool isGlobalDir = false;
1199  Oid dboid = InvalidOid;
1200  BlockNumber *relative_block_numbers = NULL;
1201 
1202  /*
1203  * Since this array is relatively large, avoid putting it on the stack.
1204  * But we don't need it at all if this is not an incremental backup.
1205  */
1206  if (ib != NULL)
1207  relative_block_numbers = palloc(sizeof(BlockNumber) * RELSEG_SIZE);
1208 
1209  /*
1210  * Determine if the current path is a database directory that can contain
1211  * relations.
1212  *
1213  * Start by finding the location of the delimiter between the parent path
1214  * and the current path.
1215  */
1216  lastDir = last_dir_separator(path);
1217 
1218  /* Does this path look like a database path (i.e. all digits)? */
1219  if (lastDir != NULL &&
1220  strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1221  {
1222  /* Part of path that contains the parent directory. */
1223  int parentPathLen = lastDir - path;
1224 
1225  /*
1226  * Mark path as a database directory if the parent path is either
1227  * $PGDATA/base or a tablespace version path.
1228  */
1229  if (strncmp(path, "./base", parentPathLen) == 0 ||
1230  (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1231  strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1233  sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1234  {
1235  isRelationDir = true;
1236  dboid = atooid(lastDir + 1);
1237  }
1238  }
1239  else if (strcmp(path, "./global") == 0)
1240  {
1241  isRelationDir = true;
1242  isGlobalDir = true;
1243  }
1244 
1245  dir = AllocateDir(path);
1246  while ((de = ReadDir(dir, path)) != NULL)
1247  {
1248  int excludeIdx;
1249  bool excludeFound;
1250  RelFileNumber relfilenumber = InvalidRelFileNumber;
1251  ForkNumber relForkNum = InvalidForkNumber;
1252  unsigned segno = 0;
1253  bool isRelationFile = false;
1254 
1255  /* Skip special stuff */
1256  if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1257  continue;
1258 
1259  /* Skip temporary files */
1260  if (strncmp(de->d_name,
1262  strlen(PG_TEMP_FILE_PREFIX)) == 0)
1263  continue;
1264 
1265  /* Skip macOS system files */
1266  if (strcmp(de->d_name, ".DS_Store") == 0)
1267  continue;
1268 
1269  /*
1270  * Check if the postmaster has signaled us to exit, and abort with an
1271  * error in that case. The error handler further up will call
1272  * do_pg_abort_backup() for us. Also check that if the backup was
1273  * started while still in recovery, the server wasn't promoted.
1274  * do_pg_backup_stop() will check that too, but it's better to stop
1275  * the backup early than continue to the end and fail there.
1276  */
1279  ereport(ERROR,
1280  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1281  errmsg("the standby was promoted during online backup"),
1282  errhint("This means that the backup being taken is corrupt "
1283  "and should not be used. "
1284  "Try taking another online backup.")));
1285 
1286  /* Scan for files that should be excluded */
1287  excludeFound = false;
1288  for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1289  {
1290  int cmplen = strlen(excludeFiles[excludeIdx].name);
1291 
1292  if (!excludeFiles[excludeIdx].match_prefix)
1293  cmplen++;
1294  if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
1295  {
1296  elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1297  excludeFound = true;
1298  break;
1299  }
1300  }
1301 
1302  if (excludeFound)
1303  continue;
1304 
1305  /*
1306  * If there could be non-temporary relation files in this directory,
1307  * try to parse the filename.
1308  */
1309  if (isRelationDir)
1310  isRelationFile =
1312  &relfilenumber,
1313  &relForkNum, &segno);
1314 
1315  /* Exclude all forks for unlogged tables except the init fork */
1316  if (isRelationFile && relForkNum != INIT_FORKNUM)
1317  {
1318  char initForkFile[MAXPGPATH];
1319 
1320  /*
1321  * If any other type of fork, check if there is an init fork with
1322  * the same RelFileNumber. If so, the file can be excluded.
1323  */
1324  snprintf(initForkFile, sizeof(initForkFile), "%s/%u_init",
1325  path, relfilenumber);
1326 
1327  if (lstat(initForkFile, &statbuf) == 0)
1328  {
1329  elog(DEBUG2,
1330  "unlogged relation file \"%s\" excluded from backup",
1331  de->d_name);
1332 
1333  continue;
1334  }
1335  }
1336 
1337  /* Exclude temporary relations */
1338  if (OidIsValid(dboid) && looks_like_temp_rel_name(de->d_name))
1339  {
1340  elog(DEBUG2,
1341  "temporary relation file \"%s\" excluded from backup",
1342  de->d_name);
1343 
1344  continue;
1345  }
1346 
1347  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1348 
1349  /* Skip pg_control here to back up it last */
1350  if (strcmp(pathbuf, "./global/pg_control") == 0)
1351  continue;
1352 
1353  if (lstat(pathbuf, &statbuf) != 0)
1354  {
1355  if (errno != ENOENT)
1356  ereport(ERROR,
1358  errmsg("could not stat file or directory \"%s\": %m",
1359  pathbuf)));
1360 
1361  /* If the file went away while scanning, it's not an error. */
1362  continue;
1363  }
1364 
1365  /* Scan for directories whose contents should be excluded */
1366  excludeFound = false;
1367  for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1368  {
1369  if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1370  {
1371  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1372  convert_link_to_directory(pathbuf, &statbuf);
1373  size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1374  &statbuf, sizeonly);
1375  excludeFound = true;
1376  break;
1377  }
1378  }
1379 
1380  if (excludeFound)
1381  continue;
1382 
1383  /*
1384  * We can skip pg_wal, the WAL segments need to be fetched from the
1385  * WAL archive anyway. But include it as an empty directory anyway, so
1386  * we get permissions right.
1387  */
1388  if (strcmp(pathbuf, "./pg_wal") == 0)
1389  {
1390  /* If pg_wal is a symlink, write it as a directory anyway */
1391  convert_link_to_directory(pathbuf, &statbuf);
1392  size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1393  &statbuf, sizeonly);
1394 
1395  /*
1396  * Also send archive_status and summaries directories (by
1397  * hackishly reusing statbuf from above ...).
1398  */
1399  size += _tarWriteHeader(sink, "./pg_wal/archive_status", NULL,
1400  &statbuf, sizeonly);
1401  size += _tarWriteHeader(sink, "./pg_wal/summaries", NULL,
1402  &statbuf, sizeonly);
1403 
1404  continue; /* don't recurse into pg_wal */
1405  }
1406 
1407  /* Allow symbolic links in pg_tblspc only */
1408  if (strcmp(path, "./pg_tblspc") == 0 && S_ISLNK(statbuf.st_mode))
1409  {
1410  char linkpath[MAXPGPATH];
1411  int rllen;
1412 
1413  rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1414  if (rllen < 0)
1415  ereport(ERROR,
1417  errmsg("could not read symbolic link \"%s\": %m",
1418  pathbuf)));
1419  if (rllen >= sizeof(linkpath))
1420  ereport(ERROR,
1421  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1422  errmsg("symbolic link \"%s\" target is too long",
1423  pathbuf)));
1424  linkpath[rllen] = '\0';
1425 
1426  size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, linkpath,
1427  &statbuf, sizeonly);
1428  }
1429  else if (S_ISDIR(statbuf.st_mode))
1430  {
1431  bool skip_this_dir = false;
1432  ListCell *lc;
1433 
1434  /*
1435  * Store a directory entry in the tar file so we can get the
1436  * permissions right.
1437  */
1438  size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, &statbuf,
1439  sizeonly);
1440 
1441  /*
1442  * Call ourselves recursively for a directory, unless it happens
1443  * to be a separate tablespace located within PGDATA.
1444  */
1445  foreach(lc, tablespaces)
1446  {
1447  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1448 
1449  /*
1450  * ti->rpath is the tablespace relative path within PGDATA, or
1451  * NULL if the tablespace has been properly located somewhere
1452  * else.
1453  *
1454  * Skip past the leading "./" in pathbuf when comparing.
1455  */
1456  if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1457  {
1458  skip_this_dir = true;
1459  break;
1460  }
1461  }
1462 
1463  /*
1464  * skip sending directories inside pg_tblspc, if not required.
1465  */
1466  if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1467  skip_this_dir = true;
1468 
1469  if (!skip_this_dir)
1470  size += sendDir(sink, pathbuf, basepathlen, sizeonly, tablespaces,
1471  sendtblspclinks, manifest, spcoid, ib);
1472  }
1473  else if (S_ISREG(statbuf.st_mode))
1474  {
1475  bool sent = false;
1476  unsigned num_blocks_required = 0;
1477  unsigned truncation_block_length = 0;
1478  char tarfilenamebuf[MAXPGPATH * 2];
1479  char *tarfilename = pathbuf + basepathlen + 1;
1481 
1482  if (ib != NULL && isRelationFile)
1483  {
1484  Oid relspcoid;
1485  char *lookup_path;
1486 
1487  if (OidIsValid(spcoid))
1488  {
1489  relspcoid = spcoid;
1490  lookup_path = psprintf("pg_tblspc/%u/%s", spcoid,
1491  tarfilename);
1492  }
1493  else
1494  {
1495  if (isGlobalDir)
1496  relspcoid = GLOBALTABLESPACE_OID;
1497  else
1498  relspcoid = DEFAULTTABLESPACE_OID;
1499  lookup_path = pstrdup(tarfilename);
1500  }
1501 
1502  method = GetFileBackupMethod(ib, lookup_path, dboid, relspcoid,
1503  relfilenumber, relForkNum,
1504  segno, statbuf.st_size,
1505  &num_blocks_required,
1506  relative_block_numbers,
1507  &truncation_block_length);
1508  if (method == BACK_UP_FILE_INCREMENTALLY)
1509  {
1510  statbuf.st_size =
1511  GetIncrementalFileSize(num_blocks_required);
1512  snprintf(tarfilenamebuf, sizeof(tarfilenamebuf),
1513  "%s/INCREMENTAL.%s",
1514  path + basepathlen + 1,
1515  de->d_name);
1516  tarfilename = tarfilenamebuf;
1517  }
1518 
1519  pfree(lookup_path);
1520  }
1521 
1522  if (!sizeonly)
1523  sent = sendFile(sink, pathbuf, tarfilename, &statbuf,
1524  true, dboid, spcoid,
1525  relfilenumber, segno, manifest,
1526  num_blocks_required,
1527  method == BACK_UP_FILE_INCREMENTALLY ? relative_block_numbers : NULL,
1528  truncation_block_length);
1529 
1530  if (sent || sizeonly)
1531  {
1532  /* Add size. */
1533  size += statbuf.st_size;
1534 
1535  /* Pad to a multiple of the tar block size. */
1536  size += tarPaddingBytesRequired(statbuf.st_size);
1537 
1538  /* Size of the header for the file. */
1539  size += TAR_BLOCK_SIZE;
1540  }
1541  }
1542  else
1543  ereport(WARNING,
1544  (errmsg("skipping special file \"%s\"", pathbuf)));
1545  }
1546 
1547  if (relative_block_numbers != NULL)
1548  pfree(relative_block_numbers);
1549 
1550  FreeDir(dir);
1551  return size;
1552 }
1553 
1554 /*
1555  * Given the member, write the TAR header & send the file.
1556  *
1557  * If 'missing_ok' is true, will not throw an error if the file is not found.
1558  *
1559  * If dboid is anything other than InvalidOid then any checksum failures
1560  * detected will get reported to the cumulative stats system.
1561  *
1562  * If the file is to be sent incrementally, then num_incremental_blocks
1563  * should be the number of blocks to be sent, and incremental_blocks
1564  * an array of block numbers relative to the start of the current segment.
1565  * If the whole file is to be sent, then incremental_blocks should be NULL,
1566  * and num_incremental_blocks can have any value, as it will be ignored.
1567  *
1568  * Returns true if the file was successfully sent, false if 'missing_ok',
1569  * and the file did not exist.
1570  */
1571 static bool
1572 sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
1573  struct stat *statbuf, bool missing_ok, Oid dboid, Oid spcoid,
1574  RelFileNumber relfilenumber, unsigned segno,
1575  backup_manifest_info *manifest, unsigned num_incremental_blocks,
1576  BlockNumber *incremental_blocks, unsigned truncation_block_length)
1577 {
1578  int fd;
1579  BlockNumber blkno = 0;
1580  int checksum_failures = 0;
1581  off_t cnt;
1582  pgoff_t bytes_done = 0;
1583  bool verify_checksum = false;
1584  pg_checksum_context checksum_ctx;
1585  int ibindex = 0;
1586 
1587  if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
1588  elog(ERROR, "could not initialize checksum of file \"%s\"",
1589  readfilename);
1590 
1591  fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY);
1592  if (fd < 0)
1593  {
1594  if (errno == ENOENT && missing_ok)
1595  return false;
1596  ereport(ERROR,
1598  errmsg("could not open file \"%s\": %m", readfilename)));
1599  }
1600 
1601  _tarWriteHeader(sink, tarfilename, NULL, statbuf, false);
1602 
1603  /*
1604  * Checksums are verified in multiples of BLCKSZ, so the buffer length
1605  * should be a multiple of the block size as well.
1606  */
1607  Assert((sink->bbs_buffer_length % BLCKSZ) == 0);
1608 
1609  /*
1610  * If we weren't told not to verify checksums, and if checksums are
1611  * enabled for this cluster, and if this is a relation file, then verify
1612  * the checksum.
1613  */
1615  RelFileNumberIsValid(relfilenumber))
1616  verify_checksum = true;
1617 
1618  /*
1619  * If we're sending an incremental file, write the file header.
1620  */
1621  if (incremental_blocks != NULL)
1622  {
1623  unsigned magic = INCREMENTAL_MAGIC;
1624  size_t header_bytes_done = 0;
1625  char padding[BLCKSZ];
1626  size_t paddinglen;
1627 
1628  /* Emit header data. */
1629  push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1630  &magic, sizeof(magic));
1631  push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1632  &num_incremental_blocks, sizeof(num_incremental_blocks));
1633  push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1634  &truncation_block_length, sizeof(truncation_block_length));
1635  push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1636  incremental_blocks,
1637  sizeof(BlockNumber) * num_incremental_blocks);
1638 
1639  /*
1640  * Add padding to align header to a multiple of BLCKSZ, but only if
1641  * the incremental file has some blocks, and the alignment is actually
1642  * needed (i.e. header is not already a multiple of BLCKSZ). If there
1643  * are no blocks we don't want to make the file unnecessarily large,
1644  * as that might make some filesystem optimizations impossible.
1645  */
1646  if ((num_incremental_blocks > 0) && (header_bytes_done % BLCKSZ != 0))
1647  {
1648  paddinglen = (BLCKSZ - (header_bytes_done % BLCKSZ));
1649 
1650  memset(padding, 0, paddinglen);
1651  bytes_done += paddinglen;
1652 
1653  push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1654  padding, paddinglen);
1655  }
1656 
1657  /* Flush out any data still in the buffer so it's again empty. */
1658  if (header_bytes_done > 0)
1659  {
1660  bbsink_archive_contents(sink, header_bytes_done);
1661  if (pg_checksum_update(&checksum_ctx,
1662  (uint8 *) sink->bbs_buffer,
1663  header_bytes_done) < 0)
1664  elog(ERROR, "could not update checksum of base backup");
1665  }
1666 
1667  /* Update our notion of file position. */
1668  bytes_done += sizeof(magic);
1669  bytes_done += sizeof(num_incremental_blocks);
1670  bytes_done += sizeof(truncation_block_length);
1671  bytes_done += sizeof(BlockNumber) * num_incremental_blocks;
1672  }
1673 
1674  /*
1675  * Loop until we read the amount of data the caller told us to expect. The
1676  * file could be longer, if it was extended while we were sending it, but
1677  * for a base backup we can ignore such extended data. It will be restored
1678  * from WAL.
1679  */
1680  while (1)
1681  {
1682  /*
1683  * Determine whether we've read all the data that we need, and if not,
1684  * read some more.
1685  */
1686  if (incremental_blocks == NULL)
1687  {
1688  size_t remaining = statbuf->st_size - bytes_done;
1689 
1690  /*
1691  * If we've read the required number of bytes, then it's time to
1692  * stop.
1693  */
1694  if (bytes_done >= statbuf->st_size)
1695  break;
1696 
1697  /*
1698  * Read as many bytes as will fit in the buffer, or however many
1699  * are left to read, whichever is less.
1700  */
1701  cnt = read_file_data_into_buffer(sink, readfilename, fd,
1702  bytes_done, remaining,
1703  blkno + segno * RELSEG_SIZE,
1704  verify_checksum,
1705  &checksum_failures);
1706  }
1707  else
1708  {
1709  BlockNumber relative_blkno;
1710 
1711  /*
1712  * If we've read all the blocks, then it's time to stop.
1713  */
1714  if (ibindex >= num_incremental_blocks)
1715  break;
1716 
1717  /*
1718  * Read just one block, whichever one is the next that we're
1719  * supposed to include.
1720  */
1721  relative_blkno = incremental_blocks[ibindex++];
1722  cnt = read_file_data_into_buffer(sink, readfilename, fd,
1723  relative_blkno * BLCKSZ,
1724  BLCKSZ,
1725  relative_blkno + segno * RELSEG_SIZE,
1726  verify_checksum,
1727  &checksum_failures);
1728 
1729  /*
1730  * If we get a partial read, that must mean that the relation is
1731  * being truncated. Ultimately, it should be truncated to a
1732  * multiple of BLCKSZ, since this path should only be reached for
1733  * relation files, but we might transiently observe an
1734  * intermediate value.
1735  *
1736  * It should be fine to treat this just as if the entire block had
1737  * been truncated away - i.e. fill this and all later blocks with
1738  * zeroes. WAL replay will fix things up.
1739  */
1740  if (cnt < BLCKSZ)
1741  break;
1742  }
1743 
1744  /*
1745  * If the amount of data we were able to read was not a multiple of
1746  * BLCKSZ, we cannot verify checksums, which are block-level.
1747  */
1748  if (verify_checksum && (cnt % BLCKSZ != 0))
1749  {
1750  ereport(WARNING,
1751  (errmsg("could not verify checksum in file \"%s\", block "
1752  "%u: read buffer size %d and page size %d "
1753  "differ",
1754  readfilename, blkno, (int) cnt, BLCKSZ)));
1755  verify_checksum = false;
1756  }
1757 
1758  /*
1759  * If we hit end-of-file, a concurrent truncation must have occurred.
1760  * That's not an error condition, because WAL replay will fix things
1761  * up.
1762  */
1763  if (cnt == 0)
1764  break;
1765 
1766  /* Update block number and # of bytes done for next loop iteration. */
1767  blkno += cnt / BLCKSZ;
1768  bytes_done += cnt;
1769 
1770  /*
1771  * Make sure incremental files with block data are properly aligned
1772  * (header is a multiple of BLCKSZ, blocks are BLCKSZ too).
1773  */
1774  Assert(!((incremental_blocks != NULL && num_incremental_blocks > 0) &&
1775  (bytes_done % BLCKSZ != 0)));
1776 
1777  /* Archive the data we just read. */
1778  bbsink_archive_contents(sink, cnt);
1779 
1780  /* Also feed it to the checksum machinery. */
1781  if (pg_checksum_update(&checksum_ctx,
1782  (uint8 *) sink->bbs_buffer, cnt) < 0)
1783  elog(ERROR, "could not update checksum of base backup");
1784  }
1785 
1786  /* If the file was truncated while we were sending it, pad it with zeros */
1787  while (bytes_done < statbuf->st_size)
1788  {
1789  size_t remaining = statbuf->st_size - bytes_done;
1790  size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1791 
1792  MemSet(sink->bbs_buffer, 0, nbytes);
1793  if (pg_checksum_update(&checksum_ctx,
1794  (uint8 *) sink->bbs_buffer,
1795  nbytes) < 0)
1796  elog(ERROR, "could not update checksum of base backup");
1797  bbsink_archive_contents(sink, nbytes);
1798  bytes_done += nbytes;
1799  }
1800 
1801  /*
1802  * Pad to a block boundary, per tar format requirements. (This small piece
1803  * of data is probably not worth throttling, and is not checksummed
1804  * because it's not actually part of the file.)
1805  */
1806  _tarWritePadding(sink, bytes_done);
1807 
1809 
1810  if (checksum_failures > 1)
1811  {
1812  ereport(WARNING,
1813  (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1814  "file \"%s\" has a total of %d checksum verification failures",
1815  checksum_failures,
1816  readfilename, checksum_failures)));
1817 
1818  pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1819  }
1820 
1821  total_checksum_failures += checksum_failures;
1822 
1823  AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
1824  (pg_time_t) statbuf->st_mtime, &checksum_ctx);
1825 
1826  return true;
1827 }
1828 
1829 /*
1830  * Read some more data from the file into the bbsink's buffer, verifying
1831  * checksums as required.
1832  *
1833  * 'offset' is the file offset from which we should begin to read, and
1834  * 'length' is the amount of data that should be read. The actual amount
1835  * of data read will be less than the requested amount if the bbsink's
1836  * buffer isn't big enough to hold it all, or if the underlying file has
1837  * been truncated. The return value is the number of bytes actually read.
1838  *
1839  * 'blkno' is the block number of the first page in the bbsink's buffer
1840  * relative to the start of the relation.
1841  *
1842  * 'verify_checksum' indicates whether we should try to verify checksums
1843  * for the blocks we read. If we do this, we'll update *checksum_failures
1844  * and issue warnings as appropriate.
1845  */
1846 static off_t
1847 read_file_data_into_buffer(bbsink *sink, const char *readfilename, int fd,
1848  off_t offset, size_t length, BlockNumber blkno,
1849  bool verify_checksum, int *checksum_failures)
1850 {
1851  off_t cnt;
1852  int i;
1853  char *page;
1854 
1855  /* Try to read some more data. */
1856  cnt = basebackup_read_file(fd, sink->bbs_buffer,
1857  Min(sink->bbs_buffer_length, length),
1858  offset, readfilename, true);
1859 
1860  /* Can't verify checksums if read length is not a multiple of BLCKSZ. */
1861  if (!verify_checksum || (cnt % BLCKSZ) != 0)
1862  return cnt;
1863 
1864  /* Verify checksum for each block. */
1865  for (i = 0; i < cnt / BLCKSZ; i++)
1866  {
1867  int reread_cnt;
1868  uint16 expected_checksum;
1869 
1870  page = sink->bbs_buffer + BLCKSZ * i;
1871 
1872  /* If the page is OK, go on to the next one. */
1873  if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i,
1874  &expected_checksum))
1875  continue;
1876 
1877  /*
1878  * Retry the block on the first failure. It's possible that we read
1879  * the first 4K page of the block just before postgres updated the
1880  * entire block so it ends up looking torn to us. If, before we retry
1881  * the read, the concurrent write of the block finishes, the page LSN
1882  * will be updated and we'll realize that we should ignore this block.
1883  *
1884  * There's no guarantee that this will actually happen, though: the
1885  * torn write could take an arbitrarily long time to complete.
1886  * Retrying multiple times wouldn't fix this problem, either, though
1887  * it would reduce the chances of it happening in practice. The only
1888  * real fix here seems to be to have some kind of interlock that
1889  * allows us to wait until we can be certain that no write to the
1890  * block is in progress. Since we don't have any such thing right now,
1891  * we just do this and hope for the best.
1892  */
1893  reread_cnt =
1894  basebackup_read_file(fd, sink->bbs_buffer + BLCKSZ * i,
1895  BLCKSZ, offset + BLCKSZ * i,
1896  readfilename, false);
1897  if (reread_cnt == 0)
1898  {
1899  /*
1900  * If we hit end-of-file, a concurrent truncation must have
1901  * occurred, so reduce cnt to reflect only the blocks already
1902  * processed and break out of this loop.
1903  */
1904  cnt = BLCKSZ * i;
1905  break;
1906  }
1907 
1908  /* If the page now looks OK, go on to the next one. */
1909  if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i,
1910  &expected_checksum))
1911  continue;
1912 
1913  /* Handle checksum failure. */
1914  (*checksum_failures)++;
1915  if (*checksum_failures <= 5)
1916  ereport(WARNING,
1917  (errmsg("checksum verification failed in "
1918  "file \"%s\", block %u: calculated "
1919  "%X but expected %X",
1920  readfilename, blkno + i, expected_checksum,
1921  ((PageHeader) page)->pd_checksum)));
1922  if (*checksum_failures == 5)
1923  ereport(WARNING,
1924  (errmsg("further checksum verification "
1925  "failures in file \"%s\" will not "
1926  "be reported", readfilename)));
1927  }
1928 
1929  return cnt;
1930 }
1931 
1932 /*
1933  * Push data into a bbsink.
1934  *
1935  * It's better, when possible, to read data directly into the bbsink's buffer,
1936  * rather than using this function to copy it into the buffer; this function is
1937  * for cases where that approach is not practical.
1938  *
1939  * bytes_done should point to a count of the number of bytes that are
1940  * currently used in the bbsink's buffer. Upon return, the bytes identified by
1941  * data and length will have been copied into the bbsink's buffer, flushing
1942  * as required, and *bytes_done will have been updated accordingly. If the
1943  * buffer was flushed, the previous contents will also have been fed to
1944  * checksum_ctx.
1945  *
1946  * Note that after one or more calls to this function it is the caller's
1947  * responsibility to perform any required final flush.
1948  */
1949 static void
1951  size_t *bytes_done, void *data, size_t length)
1952 {
1953  while (length > 0)
1954  {
1955  size_t bytes_to_copy;
1956 
1957  /*
1958  * We use < here rather than <= so that if the data exactly fills the
1959  * remaining buffer space, we trigger a flush now.
1960  */
1961  if (length < sink->bbs_buffer_length - *bytes_done)
1962  {
1963  /* Append remaining data to buffer. */
1964  memcpy(sink->bbs_buffer + *bytes_done, data, length);
1965  *bytes_done += length;
1966  return;
1967  }
1968 
1969  /* Copy until buffer is full and flush it. */
1970  bytes_to_copy = sink->bbs_buffer_length - *bytes_done;
1971  memcpy(sink->bbs_buffer + *bytes_done, data, bytes_to_copy);
1972  data = ((char *) data) + bytes_to_copy;
1973  length -= bytes_to_copy;
1975  if (pg_checksum_update(checksum_ctx, (uint8 *) sink->bbs_buffer,
1976  sink->bbs_buffer_length) < 0)
1977  elog(ERROR, "could not update checksum");
1978  *bytes_done = 0;
1979  }
1980 }
1981 
1982 /*
1983  * Try to verify the checksum for the provided page, if it seems appropriate
1984  * to do so.
1985  *
1986  * Returns true if verification succeeds or if we decide not to check it,
1987  * and false if verification fails. When return false, it also sets
1988  * *expected_checksum to the computed value.
1989  */
1990 static bool
1992  uint16 *expected_checksum)
1993 {
1994  PageHeader phdr;
1995  uint16 checksum;
1996 
1997  /*
1998  * Only check pages which have not been modified since the start of the
1999  * base backup. Otherwise, they might have been written only halfway and
2000  * the checksum would not be valid. However, replaying WAL would
2001  * reinstate the correct page in this case. We also skip completely new
2002  * pages, since they don't have a checksum yet.
2003  */
2004  if (PageIsNew(page) || PageGetLSN(page) >= start_lsn)
2005  return true;
2006 
2007  /* Perform the actual checksum calculation. */
2008  checksum = pg_checksum_page(page, blkno);
2009 
2010  /* See whether it matches the value from the page. */
2011  phdr = (PageHeader) page;
2012  if (phdr->pd_checksum == checksum)
2013  return true;
2014  *expected_checksum = checksum;
2015  return false;
2016 }
2017 
2018 static int64
2019 _tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget,
2020  struct stat *statbuf, bool sizeonly)
2021 {
2022  enum tarError rc;
2023 
2024  if (!sizeonly)
2025  {
2026  /*
2027  * As of this writing, the smallest supported block size is 1kB, which
2028  * is twice TAR_BLOCK_SIZE. Since the buffer size is required to be a
2029  * multiple of BLCKSZ, it should be safe to assume that the buffer is
2030  * large enough to fit an entire tar block. We double-check by means
2031  * of these assertions.
2032  */
2033  StaticAssertDecl(TAR_BLOCK_SIZE <= BLCKSZ,
2034  "BLCKSZ too small for tar block");
2036 
2037  rc = tarCreateHeader(sink->bbs_buffer, filename, linktarget,
2038  statbuf->st_size, statbuf->st_mode,
2039  statbuf->st_uid, statbuf->st_gid,
2040  statbuf->st_mtime);
2041 
2042  switch (rc)
2043  {
2044  case TAR_OK:
2045  break;
2046  case TAR_NAME_TOO_LONG:
2047  ereport(ERROR,
2048  (errmsg("file name too long for tar format: \"%s\"",
2049  filename)));
2050  break;
2051  case TAR_SYMLINK_TOO_LONG:
2052  ereport(ERROR,
2053  (errmsg("symbolic link target too long for tar format: "
2054  "file name \"%s\", target \"%s\"",
2055  filename, linktarget)));
2056  break;
2057  default:
2058  elog(ERROR, "unrecognized tar error: %d", rc);
2059  }
2060 
2062  }
2063 
2064  return TAR_BLOCK_SIZE;
2065 }
2066 
2067 /*
2068  * Pad with zero bytes out to a multiple of TAR_BLOCK_SIZE.
2069  */
2070 static void
2072 {
2073  int pad = tarPaddingBytesRequired(len);
2074 
2075  /*
2076  * As in _tarWriteHeader, it should be safe to assume that the buffer is
2077  * large enough that we don't need to do this in multiple chunks.
2078  */
2080  Assert(pad <= TAR_BLOCK_SIZE);
2081 
2082  if (pad > 0)
2083  {
2084  MemSet(sink->bbs_buffer, 0, pad);
2085  bbsink_archive_contents(sink, pad);
2086  }
2087 }
2088 
2089 /*
2090  * If the entry in statbuf is a link, then adjust statbuf to make it look like a
2091  * directory, so that it will be written that way.
2092  */
2093 static void
2094 convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
2095 {
2096  /* If symlink, write it as a directory anyway */
2097  if (S_ISLNK(statbuf->st_mode))
2098  statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
2099 }
2100 
2101 /*
2102  * Read some data from a file, setting a wait event and reporting any error
2103  * encountered.
2104  *
2105  * If partial_read_ok is false, also report an error if the number of bytes
2106  * read is not equal to the number of bytes requested.
2107  *
2108  * Returns the number of bytes read.
2109  */
2110 static ssize_t
2111 basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
2112  const char *filename, bool partial_read_ok)
2113 {
2114  ssize_t rc;
2115 
2116  pgstat_report_wait_start(WAIT_EVENT_BASEBACKUP_READ);
2117  rc = pg_pread(fd, buf, nbytes, offset);
2119 
2120  if (rc < 0)
2121  ereport(ERROR,
2123  errmsg("could not read file \"%s\": %m", filename)));
2124  if (!partial_read_ok && rc > 0 && rc != nbytes)
2125  ereport(ERROR,
2127  errmsg("could not read file \"%s\": read %zd of %zu",
2128  filename, rc, nbytes)));
2129 
2130  return rc;
2131 }
void InitializeBackupManifest(backup_manifest_info *manifest, backup_manifest_option want_manifest, pg_checksum_type manifest_checksum_type)
void AddFileToBackupManifest(backup_manifest_info *manifest, Oid spcoid, const char *pathname, size_t size, pg_time_t mtime, pg_checksum_context *checksum_ctx)
void AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr, TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)
void SendBackupManifest(backup_manifest_info *manifest, bbsink *sink)
void FreeBackupManifest(backup_manifest_info *manifest)
@ MANIFEST_OPTION_NO
@ MANIFEST_OPTION_FORCE_ENCODE
@ MANIFEST_OPTION_YES
enum manifest_option backup_manifest_option
static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, Oid spcoid, RelFileNumber relfilenumber, unsigned segno, backup_manifest_info *manifest, unsigned num_incremental_blocks, BlockNumber *incremental_blocks, unsigned truncation_block_length)
Definition: basebackup.c:1572
static const struct exclude_list_item excludeFiles[]
Definition: basebackup.c:191
static int compareWalFileNames(const ListCell *a, const ListCell *b)
Definition: basebackup.c:684
static void sendFileWithContent(bbsink *sink, const char *filename, const char *content, int len, backup_manifest_info *manifest)
Definition: basebackup.c:1073
static off_t read_file_data_into_buffer(bbsink *sink, const char *readfilename, int fd, off_t offset, size_t length, BlockNumber blkno, bool verify_checksum, int *checksum_failures)
Definition: basebackup.c:1847
static void push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx, size_t *bytes_done, void *data, size_t length)
Definition: basebackup.c:1950
static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, Oid spcoid, IncrementalBackupInfo *ib)
Definition: basebackup.c:1187
#define SINK_BUFFER_LENGTH
Definition: basebackup.c:60
static void parse_basebackup_options(List *options, basebackup_options *opt)
Definition: basebackup.c:696
static const char *const excludeDirContents[]
Definition: basebackup.c:151
static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
Definition: basebackup.c:2094
void SendBaseBackup(BaseBackupCmd *cmd, IncrementalBackupInfo *ib)
Definition: basebackup.c:988
static bool backup_started_in_recovery
Definition: basebackup.c:123
static int64 _tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:2019
static void perform_base_backup(basebackup_options *opt, bbsink *sink, IncrementalBackupInfo *ib)
Definition: basebackup.c:234
static int64 sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly, struct backup_manifest_info *manifest, IncrementalBackupInfo *ib)
Definition: basebackup.c:1134
static bool noverify_checksums
Definition: basebackup.c:129
static void _tarWritePadding(bbsink *sink, int len)
Definition: basebackup.c:2071
static bool verify_page_checksum(Page page, XLogRecPtr start_lsn, BlockNumber blkno, uint16 *expected_checksum)
Definition: basebackup.c:1991
static ssize_t basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset, const char *filename, bool partial_read_ok)
Definition: basebackup.c:2111
static long long int total_checksum_failures
Definition: basebackup.c:126
#define MAX_RATE_UPPER
Definition: basebackup.h:21
#define MAX_RATE_LOWER
Definition: basebackup.h:20
bbsink * bbsink_copystream_new(bool send_to_client)
bbsink * bbsink_gzip_new(bbsink *next, pg_compress_specification *compress)
size_t GetIncrementalFileSize(unsigned num_blocks_required)
FileBackupMethod GetFileBackupMethod(IncrementalBackupInfo *ib, const char *path, Oid dboid, Oid spcoid, RelFileNumber relfilenumber, ForkNumber forknum, unsigned segno, size_t size, unsigned *num_blocks_required, BlockNumber *relative_block_numbers, unsigned *truncation_block_length)
void PrepareForIncrementalBackup(IncrementalBackupInfo *ib, BackupState *backup_state)
#define INCREMENTAL_MAGIC
@ BACK_UP_FILE_INCREMENTALLY
@ BACK_UP_FILE_FULLY
bbsink * bbsink_lz4_new(bbsink *next, pg_compress_specification *compress)
void basebackup_progress_wait_checkpoint(void)
void basebackup_progress_wait_wal_archive(bbsink_state *state)
bbsink * bbsink_progress_new(bbsink *next, bool estimate_backup_size)
void basebackup_progress_done(void)
void basebackup_progress_transfer_wal(void)
void basebackup_progress_estimate_backup_size(void)
static void bbsink_begin_backup(bbsink *sink, bbsink_state *state, int buffer_length)
static void bbsink_begin_archive(bbsink *sink, const char *archive_name)
static void bbsink_end_archive(bbsink *sink)
static void bbsink_end_backup(bbsink *sink, XLogRecPtr endptr, TimeLineID endtli)
static void bbsink_cleanup(bbsink *sink)
static void bbsink_archive_contents(bbsink *sink, size_t len)
bbsink * BaseBackupGetSink(BaseBackupTargetHandle *handle, bbsink *next_sink)
BaseBackupTargetHandle * BaseBackupGetTargetHandle(char *target, char *target_detail)
bbsink * bbsink_throttle_new(bbsink *next, uint32 maxrate)
bbsink * bbsink_zstd_new(bbsink *next, pg_compress_specification *compress)
uint32 BlockNumber
Definition: block.h:31
bool parse_bool(const char *value, bool *result)
Definition: bool.c:30
PageHeaderData * PageHeader
Definition: bufpage.h:170
Pointer Page
Definition: bufpage.h:78
static bool PageIsNew(Page page)
Definition: bufpage.h:230
static XLogRecPtr PageGetLSN(Page page)
Definition: bufpage.h:383
unsigned short uint16
Definition: c.h:505
unsigned int uint32
Definition: c.h:506
#define Min(x, y)
Definition: c.h:1004
#define Assert(condition)
Definition: c.h:858
#define PG_BINARY
Definition: c.h:1273
unsigned char uint8
Definition: c.h:504
#define MemSet(start, val, len)
Definition: c.h:1020
#define StaticAssertDecl(condition, errmessage)
Definition: c.h:936
#define StaticAssertStmt(condition, errmessage)
Definition: c.h:938
#define OidIsValid(objectId)
Definition: c.h:775
uint16 pg_checksum_page(char *page, BlockNumber blkno)
bool pg_checksum_parse_type(char *name, pg_checksum_type *type)
int pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
int pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
pg_checksum_type
@ CHECKSUM_TYPE_NONE
@ CHECKSUM_TYPE_CRC32C
bool parse_compress_algorithm(char *name, pg_compress_algorithm *algorithm)
Definition: compression.c:49
void parse_compress_specification(pg_compress_algorithm algorithm, char *specification, pg_compress_specification *result)
Definition: compression.c:107
char * validate_compress_specification(pg_compress_specification *spec)
Definition: compression.c:344
pg_compress_algorithm
Definition: compression.h:22
@ PG_COMPRESSION_GZIP
Definition: compression.h:24
@ PG_COMPRESSION_LZ4
Definition: compression.h:25
@ PG_COMPRESSION_NONE
Definition: compression.h:23
@ PG_COMPRESSION_ZSTD
Definition: compression.h:26
bool defGetBoolean(DefElem *def)
Definition: define.c:107
char * defGetString(DefElem *def)
Definition: define.c:48
int64 defGetInt64(DefElem *def)
Definition: define.c:186
#define PG_DYNSHMEM_DIR
Definition: dsm_impl.h:51
int errmsg_plural(const char *fmt_singular, const char *fmt_plural, unsigned long n,...)
Definition: elog.c:1180
int errcode_for_file_access(void)
Definition: elog.c:880
int errhint(const char *fmt,...)
Definition: elog.c:1317
int errcode(int sqlerrcode)
Definition: elog.c:857
int errmsg(const char *fmt,...)
Definition: elog.c:1070
#define PG_TRY(...)
Definition: elog.h:370
#define WARNING
Definition: elog.h:36
#define DEBUG2
Definition: elog.h:29
#define PG_END_TRY(...)
Definition: elog.h:395
#define DEBUG1
Definition: elog.h:30
#define ERROR
Definition: elog.h:39
#define elog(elevel,...)
Definition: elog.h:224
#define PG_FINALLY(...)
Definition: elog.h:387
#define ereport(elevel,...)
Definition: elog.h:149
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2909
int FreeDir(DIR *dir)
Definition: fd.c:2961
int CloseTransientFile(int fd)
Definition: fd.c:2809
bool looks_like_temp_rel_name(const char *name)
Definition: fd.c:3449
int OpenTransientFile(const char *fileName, int fileFlags)
Definition: fd.c:2633
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2843
int pg_file_create_mode
Definition: file_perm.c:19
int pg_dir_create_mode
Definition: file_perm.c:18
#define PG_TEMP_FILE_PREFIX
Definition: file_utils.h:63
#define PG_AUTOCONF_FILENAME
Definition: guc.h:33
int remaining
Definition: informix.c:673
#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:47
#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:52
int b
Definition: isn.c:70
int a
Definition: isn.c:69
int i
Definition: isn.c:73
void list_sort(List *list, list_sort_comparator cmp)
Definition: list.c:1674
List * lappend(List *list, void *datum)
Definition: list.c:339
char * pstrdup(const char *in)
Definition: mcxt.c:1695
void pfree(void *pointer)
Definition: mcxt.c:1520
void * palloc0(Size size)
Definition: mcxt.c:1346
void * palloc(Size size)
Definition: mcxt.c:1316
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:122
static bool manifest
static int32 maxrate
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:41
#define MAXPGPATH
const void size_t len
const void * data
static char * filename
Definition: pg_dumpall.c:119
#define lfirst(lc)
Definition: pg_list.h:172
#define NIL
Definition: pg_list.h:68
#define linitial(l)
Definition: pg_list.h:178
static ListCell * lnext(const List *l, const ListCell *c)
Definition: pg_list.h:343
static char * buf
Definition: pg_test_fsync.c:73
#define PG_STAT_TMP_DIR
Definition: pgstat.h:32
void pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount)
static size_t tarPaddingBytesRequired(size_t len)
Definition: pgtar.h:79
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
tarError
Definition: pgtar.h:20
@ TAR_SYMLINK_TOO_LONG
Definition: pgtar.h:23
@ TAR_OK
Definition: pgtar.h:21
@ TAR_NAME_TOO_LONG
Definition: pgtar.h:22
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
int64 pg_time_t
Definition: pgtime.h:23
char * last_dir_separator(const char *filename)
Definition: path.c:139
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
#define pg_pread
Definition: port.h:225
#define snprintf
Definition: port.h:238
static Datum BoolGetDatum(bool X)
Definition: postgres.h:102
#define InvalidOid
Definition: postgres_ext.h:36
unsigned int Oid
Definition: postgres_ext.h:31
#define atooid(x)
Definition: postgres_ext.h:42
static int fd(const char *x, int i)
Definition: preproc-init.c:105
bool update_process_title
Definition: ps_status.c:29
static void set_ps_display(const char *activity)
Definition: ps_status.h:40
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
bool parse_filename_for_nontemp_relation(const char *name, RelFileNumber *relnumber, ForkNumber *fork, unsigned *segno)
Definition: reinit.c:380
#define RELCACHE_INIT_FILENAME
Definition: relcache.h:25
Oid RelFileNumber
Definition: relpath.h:25
ForkNumber
Definition: relpath.h:48
@ InvalidForkNumber
Definition: relpath.h:49
@ INIT_FORKNUM
Definition: relpath.h:53
#define InvalidRelFileNumber
Definition: relpath.h:26
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:33
#define RelFileNumberIsValid(relnumber)
Definition: relpath.h:27
ResourceOwner ResourceOwnerCreate(ResourceOwner parent, const char *name)
Definition: resowner.c:413
ResourceOwner CurrentResourceOwner
Definition: resowner.c:165
static pg_noinline void Size size
Definition: slab.c:607
void destroyStringInfo(StringInfo str)
Definition: stringinfo.c:361
StringInfo makeStringInfo(void)
Definition: stringinfo.c:41
TimeLineID starttli
Definition: xlogbackup.h:27
XLogRecPtr startpoint
Definition: xlogbackup.h:26
TimeLineID stoptli
Definition: xlogbackup.h:36
XLogRecPtr stoppoint
Definition: xlogbackup.h:35
List * options
Definition: replnodes.h:44
Definition: dirent.c:26
char * defname
Definition: parsenodes.h:815
Definition: pg_list.h:54
uint16 pd_checksum
Definition: bufpage.h:160
pg_compress_specification compression_specification
Definition: basebackup.c:77
pg_checksum_type manifest_checksum_type
Definition: basebackup.c:78
const char * label
Definition: basebackup.c:64
pg_compress_algorithm compression
Definition: basebackup.c:76
backup_manifest_option manifest
Definition: basebackup.c:75
BaseBackupTargetHandle * target_handle
Definition: basebackup.c:74
XLogRecPtr startptr
bbsink_state * bbs_state
char * bbs_buffer
size_t bbs_buffer_length
Definition: dirent.h:10
char d_name[MAX_PATH]
Definition: dirent.h:15
const char * name
Definition: basebackup.c:139
pg_compress_algorithm algorithm
Definition: compression.h:34
__time64_t st_mtime
Definition: win32_port.h:275
__int64 st_size
Definition: win32_port.h:273
short st_gid
Definition: win32_port.h:271
unsigned short st_mode
Definition: win32_port.h:268
short st_uid
Definition: win32_port.h:270
Definition: regguts.h:323
char * rpath
Definition: basebackup.h:32
#define LOG_METAINFO_DATAFILE_TMP
Definition: syslogger.h:99
static void pgstat_report_wait_start(uint32 wait_event_info)
Definition: wait_event.h:82
static void pgstat_report_wait_end(void)
Definition: wait_event.h:98
const char * name
void WalSndSetState(WalSndState state)
Definition: walsender.c:3809
void WalSndResourceCleanup(bool isCommit)
Definition: walsender.c:362
@ WALSNDSTATE_BACKUP
bool summarize_wal
#define lstat(path, sb)
Definition: win32_port.h:285
#define S_ISDIR(m)
Definition: win32_port.h:325
#define S_ISLNK(m)
Definition: win32_port.h:344
#define fstat
Definition: win32_port.h:283
#define readlink(path, buf, size)
Definition: win32_port.h:236
#define S_ISREG(m)
Definition: win32_port.h:328
#define pgoff_t
Definition: win32_port.h:207
bool RecoveryInProgress(void)
Definition: xlog.c:6290
void do_pg_abort_backup(int code, Datum arg)
Definition: xlog.c:9314
SessionBackupState get_backup_status(void)
Definition: xlog.c:9021
void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)
Definition: xlog.c:3716
int wal_segment_size
Definition: xlog.c:143
bool DataChecksumsEnabled(void)
Definition: xlog.c:4555
void do_pg_backup_start(const char *backupidstr, bool fast, List **tablespaces, BackupState *state, StringInfo tblspcmapfile)
Definition: xlog.c:8712
void do_pg_backup_stop(BackupState *state, bool waitforarchive)
Definition: xlog.c:9040
#define TABLESPACE_MAP
Definition: xlog.h:301
#define BACKUP_LABEL_FILE
Definition: xlog.h:298
SessionBackupState
Definition: xlog.h:282
@ SESSION_BACKUP_RUNNING
Definition: xlog.h:284
#define XLOG_CONTROL_FILE
static bool IsXLogFileName(const char *fname)
static void XLogFromFileName(const char *fname, TimeLineID *tli, XLogSegNo *logSegNo, int wal_segsz_bytes)
#define XLByteToPrevSeg(xlrp, logSegNo, wal_segsz_bytes)
static bool IsTLHistoryFileName(const char *fname)
#define MAXFNAMELEN
#define XLOGDIR
static void StatusFilePath(char *path, const char *xlog, const char *suffix)
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)
static void XLogFileName(char *fname, TimeLineID tli, XLogSegNo logSegNo, int wal_segsz_bytes)
char * build_backup_content(BackupState *state, bool ishistoryfile)
Definition: xlogbackup.c:29
uint64 XLogRecPtr
Definition: xlogdefs.h:21
uint32 TimeLineID
Definition: xlogdefs.h:59
uint64 XLogSegNo
Definition: xlogdefs.h:48
static BackupState * backup_state
Definition: xlogfuncs.c:40
static StringInfo tablespace_map
Definition: xlogfuncs.c:41