PostgreSQL Source Code  git master
basebackup.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * basebackup.c
4  * code for taking a base backup and streaming it to a standby
5  *
6  * Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * src/backend/replication/basebackup.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 #include "postgres.h"
14 
15 #include <sys/stat.h>
16 #include <unistd.h>
17 #include <time.h>
18 
19 #include "access/xlog_internal.h" /* for pg_start/stop_backup */
20 #include "catalog/pg_type.h"
21 #include "common/file_perm.h"
22 #include "commands/progress.h"
23 #include "lib/stringinfo.h"
24 #include "libpq/libpq.h"
25 #include "libpq/pqformat.h"
26 #include "miscadmin.h"
27 #include "nodes/pg_list.h"
28 #include "pgstat.h"
29 #include "pgtar.h"
30 #include "port.h"
31 #include "postmaster/syslogger.h"
32 #include "replication/basebackup.h"
34 #include "replication/walsender.h"
36 #include "storage/bufpage.h"
37 #include "storage/checksum.h"
38 #include "storage/dsm_impl.h"
39 #include "storage/fd.h"
40 #include "storage/ipc.h"
41 #include "storage/reinit.h"
42 #include "utils/builtins.h"
43 #include "utils/ps_status.h"
44 #include "utils/relcache.h"
45 #include "utils/resowner.h"
46 #include "utils/timestamp.h"
47 
48 typedef struct
49 {
50  const char *label;
51  bool progress;
53  bool nowait;
54  bool includewal;
60 
61 static int64 sendDir(const char *path, int basepathlen, bool sizeonly,
62  List *tablespaces, bool sendtblspclinks,
63  backup_manifest_info *manifest, const char *spcoid);
64 static bool sendFile(const char *readfilename, const char *tarfilename,
65  struct stat *statbuf, bool missing_ok, Oid dboid,
66  backup_manifest_info *manifest, const char *spcoid);
67 static void sendFileWithContent(const char *filename, const char *content,
69 static int64 _tarWriteHeader(const char *filename, const char *linktarget,
70  struct stat *statbuf, bool sizeonly);
71 static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf,
72  bool sizeonly);
73 static void send_int8_string(StringInfoData *buf, int64 intval);
74 static void SendBackupHeader(List *tablespaces);
75 static void perform_base_backup(basebackup_options *opt);
77 static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli);
78 static int compareWalFileNames(const ListCell *a, const ListCell *b);
79 static void throttle(size_t increment);
80 static void update_basebackup_progress(int64 delta);
81 static bool is_checksummed_file(const char *fullpath, const char *filename);
82 
83 /* Was the backup currently in-progress initiated in recovery mode? */
84 static bool backup_started_in_recovery = false;
85 
86 /* Relative path of temporary statistics directory */
87 static char *statrelpath = NULL;
88 
89 /*
90  * Size of each block sent into the tar stream for larger files.
91  */
92 #define TAR_SEND_SIZE 32768
93 
94 /*
95  * How frequently to throttle, as a fraction of the specified rate-second.
96  */
97 #define THROTTLING_FREQUENCY 8
98 
99 /*
100  * Checks whether we encountered any error in fread(). fread() doesn't give
101  * any clue what has happened, so we check with ferror(). Also, neither
102  * fread() nor ferror() set errno, so we just throw a generic error.
103  */
104 #define CHECK_FREAD_ERROR(fp, filename) \
105 do { \
106  if (ferror(fp)) \
107  ereport(ERROR, \
108  (errmsg("could not read from file \"%s\"", filename))); \
109 } while (0)
110 
111 /* The actual number of bytes, transfer of which may cause sleep. */
112 static uint64 throttling_sample;
113 
114 /* Amount of data already transferred but not yet throttled. */
115 static int64 throttling_counter;
116 
117 /* The minimum time required to transfer throttling_sample bytes. */
119 
120 /* The last check of the transfer rate. */
122 
123 /* The starting XLOG position of the base backup. */
125 
126 /* Total number of checksum failures during base backup. */
127 static long long int total_checksum_failures;
128 
129 /* Do not verify checksums. */
130 static bool noverify_checksums = false;
131 
132 /*
133  * Total amount of backup data that will be streamed.
134  * -1 means that the size is not estimated.
135  */
136 static int64 backup_total = 0;
137 
138 /* Amount of backup data already streamed */
139 static int64 backup_streamed = 0;
140 
141 /*
142  * Definition of one element part of an exclusion list, used for paths part
143  * of checksum validation or base backups. "name" is the name of the file
144  * or path to check for exclusion. If "match_prefix" is true, any items
145  * matching the name as prefix are excluded.
146  */
148 {
149  const char *name;
151 };
152 
153 /*
154  * The contents of these directories are removed or recreated during server
155  * start so they are not included in backups. The directories themselves are
156  * kept and included as empty to preserve access permissions.
157  *
158  * Note: this list should be kept in sync with the filter lists in pg_rewind's
159  * filemap.c.
160  */
161 static const char *const excludeDirContents[] =
162 {
163  /*
164  * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even
165  * when stats_temp_directory is set because PGSS_TEXT_FILE is always
166  * created there.
167  */
169 
170  /*
171  * It is generally not useful to backup the contents of this directory
172  * even if the intention is to restore to another master. See backup.sgml
173  * for a more detailed description.
174  */
175  "pg_replslot",
176 
177  /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
179 
180  /* Contents removed on startup, see AsyncShmemInit(). */
181  "pg_notify",
182 
183  /*
184  * Old contents are loaded for possible debugging but are not required for
185  * normal operation, see SerialInit().
186  */
187  "pg_serial",
188 
189  /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
190  "pg_snapshots",
191 
192  /* Contents zeroed on startup, see StartupSUBTRANS(). */
193  "pg_subtrans",
194 
195  /* end of list */
196  NULL
197 };
198 
199 /*
200  * List of files excluded from backups.
201  */
202 static const struct exclude_list_item excludeFiles[] =
203 {
204  /* Skip auto conf temporary file. */
205  {PG_AUTOCONF_FILENAME ".tmp", false},
206 
207  /* Skip current log file temporary file */
208  {LOG_METAINFO_DATAFILE_TMP, false},
209 
210  /*
211  * Skip relation cache because it is rebuilt on startup. This includes
212  * temporary files.
213  */
214  {RELCACHE_INIT_FILENAME, true},
215 
216  /*
217  * If there's a backup_label or tablespace_map file, it belongs to a
218  * backup started by the user with pg_start_backup(). It is *not* correct
219  * for this backup. Our backup_label/tablespace_map is injected into the
220  * tar separately.
221  */
222  {BACKUP_LABEL_FILE, false},
223  {TABLESPACE_MAP, false},
224 
225  /*
226  * If there's a backup_manifest, it belongs to a backup that was used to
227  * start this server. It is *not* correct for this backup. Our
228  * backup_manifest is injected into the backup separately if users want
229  * it.
230  */
231  {"backup_manifest", false},
232 
233  {"postmaster.pid", false},
234  {"postmaster.opts", false},
235 
236  /* end of list */
237  {NULL, false}
238 };
239 
240 /*
241  * List of files excluded from checksum validation.
242  *
243  * Note: this list should be kept in sync with what pg_checksums.c
244  * includes.
245  */
246 static const struct exclude_list_item noChecksumFiles[] = {
247  {"pg_control", false},
248  {"pg_filenode.map", false},
249  {"pg_internal.init", true},
250  {"PG_VERSION", false},
251 #ifdef EXEC_BACKEND
252  {"config_exec_params", true},
253 #endif
254  {NULL, false}
255 };
256 
257 /*
258  * Actually do a base backup for the specified tablespaces.
259  *
260  * This is split out mainly to avoid complaints about "variable might be
261  * clobbered by longjmp" from stupider versions of gcc.
262  */
263 static void
265 {
266  TimeLineID starttli;
267  XLogRecPtr endptr;
268  TimeLineID endtli;
269  StringInfo labelfile;
272  int datadirpathlen;
273  List *tablespaces = NIL;
274 
275  backup_total = 0;
276  backup_streamed = 0;
278 
279  /*
280  * If the estimation of the total backup size is disabled, make the
281  * backup_total column in the view return NULL by setting the parameter to
282  * -1.
283  */
284  if (!opt->progress)
285  {
286  backup_total = -1;
288  backup_total);
289  }
290 
291  /* we're going to use a BufFile, so we need a ResourceOwner */
292  Assert(CurrentResourceOwner == NULL);
293  CurrentResourceOwner = ResourceOwnerCreate(NULL, "base backup");
294 
295  datadirpathlen = strlen(DataDir);
296 
298 
299  labelfile = makeStringInfo();
300  tblspc_map_file = makeStringInfo();
301  InitializeBackupManifest(&manifest, opt->manifest,
303 
305 
308  startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
309  labelfile, &tablespaces,
310  tblspc_map_file,
311  opt->progress, opt->sendtblspcmapfile);
312 
313  /*
314  * Once do_pg_start_backup has been called, ensure that any failure causes
315  * us to abort the backup so we don't "leak" a backup counter. For this
316  * reason, *all* functionality between do_pg_start_backup() and the end of
317  * do_pg_stop_backup() should be inside the error cleanup block!
318  */
319 
321  {
322  ListCell *lc;
323  tablespaceinfo *ti;
324  int tblspc_streamed = 0;
325 
326  /*
327  * Calculate the relative path of temporary statistics directory in
328  * order to skip the files which are located in that directory later.
329  */
331  strncmp(pgstat_stat_directory, DataDir, datadirpathlen) == 0)
332  statrelpath = psprintf("./%s", pgstat_stat_directory + datadirpathlen + 1);
333  else if (strncmp(pgstat_stat_directory, "./", 2) != 0)
335  else
337 
338  /* Add a node for the base directory at the end */
339  ti = palloc0(sizeof(tablespaceinfo));
340  if (opt->progress)
341  ti->size = sendDir(".", 1, true, tablespaces, true, NULL, NULL);
342  else
343  ti->size = -1;
344  tablespaces = lappend(tablespaces, ti);
345 
346  /*
347  * Calculate the total backup size by summing up the size of each
348  * tablespace
349  */
350  if (opt->progress)
351  {
352  foreach(lc, tablespaces)
353  {
354  tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
355 
356  backup_total += tmp->size;
357  }
358  }
359 
360  /* Report that we are now streaming database files as a base backup */
361  {
362  const int index[] = {
366  };
367  const int64 val[] = {
369  backup_total, list_length(tablespaces)
370  };
371 
372  pgstat_progress_update_multi_param(3, index, val);
373  }
374 
375  /* Send the starting position of the backup */
376  SendXlogRecPtrResult(startptr, starttli);
377 
378  /* Send tablespace header */
379  SendBackupHeader(tablespaces);
380 
381  /* Setup and activate network throttling, if client requested it */
382  if (opt->maxrate > 0)
383  {
385  (int64) opt->maxrate * (int64) 1024 / THROTTLING_FREQUENCY;
386 
387  /*
388  * The minimum amount of time for throttling_sample bytes to be
389  * transferred.
390  */
392 
393  /* Enable throttling. */
394  throttling_counter = 0;
395 
396  /* The 'real data' starts now (header was ignored). */
398  }
399  else
400  {
401  /* Disable throttling. */
402  throttling_counter = -1;
403  }
404 
405  /* Send off our tablespaces one by one */
406  foreach(lc, tablespaces)
407  {
408  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
410 
411  /* Send CopyOutResponse message */
412  pq_beginmessage(&buf, 'H');
413  pq_sendbyte(&buf, 0); /* overall format */
414  pq_sendint16(&buf, 0); /* natts */
415  pq_endmessage(&buf);
416 
417  if (ti->path == NULL)
418  {
419  struct stat statbuf;
420 
421  /* In the main tar, include the backup_label first... */
423  &manifest);
424 
425  /*
426  * Send tablespace_map file if required and then the bulk of
427  * the files.
428  */
429  if (tblspc_map_file && opt->sendtblspcmapfile)
430  {
431  sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data,
432  &manifest);
433  sendDir(".", 1, false, tablespaces, false,
434  &manifest, NULL);
435  }
436  else
437  sendDir(".", 1, false, tablespaces, true,
438  &manifest, NULL);
439 
440  /* ... and pg_control after everything else. */
441  if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
442  ereport(ERROR,
444  errmsg("could not stat file \"%s\": %m",
447  false, InvalidOid, &manifest, NULL);
448  }
449  else
450  sendTablespace(ti->path, ti->oid, false, &manifest);
451 
452  /*
453  * If we're including WAL, and this is the main data directory we
454  * don't terminate the tar stream here. Instead, we will append
455  * the xlog files below and terminate it then. This is safe since
456  * the main data directory is always sent *last*.
457  */
458  if (opt->includewal && ti->path == NULL)
459  {
460  Assert(lnext(tablespaces, lc) == NULL);
461  }
462  else
463  pq_putemptymessage('c'); /* CopyDone */
464 
465  tblspc_streamed++;
467  tblspc_streamed);
468  }
469 
472  endptr = do_pg_stop_backup(labelfile->data, !opt->nowait, &endtli);
473  }
475 
476 
477  if (opt->includewal)
478  {
479  /*
480  * We've left the last tar file "open", so we can now append the
481  * required WAL files to it.
482  */
483  char pathbuf[MAXPGPATH];
484  XLogSegNo segno;
485  XLogSegNo startsegno;
486  XLogSegNo endsegno;
487  struct stat statbuf;
488  List *historyFileList = NIL;
489  List *walFileList = NIL;
490  char firstoff[MAXFNAMELEN];
491  char lastoff[MAXFNAMELEN];
492  DIR *dir;
493  struct dirent *de;
494  ListCell *lc;
495  TimeLineID tli;
496 
499 
500  /*
501  * I'd rather not worry about timelines here, so scan pg_wal and
502  * include all WAL files in the range between 'startptr' and 'endptr',
503  * regardless of the timeline the file is stamped with. If there are
504  * some spurious WAL files belonging to timelines that don't belong in
505  * this server's history, they will be included too. Normally there
506  * shouldn't be such files, but if there are, there's little harm in
507  * including them.
508  */
509  XLByteToSeg(startptr, startsegno, wal_segment_size);
510  XLogFileName(firstoff, ThisTimeLineID, startsegno, wal_segment_size);
511  XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
512  XLogFileName(lastoff, ThisTimeLineID, endsegno, wal_segment_size);
513 
514  dir = AllocateDir("pg_wal");
515  while ((de = ReadDir(dir, "pg_wal")) != NULL)
516  {
517  /* Does it look like a WAL segment, and is it in the range? */
518  if (IsXLogFileName(de->d_name) &&
519  strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
520  strcmp(de->d_name + 8, lastoff + 8) <= 0)
521  {
522  walFileList = lappend(walFileList, pstrdup(de->d_name));
523  }
524  /* Does it look like a timeline history file? */
525  else if (IsTLHistoryFileName(de->d_name))
526  {
527  historyFileList = lappend(historyFileList, pstrdup(de->d_name));
528  }
529  }
530  FreeDir(dir);
531 
532  /*
533  * Before we go any further, check that none of the WAL segments we
534  * need were removed.
535  */
536  CheckXLogRemoved(startsegno, ThisTimeLineID);
537 
538  /*
539  * Sort the WAL filenames. We want to send the files in order from
540  * oldest to newest, to reduce the chance that a file is recycled
541  * before we get a chance to send it over.
542  */
543  list_sort(walFileList, compareWalFileNames);
544 
545  /*
546  * There must be at least one xlog file in the pg_wal directory, since
547  * we are doing backup-including-xlog.
548  */
549  if (walFileList == NIL)
550  ereport(ERROR,
551  (errmsg("could not find any WAL files")));
552 
553  /*
554  * Sanity check: the first and last segment should cover startptr and
555  * endptr, with no gaps in between.
556  */
557  XLogFromFileName((char *) linitial(walFileList),
558  &tli, &segno, wal_segment_size);
559  if (segno != startsegno)
560  {
561  char startfname[MAXFNAMELEN];
562 
563  XLogFileName(startfname, ThisTimeLineID, startsegno,
565  ereport(ERROR,
566  (errmsg("could not find WAL file \"%s\"", startfname)));
567  }
568  foreach(lc, walFileList)
569  {
570  char *walFileName = (char *) lfirst(lc);
571  XLogSegNo currsegno = segno;
572  XLogSegNo nextsegno = segno + 1;
573 
574  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
575  if (!(nextsegno == segno || currsegno == segno))
576  {
577  char nextfname[MAXFNAMELEN];
578 
579  XLogFileName(nextfname, ThisTimeLineID, nextsegno,
581  ereport(ERROR,
582  (errmsg("could not find WAL file \"%s\"", nextfname)));
583  }
584  }
585  if (segno != endsegno)
586  {
587  char endfname[MAXFNAMELEN];
588 
589  XLogFileName(endfname, ThisTimeLineID, endsegno, wal_segment_size);
590  ereport(ERROR,
591  (errmsg("could not find WAL file \"%s\"", endfname)));
592  }
593 
594  /* Ok, we have everything we need. Send the WAL files. */
595  foreach(lc, walFileList)
596  {
597  char *walFileName = (char *) lfirst(lc);
598  FILE *fp;
599  char buf[TAR_SEND_SIZE];
600  size_t cnt;
601  pgoff_t len = 0;
602 
603  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
604  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
605 
606  fp = AllocateFile(pathbuf, "rb");
607  if (fp == NULL)
608  {
609  int save_errno = errno;
610 
611  /*
612  * Most likely reason for this is that the file was already
613  * removed by a checkpoint, so check for that to get a better
614  * error message.
615  */
616  CheckXLogRemoved(segno, tli);
617 
618  errno = save_errno;
619  ereport(ERROR,
621  errmsg("could not open file \"%s\": %m", pathbuf)));
622  }
623 
624  if (fstat(fileno(fp), &statbuf) != 0)
625  ereport(ERROR,
627  errmsg("could not stat file \"%s\": %m",
628  pathbuf)));
629  if (statbuf.st_size != wal_segment_size)
630  {
631  CheckXLogRemoved(segno, tli);
632  ereport(ERROR,
634  errmsg("unexpected WAL file size \"%s\"", walFileName)));
635  }
636 
637  /* send the WAL file itself */
638  _tarWriteHeader(pathbuf, NULL, &statbuf, false);
639 
640  while ((cnt = fread(buf, 1,
641  Min(sizeof(buf), wal_segment_size - len),
642  fp)) > 0)
643  {
644  CheckXLogRemoved(segno, tli);
645  /* Send the chunk as a CopyData message */
646  if (pq_putmessage('d', buf, cnt))
647  ereport(ERROR,
648  (errmsg("base backup could not send data, aborting backup")));
650 
651  len += cnt;
652  throttle(cnt);
653 
654  if (len == wal_segment_size)
655  break;
656  }
657 
658  CHECK_FREAD_ERROR(fp, pathbuf);
659 
660  if (len != wal_segment_size)
661  {
662  CheckXLogRemoved(segno, tli);
663  ereport(ERROR,
665  errmsg("unexpected WAL file size \"%s\"", walFileName)));
666  }
667 
668  /* wal_segment_size is a multiple of 512, so no need for padding */
669 
670  FreeFile(fp);
671 
672  /*
673  * Mark file as archived, otherwise files can get archived again
674  * after promotion of a new node. This is in line with
675  * walreceiver.c always doing an XLogArchiveForceDone() after a
676  * complete segment.
677  */
678  StatusFilePath(pathbuf, walFileName, ".done");
679  sendFileWithContent(pathbuf, "", &manifest);
680  }
681 
682  /*
683  * Send timeline history files too. Only the latest timeline history
684  * file is required for recovery, and even that only if there happens
685  * to be a timeline switch in the first WAL segment that contains the
686  * checkpoint record, or if we're taking a base backup from a standby
687  * server and the target timeline changes while the backup is taken.
688  * But they are small and highly useful for debugging purposes, so
689  * better include them all, always.
690  */
691  foreach(lc, historyFileList)
692  {
693  char *fname = lfirst(lc);
694 
695  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
696 
697  if (lstat(pathbuf, &statbuf) != 0)
698  ereport(ERROR,
700  errmsg("could not stat file \"%s\": %m", pathbuf)));
701 
702  sendFile(pathbuf, pathbuf, &statbuf, false, InvalidOid,
703  &manifest, NULL);
704 
705  /* unconditionally mark file as archived */
706  StatusFilePath(pathbuf, fname, ".done");
707  sendFileWithContent(pathbuf, "", &manifest);
708  }
709 
710  /* Send CopyDone message for the last tar file */
711  pq_putemptymessage('c');
712  }
713 
714  AddWALInfoToBackupManifest(&manifest, startptr, starttli, endptr, endtli);
715 
716  SendBackupManifest(&manifest);
717 
718  SendXlogRecPtrResult(endptr, endtli);
719 
721  {
722  if (total_checksum_failures > 1)
724  (errmsg("%lld total checksum verification failures", total_checksum_failures)));
725 
726  ereport(ERROR,
728  errmsg("checksum verification failure during base backup")));
729  }
730 
731  /* clean up the resource owner we created */
732  WalSndResourceCleanup(true);
733 
735 }
736 
737 /*
738  * list_sort comparison function, to compare log/seg portion of WAL segment
739  * filenames, ignoring the timeline portion.
740  */
741 static int
743 {
744  char *fna = (char *) lfirst(a);
745  char *fnb = (char *) lfirst(b);
746 
747  return strcmp(fna + 8, fnb + 8);
748 }
749 
750 /*
751  * Parse the base backup options passed down by the parser
752  */
753 static void
755 {
756  ListCell *lopt;
757  bool o_label = false;
758  bool o_progress = false;
759  bool o_fast = false;
760  bool o_nowait = false;
761  bool o_wal = false;
762  bool o_maxrate = false;
763  bool o_tablespace_map = false;
764  bool o_noverify_checksums = false;
765  bool o_manifest = false;
766  bool o_manifest_checksums = false;
767 
768  MemSet(opt, 0, sizeof(*opt));
771 
772  foreach(lopt, options)
773  {
774  DefElem *defel = (DefElem *) lfirst(lopt);
775 
776  if (strcmp(defel->defname, "label") == 0)
777  {
778  if (o_label)
779  ereport(ERROR,
780  (errcode(ERRCODE_SYNTAX_ERROR),
781  errmsg("duplicate option \"%s\"", defel->defname)));
782  opt->label = strVal(defel->arg);
783  o_label = true;
784  }
785  else if (strcmp(defel->defname, "progress") == 0)
786  {
787  if (o_progress)
788  ereport(ERROR,
789  (errcode(ERRCODE_SYNTAX_ERROR),
790  errmsg("duplicate option \"%s\"", defel->defname)));
791  opt->progress = true;
792  o_progress = true;
793  }
794  else if (strcmp(defel->defname, "fast") == 0)
795  {
796  if (o_fast)
797  ereport(ERROR,
798  (errcode(ERRCODE_SYNTAX_ERROR),
799  errmsg("duplicate option \"%s\"", defel->defname)));
800  opt->fastcheckpoint = true;
801  o_fast = true;
802  }
803  else if (strcmp(defel->defname, "nowait") == 0)
804  {
805  if (o_nowait)
806  ereport(ERROR,
807  (errcode(ERRCODE_SYNTAX_ERROR),
808  errmsg("duplicate option \"%s\"", defel->defname)));
809  opt->nowait = true;
810  o_nowait = true;
811  }
812  else if (strcmp(defel->defname, "wal") == 0)
813  {
814  if (o_wal)
815  ereport(ERROR,
816  (errcode(ERRCODE_SYNTAX_ERROR),
817  errmsg("duplicate option \"%s\"", defel->defname)));
818  opt->includewal = true;
819  o_wal = true;
820  }
821  else if (strcmp(defel->defname, "max_rate") == 0)
822  {
823  long maxrate;
824 
825  if (o_maxrate)
826  ereport(ERROR,
827  (errcode(ERRCODE_SYNTAX_ERROR),
828  errmsg("duplicate option \"%s\"", defel->defname)));
829 
830  maxrate = intVal(defel->arg);
831  if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
832  ereport(ERROR,
833  (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
834  errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
835  (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
836 
837  opt->maxrate = (uint32) maxrate;
838  o_maxrate = true;
839  }
840  else if (strcmp(defel->defname, "tablespace_map") == 0)
841  {
842  if (o_tablespace_map)
843  ereport(ERROR,
844  (errcode(ERRCODE_SYNTAX_ERROR),
845  errmsg("duplicate option \"%s\"", defel->defname)));
846  opt->sendtblspcmapfile = true;
847  o_tablespace_map = true;
848  }
849  else if (strcmp(defel->defname, "noverify_checksums") == 0)
850  {
851  if (o_noverify_checksums)
852  ereport(ERROR,
853  (errcode(ERRCODE_SYNTAX_ERROR),
854  errmsg("duplicate option \"%s\"", defel->defname)));
855  noverify_checksums = true;
856  o_noverify_checksums = true;
857  }
858  else if (strcmp(defel->defname, "manifest") == 0)
859  {
860  char *optval = strVal(defel->arg);
861  bool manifest_bool;
862 
863  if (o_manifest)
864  ereport(ERROR,
865  (errcode(ERRCODE_SYNTAX_ERROR),
866  errmsg("duplicate option \"%s\"", defel->defname)));
867  if (parse_bool(optval, &manifest_bool))
868  {
869  if (manifest_bool)
871  else
873  }
874  else if (pg_strcasecmp(optval, "force-encode") == 0)
876  else
877  ereport(ERROR,
878  (errcode(ERRCODE_SYNTAX_ERROR),
879  errmsg("unrecognized manifest option: \"%s\"",
880  optval)));
881  o_manifest = true;
882  }
883  else if (strcmp(defel->defname, "manifest_checksums") == 0)
884  {
885  char *optval = strVal(defel->arg);
886 
887  if (o_manifest_checksums)
888  ereport(ERROR,
889  (errcode(ERRCODE_SYNTAX_ERROR),
890  errmsg("duplicate option \"%s\"", defel->defname)));
891  if (!pg_checksum_parse_type(optval,
892  &opt->manifest_checksum_type))
893  ereport(ERROR,
894  (errcode(ERRCODE_SYNTAX_ERROR),
895  errmsg("unrecognized checksum algorithm: \"%s\"",
896  optval)));
897  o_manifest_checksums = true;
898  }
899  else
900  elog(ERROR, "option \"%s\" not recognized",
901  defel->defname);
902  }
903  if (opt->label == NULL)
904  opt->label = "base backup";
905  if (opt->manifest == MANIFEST_OPTION_NO)
906  {
907  if (o_manifest_checksums)
908  ereport(ERROR,
909  (errcode(ERRCODE_SYNTAX_ERROR),
910  errmsg("manifest checksums require a backup manifest")));
912  }
913 }
914 
915 
916 /*
917  * SendBaseBackup() - send a complete base backup.
918  *
919  * The function will put the system into backup mode like pg_start_backup()
920  * does, so that the backup is consistent even though we read directly from
921  * the filesystem, bypassing the buffer cache.
922  */
923 void
925 {
926  basebackup_options opt;
927 
928  parse_basebackup_options(cmd->options, &opt);
929 
931 
933  {
934  char activitymsg[50];
935 
936  snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
937  opt.label);
938  set_ps_display(activitymsg);
939  }
940 
941  perform_base_backup(&opt);
942 }
943 
944 static void
946 {
947  char is[32];
948 
949  sprintf(is, INT64_FORMAT, intval);
950  pq_sendint32(buf, strlen(is));
951  pq_sendbytes(buf, is, strlen(is));
952 }
953 
954 static void
955 SendBackupHeader(List *tablespaces)
956 {
958  ListCell *lc;
959 
960  /* Construct and send the directory information */
961  pq_beginmessage(&buf, 'T'); /* RowDescription */
962  pq_sendint16(&buf, 3); /* 3 fields */
963 
964  /* First field - spcoid */
965  pq_sendstring(&buf, "spcoid");
966  pq_sendint32(&buf, 0); /* table oid */
967  pq_sendint16(&buf, 0); /* attnum */
968  pq_sendint32(&buf, OIDOID); /* type oid */
969  pq_sendint16(&buf, 4); /* typlen */
970  pq_sendint32(&buf, 0); /* typmod */
971  pq_sendint16(&buf, 0); /* format code */
972 
973  /* Second field - spclocation */
974  pq_sendstring(&buf, "spclocation");
975  pq_sendint32(&buf, 0);
976  pq_sendint16(&buf, 0);
977  pq_sendint32(&buf, TEXTOID);
978  pq_sendint16(&buf, -1);
979  pq_sendint32(&buf, 0);
980  pq_sendint16(&buf, 0);
981 
982  /* Third field - size */
983  pq_sendstring(&buf, "size");
984  pq_sendint32(&buf, 0);
985  pq_sendint16(&buf, 0);
986  pq_sendint32(&buf, INT8OID);
987  pq_sendint16(&buf, 8);
988  pq_sendint32(&buf, 0);
989  pq_sendint16(&buf, 0);
990  pq_endmessage(&buf);
991 
992  foreach(lc, tablespaces)
993  {
994  tablespaceinfo *ti = lfirst(lc);
995 
996  /* Send one datarow message */
997  pq_beginmessage(&buf, 'D');
998  pq_sendint16(&buf, 3); /* number of columns */
999  if (ti->path == NULL)
1000  {
1001  pq_sendint32(&buf, -1); /* Length = -1 ==> NULL */
1002  pq_sendint32(&buf, -1);
1003  }
1004  else
1005  {
1006  Size len;
1007 
1008  len = strlen(ti->oid);
1009  pq_sendint32(&buf, len);
1010  pq_sendbytes(&buf, ti->oid, len);
1011 
1012  len = strlen(ti->path);
1013  pq_sendint32(&buf, len);
1014  pq_sendbytes(&buf, ti->path, len);
1015  }
1016  if (ti->size >= 0)
1017  send_int8_string(&buf, ti->size / 1024);
1018  else
1019  pq_sendint32(&buf, -1); /* NULL */
1020 
1021  pq_endmessage(&buf);
1022  }
1023 
1024  /* Send a CommandComplete message */
1025  pq_puttextmessage('C', "SELECT");
1026 }
1027 
1028 /*
1029  * Send a single resultset containing just a single
1030  * XLogRecPtr record (in text format)
1031  */
1032 static void
1034 {
1036  char str[MAXFNAMELEN];
1037  Size len;
1038 
1039  pq_beginmessage(&buf, 'T'); /* RowDescription */
1040  pq_sendint16(&buf, 2); /* 2 fields */
1041 
1042  /* Field headers */
1043  pq_sendstring(&buf, "recptr");
1044  pq_sendint32(&buf, 0); /* table oid */
1045  pq_sendint16(&buf, 0); /* attnum */
1046  pq_sendint32(&buf, TEXTOID); /* type oid */
1047  pq_sendint16(&buf, -1);
1048  pq_sendint32(&buf, 0);
1049  pq_sendint16(&buf, 0);
1050 
1051  pq_sendstring(&buf, "tli");
1052  pq_sendint32(&buf, 0); /* table oid */
1053  pq_sendint16(&buf, 0); /* attnum */
1054 
1055  /*
1056  * int8 may seem like a surprising data type for this, but in theory int4
1057  * would not be wide enough for this, as TimeLineID is unsigned.
1058  */
1059  pq_sendint32(&buf, INT8OID); /* type oid */
1060  pq_sendint16(&buf, -1);
1061  pq_sendint32(&buf, 0);
1062  pq_sendint16(&buf, 0);
1063  pq_endmessage(&buf);
1064 
1065  /* Data row */
1066  pq_beginmessage(&buf, 'D');
1067  pq_sendint16(&buf, 2); /* number of columns */
1068 
1069  len = snprintf(str, sizeof(str),
1070  "%X/%X", (uint32) (ptr >> 32), (uint32) ptr);
1071  pq_sendint32(&buf, len);
1072  pq_sendbytes(&buf, str, len);
1073 
1074  len = snprintf(str, sizeof(str), "%u", tli);
1075  pq_sendint32(&buf, len);
1076  pq_sendbytes(&buf, str, len);
1077 
1078  pq_endmessage(&buf);
1079 
1080  /* Send a CommandComplete message */
1081  pq_puttextmessage('C', "SELECT");
1082 }
1083 
1084 /*
1085  * Inject a file with given name and content in the output tar stream.
1086  */
1087 static void
1088 sendFileWithContent(const char *filename, const char *content,
1090 {
1091  struct stat statbuf;
1092  int pad,
1093  len;
1094  pg_checksum_context checksum_ctx;
1095 
1096  pg_checksum_init(&checksum_ctx, manifest->checksum_type);
1097 
1098  len = strlen(content);
1099 
1100  /*
1101  * Construct a stat struct for the backup_label file we're injecting in
1102  * the tar.
1103  */
1104  /* Windows doesn't have the concept of uid and gid */
1105 #ifdef WIN32
1106  statbuf.st_uid = 0;
1107  statbuf.st_gid = 0;
1108 #else
1109  statbuf.st_uid = geteuid();
1110  statbuf.st_gid = getegid();
1111 #endif
1112  statbuf.st_mtime = time(NULL);
1113  statbuf.st_mode = pg_file_create_mode;
1114  statbuf.st_size = len;
1115 
1116  _tarWriteHeader(filename, NULL, &statbuf, false);
1117  /* Send the contents as a CopyData message */
1118  pq_putmessage('d', content, len);
1120 
1121  /* Pad to 512 byte boundary, per tar format requirements */
1122  pad = ((len + 511) & ~511) - len;
1123  if (pad > 0)
1124  {
1125  char buf[512];
1126 
1127  MemSet(buf, 0, pad);
1128  pq_putmessage('d', buf, pad);
1130  }
1131 
1132  pg_checksum_update(&checksum_ctx, (uint8 *) content, len);
1133  AddFileToBackupManifest(manifest, NULL, filename, len,
1134  (pg_time_t) statbuf.st_mtime, &checksum_ctx);
1135 }
1136 
1137 /*
1138  * Include the tablespace directory pointed to by 'path' in the output tar
1139  * stream. If 'sizeonly' is true, we just calculate a total length and return
1140  * it, without actually sending anything.
1141  *
1142  * Only used to send auxiliary tablespaces, not PGDATA.
1143  */
1144 int64
1145 sendTablespace(char *path, char *spcoid, bool sizeonly,
1147 {
1148  int64 size;
1149  char pathbuf[MAXPGPATH];
1150  struct stat statbuf;
1151 
1152  /*
1153  * 'path' points to the tablespace location, but we only want to include
1154  * the version directory in it that belongs to us.
1155  */
1156  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1158 
1159  /*
1160  * Store a directory entry in the tar file so we get the permissions
1161  * right.
1162  */
1163  if (lstat(pathbuf, &statbuf) != 0)
1164  {
1165  if (errno != ENOENT)
1166  ereport(ERROR,
1168  errmsg("could not stat file or directory \"%s\": %m",
1169  pathbuf)));
1170 
1171  /* If the tablespace went away while scanning, it's no error. */
1172  return 0;
1173  }
1174 
1175  size = _tarWriteHeader(TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1176  sizeonly);
1177 
1178  /* Send all the files in the tablespace version directory */
1179  size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1180  spcoid);
1181 
1182  return size;
1183 }
1184 
1185 /*
1186  * Include all files from the given directory in the output tar stream. If
1187  * 'sizeonly' is true, we just calculate a total length and return it, without
1188  * actually sending anything.
1189  *
1190  * Omit any directory in the tablespaces list, to avoid backing up
1191  * tablespaces twice when they were created inside PGDATA.
1192  *
1193  * If sendtblspclinks is true, we need to include symlink
1194  * information in the tar file. If not, we can skip that
1195  * as it will be sent separately in the tablespace_map file.
1196  */
1197 static int64
1198 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces,
1199  bool sendtblspclinks, backup_manifest_info *manifest,
1200  const char *spcoid)
1201 {
1202  DIR *dir;
1203  struct dirent *de;
1204  char pathbuf[MAXPGPATH * 2];
1205  struct stat statbuf;
1206  int64 size = 0;
1207  const char *lastDir; /* Split last dir from parent path. */
1208  bool isDbDir = false; /* Does this directory contain relations? */
1209 
1210  /*
1211  * Determine if the current path is a database directory that can contain
1212  * relations.
1213  *
1214  * Start by finding the location of the delimiter between the parent path
1215  * and the current path.
1216  */
1217  lastDir = last_dir_separator(path);
1218 
1219  /* Does this path look like a database path (i.e. all digits)? */
1220  if (lastDir != NULL &&
1221  strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1222  {
1223  /* Part of path that contains the parent directory. */
1224  int parentPathLen = lastDir - path;
1225 
1226  /*
1227  * Mark path as a database directory if the parent path is either
1228  * $PGDATA/base or a tablespace version path.
1229  */
1230  if (strncmp(path, "./base", parentPathLen) == 0 ||
1231  (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1232  strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1234  sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1235  isDbDir = true;
1236  }
1237 
1238  dir = AllocateDir(path);
1239  while ((de = ReadDir(dir, path)) != NULL)
1240  {
1241  int excludeIdx;
1242  bool excludeFound;
1243  ForkNumber relForkNum; /* Type of fork if file is a relation */
1244  int relOidChars; /* Chars in filename that are the rel oid */
1245 
1246  /* Skip special stuff */
1247  if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1248  continue;
1249 
1250  /* Skip temporary files */
1251  if (strncmp(de->d_name,
1253  strlen(PG_TEMP_FILE_PREFIX)) == 0)
1254  continue;
1255 
1256  /*
1257  * Check if the postmaster has signaled us to exit, and abort with an
1258  * error in that case. The error handler further up will call
1259  * do_pg_abort_backup() for us. Also check that if the backup was
1260  * started while still in recovery, the server wasn't promoted.
1261  * do_pg_stop_backup() will check that too, but it's better to stop
1262  * the backup early than continue to the end and fail there.
1263  */
1266  ereport(ERROR,
1267  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1268  errmsg("the standby was promoted during online backup"),
1269  errhint("This means that the backup being taken is corrupt "
1270  "and should not be used. "
1271  "Try taking another online backup.")));
1272 
1273  /* Scan for files that should be excluded */
1274  excludeFound = false;
1275  for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1276  {
1277  int cmplen = strlen(excludeFiles[excludeIdx].name);
1278 
1279  if (!excludeFiles[excludeIdx].match_prefix)
1280  cmplen++;
1281  if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
1282  {
1283  elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1284  excludeFound = true;
1285  break;
1286  }
1287  }
1288 
1289  if (excludeFound)
1290  continue;
1291 
1292  /* Exclude all forks for unlogged tables except the init fork */
1293  if (isDbDir &&
1294  parse_filename_for_nontemp_relation(de->d_name, &relOidChars,
1295  &relForkNum))
1296  {
1297  /* Never exclude init forks */
1298  if (relForkNum != INIT_FORKNUM)
1299  {
1300  char initForkFile[MAXPGPATH];
1301  char relOid[OIDCHARS + 1];
1302 
1303  /*
1304  * If any other type of fork, check if there is an init fork
1305  * with the same OID. If so, the file can be excluded.
1306  */
1307  memcpy(relOid, de->d_name, relOidChars);
1308  relOid[relOidChars] = '\0';
1309  snprintf(initForkFile, sizeof(initForkFile), "%s/%s_init",
1310  path, relOid);
1311 
1312  if (lstat(initForkFile, &statbuf) == 0)
1313  {
1314  elog(DEBUG2,
1315  "unlogged relation file \"%s\" excluded from backup",
1316  de->d_name);
1317 
1318  continue;
1319  }
1320  }
1321  }
1322 
1323  /* Exclude temporary relations */
1324  if (isDbDir && looks_like_temp_rel_name(de->d_name))
1325  {
1326  elog(DEBUG2,
1327  "temporary relation file \"%s\" excluded from backup",
1328  de->d_name);
1329 
1330  continue;
1331  }
1332 
1333  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1334 
1335  /* Skip pg_control here to back up it last */
1336  if (strcmp(pathbuf, "./global/pg_control") == 0)
1337  continue;
1338 
1339  if (lstat(pathbuf, &statbuf) != 0)
1340  {
1341  if (errno != ENOENT)
1342  ereport(ERROR,
1344  errmsg("could not stat file or directory \"%s\": %m",
1345  pathbuf)));
1346 
1347  /* If the file went away while scanning, it's not an error. */
1348  continue;
1349  }
1350 
1351  /* Scan for directories whose contents should be excluded */
1352  excludeFound = false;
1353  for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1354  {
1355  if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1356  {
1357  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1358  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1359  excludeFound = true;
1360  break;
1361  }
1362  }
1363 
1364  if (excludeFound)
1365  continue;
1366 
1367  /*
1368  * Exclude contents of directory specified by statrelpath if not set
1369  * to the default (pg_stat_tmp) which is caught in the loop above.
1370  */
1371  if (statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0)
1372  {
1373  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath);
1374  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1375  continue;
1376  }
1377 
1378  /*
1379  * We can skip pg_wal, the WAL segments need to be fetched from the
1380  * WAL archive anyway. But include it as an empty directory anyway, so
1381  * we get permissions right.
1382  */
1383  if (strcmp(pathbuf, "./pg_wal") == 0)
1384  {
1385  /* If pg_wal is a symlink, write it as a directory anyway */
1386  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1387 
1388  /*
1389  * Also send archive_status directory (by hackishly reusing
1390  * statbuf from above ...).
1391  */
1392  size += _tarWriteHeader("./pg_wal/archive_status", NULL, &statbuf,
1393  sizeonly);
1394 
1395  continue; /* don't recurse into pg_wal */
1396  }
1397 
1398  /* Allow symbolic links in pg_tblspc only */
1399  if (strcmp(path, "./pg_tblspc") == 0 &&
1400 #ifndef WIN32
1401  S_ISLNK(statbuf.st_mode)
1402 #else
1403  pgwin32_is_junction(pathbuf)
1404 #endif
1405  )
1406  {
1407 #if defined(HAVE_READLINK) || defined(WIN32)
1408  char linkpath[MAXPGPATH];
1409  int rllen;
1410 
1411  rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1412  if (rllen < 0)
1413  ereport(ERROR,
1415  errmsg("could not read symbolic link \"%s\": %m",
1416  pathbuf)));
1417  if (rllen >= sizeof(linkpath))
1418  ereport(ERROR,
1419  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1420  errmsg("symbolic link \"%s\" target is too long",
1421  pathbuf)));
1422  linkpath[rllen] = '\0';
1423 
1424  size += _tarWriteHeader(pathbuf + basepathlen + 1, linkpath,
1425  &statbuf, sizeonly);
1426 #else
1427 
1428  /*
1429  * If the platform does not have symbolic links, it should not be
1430  * possible to have tablespaces - clearly somebody else created
1431  * them. Warn about it and ignore.
1432  */
1433  ereport(WARNING,
1434  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1435  errmsg("tablespaces are not supported on this platform")));
1436  continue;
1437 #endif /* HAVE_READLINK */
1438  }
1439  else if (S_ISDIR(statbuf.st_mode))
1440  {
1441  bool skip_this_dir = false;
1442  ListCell *lc;
1443 
1444  /*
1445  * Store a directory entry in the tar file so we can get the
1446  * permissions right.
1447  */
1448  size += _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf,
1449  sizeonly);
1450 
1451  /*
1452  * Call ourselves recursively for a directory, unless it happens
1453  * to be a separate tablespace located within PGDATA.
1454  */
1455  foreach(lc, tablespaces)
1456  {
1457  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1458 
1459  /*
1460  * ti->rpath is the tablespace relative path within PGDATA, or
1461  * NULL if the tablespace has been properly located somewhere
1462  * else.
1463  *
1464  * Skip past the leading "./" in pathbuf when comparing.
1465  */
1466  if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1467  {
1468  skip_this_dir = true;
1469  break;
1470  }
1471  }
1472 
1473  /*
1474  * skip sending directories inside pg_tblspc, if not required.
1475  */
1476  if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1477  skip_this_dir = true;
1478 
1479  if (!skip_this_dir)
1480  size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces,
1481  sendtblspclinks, manifest, spcoid);
1482  }
1483  else if (S_ISREG(statbuf.st_mode))
1484  {
1485  bool sent = false;
1486 
1487  if (!sizeonly)
1488  sent = sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf,
1489  true, isDbDir ? atooid(lastDir + 1) : InvalidOid,
1490  manifest, spcoid);
1491 
1492  if (sent || sizeonly)
1493  {
1494  /* Add size, rounded up to 512byte block */
1495  size += ((statbuf.st_size + 511) & ~511);
1496  size += 512; /* Size of the header of the file */
1497  }
1498  }
1499  else
1500  ereport(WARNING,
1501  (errmsg("skipping special file \"%s\"", pathbuf)));
1502  }
1503  FreeDir(dir);
1504  return size;
1505 }
1506 
1507 /*
1508  * Check if a file should have its checksum validated.
1509  * We validate checksums on files in regular tablespaces
1510  * (including global and default) only, and in those there
1511  * are some files that are explicitly excluded.
1512  */
1513 static bool
1514 is_checksummed_file(const char *fullpath, const char *filename)
1515 {
1516  /* Check that the file is in a tablespace */
1517  if (strncmp(fullpath, "./global/", 9) == 0 ||
1518  strncmp(fullpath, "./base/", 7) == 0 ||
1519  strncmp(fullpath, "/", 1) == 0)
1520  {
1521  int excludeIdx;
1522 
1523  /* Compare file against noChecksumFiles skip list */
1524  for (excludeIdx = 0; noChecksumFiles[excludeIdx].name != NULL; excludeIdx++)
1525  {
1526  int cmplen = strlen(noChecksumFiles[excludeIdx].name);
1527 
1528  if (!noChecksumFiles[excludeIdx].match_prefix)
1529  cmplen++;
1530  if (strncmp(filename, noChecksumFiles[excludeIdx].name,
1531  cmplen) == 0)
1532  return false;
1533  }
1534 
1535  return true;
1536  }
1537  else
1538  return false;
1539 }
1540 
1541 /*****
1542  * Functions for handling tar file format
1543  *
1544  * Copied from pg_dump, but modified to work with libpq for sending
1545  */
1546 
1547 
1548 /*
1549  * Given the member, write the TAR header & send the file.
1550  *
1551  * If 'missing_ok' is true, will not throw an error if the file is not found.
1552  *
1553  * If dboid is anything other than InvalidOid then any checksum failures detected
1554  * will get reported to the stats collector.
1555  *
1556  * Returns true if the file was successfully sent, false if 'missing_ok',
1557  * and the file did not exist.
1558  */
1559 static bool
1560 sendFile(const char *readfilename, const char *tarfilename,
1561  struct stat *statbuf, bool missing_ok, Oid dboid,
1562  backup_manifest_info *manifest, const char *spcoid)
1563 {
1564  FILE *fp;
1565  BlockNumber blkno = 0;
1566  bool block_retry = false;
1567  char buf[TAR_SEND_SIZE];
1568  uint16 checksum;
1569  int checksum_failures = 0;
1570  off_t cnt;
1571  int i;
1572  pgoff_t len = 0;
1573  char *page;
1574  size_t pad;
1575  PageHeader phdr;
1576  int segmentno = 0;
1577  char *segmentpath;
1578  bool verify_checksum = false;
1579  pg_checksum_context checksum_ctx;
1580 
1581  pg_checksum_init(&checksum_ctx, manifest->checksum_type);
1582 
1583  fp = AllocateFile(readfilename, "rb");
1584  if (fp == NULL)
1585  {
1586  if (errno == ENOENT && missing_ok)
1587  return false;
1588  ereport(ERROR,
1590  errmsg("could not open file \"%s\": %m", readfilename)));
1591  }
1592 
1593  _tarWriteHeader(tarfilename, NULL, statbuf, false);
1594 
1596  {
1597  char *filename;
1598 
1599  /*
1600  * Get the filename (excluding path). As last_dir_separator()
1601  * includes the last directory separator, we chop that off by
1602  * incrementing the pointer.
1603  */
1604  filename = last_dir_separator(readfilename) + 1;
1605 
1606  if (is_checksummed_file(readfilename, filename))
1607  {
1608  verify_checksum = true;
1609 
1610  /*
1611  * Cut off at the segment boundary (".") to get the segment number
1612  * in order to mix it into the checksum.
1613  */
1614  segmentpath = strstr(filename, ".");
1615  if (segmentpath != NULL)
1616  {
1617  segmentno = atoi(segmentpath + 1);
1618  if (segmentno == 0)
1619  ereport(ERROR,
1620  (errmsg("invalid segment number %d in file \"%s\"",
1621  segmentno, filename)));
1622  }
1623  }
1624  }
1625 
1626  while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len), fp)) > 0)
1627  {
1628  /*
1629  * The checksums are verified at block level, so we iterate over the
1630  * buffer in chunks of BLCKSZ, after making sure that
1631  * TAR_SEND_SIZE/buf is divisible by BLCKSZ and we read a multiple of
1632  * BLCKSZ bytes.
1633  */
1634  Assert(TAR_SEND_SIZE % BLCKSZ == 0);
1635 
1636  if (verify_checksum && (cnt % BLCKSZ != 0))
1637  {
1638  ereport(WARNING,
1639  (errmsg("could not verify checksum in file \"%s\", block "
1640  "%d: read buffer size %d and page size %d "
1641  "differ",
1642  readfilename, blkno, (int) cnt, BLCKSZ)));
1643  verify_checksum = false;
1644  }
1645 
1646  if (verify_checksum)
1647  {
1648  for (i = 0; i < cnt / BLCKSZ; i++)
1649  {
1650  page = buf + BLCKSZ * i;
1651 
1652  /*
1653  * Only check pages which have not been modified since the
1654  * start of the base backup. Otherwise, they might have been
1655  * written only halfway and the checksum would not be valid.
1656  * However, replaying WAL would reinstate the correct page in
1657  * this case. We also skip completely new pages, since they
1658  * don't have a checksum yet.
1659  */
1660  if (!PageIsNew(page) && PageGetLSN(page) < startptr)
1661  {
1662  checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE);
1663  phdr = (PageHeader) page;
1664  if (phdr->pd_checksum != checksum)
1665  {
1666  /*
1667  * Retry the block on the first failure. It's
1668  * possible that we read the first 4K page of the
1669  * block just before postgres updated the entire block
1670  * so it ends up looking torn to us. We only need to
1671  * retry once because the LSN should be updated to
1672  * something we can ignore on the next pass. If the
1673  * error happens again then it is a true validation
1674  * failure.
1675  */
1676  if (block_retry == false)
1677  {
1678  /* Reread the failed block */
1679  if (fseek(fp, -(cnt - BLCKSZ * i), SEEK_CUR) == -1)
1680  {
1681  ereport(ERROR,
1683  errmsg("could not fseek in file \"%s\": %m",
1684  readfilename)));
1685  }
1686 
1687  if (fread(buf + BLCKSZ * i, 1, BLCKSZ, fp) != BLCKSZ)
1688  {
1689  /*
1690  * If we hit end-of-file, a concurrent
1691  * truncation must have occurred, so break out
1692  * of this loop just as if the initial fread()
1693  * returned 0. We'll drop through to the same
1694  * code that handles that case. (We must fix
1695  * up cnt first, though.)
1696  */
1697  if (feof(fp))
1698  {
1699  cnt = BLCKSZ * i;
1700  break;
1701  }
1702 
1703  ereport(ERROR,
1705  errmsg("could not reread block %d of file \"%s\": %m",
1706  blkno, readfilename)));
1707  }
1708 
1709  if (fseek(fp, cnt - BLCKSZ * i - BLCKSZ, SEEK_CUR) == -1)
1710  {
1711  ereport(ERROR,
1713  errmsg("could not fseek in file \"%s\": %m",
1714  readfilename)));
1715  }
1716 
1717  /* Set flag so we know a retry was attempted */
1718  block_retry = true;
1719 
1720  /* Reset loop to validate the block again */
1721  i--;
1722  continue;
1723  }
1724 
1725  checksum_failures++;
1726 
1727  if (checksum_failures <= 5)
1728  ereport(WARNING,
1729  (errmsg("checksum verification failed in "
1730  "file \"%s\", block %d: calculated "
1731  "%X but expected %X",
1732  readfilename, blkno, checksum,
1733  phdr->pd_checksum)));
1734  if (checksum_failures == 5)
1735  ereport(WARNING,
1736  (errmsg("further checksum verification "
1737  "failures in file \"%s\" will not "
1738  "be reported", readfilename)));
1739  }
1740  }
1741  block_retry = false;
1742  blkno++;
1743  }
1744  }
1745 
1746  /* Send the chunk as a CopyData message */
1747  if (pq_putmessage('d', buf, cnt))
1748  ereport(ERROR,
1749  (errmsg("base backup could not send data, aborting backup")));
1751 
1752  /* Also feed it to the checksum machinery. */
1753  pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt);
1754 
1755  len += cnt;
1756  throttle(cnt);
1757 
1758  if (feof(fp) || len >= statbuf->st_size)
1759  {
1760  /*
1761  * Reached end of file. The file could be longer, if it was
1762  * extended while we were sending it, but for a base backup we can
1763  * ignore such extended data. It will be restored from WAL.
1764  */
1765  break;
1766  }
1767  }
1768 
1769  CHECK_FREAD_ERROR(fp, readfilename);
1770 
1771  /* If the file was truncated while we were sending it, pad it with zeros */
1772  if (len < statbuf->st_size)
1773  {
1774  MemSet(buf, 0, sizeof(buf));
1775  while (len < statbuf->st_size)
1776  {
1777  cnt = Min(sizeof(buf), statbuf->st_size - len);
1778  pq_putmessage('d', buf, cnt);
1779  pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt);
1781  len += cnt;
1782  throttle(cnt);
1783  }
1784  }
1785 
1786  /*
1787  * Pad to 512 byte boundary, per tar format requirements. (This small
1788  * piece of data is probably not worth throttling, and is not checksummed
1789  * because it's not actually part of the file.)
1790  */
1791  pad = ((len + 511) & ~511) - len;
1792  if (pad > 0)
1793  {
1794  MemSet(buf, 0, pad);
1795  pq_putmessage('d', buf, pad);
1797  }
1798 
1799  FreeFile(fp);
1800 
1801  if (checksum_failures > 1)
1802  {
1803  ereport(WARNING,
1804  (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1805  "file \"%s\" has a total of %d checksum verification failures",
1806  checksum_failures,
1807  readfilename, checksum_failures)));
1808 
1809  pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1810  }
1811 
1812  total_checksum_failures += checksum_failures;
1813 
1814  AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
1815  (pg_time_t) statbuf->st_mtime, &checksum_ctx);
1816 
1817  return true;
1818 }
1819 
1820 
1821 static int64
1822 _tarWriteHeader(const char *filename, const char *linktarget,
1823  struct stat *statbuf, bool sizeonly)
1824 {
1825  char h[512];
1826  enum tarError rc;
1827 
1828  if (!sizeonly)
1829  {
1830  rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size,
1831  statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
1832  statbuf->st_mtime);
1833 
1834  switch (rc)
1835  {
1836  case TAR_OK:
1837  break;
1838  case TAR_NAME_TOO_LONG:
1839  ereport(ERROR,
1840  (errmsg("file name too long for tar format: \"%s\"",
1841  filename)));
1842  break;
1843  case TAR_SYMLINK_TOO_LONG:
1844  ereport(ERROR,
1845  (errmsg("symbolic link target too long for tar format: "
1846  "file name \"%s\", target \"%s\"",
1847  filename, linktarget)));
1848  break;
1849  default:
1850  elog(ERROR, "unrecognized tar error: %d", rc);
1851  }
1852 
1853  pq_putmessage('d', h, sizeof(h));
1854  update_basebackup_progress(sizeof(h));
1855  }
1856 
1857  return sizeof(h);
1858 }
1859 
1860 /*
1861  * Write tar header for a directory. If the entry in statbuf is a link then
1862  * write it as a directory anyway.
1863  */
1864 static int64
1865 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf,
1866  bool sizeonly)
1867 {
1868  /* If symlink, write it as a directory anyway */
1869 #ifndef WIN32
1870  if (S_ISLNK(statbuf->st_mode))
1871 #else
1872  if (pgwin32_is_junction(pathbuf))
1873 #endif
1874  statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
1875 
1876  return _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf, sizeonly);
1877 }
1878 
1879 /*
1880  * Increment the network transfer counter by the given number of bytes,
1881  * and sleep if necessary to comply with the requested network transfer
1882  * rate.
1883  */
1884 static void
1885 throttle(size_t increment)
1886 {
1887  TimeOffset elapsed_min;
1888 
1889  if (throttling_counter < 0)
1890  return;
1891 
1892  throttling_counter += increment;
1894  return;
1895 
1896  /* How much time should have elapsed at minimum? */
1897  elapsed_min = elapsed_min_unit *
1899 
1900  /*
1901  * Since the latch could be set repeatedly because of concurrently WAL
1902  * activity, sleep in a loop to ensure enough time has passed.
1903  */
1904  for (;;)
1905  {
1906  TimeOffset elapsed,
1907  sleep;
1908  int wait_result;
1909 
1910  /* Time elapsed since the last measurement (and possible wake up). */
1911  elapsed = GetCurrentTimestamp() - throttled_last;
1912 
1913  /* sleep if the transfer is faster than it should be */
1914  sleep = elapsed_min - elapsed;
1915  if (sleep <= 0)
1916  break;
1917 
1919 
1920  /* We're eating a potentially set latch, so check for interrupts */
1922 
1923  /*
1924  * (TAR_SEND_SIZE / throttling_sample * elapsed_min_unit) should be
1925  * the maximum time to sleep. Thus the cast to long is safe.
1926  */
1927  wait_result = WaitLatch(MyLatch,
1929  (long) (sleep / 1000),
1931 
1932  if (wait_result & WL_LATCH_SET)
1934 
1935  /* Done waiting? */
1936  if (wait_result & WL_TIMEOUT)
1937  break;
1938  }
1939 
1940  /*
1941  * As we work with integers, only whole multiple of throttling_sample was
1942  * processed. The rest will be done during the next call of this function.
1943  */
1945 
1946  /*
1947  * Time interval for the remaining amount and possible next increments
1948  * starts now.
1949  */
1951 }
1952 
1953 /*
1954  * Increment the counter for the amount of data already streamed
1955  * by the given number of bytes, and update the progress report for
1956  * pg_stat_progress_basebackup.
1957  */
1958 static void
1960 {
1961  const int index[] = {
1964  };
1965  int64 val[2];
1966  int nparam = 0;
1967 
1968  backup_streamed += delta;
1969  val[nparam++] = backup_streamed;
1970 
1971  /*
1972  * Avoid overflowing past 100% or the full size. This may make the total
1973  * size number change as we approach the end of the backup (the estimate
1974  * will always be wrong if WAL is included), but that's better than having
1975  * the done column be bigger than the total.
1976  */
1978  {
1980  val[nparam++] = backup_total;
1981  }
1982 
1983  pgstat_progress_update_multi_param(nparam, index, val);
1984 }
#define StatusFilePath(path, xlog, suffix)
List * options
Definition: replnodes.h:44
#define NIL
Definition: pg_list.h:65
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
void InitializeBackupManifest(backup_manifest_info *manifest, backup_manifest_option want_manifest, pg_checksum_type manifest_checksum_type)
int pg_file_create_mode
Definition: file_perm.c:19
#define MAX_RATE_LOWER
Definition: basebackup.h:23
#define DEBUG1
Definition: elog.h:25
int errhint(const char *fmt,...)
Definition: elog.c:1071
static void throttle(size_t increment)
Definition: basebackup.c:1885
static int64 backup_total
Definition: basebackup.c:136
uint32 TimeLineID
Definition: xlogdefs.h:52
int64 pg_time_t
Definition: pgtime.h:23
#define PROGRESS_BASEBACKUP_PHASE
Definition: progress.h:123
void do_pg_abort_backup(int code, Datum arg)
Definition: xlog.c:11415
#define WL_TIMEOUT
Definition: latch.h:127
int wal_segment_size
Definition: xlog.c:116
#define PG_DYNSHMEM_DIR
Definition: dsm_impl.h:50
static ListCell * lnext(const List *l, const ListCell *c)
Definition: pg_list.h:321
void pgstat_progress_start_command(ProgressCommandType cmdtype, Oid relid)
Definition: pgstat.c:3214
#define USECS_PER_SEC
Definition: timestamp.h:94
bool update_process_title
Definition: ps_status.c:36
int errmsg_plural(const char *fmt_singular, const char *fmt_plural, unsigned long n,...)
Definition: elog.c:934
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1583
bool DataChecksumsEnabled(void)
Definition: xlog.c:4930
int64 TimestampTz
Definition: timestamp.h:39
#define XLByteToPrevSeg(xlrp, logSegNo, wal_segsz_bytes)
static const struct exclude_list_item noChecksumFiles[]
Definition: basebackup.c:246
ResourceOwner CurrentResourceOwner
Definition: resowner.c:142
void pgstat_progress_update_param(int index, int64 val)
Definition: pgstat.c:3235
char * pstrdup(const char *in)
Definition: mcxt.c:1186
Definition: pgtar.h:17
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
char * rpath
Definition: basebackup.h:30
StringInfo makeStringInfo(void)
Definition: stringinfo.c:41
#define Min(x, y)
Definition: c.h:920
unsigned char uint8
Definition: c.h:365
static void SendBackupHeader(List *tablespaces)
Definition: basebackup.c:955
static void parse_basebackup_options(List *options, basebackup_options *opt)
Definition: basebackup.c:754
#define PROGRESS_BASEBACKUP_TBLSPC_STREAMED
Definition: progress.h:127
void pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
#define strVal(v)
Definition: value.h:54
int errcode(int sqlerrcode)
Definition: elog.c:610
#define PROGRESS_BASEBACKUP_PHASE_WAIT_WAL_ARCHIVE
Definition: progress.h:133
#define MemSet(start, val, len)
Definition: c.h:971
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:390
bool pg_checksum_parse_type(char *name, pg_checksum_type *type)
uint32 BlockNumber
Definition: block.h:31
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:197
bool parse_bool(const char *value, bool *result)
Definition: bool.c:30
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
unsigned int Oid
Definition: postgres_ext.h:31
bool RecoveryInProgress(void)
Definition: xlog.c:8069
static bool backup_started_in_recovery
Definition: basebackup.c:84
static void send_int8_string(StringInfoData *buf, int64 intval)
Definition: basebackup.c:945
Definition: dirent.h:9
void ResetLatch(Latch *latch)
Definition: latch.c:540
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
int64 sendTablespace(char *path, char *spcoid, bool sizeonly, backup_manifest_info *manifest)
Definition: basebackup.c:1145
void pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount)
Definition: pgstat.c:1585
XLogRecPtr do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p, StringInfo labelfile, List **tablespaces, StringInfo tblspcmapfile, bool infotbssize, bool needtblspcmapfile)
Definition: xlog.c:10474
int WaitLatch(Latch *latch, int wakeEvents, long timeout, uint32 wait_event_info)
Definition: latch.c:365
uint16 pd_checksum
Definition: bufpage.h:156
static void pq_sendbyte(StringInfo buf, uint8 byt)
Definition: pqformat.h:161
Definition: type.h:89
#define PROGRESS_BASEBACKUP_PHASE_WAIT_CHECKPOINT
Definition: progress.h:130
#define pgoff_t
Definition: win32_port.h:194
void set_ps_display(const char *activity)
Definition: ps_status.c:349
static void pq_sendint32(StringInfo buf, uint32 i)
Definition: pqformat.h:145
#define sprintf
Definition: port.h:195
#define TABLESPACE_MAP
Definition: xlog.h:389
#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:47
static void sendFileWithContent(const char *filename, const char *content, backup_manifest_info *manifest)
Definition: basebackup.c:1088
unsigned short uint16
Definition: c.h:366
#define linitial(l)
Definition: pg_list.h:195
static const char *const excludeDirContents[]
Definition: basebackup.c:161
#define MAX_RATE_UPPER
Definition: basebackup.h:24
Definition: dirent.c:25
void pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
tarError
Definition: pgtar.h:15
#define ERROR
Definition: elog.h:43
#define PG_TEMP_FILE_PREFIX
Definition: pg_checksums.c:59
#define IsXLogFileName(fname)
backup_manifest_option manifest
Definition: basebackup.c:57
XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive, TimeLineID *stoptli_p)
Definition: xlog.c:11002
static bool noverify_checksums
Definition: basebackup.c:130
void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)
Definition: xlog.c:3925
void SendBaseBackup(BaseBackupCmd *cmd)
Definition: basebackup.c:924
#define MAXPGPATH
#define DEBUG2
Definition: elog.h:24
#define XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes)
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:26
static int32 maxrate
#define PROGRESS_BASEBACKUP_BACKUP_STREAMED
Definition: progress.h:125
static char * buf
Definition: pg_test_fsync.c:67
uint64 XLogSegNo
Definition: xlogdefs.h:41
void SendBackupManifest(backup_manifest_info *manifest)
#define readlink(path, buf, size)
Definition: win32_port.h:222
static int64 backup_streamed
Definition: basebackup.c:139
int errcode_for_file_access(void)
Definition: elog.c:633
#define is_absolute_path(filename)
Definition: port.h:86
const char * label
Definition: basebackup.c:50
enum manifest_option backup_manifest_option
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2320
static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli)
Definition: basebackup.c:1033
unsigned int uint32
Definition: c.h:367
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1822
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2581
char * pgstat_stat_directory
Definition: pgstat.c:133
#define atooid(x)
Definition: postgres_ext.h:42
pg_checksum_type
#define CHECK_FREAD_ERROR(fp, filename)
Definition: basebackup.c:104
static char * statrelpath
Definition: basebackup.c:87
void AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr, TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)
ForkNumber
Definition: relpath.h:40
Node * arg
Definition: parsenodes.h:733
#define S_ISREG(m)
Definition: win32_port.h:299
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:45
List * lappend(List *list, void *datum)
Definition: list.c:321
#define WARNING
Definition: elog.h:40
#define stat(a, b)
Definition: win32_port.h:255
#define MAXFNAMELEN
#define PROGRESS_BASEBACKUP_BACKUP_TOTAL
Definition: progress.h:124
int pg_dir_create_mode
Definition: file_perm.c:18
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
int64 TimeOffset
Definition: timestamp.h:40
static void perform_base_backup(basebackup_options *opt)
Definition: basebackup.c:264
void * palloc0(Size size)
Definition: mcxt.c:980
void pgstat_progress_end_command(void)
Definition: pgstat.c:3286
#define XLOGDIR
#define PROGRESS_BASEBACKUP_TBLSPC_TOTAL
Definition: progress.h:126
pg_checksum_type checksum_type
#define PROGRESS_BASEBACKUP_PHASE_TRANSFER_WAL
Definition: progress.h:134
void WalSndResourceCleanup(bool isCommit)
Definition: walsender.c:347
char * last_dir_separator(const char *filename)
Definition: path.c:138
#define BoolGetDatum(X)
Definition: postgres.h:402
#define XLOG_CONTROL_FILE
#define InvalidOid
Definition: postgres_ext.h:36
TimeLineID ThisTimeLineID
Definition: xlog.c:191
static TimestampTz throttled_last
Definition: basebackup.c:121
#define RELCACHE_INIT_FILENAME
Definition: relcache.h:25
#define ereport(elevel,...)
Definition: elog.h:144
static StringInfo tblspc_map_file
Definition: xlogfuncs.c:45
static bool is_checksummed_file(const char *fullpath, const char *filename)
Definition: basebackup.c:1514
PageHeaderData * PageHeader
Definition: bufpage.h:166
static void update_basebackup_progress(int64 delta)
Definition: basebackup.c:1959
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:738
#define lfirst(lc)
Definition: pg_list.h:190
void WalSndSetState(WalSndState state)
Definition: walsender.c:3205
#define PG_STAT_TMP_DIR
Definition: pgstat.h:34
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2647
#define THROTTLING_FREQUENCY
Definition: basebackup.c:97
void pgstat_progress_update_multi_param(int nparam, const int *index, const int64 *val)
Definition: pgstat.c:3257
size_t Size
Definition: c.h:466
static bool manifest
#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)
static int list_length(const List *l)
Definition: pg_list.h:169
#define PG_AUTOCONF_FILENAME
Definition: guc.h:34
#define LOG_METAINFO_DATAFILE_TMP
Definition: syslogger.h:96
#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:52
bool looks_like_temp_rel_name(const char *name)
Definition: fd.c:3179
bool parse_filename_for_nontemp_relation(const char *name, int *oidchars, ForkNumber *fork)
Definition: reinit.c:374
#define INT64_FORMAT
Definition: c.h:409
#define S_ISDIR(m)
Definition: win32_port.h:296
#define PageGetLSN(page)
Definition: bufpage.h:366
#define lstat(path, sb)
Definition: win32_port.h:244
#define PROGRESS_BASEBACKUP_PHASE_STREAM_BACKUP
Definition: progress.h:132
const char * name
Definition: basebackup.c:149
int FreeFile(FILE *file)
Definition: fd.c:2519
#define IsTLHistoryFileName(fname)
static int compareWalFileNames(const ListCell *a, const ListCell *b)
Definition: basebackup.c:742
static char * filename
Definition: pg_dumpall.c:90
static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1198
#define intVal(v)
Definition: value.h:52
#define PageIsNew(page)
Definition: bufpage.h:229
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125
void list_sort(List *list, list_sort_comparator cmp)
Definition: list.c:1481
int errmsg(const char *fmt,...)
Definition: elog.c:824
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
#define TAR_SEND_SIZE
Definition: basebackup.c:92
#define elog(elevel,...)
Definition: elog.h:214
int i
static bool sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1560
char * DataDir
Definition: globals.c:62
struct Latch * MyLatch
Definition: globals.c:54
#define BACKUP_LABEL_FILE
Definition: xlog.h:386
void AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid, const char *pathname, size_t size, pg_time_t mtime, pg_checksum_context *checksum_ctx)
char * defname
Definition: parsenodes.h:732
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:99
pg_checksum_type manifest_checksum_type
Definition: basebackup.c:58
char d_name[MAX_PATH]
Definition: dirent.h:14
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
static int64 throttling_counter
Definition: basebackup.c:115
static const struct exclude_list_item excludeFiles[]
Definition: basebackup.c:202
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:369
Definition: pg_list.h:50
#define snprintf
Definition: port.h:193
#define WL_LATCH_SET
Definition: latch.h:124
#define OIDCHARS
Definition: relpath.h:30
static uint64 throttling_sample
Definition: basebackup.c:112
uint16 pg_checksum_page(char *page, BlockNumber blkno)
long val
Definition: informix.c:664
int FreeDir(DIR *dir)
Definition: fd.c:2699
static TimeOffset elapsed_min_unit
Definition: basebackup.c:118
#define WL_EXIT_ON_PM_DEATH
Definition: latch.h:129
static XLogRecPtr startptr
Definition: basebackup.c:124
bool pgwin32_is_junction(const char *path)
ResourceOwner ResourceOwnerCreate(ResourceOwner parent, const char *name)
Definition: resowner.c:422
static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1865
static long long int total_checksum_failures
Definition: basebackup.c:127
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)