PostgreSQL Source Code  git master
basebackup.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * basebackup.c
4  * code for taking a base backup and streaming it to a standby
5  *
6  * Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * src/backend/replication/basebackup.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 #include "postgres.h"
14 
15 #include <sys/stat.h>
16 #include <unistd.h>
17 #include <time.h>
18 
19 #include "access/xlog_internal.h" /* for pg_start/stop_backup */
20 #include "catalog/pg_type.h"
21 #include "common/file_perm.h"
22 #include "commands/progress.h"
23 #include "lib/stringinfo.h"
24 #include "libpq/libpq.h"
25 #include "libpq/pqformat.h"
26 #include "miscadmin.h"
27 #include "nodes/pg_list.h"
28 #include "pgstat.h"
29 #include "pgtar.h"
30 #include "port.h"
31 #include "postmaster/syslogger.h"
32 #include "replication/basebackup.h"
34 #include "replication/walsender.h"
36 #include "storage/bufpage.h"
37 #include "storage/checksum.h"
38 #include "storage/dsm_impl.h"
39 #include "storage/fd.h"
40 #include "storage/ipc.h"
41 #include "storage/reinit.h"
42 #include "utils/builtins.h"
43 #include "utils/ps_status.h"
44 #include "utils/relcache.h"
45 #include "utils/resowner.h"
46 #include "utils/timestamp.h"
47 
48 typedef struct
49 {
50  const char *label;
51  bool progress;
53  bool nowait;
54  bool includewal;
60 
61 static int64 sendTablespace(char *path, char *oid, bool sizeonly,
63 static int64 sendDir(const char *path, int basepathlen, bool sizeonly,
64  List *tablespaces, bool sendtblspclinks,
65  backup_manifest_info *manifest, const char *spcoid);
66 static bool sendFile(const char *readfilename, const char *tarfilename,
67  struct stat *statbuf, bool missing_ok, Oid dboid,
68  backup_manifest_info *manifest, const char *spcoid);
69 static void sendFileWithContent(const char *filename, const char *content,
71 static int64 _tarWriteHeader(const char *filename, const char *linktarget,
72  struct stat *statbuf, bool sizeonly);
73 static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf,
74  bool sizeonly);
75 static void send_int8_string(StringInfoData *buf, int64 intval);
76 static void SendBackupHeader(List *tablespaces);
77 static void perform_base_backup(basebackup_options *opt);
79 static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli);
80 static int compareWalFileNames(const ListCell *a, const ListCell *b);
81 static void throttle(size_t increment);
82 static void update_basebackup_progress(int64 delta);
83 static bool is_checksummed_file(const char *fullpath, const char *filename);
84 static int basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
85  const char *filename, bool partial_read_ok);
86 
87 /* Was the backup currently in-progress initiated in recovery mode? */
88 static bool backup_started_in_recovery = false;
89 
90 /* Relative path of temporary statistics directory */
91 static char *statrelpath = NULL;
92 
93 /*
94  * Size of each block sent into the tar stream for larger files.
95  */
96 #define TAR_SEND_SIZE 32768
97 
98 /*
99  * How frequently to throttle, as a fraction of the specified rate-second.
100  */
101 #define THROTTLING_FREQUENCY 8
102 
103 /* The actual number of bytes, transfer of which may cause sleep. */
104 static uint64 throttling_sample;
105 
106 /* Amount of data already transferred but not yet throttled. */
107 static int64 throttling_counter;
108 
109 /* The minimum time required to transfer throttling_sample bytes. */
111 
112 /* The last check of the transfer rate. */
114 
115 /* The starting XLOG position of the base backup. */
117 
118 /* Total number of checksum failures during base backup. */
119 static long long int total_checksum_failures;
120 
121 /* Do not verify checksums. */
122 static bool noverify_checksums = false;
123 
124 /*
125  * Total amount of backup data that will be streamed.
126  * -1 means that the size is not estimated.
127  */
128 static int64 backup_total = 0;
129 
130 /* Amount of backup data already streamed */
131 static int64 backup_streamed = 0;
132 
133 /*
134  * Definition of one element part of an exclusion list, used for paths part
135  * of checksum validation or base backups. "name" is the name of the file
136  * or path to check for exclusion. If "match_prefix" is true, any items
137  * matching the name as prefix are excluded.
138  */
140 {
141  const char *name;
143 };
144 
145 /*
146  * The contents of these directories are removed or recreated during server
147  * start so they are not included in backups. The directories themselves are
148  * kept and included as empty to preserve access permissions.
149  *
150  * Note: this list should be kept in sync with the filter lists in pg_rewind's
151  * filemap.c.
152  */
153 static const char *const excludeDirContents[] =
154 {
155  /*
156  * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even
157  * when stats_temp_directory is set because PGSS_TEXT_FILE is always
158  * created there.
159  */
161 
162  /*
163  * It is generally not useful to backup the contents of this directory
164  * even if the intention is to restore to another primary. See backup.sgml
165  * for a more detailed description.
166  */
167  "pg_replslot",
168 
169  /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
171 
172  /* Contents removed on startup, see AsyncShmemInit(). */
173  "pg_notify",
174 
175  /*
176  * Old contents are loaded for possible debugging but are not required for
177  * normal operation, see SerialInit().
178  */
179  "pg_serial",
180 
181  /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
182  "pg_snapshots",
183 
184  /* Contents zeroed on startup, see StartupSUBTRANS(). */
185  "pg_subtrans",
186 
187  /* end of list */
188  NULL
189 };
190 
191 /*
192  * List of files excluded from backups.
193  */
194 static const struct exclude_list_item excludeFiles[] =
195 {
196  /* Skip auto conf temporary file. */
197  {PG_AUTOCONF_FILENAME ".tmp", false},
198 
199  /* Skip current log file temporary file */
200  {LOG_METAINFO_DATAFILE_TMP, false},
201 
202  /*
203  * Skip relation cache because it is rebuilt on startup. This includes
204  * temporary files.
205  */
206  {RELCACHE_INIT_FILENAME, true},
207 
208  /*
209  * If there's a backup_label or tablespace_map file, it belongs to a
210  * backup started by the user with pg_start_backup(). It is *not* correct
211  * for this backup. Our backup_label/tablespace_map is injected into the
212  * tar separately.
213  */
214  {BACKUP_LABEL_FILE, false},
215  {TABLESPACE_MAP, false},
216 
217  /*
218  * If there's a backup_manifest, it belongs to a backup that was used to
219  * start this server. It is *not* correct for this backup. Our
220  * backup_manifest is injected into the backup separately if users want
221  * it.
222  */
223  {"backup_manifest", false},
224 
225  {"postmaster.pid", false},
226  {"postmaster.opts", false},
227 
228  /* end of list */
229  {NULL, false}
230 };
231 
232 /*
233  * List of files excluded from checksum validation.
234  *
235  * Note: this list should be kept in sync with what pg_checksums.c
236  * includes.
237  */
238 static const struct exclude_list_item noChecksumFiles[] = {
239  {"pg_control", false},
240  {"pg_filenode.map", false},
241  {"pg_internal.init", true},
242  {"PG_VERSION", false},
243 #ifdef EXEC_BACKEND
244  {"config_exec_params", true},
245 #endif
246  {NULL, false}
247 };
248 
249 /*
250  * Actually do a base backup for the specified tablespaces.
251  *
252  * This is split out mainly to avoid complaints about "variable might be
253  * clobbered by longjmp" from stupider versions of gcc.
254  */
255 static void
257 {
258  TimeLineID starttli;
259  XLogRecPtr endptr;
260  TimeLineID endtli;
261  StringInfo labelfile;
264  int datadirpathlen;
265  List *tablespaces = NIL;
266 
267  backup_total = 0;
268  backup_streamed = 0;
270 
271  /*
272  * If the estimation of the total backup size is disabled, make the
273  * backup_total column in the view return NULL by setting the parameter to
274  * -1.
275  */
276  if (!opt->progress)
277  {
278  backup_total = -1;
280  backup_total);
281  }
282 
283  /* we're going to use a BufFile, so we need a ResourceOwner */
284  Assert(CurrentResourceOwner == NULL);
285  CurrentResourceOwner = ResourceOwnerCreate(NULL, "base backup");
286 
287  datadirpathlen = strlen(DataDir);
288 
290 
291  labelfile = makeStringInfo();
292  tblspc_map_file = makeStringInfo();
293  InitializeBackupManifest(&manifest, opt->manifest,
295 
297 
300  startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
301  labelfile, &tablespaces,
302  tblspc_map_file, opt->sendtblspcmapfile);
303 
304  /*
305  * Once do_pg_start_backup has been called, ensure that any failure causes
306  * us to abort the backup so we don't "leak" a backup counter. For this
307  * reason, *all* functionality between do_pg_start_backup() and the end of
308  * do_pg_stop_backup() should be inside the error cleanup block!
309  */
310 
312  {
313  ListCell *lc;
314  tablespaceinfo *ti;
315  int tblspc_streamed = 0;
316 
317  /*
318  * Calculate the relative path of temporary statistics directory in
319  * order to skip the files which are located in that directory later.
320  */
322  strncmp(pgstat_stat_directory, DataDir, datadirpathlen) == 0)
323  statrelpath = psprintf("./%s", pgstat_stat_directory + datadirpathlen + 1);
324  else if (strncmp(pgstat_stat_directory, "./", 2) != 0)
326  else
328 
329  /* Add a node for the base directory at the end */
330  ti = palloc0(sizeof(tablespaceinfo));
331  ti->size = -1;
332  tablespaces = lappend(tablespaces, ti);
333 
334  /*
335  * Calculate the total backup size by summing up the size of each
336  * tablespace
337  */
338  if (opt->progress)
339  {
342 
343  foreach(lc, tablespaces)
344  {
345  tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
346 
347  if (tmp->path == NULL)
348  tmp->size = sendDir(".", 1, true, tablespaces, true, NULL,
349  NULL);
350  else
351  tmp->size = sendTablespace(tmp->path, tmp->oid, true,
352  NULL);
353  backup_total += tmp->size;
354  }
355  }
356 
357  /* Report that we are now streaming database files as a base backup */
358  {
359  const int index[] = {
363  };
364  const int64 val[] = {
366  backup_total, list_length(tablespaces)
367  };
368 
369  pgstat_progress_update_multi_param(3, index, val);
370  }
371 
372  /* Send the starting position of the backup */
373  SendXlogRecPtrResult(startptr, starttli);
374 
375  /* Send tablespace header */
376  SendBackupHeader(tablespaces);
377 
378  /* Setup and activate network throttling, if client requested it */
379  if (opt->maxrate > 0)
380  {
382  (int64) opt->maxrate * (int64) 1024 / THROTTLING_FREQUENCY;
383 
384  /*
385  * The minimum amount of time for throttling_sample bytes to be
386  * transferred.
387  */
389 
390  /* Enable throttling. */
391  throttling_counter = 0;
392 
393  /* The 'real data' starts now (header was ignored). */
395  }
396  else
397  {
398  /* Disable throttling. */
399  throttling_counter = -1;
400  }
401 
402  /* Send off our tablespaces one by one */
403  foreach(lc, tablespaces)
404  {
405  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
407 
408  /* Send CopyOutResponse message */
409  pq_beginmessage(&buf, 'H');
410  pq_sendbyte(&buf, 0); /* overall format */
411  pq_sendint16(&buf, 0); /* natts */
412  pq_endmessage(&buf);
413 
414  if (ti->path == NULL)
415  {
416  struct stat statbuf;
417  bool sendtblspclinks = true;
418 
419  /* In the main tar, include the backup_label first... */
421  &manifest);
422 
423  /* Then the tablespace_map file, if required... */
424  if (opt->sendtblspcmapfile)
425  {
426  sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data,
427  &manifest);
428  sendtblspclinks = false;
429  }
430 
431  /* Then the bulk of the files... */
432  sendDir(".", 1, false, tablespaces, sendtblspclinks,
433  &manifest, NULL);
434 
435  /* ... and pg_control after everything else. */
436  if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
437  ereport(ERROR,
439  errmsg("could not stat file \"%s\": %m",
442  false, InvalidOid, &manifest, NULL);
443  }
444  else
445  sendTablespace(ti->path, ti->oid, false, &manifest);
446 
447  /*
448  * If we're including WAL, and this is the main data directory we
449  * don't terminate the tar stream here. Instead, we will append
450  * the xlog files below and terminate it then. This is safe since
451  * the main data directory is always sent *last*.
452  */
453  if (opt->includewal && ti->path == NULL)
454  {
455  Assert(lnext(tablespaces, lc) == NULL);
456  }
457  else
458  pq_putemptymessage('c'); /* CopyDone */
459 
460  tblspc_streamed++;
462  tblspc_streamed);
463  }
464 
467  endptr = do_pg_stop_backup(labelfile->data, !opt->nowait, &endtli);
468  }
470 
471 
472  if (opt->includewal)
473  {
474  /*
475  * We've left the last tar file "open", so we can now append the
476  * required WAL files to it.
477  */
478  char pathbuf[MAXPGPATH];
479  XLogSegNo segno;
480  XLogSegNo startsegno;
481  XLogSegNo endsegno;
482  struct stat statbuf;
483  List *historyFileList = NIL;
484  List *walFileList = NIL;
485  char firstoff[MAXFNAMELEN];
486  char lastoff[MAXFNAMELEN];
487  DIR *dir;
488  struct dirent *de;
489  ListCell *lc;
490  TimeLineID tli;
491 
494 
495  /*
496  * I'd rather not worry about timelines here, so scan pg_wal and
497  * include all WAL files in the range between 'startptr' and 'endptr',
498  * regardless of the timeline the file is stamped with. If there are
499  * some spurious WAL files belonging to timelines that don't belong in
500  * this server's history, they will be included too. Normally there
501  * shouldn't be such files, but if there are, there's little harm in
502  * including them.
503  */
504  XLByteToSeg(startptr, startsegno, wal_segment_size);
505  XLogFileName(firstoff, ThisTimeLineID, startsegno, wal_segment_size);
506  XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
507  XLogFileName(lastoff, ThisTimeLineID, endsegno, wal_segment_size);
508 
509  dir = AllocateDir("pg_wal");
510  while ((de = ReadDir(dir, "pg_wal")) != NULL)
511  {
512  /* Does it look like a WAL segment, and is it in the range? */
513  if (IsXLogFileName(de->d_name) &&
514  strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
515  strcmp(de->d_name + 8, lastoff + 8) <= 0)
516  {
517  walFileList = lappend(walFileList, pstrdup(de->d_name));
518  }
519  /* Does it look like a timeline history file? */
520  else if (IsTLHistoryFileName(de->d_name))
521  {
522  historyFileList = lappend(historyFileList, pstrdup(de->d_name));
523  }
524  }
525  FreeDir(dir);
526 
527  /*
528  * Before we go any further, check that none of the WAL segments we
529  * need were removed.
530  */
531  CheckXLogRemoved(startsegno, ThisTimeLineID);
532 
533  /*
534  * Sort the WAL filenames. We want to send the files in order from
535  * oldest to newest, to reduce the chance that a file is recycled
536  * before we get a chance to send it over.
537  */
538  list_sort(walFileList, compareWalFileNames);
539 
540  /*
541  * There must be at least one xlog file in the pg_wal directory, since
542  * we are doing backup-including-xlog.
543  */
544  if (walFileList == NIL)
545  ereport(ERROR,
546  (errmsg("could not find any WAL files")));
547 
548  /*
549  * Sanity check: the first and last segment should cover startptr and
550  * endptr, with no gaps in between.
551  */
552  XLogFromFileName((char *) linitial(walFileList),
553  &tli, &segno, wal_segment_size);
554  if (segno != startsegno)
555  {
556  char startfname[MAXFNAMELEN];
557 
558  XLogFileName(startfname, ThisTimeLineID, startsegno,
560  ereport(ERROR,
561  (errmsg("could not find WAL file \"%s\"", startfname)));
562  }
563  foreach(lc, walFileList)
564  {
565  char *walFileName = (char *) lfirst(lc);
566  XLogSegNo currsegno = segno;
567  XLogSegNo nextsegno = segno + 1;
568 
569  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
570  if (!(nextsegno == segno || currsegno == segno))
571  {
572  char nextfname[MAXFNAMELEN];
573 
574  XLogFileName(nextfname, ThisTimeLineID, nextsegno,
576  ereport(ERROR,
577  (errmsg("could not find WAL file \"%s\"", nextfname)));
578  }
579  }
580  if (segno != endsegno)
581  {
582  char endfname[MAXFNAMELEN];
583 
584  XLogFileName(endfname, ThisTimeLineID, endsegno, wal_segment_size);
585  ereport(ERROR,
586  (errmsg("could not find WAL file \"%s\"", endfname)));
587  }
588 
589  /* Ok, we have everything we need. Send the WAL files. */
590  foreach(lc, walFileList)
591  {
592  char *walFileName = (char *) lfirst(lc);
593  int fd;
594  char buf[TAR_SEND_SIZE];
595  size_t cnt;
596  pgoff_t len = 0;
597 
598  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
599  XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
600 
601  fd = OpenTransientFile(pathbuf, O_RDONLY | PG_BINARY);
602  if (fd < 0)
603  {
604  int save_errno = errno;
605 
606  /*
607  * Most likely reason for this is that the file was already
608  * removed by a checkpoint, so check for that to get a better
609  * error message.
610  */
611  CheckXLogRemoved(segno, tli);
612 
613  errno = save_errno;
614  ereport(ERROR,
616  errmsg("could not open file \"%s\": %m", pathbuf)));
617  }
618 
619  if (fstat(fd, &statbuf) != 0)
620  ereport(ERROR,
622  errmsg("could not stat file \"%s\": %m",
623  pathbuf)));
624  if (statbuf.st_size != wal_segment_size)
625  {
626  CheckXLogRemoved(segno, tli);
627  ereport(ERROR,
629  errmsg("unexpected WAL file size \"%s\"", walFileName)));
630  }
631 
632  /* send the WAL file itself */
633  _tarWriteHeader(pathbuf, NULL, &statbuf, false);
634 
635  while ((cnt = basebackup_read_file(fd, buf,
636  Min(sizeof(buf),
637  wal_segment_size - len),
638  len, pathbuf, true)) > 0)
639  {
640  CheckXLogRemoved(segno, tli);
641  /* Send the chunk as a CopyData message */
642  if (pq_putmessage('d', buf, cnt))
643  ereport(ERROR,
644  (errmsg("base backup could not send data, aborting backup")));
646 
647  len += cnt;
648  throttle(cnt);
649 
650  if (len == wal_segment_size)
651  break;
652  }
653 
654  if (len != wal_segment_size)
655  {
656  CheckXLogRemoved(segno, tli);
657  ereport(ERROR,
659  errmsg("unexpected WAL file size \"%s\"", walFileName)));
660  }
661 
662  /*
663  * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need
664  * for padding.
665  */
667 
668  CloseTransientFile(fd);
669 
670  /*
671  * Mark file as archived, otherwise files can get archived again
672  * after promotion of a new node. This is in line with
673  * walreceiver.c always doing an XLogArchiveForceDone() after a
674  * complete segment.
675  */
676  StatusFilePath(pathbuf, walFileName, ".done");
677  sendFileWithContent(pathbuf, "", &manifest);
678  }
679 
680  /*
681  * Send timeline history files too. Only the latest timeline history
682  * file is required for recovery, and even that only if there happens
683  * to be a timeline switch in the first WAL segment that contains the
684  * checkpoint record, or if we're taking a base backup from a standby
685  * server and the target timeline changes while the backup is taken.
686  * But they are small and highly useful for debugging purposes, so
687  * better include them all, always.
688  */
689  foreach(lc, historyFileList)
690  {
691  char *fname = lfirst(lc);
692 
693  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
694 
695  if (lstat(pathbuf, &statbuf) != 0)
696  ereport(ERROR,
698  errmsg("could not stat file \"%s\": %m", pathbuf)));
699 
700  sendFile(pathbuf, pathbuf, &statbuf, false, InvalidOid,
701  &manifest, NULL);
702 
703  /* unconditionally mark file as archived */
704  StatusFilePath(pathbuf, fname, ".done");
705  sendFileWithContent(pathbuf, "", &manifest);
706  }
707 
708  /* Send CopyDone message for the last tar file */
709  pq_putemptymessage('c');
710  }
711 
712  AddWALInfoToBackupManifest(&manifest, startptr, starttli, endptr, endtli);
713 
714  SendBackupManifest(&manifest);
715 
716  SendXlogRecPtrResult(endptr, endtli);
717 
719  {
720  if (total_checksum_failures > 1)
722  (errmsg("%lld total checksum verification failures", total_checksum_failures)));
723 
724  ereport(ERROR,
726  errmsg("checksum verification failure during base backup")));
727  }
728 
729  /* clean up the resource owner we created */
730  WalSndResourceCleanup(true);
731 
733 }
734 
735 /*
736  * list_sort comparison function, to compare log/seg portion of WAL segment
737  * filenames, ignoring the timeline portion.
738  */
739 static int
741 {
742  char *fna = (char *) lfirst(a);
743  char *fnb = (char *) lfirst(b);
744 
745  return strcmp(fna + 8, fnb + 8);
746 }
747 
748 /*
749  * Parse the base backup options passed down by the parser
750  */
751 static void
753 {
754  ListCell *lopt;
755  bool o_label = false;
756  bool o_progress = false;
757  bool o_fast = false;
758  bool o_nowait = false;
759  bool o_wal = false;
760  bool o_maxrate = false;
761  bool o_tablespace_map = false;
762  bool o_noverify_checksums = false;
763  bool o_manifest = false;
764  bool o_manifest_checksums = false;
765 
766  MemSet(opt, 0, sizeof(*opt));
769 
770  foreach(lopt, options)
771  {
772  DefElem *defel = (DefElem *) lfirst(lopt);
773 
774  if (strcmp(defel->defname, "label") == 0)
775  {
776  if (o_label)
777  ereport(ERROR,
778  (errcode(ERRCODE_SYNTAX_ERROR),
779  errmsg("duplicate option \"%s\"", defel->defname)));
780  opt->label = strVal(defel->arg);
781  o_label = true;
782  }
783  else if (strcmp(defel->defname, "progress") == 0)
784  {
785  if (o_progress)
786  ereport(ERROR,
787  (errcode(ERRCODE_SYNTAX_ERROR),
788  errmsg("duplicate option \"%s\"", defel->defname)));
789  opt->progress = true;
790  o_progress = true;
791  }
792  else if (strcmp(defel->defname, "fast") == 0)
793  {
794  if (o_fast)
795  ereport(ERROR,
796  (errcode(ERRCODE_SYNTAX_ERROR),
797  errmsg("duplicate option \"%s\"", defel->defname)));
798  opt->fastcheckpoint = true;
799  o_fast = true;
800  }
801  else if (strcmp(defel->defname, "nowait") == 0)
802  {
803  if (o_nowait)
804  ereport(ERROR,
805  (errcode(ERRCODE_SYNTAX_ERROR),
806  errmsg("duplicate option \"%s\"", defel->defname)));
807  opt->nowait = true;
808  o_nowait = true;
809  }
810  else if (strcmp(defel->defname, "wal") == 0)
811  {
812  if (o_wal)
813  ereport(ERROR,
814  (errcode(ERRCODE_SYNTAX_ERROR),
815  errmsg("duplicate option \"%s\"", defel->defname)));
816  opt->includewal = true;
817  o_wal = true;
818  }
819  else if (strcmp(defel->defname, "max_rate") == 0)
820  {
821  long maxrate;
822 
823  if (o_maxrate)
824  ereport(ERROR,
825  (errcode(ERRCODE_SYNTAX_ERROR),
826  errmsg("duplicate option \"%s\"", defel->defname)));
827 
828  maxrate = intVal(defel->arg);
829  if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
830  ereport(ERROR,
831  (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
832  errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
833  (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
834 
835  opt->maxrate = (uint32) maxrate;
836  o_maxrate = true;
837  }
838  else if (strcmp(defel->defname, "tablespace_map") == 0)
839  {
840  if (o_tablespace_map)
841  ereport(ERROR,
842  (errcode(ERRCODE_SYNTAX_ERROR),
843  errmsg("duplicate option \"%s\"", defel->defname)));
844  opt->sendtblspcmapfile = true;
845  o_tablespace_map = true;
846  }
847  else if (strcmp(defel->defname, "noverify_checksums") == 0)
848  {
849  if (o_noverify_checksums)
850  ereport(ERROR,
851  (errcode(ERRCODE_SYNTAX_ERROR),
852  errmsg("duplicate option \"%s\"", defel->defname)));
853  noverify_checksums = true;
854  o_noverify_checksums = true;
855  }
856  else if (strcmp(defel->defname, "manifest") == 0)
857  {
858  char *optval = strVal(defel->arg);
859  bool manifest_bool;
860 
861  if (o_manifest)
862  ereport(ERROR,
863  (errcode(ERRCODE_SYNTAX_ERROR),
864  errmsg("duplicate option \"%s\"", defel->defname)));
865  if (parse_bool(optval, &manifest_bool))
866  {
867  if (manifest_bool)
869  else
871  }
872  else if (pg_strcasecmp(optval, "force-encode") == 0)
874  else
875  ereport(ERROR,
876  (errcode(ERRCODE_SYNTAX_ERROR),
877  errmsg("unrecognized manifest option: \"%s\"",
878  optval)));
879  o_manifest = true;
880  }
881  else if (strcmp(defel->defname, "manifest_checksums") == 0)
882  {
883  char *optval = strVal(defel->arg);
884 
885  if (o_manifest_checksums)
886  ereport(ERROR,
887  (errcode(ERRCODE_SYNTAX_ERROR),
888  errmsg("duplicate option \"%s\"", defel->defname)));
889  if (!pg_checksum_parse_type(optval,
890  &opt->manifest_checksum_type))
891  ereport(ERROR,
892  (errcode(ERRCODE_SYNTAX_ERROR),
893  errmsg("unrecognized checksum algorithm: \"%s\"",
894  optval)));
895  o_manifest_checksums = true;
896  }
897  else
898  elog(ERROR, "option \"%s\" not recognized",
899  defel->defname);
900  }
901  if (opt->label == NULL)
902  opt->label = "base backup";
903  if (opt->manifest == MANIFEST_OPTION_NO)
904  {
905  if (o_manifest_checksums)
906  ereport(ERROR,
907  (errcode(ERRCODE_SYNTAX_ERROR),
908  errmsg("manifest checksums require a backup manifest")));
910  }
911 }
912 
913 
914 /*
915  * SendBaseBackup() - send a complete base backup.
916  *
917  * The function will put the system into backup mode like pg_start_backup()
918  * does, so that the backup is consistent even though we read directly from
919  * the filesystem, bypassing the buffer cache.
920  */
921 void
923 {
924  basebackup_options opt;
925 
926  parse_basebackup_options(cmd->options, &opt);
927 
929 
931  {
932  char activitymsg[50];
933 
934  snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
935  opt.label);
936  set_ps_display(activitymsg);
937  }
938 
939  perform_base_backup(&opt);
940 }
941 
942 static void
944 {
945  char is[32];
946 
947  sprintf(is, INT64_FORMAT, intval);
948  pq_sendint32(buf, strlen(is));
949  pq_sendbytes(buf, is, strlen(is));
950 }
951 
952 static void
953 SendBackupHeader(List *tablespaces)
954 {
956  ListCell *lc;
957 
958  /* Construct and send the directory information */
959  pq_beginmessage(&buf, 'T'); /* RowDescription */
960  pq_sendint16(&buf, 3); /* 3 fields */
961 
962  /* First field - spcoid */
963  pq_sendstring(&buf, "spcoid");
964  pq_sendint32(&buf, 0); /* table oid */
965  pq_sendint16(&buf, 0); /* attnum */
966  pq_sendint32(&buf, OIDOID); /* type oid */
967  pq_sendint16(&buf, 4); /* typlen */
968  pq_sendint32(&buf, 0); /* typmod */
969  pq_sendint16(&buf, 0); /* format code */
970 
971  /* Second field - spclocation */
972  pq_sendstring(&buf, "spclocation");
973  pq_sendint32(&buf, 0);
974  pq_sendint16(&buf, 0);
975  pq_sendint32(&buf, TEXTOID);
976  pq_sendint16(&buf, -1);
977  pq_sendint32(&buf, 0);
978  pq_sendint16(&buf, 0);
979 
980  /* Third field - size */
981  pq_sendstring(&buf, "size");
982  pq_sendint32(&buf, 0);
983  pq_sendint16(&buf, 0);
984  pq_sendint32(&buf, INT8OID);
985  pq_sendint16(&buf, 8);
986  pq_sendint32(&buf, 0);
987  pq_sendint16(&buf, 0);
988  pq_endmessage(&buf);
989 
990  foreach(lc, tablespaces)
991  {
992  tablespaceinfo *ti = lfirst(lc);
993 
994  /* Send one datarow message */
995  pq_beginmessage(&buf, 'D');
996  pq_sendint16(&buf, 3); /* number of columns */
997  if (ti->path == NULL)
998  {
999  pq_sendint32(&buf, -1); /* Length = -1 ==> NULL */
1000  pq_sendint32(&buf, -1);
1001  }
1002  else
1003  {
1004  Size len;
1005 
1006  len = strlen(ti->oid);
1007  pq_sendint32(&buf, len);
1008  pq_sendbytes(&buf, ti->oid, len);
1009 
1010  len = strlen(ti->path);
1011  pq_sendint32(&buf, len);
1012  pq_sendbytes(&buf, ti->path, len);
1013  }
1014  if (ti->size >= 0)
1015  send_int8_string(&buf, ti->size / 1024);
1016  else
1017  pq_sendint32(&buf, -1); /* NULL */
1018 
1019  pq_endmessage(&buf);
1020  }
1021 
1022  /* Send a CommandComplete message */
1023  pq_puttextmessage('C', "SELECT");
1024 }
1025 
1026 /*
1027  * Send a single resultset containing just a single
1028  * XLogRecPtr record (in text format)
1029  */
1030 static void
1032 {
1034  char str[MAXFNAMELEN];
1035  Size len;
1036 
1037  pq_beginmessage(&buf, 'T'); /* RowDescription */
1038  pq_sendint16(&buf, 2); /* 2 fields */
1039 
1040  /* Field headers */
1041  pq_sendstring(&buf, "recptr");
1042  pq_sendint32(&buf, 0); /* table oid */
1043  pq_sendint16(&buf, 0); /* attnum */
1044  pq_sendint32(&buf, TEXTOID); /* type oid */
1045  pq_sendint16(&buf, -1);
1046  pq_sendint32(&buf, 0);
1047  pq_sendint16(&buf, 0);
1048 
1049  pq_sendstring(&buf, "tli");
1050  pq_sendint32(&buf, 0); /* table oid */
1051  pq_sendint16(&buf, 0); /* attnum */
1052 
1053  /*
1054  * int8 may seem like a surprising data type for this, but in theory int4
1055  * would not be wide enough for this, as TimeLineID is unsigned.
1056  */
1057  pq_sendint32(&buf, INT8OID); /* type oid */
1058  pq_sendint16(&buf, -1);
1059  pq_sendint32(&buf, 0);
1060  pq_sendint16(&buf, 0);
1061  pq_endmessage(&buf);
1062 
1063  /* Data row */
1064  pq_beginmessage(&buf, 'D');
1065  pq_sendint16(&buf, 2); /* number of columns */
1066 
1067  len = snprintf(str, sizeof(str),
1068  "%X/%X", (uint32) (ptr >> 32), (uint32) ptr);
1069  pq_sendint32(&buf, len);
1070  pq_sendbytes(&buf, str, len);
1071 
1072  len = snprintf(str, sizeof(str), "%u", tli);
1073  pq_sendint32(&buf, len);
1074  pq_sendbytes(&buf, str, len);
1075 
1076  pq_endmessage(&buf);
1077 
1078  /* Send a CommandComplete message */
1079  pq_puttextmessage('C', "SELECT");
1080 }
1081 
1082 /*
1083  * Inject a file with given name and content in the output tar stream.
1084  */
1085 static void
1086 sendFileWithContent(const char *filename, const char *content,
1088 {
1089  struct stat statbuf;
1090  int pad,
1091  len;
1092  pg_checksum_context checksum_ctx;
1093 
1094  pg_checksum_init(&checksum_ctx, manifest->checksum_type);
1095 
1096  len = strlen(content);
1097 
1098  /*
1099  * Construct a stat struct for the backup_label file we're injecting in
1100  * the tar.
1101  */
1102  /* Windows doesn't have the concept of uid and gid */
1103 #ifdef WIN32
1104  statbuf.st_uid = 0;
1105  statbuf.st_gid = 0;
1106 #else
1107  statbuf.st_uid = geteuid();
1108  statbuf.st_gid = getegid();
1109 #endif
1110  statbuf.st_mtime = time(NULL);
1111  statbuf.st_mode = pg_file_create_mode;
1112  statbuf.st_size = len;
1113 
1114  _tarWriteHeader(filename, NULL, &statbuf, false);
1115  /* Send the contents as a CopyData message */
1116  pq_putmessage('d', content, len);
1118 
1119  /* Pad to a multiple of the tar block size. */
1120  pad = tarPaddingBytesRequired(len);
1121  if (pad > 0)
1122  {
1123  char buf[TAR_BLOCK_SIZE];
1124 
1125  MemSet(buf, 0, pad);
1126  pq_putmessage('d', buf, pad);
1128  }
1129 
1130  pg_checksum_update(&checksum_ctx, (uint8 *) content, len);
1131  AddFileToBackupManifest(manifest, NULL, filename, len,
1132  (pg_time_t) statbuf.st_mtime, &checksum_ctx);
1133 }
1134 
1135 /*
1136  * Include the tablespace directory pointed to by 'path' in the output tar
1137  * stream. If 'sizeonly' is true, we just calculate a total length and return
1138  * it, without actually sending anything.
1139  *
1140  * Only used to send auxiliary tablespaces, not PGDATA.
1141  */
1142 static int64
1143 sendTablespace(char *path, char *spcoid, bool sizeonly,
1145 {
1146  int64 size;
1147  char pathbuf[MAXPGPATH];
1148  struct stat statbuf;
1149 
1150  /*
1151  * 'path' points to the tablespace location, but we only want to include
1152  * the version directory in it that belongs to us.
1153  */
1154  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1156 
1157  /*
1158  * Store a directory entry in the tar file so we get the permissions
1159  * right.
1160  */
1161  if (lstat(pathbuf, &statbuf) != 0)
1162  {
1163  if (errno != ENOENT)
1164  ereport(ERROR,
1166  errmsg("could not stat file or directory \"%s\": %m",
1167  pathbuf)));
1168 
1169  /* If the tablespace went away while scanning, it's no error. */
1170  return 0;
1171  }
1172 
1173  size = _tarWriteHeader(TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1174  sizeonly);
1175 
1176  /* Send all the files in the tablespace version directory */
1177  size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1178  spcoid);
1179 
1180  return size;
1181 }
1182 
1183 /*
1184  * Include all files from the given directory in the output tar stream. If
1185  * 'sizeonly' is true, we just calculate a total length and return it, without
1186  * actually sending anything.
1187  *
1188  * Omit any directory in the tablespaces list, to avoid backing up
1189  * tablespaces twice when they were created inside PGDATA.
1190  *
1191  * If sendtblspclinks is true, we need to include symlink
1192  * information in the tar file. If not, we can skip that
1193  * as it will be sent separately in the tablespace_map file.
1194  */
1195 static int64
1196 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces,
1197  bool sendtblspclinks, backup_manifest_info *manifest,
1198  const char *spcoid)
1199 {
1200  DIR *dir;
1201  struct dirent *de;
1202  char pathbuf[MAXPGPATH * 2];
1203  struct stat statbuf;
1204  int64 size = 0;
1205  const char *lastDir; /* Split last dir from parent path. */
1206  bool isDbDir = false; /* Does this directory contain relations? */
1207 
1208  /*
1209  * Determine if the current path is a database directory that can contain
1210  * relations.
1211  *
1212  * Start by finding the location of the delimiter between the parent path
1213  * and the current path.
1214  */
1215  lastDir = last_dir_separator(path);
1216 
1217  /* Does this path look like a database path (i.e. all digits)? */
1218  if (lastDir != NULL &&
1219  strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1220  {
1221  /* Part of path that contains the parent directory. */
1222  int parentPathLen = lastDir - path;
1223 
1224  /*
1225  * Mark path as a database directory if the parent path is either
1226  * $PGDATA/base or a tablespace version path.
1227  */
1228  if (strncmp(path, "./base", parentPathLen) == 0 ||
1229  (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1230  strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1232  sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1233  isDbDir = true;
1234  }
1235 
1236  dir = AllocateDir(path);
1237  while ((de = ReadDir(dir, path)) != NULL)
1238  {
1239  int excludeIdx;
1240  bool excludeFound;
1241  ForkNumber relForkNum; /* Type of fork if file is a relation */
1242  int relOidChars; /* Chars in filename that are the rel oid */
1243 
1244  /* Skip special stuff */
1245  if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1246  continue;
1247 
1248  /* Skip temporary files */
1249  if (strncmp(de->d_name,
1251  strlen(PG_TEMP_FILE_PREFIX)) == 0)
1252  continue;
1253 
1254  /*
1255  * Check if the postmaster has signaled us to exit, and abort with an
1256  * error in that case. The error handler further up will call
1257  * do_pg_abort_backup() for us. Also check that if the backup was
1258  * started while still in recovery, the server wasn't promoted.
1259  * do_pg_stop_backup() will check that too, but it's better to stop
1260  * the backup early than continue to the end and fail there.
1261  */
1264  ereport(ERROR,
1265  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1266  errmsg("the standby was promoted during online backup"),
1267  errhint("This means that the backup being taken is corrupt "
1268  "and should not be used. "
1269  "Try taking another online backup.")));
1270 
1271  /* Scan for files that should be excluded */
1272  excludeFound = false;
1273  for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1274  {
1275  int cmplen = strlen(excludeFiles[excludeIdx].name);
1276 
1277  if (!excludeFiles[excludeIdx].match_prefix)
1278  cmplen++;
1279  if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
1280  {
1281  elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1282  excludeFound = true;
1283  break;
1284  }
1285  }
1286 
1287  if (excludeFound)
1288  continue;
1289 
1290  /* Exclude all forks for unlogged tables except the init fork */
1291  if (isDbDir &&
1292  parse_filename_for_nontemp_relation(de->d_name, &relOidChars,
1293  &relForkNum))
1294  {
1295  /* Never exclude init forks */
1296  if (relForkNum != INIT_FORKNUM)
1297  {
1298  char initForkFile[MAXPGPATH];
1299  char relOid[OIDCHARS + 1];
1300 
1301  /*
1302  * If any other type of fork, check if there is an init fork
1303  * with the same OID. If so, the file can be excluded.
1304  */
1305  memcpy(relOid, de->d_name, relOidChars);
1306  relOid[relOidChars] = '\0';
1307  snprintf(initForkFile, sizeof(initForkFile), "%s/%s_init",
1308  path, relOid);
1309 
1310  if (lstat(initForkFile, &statbuf) == 0)
1311  {
1312  elog(DEBUG2,
1313  "unlogged relation file \"%s\" excluded from backup",
1314  de->d_name);
1315 
1316  continue;
1317  }
1318  }
1319  }
1320 
1321  /* Exclude temporary relations */
1322  if (isDbDir && looks_like_temp_rel_name(de->d_name))
1323  {
1324  elog(DEBUG2,
1325  "temporary relation file \"%s\" excluded from backup",
1326  de->d_name);
1327 
1328  continue;
1329  }
1330 
1331  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1332 
1333  /* Skip pg_control here to back up it last */
1334  if (strcmp(pathbuf, "./global/pg_control") == 0)
1335  continue;
1336 
1337  if (lstat(pathbuf, &statbuf) != 0)
1338  {
1339  if (errno != ENOENT)
1340  ereport(ERROR,
1342  errmsg("could not stat file or directory \"%s\": %m",
1343  pathbuf)));
1344 
1345  /* If the file went away while scanning, it's not an error. */
1346  continue;
1347  }
1348 
1349  /* Scan for directories whose contents should be excluded */
1350  excludeFound = false;
1351  for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1352  {
1353  if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1354  {
1355  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1356  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1357  excludeFound = true;
1358  break;
1359  }
1360  }
1361 
1362  if (excludeFound)
1363  continue;
1364 
1365  /*
1366  * Exclude contents of directory specified by statrelpath if not set
1367  * to the default (pg_stat_tmp) which is caught in the loop above.
1368  */
1369  if (statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0)
1370  {
1371  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath);
1372  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1373  continue;
1374  }
1375 
1376  /*
1377  * We can skip pg_wal, the WAL segments need to be fetched from the
1378  * WAL archive anyway. But include it as an empty directory anyway, so
1379  * we get permissions right.
1380  */
1381  if (strcmp(pathbuf, "./pg_wal") == 0)
1382  {
1383  /* If pg_wal is a symlink, write it as a directory anyway */
1384  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1385 
1386  /*
1387  * Also send archive_status directory (by hackishly reusing
1388  * statbuf from above ...).
1389  */
1390  size += _tarWriteHeader("./pg_wal/archive_status", NULL, &statbuf,
1391  sizeonly);
1392 
1393  continue; /* don't recurse into pg_wal */
1394  }
1395 
1396  /* Allow symbolic links in pg_tblspc only */
1397  if (strcmp(path, "./pg_tblspc") == 0 &&
1398 #ifndef WIN32
1399  S_ISLNK(statbuf.st_mode)
1400 #else
1401  pgwin32_is_junction(pathbuf)
1402 #endif
1403  )
1404  {
1405 #if defined(HAVE_READLINK) || defined(WIN32)
1406  char linkpath[MAXPGPATH];
1407  int rllen;
1408 
1409  rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1410  if (rllen < 0)
1411  ereport(ERROR,
1413  errmsg("could not read symbolic link \"%s\": %m",
1414  pathbuf)));
1415  if (rllen >= sizeof(linkpath))
1416  ereport(ERROR,
1417  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1418  errmsg("symbolic link \"%s\" target is too long",
1419  pathbuf)));
1420  linkpath[rllen] = '\0';
1421 
1422  size += _tarWriteHeader(pathbuf + basepathlen + 1, linkpath,
1423  &statbuf, sizeonly);
1424 #else
1425 
1426  /*
1427  * If the platform does not have symbolic links, it should not be
1428  * possible to have tablespaces - clearly somebody else created
1429  * them. Warn about it and ignore.
1430  */
1431  ereport(WARNING,
1432  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1433  errmsg("tablespaces are not supported on this platform")));
1434  continue;
1435 #endif /* HAVE_READLINK */
1436  }
1437  else if (S_ISDIR(statbuf.st_mode))
1438  {
1439  bool skip_this_dir = false;
1440  ListCell *lc;
1441 
1442  /*
1443  * Store a directory entry in the tar file so we can get the
1444  * permissions right.
1445  */
1446  size += _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf,
1447  sizeonly);
1448 
1449  /*
1450  * Call ourselves recursively for a directory, unless it happens
1451  * to be a separate tablespace located within PGDATA.
1452  */
1453  foreach(lc, tablespaces)
1454  {
1455  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1456 
1457  /*
1458  * ti->rpath is the tablespace relative path within PGDATA, or
1459  * NULL if the tablespace has been properly located somewhere
1460  * else.
1461  *
1462  * Skip past the leading "./" in pathbuf when comparing.
1463  */
1464  if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1465  {
1466  skip_this_dir = true;
1467  break;
1468  }
1469  }
1470 
1471  /*
1472  * skip sending directories inside pg_tblspc, if not required.
1473  */
1474  if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1475  skip_this_dir = true;
1476 
1477  if (!skip_this_dir)
1478  size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces,
1479  sendtblspclinks, manifest, spcoid);
1480  }
1481  else if (S_ISREG(statbuf.st_mode))
1482  {
1483  bool sent = false;
1484 
1485  if (!sizeonly)
1486  sent = sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf,
1487  true, isDbDir ? atooid(lastDir + 1) : InvalidOid,
1488  manifest, spcoid);
1489 
1490  if (sent || sizeonly)
1491  {
1492  /* Add size. */
1493  size += statbuf.st_size;
1494 
1495  /* Pad to a multiple of the tar block size. */
1496  size += tarPaddingBytesRequired(statbuf.st_size);
1497 
1498  /* Size of the header for the file. */
1499  size += TAR_BLOCK_SIZE;
1500  }
1501  }
1502  else
1503  ereport(WARNING,
1504  (errmsg("skipping special file \"%s\"", pathbuf)));
1505  }
1506  FreeDir(dir);
1507  return size;
1508 }
1509 
1510 /*
1511  * Check if a file should have its checksum validated.
1512  * We validate checksums on files in regular tablespaces
1513  * (including global and default) only, and in those there
1514  * are some files that are explicitly excluded.
1515  */
1516 static bool
1517 is_checksummed_file(const char *fullpath, const char *filename)
1518 {
1519  /* Check that the file is in a tablespace */
1520  if (strncmp(fullpath, "./global/", 9) == 0 ||
1521  strncmp(fullpath, "./base/", 7) == 0 ||
1522  strncmp(fullpath, "/", 1) == 0)
1523  {
1524  int excludeIdx;
1525 
1526  /* Compare file against noChecksumFiles skip list */
1527  for (excludeIdx = 0; noChecksumFiles[excludeIdx].name != NULL; excludeIdx++)
1528  {
1529  int cmplen = strlen(noChecksumFiles[excludeIdx].name);
1530 
1531  if (!noChecksumFiles[excludeIdx].match_prefix)
1532  cmplen++;
1533  if (strncmp(filename, noChecksumFiles[excludeIdx].name,
1534  cmplen) == 0)
1535  return false;
1536  }
1537 
1538  return true;
1539  }
1540  else
1541  return false;
1542 }
1543 
1544 /*****
1545  * Functions for handling tar file format
1546  *
1547  * Copied from pg_dump, but modified to work with libpq for sending
1548  */
1549 
1550 
1551 /*
1552  * Given the member, write the TAR header & send the file.
1553  *
1554  * If 'missing_ok' is true, will not throw an error if the file is not found.
1555  *
1556  * If dboid is anything other than InvalidOid then any checksum failures detected
1557  * will get reported to the stats collector.
1558  *
1559  * Returns true if the file was successfully sent, false if 'missing_ok',
1560  * and the file did not exist.
1561  */
1562 static bool
1563 sendFile(const char *readfilename, const char *tarfilename,
1564  struct stat *statbuf, bool missing_ok, Oid dboid,
1565  backup_manifest_info *manifest, const char *spcoid)
1566 {
1567  int fd;
1568  BlockNumber blkno = 0;
1569  bool block_retry = false;
1570  char buf[TAR_SEND_SIZE];
1571  uint16 checksum;
1572  int checksum_failures = 0;
1573  off_t cnt;
1574  int i;
1575  pgoff_t len = 0;
1576  char *page;
1577  size_t pad;
1578  PageHeader phdr;
1579  int segmentno = 0;
1580  char *segmentpath;
1581  bool verify_checksum = false;
1582  pg_checksum_context checksum_ctx;
1583 
1584  pg_checksum_init(&checksum_ctx, manifest->checksum_type);
1585 
1586  fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY);
1587  if (fd < 0)
1588  {
1589  if (errno == ENOENT && missing_ok)
1590  return false;
1591  ereport(ERROR,
1593  errmsg("could not open file \"%s\": %m", readfilename)));
1594  }
1595 
1596  _tarWriteHeader(tarfilename, NULL, statbuf, false);
1597 
1599  {
1600  char *filename;
1601 
1602  /*
1603  * Get the filename (excluding path). As last_dir_separator()
1604  * includes the last directory separator, we chop that off by
1605  * incrementing the pointer.
1606  */
1607  filename = last_dir_separator(readfilename) + 1;
1608 
1609  if (is_checksummed_file(readfilename, filename))
1610  {
1611  verify_checksum = true;
1612 
1613  /*
1614  * Cut off at the segment boundary (".") to get the segment number
1615  * in order to mix it into the checksum.
1616  */
1617  segmentpath = strstr(filename, ".");
1618  if (segmentpath != NULL)
1619  {
1620  segmentno = atoi(segmentpath + 1);
1621  if (segmentno == 0)
1622  ereport(ERROR,
1623  (errmsg("invalid segment number %d in file \"%s\"",
1624  segmentno, filename)));
1625  }
1626  }
1627  }
1628 
1629  /*
1630  * Loop until we read the amount of data the caller told us to expect. The
1631  * file could be longer, if it was extended while we were sending it, but
1632  * for a base backup we can ignore such extended data. It will be restored
1633  * from WAL.
1634  */
1635  while (len < statbuf->st_size)
1636  {
1637  /* Try to read some more data. */
1638  cnt = basebackup_read_file(fd, buf,
1639  Min(sizeof(buf), statbuf->st_size - len),
1640  len, readfilename, true);
1641 
1642  /*
1643  * If we hit end-of-file, a concurrent truncation must have occurred.
1644  * That's not an error condition, because WAL replay will fix things
1645  * up.
1646  */
1647  if (cnt == 0)
1648  break;
1649 
1650  /*
1651  * The checksums are verified at block level, so we iterate over the
1652  * buffer in chunks of BLCKSZ, after making sure that
1653  * TAR_SEND_SIZE/buf is divisible by BLCKSZ and we read a multiple of
1654  * BLCKSZ bytes.
1655  */
1656  Assert(TAR_SEND_SIZE % BLCKSZ == 0);
1657 
1658  if (verify_checksum && (cnt % BLCKSZ != 0))
1659  {
1660  ereport(WARNING,
1661  (errmsg("could not verify checksum in file \"%s\", block "
1662  "%d: read buffer size %d and page size %d "
1663  "differ",
1664  readfilename, blkno, (int) cnt, BLCKSZ)));
1665  verify_checksum = false;
1666  }
1667 
1668  if (verify_checksum)
1669  {
1670  for (i = 0; i < cnt / BLCKSZ; i++)
1671  {
1672  page = buf + BLCKSZ * i;
1673 
1674  /*
1675  * Only check pages which have not been modified since the
1676  * start of the base backup. Otherwise, they might have been
1677  * written only halfway and the checksum would not be valid.
1678  * However, replaying WAL would reinstate the correct page in
1679  * this case. We also skip completely new pages, since they
1680  * don't have a checksum yet.
1681  */
1682  if (!PageIsNew(page) && PageGetLSN(page) < startptr)
1683  {
1684  checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE);
1685  phdr = (PageHeader) page;
1686  if (phdr->pd_checksum != checksum)
1687  {
1688  /*
1689  * Retry the block on the first failure. It's
1690  * possible that we read the first 4K page of the
1691  * block just before postgres updated the entire block
1692  * so it ends up looking torn to us. We only need to
1693  * retry once because the LSN should be updated to
1694  * something we can ignore on the next pass. If the
1695  * error happens again then it is a true validation
1696  * failure.
1697  */
1698  if (block_retry == false)
1699  {
1700  int reread_cnt;
1701 
1702  /* Reread the failed block */
1703  reread_cnt =
1704  basebackup_read_file(fd, buf + BLCKSZ * i,
1705  BLCKSZ, len + BLCKSZ * i,
1706  readfilename,
1707  false);
1708  if (reread_cnt == 0)
1709  {
1710  /*
1711  * If we hit end-of-file, a concurrent
1712  * truncation must have occurred, so break out
1713  * of this loop just as if the initial fread()
1714  * returned 0. We'll drop through to the same
1715  * code that handles that case. (We must fix
1716  * up cnt first, though.)
1717  */
1718  cnt = BLCKSZ * i;
1719  break;
1720  }
1721 
1722  /* Set flag so we know a retry was attempted */
1723  block_retry = true;
1724 
1725  /* Reset loop to validate the block again */
1726  i--;
1727  continue;
1728  }
1729 
1730  checksum_failures++;
1731 
1732  if (checksum_failures <= 5)
1733  ereport(WARNING,
1734  (errmsg("checksum verification failed in "
1735  "file \"%s\", block %d: calculated "
1736  "%X but expected %X",
1737  readfilename, blkno, checksum,
1738  phdr->pd_checksum)));
1739  if (checksum_failures == 5)
1740  ereport(WARNING,
1741  (errmsg("further checksum verification "
1742  "failures in file \"%s\" will not "
1743  "be reported", readfilename)));
1744  }
1745  }
1746  block_retry = false;
1747  blkno++;
1748  }
1749  }
1750 
1751  /* Send the chunk as a CopyData message */
1752  if (pq_putmessage('d', buf, cnt))
1753  ereport(ERROR,
1754  (errmsg("base backup could not send data, aborting backup")));
1756 
1757  /* Also feed it to the checksum machinery. */
1758  pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt);
1759 
1760  len += cnt;
1761  throttle(cnt);
1762  }
1763 
1764  /* If the file was truncated while we were sending it, pad it with zeros */
1765  if (len < statbuf->st_size)
1766  {
1767  MemSet(buf, 0, sizeof(buf));
1768  while (len < statbuf->st_size)
1769  {
1770  cnt = Min(sizeof(buf), statbuf->st_size - len);
1771  pq_putmessage('d', buf, cnt);
1772  pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt);
1774  len += cnt;
1775  throttle(cnt);
1776  }
1777  }
1778 
1779  /*
1780  * Pad to a block boundary, per tar format requirements. (This small
1781  * piece of data is probably not worth throttling, and is not checksummed
1782  * because it's not actually part of the file.)
1783  */
1784  pad = tarPaddingBytesRequired(len);
1785  if (pad > 0)
1786  {
1787  MemSet(buf, 0, pad);
1788  pq_putmessage('d', buf, pad);
1790  }
1791 
1792  CloseTransientFile(fd);
1793 
1794  if (checksum_failures > 1)
1795  {
1796  ereport(WARNING,
1797  (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1798  "file \"%s\" has a total of %d checksum verification failures",
1799  checksum_failures,
1800  readfilename, checksum_failures)));
1801 
1802  pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1803  }
1804 
1805  total_checksum_failures += checksum_failures;
1806 
1807  AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
1808  (pg_time_t) statbuf->st_mtime, &checksum_ctx);
1809 
1810  return true;
1811 }
1812 
1813 
1814 static int64
1815 _tarWriteHeader(const char *filename, const char *linktarget,
1816  struct stat *statbuf, bool sizeonly)
1817 {
1818  char h[TAR_BLOCK_SIZE];
1819  enum tarError rc;
1820 
1821  if (!sizeonly)
1822  {
1823  rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size,
1824  statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
1825  statbuf->st_mtime);
1826 
1827  switch (rc)
1828  {
1829  case TAR_OK:
1830  break;
1831  case TAR_NAME_TOO_LONG:
1832  ereport(ERROR,
1833  (errmsg("file name too long for tar format: \"%s\"",
1834  filename)));
1835  break;
1836  case TAR_SYMLINK_TOO_LONG:
1837  ereport(ERROR,
1838  (errmsg("symbolic link target too long for tar format: "
1839  "file name \"%s\", target \"%s\"",
1840  filename, linktarget)));
1841  break;
1842  default:
1843  elog(ERROR, "unrecognized tar error: %d", rc);
1844  }
1845 
1846  pq_putmessage('d', h, sizeof(h));
1847  update_basebackup_progress(sizeof(h));
1848  }
1849 
1850  return sizeof(h);
1851 }
1852 
1853 /*
1854  * Write tar header for a directory. If the entry in statbuf is a link then
1855  * write it as a directory anyway.
1856  */
1857 static int64
1858 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf,
1859  bool sizeonly)
1860 {
1861  /* If symlink, write it as a directory anyway */
1862 #ifndef WIN32
1863  if (S_ISLNK(statbuf->st_mode))
1864 #else
1865  if (pgwin32_is_junction(pathbuf))
1866 #endif
1867  statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
1868 
1869  return _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf, sizeonly);
1870 }
1871 
1872 /*
1873  * Increment the network transfer counter by the given number of bytes,
1874  * and sleep if necessary to comply with the requested network transfer
1875  * rate.
1876  */
1877 static void
1878 throttle(size_t increment)
1879 {
1880  TimeOffset elapsed_min;
1881 
1882  if (throttling_counter < 0)
1883  return;
1884 
1885  throttling_counter += increment;
1887  return;
1888 
1889  /* How much time should have elapsed at minimum? */
1890  elapsed_min = elapsed_min_unit *
1892 
1893  /*
1894  * Since the latch could be set repeatedly because of concurrently WAL
1895  * activity, sleep in a loop to ensure enough time has passed.
1896  */
1897  for (;;)
1898  {
1899  TimeOffset elapsed,
1900  sleep;
1901  int wait_result;
1902 
1903  /* Time elapsed since the last measurement (and possible wake up). */
1904  elapsed = GetCurrentTimestamp() - throttled_last;
1905 
1906  /* sleep if the transfer is faster than it should be */
1907  sleep = elapsed_min - elapsed;
1908  if (sleep <= 0)
1909  break;
1910 
1912 
1913  /* We're eating a potentially set latch, so check for interrupts */
1915 
1916  /*
1917  * (TAR_SEND_SIZE / throttling_sample * elapsed_min_unit) should be
1918  * the maximum time to sleep. Thus the cast to long is safe.
1919  */
1920  wait_result = WaitLatch(MyLatch,
1922  (long) (sleep / 1000),
1924 
1925  if (wait_result & WL_LATCH_SET)
1927 
1928  /* Done waiting? */
1929  if (wait_result & WL_TIMEOUT)
1930  break;
1931  }
1932 
1933  /*
1934  * As we work with integers, only whole multiple of throttling_sample was
1935  * processed. The rest will be done during the next call of this function.
1936  */
1938 
1939  /*
1940  * Time interval for the remaining amount and possible next increments
1941  * starts now.
1942  */
1944 }
1945 
1946 /*
1947  * Increment the counter for the amount of data already streamed
1948  * by the given number of bytes, and update the progress report for
1949  * pg_stat_progress_basebackup.
1950  */
1951 static void
1953 {
1954  const int index[] = {
1957  };
1958  int64 val[2];
1959  int nparam = 0;
1960 
1961  backup_streamed += delta;
1962  val[nparam++] = backup_streamed;
1963 
1964  /*
1965  * Avoid overflowing past 100% or the full size. This may make the total
1966  * size number change as we approach the end of the backup (the estimate
1967  * will always be wrong if WAL is included), but that's better than having
1968  * the done column be bigger than the total.
1969  */
1971  {
1973  val[nparam++] = backup_total;
1974  }
1975 
1976  pgstat_progress_update_multi_param(nparam, index, val);
1977 }
1978 
1979 /*
1980  * Read some data from a file, setting a wait event and reporting any error
1981  * encountered.
1982  *
1983  * If partial_read_ok is false, also report an error if the number of bytes
1984  * read is not equal to the number of bytes requested.
1985  *
1986  * Returns the number of bytes read.
1987  */
1988 static int
1989 basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
1990  const char *filename, bool partial_read_ok)
1991 {
1992  int rc;
1993 
1995  rc = pg_pread(fd, buf, nbytes, offset);
1997 
1998  if (rc < 0)
1999  ereport(ERROR,
2001  errmsg("could not read file \"%s\": %m", filename)));
2002  if (!partial_read_ok && rc > 0 && rc != nbytes)
2003  ereport(ERROR,
2005  errmsg("could not read file \"%s\": read %d of %zu",
2006  filename, rc, nbytes)));
2007 
2008  return rc;
2009 }
#define StatusFilePath(path, xlog, suffix)
List * options
Definition: replnodes.h:44
#define NIL
Definition: pg_list.h:65
static void pq_sendint16(StringInfo buf, uint16 i)
Definition: pqformat.h:137
void InitializeBackupManifest(backup_manifest_info *manifest, backup_manifest_option want_manifest, pg_checksum_type manifest_checksum_type)
int pg_file_create_mode
Definition: file_perm.c:19
static int basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset, const char *filename, bool partial_read_ok)
Definition: basebackup.c:1989
#define MAX_RATE_LOWER
Definition: basebackup.h:20
#define DEBUG1
Definition: elog.h:25
int errhint(const char *fmt,...)
Definition: elog.c:1071
static void throttle(size_t increment)
Definition: basebackup.c:1878
static int64 backup_total
Definition: basebackup.c:128
uint32 TimeLineID
Definition: xlogdefs.h:52
int64 pg_time_t
Definition: pgtime.h:23
#define PROGRESS_BASEBACKUP_PHASE
Definition: progress.h:123
void do_pg_abort_backup(int code, Datum arg)
Definition: xlog.c:11427
#define WL_TIMEOUT
Definition: latch.h:127
int wal_segment_size
Definition: xlog.c:116
#define PG_DYNSHMEM_DIR
Definition: dsm_impl.h:51
static ListCell * lnext(const List *l, const ListCell *c)
Definition: pg_list.h:321
void pgstat_progress_start_command(ProgressCommandType cmdtype, Oid relid)
Definition: pgstat.c:3210
#define USECS_PER_SEC
Definition: timestamp.h:94
bool update_process_title
Definition: ps_status.c:36
int errmsg_plural(const char *fmt_singular, const char *fmt_plural, unsigned long n,...)
Definition: elog.c:934
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1574
bool DataChecksumsEnabled(void)
Definition: xlog.c:4930
int64 TimestampTz
Definition: timestamp.h:39
#define XLByteToPrevSeg(xlrp, logSegNo, wal_segsz_bytes)
static const struct exclude_list_item noChecksumFiles[]
Definition: basebackup.c:238
ResourceOwner CurrentResourceOwner
Definition: resowner.c:142
void pgstat_progress_update_param(int index, int64 val)
Definition: pgstat.c:3231
char * pstrdup(const char *in)
Definition: mcxt.c:1186
Definition: pgtar.h:21
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
char * rpath
Definition: basebackup.h:27
StringInfo makeStringInfo(void)
Definition: stringinfo.c:41
#define Min(x, y)
Definition: c.h:927
unsigned char uint8
Definition: c.h:372
static void SendBackupHeader(List *tablespaces)
Definition: basebackup.c:953
static void parse_basebackup_options(List *options, basebackup_options *opt)
Definition: basebackup.c:752
#define PROGRESS_BASEBACKUP_TBLSPC_STREAMED
Definition: progress.h:127
void pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
#define strVal(v)
Definition: value.h:54
int errcode(int sqlerrcode)
Definition: elog.c:610
#define PROGRESS_BASEBACKUP_PHASE_WAIT_WAL_ARCHIVE
Definition: progress.h:133
#define MemSet(start, val, len)
Definition: c.h:949
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:390
bool pg_checksum_parse_type(char *name, pg_checksum_type *type)
uint32 BlockNumber
Definition: block.h:31
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:197
bool parse_bool(const char *value, bool *result)
Definition: bool.c:30
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
unsigned int Oid
Definition: postgres_ext.h:31
bool RecoveryInProgress(void)
Definition: xlog.c:8069
static bool backup_started_in_recovery
Definition: basebackup.c:88
static void send_int8_string(StringInfoData *buf, int64 intval)
Definition: basebackup.c:943
Definition: dirent.h:9
static int fd(const char *x, int i)
Definition: preproc-init.c:105
#define PG_BINARY
Definition: c.h:1211
void ResetLatch(Latch *latch)
Definition: latch.c:588
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:87
XLogRecPtr do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p, StringInfo labelfile, List **tablespaces, StringInfo tblspcmapfile, bool needtblspcmapfile)
Definition: xlog.c:10496
void pgstat_report_checksum_failures_in_db(Oid dboid, int failurecount)
Definition: pgstat.c:1585
int WaitLatch(Latch *latch, int wakeEvents, long timeout, uint32 wait_event_info)
Definition: latch.c:390
ssize_t pg_pread(int fd, void *buf, size_t nbyte, off_t offset)
Definition: pread.c:27
uint16 pd_checksum
Definition: bufpage.h:156
static void pq_sendbyte(StringInfo buf, uint8 byt)
Definition: pqformat.h:161
Definition: type.h:89
#define PROGRESS_BASEBACKUP_PHASE_WAIT_CHECKPOINT
Definition: progress.h:130
#define pgoff_t
Definition: win32_port.h:194
void set_ps_display(const char *activity)
Definition: ps_status.c:349
static void pq_sendint32(StringInfo buf, uint32 i)
Definition: pqformat.h:145
#define sprintf
Definition: port.h:195
#define TABLESPACE_MAP
Definition: xlog.h:392
#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:47
static void sendFileWithContent(const char *filename, const char *content, backup_manifest_info *manifest)
Definition: basebackup.c:1086
unsigned short uint16
Definition: c.h:373
#define linitial(l)
Definition: pg_list.h:195
static const char *const excludeDirContents[]
Definition: basebackup.c:153
#define MAX_RATE_UPPER
Definition: basebackup.h:21
Definition: dirent.c:25
void pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
tarError
Definition: pgtar.h:19
#define ERROR
Definition: elog.h:43
#define PG_TEMP_FILE_PREFIX
Definition: pg_checksums.c:59
#define IsXLogFileName(fname)
backup_manifest_option manifest
Definition: basebackup.c:57
XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive, TimeLineID *stoptli_p)
Definition: xlog.c:11014
int OpenTransientFile(const char *fileName, int fileFlags)
Definition: fd.c:2372
static bool noverify_checksums
Definition: basebackup.c:122
void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)
Definition: xlog.c:3925
void SendBaseBackup(BaseBackupCmd *cmd)
Definition: basebackup.c:922
#define MAXPGPATH
#define DEBUG2
Definition: elog.h:24
#define XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes)
#define TAR_BLOCK_SIZE
Definition: pgtar.h:17
#define TABLESPACE_VERSION_DIRECTORY
Definition: relpath.h:26
static int32 maxrate
#define PROGRESS_BASEBACKUP_BACKUP_STREAMED
Definition: progress.h:125
static char * buf
Definition: pg_test_fsync.c:67
uint64 XLogSegNo
Definition: xlogdefs.h:41
void SendBackupManifest(backup_manifest_info *manifest)
#define readlink(path, buf, size)
Definition: win32_port.h:222
static int64 backup_streamed
Definition: basebackup.c:131
int errcode_for_file_access(void)
Definition: elog.c:633
#define is_absolute_path(filename)
Definition: port.h:86
const char * label
Definition: basebackup.c:50
enum manifest_option backup_manifest_option
static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli)
Definition: basebackup.c:1031
unsigned int uint32
Definition: c.h:374
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1815
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2583
static void pgstat_report_wait_end(void)
Definition: pgstat.h:1381
static int64 sendTablespace(char *path, char *oid, bool sizeonly, struct backup_manifest_info *manifest)
Definition: basebackup.c:1143
char * pgstat_stat_directory
Definition: pgstat.c:133
#define atooid(x)
Definition: postgres_ext.h:42
pg_checksum_type
static char * statrelpath
Definition: basebackup.c:91
void AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr, TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)
ForkNumber
Definition: relpath.h:40
Node * arg
Definition: parsenodes.h:733
#define S_ISREG(m)
Definition: win32_port.h:299
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:45
List * lappend(List *list, void *datum)
Definition: list.c:321
int CloseTransientFile(int fd)
Definition: fd.c:2549
#define WARNING
Definition: elog.h:40
#define stat(a, b)
Definition: win32_port.h:255
#define MAXFNAMELEN
#define PROGRESS_BASEBACKUP_BACKUP_TOTAL
Definition: progress.h:124
int pg_dir_create_mode
Definition: file_perm.c:18
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
int64 TimeOffset
Definition: timestamp.h:40
static void perform_base_backup(basebackup_options *opt)
Definition: basebackup.c:256
void * palloc0(Size size)
Definition: mcxt.c:980
void pgstat_progress_end_command(void)
Definition: pgstat.c:3282
#define XLOGDIR
#define PROGRESS_BASEBACKUP_TBLSPC_TOTAL
Definition: progress.h:126
pg_checksum_type checksum_type
#define PROGRESS_BASEBACKUP_PHASE_TRANSFER_WAL
Definition: progress.h:134
static size_t tarPaddingBytesRequired(size_t len)
Definition: pgtar.h:40
void WalSndResourceCleanup(bool isCommit)
Definition: walsender.c:330
char * last_dir_separator(const char *filename)
Definition: path.c:138
#define BoolGetDatum(X)
Definition: postgres.h:402
#define XLOG_CONTROL_FILE
#define InvalidOid
Definition: postgres_ext.h:36
TimeLineID ThisTimeLineID
Definition: xlog.c:191
static TimestampTz throttled_last
Definition: basebackup.c:113
#define RELCACHE_INIT_FILENAME
Definition: relcache.h:25
#define ereport(elevel,...)
Definition: elog.h:144
static StringInfo tblspc_map_file
Definition: xlogfuncs.c:45
static bool is_checksummed_file(const char *fullpath, const char *filename)
Definition: basebackup.c:1517
PageHeaderData * PageHeader
Definition: bufpage.h:166
static void update_basebackup_progress(int64 delta)
Definition: basebackup.c:1952
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:745
#define lfirst(lc)
Definition: pg_list.h:190
void WalSndSetState(WalSndState state)
Definition: walsender.c:3188
#define PG_STAT_TMP_DIR
Definition: pgstat.h:34
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2649
#define THROTTLING_FREQUENCY
Definition: basebackup.c:101
void pgstat_progress_update_multi_param(int nparam, const int *index, const int64 *val)
Definition: pgstat.c:3253
size_t Size
Definition: c.h:473
static bool manifest
#define XLogFileName(fname, tli, logSegNo, wal_segsz_bytes)
static void pgstat_report_wait_start(uint32 wait_event_info)
Definition: pgstat.h:1357
static int list_length(const List *l)
Definition: pg_list.h:169
#define PG_AUTOCONF_FILENAME
Definition: guc.h:34
#define LOG_METAINFO_DATAFILE_TMP
Definition: syslogger.h:96
#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:52
bool looks_like_temp_rel_name(const char *name)
Definition: fd.c:3187
bool parse_filename_for_nontemp_relation(const char *name, int *oidchars, ForkNumber *fork)
Definition: reinit.c:374
#define INT64_FORMAT
Definition: c.h:416
#define S_ISDIR(m)
Definition: win32_port.h:296
#define PageGetLSN(page)
Definition: bufpage.h:366
#define PROGRESS_BASEBACKUP_PHASE_ESTIMATE_BACKUP_SIZE
Definition: progress.h:131
#define lstat(path, sb)
Definition: win32_port.h:244
#define PROGRESS_BASEBACKUP_PHASE_STREAM_BACKUP
Definition: progress.h:132
const char * name
Definition: basebackup.c:141
#define IsTLHistoryFileName(fname)
static int compareWalFileNames(const ListCell *a, const ListCell *b)
Definition: basebackup.c:740
static char * filename
Definition: pg_dumpall.c:90
static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1196
#define intVal(v)
Definition: value.h:52
#define PageIsNew(page)
Definition: bufpage.h:229
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:125
void list_sort(List *list, list_sort_comparator cmp)
Definition: list.c:1481
int errmsg(const char *fmt,...)
Definition: elog.c:824
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:298
#define TAR_SEND_SIZE
Definition: basebackup.c:96
#define elog(elevel,...)
Definition: elog.h:214
int i
static bool sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf, bool missing_ok, Oid dboid, backup_manifest_info *manifest, const char *spcoid)
Definition: basebackup.c:1563
char * DataDir
Definition: globals.c:62
struct Latch * MyLatch
Definition: globals.c:54
#define BACKUP_LABEL_FILE
Definition: xlog.h:389
void AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid, const char *pathname, size_t size, pg_time_t mtime, pg_checksum_context *checksum_ctx)
char * defname
Definition: parsenodes.h:732
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:99
pg_checksum_type manifest_checksum_type
Definition: basebackup.c:58
char d_name[MAX_PATH]
Definition: dirent.h:14
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:42
static int64 throttling_counter
Definition: basebackup.c:107
static const struct exclude_list_item excludeFiles[]
Definition: basebackup.c:194
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:369
Definition: pg_list.h:50
#define snprintf
Definition: port.h:193
#define WL_LATCH_SET
Definition: latch.h:124
#define OIDCHARS
Definition: relpath.h:30
static uint64 throttling_sample
Definition: basebackup.c:104
uint16 pg_checksum_page(char *page, BlockNumber blkno)
long val
Definition: informix.c:664
int FreeDir(DIR *dir)
Definition: fd.c:2701
static TimeOffset elapsed_min_unit
Definition: basebackup.c:110
#define WL_EXIT_ON_PM_DEATH
Definition: latch.h:129
static XLogRecPtr startptr
Definition: basebackup.c:116
bool pgwin32_is_junction(const char *path)
ResourceOwner ResourceOwnerCreate(ResourceOwner parent, const char *name)
Definition: resowner.c:422
static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1858
static long long int total_checksum_failures
Definition: basebackup.c:119
#define XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes)