PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
basebackup.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * basebackup.c
4  * code for taking a base backup and streaming it to a standby
5  *
6  * Portions Copyright (c) 2010-2017, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * src/backend/replication/basebackup.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 #include "postgres.h"
14 
15 #include <sys/types.h>
16 #include <sys/stat.h>
17 #include <unistd.h>
18 #include <time.h>
19 
20 #include "access/xlog_internal.h" /* for pg_start/stop_backup */
21 #include "catalog/catalog.h"
22 #include "catalog/pg_type.h"
23 #include "lib/stringinfo.h"
24 #include "libpq/libpq.h"
25 #include "libpq/pqformat.h"
26 #include "miscadmin.h"
27 #include "nodes/pg_list.h"
28 #include "pgtar.h"
29 #include "pgstat.h"
30 #include "replication/basebackup.h"
31 #include "replication/walsender.h"
33 #include "storage/dsm_impl.h"
34 #include "storage/fd.h"
35 #include "storage/ipc.h"
36 #include "utils/builtins.h"
37 #include "utils/elog.h"
38 #include "utils/ps_status.h"
39 #include "utils/timestamp.h"
40 
41 
42 typedef struct
43 {
44  const char *label;
45  bool progress;
47  bool nowait;
48  bool includewal;
52 
53 
54 static int64 sendDir(char *path, int basepathlen, bool sizeonly,
55  List *tablespaces, bool sendtblspclinks);
56 static bool sendFile(char *readfilename, char *tarfilename,
57  struct stat * statbuf, bool missing_ok);
58 static void sendFileWithContent(const char *filename, const char *content);
59 static int64 _tarWriteHeader(const char *filename, const char *linktarget,
60  struct stat * statbuf, bool sizeonly);
61 static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf,
62  bool sizeonly);
63 static void send_int8_string(StringInfoData *buf, int64 intval);
64 static void SendBackupHeader(List *tablespaces);
65 static void base_backup_cleanup(int code, Datum arg);
66 static void perform_base_backup(basebackup_options *opt, DIR *tblspcdir);
68 static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli);
69 static int compareWalFileNames(const void *a, const void *b);
70 static void throttle(size_t increment);
71 
72 /* Was the backup currently in-progress initiated in recovery mode? */
73 static bool backup_started_in_recovery = false;
74 
75 /* Relative path of temporary statistics directory */
76 static char *statrelpath = NULL;
77 
78 /*
79  * Size of each block sent into the tar stream for larger files.
80  */
81 #define TAR_SEND_SIZE 32768
82 
83 /*
84  * How frequently to throttle, as a fraction of the specified rate-second.
85  */
86 #define THROTTLING_FREQUENCY 8
87 
88 /* The actual number of bytes, transfer of which may cause sleep. */
89 static uint64 throttling_sample;
90 
91 /* Amount of data already transferred but not yet throttled. */
92 static int64 throttling_counter;
93 
94 /* The minimum time required to transfer throttling_sample bytes. */
96 
97 /* The last check of the transfer rate. */
99 
100 /*
101  * The contents of these directories are removed or recreated during server
102  * start so they are not included in backups. The directories themselves are
103  * kept and included as empty to preserve access permissions.
104  */
105 static const char *excludeDirContents[] =
106 {
107  /*
108  * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even
109  * when stats_temp_directory is set because PGSS_TEXT_FILE is always created
110  * there.
111  */
113 
114  /*
115  * It is generally not useful to backup the contents of this directory even
116  * if the intention is to restore to another master. See backup.sgml for a
117  * more detailed description.
118  */
119  "pg_replslot",
120 
121  /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
123 
124  /* Contents removed on startup, see AsyncShmemInit(). */
125  "pg_notify",
126 
127  /*
128  * Old contents are loaded for possible debugging but are not required for
129  * normal operation, see OldSerXidInit().
130  */
131  "pg_serial",
132 
133  /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
134  "pg_snapshots",
135 
136  /* Contents zeroed on startup, see StartupSUBTRANS(). */
137  "pg_subtrans",
138 
139  /* end of list */
140  NULL
141 };
142 
143 /*
144  * List of files excluded from backups.
145  */
146 static const char *excludeFiles[] =
147 {
148  /* Skip auto conf temporary file. */
149  PG_AUTOCONF_FILENAME ".tmp",
150 
151  /*
152  * If there's a backup_label or tablespace_map file, it belongs to a
153  * backup started by the user with pg_start_backup(). It is *not* correct
154  * for this backup. Our backup_label/tablespace_map is injected into the
155  * tar separately.
156  */
159 
160  "postmaster.pid",
161  "postmaster.opts",
162 
163  /* end of list */
164  NULL
165 };
166 
167 /*
168  * Called when ERROR or FATAL happens in perform_base_backup() after
169  * we have started the backup - make sure we end it!
170  */
171 static void
173 {
175 }
176 
177 /*
178  * Actually do a base backup for the specified tablespaces.
179  *
180  * This is split out mainly to avoid complaints about "variable might be
181  * clobbered by longjmp" from stupider versions of gcc.
182  */
183 static void
185 {
186  XLogRecPtr startptr;
187  TimeLineID starttli;
188  XLogRecPtr endptr;
189  TimeLineID endtli;
190  StringInfo labelfile;
192  int datadirpathlen;
193  List *tablespaces = NIL;
194 
195  datadirpathlen = strlen(DataDir);
196 
198 
199  labelfile = makeStringInfo();
200  tblspc_map_file = makeStringInfo();
201 
202  startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
203  labelfile, tblspcdir, &tablespaces,
204  tblspc_map_file,
205  opt->progress, opt->sendtblspcmapfile);
206 
207  /*
208  * Once do_pg_start_backup has been called, ensure that any failure causes
209  * us to abort the backup so we don't "leak" a backup counter. For this
210  * reason, *all* functionality between do_pg_start_backup() and
211  * do_pg_stop_backup() should be inside the error cleanup block!
212  */
213 
215  {
216  ListCell *lc;
217  tablespaceinfo *ti;
218 
219  SendXlogRecPtrResult(startptr, starttli);
220 
221  /*
222  * Calculate the relative path of temporary statistics directory in
223  * order to skip the files which are located in that directory later.
224  */
226  strncmp(pgstat_stat_directory, DataDir, datadirpathlen) == 0)
227  statrelpath = psprintf("./%s", pgstat_stat_directory + datadirpathlen + 1);
228  else if (strncmp(pgstat_stat_directory, "./", 2) != 0)
230  else
232 
233  /* Add a node for the base directory at the end */
234  ti = palloc0(sizeof(tablespaceinfo));
235  ti->size = opt->progress ? sendDir(".", 1, true, tablespaces, true) : -1;
236  tablespaces = lappend(tablespaces, ti);
237 
238  /* Send tablespace header */
239  SendBackupHeader(tablespaces);
240 
241  /* Setup and activate network throttling, if client requested it */
242  if (opt->maxrate > 0)
243  {
245  (int64) opt->maxrate * (int64) 1024 / THROTTLING_FREQUENCY;
246 
247  /*
248  * The minimum amount of time for throttling_sample bytes to be
249  * transferred.
250  */
252 
253  /* Enable throttling. */
254  throttling_counter = 0;
255 
256  /* The 'real data' starts now (header was ignored). */
258  }
259  else
260  {
261  /* Disable throttling. */
262  throttling_counter = -1;
263  }
264 
265  /* Send off our tablespaces one by one */
266  foreach(lc, tablespaces)
267  {
268  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
270 
271  /* Send CopyOutResponse message */
272  pq_beginmessage(&buf, 'H');
273  pq_sendbyte(&buf, 0); /* overall format */
274  pq_sendint(&buf, 0, 2); /* natts */
275  pq_endmessage(&buf);
276 
277  if (ti->path == NULL)
278  {
279  struct stat statbuf;
280 
281  /* In the main tar, include the backup_label first... */
283 
284  /*
285  * Send tablespace_map file if required and then the bulk of
286  * the files.
287  */
288  if (tblspc_map_file && opt->sendtblspcmapfile)
289  {
290  sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data);
291  sendDir(".", 1, false, tablespaces, false);
292  }
293  else
294  sendDir(".", 1, false, tablespaces, true);
295 
296  /* ... and pg_control after everything else. */
297  if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
298  ereport(ERROR,
300  errmsg("could not stat control file \"%s\": %m",
302  sendFile(XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf, false);
303  }
304  else
305  sendTablespace(ti->path, false);
306 
307  /*
308  * If we're including WAL, and this is the main data directory we
309  * don't terminate the tar stream here. Instead, we will append
310  * the xlog files below and terminate it then. This is safe since
311  * the main data directory is always sent *last*.
312  */
313  if (opt->includewal && ti->path == NULL)
314  {
315  Assert(lnext(lc) == NULL);
316  }
317  else
318  pq_putemptymessage('c'); /* CopyDone */
319  }
320  }
322 
323  endptr = do_pg_stop_backup(labelfile->data, !opt->nowait, &endtli);
324 
325  if (opt->includewal)
326  {
327  /*
328  * We've left the last tar file "open", so we can now append the
329  * required WAL files to it.
330  */
331  char pathbuf[MAXPGPATH];
332  XLogSegNo segno;
333  XLogSegNo startsegno;
334  XLogSegNo endsegno;
335  struct stat statbuf;
336  List *historyFileList = NIL;
337  List *walFileList = NIL;
338  char **walFiles;
339  int nWalFiles;
340  char firstoff[MAXFNAMELEN];
341  char lastoff[MAXFNAMELEN];
342  DIR *dir;
343  struct dirent *de;
344  int i;
345  ListCell *lc;
346  TimeLineID tli;
347 
348  /*
349  * I'd rather not worry about timelines here, so scan pg_wal and
350  * include all WAL files in the range between 'startptr' and 'endptr',
351  * regardless of the timeline the file is stamped with. If there are
352  * some spurious WAL files belonging to timelines that don't belong in
353  * this server's history, they will be included too. Normally there
354  * shouldn't be such files, but if there are, there's little harm in
355  * including them.
356  */
357  XLByteToSeg(startptr, startsegno);
358  XLogFileName(firstoff, ThisTimeLineID, startsegno);
359  XLByteToPrevSeg(endptr, endsegno);
360  XLogFileName(lastoff, ThisTimeLineID, endsegno);
361 
362  dir = AllocateDir("pg_wal");
363  if (!dir)
364  ereport(ERROR,
365  (errmsg("could not open directory \"%s\": %m", "pg_wal")));
366  while ((de = ReadDir(dir, "pg_wal")) != NULL)
367  {
368  /* Does it look like a WAL segment, and is it in the range? */
369  if (IsXLogFileName(de->d_name) &&
370  strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
371  strcmp(de->d_name + 8, lastoff + 8) <= 0)
372  {
373  walFileList = lappend(walFileList, pstrdup(de->d_name));
374  }
375  /* Does it look like a timeline history file? */
376  else if (IsTLHistoryFileName(de->d_name))
377  {
378  historyFileList = lappend(historyFileList, pstrdup(de->d_name));
379  }
380  }
381  FreeDir(dir);
382 
383  /*
384  * Before we go any further, check that none of the WAL segments we
385  * need were removed.
386  */
387  CheckXLogRemoved(startsegno, ThisTimeLineID);
388 
389  /*
390  * Put the WAL filenames into an array, and sort. We send the files in
391  * order from oldest to newest, to reduce the chance that a file is
392  * recycled before we get a chance to send it over.
393  */
394  nWalFiles = list_length(walFileList);
395  walFiles = palloc(nWalFiles * sizeof(char *));
396  i = 0;
397  foreach(lc, walFileList)
398  {
399  walFiles[i++] = lfirst(lc);
400  }
401  qsort(walFiles, nWalFiles, sizeof(char *), compareWalFileNames);
402 
403  /*
404  * There must be at least one xlog file in the pg_wal directory,
405  * since we are doing backup-including-xlog.
406  */
407  if (nWalFiles < 1)
408  ereport(ERROR,
409  (errmsg("could not find any WAL files")));
410 
411  /*
412  * Sanity check: the first and last segment should cover startptr and
413  * endptr, with no gaps in between.
414  */
415  XLogFromFileName(walFiles[0], &tli, &segno);
416  if (segno != startsegno)
417  {
418  char startfname[MAXFNAMELEN];
419 
420  XLogFileName(startfname, ThisTimeLineID, startsegno);
421  ereport(ERROR,
422  (errmsg("could not find WAL file \"%s\"", startfname)));
423  }
424  for (i = 0; i < nWalFiles; i++)
425  {
426  XLogSegNo currsegno = segno;
427  XLogSegNo nextsegno = segno + 1;
428 
429  XLogFromFileName(walFiles[i], &tli, &segno);
430  if (!(nextsegno == segno || currsegno == segno))
431  {
432  char nextfname[MAXFNAMELEN];
433 
434  XLogFileName(nextfname, ThisTimeLineID, nextsegno);
435  ereport(ERROR,
436  (errmsg("could not find WAL file \"%s\"", nextfname)));
437  }
438  }
439  if (segno != endsegno)
440  {
441  char endfname[MAXFNAMELEN];
442 
443  XLogFileName(endfname, ThisTimeLineID, endsegno);
444  ereport(ERROR,
445  (errmsg("could not find WAL file \"%s\"", endfname)));
446  }
447 
448  /* Ok, we have everything we need. Send the WAL files. */
449  for (i = 0; i < nWalFiles; i++)
450  {
451  FILE *fp;
452  char buf[TAR_SEND_SIZE];
453  size_t cnt;
454  pgoff_t len = 0;
455 
456  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFiles[i]);
457  XLogFromFileName(walFiles[i], &tli, &segno);
458 
459  fp = AllocateFile(pathbuf, "rb");
460  if (fp == NULL)
461  {
462  /*
463  * Most likely reason for this is that the file was already
464  * removed by a checkpoint, so check for that to get a better
465  * error message.
466  */
467  CheckXLogRemoved(segno, tli);
468 
469  ereport(ERROR,
471  errmsg("could not open file \"%s\": %m", pathbuf)));
472  }
473 
474  if (fstat(fileno(fp), &statbuf) != 0)
475  ereport(ERROR,
477  errmsg("could not stat file \"%s\": %m",
478  pathbuf)));
479  if (statbuf.st_size != XLogSegSize)
480  {
481  CheckXLogRemoved(segno, tli);
482  ereport(ERROR,
484  errmsg("unexpected WAL file size \"%s\"", walFiles[i])));
485  }
486 
487  /* send the WAL file itself */
488  _tarWriteHeader(pathbuf, NULL, &statbuf, false);
489 
490  while ((cnt = fread(buf, 1, Min(sizeof(buf), XLogSegSize - len), fp)) > 0)
491  {
492  CheckXLogRemoved(segno, tli);
493  /* Send the chunk as a CopyData message */
494  if (pq_putmessage('d', buf, cnt))
495  ereport(ERROR,
496  (errmsg("base backup could not send data, aborting backup")));
497 
498  len += cnt;
499  throttle(cnt);
500 
501  if (len == XLogSegSize)
502  break;
503  }
504 
505  if (len != XLogSegSize)
506  {
507  CheckXLogRemoved(segno, tli);
508  ereport(ERROR,
510  errmsg("unexpected WAL file size \"%s\"", walFiles[i])));
511  }
512 
513  /* XLogSegSize is a multiple of 512, so no need for padding */
514 
515  FreeFile(fp);
516 
517  /*
518  * Mark file as archived, otherwise files can get archived again
519  * after promotion of a new node. This is in line with
520  * walreceiver.c always doing an XLogArchiveForceDone() after a
521  * complete segment.
522  */
523  StatusFilePath(pathbuf, walFiles[i], ".done");
524  sendFileWithContent(pathbuf, "");
525  }
526 
527  /*
528  * Send timeline history files too. Only the latest timeline history
529  * file is required for recovery, and even that only if there happens
530  * to be a timeline switch in the first WAL segment that contains the
531  * checkpoint record, or if we're taking a base backup from a standby
532  * server and the target timeline changes while the backup is taken.
533  * But they are small and highly useful for debugging purposes, so
534  * better include them all, always.
535  */
536  foreach(lc, historyFileList)
537  {
538  char *fname = lfirst(lc);
539 
540  snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
541 
542  if (lstat(pathbuf, &statbuf) != 0)
543  ereport(ERROR,
545  errmsg("could not stat file \"%s\": %m", pathbuf)));
546 
547  sendFile(pathbuf, pathbuf, &statbuf, false);
548 
549  /* unconditionally mark file as archived */
550  StatusFilePath(pathbuf, fname, ".done");
551  sendFileWithContent(pathbuf, "");
552  }
553 
554  /* Send CopyDone message for the last tar file */
555  pq_putemptymessage('c');
556  }
557  SendXlogRecPtrResult(endptr, endtli);
558 }
559 
560 /*
561  * qsort comparison function, to compare log/seg portion of WAL segment
562  * filenames, ignoring the timeline portion.
563  */
564 static int
565 compareWalFileNames(const void *a, const void *b)
566 {
567  char *fna = *((char **) a);
568  char *fnb = *((char **) b);
569 
570  return strcmp(fna + 8, fnb + 8);
571 }
572 
573 /*
574  * Parse the base backup options passed down by the parser
575  */
576 static void
578 {
579  ListCell *lopt;
580  bool o_label = false;
581  bool o_progress = false;
582  bool o_fast = false;
583  bool o_nowait = false;
584  bool o_wal = false;
585  bool o_maxrate = false;
586  bool o_tablespace_map = false;
587 
588  MemSet(opt, 0, sizeof(*opt));
589  foreach(lopt, options)
590  {
591  DefElem *defel = (DefElem *) lfirst(lopt);
592 
593  if (strcmp(defel->defname, "label") == 0)
594  {
595  if (o_label)
596  ereport(ERROR,
597  (errcode(ERRCODE_SYNTAX_ERROR),
598  errmsg("duplicate option \"%s\"", defel->defname)));
599  opt->label = strVal(defel->arg);
600  o_label = true;
601  }
602  else if (strcmp(defel->defname, "progress") == 0)
603  {
604  if (o_progress)
605  ereport(ERROR,
606  (errcode(ERRCODE_SYNTAX_ERROR),
607  errmsg("duplicate option \"%s\"", defel->defname)));
608  opt->progress = true;
609  o_progress = true;
610  }
611  else if (strcmp(defel->defname, "fast") == 0)
612  {
613  if (o_fast)
614  ereport(ERROR,
615  (errcode(ERRCODE_SYNTAX_ERROR),
616  errmsg("duplicate option \"%s\"", defel->defname)));
617  opt->fastcheckpoint = true;
618  o_fast = true;
619  }
620  else if (strcmp(defel->defname, "nowait") == 0)
621  {
622  if (o_nowait)
623  ereport(ERROR,
624  (errcode(ERRCODE_SYNTAX_ERROR),
625  errmsg("duplicate option \"%s\"", defel->defname)));
626  opt->nowait = true;
627  o_nowait = true;
628  }
629  else if (strcmp(defel->defname, "wal") == 0)
630  {
631  if (o_wal)
632  ereport(ERROR,
633  (errcode(ERRCODE_SYNTAX_ERROR),
634  errmsg("duplicate option \"%s\"", defel->defname)));
635  opt->includewal = true;
636  o_wal = true;
637  }
638  else if (strcmp(defel->defname, "max_rate") == 0)
639  {
640  long maxrate;
641 
642  if (o_maxrate)
643  ereport(ERROR,
644  (errcode(ERRCODE_SYNTAX_ERROR),
645  errmsg("duplicate option \"%s\"", defel->defname)));
646 
647  maxrate = intVal(defel->arg);
648  if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
649  ereport(ERROR,
650  (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
651  errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
652  (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
653 
654  opt->maxrate = (uint32) maxrate;
655  o_maxrate = true;
656  }
657  else if (strcmp(defel->defname, "tablespace_map") == 0)
658  {
659  if (o_tablespace_map)
660  ereport(ERROR,
661  (errcode(ERRCODE_SYNTAX_ERROR),
662  errmsg("duplicate option \"%s\"", defel->defname)));
663  opt->sendtblspcmapfile = true;
664  o_tablespace_map = true;
665  }
666  else
667  elog(ERROR, "option \"%s\" not recognized",
668  defel->defname);
669  }
670  if (opt->label == NULL)
671  opt->label = "base backup";
672 }
673 
674 
675 /*
676  * SendBaseBackup() - send a complete base backup.
677  *
678  * The function will put the system into backup mode like pg_start_backup()
679  * does, so that the backup is consistent even though we read directly from
680  * the filesystem, bypassing the buffer cache.
681  */
682 void
684 {
685  DIR *dir;
686  basebackup_options opt;
687 
688  parse_basebackup_options(cmd->options, &opt);
689 
691 
693  {
694  char activitymsg[50];
695 
696  snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
697  opt.label);
698  set_ps_display(activitymsg, false);
699  }
700 
701  /* Make sure we can open the directory with tablespaces in it */
702  dir = AllocateDir("pg_tblspc");
703  if (!dir)
704  ereport(ERROR,
705  (errmsg("could not open directory \"%s\": %m", "pg_tblspc")));
706 
707  perform_base_backup(&opt, dir);
708 
709  FreeDir(dir);
710 }
711 
712 static void
714 {
715  char is[32];
716 
717  sprintf(is, INT64_FORMAT, intval);
718  pq_sendint(buf, strlen(is), 4);
719  pq_sendbytes(buf, is, strlen(is));
720 }
721 
722 static void
723 SendBackupHeader(List *tablespaces)
724 {
726  ListCell *lc;
727 
728  /* Construct and send the directory information */
729  pq_beginmessage(&buf, 'T'); /* RowDescription */
730  pq_sendint(&buf, 3, 2); /* 3 fields */
731 
732  /* First field - spcoid */
733  pq_sendstring(&buf, "spcoid");
734  pq_sendint(&buf, 0, 4); /* table oid */
735  pq_sendint(&buf, 0, 2); /* attnum */
736  pq_sendint(&buf, OIDOID, 4); /* type oid */
737  pq_sendint(&buf, 4, 2); /* typlen */
738  pq_sendint(&buf, 0, 4); /* typmod */
739  pq_sendint(&buf, 0, 2); /* format code */
740 
741  /* Second field - spcpath */
742  pq_sendstring(&buf, "spclocation");
743  pq_sendint(&buf, 0, 4);
744  pq_sendint(&buf, 0, 2);
745  pq_sendint(&buf, TEXTOID, 4);
746  pq_sendint(&buf, -1, 2);
747  pq_sendint(&buf, 0, 4);
748  pq_sendint(&buf, 0, 2);
749 
750  /* Third field - size */
751  pq_sendstring(&buf, "size");
752  pq_sendint(&buf, 0, 4);
753  pq_sendint(&buf, 0, 2);
754  pq_sendint(&buf, INT8OID, 4);
755  pq_sendint(&buf, 8, 2);
756  pq_sendint(&buf, 0, 4);
757  pq_sendint(&buf, 0, 2);
758  pq_endmessage(&buf);
759 
760  foreach(lc, tablespaces)
761  {
762  tablespaceinfo *ti = lfirst(lc);
763 
764  /* Send one datarow message */
765  pq_beginmessage(&buf, 'D');
766  pq_sendint(&buf, 3, 2); /* number of columns */
767  if (ti->path == NULL)
768  {
769  pq_sendint(&buf, -1, 4); /* Length = -1 ==> NULL */
770  pq_sendint(&buf, -1, 4);
771  }
772  else
773  {
774  Size len;
775 
776  len = strlen(ti->oid);
777  pq_sendint(&buf, len, 4);
778  pq_sendbytes(&buf, ti->oid, len);
779 
780  len = strlen(ti->path);
781  pq_sendint(&buf, len, 4);
782  pq_sendbytes(&buf, ti->path, len);
783  }
784  if (ti->size >= 0)
785  send_int8_string(&buf, ti->size / 1024);
786  else
787  pq_sendint(&buf, -1, 4); /* NULL */
788 
789  pq_endmessage(&buf);
790  }
791 
792  /* Send a CommandComplete message */
793  pq_puttextmessage('C', "SELECT");
794 }
795 
796 /*
797  * Send a single resultset containing just a single
798  * XLogRecPtr record (in text format)
799  */
800 static void
802 {
804  char str[MAXFNAMELEN];
805  Size len;
806 
807  pq_beginmessage(&buf, 'T'); /* RowDescription */
808  pq_sendint(&buf, 2, 2); /* 2 fields */
809 
810  /* Field headers */
811  pq_sendstring(&buf, "recptr");
812  pq_sendint(&buf, 0, 4); /* table oid */
813  pq_sendint(&buf, 0, 2); /* attnum */
814  pq_sendint(&buf, TEXTOID, 4); /* type oid */
815  pq_sendint(&buf, -1, 2);
816  pq_sendint(&buf, 0, 4);
817  pq_sendint(&buf, 0, 2);
818 
819  pq_sendstring(&buf, "tli");
820  pq_sendint(&buf, 0, 4); /* table oid */
821  pq_sendint(&buf, 0, 2); /* attnum */
822 
823  /*
824  * int8 may seem like a surprising data type for this, but in theory int4
825  * would not be wide enough for this, as TimeLineID is unsigned.
826  */
827  pq_sendint(&buf, INT8OID, 4); /* type oid */
828  pq_sendint(&buf, -1, 2);
829  pq_sendint(&buf, 0, 4);
830  pq_sendint(&buf, 0, 2);
831  pq_endmessage(&buf);
832 
833  /* Data row */
834  pq_beginmessage(&buf, 'D');
835  pq_sendint(&buf, 2, 2); /* number of columns */
836 
837  len = snprintf(str, sizeof(str),
838  "%X/%X", (uint32) (ptr >> 32), (uint32) ptr);
839  pq_sendint(&buf, len, 4);
840  pq_sendbytes(&buf, str, len);
841 
842  len = snprintf(str, sizeof(str), "%u", tli);
843  pq_sendint(&buf, len, 4);
844  pq_sendbytes(&buf, str, len);
845 
846  pq_endmessage(&buf);
847 
848  /* Send a CommandComplete message */
849  pq_puttextmessage('C', "SELECT");
850 }
851 
852 /*
853  * Inject a file with given name and content in the output tar stream.
854  */
855 static void
856 sendFileWithContent(const char *filename, const char *content)
857 {
858  struct stat statbuf;
859  int pad,
860  len;
861 
862  len = strlen(content);
863 
864  /*
865  * Construct a stat struct for the backup_label file we're injecting in
866  * the tar.
867  */
868  /* Windows doesn't have the concept of uid and gid */
869 #ifdef WIN32
870  statbuf.st_uid = 0;
871  statbuf.st_gid = 0;
872 #else
873  statbuf.st_uid = geteuid();
874  statbuf.st_gid = getegid();
875 #endif
876  statbuf.st_mtime = time(NULL);
877  statbuf.st_mode = S_IRUSR | S_IWUSR;
878  statbuf.st_size = len;
879 
880  _tarWriteHeader(filename, NULL, &statbuf, false);
881  /* Send the contents as a CopyData message */
882  pq_putmessage('d', content, len);
883 
884  /* Pad to 512 byte boundary, per tar format requirements */
885  pad = ((len + 511) & ~511) - len;
886  if (pad > 0)
887  {
888  char buf[512];
889 
890  MemSet(buf, 0, pad);
891  pq_putmessage('d', buf, pad);
892  }
893 }
894 
895 /*
896  * Include the tablespace directory pointed to by 'path' in the output tar
897  * stream. If 'sizeonly' is true, we just calculate a total length and return
898  * it, without actually sending anything.
899  *
900  * Only used to send auxiliary tablespaces, not PGDATA.
901  */
902 int64
903 sendTablespace(char *path, bool sizeonly)
904 {
905  int64 size;
906  char pathbuf[MAXPGPATH];
907  struct stat statbuf;
908 
909  /*
910  * 'path' points to the tablespace location, but we only want to include
911  * the version directory in it that belongs to us.
912  */
913  snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
915 
916  /*
917  * Store a directory entry in the tar file so we get the permissions
918  * right.
919  */
920  if (lstat(pathbuf, &statbuf) != 0)
921  {
922  if (errno != ENOENT)
923  ereport(ERROR,
925  errmsg("could not stat file or directory \"%s\": %m",
926  pathbuf)));
927 
928  /* If the tablespace went away while scanning, it's no error. */
929  return 0;
930  }
931 
933  sizeonly);
934 
935  /* Send all the files in the tablespace version directory */
936  size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true);
937 
938  return size;
939 }
940 
941 /*
942  * Include all files from the given directory in the output tar stream. If
943  * 'sizeonly' is true, we just calculate a total length and return it, without
944  * actually sending anything.
945  *
946  * Omit any directory in the tablespaces list, to avoid backing up
947  * tablespaces twice when they were created inside PGDATA.
948  *
949  * If sendtblspclinks is true, we need to include symlink
950  * information in the tar file. If not, we can skip that
951  * as it will be sent separately in the tablespace_map file.
952  */
953 static int64
954 sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces,
955  bool sendtblspclinks)
956 {
957  DIR *dir;
958  struct dirent *de;
959  char pathbuf[MAXPGPATH];
960  struct stat statbuf;
961  int64 size = 0;
962 
963  dir = AllocateDir(path);
964  while ((de = ReadDir(dir, path)) != NULL)
965  {
966  int excludeIdx;
967  bool excludeFound;
968 
969  /* Skip special stuff */
970  if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
971  continue;
972 
973  /* Skip temporary files */
974  if (strncmp(de->d_name,
976  strlen(PG_TEMP_FILE_PREFIX)) == 0)
977  continue;
978 
979  /*
980  * Check if the postmaster has signaled us to exit, and abort with an
981  * error in that case. The error handler further up will call
982  * do_pg_abort_backup() for us. Also check that if the backup was
983  * started while still in recovery, the server wasn't promoted.
984  * dp_pg_stop_backup() will check that too, but it's better to stop
985  * the backup early than continue to the end and fail there.
986  */
989  ereport(ERROR,
990  (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
991  errmsg("the standby was promoted during online backup"),
992  errhint("This means that the backup being taken is corrupt "
993  "and should not be used. "
994  "Try taking another online backup.")));
995 
996  /* Scan for files that should be excluded */
997  excludeFound = false;
998  for (excludeIdx = 0; excludeFiles[excludeIdx] != NULL; excludeIdx++)
999  {
1000  if (strcmp(de->d_name, excludeFiles[excludeIdx]) == 0)
1001  {
1002  elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1003  excludeFound = true;
1004  break;
1005  }
1006  }
1007 
1008  if (excludeFound)
1009  continue;
1010 
1011  snprintf(pathbuf, MAXPGPATH, "%s/%s", path, de->d_name);
1012 
1013  /* Skip pg_control here to back up it last */
1014  if (strcmp(pathbuf, "./global/pg_control") == 0)
1015  continue;
1016 
1017  if (lstat(pathbuf, &statbuf) != 0)
1018  {
1019  if (errno != ENOENT)
1020  ereport(ERROR,
1022  errmsg("could not stat file or directory \"%s\": %m",
1023  pathbuf)));
1024 
1025  /* If the file went away while scanning, it's not an error. */
1026  continue;
1027  }
1028 
1029  /* Scan for directories whose contents should be excluded */
1030  excludeFound = false;
1031  for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1032  {
1033  if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1034  {
1035  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1036  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1037  excludeFound = true;
1038  break;
1039  }
1040  }
1041 
1042  if (excludeFound)
1043  continue;
1044 
1045  /*
1046  * Exclude contents of directory specified by statrelpath if not set
1047  * to the default (pg_stat_tmp) which is caught in the loop above.
1048  */
1049  if (statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0)
1050  {
1051  elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath);
1052  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1053  continue;
1054  }
1055 
1056  /*
1057  * We can skip pg_wal, the WAL segments need to be fetched from the
1058  * WAL archive anyway. But include it as an empty directory anyway, so
1059  * we get permissions right.
1060  */
1061  if (strcmp(pathbuf, "./pg_wal") == 0)
1062  {
1063  /* If pg_wal is a symlink, write it as a directory anyway */
1064  size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
1065 
1066  /*
1067  * Also send archive_status directory (by hackishly reusing
1068  * statbuf from above ...).
1069  */
1070  size += _tarWriteHeader("./pg_wal/archive_status", NULL, &statbuf,
1071  sizeonly);
1072 
1073  continue; /* don't recurse into pg_wal */
1074  }
1075 
1076  /* Allow symbolic links in pg_tblspc only */
1077  if (strcmp(path, "./pg_tblspc") == 0 &&
1078 #ifndef WIN32
1079  S_ISLNK(statbuf.st_mode)
1080 #else
1081  pgwin32_is_junction(pathbuf)
1082 #endif
1083  )
1084  {
1085 #if defined(HAVE_READLINK) || defined(WIN32)
1086  char linkpath[MAXPGPATH];
1087  int rllen;
1088 
1089  rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1090  if (rllen < 0)
1091  ereport(ERROR,
1093  errmsg("could not read symbolic link \"%s\": %m",
1094  pathbuf)));
1095  if (rllen >= sizeof(linkpath))
1096  ereport(ERROR,
1097  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1098  errmsg("symbolic link \"%s\" target is too long",
1099  pathbuf)));
1100  linkpath[rllen] = '\0';
1101 
1102  size += _tarWriteHeader(pathbuf + basepathlen + 1, linkpath,
1103  &statbuf, sizeonly);
1104 #else
1105 
1106  /*
1107  * If the platform does not have symbolic links, it should not be
1108  * possible to have tablespaces - clearly somebody else created
1109  * them. Warn about it and ignore.
1110  */
1111  ereport(WARNING,
1112  (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
1113  errmsg("tablespaces are not supported on this platform")));
1114  continue;
1115 #endif /* HAVE_READLINK */
1116  }
1117  else if (S_ISDIR(statbuf.st_mode))
1118  {
1119  bool skip_this_dir = false;
1120  ListCell *lc;
1121 
1122  /*
1123  * Store a directory entry in the tar file so we can get the
1124  * permissions right.
1125  */
1126  size += _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf,
1127  sizeonly);
1128 
1129  /*
1130  * Call ourselves recursively for a directory, unless it happens
1131  * to be a separate tablespace located within PGDATA.
1132  */
1133  foreach(lc, tablespaces)
1134  {
1135  tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1136 
1137  /*
1138  * ti->rpath is the tablespace relative path within PGDATA, or
1139  * NULL if the tablespace has been properly located somewhere
1140  * else.
1141  *
1142  * Skip past the leading "./" in pathbuf when comparing.
1143  */
1144  if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1145  {
1146  skip_this_dir = true;
1147  break;
1148  }
1149  }
1150 
1151  /*
1152  * skip sending directories inside pg_tblspc, if not required.
1153  */
1154  if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1155  skip_this_dir = true;
1156 
1157  if (!skip_this_dir)
1158  size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces, sendtblspclinks);
1159  }
1160  else if (S_ISREG(statbuf.st_mode))
1161  {
1162  bool sent = false;
1163 
1164  if (!sizeonly)
1165  sent = sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf,
1166  true);
1167 
1168  if (sent || sizeonly)
1169  {
1170  /* Add size, rounded up to 512byte block */
1171  size += ((statbuf.st_size + 511) & ~511);
1172  size += 512; /* Size of the header of the file */
1173  }
1174  }
1175  else
1176  ereport(WARNING,
1177  (errmsg("skipping special file \"%s\"", pathbuf)));
1178  }
1179  FreeDir(dir);
1180  return size;
1181 }
1182 
1183 /*****
1184  * Functions for handling tar file format
1185  *
1186  * Copied from pg_dump, but modified to work with libpq for sending
1187  */
1188 
1189 
1190 /*
1191  * Given the member, write the TAR header & send the file.
1192  *
1193  * If 'missing_ok' is true, will not throw an error if the file is not found.
1194  *
1195  * Returns true if the file was successfully sent, false if 'missing_ok',
1196  * and the file did not exist.
1197  */
1198 static bool
1199 sendFile(char *readfilename, char *tarfilename, struct stat * statbuf,
1200  bool missing_ok)
1201 {
1202  FILE *fp;
1203  char buf[TAR_SEND_SIZE];
1204  size_t cnt;
1205  pgoff_t len = 0;
1206  size_t pad;
1207 
1208  fp = AllocateFile(readfilename, "rb");
1209  if (fp == NULL)
1210  {
1211  if (errno == ENOENT && missing_ok)
1212  return false;
1213  ereport(ERROR,
1215  errmsg("could not open file \"%s\": %m", readfilename)));
1216  }
1217 
1218  _tarWriteHeader(tarfilename, NULL, statbuf, false);
1219 
1220  while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len), fp)) > 0)
1221  {
1222  /* Send the chunk as a CopyData message */
1223  if (pq_putmessage('d', buf, cnt))
1224  ereport(ERROR,
1225  (errmsg("base backup could not send data, aborting backup")));
1226 
1227  len += cnt;
1228  throttle(cnt);
1229 
1230  if (len >= statbuf->st_size)
1231  {
1232  /*
1233  * Reached end of file. The file could be longer, if it was
1234  * extended while we were sending it, but for a base backup we can
1235  * ignore such extended data. It will be restored from WAL.
1236  */
1237  break;
1238  }
1239  }
1240 
1241  /* If the file was truncated while we were sending it, pad it with zeros */
1242  if (len < statbuf->st_size)
1243  {
1244  MemSet(buf, 0, sizeof(buf));
1245  while (len < statbuf->st_size)
1246  {
1247  cnt = Min(sizeof(buf), statbuf->st_size - len);
1248  pq_putmessage('d', buf, cnt);
1249  len += cnt;
1250  throttle(cnt);
1251  }
1252  }
1253 
1254  /*
1255  * Pad to 512 byte boundary, per tar format requirements. (This small
1256  * piece of data is probably not worth throttling.)
1257  */
1258  pad = ((len + 511) & ~511) - len;
1259  if (pad > 0)
1260  {
1261  MemSet(buf, 0, pad);
1262  pq_putmessage('d', buf, pad);
1263  }
1264 
1265  FreeFile(fp);
1266 
1267  return true;
1268 }
1269 
1270 
1271 static int64
1272 _tarWriteHeader(const char *filename, const char *linktarget,
1273  struct stat * statbuf, bool sizeonly)
1274 {
1275  char h[512];
1276  enum tarError rc;
1277 
1278  if (!sizeonly)
1279  {
1280  rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size,
1281  statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
1282  statbuf->st_mtime);
1283 
1284  switch (rc)
1285  {
1286  case TAR_OK:
1287  break;
1288  case TAR_NAME_TOO_LONG:
1289  ereport(ERROR,
1290  (errmsg("file name too long for tar format: \"%s\"",
1291  filename)));
1292  break;
1293  case TAR_SYMLINK_TOO_LONG:
1294  ereport(ERROR,
1295  (errmsg("symbolic link target too long for tar format: "
1296  "file name \"%s\", target \"%s\"",
1297  filename, linktarget)));
1298  break;
1299  default:
1300  elog(ERROR, "unrecognized tar error: %d", rc);
1301  }
1302 
1303  pq_putmessage('d', h, sizeof(h));
1304  }
1305 
1306  return sizeof(h);
1307 }
1308 
1309 /*
1310  * Write tar header for a directory. If the entry in statbuf is a link then
1311  * write it as a directory anyway.
1312  */
1313 static int64
1314 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf,
1315  bool sizeonly)
1316 {
1317  /* If symlink, write it as a directory anyway */
1318 #ifndef WIN32
1319  if (S_ISLNK(statbuf->st_mode))
1320 #else
1321  if (pgwin32_is_junction(pathbuf))
1322 #endif
1323  statbuf->st_mode = S_IFDIR | S_IRWXU;
1324 
1325  return _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf, sizeonly);
1326 }
1327 
1328 /*
1329  * Increment the network transfer counter by the given number of bytes,
1330  * and sleep if necessary to comply with the requested network transfer
1331  * rate.
1332  */
1333 static void
1334 throttle(size_t increment)
1335 {
1336  TimeOffset elapsed,
1337  elapsed_min,
1338  sleep;
1339  int wait_result;
1340 
1341  if (throttling_counter < 0)
1342  return;
1343 
1344  throttling_counter += increment;
1346  return;
1347 
1348  /* Time elapsed since the last measurement (and possible wake up). */
1349  elapsed = GetCurrentTimestamp() - throttled_last;
1350  /* How much should have elapsed at minimum? */
1352  sleep = elapsed_min - elapsed;
1353  /* Only sleep if the transfer is faster than it should be. */
1354  if (sleep > 0)
1355  {
1357 
1358  /* We're eating a potentially set latch, so check for interrupts */
1360 
1361  /*
1362  * (TAR_SEND_SIZE / throttling_sample * elapsed_min_unit) should be
1363  * the maximum time to sleep. Thus the cast to long is safe.
1364  */
1365  wait_result = WaitLatch(MyLatch,
1367  (long) (sleep / 1000),
1369 
1370  if (wait_result & WL_LATCH_SET)
1372  }
1373 
1374  /*
1375  * As we work with integers, only whole multiple of throttling_sample was
1376  * processed. The rest will be done during the next call of this function.
1377  */
1379 
1380  /*
1381  * Time interval for the remaining amount and possible next increments
1382  * starts now.
1383  */
1385 }
#define StatusFilePath(path, xlog, suffix)
static int64 sendDir(char *path, int basepathlen, bool sizeonly, List *tablespaces, bool sendtblspclinks)
Definition: basebackup.c:954
List * options
Definition: replnodes.h:44
#define NIL
Definition: pg_list.h:69
#define XLogSegSize
Definition: xlog_internal.h:92
static bool sendFile(char *readfilename, char *tarfilename, struct stat *statbuf, bool missing_ok)
Definition: basebackup.c:1199
#define MAX_RATE_LOWER
Definition: basebackup.h:20
#define DEBUG1
Definition: elog.h:25
int errhint(const char *fmt,...)
Definition: elog.c:987
static void throttle(size_t increment)
Definition: basebackup.c:1334
uint32 TimeLineID
Definition: xlogdefs.h:45
void pq_sendbyte(StringInfo buf, int byt)
Definition: pqformat.c:105
#define WL_TIMEOUT
Definition: latch.h:127
#define PG_DYNSHMEM_DIR
Definition: dsm_impl.h:51
#define USECS_PER_SEC
Definition: timestamp.h:94
bool update_process_title
Definition: ps_status.c:35
TimestampTz GetCurrentTimestamp(void)
Definition: timestamp.c:1569
#define OIDOID
Definition: pg_type.h:328
#define TEXTOID
Definition: pg_type.h:324
int64 TimestampTz
Definition: timestamp.h:39
char * pstrdup(const char *in)
Definition: mcxt.c:1165
Definition: pgtar.h:17
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
char * rpath
Definition: basebackup.h:28
StringInfo makeStringInfo(void)
Definition: stringinfo.c:29
#define Min(x, y)
Definition: c.h:802
#define XLogFileName(fname, tli, logSegNo)
static void SendBackupHeader(List *tablespaces)
Definition: basebackup.c:723
void set_ps_display(const char *activity, bool force)
Definition: ps_status.c:326
static void parse_basebackup_options(List *options, basebackup_options *opt)
Definition: basebackup.c:577
#define strVal(v)
Definition: value.h:54
int errcode(int sqlerrcode)
Definition: elog.c:575
#define MemSet(start, val, len)
Definition: c.h:853
void pq_putemptymessage(char msgtype)
Definition: pqformat.c:421
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
#define PG_TEMP_FILE_PREFIX
Definition: fd.h:127
void pq_sendstring(StringInfo buf, const char *str)
Definition: pqformat.c:186
void ResetLatch(volatile Latch *latch)
Definition: latch.c:462
bool RecoveryInProgress(void)
Definition: xlog.c:7805
static bool backup_started_in_recovery
Definition: basebackup.c:73
static void send_int8_string(StringInfoData *buf, int64 intval)
Definition: basebackup.c:713
Definition: dirent.h:9
void pq_beginmessage(StringInfo buf, char msgtype)
Definition: pqformat.c:88
#define TABLESPACE_MAP
Definition: xlog.h:304
#define PG_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:47
int WaitLatch(volatile Latch *latch, int wakeEvents, long timeout, uint32 wait_event_info)
Definition: latch.c:301
#define MAX_RATE_UPPER
Definition: basebackup.h:21
Definition: dirent.c:25
tarError
Definition: pgtar.h:15
#define ERROR
Definition: elog.h:43
#define IsXLogFileName(fname)
XLogRecPtr do_pg_stop_backup(char *labelfile, bool waitforarchive, TimeLineID *stoptli_p)
Definition: xlog.c:10581
void CheckXLogRemoved(XLogSegNo segno, TimeLineID tli)
Definition: xlog.c:3743
void SendBaseBackup(BaseBackupCmd *cmd)
Definition: basebackup.c:683
#define MAXPGPATH
static int32 maxrate
Definition: pg_basebackup.c:95
static char * buf
Definition: pg_test_fsync.c:65
static void perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
Definition: basebackup.c:184
uint64 XLogSegNo
Definition: xlogdefs.h:34
int errcode_for_file_access(void)
Definition: elog.c:598
#define is_absolute_path(filename)
Definition: port.h:77
const char * label
Definition: basebackup.c:44
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2043
static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli)
Definition: basebackup.c:801
unsigned int uint32
Definition: c.h:265
int64 sendTablespace(char *path, bool sizeonly)
Definition: basebackup.c:903
static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1272
DIR * AllocateDir(const char *dirname)
Definition: fd.c:2284
char * pgstat_stat_directory
Definition: pgstat.c:118
static const char * excludeFiles[]
Definition: basebackup.c:146
#define XLByteToPrevSeg(xlrp, logSegNo)
#define lnext(lc)
Definition: pg_list.h:105
#define ereport(elevel, rest)
Definition: elog.h:122
static char * statrelpath
Definition: basebackup.c:76
Node * arg
Definition: parsenodes.h:676
List * lappend(List *list, void *datum)
Definition: list.c:128
#define WARNING
Definition: elog.h:40
#define MAXFNAMELEN
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:112
int64 TimeOffset
Definition: timestamp.h:40
#define pgoff_t
Definition: win32.h:241
#define WL_POSTMASTER_DEATH
Definition: latch.h:128
void * palloc0(Size size)
Definition: mcxt.c:920
#define XLOGDIR
uintptr_t Datum
Definition: postgres.h:374
#define XLOG_CONTROL_FILE
TimeLineID ThisTimeLineID
Definition: xlog.c:178
static TimestampTz throttled_last
Definition: basebackup.c:98
#define INT8OID
Definition: pg_type.h:304
static StringInfo tblspc_map_file
Definition: xlogfuncs.c:44
#define XLByteToSeg(xlrp, logSegNo)
#define NULL
Definition: c.h:226
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:671
#define lfirst(lc)
Definition: pg_list.h:106
void WalSndSetState(WalSndState state)
Definition: walsender.c:2655
#define PG_STAT_TMP_DIR
Definition: pgstat.h:34
struct dirent * ReadDir(DIR *dir, const char *dirname)
Definition: fd.c:2350
#define THROTTLING_FREQUENCY
Definition: basebackup.c:86
size_t Size
Definition: c.h:353
static int list_length(const List *l)
Definition: pg_list.h:89
#define PG_AUTOCONF_FILENAME
Definition: guc.h:34
#define PG_END_ENSURE_ERROR_CLEANUP(cleanup_function, arg)
Definition: ipc.h:52
#define TABLESPACE_VERSION_DIRECTORY
Definition: catalog.h:26
#define INT64_FORMAT
Definition: c.h:312
int FreeFile(FILE *file)
Definition: fd.c:2226
#define IsTLHistoryFileName(fname)
static char * filename
Definition: pg_dumpall.c:84
#define intVal(v)
Definition: value.h:52
void pq_sendbytes(StringInfo buf, const char *data, int datalen)
Definition: pqformat.c:115
void * palloc(Size size)
Definition: mcxt.c:891
int errmsg(const char *fmt,...)
Definition: elog.c:797
void pq_sendint(StringInfo buf, int i, int b)
Definition: pqformat.c:236
void pq_endmessage(StringInfo buf)
Definition: pqformat.c:344
static const char * excludeDirContents[]
Definition: basebackup.c:105
#define TAR_SEND_SIZE
Definition: basebackup.c:81
int i
void do_pg_abort_backup(void)
Definition: xlog.c:10964
void * arg
char * DataDir
Definition: globals.c:59
struct Latch * MyLatch
Definition: globals.c:51
#define BACKUP_LABEL_FILE
Definition: xlog.h:301
char * defname
Definition: parsenodes.h:675
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:97
char d_name[MAX_PATH]
Definition: dirent.h:14
#define elog
Definition: elog.h:219
#define pq_putmessage(msgtype, s, len)
Definition: libpq.h:43
static int64 throttling_counter
Definition: basebackup.c:92
#define qsort(a, b, c, d)
Definition: port.h:440
void pq_puttextmessage(char msgtype, const char *str)
Definition: pqformat.c:400
#define XLogFromFileName(fname, tli, logSegNo)
static void base_backup_cleanup(int code, Datum arg)
Definition: basebackup.c:172
#define lstat(path, sb)
Definition: win32.h:272
Definition: pg_list.h:45
#define WL_LATCH_SET
Definition: latch.h:124
static uint64 throttling_sample
Definition: basebackup.c:89
static void sendFileWithContent(const char *filename, const char *content)
Definition: basebackup.c:856
int FreeDir(DIR *dir)
Definition: fd.c:2393
static TimeOffset elapsed_min_unit
Definition: basebackup.c:95
XLogRecPtr do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p, StringInfo labelfile, DIR *tblspcdir, List **tablespaces, StringInfo tblspcmapfile, bool infotbssize, bool needtblspcmapfile)
Definition: xlog.c:10087
static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly)
Definition: basebackup.c:1314
static int compareWalFileNames(const void *a, const void *b)
Definition: basebackup.c:565