PostgreSQL Source Code  git master
pg_basebackup.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_basebackup.c - receive a base backup using streaming replication protocol
4  *
5  * Author: Magnus Hagander <magnus@hagander.net>
6  *
7  * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
8  *
9  * IDENTIFICATION
10  * src/bin/pg_basebackup/pg_basebackup.c
11  *-------------------------------------------------------------------------
12  */
13 
14 #include "postgres_fe.h"
15 
16 #include <unistd.h>
17 #include <dirent.h>
18 #include <sys/stat.h>
19 #include <sys/wait.h>
20 #include <signal.h>
21 #include <time.h>
22 #ifdef HAVE_SYS_SELECT_H
23 #include <sys/select.h>
24 #endif
25 #ifdef HAVE_LIBZ
26 #include <zlib.h>
27 #endif
28 
29 #include "access/xlog_internal.h"
30 #include "common/file_perm.h"
31 #include "common/file_utils.h"
32 #include "common/logging.h"
33 #include "common/string.h"
34 #include "fe_utils/recovery_gen.h"
35 #include "fe_utils/string_utils.h"
36 #include "getopt_long.h"
37 #include "libpq-fe.h"
38 #include "pgtar.h"
39 #include "pgtime.h"
40 #include "pqexpbuffer.h"
41 #include "receivelog.h"
42 #include "replication/basebackup.h"
43 #include "streamutil.h"
44 
45 #define ERRCODE_DATA_CORRUPTED "XX001"
46 
47 typedef struct TablespaceListCell
48 {
53 
54 typedef struct TablespaceList
55 {
59 
60 typedef struct WriteTarState
61 {
64  FILE *tarfile;
65  char tarhdr[512];
67  bool in_tarhdr;
68  bool skip_file;
73  size_t tarhdrsz;
75 #ifdef HAVE_LIBZ
76  gzFile ztarfile;
77 #endif
79 
80 typedef struct UnpackTarState
81 {
83  char current_path[MAXPGPATH];
85  const char *mapped_tblspc_path;
88  FILE *file;
90 
91 typedef struct WriteManifestState
92 {
94  FILE *file;
96 
97 typedef void (*WriteDataCallback) (size_t nbytes, char *buf,
98  void *callback_data);
99 
100 /*
101  * pg_xlog has been renamed to pg_wal in version 10. This version number
102  * should be compared with PQserverVersion().
103  */
104 #define MINIMUM_VERSION_FOR_PG_WAL 100000
105 
106 /*
107  * Temporary replication slots are supported from version 10.
108  */
109 #define MINIMUM_VERSION_FOR_TEMP_SLOTS 100000
110 
111 /*
112  * Different ways to include WAL
113  */
114 typedef enum
115 {
119 } IncludeWal;
120 
121 /* Global options */
122 static char *basedir = NULL;
123 static TablespaceList tablespace_dirs = {NULL, NULL};
124 static char *xlog_dir = NULL;
125 static char format = 'p'; /* p(lain)/t(ar) */
126 static char *label = "pg_basebackup base backup";
127 static bool noclean = false;
128 static bool checksum_failure = false;
129 static bool showprogress = false;
130 static bool estimatesize = true;
131 static int verbose = 0;
132 static int compresslevel = 0;
134 static bool fastcheckpoint = false;
135 static bool writerecoveryconf = false;
136 static bool do_sync = true;
137 static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
139 static int32 maxrate = 0; /* no limit by default */
140 static char *replication_slot = NULL;
141 static bool temp_replication_slot = true;
142 static bool create_slot = false;
143 static bool no_slot = false;
144 static bool verify_checksums = true;
145 static bool manifest = true;
146 static bool manifest_force_encode = false;
147 static char *manifest_checksums = NULL;
148 
149 static bool success = false;
150 static bool made_new_pgdata = false;
151 static bool found_existing_pgdata = false;
152 static bool made_new_xlogdir = false;
153 static bool found_existing_xlogdir = false;
154 static bool made_tablespace_dirs = false;
155 static bool found_tablespace_dirs = false;
156 
157 /* Progress counters */
158 static uint64 totalsize_kb;
159 static uint64 totaldone;
160 static int tablespacecount;
161 
162 /* Pipe to communicate with background wal receiver process */
163 #ifndef WIN32
164 static int bgpipe[2] = {-1, -1};
165 #endif
166 
167 /* Handle to child process */
168 static pid_t bgchild = -1;
169 static bool in_log_streamer = false;
170 
171 /* End position for xlog streaming, empty string if unknown yet */
173 
174 #ifndef WIN32
175 static int has_xlogendptr = 0;
176 #else
177 static volatile LONG has_xlogendptr = 0;
178 #endif
179 
180 /* Contents of configuration file to be generated */
182 
183 /* Function headers */
184 static void usage(void);
185 static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found);
186 static void progress_report(int tablespacenum, const char *filename, bool force);
187 
188 static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum);
189 static void ReceiveTarCopyChunk(size_t r, char *copybuf, void *callback_data);
190 static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum);
191 static void ReceiveTarAndUnpackCopyChunk(size_t r, char *copybuf,
192  void *callback_data);
193 static void ReceiveBackupManifest(PGconn *conn);
194 static void ReceiveBackupManifestChunk(size_t r, char *copybuf,
195  void *callback_data);
197 static void ReceiveBackupManifestInMemoryChunk(size_t r, char *copybuf,
198  void *callback_data);
199 static void BaseBackup(void);
200 
201 static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline,
202  bool segment_finished);
203 
204 static const char *get_tablespace_mapping(const char *dir);
205 static void tablespace_list_append(const char *arg);
206 
207 
208 static void
210 {
211  if (success || in_log_streamer)
212  return;
213 
214  if (!noclean && !checksum_failure)
215  {
216  if (made_new_pgdata)
217  {
218  pg_log_info("removing data directory \"%s\"", basedir);
219  if (!rmtree(basedir, true))
220  pg_log_error("failed to remove data directory");
221  }
222  else if (found_existing_pgdata)
223  {
224  pg_log_info("removing contents of data directory \"%s\"", basedir);
225  if (!rmtree(basedir, false))
226  pg_log_error("failed to remove contents of data directory");
227  }
228 
229  if (made_new_xlogdir)
230  {
231  pg_log_info("removing WAL directory \"%s\"", xlog_dir);
232  if (!rmtree(xlog_dir, true))
233  pg_log_error("failed to remove WAL directory");
234  }
235  else if (found_existing_xlogdir)
236  {
237  pg_log_info("removing contents of WAL directory \"%s\"", xlog_dir);
238  if (!rmtree(xlog_dir, false))
239  pg_log_error("failed to remove contents of WAL directory");
240  }
241  }
242  else
243  {
245  pg_log_info("data directory \"%s\" not removed at user's request", basedir);
246 
248  pg_log_info("WAL directory \"%s\" not removed at user's request", xlog_dir);
249  }
250 
252  pg_log_info("changes to tablespace directories will not be undone");
253 }
254 
255 static void
257 {
258  if (conn != NULL)
259  PQfinish(conn);
260 }
261 
262 #ifndef WIN32
263 /*
264  * On windows, our background thread dies along with the process. But on
265  * Unix, if we have started a subprocess, we want to kill it off so it
266  * doesn't remain running trying to stream data.
267  */
268 static void
270 {
271  if (bgchild > 0)
272  kill(bgchild, SIGTERM);
273 }
274 #endif
275 
276 /*
277  * Split argument into old_dir and new_dir and append to tablespace mapping
278  * list.
279  */
280 static void
282 {
284  char *dst;
285  char *dst_ptr;
286  const char *arg_ptr;
287 
288  dst_ptr = dst = cell->old_dir;
289  for (arg_ptr = arg; *arg_ptr; arg_ptr++)
290  {
291  if (dst_ptr - dst >= MAXPGPATH)
292  {
293  pg_log_error("directory name too long");
294  exit(1);
295  }
296 
297  if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
298  ; /* skip backslash escaping = */
299  else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
300  {
301  if (*cell->new_dir)
302  {
303  pg_log_error("multiple \"=\" signs in tablespace mapping");
304  exit(1);
305  }
306  else
307  dst = dst_ptr = cell->new_dir;
308  }
309  else
310  *dst_ptr++ = *arg_ptr;
311  }
312 
313  if (!*cell->old_dir || !*cell->new_dir)
314  {
315  pg_log_error("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
316  exit(1);
317  }
318 
319  /*
320  * This check isn't absolutely necessary. But all tablespaces are created
321  * with absolute directories, so specifying a non-absolute path here would
322  * just never match, possibly confusing users. It's also good to be
323  * consistent with the new_dir check.
324  */
325  if (!is_absolute_path(cell->old_dir))
326  {
327  pg_log_error("old directory is not an absolute path in tablespace mapping: %s",
328  cell->old_dir);
329  exit(1);
330  }
331 
332  if (!is_absolute_path(cell->new_dir))
333  {
334  pg_log_error("new directory is not an absolute path in tablespace mapping: %s",
335  cell->new_dir);
336  exit(1);
337  }
338 
339  /*
340  * Comparisons done with these values should involve similarly
341  * canonicalized path values. This is particularly sensitive on Windows
342  * where path values may not necessarily use Unix slashes.
343  */
344  canonicalize_path(cell->old_dir);
345  canonicalize_path(cell->new_dir);
346 
347  if (tablespace_dirs.tail)
348  tablespace_dirs.tail->next = cell;
349  else
350  tablespace_dirs.head = cell;
351  tablespace_dirs.tail = cell;
352 }
353 
354 
355 #ifdef HAVE_LIBZ
356 static const char *
357 get_gz_error(gzFile gzf)
358 {
359  int errnum;
360  const char *errmsg;
361 
362  errmsg = gzerror(gzf, &errnum);
363  if (errnum == Z_ERRNO)
364  return strerror(errno);
365  else
366  return errmsg;
367 }
368 #endif
369 
370 static void
371 usage(void)
372 {
373  printf(_("%s takes a base backup of a running PostgreSQL server.\n\n"),
374  progname);
375  printf(_("Usage:\n"));
376  printf(_(" %s [OPTION]...\n"), progname);
377  printf(_("\nOptions controlling the output:\n"));
378  printf(_(" -D, --pgdata=DIRECTORY receive base backup into directory\n"));
379  printf(_(" -F, --format=p|t output format (plain (default), tar)\n"));
380  printf(_(" -r, --max-rate=RATE maximum transfer rate to transfer data directory\n"
381  " (in kB/s, or use suffix \"k\" or \"M\")\n"));
382  printf(_(" -R, --write-recovery-conf\n"
383  " write configuration for replication\n"));
384  printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
385  " relocate tablespace in OLDDIR to NEWDIR\n"));
386  printf(_(" --waldir=WALDIR location for the write-ahead log directory\n"));
387  printf(_(" -X, --wal-method=none|fetch|stream\n"
388  " include required WAL files with specified method\n"));
389  printf(_(" -z, --gzip compress tar output\n"));
390  printf(_(" -Z, --compress=0-9 compress tar output with given compression level\n"));
391  printf(_("\nGeneral options:\n"));
392  printf(_(" -c, --checkpoint=fast|spread\n"
393  " set fast or spread checkpointing\n"));
394  printf(_(" -C, --create-slot create replication slot\n"));
395  printf(_(" -l, --label=LABEL set backup label\n"));
396  printf(_(" -n, --no-clean do not clean up after errors\n"));
397  printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
398  printf(_(" -P, --progress show progress information\n"));
399  printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
400  printf(_(" -v, --verbose output verbose messages\n"));
401  printf(_(" -V, --version output version information, then exit\n"));
402  printf(_(" --no-slot prevent creation of temporary replication slot\n"));
403  printf(_(" --no-verify-checksums\n"
404  " do not verify checksums\n"));
405  printf(_(" --no-estimate-size do not estimate backup size in server side\n"));
406  printf(_(" --no-manifest suppress generation of backup manifest\n"));
407  printf(_(" --manifest-force-encode\n"
408  " hex encode all filenames in manifest\n"));
409  printf(_(" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
410  " use algorithm for manifest checksums\n"));
411  printf(_(" -?, --help show this help, then exit\n"));
412  printf(_("\nConnection options:\n"));
413  printf(_(" -d, --dbname=CONNSTR connection string\n"));
414  printf(_(" -h, --host=HOSTNAME database server host or socket directory\n"));
415  printf(_(" -p, --port=PORT database server port number\n"));
416  printf(_(" -s, --status-interval=INTERVAL\n"
417  " time between status packets sent to server (in seconds)\n"));
418  printf(_(" -U, --username=NAME connect as specified database user\n"));
419  printf(_(" -w, --no-password never prompt for password\n"));
420  printf(_(" -W, --password force password prompt (should happen automatically)\n"));
421  printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
422  printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
423 }
424 
425 
426 /*
427  * Called in the background process every time data is received.
428  * On Unix, we check to see if there is any data on our pipe
429  * (which would mean we have a stop position), and if it is, check if
430  * it is time to stop.
431  * On Windows, we are in a single process, so we can just check if it's
432  * time to stop.
433  */
434 static bool
436  bool segment_finished)
437 {
438  if (!has_xlogendptr)
439  {
440 #ifndef WIN32
441  fd_set fds;
442  struct timeval tv;
443  int r;
444 
445  /*
446  * Don't have the end pointer yet - check our pipe to see if it has
447  * been sent yet.
448  */
449  FD_ZERO(&fds);
450  FD_SET(bgpipe[0], &fds);
451 
452  MemSet(&tv, 0, sizeof(tv));
453 
454  r = select(bgpipe[0] + 1, &fds, NULL, NULL, &tv);
455  if (r == 1)
456  {
457  char xlogend[64];
458  uint32 hi,
459  lo;
460 
461  MemSet(xlogend, 0, sizeof(xlogend));
462  r = read(bgpipe[0], xlogend, sizeof(xlogend) - 1);
463  if (r < 0)
464  {
465  pg_log_error("could not read from ready pipe: %m");
466  exit(1);
467  }
468 
469  if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
470  {
471  pg_log_error("could not parse write-ahead log location \"%s\"",
472  xlogend);
473  exit(1);
474  }
475  xlogendptr = ((uint64) hi) << 32 | lo;
476  has_xlogendptr = 1;
477 
478  /*
479  * Fall through to check if we've reached the point further
480  * already.
481  */
482  }
483  else
484  {
485  /*
486  * No data received on the pipe means we don't know the end
487  * position yet - so just say it's not time to stop yet.
488  */
489  return false;
490  }
491 #else
492 
493  /*
494  * On win32, has_xlogendptr is set by the main thread, so if it's not
495  * set here, we just go back and wait until it shows up.
496  */
497  return false;
498 #endif
499  }
500 
501  /*
502  * At this point we have an end pointer, so compare it to the current
503  * position to figure out if it's time to stop.
504  */
505  if (segendpos >= xlogendptr)
506  return true;
507 
508  /*
509  * Have end pointer, but haven't reached it yet - so tell the caller to
510  * keep streaming.
511  */
512  return false;
513 }
514 
515 typedef struct
516 {
519  char xlog[MAXPGPATH]; /* directory or tarfile depending on mode */
521  int timeline;
523 
524 static int
526 {
527  StreamCtl stream;
528 
529  in_log_streamer = true;
530 
531  MemSet(&stream, 0, sizeof(stream));
532  stream.startpos = param->startptr;
533  stream.timeline = param->timeline;
534  stream.sysidentifier = param->sysidentifier;
536 #ifndef WIN32
537  stream.stop_socket = bgpipe[0];
538 #else
539  stream.stop_socket = PGINVALID_SOCKET;
540 #endif
542  stream.synchronous = false;
543  /* fsync happens at the end of pg_basebackup for all data */
544  stream.do_sync = false;
545  stream.mark_done = true;
546  stream.partial_suffix = NULL;
548 
549  if (format == 'p')
550  stream.walmethod = CreateWalDirectoryMethod(param->xlog, 0,
551  stream.do_sync);
552  else
554  stream.do_sync);
555 
556  if (!ReceiveXlogStream(param->bgconn, &stream))
557 
558  /*
559  * Any errors will already have been reported in the function process,
560  * but we need to tell the parent that we didn't shutdown in a nice
561  * way.
562  */
563  return 1;
564 
565  if (!stream.walmethod->finish())
566  {
567  pg_log_error("could not finish writing WAL files: %m");
568  return 1;
569  }
570 
571  PQfinish(param->bgconn);
572 
573  if (format == 'p')
575  else
577  pg_free(stream.walmethod);
578 
579  return 0;
580 }
581 
582 /*
583  * Initiate background process for receiving xlog during the backup.
584  * The background stream will use its own database connection so we can
585  * stream the logfile in parallel with the backups.
586  */
587 static void
588 StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
589 {
590  logstreamer_param *param;
591  uint32 hi,
592  lo;
593  char statusdir[MAXPGPATH];
594 
595  param = pg_malloc0(sizeof(logstreamer_param));
596  param->timeline = timeline;
597  param->sysidentifier = sysidentifier;
598 
599  /* Convert the starting position */
600  if (sscanf(startpos, "%X/%X", &hi, &lo) != 2)
601  {
602  pg_log_error("could not parse write-ahead log location \"%s\"",
603  startpos);
604  exit(1);
605  }
606  param->startptr = ((uint64) hi) << 32 | lo;
607  /* Round off to even segment position */
608  param->startptr -= XLogSegmentOffset(param->startptr, WalSegSz);
609 
610 #ifndef WIN32
611  /* Create our background pipe */
612  if (pipe(bgpipe) < 0)
613  {
614  pg_log_error("could not create pipe for background process: %m");
615  exit(1);
616  }
617 #endif
618 
619  /* Get a second connection */
620  param->bgconn = GetConnection();
621  if (!param->bgconn)
622  /* Error message already written in GetConnection() */
623  exit(1);
624 
625  /* In post-10 cluster, pg_xlog has been renamed to pg_wal */
626  snprintf(param->xlog, sizeof(param->xlog), "%s/%s",
627  basedir,
629  "pg_xlog" : "pg_wal");
630 
631  /* Temporary replication slots are only supported in 10 and newer */
633  temp_replication_slot = false;
634 
635  /*
636  * Create replication slot if requested
637  */
639  replication_slot = psprintf("pg_basebackup_%d", (int) PQbackendPID(param->bgconn));
641  {
642  if (!CreateReplicationSlot(param->bgconn, replication_slot, NULL,
643  temp_replication_slot, true, true, false))
644  exit(1);
645 
646  if (verbose)
647  {
649  pg_log_info("created temporary replication slot \"%s\"",
651  else
652  pg_log_info("created replication slot \"%s\"",
654  }
655  }
656 
657  if (format == 'p')
658  {
659  /*
660  * Create pg_wal/archive_status or pg_xlog/archive_status (and thus
661  * pg_wal or pg_xlog) depending on the target server so we can write
662  * to basedir/pg_wal or basedir/pg_xlog as the directory entry in the
663  * tar file may arrive later.
664  */
665  snprintf(statusdir, sizeof(statusdir), "%s/%s/archive_status",
666  basedir,
668  "pg_xlog" : "pg_wal");
669 
670  if (pg_mkdir_p(statusdir, pg_dir_create_mode) != 0 && errno != EEXIST)
671  {
672  pg_log_error("could not create directory \"%s\": %m", statusdir);
673  exit(1);
674  }
675  }
676 
677  /*
678  * Start a child process and tell it to start streaming. On Unix, this is
679  * a fork(). On Windows, we create a thread.
680  */
681 #ifndef WIN32
682  bgchild = fork();
683  if (bgchild == 0)
684  {
685  /* in child process */
686  exit(LogStreamerMain(param));
687  }
688  else if (bgchild < 0)
689  {
690  pg_log_error("could not create background process: %m");
691  exit(1);
692  }
693 
694  /*
695  * Else we are in the parent process and all is well.
696  */
697  atexit(kill_bgchild_atexit);
698 #else /* WIN32 */
699  bgchild = _beginthreadex(NULL, 0, (void *) LogStreamerMain, param, 0, NULL);
700  if (bgchild == 0)
701  {
702  pg_log_error("could not create background thread: %m");
703  exit(1);
704  }
705 #endif
706 }
707 
708 /*
709  * Verify that the given directory exists and is empty. If it does not
710  * exist, it is created. If it exists but is not empty, an error will
711  * be given and the process ended.
712  */
713 static void
714 verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
715 {
716  switch (pg_check_dir(dirname))
717  {
718  case 0:
719 
720  /*
721  * Does not exist, so create
722  */
723  if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
724  {
725  pg_log_error("could not create directory \"%s\": %m", dirname);
726  exit(1);
727  }
728  if (created)
729  *created = true;
730  return;
731  case 1:
732 
733  /*
734  * Exists, empty
735  */
736  if (found)
737  *found = true;
738  return;
739  case 2:
740  case 3:
741  case 4:
742 
743  /*
744  * Exists, not empty
745  */
746  pg_log_error("directory \"%s\" exists but is not empty", dirname);
747  exit(1);
748  case -1:
749 
750  /*
751  * Access problem
752  */
753  pg_log_error("could not access directory \"%s\": %m", dirname);
754  exit(1);
755  }
756 }
757 
758 
759 /*
760  * Print a progress report based on the global variables. If verbose output
761  * is enabled, also print the current file name.
762  *
763  * Progress report is written at maximum once per second, unless the
764  * force parameter is set to true.
765  */
766 static void
767 progress_report(int tablespacenum, const char *filename, bool force)
768 {
769  int percent;
770  char totaldone_str[32];
771  char totalsize_str[32];
772  pg_time_t now;
773 
774  if (!showprogress)
775  return;
776 
777  now = time(NULL);
778  if (now == last_progress_report && !force)
779  return; /* Max once per second */
780 
782  percent = totalsize_kb ? (int) ((totaldone / 1024) * 100 / totalsize_kb) : 0;
783 
784  /*
785  * Avoid overflowing past 100% or the full size. This may make the total
786  * size number change as we approach the end of the backup (the estimate
787  * will always be wrong if WAL is included), but that's better than having
788  * the done column be bigger than the total.
789  */
790  if (percent > 100)
791  percent = 100;
792  if (totaldone / 1024 > totalsize_kb)
793  totalsize_kb = totaldone / 1024;
794 
795  /*
796  * Separate step to keep platform-dependent format code out of
797  * translatable strings. And we only test for INT64_FORMAT availability
798  * in snprintf, not fprintf.
799  */
800  snprintf(totaldone_str, sizeof(totaldone_str), INT64_FORMAT,
801  totaldone / 1024);
802  snprintf(totalsize_str, sizeof(totalsize_str), INT64_FORMAT, totalsize_kb);
803 
804 #define VERBOSE_FILENAME_LENGTH 35
805  if (verbose)
806  {
807  if (!filename)
808 
809  /*
810  * No filename given, so clear the status line (used for last
811  * call)
812  */
813  fprintf(stderr,
814  ngettext("%*s/%s kB (100%%), %d/%d tablespace %*s",
815  "%*s/%s kB (100%%), %d/%d tablespaces %*s",
817  (int) strlen(totalsize_str),
818  totaldone_str, totalsize_str,
819  tablespacenum, tablespacecount,
820  VERBOSE_FILENAME_LENGTH + 5, "");
821  else
822  {
823  bool truncate = (strlen(filename) > VERBOSE_FILENAME_LENGTH);
824 
825  fprintf(stderr,
826  ngettext("%*s/%s kB (%d%%), %d/%d tablespace (%s%-*.*s)",
827  "%*s/%s kB (%d%%), %d/%d tablespaces (%s%-*.*s)",
829  (int) strlen(totalsize_str),
830  totaldone_str, totalsize_str, percent,
831  tablespacenum, tablespacecount,
832  /* Prefix with "..." if we do leading truncation */
833  truncate ? "..." : "",
836  /* Truncate filename at beginning if it's too long */
837  truncate ? filename + strlen(filename) - VERBOSE_FILENAME_LENGTH + 3 : filename);
838  }
839  }
840  else
841  fprintf(stderr,
842  ngettext("%*s/%s kB (%d%%), %d/%d tablespace",
843  "%*s/%s kB (%d%%), %d/%d tablespaces",
845  (int) strlen(totalsize_str),
846  totaldone_str, totalsize_str, percent,
847  tablespacenum, tablespacecount);
848 
849  if (isatty(fileno(stderr)))
850  fprintf(stderr, "\r");
851  else
852  fprintf(stderr, "\n");
853 }
854 
855 static int32
856 parse_max_rate(char *src)
857 {
858  double result;
859  char *after_num;
860  char *suffix = NULL;
861 
862  errno = 0;
863  result = strtod(src, &after_num);
864  if (src == after_num)
865  {
866  pg_log_error("transfer rate \"%s\" is not a valid value", src);
867  exit(1);
868  }
869  if (errno != 0)
870  {
871  pg_log_error("invalid transfer rate \"%s\": %m", src);
872  exit(1);
873  }
874 
875  if (result <= 0)
876  {
877  /*
878  * Reject obviously wrong values here.
879  */
880  pg_log_error("transfer rate must be greater than zero");
881  exit(1);
882  }
883 
884  /*
885  * Evaluate suffix, after skipping over possible whitespace. Lack of
886  * suffix means kilobytes.
887  */
888  while (*after_num != '\0' && isspace((unsigned char) *after_num))
889  after_num++;
890 
891  if (*after_num != '\0')
892  {
893  suffix = after_num;
894  if (*after_num == 'k')
895  {
896  /* kilobyte is the expected unit. */
897  after_num++;
898  }
899  else if (*after_num == 'M')
900  {
901  after_num++;
902  result *= 1024.0;
903  }
904  }
905 
906  /* The rest can only consist of white space. */
907  while (*after_num != '\0' && isspace((unsigned char) *after_num))
908  after_num++;
909 
910  if (*after_num != '\0')
911  {
912  pg_log_error("invalid --max-rate unit: \"%s\"", suffix);
913  exit(1);
914  }
915 
916  /* Valid integer? */
917  if ((uint64) result != (uint64) ((uint32) result))
918  {
919  pg_log_error("transfer rate \"%s\" exceeds integer range", src);
920  exit(1);
921  }
922 
923  /*
924  * The range is checked on the server side too, but avoid the server
925  * connection if a nonsensical value was passed.
926  */
927  if (result < MAX_RATE_LOWER || result > MAX_RATE_UPPER)
928  {
929  pg_log_error("transfer rate \"%s\" is out of range", src);
930  exit(1);
931  }
932 
933  return (int32) result;
934 }
935 
936 /*
937  * Read a stream of COPY data and invoke the provided callback for each
938  * chunk.
939  */
940 static void
942  void *callback_data)
943 {
944  PGresult *res;
945 
946  /* Get the COPY data stream. */
947  res = PQgetResult(conn);
948  if (PQresultStatus(res) != PGRES_COPY_OUT)
949  {
950  pg_log_error("could not get COPY data stream: %s",
951  PQerrorMessage(conn));
952  exit(1);
953  }
954  PQclear(res);
955 
956  /* Loop over chunks until done. */
957  while (1)
958  {
959  int r;
960  char *copybuf;
961 
962  r = PQgetCopyData(conn, &copybuf, 0);
963  if (r == -1)
964  {
965  /* End of chunk. */
966  break;
967  }
968  else if (r == -2)
969  {
970  pg_log_error("could not read COPY data: %s",
971  PQerrorMessage(conn));
972  exit(1);
973  }
974 
975  (*callback) (r, copybuf, callback_data);
976 
977  PQfreemem(copybuf);
978  }
979 }
980 
981 /*
982  * Write a piece of tar data
983  */
984 static void
986 {
987 #ifdef HAVE_LIBZ
988  if (state->ztarfile != NULL)
989  {
990  if (gzwrite(state->ztarfile, buf, r) != r)
991  {
992  pg_log_error("could not write to compressed file \"%s\": %s",
993  state->filename, get_gz_error(state->ztarfile));
994  exit(1);
995  }
996  }
997  else
998 #endif
999  {
1000  if (fwrite(buf, r, 1, state->tarfile) != 1)
1001  {
1002  pg_log_error("could not write to file \"%s\": %m",
1003  state->filename);
1004  exit(1);
1005  }
1006  }
1007 }
1008 
1009 /*
1010  * Receive a tar format file from the connection to the server, and write
1011  * the data from this file directly into a tar file. If compression is
1012  * enabled, the data will be compressed while written to the file.
1013  *
1014  * The file will be named base.tar[.gz] if it's for the main data directory
1015  * or <tablespaceoid>.tar[.gz] if it's for another tablespace.
1016  *
1017  * No attempt to inspect or validate the contents of the file is done.
1018  */
1019 static void
1020 ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
1021 {
1022  char zerobuf[1024];
1024 
1025  memset(&state, 0, sizeof(state));
1026  state.tablespacenum = rownum;
1027  state.basetablespace = PQgetisnull(res, rownum, 0);
1028  state.in_tarhdr = true;
1029 
1030  /* recovery.conf is integrated into postgresql.conf in 12 and newer */
1032  state.is_recovery_guc_supported = true;
1033 
1034  if (state.basetablespace)
1035  {
1036  /*
1037  * Base tablespaces
1038  */
1039  if (strcmp(basedir, "-") == 0)
1040  {
1041 #ifdef WIN32
1042  _setmode(fileno(stdout), _O_BINARY);
1043 #endif
1044 
1045 #ifdef HAVE_LIBZ
1046  if (compresslevel != 0)
1047  {
1048  int fd = dup(fileno(stdout));
1049  if (fd < 0)
1050  {
1051  pg_log_error("could not duplicate stdout: %m");
1052  exit(1);
1053  }
1054 
1055  state.ztarfile = gzdopen(fd, "wb");
1056  if (state.ztarfile == NULL)
1057  {
1058  pg_log_error("could not open output file: %m");
1059  exit(1);
1060  }
1061 
1062  if (gzsetparams(state.ztarfile, compresslevel,
1063  Z_DEFAULT_STRATEGY) != Z_OK)
1064  {
1065  pg_log_error("could not set compression level %d: %s",
1066  compresslevel, get_gz_error(state.ztarfile));
1067  exit(1);
1068  }
1069  }
1070  else
1071 #endif
1072  state.tarfile = stdout;
1073  strcpy(state.filename, "-");
1074  }
1075  else
1076  {
1077 #ifdef HAVE_LIBZ
1078  if (compresslevel != 0)
1079  {
1080  snprintf(state.filename, sizeof(state.filename),
1081  "%s/base.tar.gz", basedir);
1082  state.ztarfile = gzopen(state.filename, "wb");
1083  if (gzsetparams(state.ztarfile, compresslevel,
1084  Z_DEFAULT_STRATEGY) != Z_OK)
1085  {
1086  pg_log_error("could not set compression level %d: %s",
1087  compresslevel, get_gz_error(state.ztarfile));
1088  exit(1);
1089  }
1090  }
1091  else
1092 #endif
1093  {
1094  snprintf(state.filename, sizeof(state.filename),
1095  "%s/base.tar", basedir);
1096  state.tarfile = fopen(state.filename, "wb");
1097  }
1098  }
1099  }
1100  else
1101  {
1102  /*
1103  * Specific tablespace
1104  */
1105 #ifdef HAVE_LIBZ
1106  if (compresslevel != 0)
1107  {
1108  snprintf(state.filename, sizeof(state.filename),
1109  "%s/%s.tar.gz",
1110  basedir, PQgetvalue(res, rownum, 0));
1111  state.ztarfile = gzopen(state.filename, "wb");
1112  if (gzsetparams(state.ztarfile, compresslevel,
1113  Z_DEFAULT_STRATEGY) != Z_OK)
1114  {
1115  pg_log_error("could not set compression level %d: %s",
1116  compresslevel, get_gz_error(state.ztarfile));
1117  exit(1);
1118  }
1119  }
1120  else
1121 #endif
1122  {
1123  snprintf(state.filename, sizeof(state.filename), "%s/%s.tar",
1124  basedir, PQgetvalue(res, rownum, 0));
1125  state.tarfile = fopen(state.filename, "wb");
1126  }
1127  }
1128 
1129 #ifdef HAVE_LIBZ
1130  if (compresslevel != 0)
1131  {
1132  if (!state.ztarfile)
1133  {
1134  /* Compression is in use */
1135  pg_log_error("could not create compressed file \"%s\": %s",
1136  state.filename, get_gz_error(state.ztarfile));
1137  exit(1);
1138  }
1139  }
1140  else
1141 #endif
1142  {
1143  /* Either no zlib support, or zlib support but compresslevel = 0 */
1144  if (!state.tarfile)
1145  {
1146  pg_log_error("could not create file \"%s\": %m", state.filename);
1147  exit(1);
1148  }
1149  }
1150 
1151  ReceiveCopyData(conn, ReceiveTarCopyChunk, &state);
1152 
1153  /*
1154  * End of copy data. If requested, and this is the base tablespace, write
1155  * configuration file into the tarfile. When done, close the file (but not
1156  * stdout).
1157  *
1158  * Also, write two completely empty blocks at the end of the tar file, as
1159  * required by some tar programs.
1160  */
1161 
1162  MemSet(zerobuf, 0, sizeof(zerobuf));
1163 
1164  if (state.basetablespace && writerecoveryconf)
1165  {
1166  char header[512];
1167 
1168  /*
1169  * If postgresql.auto.conf has not been found in the streamed data,
1170  * add recovery configuration to postgresql.auto.conf if recovery
1171  * parameters are GUCs. If the instance connected to is older than
1172  * 12, create recovery.conf with this data otherwise.
1173  */
1175  {
1176  int padding;
1177 
1178  tarCreateHeader(header,
1179  state.is_recovery_guc_supported ? "postgresql.auto.conf" : "recovery.conf",
1180  NULL,
1181  recoveryconfcontents->len,
1182  pg_file_create_mode, 04000, 02000,
1183  time(NULL));
1184 
1185  padding = ((recoveryconfcontents->len + 511) & ~511) - recoveryconfcontents->len;
1186 
1187  writeTarData(&state, header, sizeof(header));
1188  writeTarData(&state, recoveryconfcontents->data,
1189  recoveryconfcontents->len);
1190  if (padding)
1191  writeTarData(&state, zerobuf, padding);
1192  }
1193 
1194  /*
1195  * standby.signal is supported only if recovery parameters are GUCs.
1196  */
1197  if (state.is_recovery_guc_supported)
1198  {
1199  tarCreateHeader(header, "standby.signal", NULL,
1200  0, /* zero-length file */
1201  pg_file_create_mode, 04000, 02000,
1202  time(NULL));
1203 
1204  writeTarData(&state, header, sizeof(header));
1205  writeTarData(&state, zerobuf, 511);
1206  }
1207  }
1208 
1209  /*
1210  * Normally, we emit the backup manifest as a separate file, but when
1211  * we're writing a tarfile to stdout, we don't have that option, so
1212  * include it in the one tarfile we've got.
1213  */
1214  if (strcmp(basedir, "-") == 0)
1215  {
1216  char header[512];
1218 
1219  initPQExpBuffer(&buf);
1220  ReceiveBackupManifestInMemory(conn, &buf);
1221  if (PQExpBufferDataBroken(buf))
1222  {
1223  pg_log_error("out of memory");
1224  exit(1);
1225  }
1226  tarCreateHeader(header, "backup_manifest", NULL, buf.len,
1227  pg_file_create_mode, 04000, 02000,
1228  time(NULL));
1229  writeTarData(&state, header, sizeof(header));
1230  writeTarData(&state, buf.data, buf.len);
1231  termPQExpBuffer(&buf);
1232  }
1233 
1234  /* 2 * 512 bytes empty data at end of file */
1235  writeTarData(&state, zerobuf, sizeof(zerobuf));
1236 
1237 #ifdef HAVE_LIBZ
1238  if (state.ztarfile != NULL)
1239  {
1240  if (gzclose(state.ztarfile) != 0)
1241  {
1242  pg_log_error("could not close compressed file \"%s\": %s",
1243  state.filename, get_gz_error(state.ztarfile));
1244  exit(1);
1245  }
1246  }
1247  else
1248 #endif
1249  {
1250  if (strcmp(basedir, "-") != 0)
1251  {
1252  if (fclose(state.tarfile) != 0)
1253  {
1254  pg_log_error("could not close file \"%s\": %m",
1255  state.filename);
1256  exit(1);
1257  }
1258  }
1259  }
1260 
1261  progress_report(rownum, state.filename, true);
1262 
1263  /*
1264  * Do not sync the resulting tar file yet, all files are synced once at
1265  * the end.
1266  */
1267 }
1268 
1269 /*
1270  * Receive one chunk of tar-format data from the server.
1271  */
1272 static void
1273 ReceiveTarCopyChunk(size_t r, char *copybuf, void *callback_data)
1274 {
1275  WriteTarState *state = callback_data;
1276 
1277  if (!writerecoveryconf || !state->basetablespace)
1278  {
1279  /*
1280  * When not writing config file, or when not working on the base
1281  * tablespace, we never have to look for an existing configuration
1282  * file in the stream.
1283  */
1284  writeTarData(state, copybuf, r);
1285  }
1286  else
1287  {
1288  /*
1289  * Look for a config file in the existing tar stream. If it's there,
1290  * we must skip it so we can later overwrite it with our own version
1291  * of the file.
1292  *
1293  * To do this, we have to process the individual files inside the TAR
1294  * stream. The stream consists of a header and zero or more chunks,
1295  * all 512 bytes long. The stream from the server is broken up into
1296  * smaller pieces, so we have to track the size of the files to find
1297  * the next header structure.
1298  */
1299  int rr = r;
1300  int pos = 0;
1301 
1302  while (rr > 0)
1303  {
1304  if (state->in_tarhdr)
1305  {
1306  /*
1307  * We're currently reading a header structure inside the TAR
1308  * stream, i.e. the file metadata.
1309  */
1310  if (state->tarhdrsz < 512)
1311  {
1312  /*
1313  * Copy the header structure into tarhdr in case the
1314  * header is not aligned to 512 bytes or it's not returned
1315  * in whole by the last PQgetCopyData call.
1316  */
1317  int hdrleft;
1318  int bytes2copy;
1319 
1320  hdrleft = 512 - state->tarhdrsz;
1321  bytes2copy = (rr > hdrleft ? hdrleft : rr);
1322 
1323  memcpy(&state->tarhdr[state->tarhdrsz], copybuf + pos,
1324  bytes2copy);
1325 
1326  rr -= bytes2copy;
1327  pos += bytes2copy;
1328  state->tarhdrsz += bytes2copy;
1329  }
1330  else
1331  {
1332  /*
1333  * We have the complete header structure in tarhdr, look
1334  * at the file metadata: we may want append recovery info
1335  * into postgresql.auto.conf and skip standby.signal file
1336  * if recovery parameters are integrated as GUCs, and
1337  * recovery.conf otherwise. In both cases we must
1338  * calculate tar padding.
1339  */
1340  if (state->is_recovery_guc_supported)
1341  {
1342  state->skip_file =
1343  (strcmp(&state->tarhdr[0], "standby.signal") == 0);
1344  state->is_postgresql_auto_conf =
1345  (strcmp(&state->tarhdr[0], "postgresql.auto.conf") == 0);
1346  }
1347  else
1348  state->skip_file =
1349  (strcmp(&state->tarhdr[0], "recovery.conf") == 0);
1350 
1351  state->filesz = read_tar_number(&state->tarhdr[124], 12);
1352  state->file_padding_len =
1353  ((state->filesz + 511) & ~511) - state->filesz;
1354 
1355  if (state->is_recovery_guc_supported &&
1356  state->is_postgresql_auto_conf &&
1358  {
1359  /* replace tar header */
1360  char header[512];
1361 
1362  tarCreateHeader(header, "postgresql.auto.conf", NULL,
1363  state->filesz + recoveryconfcontents->len,
1364  pg_file_create_mode, 04000, 02000,
1365  time(NULL));
1366 
1367  writeTarData(state, header, sizeof(header));
1368  }
1369  else
1370  {
1371  /* copy stream with padding */
1372  state->filesz += state->file_padding_len;
1373 
1374  if (!state->skip_file)
1375  {
1376  /*
1377  * If we're not skipping the file, write the tar
1378  * header unmodified.
1379  */
1380  writeTarData(state, state->tarhdr, 512);
1381  }
1382  }
1383 
1384  /* Next part is the file, not the header */
1385  state->in_tarhdr = false;
1386  }
1387  }
1388  else
1389  {
1390  /*
1391  * We're processing a file's contents.
1392  */
1393  if (state->filesz > 0)
1394  {
1395  /*
1396  * We still have data to read (and possibly write).
1397  */
1398  int bytes2write;
1399 
1400  bytes2write = (state->filesz > rr ? rr : state->filesz);
1401 
1402  if (!state->skip_file)
1403  writeTarData(state, copybuf + pos, bytes2write);
1404 
1405  rr -= bytes2write;
1406  pos += bytes2write;
1407  state->filesz -= bytes2write;
1408  }
1409  else if (state->is_recovery_guc_supported &&
1410  state->is_postgresql_auto_conf &&
1412  {
1413  /* append recovery config to postgresql.auto.conf */
1414  int padding;
1415  int tailsize;
1416 
1417  tailsize = (512 - state->file_padding_len) + recoveryconfcontents->len;
1418  padding = ((tailsize + 511) & ~511) - tailsize;
1419 
1420  writeTarData(state, recoveryconfcontents->data,
1421  recoveryconfcontents->len);
1422 
1423  if (padding)
1424  {
1425  char zerobuf[512];
1426 
1427  MemSet(zerobuf, 0, sizeof(zerobuf));
1428  writeTarData(state, zerobuf, padding);
1429  }
1430 
1431  /* skip original file padding */
1432  state->is_postgresql_auto_conf = false;
1433  state->skip_file = true;
1434  state->filesz += state->file_padding_len;
1435 
1436  state->found_postgresql_auto_conf = true;
1437  }
1438  else
1439  {
1440  /*
1441  * No more data in the current file, the next piece of
1442  * data (if any) will be a new file header structure.
1443  */
1444  state->in_tarhdr = true;
1445  state->skip_file = false;
1446  state->is_postgresql_auto_conf = false;
1447  state->tarhdrsz = 0;
1448  state->filesz = 0;
1449  }
1450  }
1451  }
1452  }
1453  totaldone += r;
1454  progress_report(state->tablespacenum, state->filename, false);
1455 }
1456 
1457 
1458 /*
1459  * Retrieve tablespace path, either relocated or original depending on whether
1460  * -T was passed or not.
1461  */
1462 static const char *
1463 get_tablespace_mapping(const char *dir)
1464 {
1465  TablespaceListCell *cell;
1466  char canon_dir[MAXPGPATH];
1467 
1468  /* Canonicalize path for comparison consistency */
1469  strlcpy(canon_dir, dir, sizeof(canon_dir));
1470  canonicalize_path(canon_dir);
1471 
1472  for (cell = tablespace_dirs.head; cell; cell = cell->next)
1473  if (strcmp(canon_dir, cell->old_dir) == 0)
1474  return cell->new_dir;
1475 
1476  return dir;
1477 }
1478 
1479 
1480 /*
1481  * Receive a tar format stream from the connection to the server, and unpack
1482  * the contents of it into a directory. Only files, directories and
1483  * symlinks are supported, no other kinds of special files.
1484  *
1485  * If the data is for the main data directory, it will be restored in the
1486  * specified directory. If it's for another tablespace, it will be restored
1487  * in the original or mapped directory.
1488  */
1489 static void
1491 {
1493  bool basetablespace;
1494 
1495  memset(&state, 0, sizeof(state));
1496  state.tablespacenum = rownum;
1497 
1498  basetablespace = PQgetisnull(res, rownum, 0);
1499  if (basetablespace)
1500  strlcpy(state.current_path, basedir, sizeof(state.current_path));
1501  else
1502  strlcpy(state.current_path,
1503  get_tablespace_mapping(PQgetvalue(res, rownum, 1)),
1504  sizeof(state.current_path));
1505 
1507 
1508 
1509  if (state.file)
1510  fclose(state.file);
1511 
1512  progress_report(rownum, state.filename, true);
1513 
1514  if (state.file != NULL)
1515  {
1516  pg_log_error("COPY stream ended before last file was finished");
1517  exit(1);
1518  }
1519 
1520  if (basetablespace && writerecoveryconf)
1521  WriteRecoveryConfig(conn, basedir, recoveryconfcontents);
1522 
1523  /*
1524  * No data is synced here, everything is done for all tablespaces at the
1525  * end.
1526  */
1527 }
1528 
1529 static void
1530 ReceiveTarAndUnpackCopyChunk(size_t r, char *copybuf, void *callback_data)
1531 {
1532  UnpackTarState *state = callback_data;
1533 
1534  if (state->file == NULL)
1535  {
1536 #ifndef WIN32
1537  int filemode;
1538 #endif
1539 
1540  /*
1541  * No current file, so this must be the header for a new file
1542  */
1543  if (r != 512)
1544  {
1545  pg_log_error("invalid tar block header size: %zu", r);
1546  exit(1);
1547  }
1548  totaldone += 512;
1549 
1550  state->current_len_left = read_tar_number(&copybuf[124], 12);
1551 
1552 #ifndef WIN32
1553  /* Set permissions on the file */
1554  filemode = read_tar_number(&copybuf[100], 8);
1555 #endif
1556 
1557  /*
1558  * All files are padded up to 512 bytes
1559  */
1560  state->current_padding =
1561  ((state->current_len_left + 511) & ~511) - state->current_len_left;
1562 
1563  /*
1564  * First part of header is zero terminated filename
1565  */
1566  snprintf(state->filename, sizeof(state->filename),
1567  "%s/%s", state->current_path, copybuf);
1568  if (state->filename[strlen(state->filename) - 1] == '/')
1569  {
1570  /*
1571  * Ends in a slash means directory or symlink to directory
1572  */
1573  if (copybuf[156] == '5')
1574  {
1575  /*
1576  * Directory. Remove trailing slash first.
1577  */
1578  state->filename[strlen(state->filename) - 1] = '\0';
1579  if (mkdir(state->filename, pg_dir_create_mode) != 0)
1580  {
1581  /*
1582  * When streaming WAL, pg_wal (or pg_xlog for pre-9.6
1583  * clusters) will have been created by the wal receiver
1584  * process. Also, when the WAL directory location was
1585  * specified, pg_wal (or pg_xlog) has already been created
1586  * as a symbolic link before starting the actual backup.
1587  * So just ignore creation failures on related
1588  * directories.
1589  */
1590  if (!((pg_str_endswith(state->filename, "/pg_wal") ||
1591  pg_str_endswith(state->filename, "/pg_xlog") ||
1592  pg_str_endswith(state->filename, "/archive_status")) &&
1593  errno == EEXIST))
1594  {
1595  pg_log_error("could not create directory \"%s\": %m",
1596  state->filename);
1597  exit(1);
1598  }
1599  }
1600 #ifndef WIN32
1601  if (chmod(state->filename, (mode_t) filemode))
1602  pg_log_error("could not set permissions on directory \"%s\": %m",
1603  state->filename);
1604 #endif
1605  }
1606  else if (copybuf[156] == '2')
1607  {
1608  /*
1609  * Symbolic link
1610  *
1611  * It's most likely a link in pg_tblspc directory, to the
1612  * location of a tablespace. Apply any tablespace mapping
1613  * given on the command line (--tablespace-mapping). (We
1614  * blindly apply the mapping without checking that the link
1615  * really is inside pg_tblspc. We don't expect there to be
1616  * other symlinks in a data directory, but if there are, you
1617  * can call it an undocumented feature that you can map them
1618  * too.)
1619  */
1620  state->filename[strlen(state->filename) - 1] = '\0'; /* Remove trailing slash */
1621 
1622  state->mapped_tblspc_path =
1623  get_tablespace_mapping(&copybuf[157]);
1624  if (symlink(state->mapped_tblspc_path, state->filename) != 0)
1625  {
1626  pg_log_error("could not create symbolic link from \"%s\" to \"%s\": %m",
1627  state->filename, state->mapped_tblspc_path);
1628  exit(1);
1629  }
1630  }
1631  else
1632  {
1633  pg_log_error("unrecognized link indicator \"%c\"",
1634  copybuf[156]);
1635  exit(1);
1636  }
1637  return; /* directory or link handled */
1638  }
1639 
1640  /*
1641  * regular file
1642  */
1643  state->file = fopen(state->filename, "wb");
1644  if (!state->file)
1645  {
1646  pg_log_error("could not create file \"%s\": %m", state->filename);
1647  exit(1);
1648  }
1649 
1650 #ifndef WIN32
1651  if (chmod(state->filename, (mode_t) filemode))
1652  pg_log_error("could not set permissions on file \"%s\": %m",
1653  state->filename);
1654 #endif
1655 
1656  if (state->current_len_left == 0)
1657  {
1658  /*
1659  * Done with this file, next one will be a new tar header
1660  */
1661  fclose(state->file);
1662  state->file = NULL;
1663  return;
1664  }
1665  } /* new file */
1666  else
1667  {
1668  /*
1669  * Continuing blocks in existing file
1670  */
1671  if (state->current_len_left == 0 && r == state->current_padding)
1672  {
1673  /*
1674  * Received the padding block for this file, ignore it and close
1675  * the file, then move on to the next tar header.
1676  */
1677  fclose(state->file);
1678  state->file = NULL;
1679  totaldone += r;
1680  return;
1681  }
1682 
1683  if (fwrite(copybuf, r, 1, state->file) != 1)
1684  {
1685  pg_log_error("could not write to file \"%s\": %m", state->filename);
1686  exit(1);
1687  }
1688  totaldone += r;
1689  progress_report(state->tablespacenum, state->filename, false);
1690 
1691  state->current_len_left -= r;
1692  if (state->current_len_left == 0 && state->current_padding == 0)
1693  {
1694  /*
1695  * Received the last block, and there is no padding to be
1696  * expected. Close the file and move on to the next tar header.
1697  */
1698  fclose(state->file);
1699  state->file = NULL;
1700  return;
1701  }
1702  } /* continuing data in existing file */
1703 }
1704 
1705 /*
1706  * Receive the backup manifest file and write it out to a file.
1707  */
1708 static void
1710 {
1712 
1713  snprintf(state.filename, sizeof(state.filename),
1714  "%s/backup_manifest.tmp", basedir);
1715  state.file = fopen(state.filename, "wb");
1716  if (state.file == NULL)
1717  {
1718  pg_log_error("could not create file \"%s\": %m", state.filename);
1719  exit(1);
1720  }
1721 
1723 
1724  fclose(state.file);
1725 }
1726 
1727 /*
1728  * Receive one chunk of the backup manifest file and write it out to a file.
1729  */
1730 static void
1731 ReceiveBackupManifestChunk(size_t r, char *copybuf, void *callback_data)
1732 {
1733  WriteManifestState *state = callback_data;
1734 
1735  if (fwrite(copybuf, r, 1, state->file) != 1)
1736  {
1737  pg_log_error("could not write to file \"%s\": %m", state->filename);
1738  exit(1);
1739  }
1740 }
1741 
1742 /*
1743  * Receive the backup manifest file and write it out to a file.
1744  */
1745 static void
1747 {
1749 }
1750 
1751 /*
1752  * Receive one chunk of the backup manifest file and write it out to a file.
1753  */
1754 static void
1756  void *callback_data)
1757 {
1758  PQExpBuffer buf = callback_data;
1759 
1760  appendPQExpBuffer(buf, copybuf, r);
1761 }
1762 
1763 static void
1765 {
1766  PGresult *res;
1767  char *sysidentifier;
1768  TimeLineID latesttli;
1769  TimeLineID starttli;
1770  char *basebkp;
1771  char escaped_label[MAXPGPATH];
1772  char *maxrate_clause = NULL;
1773  char *manifest_clause;
1774  char *manifest_checksums_clause = "";
1775  int i;
1776  char xlogstart[64];
1777  char xlogend[64];
1778  int minServerMajor,
1779  maxServerMajor;
1780  int serverVersion,
1781  serverMajor;
1782  int writing_to_stdout;
1783 
1784  Assert(conn != NULL);
1785 
1786  /*
1787  * Check server version. BASE_BACKUP command was introduced in 9.1, so we
1788  * can't work with servers older than 9.1.
1789  */
1790  minServerMajor = 901;
1791  maxServerMajor = PG_VERSION_NUM / 100;
1792  serverVersion = PQserverVersion(conn);
1793  serverMajor = serverVersion / 100;
1794  if (serverMajor < minServerMajor || serverMajor > maxServerMajor)
1795  {
1796  const char *serverver = PQparameterStatus(conn, "server_version");
1797 
1798  pg_log_error("incompatible server version %s",
1799  serverver ? serverver : "'unknown'");
1800  exit(1);
1801  }
1802 
1803  /*
1804  * If WAL streaming was requested, also check that the server is new
1805  * enough for that.
1806  */
1808  {
1809  /*
1810  * Error message already written in CheckServerVersionForStreaming(),
1811  * but add a hint about using -X none.
1812  */
1813  pg_log_info("HINT: use -X none or -X fetch to disable log streaming");
1814  exit(1);
1815  }
1816 
1817  /*
1818  * Build contents of configuration file if requested
1819  */
1820  if (writerecoveryconf)
1821  recoveryconfcontents = GenerateRecoveryConfig(conn, replication_slot);
1822 
1823  /*
1824  * Run IDENTIFY_SYSTEM so we can get the timeline
1825  */
1826  if (!RunIdentifySystem(conn, &sysidentifier, &latesttli, NULL, NULL))
1827  exit(1);
1828 
1829  /*
1830  * Start the actual backup
1831  */
1832  PQescapeStringConn(conn, escaped_label, label, sizeof(escaped_label), &i);
1833 
1834  if (maxrate > 0)
1835  maxrate_clause = psprintf("MAX_RATE %u", maxrate);
1836 
1837  if (manifest)
1838  {
1839  if (serverMajor < 1300)
1840  {
1841  const char *serverver = PQparameterStatus(conn, "server_version");
1842 
1843  pg_log_error("backup manifests are not supported by server version %s",
1844  serverver ? serverver : "'unknown'");
1845  exit(1);
1846  }
1847 
1849  manifest_clause = "MANIFEST 'force-encode'";
1850  else
1851  manifest_clause = "MANIFEST 'yes'";
1852  if (manifest_checksums != NULL)
1853  manifest_checksums_clause = psprintf("MANIFEST_CHECKSUMS '%s'",
1855  }
1856  else
1857  {
1858  if (serverMajor < 1300)
1859  manifest_clause = "";
1860  else
1861  manifest_clause = "MANIFEST 'no'";
1862  }
1863 
1864  if (verbose)
1865  pg_log_info("initiating base backup, waiting for checkpoint to complete");
1866 
1867  if (showprogress && !verbose)
1868  {
1869  fprintf(stderr, "waiting for checkpoint");
1870  if (isatty(fileno(stderr)))
1871  fprintf(stderr, "\r");
1872  else
1873  fprintf(stderr, "\n");
1874  }
1875 
1876  basebkp =
1877  psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s %s %s",
1878  escaped_label,
1879  estimatesize ? "PROGRESS" : "",
1880  includewal == FETCH_WAL ? "WAL" : "",
1881  fastcheckpoint ? "FAST" : "",
1882  includewal == NO_WAL ? "" : "NOWAIT",
1883  maxrate_clause ? maxrate_clause : "",
1884  format == 't' ? "TABLESPACE_MAP" : "",
1885  verify_checksums ? "" : "NOVERIFY_CHECKSUMS",
1886  manifest_clause,
1887  manifest_checksums_clause);
1888 
1889  if (PQsendQuery(conn, basebkp) == 0)
1890  {
1891  pg_log_error("could not send replication command \"%s\": %s",
1892  "BASE_BACKUP", PQerrorMessage(conn));
1893  exit(1);
1894  }
1895 
1896  /*
1897  * Get the starting WAL location
1898  */
1899  res = PQgetResult(conn);
1900  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1901  {
1902  pg_log_error("could not initiate base backup: %s",
1903  PQerrorMessage(conn));
1904  exit(1);
1905  }
1906  if (PQntuples(res) != 1)
1907  {
1908  pg_log_error("server returned unexpected response to BASE_BACKUP command; got %d rows and %d fields, expected %d rows and %d fields",
1909  PQntuples(res), PQnfields(res), 1, 2);
1910  exit(1);
1911  }
1912 
1913  strlcpy(xlogstart, PQgetvalue(res, 0, 0), sizeof(xlogstart));
1914 
1915  if (verbose)
1916  pg_log_info("checkpoint completed");
1917 
1918  /*
1919  * 9.3 and later sends the TLI of the starting point. With older servers,
1920  * assume it's the same as the latest timeline reported by
1921  * IDENTIFY_SYSTEM.
1922  */
1923  if (PQnfields(res) >= 2)
1924  starttli = atoi(PQgetvalue(res, 0, 1));
1925  else
1926  starttli = latesttli;
1927  PQclear(res);
1928  MemSet(xlogend, 0, sizeof(xlogend));
1929 
1930  if (verbose && includewal != NO_WAL)
1931  pg_log_info("write-ahead log start point: %s on timeline %u",
1932  xlogstart, starttli);
1933 
1934  /*
1935  * Get the header
1936  */
1937  res = PQgetResult(conn);
1938  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1939  {
1940  pg_log_error("could not get backup header: %s",
1941  PQerrorMessage(conn));
1942  exit(1);
1943  }
1944  if (PQntuples(res) < 1)
1945  {
1946  pg_log_error("no data returned from server");
1947  exit(1);
1948  }
1949 
1950  /*
1951  * Sum up the total size, for progress reporting
1952  */
1953  totalsize_kb = totaldone = 0;
1954  tablespacecount = PQntuples(res);
1955  for (i = 0; i < PQntuples(res); i++)
1956  {
1957  totalsize_kb += atol(PQgetvalue(res, i, 2));
1958 
1959  /*
1960  * Verify tablespace directories are empty. Don't bother with the
1961  * first once since it can be relocated, and it will be checked before
1962  * we do anything anyway.
1963  */
1964  if (format == 'p' && !PQgetisnull(res, i, 1))
1965  {
1966  char *path = unconstify(char *, get_tablespace_mapping(PQgetvalue(res, i, 1)));
1967 
1969  }
1970  }
1971 
1972  /*
1973  * When writing to stdout, require a single tablespace
1974  */
1975  writing_to_stdout = format == 't' && strcmp(basedir, "-") == 0;
1976  if (writing_to_stdout && PQntuples(res) > 1)
1977  {
1978  pg_log_error("can only write single tablespace to stdout, database has %d",
1979  PQntuples(res));
1980  exit(1);
1981  }
1982 
1983  /*
1984  * If we're streaming WAL, start the streaming session before we start
1985  * receiving the actual data chunks.
1986  */
1987  if (includewal == STREAM_WAL)
1988  {
1989  if (verbose)
1990  pg_log_info("starting background WAL receiver");
1991  StartLogStreamer(xlogstart, starttli, sysidentifier);
1992  }
1993 
1994  /*
1995  * Start receiving chunks
1996  */
1997  for (i = 0; i < PQntuples(res); i++)
1998  {
1999  if (format == 't')
2000  ReceiveTarFile(conn, res, i);
2001  else
2002  ReceiveAndUnpackTarFile(conn, res, i);
2003  } /* Loop over all tablespaces */
2004 
2005  /*
2006  * Now receive backup manifest, if appropriate.
2007  *
2008  * If we're writing a tarfile to stdout, ReceiveTarFile will have already
2009  * processed the backup manifest and included it in the output tarfile.
2010  * Such a configuration doesn't allow for writing multiple files.
2011  *
2012  * If we're talking to an older server, it won't send a backup manifest,
2013  * so don't try to receive one.
2014  */
2015  if (!writing_to_stdout && manifest)
2017 
2018  if (showprogress)
2019  {
2020  progress_report(PQntuples(res), NULL, true);
2021  if (isatty(fileno(stderr)))
2022  fprintf(stderr, "\n"); /* Need to move to next line */
2023  }
2024 
2025  PQclear(res);
2026 
2027  /*
2028  * Get the stop position
2029  */
2030  res = PQgetResult(conn);
2031  if (PQresultStatus(res) != PGRES_TUPLES_OK)
2032  {
2033  pg_log_error("could not get write-ahead log end position from server: %s",
2034  PQerrorMessage(conn));
2035  exit(1);
2036  }
2037  if (PQntuples(res) != 1)
2038  {
2039  pg_log_error("no write-ahead log end position returned from server");
2040  exit(1);
2041  }
2042  strlcpy(xlogend, PQgetvalue(res, 0, 0), sizeof(xlogend));
2043  if (verbose && includewal != NO_WAL)
2044  pg_log_info("write-ahead log end point: %s", xlogend);
2045  PQclear(res);
2046 
2047  res = PQgetResult(conn);
2048  if (PQresultStatus(res) != PGRES_COMMAND_OK)
2049  {
2050  const char *sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
2051 
2052  if (sqlstate &&
2053  strcmp(sqlstate, ERRCODE_DATA_CORRUPTED) == 0)
2054  {
2055  pg_log_error("checksum error occurred");
2056  checksum_failure = true;
2057  }
2058  else
2059  {
2060  pg_log_error("final receive failed: %s",
2061  PQerrorMessage(conn));
2062  }
2063  exit(1);
2064  }
2065 
2066  if (bgchild > 0)
2067  {
2068 #ifndef WIN32
2069  int status;
2070  pid_t r;
2071 #else
2072  DWORD status;
2073 
2074  /*
2075  * get a pointer sized version of bgchild to avoid warnings about
2076  * casting to a different size on WIN64.
2077  */
2078  intptr_t bgchild_handle = bgchild;
2079  uint32 hi,
2080  lo;
2081 #endif
2082 
2083  if (verbose)
2084  pg_log_info("waiting for background process to finish streaming ...");
2085 
2086 #ifndef WIN32
2087  if (write(bgpipe[1], xlogend, strlen(xlogend)) != strlen(xlogend))
2088  {
2089  pg_log_info("could not send command to background pipe: %m");
2090  exit(1);
2091  }
2092 
2093  /* Just wait for the background process to exit */
2094  r = waitpid(bgchild, &status, 0);
2095  if (r == (pid_t) -1)
2096  {
2097  pg_log_error("could not wait for child process: %m");
2098  exit(1);
2099  }
2100  if (r != bgchild)
2101  {
2102  pg_log_error("child %d died, expected %d", (int) r, (int) bgchild);
2103  exit(1);
2104  }
2105  if (status != 0)
2106  {
2107  pg_log_error("%s", wait_result_to_str(status));
2108  exit(1);
2109  }
2110  /* Exited normally, we're happy! */
2111 #else /* WIN32 */
2112 
2113  /*
2114  * On Windows, since we are in the same process, we can just store the
2115  * value directly in the variable, and then set the flag that says
2116  * it's there.
2117  */
2118  if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
2119  {
2120  pg_log_error("could not parse write-ahead log location \"%s\"",
2121  xlogend);
2122  exit(1);
2123  }
2124  xlogendptr = ((uint64) hi) << 32 | lo;
2125  InterlockedIncrement(&has_xlogendptr);
2126 
2127  /* First wait for the thread to exit */
2128  if (WaitForSingleObjectEx((HANDLE) bgchild_handle, INFINITE, FALSE) !=
2129  WAIT_OBJECT_0)
2130  {
2131  _dosmaperr(GetLastError());
2132  pg_log_error("could not wait for child thread: %m");
2133  exit(1);
2134  }
2135  if (GetExitCodeThread((HANDLE) bgchild_handle, &status) == 0)
2136  {
2137  _dosmaperr(GetLastError());
2138  pg_log_error("could not get child thread exit status: %m");
2139  exit(1);
2140  }
2141  if (status != 0)
2142  {
2143  pg_log_error("child thread exited with error %u",
2144  (unsigned int) status);
2145  exit(1);
2146  }
2147  /* Exited normally, we're happy */
2148 #endif
2149  }
2150 
2151  /* Free the configuration file contents */
2152  destroyPQExpBuffer(recoveryconfcontents);
2153 
2154  /*
2155  * End of copy data. Final result is already checked inside the loop.
2156  */
2157  PQclear(res);
2158  PQfinish(conn);
2159  conn = NULL;
2160 
2161  /*
2162  * Make data persistent on disk once backup is completed. For tar format
2163  * sync the parent directory and all its contents as each tar file was not
2164  * synced after being completed. In plain format, all the data of the
2165  * base directory is synced, taking into account all the tablespaces.
2166  * Errors are not considered fatal.
2167  */
2168  if (do_sync)
2169  {
2170  if (verbose)
2171  pg_log_info("syncing data to disk ...");
2172  if (format == 't')
2173  {
2174  if (strcmp(basedir, "-") != 0)
2175  (void) fsync_dir_recurse(basedir);
2176  }
2177  else
2178  {
2179  (void) fsync_pgdata(basedir, serverVersion);
2180  }
2181  }
2182 
2183  /*
2184  * After synchronizing data to disk, perform a durable rename of
2185  * backup_manifest.tmp to backup_manifest, if we wrote such a file. This
2186  * way, a failure or system crash before we reach this point will leave us
2187  * without a backup_manifest file, decreasing the chances that a directory
2188  * we leave behind will be mistaken for a valid backup.
2189  */
2190  if (!writing_to_stdout && manifest)
2191  {
2192  char tmp_filename[MAXPGPATH];
2193  char filename[MAXPGPATH];
2194 
2195  if (verbose)
2196  pg_log_info("renaming backup_manifest.tmp to backup_manifest");
2197 
2198  snprintf(tmp_filename, MAXPGPATH, "%s/backup_manifest.tmp", basedir);
2199  snprintf(filename, MAXPGPATH, "%s/backup_manifest", basedir);
2200 
2201  /* durable_rename emits its own log message in case of failure */
2202  if (durable_rename(tmp_filename, filename) != 0)
2203  exit(1);
2204  }
2205 
2206  if (verbose)
2207  pg_log_info("base backup completed");
2208 }
2209 
2210 
2211 int
2212 main(int argc, char **argv)
2213 {
2214  static struct option long_options[] = {
2215  {"help", no_argument, NULL, '?'},
2216  {"version", no_argument, NULL, 'V'},
2217  {"pgdata", required_argument, NULL, 'D'},
2218  {"format", required_argument, NULL, 'F'},
2219  {"checkpoint", required_argument, NULL, 'c'},
2220  {"create-slot", no_argument, NULL, 'C'},
2221  {"max-rate", required_argument, NULL, 'r'},
2222  {"write-recovery-conf", no_argument, NULL, 'R'},
2223  {"slot", required_argument, NULL, 'S'},
2224  {"tablespace-mapping", required_argument, NULL, 'T'},
2225  {"wal-method", required_argument, NULL, 'X'},
2226  {"gzip", no_argument, NULL, 'z'},
2227  {"compress", required_argument, NULL, 'Z'},
2228  {"label", required_argument, NULL, 'l'},
2229  {"no-clean", no_argument, NULL, 'n'},
2230  {"no-sync", no_argument, NULL, 'N'},
2231  {"dbname", required_argument, NULL, 'd'},
2232  {"host", required_argument, NULL, 'h'},
2233  {"port", required_argument, NULL, 'p'},
2234  {"username", required_argument, NULL, 'U'},
2235  {"no-password", no_argument, NULL, 'w'},
2236  {"password", no_argument, NULL, 'W'},
2237  {"status-interval", required_argument, NULL, 's'},
2238  {"verbose", no_argument, NULL, 'v'},
2239  {"progress", no_argument, NULL, 'P'},
2240  {"waldir", required_argument, NULL, 1},
2241  {"no-slot", no_argument, NULL, 2},
2242  {"no-verify-checksums", no_argument, NULL, 3},
2243  {"no-estimate-size", no_argument, NULL, 4},
2244  {"no-manifest", no_argument, NULL, 5},
2245  {"manifest-force-encode", no_argument, NULL, 6},
2246  {"manifest-checksums", required_argument, NULL, 7},
2247  {NULL, 0, NULL, 0}
2248  };
2249  int c;
2250 
2251  int option_index;
2252 
2253  pg_logging_init(argv[0]);
2254  progname = get_progname(argv[0]);
2255  set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup"));
2256 
2257  if (argc > 1)
2258  {
2259  if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
2260  {
2261  usage();
2262  exit(0);
2263  }
2264  else if (strcmp(argv[1], "-V") == 0
2265  || strcmp(argv[1], "--version") == 0)
2266  {
2267  puts("pg_basebackup (PostgreSQL) " PG_VERSION);
2268  exit(0);
2269  }
2270  }
2271 
2273 
2274  while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvPm:",
2275  long_options, &option_index)) != -1)
2276  {
2277  switch (c)
2278  {
2279  case 'C':
2280  create_slot = true;
2281  break;
2282  case 'D':
2284  break;
2285  case 'F':
2286  if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
2287  format = 'p';
2288  else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
2289  format = 't';
2290  else
2291  {
2292  pg_log_error("invalid output format \"%s\", must be \"plain\" or \"tar\"",
2293  optarg);
2294  exit(1);
2295  }
2296  break;
2297  case 'r':
2299  break;
2300  case 'R':
2301  writerecoveryconf = true;
2302  break;
2303  case 'S':
2304 
2305  /*
2306  * When specifying replication slot name, use a permanent
2307  * slot.
2308  */
2310  temp_replication_slot = false;
2311  break;
2312  case 2:
2313  no_slot = true;
2314  break;
2315  case 'T':
2317  break;
2318  case 'X':
2319  if (strcmp(optarg, "n") == 0 ||
2320  strcmp(optarg, "none") == 0)
2321  {
2322  includewal = NO_WAL;
2323  }
2324  else if (strcmp(optarg, "f") == 0 ||
2325  strcmp(optarg, "fetch") == 0)
2326  {
2328  }
2329  else if (strcmp(optarg, "s") == 0 ||
2330  strcmp(optarg, "stream") == 0)
2331  {
2333  }
2334  else
2335  {
2336  pg_log_error("invalid wal-method option \"%s\", must be \"fetch\", \"stream\", or \"none\"",
2337  optarg);
2338  exit(1);
2339  }
2340  break;
2341  case 1:
2343  break;
2344  case 'l':
2345  label = pg_strdup(optarg);
2346  break;
2347  case 'n':
2348  noclean = true;
2349  break;
2350  case 'N':
2351  do_sync = false;
2352  break;
2353  case 'z':
2354 #ifdef HAVE_LIBZ
2356 #else
2357  compresslevel = 1; /* will be rejected below */
2358 #endif
2359  break;
2360  case 'Z':
2361  compresslevel = atoi(optarg);
2362  if (compresslevel < 0 || compresslevel > 9)
2363  {
2364  pg_log_error("invalid compression level \"%s\"", optarg);
2365  exit(1);
2366  }
2367  break;
2368  case 'c':
2369  if (pg_strcasecmp(optarg, "fast") == 0)
2370  fastcheckpoint = true;
2371  else if (pg_strcasecmp(optarg, "spread") == 0)
2372  fastcheckpoint = false;
2373  else
2374  {
2375  pg_log_error("invalid checkpoint argument \"%s\", must be \"fast\" or \"spread\"",
2376  optarg);
2377  exit(1);
2378  }
2379  break;
2380  case 'd':
2382  break;
2383  case 'h':
2384  dbhost = pg_strdup(optarg);
2385  break;
2386  case 'p':
2387  dbport = pg_strdup(optarg);
2388  break;
2389  case 'U':
2390  dbuser = pg_strdup(optarg);
2391  break;
2392  case 'w':
2393  dbgetpassword = -1;
2394  break;
2395  case 'W':
2396  dbgetpassword = 1;
2397  break;
2398  case 's':
2399  standby_message_timeout = atoi(optarg) * 1000;
2400  if (standby_message_timeout < 0)
2401  {
2402  pg_log_error("invalid status interval \"%s\"", optarg);
2403  exit(1);
2404  }
2405  break;
2406  case 'v':
2407  verbose++;
2408  break;
2409  case 'P':
2410  showprogress = true;
2411  break;
2412  case 3:
2413  verify_checksums = false;
2414  break;
2415  case 4:
2416  estimatesize = false;
2417  break;
2418  case 5:
2419  manifest = false;
2420  break;
2421  case 6:
2422  manifest_force_encode = true;
2423  break;
2424  case 7:
2426  break;
2427  default:
2428 
2429  /*
2430  * getopt_long already emitted a complaint
2431  */
2432  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2433  progname);
2434  exit(1);
2435  }
2436  }
2437 
2438  /*
2439  * Any non-option arguments?
2440  */
2441  if (optind < argc)
2442  {
2443  pg_log_error("too many command-line arguments (first is \"%s\")",
2444  argv[optind]);
2445  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2446  progname);
2447  exit(1);
2448  }
2449 
2450  /*
2451  * Required arguments
2452  */
2453  if (basedir == NULL)
2454  {
2455  pg_log_error("no target directory specified");
2456  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2457  progname);
2458  exit(1);
2459  }
2460 
2461  /*
2462  * Mutually exclusive arguments
2463  */
2464  if (format == 'p' && compresslevel != 0)
2465  {
2466  pg_log_error("only tar mode backups can be compressed");
2467  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2468  progname);
2469  exit(1);
2470  }
2471 
2472  if (format == 't' && includewal == STREAM_WAL && strcmp(basedir, "-") == 0)
2473  {
2474  pg_log_error("cannot stream write-ahead logs in tar mode to stdout");
2475  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2476  progname);
2477  exit(1);
2478  }
2479 
2481  {
2482  pg_log_error("replication slots can only be used with WAL streaming");
2483  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2484  progname);
2485  exit(1);
2486  }
2487 
2488  if (no_slot)
2489  {
2490  if (replication_slot)
2491  {
2492  pg_log_error("--no-slot cannot be used with slot name");
2493  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2494  progname);
2495  exit(1);
2496  }
2497  temp_replication_slot = false;
2498  }
2499 
2500  if (create_slot)
2501  {
2502  if (!replication_slot)
2503  {
2504  pg_log_error("%s needs a slot to be specified using --slot",
2505  "--create-slot");
2506  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2507  progname);
2508  exit(1);
2509  }
2510 
2511  if (no_slot)
2512  {
2513  pg_log_error("--create-slot and --no-slot are incompatible options");
2514  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2515  progname);
2516  exit(1);
2517  }
2518  }
2519 
2520  if (xlog_dir)
2521  {
2522  if (format != 'p')
2523  {
2524  pg_log_error("WAL directory location can only be specified in plain mode");
2525  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2526  progname);
2527  exit(1);
2528  }
2529 
2530  /* clean up xlog directory name, check it's absolute */
2532  if (!is_absolute_path(xlog_dir))
2533  {
2534  pg_log_error("WAL directory location must be an absolute path");
2535  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2536  progname);
2537  exit(1);
2538  }
2539  }
2540 
2541 #ifndef HAVE_LIBZ
2542  if (compresslevel != 0)
2543  {
2544  pg_log_error("this build does not support compression");
2545  exit(1);
2546  }
2547 #endif
2548 
2549  if (showprogress && !estimatesize)
2550  {
2551  pg_log_error("--progress and --no-estimate-size are incompatible options");
2552  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2553  progname);
2554  exit(1);
2555  }
2556 
2557  if (!manifest && manifest_checksums != NULL)
2558  {
2559  pg_log_error("--no-manifest and --manifest-checksums are incompatible options");
2560  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2561  progname);
2562  exit(1);
2563  }
2564 
2566  {
2567  pg_log_error("--no-manifest and --manifest-force-encode are incompatible options");
2568  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2569  progname);
2570  exit(1);
2571  }
2572 
2573  /* connection in replication mode to server */
2574  conn = GetConnection();
2575  if (!conn)
2576  {
2577  /* Error message already written in GetConnection() */
2578  exit(1);
2579  }
2580  atexit(disconnect_atexit);
2581 
2582  /*
2583  * Set umask so that directories/files are created with the same
2584  * permissions as directories/files in the source data directory.
2585  *
2586  * pg_mode_mask is set to owner-only by default and then updated in
2587  * GetConnection() where we get the mode from the server-side with
2588  * RetrieveDataDirCreatePerm() and then call SetDataDirectoryCreatePerm().
2589  */
2590  umask(pg_mode_mask);
2591 
2592  /*
2593  * Verify that the target directory exists, or create it. For plaintext
2594  * backups, always require the directory. For tar backups, require it
2595  * unless we are writing to stdout.
2596  */
2597  if (format == 'p' || strcmp(basedir, "-") != 0)
2599 
2600  /* determine remote server's xlog segment size */
2601  if (!RetrieveWalSegSize(conn))
2602  exit(1);
2603 
2604  /* Create pg_wal symlink, if required */
2605  if (xlog_dir)
2606  {
2607  char *linkloc;
2608 
2610 
2611  /*
2612  * Form name of the place where the symlink must go. pg_xlog has been
2613  * renamed to pg_wal in post-10 clusters.
2614  */
2615  linkloc = psprintf("%s/%s", basedir,
2617  "pg_xlog" : "pg_wal");
2618 
2619 #ifdef HAVE_SYMLINK
2620  if (symlink(xlog_dir, linkloc) != 0)
2621  {
2622  pg_log_error("could not create symbolic link \"%s\": %m", linkloc);
2623  exit(1);
2624  }
2625 #else
2626  pg_log_error("symlinks are not supported on this platform");
2627  exit(1);
2628 #endif
2629  free(linkloc);
2630  }
2631 
2632  BaseBackup();
2633 
2634  success = true;
2635  return 0;
2636 }
bool found_postgresql_auto_conf
Definition: pg_basebackup.c:71
int PQnfields(const PGresult *res)
Definition: fe-exec.c:2777
char current_path[MAXPGPATH]
Definition: pg_basebackup.c:83
static PQExpBuffer recoveryconfcontents
static IncludeWal includewal
char * PQerrorMessage(const PGconn *conn)
Definition: fe-connect.c:6684
#define Z_DEFAULT_COMPRESSION
int pg_file_create_mode
Definition: file_perm.c:19
static bool verify_checksums
static int bgpipe[2]
static bool found_existing_pgdata
uint32 TimeLineID
Definition: xlogdefs.h:52
int64 pg_time_t
Definition: pgtime.h:23
bool is_recovery_guc_supported
Definition: pg_basebackup.c:69
static void usage(void)
static bool found_tablespace_dirs
const char * mapped_tblspc_path
Definition: pg_basebackup.c:85
char * PQgetvalue(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3163
const char * PQparameterStatus(const PGconn *conn, const char *paramName)
Definition: fe-connect.c:6649
bool pg_str_endswith(const char *str, const char *end)
Definition: string.c:31
int pg_mkdir_p(char *path, int omode)
Definition: pgmkdirp.c:57
static TablespaceList tablespace_dirs
#define write(a, b, c)
Definition: win32.h:14
bool do_sync
Definition: receivelog.h:38
const char * get_progname(const char *argv0)
Definition: path.c:453
void termPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:131
uint64 read_tar_number(const char *s, int len)
Definition: tar.c:58
#define pg_log_error(...)
Definition: logging.h:79
int getopt_long(int argc, char *const argv[], const char *optstring, const struct option *longopts, int *longindex)
Definition: getopt_long.c:57
static int32 parse_max_rate(char *src)
static bool manifest_force_encode
void pg_logging_init(const char *argv0)
Definition: logging.c:76
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
void WriteRecoveryConfig(PGconn *pgconn, char *target_dir, PQExpBuffer contents)
Definition: recovery_gen.c:117
static pg_time_t last_progress_report
char old_dir[MAXPGPATH]
Definition: pg_basebackup.c:50
static int tablespacecount
static void writeTarData(WriteTarState *state, char *buf, int r)
void _dosmaperr(unsigned long)
Definition: win32error.c:171
bool RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli, XLogRecPtr *startpos, char **db_name)
Definition: streamutil.c:407
#define VERBOSE_FILENAME_LENGTH
struct UnpackTarState UnpackTarState
IncludeWal
void canonicalize_path(char *path)
Definition: path.c:254
char * sysidentifier
Definition: receivelog.h:33
static bool writerecoveryconf
static bool create_slot
void PQfinish(PGconn *conn)
Definition: fe-connect.c:4182
static int compresslevel
static bool checksum_failure
#define MemSet(start, val, len)
Definition: c.h:971
#define MINIMUM_VERSION_FOR_RECOVERY_GUC
Definition: recovery_gen.h:21
#define kill(pid, sig)
Definition: win32_port.h:426
static void ReceiveBackupManifest(PGconn *conn)
#define printf(...)
Definition: port.h:199
char filename[MAXPGPATH]
Definition: pg_basebackup.c:84
XLogRecPtr startpos
Definition: receivelog.h:31
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
char * partial_suffix
Definition: receivelog.h:47
char new_dir[MAXPGPATH]
Definition: pg_basebackup.c:51
int PQserverVersion(const PGconn *conn)
Definition: fe-connect.c:6674
const char * progname
Definition: pg_standby.c:36
int PQntuples(const PGresult *res)
Definition: fe-exec.c:2769
#define PG_DIAG_SQLSTATE
Definition: postgres_ext.h:57
#define fprintf
Definition: port.h:197
static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline, bool segment_finished)
char * wait_result_to_str(int exitstatus)
Definition: wait_error.c:32
static int fd(const char *x, int i)
Definition: preproc-init.c:105
TimeLineID timeline
Definition: receivelog.h:32
static char * basedir
struct TablespaceListCell TablespaceListCell
WalWriteMethod * CreateWalDirectoryMethod(const char *basedir, int compression, bool sync)
Definition: walmethods.c:349
ExecStatusType PQresultStatus(const PGresult *res)
Definition: fe-exec.c:2692
static void ReceiveBackupManifestInMemory(PGconn *conn, PQExpBuffer buf)
int PQgetCopyData(PGconn *conn, char **buffer, int async)
Definition: fe-exec.c:2473
signed int int32
Definition: c.h:355
int main(int argc, char **argv)
static void BaseBackup(void)
pgoff_t filesz
Definition: pg_basebackup.c:74
void fsync_pgdata(const char *pg_data, int serverVersion)
Definition: file_utils.c:56
#define pgoff_t
Definition: win32_port.h:194
int PQsendQuery(PGconn *conn, const char *query)
Definition: fe-exec.c:1234
void destroyPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:116
#define required_argument
Definition: getopt_long.h:25
static void ReceiveTarAndUnpackCopyChunk(size_t r, char *copybuf, void *callback_data)
int optind
Definition: getopt.c:50
#define MAX_RATE_UPPER
Definition: basebackup.h:22
void fsync_dir_recurse(const char *dir)
Definition: file_utils.c:120
char * connection_string
Definition: streamutil.c:46
static void ReceiveCopyData(PGconn *conn, WriteDataCallback callback, void *callback_data)
struct TablespaceList TablespaceList
bool is_postgresql_auto_conf
Definition: pg_basebackup.c:70
bool RetrieveWalSegSize(PGconn *conn)
Definition: streamutil.c:277
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
PGconn * conn
Definition: streamutil.c:54
#define MAXPGPATH
void appendPQExpBuffer(PQExpBuffer str, const char *fmt,...)
Definition: pqexpbuffer.c:267
static void callback(struct sockaddr *addr, struct sockaddr *mask, void *unused)
Definition: test_ifaddrs.c:48
#define MINIMUM_VERSION_FOR_PG_WAL
static bool found_existing_xlogdir
static uint64 totaldone
static bool estimatesize
char * replication_slot
Definition: receivelog.h:48
bool mark_done
Definition: receivelog.h:37
static bool success
static int32 maxrate
char * c
static char * buf
Definition: pg_test_fsync.c:67
#define symlink(oldpath, newpath)
Definition: win32_port.h:221
static int has_xlogendptr
static bool made_new_pgdata
#define is_absolute_path(filename)
Definition: port.h:86
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
size_t PQescapeStringConn(PGconn *conn, char *to, const char *from, size_t length, int *error)
Definition: fe-exec.c:3410
#define select(n, r, w, e, timeout)
Definition: win32_port.h:436
unsigned int uint32
Definition: c.h:367
static char * manifest_checksums
char tarhdr[512]
Definition: pg_basebackup.c:65
static const char * get_tablespace_mapping(const char *dir)
static void progress_report(int tablespacenum, const char *filename, bool force)
static void ReceiveBackupManifestChunk(size_t r, char *copybuf, void *callback_data)
int durable_rename(const char *oldfile, const char *newfile, int elevel)
Definition: fd.c:656
XLogRecPtr startptr
PQExpBuffer GenerateRecoveryConfig(PGconn *pgconn, char *replication_slot)
Definition: recovery_gen.c:23
stream_stop_callback stream_stop
Definition: receivelog.h:41
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:45
WalWriteMethod * walmethod
Definition: receivelog.h:46
static int verbose
#define unconstify(underlying_type, expr)
Definition: c.h:1206
bool rmtree(const char *path, bool rmtopdir)
Definition: rmtree.c:42
static int standby_message_timeout
int pg_dir_create_mode
Definition: file_perm.c:18
static bool showprogress
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
int dbgetpassword
Definition: streamutil.c:51
#define no_argument
Definition: getopt_long.h:24
void FreeWalTarMethod(void)
Definition: walmethods.c:1013
#define ngettext(s, p, n)
Definition: c.h:1146
#define PG_TEXTDOMAIN(domain)
Definition: c.h:1178
static bool temp_replication_slot
int PQbackendPID(const PGconn *conn)
Definition: fe-connect.c:6710
#define PGINVALID_SOCKET
Definition: port.h:33
static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
struct TablespaceListCell * next
Definition: pg_basebackup.c:49
static char * label
#define XLogSegmentOffset(xlogptr, wal_segsz_bytes)
StringInfo copybuf
Definition: tablesync.c:108
static void disconnect_atexit(void)
static void StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
TablespaceListCell * tail
Definition: pg_basebackup.c:57
char * dbport
Definition: streamutil.c:49
static void cleanup_directories_atexit(void)
void PQclear(PGresult *res)
Definition: fe-exec.c:694
static void ReceiveBackupManifestInMemoryChunk(size_t r, char *copybuf, void *callback_data)
#define free(a)
Definition: header.h:65
static bool in_log_streamer
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
PGconn * GetConnection(UserMapping *user, bool will_prep_stmt)
Definition: connection.c:107
#define PQExpBufferDataBroken(buf)
Definition: pqexpbuffer.h:67
char * PQresultErrorField(const PGresult *res, int fieldcode)
Definition: fe-exec.c:2754
static void ReceiveTarCopyChunk(size_t r, char *copybuf, void *callback_data)
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:738
Definition: regguts.h:298
WalWriteMethod * CreateWalTarMethod(const char *tarbase, int compression, bool sync)
Definition: walmethods.c:982
static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
char * dbhost
Definition: streamutil.c:47
void(* WriteDataCallback)(size_t nbytes, char *buf, void *callback_data)
Definition: pg_basebackup.c:97
bool ReceiveXlogStream(PGconn *conn, StreamCtl *stream)
Definition: receivelog.c:436
static void kill_bgchild_atexit(void)
#define strerror
Definition: port.h:206
static char * xlog_dir
static bool manifest
static XLogRecPtr xlogendptr
static XLogRecPtr startpos
static bool no_slot
TablespaceListCell * head
Definition: pg_basebackup.c:56
bool synchronous
Definition: receivelog.h:36
void pg_free(void *ptr)
Definition: fe_memutils.c:105
bool(* finish)(void)
Definition: walmethods.h:75
struct WriteManifestState WriteManifestState
pgsocket stop_socket
Definition: receivelog.h:43
static void header(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:208
#define INT64_FORMAT
Definition: c.h:409
static bool made_new_xlogdir
static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
Definition: walsender.c:911
int WalSegSz
Definition: pg_standby.c:38
static bool fastcheckpoint
static void tablespace_list_append(const char *arg)
static bool made_tablespace_dirs
static char * filename
Definition: pg_dumpall.c:90
void set_pglocale_pgservice(const char *argv0, const char *app)
Definition: exec.c:434
int errmsg(const char *fmt,...)
Definition: elog.c:824
static pid_t bgchild
char * dbuser
Definition: streamutil.c:48
char * optarg
Definition: getopt.c:52
static uint64 totalsize_kb
struct WriteTarState WriteTarState
int pg_check_dir(const char *dir)
Definition: pgcheckdir.c:31
int i
int standby_message_timeout
Definition: receivelog.h:35
void * arg
static bool do_sync
static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
static char format
char filename[MAXPGPATH]
Definition: pg_basebackup.c:93
#define mkdir(a, b)
Definition: win32_port.h:57
static char * replication_slot
static void static void status(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:225
static int LogStreamerMain(logstreamer_param *param)
int PQgetisnull(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3188
#define snprintf
Definition: port.h:193
#define _(x)
Definition: elog.c:88
void PQfreemem(void *ptr)
Definition: fe-exec.c:3296
int pg_mode_mask
Definition: file_perm.c:25
Datum now(PG_FUNCTION_ARGS)
Definition: timestamp.c:1547
PGresult * PQgetResult(PGconn *conn)
Definition: fe-exec.c:1778
#define MINIMUM_VERSION_FOR_TEMP_SLOTS
#define read(a, b, c)
Definition: win32.h:13
void initPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:92
#define pg_log_info(...)
Definition: logging.h:87
void FreeWalDirectoryMethod(void)
Definition: walmethods.c:373
bool CheckServerVersionForStreaming(PGconn *conn)
Definition: receivelog.c:358
char filename[MAXPGPATH]
Definition: pg_basebackup.c:63
char xlog[MAXPGPATH]
pgoff_t current_len_left
Definition: pg_basebackup.c:86
static bool noclean