PostgreSQL Source Code  git master
pg_basebackup.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_basebackup.c - receive a base backup using streaming replication protocol
4  *
5  * Author: Magnus Hagander <magnus@hagander.net>
6  *
7  * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
8  *
9  * IDENTIFICATION
10  * src/bin/pg_basebackup/pg_basebackup.c
11  *-------------------------------------------------------------------------
12  */
13 
14 #include "postgres_fe.h"
15 
16 #include <unistd.h>
17 #include <dirent.h>
18 #include <sys/stat.h>
19 #include <sys/wait.h>
20 #include <signal.h>
21 #include <time.h>
22 #ifdef HAVE_SYS_SELECT_H
23 #include <sys/select.h>
24 #endif
25 #ifdef HAVE_LIBZ
26 #include <zlib.h>
27 #endif
28 
29 #include "access/xlog_internal.h"
30 #include "common/file_perm.h"
31 #include "common/file_utils.h"
32 #include "common/logging.h"
33 #include "common/string.h"
34 #include "fe_utils/string_utils.h"
35 #include "getopt_long.h"
36 #include "libpq-fe.h"
37 #include "pqexpbuffer.h"
38 #include "pgtar.h"
39 #include "pgtime.h"
40 #include "receivelog.h"
41 #include "replication/basebackup.h"
42 #include "streamutil.h"
43 
44 #define ERRCODE_DATA_CORRUPTED "XX001"
45 
46 typedef struct TablespaceListCell
47 {
52 
53 typedef struct TablespaceList
54 {
58 
59 /*
60  * pg_xlog has been renamed to pg_wal in version 10. This version number
61  * should be compared with PQserverVersion().
62  */
63 #define MINIMUM_VERSION_FOR_PG_WAL 100000
64 
65 /*
66  * Temporary replication slots are supported from version 10.
67  */
68 #define MINIMUM_VERSION_FOR_TEMP_SLOTS 100000
69 
70 /*
71  * recovery.conf is integrated into postgresql.conf from version 12.
72  */
73 #define MINIMUM_VERSION_FOR_RECOVERY_GUC 120000
74 
75 /*
76  * Different ways to include WAL
77  */
78 typedef enum
79 {
83 } IncludeWal;
84 
85 /* Global options */
86 static char *basedir = NULL;
87 static TablespaceList tablespace_dirs = {NULL, NULL};
88 static char *xlog_dir = NULL;
89 static char format = 'p'; /* p(lain)/t(ar) */
90 static char *label = "pg_basebackup base backup";
91 static bool noclean = false;
92 static bool checksum_failure = false;
93 static bool showprogress = false;
94 static int verbose = 0;
95 static int compresslevel = 0;
97 static bool fastcheckpoint = false;
98 static bool writerecoveryconf = false;
99 static bool do_sync = true;
100 static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
102 static int32 maxrate = 0; /* no limit by default */
103 static char *replication_slot = NULL;
104 static bool temp_replication_slot = true;
105 static bool create_slot = false;
106 static bool no_slot = false;
107 static bool verify_checksums = true;
108 
109 static bool success = false;
110 static bool made_new_pgdata = false;
111 static bool found_existing_pgdata = false;
112 static bool made_new_xlogdir = false;
113 static bool found_existing_xlogdir = false;
114 static bool made_tablespace_dirs = false;
115 static bool found_tablespace_dirs = false;
116 
117 /* Progress counters */
118 static uint64 totalsize;
119 static uint64 totaldone;
120 static int tablespacecount;
121 
122 /* Pipe to communicate with background wal receiver process */
123 #ifndef WIN32
124 static int bgpipe[2] = {-1, -1};
125 #endif
126 
127 /* Handle to child process */
128 static pid_t bgchild = -1;
129 static bool in_log_streamer = false;
130 
131 /* End position for xlog streaming, empty string if unknown yet */
133 
134 #ifndef WIN32
135 static int has_xlogendptr = 0;
136 #else
137 static volatile LONG has_xlogendptr = 0;
138 #endif
139 
140 /* Contents of configuration file to be generated */
142 
143 /* Function headers */
144 static void usage(void);
145 static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found);
146 static void progress_report(int tablespacenum, const char *filename, bool force);
147 
148 static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum);
149 static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum);
150 static void GenerateRecoveryConf(PGconn *conn);
151 static void WriteRecoveryConf(void);
152 static void BaseBackup(void);
153 
154 static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline,
155  bool segment_finished);
156 
157 static const char *get_tablespace_mapping(const char *dir);
158 static void tablespace_list_append(const char *arg);
159 
160 
161 static void
163 {
164  if (success || in_log_streamer)
165  return;
166 
167  if (!noclean && !checksum_failure)
168  {
169  if (made_new_pgdata)
170  {
171  pg_log_info("removing data directory \"%s\"", basedir);
172  if (!rmtree(basedir, true))
173  pg_log_error("failed to remove data directory");
174  }
175  else if (found_existing_pgdata)
176  {
177  pg_log_info("removing contents of data directory \"%s\"", basedir);
178  if (!rmtree(basedir, false))
179  pg_log_error("failed to remove contents of data directory");
180  }
181 
182  if (made_new_xlogdir)
183  {
184  pg_log_info("removing WAL directory \"%s\"", xlog_dir);
185  if (!rmtree(xlog_dir, true))
186  pg_log_error("failed to remove WAL directory");
187  }
188  else if (found_existing_xlogdir)
189  {
190  pg_log_info("removing contents of WAL directory \"%s\"", xlog_dir);
191  if (!rmtree(xlog_dir, false))
192  pg_log_error("failed to remove contents of WAL directory");
193  }
194  }
195  else
196  {
198  pg_log_info("data directory \"%s\" not removed at user's request", basedir);
199 
201  pg_log_info("WAL directory \"%s\" not removed at user's request", xlog_dir);
202  }
203 
205  pg_log_info("changes to tablespace directories will not be undone");
206 }
207 
208 static void
210 {
211  if (conn != NULL)
212  PQfinish(conn);
213 }
214 
215 #ifndef WIN32
216 /*
217  * On windows, our background thread dies along with the process. But on
218  * Unix, if we have started a subprocess, we want to kill it off so it
219  * doesn't remain running trying to stream data.
220  */
221 static void
223 {
224  if (bgchild > 0)
225  kill(bgchild, SIGTERM);
226 }
227 #endif
228 
229 /*
230  * Split argument into old_dir and new_dir and append to tablespace mapping
231  * list.
232  */
233 static void
235 {
237  char *dst;
238  char *dst_ptr;
239  const char *arg_ptr;
240 
241  dst_ptr = dst = cell->old_dir;
242  for (arg_ptr = arg; *arg_ptr; arg_ptr++)
243  {
244  if (dst_ptr - dst >= MAXPGPATH)
245  {
246  pg_log_error("directory name too long");
247  exit(1);
248  }
249 
250  if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
251  ; /* skip backslash escaping = */
252  else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
253  {
254  if (*cell->new_dir)
255  {
256  pg_log_error("multiple \"=\" signs in tablespace mapping");
257  exit(1);
258  }
259  else
260  dst = dst_ptr = cell->new_dir;
261  }
262  else
263  *dst_ptr++ = *arg_ptr;
264  }
265 
266  if (!*cell->old_dir || !*cell->new_dir)
267  {
268  pg_log_error("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
269  exit(1);
270  }
271 
272  /*
273  * This check isn't absolutely necessary. But all tablespaces are created
274  * with absolute directories, so specifying a non-absolute path here would
275  * just never match, possibly confusing users. It's also good to be
276  * consistent with the new_dir check.
277  */
278  if (!is_absolute_path(cell->old_dir))
279  {
280  pg_log_error("old directory is not an absolute path in tablespace mapping: %s",
281  cell->old_dir);
282  exit(1);
283  }
284 
285  if (!is_absolute_path(cell->new_dir))
286  {
287  pg_log_error("new directory is not an absolute path in tablespace mapping: %s",
288  cell->new_dir);
289  exit(1);
290  }
291 
292  /*
293  * Comparisons done with these values should involve similarly
294  * canonicalized path values. This is particularly sensitive on Windows
295  * where path values may not necessarily use Unix slashes.
296  */
297  canonicalize_path(cell->old_dir);
298  canonicalize_path(cell->new_dir);
299 
300  if (tablespace_dirs.tail)
301  tablespace_dirs.tail->next = cell;
302  else
303  tablespace_dirs.head = cell;
304  tablespace_dirs.tail = cell;
305 }
306 
307 
308 #ifdef HAVE_LIBZ
309 static const char *
310 get_gz_error(gzFile gzf)
311 {
312  int errnum;
313  const char *errmsg;
314 
315  errmsg = gzerror(gzf, &errnum);
316  if (errnum == Z_ERRNO)
317  return strerror(errno);
318  else
319  return errmsg;
320 }
321 #endif
322 
323 static void
324 usage(void)
325 {
326  printf(_("%s takes a base backup of a running PostgreSQL server.\n\n"),
327  progname);
328  printf(_("Usage:\n"));
329  printf(_(" %s [OPTION]...\n"), progname);
330  printf(_("\nOptions controlling the output:\n"));
331  printf(_(" -D, --pgdata=DIRECTORY receive base backup into directory\n"));
332  printf(_(" -F, --format=p|t output format (plain (default), tar)\n"));
333  printf(_(" -r, --max-rate=RATE maximum transfer rate to transfer data directory\n"
334  " (in kB/s, or use suffix \"k\" or \"M\")\n"));
335  printf(_(" -R, --write-recovery-conf\n"
336  " write configuration for replication\n"));
337  printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
338  " relocate tablespace in OLDDIR to NEWDIR\n"));
339  printf(_(" --waldir=WALDIR location for the write-ahead log directory\n"));
340  printf(_(" -X, --wal-method=none|fetch|stream\n"
341  " include required WAL files with specified method\n"));
342  printf(_(" -z, --gzip compress tar output\n"));
343  printf(_(" -Z, --compress=0-9 compress tar output with given compression level\n"));
344  printf(_("\nGeneral options:\n"));
345  printf(_(" -c, --checkpoint=fast|spread\n"
346  " set fast or spread checkpointing\n"));
347  printf(_(" -C, --create-slot create replication slot\n"));
348  printf(_(" -l, --label=LABEL set backup label\n"));
349  printf(_(" -n, --no-clean do not clean up after errors\n"));
350  printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
351  printf(_(" -P, --progress show progress information\n"));
352  printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
353  printf(_(" -v, --verbose output verbose messages\n"));
354  printf(_(" -V, --version output version information, then exit\n"));
355  printf(_(" --no-slot prevent creation of temporary replication slot\n"));
356  printf(_(" --no-verify-checksums\n"
357  " do not verify checksums\n"));
358  printf(_(" -?, --help show this help, then exit\n"));
359  printf(_("\nConnection options:\n"));
360  printf(_(" -d, --dbname=CONNSTR connection string\n"));
361  printf(_(" -h, --host=HOSTNAME database server host or socket directory\n"));
362  printf(_(" -p, --port=PORT database server port number\n"));
363  printf(_(" -s, --status-interval=INTERVAL\n"
364  " time between status packets sent to server (in seconds)\n"));
365  printf(_(" -U, --username=NAME connect as specified database user\n"));
366  printf(_(" -w, --no-password never prompt for password\n"));
367  printf(_(" -W, --password force password prompt (should happen automatically)\n"));
368  printf(_("\nReport bugs to <pgsql-bugs@lists.postgresql.org>.\n"));
369 }
370 
371 
372 /*
373  * Called in the background process every time data is received.
374  * On Unix, we check to see if there is any data on our pipe
375  * (which would mean we have a stop position), and if it is, check if
376  * it is time to stop.
377  * On Windows, we are in a single process, so we can just check if it's
378  * time to stop.
379  */
380 static bool
382  bool segment_finished)
383 {
384  if (!has_xlogendptr)
385  {
386 #ifndef WIN32
387  fd_set fds;
388  struct timeval tv;
389  int r;
390 
391  /*
392  * Don't have the end pointer yet - check our pipe to see if it has
393  * been sent yet.
394  */
395  FD_ZERO(&fds);
396  FD_SET(bgpipe[0], &fds);
397 
398  MemSet(&tv, 0, sizeof(tv));
399 
400  r = select(bgpipe[0] + 1, &fds, NULL, NULL, &tv);
401  if (r == 1)
402  {
403  char xlogend[64];
404  uint32 hi,
405  lo;
406 
407  MemSet(xlogend, 0, sizeof(xlogend));
408  r = read(bgpipe[0], xlogend, sizeof(xlogend) - 1);
409  if (r < 0)
410  {
411  pg_log_error("could not read from ready pipe: %m");
412  exit(1);
413  }
414 
415  if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
416  {
417  pg_log_error("could not parse write-ahead log location \"%s\"",
418  xlogend);
419  exit(1);
420  }
421  xlogendptr = ((uint64) hi) << 32 | lo;
422  has_xlogendptr = 1;
423 
424  /*
425  * Fall through to check if we've reached the point further
426  * already.
427  */
428  }
429  else
430  {
431  /*
432  * No data received on the pipe means we don't know the end
433  * position yet - so just say it's not time to stop yet.
434  */
435  return false;
436  }
437 #else
438 
439  /*
440  * On win32, has_xlogendptr is set by the main thread, so if it's not
441  * set here, we just go back and wait until it shows up.
442  */
443  return false;
444 #endif
445  }
446 
447  /*
448  * At this point we have an end pointer, so compare it to the current
449  * position to figure out if it's time to stop.
450  */
451  if (segendpos >= xlogendptr)
452  return true;
453 
454  /*
455  * Have end pointer, but haven't reached it yet - so tell the caller to
456  * keep streaming.
457  */
458  return false;
459 }
460 
461 typedef struct
462 {
465  char xlog[MAXPGPATH]; /* directory or tarfile depending on mode */
467  int timeline;
469 
470 static int
472 {
473  StreamCtl stream;
474 
475  in_log_streamer = true;
476 
477  MemSet(&stream, 0, sizeof(stream));
478  stream.startpos = param->startptr;
479  stream.timeline = param->timeline;
480  stream.sysidentifier = param->sysidentifier;
482 #ifndef WIN32
483  stream.stop_socket = bgpipe[0];
484 #else
485  stream.stop_socket = PGINVALID_SOCKET;
486 #endif
488  stream.synchronous = false;
489  stream.do_sync = do_sync;
490  stream.mark_done = true;
491  stream.partial_suffix = NULL;
493 
494  if (format == 'p')
495  stream.walmethod = CreateWalDirectoryMethod(param->xlog, 0, do_sync);
496  else
498 
499  if (!ReceiveXlogStream(param->bgconn, &stream))
500 
501  /*
502  * Any errors will already have been reported in the function process,
503  * but we need to tell the parent that we didn't shutdown in a nice
504  * way.
505  */
506  return 1;
507 
508  if (!stream.walmethod->finish())
509  {
510  pg_log_error("could not finish writing WAL files: %m");
511  return 1;
512  }
513 
514  PQfinish(param->bgconn);
515 
516  if (format == 'p')
518  else
520  pg_free(stream.walmethod);
521 
522  return 0;
523 }
524 
525 /*
526  * Initiate background process for receiving xlog during the backup.
527  * The background stream will use its own database connection so we can
528  * stream the logfile in parallel with the backups.
529  */
530 static void
531 StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
532 {
533  logstreamer_param *param;
534  uint32 hi,
535  lo;
536  char statusdir[MAXPGPATH];
537 
538  param = pg_malloc0(sizeof(logstreamer_param));
539  param->timeline = timeline;
540  param->sysidentifier = sysidentifier;
541 
542  /* Convert the starting position */
543  if (sscanf(startpos, "%X/%X", &hi, &lo) != 2)
544  {
545  pg_log_error("could not parse write-ahead log location \"%s\"",
546  startpos);
547  exit(1);
548  }
549  param->startptr = ((uint64) hi) << 32 | lo;
550  /* Round off to even segment position */
551  param->startptr -= XLogSegmentOffset(param->startptr, WalSegSz);
552 
553 #ifndef WIN32
554  /* Create our background pipe */
555  if (pipe(bgpipe) < 0)
556  {
557  pg_log_error("could not create pipe for background process: %m");
558  exit(1);
559  }
560 #endif
561 
562  /* Get a second connection */
563  param->bgconn = GetConnection();
564  if (!param->bgconn)
565  /* Error message already written in GetConnection() */
566  exit(1);
567 
568  /* In post-10 cluster, pg_xlog has been renamed to pg_wal */
569  snprintf(param->xlog, sizeof(param->xlog), "%s/%s",
570  basedir,
572  "pg_xlog" : "pg_wal");
573 
574  /* Temporary replication slots are only supported in 10 and newer */
576  temp_replication_slot = false;
577 
578  /*
579  * Create replication slot if requested
580  */
582  replication_slot = psprintf("pg_basebackup_%d", (int) PQbackendPID(param->bgconn));
584  {
585  if (!CreateReplicationSlot(param->bgconn, replication_slot, NULL,
586  temp_replication_slot, true, true, false))
587  exit(1);
588 
589  if (verbose)
590  {
592  pg_log_info("created temporary replication slot \"%s\"",
594  else
595  pg_log_info("created replication slot \"%s\"",
597  }
598  }
599 
600  if (format == 'p')
601  {
602  /*
603  * Create pg_wal/archive_status or pg_xlog/archive_status (and thus
604  * pg_wal or pg_xlog) depending on the target server so we can write
605  * to basedir/pg_wal or basedir/pg_xlog as the directory entry in the
606  * tar file may arrive later.
607  */
608  snprintf(statusdir, sizeof(statusdir), "%s/%s/archive_status",
609  basedir,
611  "pg_xlog" : "pg_wal");
612 
613  if (pg_mkdir_p(statusdir, pg_dir_create_mode) != 0 && errno != EEXIST)
614  {
615  pg_log_error("could not create directory \"%s\": %m", statusdir);
616  exit(1);
617  }
618  }
619 
620  /*
621  * Start a child process and tell it to start streaming. On Unix, this is
622  * a fork(). On Windows, we create a thread.
623  */
624 #ifndef WIN32
625  bgchild = fork();
626  if (bgchild == 0)
627  {
628  /* in child process */
629  exit(LogStreamerMain(param));
630  }
631  else if (bgchild < 0)
632  {
633  pg_log_error("could not create background process: %m");
634  exit(1);
635  }
636 
637  /*
638  * Else we are in the parent process and all is well.
639  */
640  atexit(kill_bgchild_atexit);
641 #else /* WIN32 */
642  bgchild = _beginthreadex(NULL, 0, (void *) LogStreamerMain, param, 0, NULL);
643  if (bgchild == 0)
644  {
645  pg_log_error("could not create background thread: %m");
646  exit(1);
647  }
648 #endif
649 }
650 
651 /*
652  * Verify that the given directory exists and is empty. If it does not
653  * exist, it is created. If it exists but is not empty, an error will
654  * be given and the process ended.
655  */
656 static void
657 verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
658 {
659  switch (pg_check_dir(dirname))
660  {
661  case 0:
662 
663  /*
664  * Does not exist, so create
665  */
666  if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
667  {
668  pg_log_error("could not create directory \"%s\": %m", dirname);
669  exit(1);
670  }
671  if (created)
672  *created = true;
673  return;
674  case 1:
675 
676  /*
677  * Exists, empty
678  */
679  if (found)
680  *found = true;
681  return;
682  case 2:
683  case 3:
684  case 4:
685 
686  /*
687  * Exists, not empty
688  */
689  pg_log_error("directory \"%s\" exists but is not empty", dirname);
690  exit(1);
691  case -1:
692 
693  /*
694  * Access problem
695  */
696  pg_log_error("could not access directory \"%s\": %m", dirname);
697  exit(1);
698  }
699 }
700 
701 
702 /*
703  * Print a progress report based on the global variables. If verbose output
704  * is enabled, also print the current file name.
705  *
706  * Progress report is written at maximum once per second, unless the
707  * force parameter is set to true.
708  */
709 static void
710 progress_report(int tablespacenum, const char *filename, bool force)
711 {
712  int percent;
713  char totaldone_str[32];
714  char totalsize_str[32];
715  pg_time_t now;
716 
717  if (!showprogress)
718  return;
719 
720  now = time(NULL);
721  if (now == last_progress_report && !force)
722  return; /* Max once per second */
723 
725  percent = totalsize ? (int) ((totaldone / 1024) * 100 / totalsize) : 0;
726 
727  /*
728  * Avoid overflowing past 100% or the full size. This may make the total
729  * size number change as we approach the end of the backup (the estimate
730  * will always be wrong if WAL is included), but that's better than having
731  * the done column be bigger than the total.
732  */
733  if (percent > 100)
734  percent = 100;
735  if (totaldone / 1024 > totalsize)
736  totalsize = totaldone / 1024;
737 
738  /*
739  * Separate step to keep platform-dependent format code out of
740  * translatable strings. And we only test for INT64_FORMAT availability
741  * in snprintf, not fprintf.
742  */
743  snprintf(totaldone_str, sizeof(totaldone_str), INT64_FORMAT,
744  totaldone / 1024);
745  snprintf(totalsize_str, sizeof(totalsize_str), INT64_FORMAT, totalsize);
746 
747 #define VERBOSE_FILENAME_LENGTH 35
748  if (verbose)
749  {
750  if (!filename)
751 
752  /*
753  * No filename given, so clear the status line (used for last
754  * call)
755  */
756  fprintf(stderr,
757  ngettext("%*s/%s kB (100%%), %d/%d tablespace %*s",
758  "%*s/%s kB (100%%), %d/%d tablespaces %*s",
760  (int) strlen(totalsize_str),
761  totaldone_str, totalsize_str,
762  tablespacenum, tablespacecount,
763  VERBOSE_FILENAME_LENGTH + 5, "");
764  else
765  {
766  bool truncate = (strlen(filename) > VERBOSE_FILENAME_LENGTH);
767 
768  fprintf(stderr,
769  ngettext("%*s/%s kB (%d%%), %d/%d tablespace (%s%-*.*s)",
770  "%*s/%s kB (%d%%), %d/%d tablespaces (%s%-*.*s)",
772  (int) strlen(totalsize_str),
773  totaldone_str, totalsize_str, percent,
774  tablespacenum, tablespacecount,
775  /* Prefix with "..." if we do leading truncation */
776  truncate ? "..." : "",
779  /* Truncate filename at beginning if it's too long */
780  truncate ? filename + strlen(filename) - VERBOSE_FILENAME_LENGTH + 3 : filename);
781  }
782  }
783  else
784  fprintf(stderr,
785  ngettext("%*s/%s kB (%d%%), %d/%d tablespace",
786  "%*s/%s kB (%d%%), %d/%d tablespaces",
788  (int) strlen(totalsize_str),
789  totaldone_str, totalsize_str, percent,
790  tablespacenum, tablespacecount);
791 
792  if (isatty(fileno(stderr)))
793  fprintf(stderr, "\r");
794  else
795  fprintf(stderr, "\n");
796 }
797 
798 static int32
799 parse_max_rate(char *src)
800 {
801  double result;
802  char *after_num;
803  char *suffix = NULL;
804 
805  errno = 0;
806  result = strtod(src, &after_num);
807  if (src == after_num)
808  {
809  pg_log_error("transfer rate \"%s\" is not a valid value", src);
810  exit(1);
811  }
812  if (errno != 0)
813  {
814  pg_log_error("invalid transfer rate \"%s\": %m", src);
815  exit(1);
816  }
817 
818  if (result <= 0)
819  {
820  /*
821  * Reject obviously wrong values here.
822  */
823  pg_log_error("transfer rate must be greater than zero");
824  exit(1);
825  }
826 
827  /*
828  * Evaluate suffix, after skipping over possible whitespace. Lack of
829  * suffix means kilobytes.
830  */
831  while (*after_num != '\0' && isspace((unsigned char) *after_num))
832  after_num++;
833 
834  if (*after_num != '\0')
835  {
836  suffix = after_num;
837  if (*after_num == 'k')
838  {
839  /* kilobyte is the expected unit. */
840  after_num++;
841  }
842  else if (*after_num == 'M')
843  {
844  after_num++;
845  result *= 1024.0;
846  }
847  }
848 
849  /* The rest can only consist of white space. */
850  while (*after_num != '\0' && isspace((unsigned char) *after_num))
851  after_num++;
852 
853  if (*after_num != '\0')
854  {
855  pg_log_error("invalid --max-rate unit: \"%s\"", suffix);
856  exit(1);
857  }
858 
859  /* Valid integer? */
860  if ((uint64) result != (uint64) ((uint32) result))
861  {
862  pg_log_error("transfer rate \"%s\" exceeds integer range", src);
863  exit(1);
864  }
865 
866  /*
867  * The range is checked on the server side too, but avoid the server
868  * connection if a nonsensical value was passed.
869  */
870  if (result < MAX_RATE_LOWER || result > MAX_RATE_UPPER)
871  {
872  pg_log_error("transfer rate \"%s\" is out of range", src);
873  exit(1);
874  }
875 
876  return (int32) result;
877 }
878 
879 /*
880  * Write a piece of tar data
881  */
882 static void
884 #ifdef HAVE_LIBZ
885  gzFile ztarfile,
886 #endif
887  FILE *tarfile, char *buf, int r, char *current_file)
888 {
889 #ifdef HAVE_LIBZ
890  if (ztarfile != NULL)
891  {
892  if (gzwrite(ztarfile, buf, r) != r)
893  {
894  pg_log_error("could not write to compressed file \"%s\": %s",
895  current_file, get_gz_error(ztarfile));
896  exit(1);
897  }
898  }
899  else
900 #endif
901  {
902  if (fwrite(buf, r, 1, tarfile) != 1)
903  {
904  pg_log_error("could not write to file \"%s\": %m", current_file);
905  exit(1);
906  }
907  }
908 }
909 
910 #ifdef HAVE_LIBZ
911 #define WRITE_TAR_DATA(buf, sz) writeTarData(ztarfile, tarfile, buf, sz, filename)
912 #else
913 #define WRITE_TAR_DATA(buf, sz) writeTarData(tarfile, buf, sz, filename)
914 #endif
915 
916 /*
917  * Receive a tar format file from the connection to the server, and write
918  * the data from this file directly into a tar file. If compression is
919  * enabled, the data will be compressed while written to the file.
920  *
921  * The file will be named base.tar[.gz] if it's for the main data directory
922  * or <tablespaceoid>.tar[.gz] if it's for another tablespace.
923  *
924  * No attempt to inspect or validate the contents of the file is done.
925  */
926 static void
927 ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
928 {
929  char filename[MAXPGPATH];
930  char *copybuf = NULL;
931  FILE *tarfile = NULL;
932  char tarhdr[512];
933  bool basetablespace = PQgetisnull(res, rownum, 0);
934  bool in_tarhdr = true;
935  bool skip_file = false;
936  bool is_recovery_guc_supported = true;
937  bool is_postgresql_auto_conf = false;
938  bool found_postgresql_auto_conf = false;
939  int file_padding_len = 0;
940  size_t tarhdrsz = 0;
941  pgoff_t filesz = 0;
942 
943 #ifdef HAVE_LIBZ
944  gzFile ztarfile = NULL;
945 #endif
946 
947  /* recovery.conf is integrated into postgresql.conf in 12 and newer */
949  is_recovery_guc_supported = false;
950 
951  if (basetablespace)
952  {
953  /*
954  * Base tablespaces
955  */
956  if (strcmp(basedir, "-") == 0)
957  {
958 #ifdef WIN32
959  _setmode(fileno(stdout), _O_BINARY);
960 #endif
961 
962 #ifdef HAVE_LIBZ
963  if (compresslevel != 0)
964  {
965  ztarfile = gzdopen(dup(fileno(stdout)), "wb");
966  if (gzsetparams(ztarfile, compresslevel,
967  Z_DEFAULT_STRATEGY) != Z_OK)
968  {
969  pg_log_error("could not set compression level %d: %s",
970  compresslevel, get_gz_error(ztarfile));
971  exit(1);
972  }
973  }
974  else
975 #endif
976  tarfile = stdout;
977  strcpy(filename, "-");
978  }
979  else
980  {
981 #ifdef HAVE_LIBZ
982  if (compresslevel != 0)
983  {
984  snprintf(filename, sizeof(filename), "%s/base.tar.gz", basedir);
985  ztarfile = gzopen(filename, "wb");
986  if (gzsetparams(ztarfile, compresslevel,
987  Z_DEFAULT_STRATEGY) != Z_OK)
988  {
989  pg_log_error("could not set compression level %d: %s",
990  compresslevel, get_gz_error(ztarfile));
991  exit(1);
992  }
993  }
994  else
995 #endif
996  {
997  snprintf(filename, sizeof(filename), "%s/base.tar", basedir);
998  tarfile = fopen(filename, "wb");
999  }
1000  }
1001  }
1002  else
1003  {
1004  /*
1005  * Specific tablespace
1006  */
1007 #ifdef HAVE_LIBZ
1008  if (compresslevel != 0)
1009  {
1010  snprintf(filename, sizeof(filename), "%s/%s.tar.gz", basedir,
1011  PQgetvalue(res, rownum, 0));
1012  ztarfile = gzopen(filename, "wb");
1013  if (gzsetparams(ztarfile, compresslevel,
1014  Z_DEFAULT_STRATEGY) != Z_OK)
1015  {
1016  pg_log_error("could not set compression level %d: %s",
1017  compresslevel, get_gz_error(ztarfile));
1018  exit(1);
1019  }
1020  }
1021  else
1022 #endif
1023  {
1024  snprintf(filename, sizeof(filename), "%s/%s.tar", basedir,
1025  PQgetvalue(res, rownum, 0));
1026  tarfile = fopen(filename, "wb");
1027  }
1028  }
1029 
1030 #ifdef HAVE_LIBZ
1031  if (compresslevel != 0)
1032  {
1033  if (!ztarfile)
1034  {
1035  /* Compression is in use */
1036  pg_log_error("could not create compressed file \"%s\": %s",
1037  filename, get_gz_error(ztarfile));
1038  exit(1);
1039  }
1040  }
1041  else
1042 #endif
1043  {
1044  /* Either no zlib support, or zlib support but compresslevel = 0 */
1045  if (!tarfile)
1046  {
1047  pg_log_error("could not create file \"%s\": %m", filename);
1048  exit(1);
1049  }
1050  }
1051 
1052  /*
1053  * Get the COPY data stream
1054  */
1055  res = PQgetResult(conn);
1056  if (PQresultStatus(res) != PGRES_COPY_OUT)
1057  {
1058  pg_log_error("could not get COPY data stream: %s",
1059  PQerrorMessage(conn));
1060  exit(1);
1061  }
1062 
1063  while (1)
1064  {
1065  int r;
1066 
1067  if (copybuf != NULL)
1068  {
1069  PQfreemem(copybuf);
1070  copybuf = NULL;
1071  }
1072 
1073  r = PQgetCopyData(conn, &copybuf, 0);
1074  if (r == -1)
1075  {
1076  /*
1077  * End of chunk. If requested, and this is the base tablespace,
1078  * write configuration file into the tarfile. When done, close the
1079  * file (but not stdout).
1080  *
1081  * Also, write two completely empty blocks at the end of the tar
1082  * file, as required by some tar programs.
1083  */
1084  char zerobuf[1024];
1085 
1086  MemSet(zerobuf, 0, sizeof(zerobuf));
1087 
1088  if (basetablespace && writerecoveryconf)
1089  {
1090  char header[512];
1091 
1092  /*
1093  * If postgresql.auto.conf has not been found in the streamed
1094  * data, add recovery configuration to postgresql.auto.conf if
1095  * recovery parameters are GUCs. If the instance connected to
1096  * is older than 12, create recovery.conf with this data
1097  * otherwise.
1098  */
1099  if (!found_postgresql_auto_conf || !is_recovery_guc_supported)
1100  {
1101  int padding;
1102 
1103  tarCreateHeader(header,
1104  is_recovery_guc_supported ? "postgresql.auto.conf" : "recovery.conf",
1105  NULL,
1106  recoveryconfcontents->len,
1107  pg_file_create_mode, 04000, 02000,
1108  time(NULL));
1109 
1110  padding = ((recoveryconfcontents->len + 511) & ~511) - recoveryconfcontents->len;
1111 
1112  WRITE_TAR_DATA(header, sizeof(header));
1113  WRITE_TAR_DATA(recoveryconfcontents->data,
1114  recoveryconfcontents->len);
1115  if (padding)
1116  WRITE_TAR_DATA(zerobuf, padding);
1117  }
1118 
1119  /*
1120  * standby.signal is supported only if recovery parameters are
1121  * GUCs.
1122  */
1123  if (is_recovery_guc_supported)
1124  {
1125  tarCreateHeader(header, "standby.signal", NULL,
1126  0, /* zero-length file */
1127  pg_file_create_mode, 04000, 02000,
1128  time(NULL));
1129 
1130  WRITE_TAR_DATA(header, sizeof(header));
1131  WRITE_TAR_DATA(zerobuf, 511);
1132  }
1133  }
1134 
1135  /* 2 * 512 bytes empty data at end of file */
1136  WRITE_TAR_DATA(zerobuf, sizeof(zerobuf));
1137 
1138 #ifdef HAVE_LIBZ
1139  if (ztarfile != NULL)
1140  {
1141  if (gzclose(ztarfile) != 0)
1142  {
1143  pg_log_error("could not close compressed file \"%s\": %s",
1144  filename, get_gz_error(ztarfile));
1145  exit(1);
1146  }
1147  }
1148  else
1149 #endif
1150  {
1151  if (strcmp(basedir, "-") != 0)
1152  {
1153  if (fclose(tarfile) != 0)
1154  {
1155  pg_log_error("could not close file \"%s\": %m",
1156  filename);
1157  exit(1);
1158  }
1159  }
1160  }
1161 
1162  break;
1163  }
1164  else if (r == -2)
1165  {
1166  pg_log_error("could not read COPY data: %s",
1167  PQerrorMessage(conn));
1168  exit(1);
1169  }
1170 
1171  if (!writerecoveryconf || !basetablespace)
1172  {
1173  /*
1174  * When not writing config file, or when not working on the base
1175  * tablespace, we never have to look for an existing configuration
1176  * file in the stream.
1177  */
1178  WRITE_TAR_DATA(copybuf, r);
1179  }
1180  else
1181  {
1182  /*
1183  * Look for a config file in the existing tar stream. If it's
1184  * there, we must skip it so we can later overwrite it with our
1185  * own version of the file.
1186  *
1187  * To do this, we have to process the individual files inside the
1188  * TAR stream. The stream consists of a header and zero or more
1189  * chunks, all 512 bytes long. The stream from the server is
1190  * broken up into smaller pieces, so we have to track the size of
1191  * the files to find the next header structure.
1192  */
1193  int rr = r;
1194  int pos = 0;
1195 
1196  while (rr > 0)
1197  {
1198  if (in_tarhdr)
1199  {
1200  /*
1201  * We're currently reading a header structure inside the
1202  * TAR stream, i.e. the file metadata.
1203  */
1204  if (tarhdrsz < 512)
1205  {
1206  /*
1207  * Copy the header structure into tarhdr in case the
1208  * header is not aligned to 512 bytes or it's not
1209  * returned in whole by the last PQgetCopyData call.
1210  */
1211  int hdrleft;
1212  int bytes2copy;
1213 
1214  hdrleft = 512 - tarhdrsz;
1215  bytes2copy = (rr > hdrleft ? hdrleft : rr);
1216 
1217  memcpy(&tarhdr[tarhdrsz], copybuf + pos, bytes2copy);
1218 
1219  rr -= bytes2copy;
1220  pos += bytes2copy;
1221  tarhdrsz += bytes2copy;
1222  }
1223  else
1224  {
1225  /*
1226  * We have the complete header structure in tarhdr,
1227  * look at the file metadata: we may want append
1228  * recovery info into postgresql.auto.conf and skip
1229  * standby.signal file if recovery parameters are
1230  * integrated as GUCs, and recovery.conf otherwise. In
1231  * both cases we must calculate tar padding.
1232  */
1233  if (is_recovery_guc_supported)
1234  {
1235  skip_file = (strcmp(&tarhdr[0], "standby.signal") == 0);
1236  is_postgresql_auto_conf = (strcmp(&tarhdr[0], "postgresql.auto.conf") == 0);
1237  }
1238  else
1239  skip_file = (strcmp(&tarhdr[0], "recovery.conf") == 0);
1240 
1241  filesz = read_tar_number(&tarhdr[124], 12);
1242  file_padding_len = ((filesz + 511) & ~511) - filesz;
1243 
1244  if (is_recovery_guc_supported &&
1245  is_postgresql_auto_conf &&
1247  {
1248  /* replace tar header */
1249  char header[512];
1250 
1251  tarCreateHeader(header, "postgresql.auto.conf", NULL,
1252  filesz + recoveryconfcontents->len,
1253  pg_file_create_mode, 04000, 02000,
1254  time(NULL));
1255 
1256  WRITE_TAR_DATA(header, sizeof(header));
1257  }
1258  else
1259  {
1260  /* copy stream with padding */
1261  filesz += file_padding_len;
1262 
1263  if (!skip_file)
1264  {
1265  /*
1266  * If we're not skipping the file, write the
1267  * tar header unmodified.
1268  */
1269  WRITE_TAR_DATA(tarhdr, 512);
1270  }
1271  }
1272 
1273  /* Next part is the file, not the header */
1274  in_tarhdr = false;
1275  }
1276  }
1277  else
1278  {
1279  /*
1280  * We're processing a file's contents.
1281  */
1282  if (filesz > 0)
1283  {
1284  /*
1285  * We still have data to read (and possibly write).
1286  */
1287  int bytes2write;
1288 
1289  bytes2write = (filesz > rr ? rr : filesz);
1290 
1291  if (!skip_file)
1292  WRITE_TAR_DATA(copybuf + pos, bytes2write);
1293 
1294  rr -= bytes2write;
1295  pos += bytes2write;
1296  filesz -= bytes2write;
1297  }
1298  else if (is_recovery_guc_supported &&
1299  is_postgresql_auto_conf &&
1301  {
1302  /* append recovery config to postgresql.auto.conf */
1303  int padding;
1304  int tailsize;
1305 
1306  tailsize = (512 - file_padding_len) + recoveryconfcontents->len;
1307  padding = ((tailsize + 511) & ~511) - tailsize;
1308 
1309  WRITE_TAR_DATA(recoveryconfcontents->data, recoveryconfcontents->len);
1310 
1311  if (padding)
1312  {
1313  char zerobuf[512];
1314 
1315  MemSet(zerobuf, 0, sizeof(zerobuf));
1316  WRITE_TAR_DATA(zerobuf, padding);
1317  }
1318 
1319  /* skip original file padding */
1320  is_postgresql_auto_conf = false;
1321  skip_file = true;
1322  filesz += file_padding_len;
1323 
1324  found_postgresql_auto_conf = true;
1325  }
1326  else
1327  {
1328  /*
1329  * No more data in the current file, the next piece of
1330  * data (if any) will be a new file header structure.
1331  */
1332  in_tarhdr = true;
1333  skip_file = false;
1334  is_postgresql_auto_conf = false;
1335  tarhdrsz = 0;
1336  filesz = 0;
1337  }
1338  }
1339  }
1340  }
1341  totaldone += r;
1342  progress_report(rownum, filename, false);
1343  } /* while (1) */
1344  progress_report(rownum, filename, true);
1345 
1346  if (copybuf != NULL)
1347  PQfreemem(copybuf);
1348 
1349  /* sync the resulting tar file, errors are not considered fatal */
1350  if (do_sync && strcmp(basedir, "-") != 0)
1351  (void) fsync_fname(filename, false);
1352 }
1353 
1354 
1355 /*
1356  * Retrieve tablespace path, either relocated or original depending on whether
1357  * -T was passed or not.
1358  */
1359 static const char *
1360 get_tablespace_mapping(const char *dir)
1361 {
1362  TablespaceListCell *cell;
1363  char canon_dir[MAXPGPATH];
1364 
1365  /* Canonicalize path for comparison consistency */
1366  strlcpy(canon_dir, dir, sizeof(canon_dir));
1367  canonicalize_path(canon_dir);
1368 
1369  for (cell = tablespace_dirs.head; cell; cell = cell->next)
1370  if (strcmp(canon_dir, cell->old_dir) == 0)
1371  return cell->new_dir;
1372 
1373  return dir;
1374 }
1375 
1376 
1377 /*
1378  * Receive a tar format stream from the connection to the server, and unpack
1379  * the contents of it into a directory. Only files, directories and
1380  * symlinks are supported, no other kinds of special files.
1381  *
1382  * If the data is for the main data directory, it will be restored in the
1383  * specified directory. If it's for another tablespace, it will be restored
1384  * in the original or mapped directory.
1385  */
1386 static void
1388 {
1389  char current_path[MAXPGPATH];
1390  char filename[MAXPGPATH];
1391  const char *mapped_tblspc_path;
1392  pgoff_t current_len_left = 0;
1393  int current_padding = 0;
1394  bool basetablespace;
1395  char *copybuf = NULL;
1396  FILE *file = NULL;
1397 
1398  basetablespace = PQgetisnull(res, rownum, 0);
1399  if (basetablespace)
1400  strlcpy(current_path, basedir, sizeof(current_path));
1401  else
1402  strlcpy(current_path,
1403  get_tablespace_mapping(PQgetvalue(res, rownum, 1)),
1404  sizeof(current_path));
1405 
1406  /*
1407  * Get the COPY data
1408  */
1409  res = PQgetResult(conn);
1410  if (PQresultStatus(res) != PGRES_COPY_OUT)
1411  {
1412  pg_log_error("could not get COPY data stream: %s",
1413  PQerrorMessage(conn));
1414  exit(1);
1415  }
1416 
1417  while (1)
1418  {
1419  int r;
1420 
1421  if (copybuf != NULL)
1422  {
1423  PQfreemem(copybuf);
1424  copybuf = NULL;
1425  }
1426 
1427  r = PQgetCopyData(conn, &copybuf, 0);
1428 
1429  if (r == -1)
1430  {
1431  /*
1432  * End of chunk
1433  */
1434  if (file)
1435  fclose(file);
1436 
1437  break;
1438  }
1439  else if (r == -2)
1440  {
1441  pg_log_error("could not read COPY data: %s",
1442  PQerrorMessage(conn));
1443  exit(1);
1444  }
1445 
1446  if (file == NULL)
1447  {
1448 #ifndef WIN32
1449  int filemode;
1450 #endif
1451 
1452  /*
1453  * No current file, so this must be the header for a new file
1454  */
1455  if (r != 512)
1456  {
1457  pg_log_error("invalid tar block header size: %d", r);
1458  exit(1);
1459  }
1460  totaldone += 512;
1461 
1462  current_len_left = read_tar_number(&copybuf[124], 12);
1463 
1464 #ifndef WIN32
1465  /* Set permissions on the file */
1466  filemode = read_tar_number(&copybuf[100], 8);
1467 #endif
1468 
1469  /*
1470  * All files are padded up to 512 bytes
1471  */
1472  current_padding =
1473  ((current_len_left + 511) & ~511) - current_len_left;
1474 
1475  /*
1476  * First part of header is zero terminated filename
1477  */
1478  snprintf(filename, sizeof(filename), "%s/%s", current_path,
1479  copybuf);
1480  if (filename[strlen(filename) - 1] == '/')
1481  {
1482  /*
1483  * Ends in a slash means directory or symlink to directory
1484  */
1485  if (copybuf[156] == '5')
1486  {
1487  /*
1488  * Directory
1489  */
1490  filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */
1491  if (mkdir(filename, pg_dir_create_mode) != 0)
1492  {
1493  /*
1494  * When streaming WAL, pg_wal (or pg_xlog for pre-9.6
1495  * clusters) will have been created by the wal
1496  * receiver process. Also, when the WAL directory
1497  * location was specified, pg_wal (or pg_xlog) has
1498  * already been created as a symbolic link before
1499  * starting the actual backup. So just ignore creation
1500  * failures on related directories.
1501  */
1502  if (!((pg_str_endswith(filename, "/pg_wal") ||
1503  pg_str_endswith(filename, "/pg_xlog") ||
1504  pg_str_endswith(filename, "/archive_status")) &&
1505  errno == EEXIST))
1506  {
1507  pg_log_error("could not create directory \"%s\": %m",
1508  filename);
1509  exit(1);
1510  }
1511  }
1512 #ifndef WIN32
1513  if (chmod(filename, (mode_t) filemode))
1514  pg_log_error("could not set permissions on directory \"%s\": %m",
1515  filename);
1516 #endif
1517  }
1518  else if (copybuf[156] == '2')
1519  {
1520  /*
1521  * Symbolic link
1522  *
1523  * It's most likely a link in pg_tblspc directory, to the
1524  * location of a tablespace. Apply any tablespace mapping
1525  * given on the command line (--tablespace-mapping). (We
1526  * blindly apply the mapping without checking that the
1527  * link really is inside pg_tblspc. We don't expect there
1528  * to be other symlinks in a data directory, but if there
1529  * are, you can call it an undocumented feature that you
1530  * can map them too.)
1531  */
1532  filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */
1533 
1534  mapped_tblspc_path = get_tablespace_mapping(&copybuf[157]);
1535  if (symlink(mapped_tblspc_path, filename) != 0)
1536  {
1537  pg_log_error("could not create symbolic link from \"%s\" to \"%s\": %m",
1538  filename, mapped_tblspc_path);
1539  exit(1);
1540  }
1541  }
1542  else
1543  {
1544  pg_log_error("unrecognized link indicator \"%c\"",
1545  copybuf[156]);
1546  exit(1);
1547  }
1548  continue; /* directory or link handled */
1549  }
1550 
1551  /*
1552  * regular file
1553  */
1554  file = fopen(filename, "wb");
1555  if (!file)
1556  {
1557  pg_log_error("could not create file \"%s\": %m", filename);
1558  exit(1);
1559  }
1560 
1561 #ifndef WIN32
1562  if (chmod(filename, (mode_t) filemode))
1563  pg_log_error("could not set permissions on file \"%s\": %m",
1564  filename);
1565 #endif
1566 
1567  if (current_len_left == 0)
1568  {
1569  /*
1570  * Done with this file, next one will be a new tar header
1571  */
1572  fclose(file);
1573  file = NULL;
1574  continue;
1575  }
1576  } /* new file */
1577  else
1578  {
1579  /*
1580  * Continuing blocks in existing file
1581  */
1582  if (current_len_left == 0 && r == current_padding)
1583  {
1584  /*
1585  * Received the padding block for this file, ignore it and
1586  * close the file, then move on to the next tar header.
1587  */
1588  fclose(file);
1589  file = NULL;
1590  totaldone += r;
1591  continue;
1592  }
1593 
1594  if (fwrite(copybuf, r, 1, file) != 1)
1595  {
1596  pg_log_error("could not write to file \"%s\": %m", filename);
1597  exit(1);
1598  }
1599  totaldone += r;
1600  progress_report(rownum, filename, false);
1601 
1602  current_len_left -= r;
1603  if (current_len_left == 0 && current_padding == 0)
1604  {
1605  /*
1606  * Received the last block, and there is no padding to be
1607  * expected. Close the file and move on to the next tar
1608  * header.
1609  */
1610  fclose(file);
1611  file = NULL;
1612  continue;
1613  }
1614  } /* continuing data in existing file */
1615  } /* loop over all data blocks */
1616  progress_report(rownum, filename, true);
1617 
1618  if (file != NULL)
1619  {
1620  pg_log_error("COPY stream ended before last file was finished");
1621  exit(1);
1622  }
1623 
1624  if (copybuf != NULL)
1625  PQfreemem(copybuf);
1626 
1627  if (basetablespace && writerecoveryconf)
1629 
1630  /*
1631  * No data is synced here, everything is done for all tablespaces at the
1632  * end.
1633  */
1634 }
1635 
1636 /*
1637  * Escape a string so that it can be used as a value in a key-value pair
1638  * a configuration file.
1639  */
1640 static char *
1641 escape_quotes(const char *src)
1642 {
1643  char *result = escape_single_quotes_ascii(src);
1644 
1645  if (!result)
1646  {
1647  pg_log_error("out of memory");
1648  exit(1);
1649  }
1650  return result;
1651 }
1652 
1653 /*
1654  * Create a configuration file in memory using a PQExpBuffer
1655  */
1656 static void
1658 {
1659  PQconninfoOption *connOptions;
1661  PQExpBufferData conninfo_buf;
1662  char *escaped;
1663 
1664  recoveryconfcontents = createPQExpBuffer();
1665  if (!recoveryconfcontents)
1666  {
1667  pg_log_error("out of memory");
1668  exit(1);
1669  }
1670 
1671  /*
1672  * In PostgreSQL 12 and newer versions, standby_mode is gone, replaced by
1673  * standby.signal to trigger a standby state at recovery.
1674  */
1676  appendPQExpBufferStr(recoveryconfcontents, "standby_mode = 'on'\n");
1677 
1678  connOptions = PQconninfo(conn);
1679  if (connOptions == NULL)
1680  {
1681  pg_log_error("out of memory");
1682  exit(1);
1683  }
1684 
1685  initPQExpBuffer(&conninfo_buf);
1686  for (option = connOptions; option && option->keyword; option++)
1687  {
1688  /* Omit empty settings and those libpqwalreceiver overrides. */
1689  if (strcmp(option->keyword, "replication") == 0 ||
1690  strcmp(option->keyword, "dbname") == 0 ||
1691  strcmp(option->keyword, "fallback_application_name") == 0 ||
1692  (option->val == NULL) ||
1693  (option->val != NULL && option->val[0] == '\0'))
1694  continue;
1695 
1696  /* Separate key-value pairs with spaces */
1697  if (conninfo_buf.len != 0)
1698  appendPQExpBufferChar(&conninfo_buf, ' ');
1699 
1700  /*
1701  * Write "keyword=value" pieces, the value string is escaped and/or
1702  * quoted if necessary.
1703  */
1704  appendPQExpBuffer(&conninfo_buf, "%s=", option->keyword);
1705  appendConnStrVal(&conninfo_buf, option->val);
1706  }
1707 
1708  /*
1709  * Escape the connection string, so that it can be put in the config file.
1710  * Note that this is different from the escaping of individual connection
1711  * options above!
1712  */
1713  escaped = escape_quotes(conninfo_buf.data);
1714  appendPQExpBuffer(recoveryconfcontents, "primary_conninfo = '%s'\n", escaped);
1715  free(escaped);
1716 
1717  if (replication_slot)
1718  {
1719  /* unescaped: ReplicationSlotValidateName allows [a-z0-9_] only */
1720  appendPQExpBuffer(recoveryconfcontents, "primary_slot_name = '%s'\n",
1722  }
1723 
1724  if (PQExpBufferBroken(recoveryconfcontents) ||
1725  PQExpBufferDataBroken(conninfo_buf))
1726  {
1727  pg_log_error("out of memory");
1728  exit(1);
1729  }
1730 
1731  termPQExpBuffer(&conninfo_buf);
1732 
1733  PQconninfoFree(connOptions);
1734 }
1735 
1736 
1737 /*
1738  * Write the configuration file into the directory specified in basedir,
1739  * with the contents already collected in memory appended. Then write
1740  * the signal file into the basedir. If the server does not support
1741  * recovery parameters as GUCs, the signal file is not necessary, and
1742  * configuration is written to recovery.conf.
1743  */
1744 static void
1746 {
1747  char filename[MAXPGPATH];
1748  FILE *cf;
1749  bool is_recovery_guc_supported = true;
1750 
1752  is_recovery_guc_supported = false;
1753 
1754  snprintf(filename, MAXPGPATH, "%s/%s", basedir,
1755  is_recovery_guc_supported ? "postgresql.auto.conf" : "recovery.conf");
1756 
1757  cf = fopen(filename, is_recovery_guc_supported ? "a" : "w");
1758  if (cf == NULL)
1759  {
1760  pg_log_error("could not open file \"%s\": %m", filename);
1761  exit(1);
1762  }
1763 
1764  if (fwrite(recoveryconfcontents->data, recoveryconfcontents->len, 1, cf) != 1)
1765  {
1766  pg_log_error("could not write to file \"%s\": %m", filename);
1767  exit(1);
1768  }
1769 
1770  fclose(cf);
1771 
1772  if (is_recovery_guc_supported)
1773  {
1774  snprintf(filename, MAXPGPATH, "%s/%s", basedir, "standby.signal");
1775  cf = fopen(filename, "w");
1776  if (cf == NULL)
1777  {
1778  pg_log_error("could not create file \"%s\": %m", filename);
1779  exit(1);
1780  }
1781 
1782  fclose(cf);
1783  }
1784 }
1785 
1786 
1787 static void
1789 {
1790  PGresult *res;
1791  char *sysidentifier;
1792  TimeLineID latesttli;
1793  TimeLineID starttli;
1794  char *basebkp;
1795  char escaped_label[MAXPGPATH];
1796  char *maxrate_clause = NULL;
1797  int i;
1798  char xlogstart[64];
1799  char xlogend[64];
1800  int minServerMajor,
1801  maxServerMajor;
1802  int serverVersion,
1803  serverMajor;
1804 
1805  Assert(conn != NULL);
1806 
1807  /*
1808  * Check server version. BASE_BACKUP command was introduced in 9.1, so we
1809  * can't work with servers older than 9.1.
1810  */
1811  minServerMajor = 901;
1812  maxServerMajor = PG_VERSION_NUM / 100;
1813  serverVersion = PQserverVersion(conn);
1814  serverMajor = serverVersion / 100;
1815  if (serverMajor < minServerMajor || serverMajor > maxServerMajor)
1816  {
1817  const char *serverver = PQparameterStatus(conn, "server_version");
1818 
1819  pg_log_error("incompatible server version %s",
1820  serverver ? serverver : "'unknown'");
1821  exit(1);
1822  }
1823 
1824  /*
1825  * If WAL streaming was requested, also check that the server is new
1826  * enough for that.
1827  */
1829  {
1830  /*
1831  * Error message already written in CheckServerVersionForStreaming(),
1832  * but add a hint about using -X none.
1833  */
1834  pg_log_info("HINT: use -X none or -X fetch to disable log streaming");
1835  exit(1);
1836  }
1837 
1838  /*
1839  * Build contents of configuration file if requested
1840  */
1841  if (writerecoveryconf)
1843 
1844  /*
1845  * Run IDENTIFY_SYSTEM so we can get the timeline
1846  */
1847  if (!RunIdentifySystem(conn, &sysidentifier, &latesttli, NULL, NULL))
1848  exit(1);
1849 
1850  /*
1851  * Start the actual backup
1852  */
1853  PQescapeStringConn(conn, escaped_label, label, sizeof(escaped_label), &i);
1854 
1855  if (maxrate > 0)
1856  maxrate_clause = psprintf("MAX_RATE %u", maxrate);
1857 
1858  if (verbose)
1859  pg_log_info("initiating base backup, waiting for checkpoint to complete");
1860 
1861  if (showprogress && !verbose)
1862  {
1863  fprintf(stderr, "waiting for checkpoint");
1864  if (isatty(fileno(stderr)))
1865  fprintf(stderr, "\r");
1866  else
1867  fprintf(stderr, "\n");
1868  }
1869 
1870  basebkp =
1871  psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s",
1872  escaped_label,
1873  showprogress ? "PROGRESS" : "",
1874  includewal == FETCH_WAL ? "WAL" : "",
1875  fastcheckpoint ? "FAST" : "",
1876  includewal == NO_WAL ? "" : "NOWAIT",
1877  maxrate_clause ? maxrate_clause : "",
1878  format == 't' ? "TABLESPACE_MAP" : "",
1879  verify_checksums ? "" : "NOVERIFY_CHECKSUMS");
1880 
1881  if (PQsendQuery(conn, basebkp) == 0)
1882  {
1883  pg_log_error("could not send replication command \"%s\": %s",
1884  "BASE_BACKUP", PQerrorMessage(conn));
1885  exit(1);
1886  }
1887 
1888  /*
1889  * Get the starting WAL location
1890  */
1891  res = PQgetResult(conn);
1892  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1893  {
1894  pg_log_error("could not initiate base backup: %s",
1895  PQerrorMessage(conn));
1896  exit(1);
1897  }
1898  if (PQntuples(res) != 1)
1899  {
1900  pg_log_error("server returned unexpected response to BASE_BACKUP command; got %d rows and %d fields, expected %d rows and %d fields",
1901  PQntuples(res), PQnfields(res), 1, 2);
1902  exit(1);
1903  }
1904 
1905  strlcpy(xlogstart, PQgetvalue(res, 0, 0), sizeof(xlogstart));
1906 
1907  if (verbose)
1908  pg_log_info("checkpoint completed");
1909 
1910  /*
1911  * 9.3 and later sends the TLI of the starting point. With older servers,
1912  * assume it's the same as the latest timeline reported by
1913  * IDENTIFY_SYSTEM.
1914  */
1915  if (PQnfields(res) >= 2)
1916  starttli = atoi(PQgetvalue(res, 0, 1));
1917  else
1918  starttli = latesttli;
1919  PQclear(res);
1920  MemSet(xlogend, 0, sizeof(xlogend));
1921 
1922  if (verbose && includewal != NO_WAL)
1923  pg_log_info("write-ahead log start point: %s on timeline %u",
1924  xlogstart, starttli);
1925 
1926  /*
1927  * Get the header
1928  */
1929  res = PQgetResult(conn);
1930  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1931  {
1932  pg_log_error("could not get backup header: %s",
1933  PQerrorMessage(conn));
1934  exit(1);
1935  }
1936  if (PQntuples(res) < 1)
1937  {
1938  pg_log_error("no data returned from server");
1939  exit(1);
1940  }
1941 
1942  /*
1943  * Sum up the total size, for progress reporting
1944  */
1945  totalsize = totaldone = 0;
1946  tablespacecount = PQntuples(res);
1947  for (i = 0; i < PQntuples(res); i++)
1948  {
1949  totalsize += atol(PQgetvalue(res, i, 2));
1950 
1951  /*
1952  * Verify tablespace directories are empty. Don't bother with the
1953  * first once since it can be relocated, and it will be checked before
1954  * we do anything anyway.
1955  */
1956  if (format == 'p' && !PQgetisnull(res, i, 1))
1957  {
1958  char *path = unconstify(char *, get_tablespace_mapping(PQgetvalue(res, i, 1)));
1959 
1961  }
1962  }
1963 
1964  /*
1965  * When writing to stdout, require a single tablespace
1966  */
1967  if (format == 't' && strcmp(basedir, "-") == 0 && PQntuples(res) > 1)
1968  {
1969  pg_log_error("can only write single tablespace to stdout, database has %d",
1970  PQntuples(res));
1971  exit(1);
1972  }
1973 
1974  /*
1975  * If we're streaming WAL, start the streaming session before we start
1976  * receiving the actual data chunks.
1977  */
1978  if (includewal == STREAM_WAL)
1979  {
1980  if (verbose)
1981  pg_log_info("starting background WAL receiver");
1982  StartLogStreamer(xlogstart, starttli, sysidentifier);
1983  }
1984 
1985  /*
1986  * Start receiving chunks
1987  */
1988  for (i = 0; i < PQntuples(res); i++)
1989  {
1990  if (format == 't')
1991  ReceiveTarFile(conn, res, i);
1992  else
1993  ReceiveAndUnpackTarFile(conn, res, i);
1994  } /* Loop over all tablespaces */
1995 
1996  if (showprogress)
1997  {
1998  progress_report(PQntuples(res), NULL, true);
1999  if (isatty(fileno(stderr)))
2000  fprintf(stderr, "\n"); /* Need to move to next line */
2001  }
2002 
2003  PQclear(res);
2004 
2005  /*
2006  * Get the stop position
2007  */
2008  res = PQgetResult(conn);
2009  if (PQresultStatus(res) != PGRES_TUPLES_OK)
2010  {
2011  pg_log_error("could not get write-ahead log end position from server: %s",
2012  PQerrorMessage(conn));
2013  exit(1);
2014  }
2015  if (PQntuples(res) != 1)
2016  {
2017  pg_log_error("no write-ahead log end position returned from server");
2018  exit(1);
2019  }
2020  strlcpy(xlogend, PQgetvalue(res, 0, 0), sizeof(xlogend));
2021  if (verbose && includewal != NO_WAL)
2022  pg_log_info("write-ahead log end point: %s", xlogend);
2023  PQclear(res);
2024 
2025  res = PQgetResult(conn);
2026  if (PQresultStatus(res) != PGRES_COMMAND_OK)
2027  {
2028  const char *sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
2029 
2030  if (sqlstate &&
2031  strcmp(sqlstate, ERRCODE_DATA_CORRUPTED) == 0)
2032  {
2033  pg_log_error("checksum error occurred");
2034  checksum_failure = true;
2035  }
2036  else
2037  {
2038  pg_log_error("final receive failed: %s",
2039  PQerrorMessage(conn));
2040  }
2041  exit(1);
2042  }
2043 
2044  if (bgchild > 0)
2045  {
2046 #ifndef WIN32
2047  int status;
2048  pid_t r;
2049 #else
2050  DWORD status;
2051 
2052  /*
2053  * get a pointer sized version of bgchild to avoid warnings about
2054  * casting to a different size on WIN64.
2055  */
2056  intptr_t bgchild_handle = bgchild;
2057  uint32 hi,
2058  lo;
2059 #endif
2060 
2061  if (verbose)
2062  pg_log_info("waiting for background process to finish streaming ...");
2063 
2064 #ifndef WIN32
2065  if (write(bgpipe[1], xlogend, strlen(xlogend)) != strlen(xlogend))
2066  {
2067  pg_log_info("could not send command to background pipe: %m");
2068  exit(1);
2069  }
2070 
2071  /* Just wait for the background process to exit */
2072  r = waitpid(bgchild, &status, 0);
2073  if (r == (pid_t) -1)
2074  {
2075  pg_log_error("could not wait for child process: %m");
2076  exit(1);
2077  }
2078  if (r != bgchild)
2079  {
2080  pg_log_error("child %d died, expected %d", (int) r, (int) bgchild);
2081  exit(1);
2082  }
2083  if (status != 0)
2084  {
2085  pg_log_error("%s", wait_result_to_str(status));
2086  exit(1);
2087  }
2088  /* Exited normally, we're happy! */
2089 #else /* WIN32 */
2090 
2091  /*
2092  * On Windows, since we are in the same process, we can just store the
2093  * value directly in the variable, and then set the flag that says
2094  * it's there.
2095  */
2096  if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
2097  {
2098  pg_log_error("could not parse write-ahead log location \"%s\"",
2099  xlogend);
2100  exit(1);
2101  }
2102  xlogendptr = ((uint64) hi) << 32 | lo;
2103  InterlockedIncrement(&has_xlogendptr);
2104 
2105  /* First wait for the thread to exit */
2106  if (WaitForSingleObjectEx((HANDLE) bgchild_handle, INFINITE, FALSE) !=
2107  WAIT_OBJECT_0)
2108  {
2109  _dosmaperr(GetLastError());
2110  pg_log_error("could not wait for child thread: %m");
2111  exit(1);
2112  }
2113  if (GetExitCodeThread((HANDLE) bgchild_handle, &status) == 0)
2114  {
2115  _dosmaperr(GetLastError());
2116  pg_log_error("could not get child thread exit status: %m");
2117  exit(1);
2118  }
2119  if (status != 0)
2120  {
2121  pg_log_error("child thread exited with error %u",
2122  (unsigned int) status);
2123  exit(1);
2124  }
2125  /* Exited normally, we're happy */
2126 #endif
2127  }
2128 
2129  /* Free the configuration file contents */
2130  destroyPQExpBuffer(recoveryconfcontents);
2131 
2132  /*
2133  * End of copy data. Final result is already checked inside the loop.
2134  */
2135  PQclear(res);
2136  PQfinish(conn);
2137  conn = NULL;
2138 
2139  /*
2140  * Make data persistent on disk once backup is completed. For tar format
2141  * once syncing the parent directory is fine, each tar file created per
2142  * tablespace has been already synced. In plain format, all the data of
2143  * the base directory is synced, taking into account all the tablespaces.
2144  * Errors are not considered fatal.
2145  */
2146  if (do_sync)
2147  {
2148  if (verbose)
2149  pg_log_info("syncing data to disk ...");
2150  if (format == 't')
2151  {
2152  if (strcmp(basedir, "-") != 0)
2153  (void) fsync_fname(basedir, true);
2154  }
2155  else
2156  {
2157  (void) fsync_pgdata(basedir, serverVersion);
2158  }
2159  }
2160 
2161  if (verbose)
2162  pg_log_info("base backup completed");
2163 }
2164 
2165 
2166 int
2167 main(int argc, char **argv)
2168 {
2169  static struct option long_options[] = {
2170  {"help", no_argument, NULL, '?'},
2171  {"version", no_argument, NULL, 'V'},
2172  {"pgdata", required_argument, NULL, 'D'},
2173  {"format", required_argument, NULL, 'F'},
2174  {"checkpoint", required_argument, NULL, 'c'},
2175  {"create-slot", no_argument, NULL, 'C'},
2176  {"max-rate", required_argument, NULL, 'r'},
2177  {"write-recovery-conf", no_argument, NULL, 'R'},
2178  {"slot", required_argument, NULL, 'S'},
2179  {"tablespace-mapping", required_argument, NULL, 'T'},
2180  {"wal-method", required_argument, NULL, 'X'},
2181  {"gzip", no_argument, NULL, 'z'},
2182  {"compress", required_argument, NULL, 'Z'},
2183  {"label", required_argument, NULL, 'l'},
2184  {"no-clean", no_argument, NULL, 'n'},
2185  {"no-sync", no_argument, NULL, 'N'},
2186  {"dbname", required_argument, NULL, 'd'},
2187  {"host", required_argument, NULL, 'h'},
2188  {"port", required_argument, NULL, 'p'},
2189  {"username", required_argument, NULL, 'U'},
2190  {"no-password", no_argument, NULL, 'w'},
2191  {"password", no_argument, NULL, 'W'},
2192  {"status-interval", required_argument, NULL, 's'},
2193  {"verbose", no_argument, NULL, 'v'},
2194  {"progress", no_argument, NULL, 'P'},
2195  {"waldir", required_argument, NULL, 1},
2196  {"no-slot", no_argument, NULL, 2},
2197  {"no-verify-checksums", no_argument, NULL, 3},
2198  {NULL, 0, NULL, 0}
2199  };
2200  int c;
2201 
2202  int option_index;
2203 
2204  pg_logging_init(argv[0]);
2205  progname = get_progname(argv[0]);
2206  set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup"));
2207 
2208  if (argc > 1)
2209  {
2210  if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
2211  {
2212  usage();
2213  exit(0);
2214  }
2215  else if (strcmp(argv[1], "-V") == 0
2216  || strcmp(argv[1], "--version") == 0)
2217  {
2218  puts("pg_basebackup (PostgreSQL) " PG_VERSION);
2219  exit(0);
2220  }
2221  }
2222 
2224 
2225  while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP",
2226  long_options, &option_index)) != -1)
2227  {
2228  switch (c)
2229  {
2230  case 'C':
2231  create_slot = true;
2232  break;
2233  case 'D':
2235  break;
2236  case 'F':
2237  if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
2238  format = 'p';
2239  else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
2240  format = 't';
2241  else
2242  {
2243  pg_log_error("invalid output format \"%s\", must be \"plain\" or \"tar\"",
2244  optarg);
2245  exit(1);
2246  }
2247  break;
2248  case 'r':
2250  break;
2251  case 'R':
2252  writerecoveryconf = true;
2253  break;
2254  case 'S':
2255 
2256  /*
2257  * When specifying replication slot name, use a permanent
2258  * slot.
2259  */
2261  temp_replication_slot = false;
2262  break;
2263  case 2:
2264  no_slot = true;
2265  break;
2266  case 'T':
2268  break;
2269  case 'X':
2270  if (strcmp(optarg, "n") == 0 ||
2271  strcmp(optarg, "none") == 0)
2272  {
2273  includewal = NO_WAL;
2274  }
2275  else if (strcmp(optarg, "f") == 0 ||
2276  strcmp(optarg, "fetch") == 0)
2277  {
2279  }
2280  else if (strcmp(optarg, "s") == 0 ||
2281  strcmp(optarg, "stream") == 0)
2282  {
2284  }
2285  else
2286  {
2287  pg_log_error("invalid wal-method option \"%s\", must be \"fetch\", \"stream\", or \"none\"",
2288  optarg);
2289  exit(1);
2290  }
2291  break;
2292  case 1:
2294  break;
2295  case 'l':
2296  label = pg_strdup(optarg);
2297  break;
2298  case 'n':
2299  noclean = true;
2300  break;
2301  case 'N':
2302  do_sync = false;
2303  break;
2304  case 'z':
2305 #ifdef HAVE_LIBZ
2307 #else
2308  compresslevel = 1; /* will be rejected below */
2309 #endif
2310  break;
2311  case 'Z':
2312  compresslevel = atoi(optarg);
2313  if (compresslevel < 0 || compresslevel > 9)
2314  {
2315  pg_log_error("invalid compression level \"%s\"", optarg);
2316  exit(1);
2317  }
2318  break;
2319  case 'c':
2320  if (pg_strcasecmp(optarg, "fast") == 0)
2321  fastcheckpoint = true;
2322  else if (pg_strcasecmp(optarg, "spread") == 0)
2323  fastcheckpoint = false;
2324  else
2325  {
2326  pg_log_error("invalid checkpoint argument \"%s\", must be \"fast\" or \"spread\"",
2327  optarg);
2328  exit(1);
2329  }
2330  break;
2331  case 'd':
2333  break;
2334  case 'h':
2335  dbhost = pg_strdup(optarg);
2336  break;
2337  case 'p':
2338  dbport = pg_strdup(optarg);
2339  break;
2340  case 'U':
2341  dbuser = pg_strdup(optarg);
2342  break;
2343  case 'w':
2344  dbgetpassword = -1;
2345  break;
2346  case 'W':
2347  dbgetpassword = 1;
2348  break;
2349  case 's':
2350  standby_message_timeout = atoi(optarg) * 1000;
2351  if (standby_message_timeout < 0)
2352  {
2353  pg_log_error("invalid status interval \"%s\"", optarg);
2354  exit(1);
2355  }
2356  break;
2357  case 'v':
2358  verbose++;
2359  break;
2360  case 'P':
2361  showprogress = true;
2362  break;
2363  case 3:
2364  verify_checksums = false;
2365  break;
2366  default:
2367 
2368  /*
2369  * getopt_long already emitted a complaint
2370  */
2371  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2372  progname);
2373  exit(1);
2374  }
2375  }
2376 
2377  /*
2378  * Any non-option arguments?
2379  */
2380  if (optind < argc)
2381  {
2382  pg_log_error("too many command-line arguments (first is \"%s\")",
2383  argv[optind]);
2384  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2385  progname);
2386  exit(1);
2387  }
2388 
2389  /*
2390  * Required arguments
2391  */
2392  if (basedir == NULL)
2393  {
2394  pg_log_error("no target directory specified");
2395  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2396  progname);
2397  exit(1);
2398  }
2399 
2400  /*
2401  * Mutually exclusive arguments
2402  */
2403  if (format == 'p' && compresslevel != 0)
2404  {
2405  pg_log_error("only tar mode backups can be compressed");
2406  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2407  progname);
2408  exit(1);
2409  }
2410 
2411  if (format == 't' && includewal == STREAM_WAL && strcmp(basedir, "-") == 0)
2412  {
2413  pg_log_error("cannot stream write-ahead logs in tar mode to stdout");
2414  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2415  progname);
2416  exit(1);
2417  }
2418 
2420  {
2421  pg_log_error("replication slots can only be used with WAL streaming");
2422  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2423  progname);
2424  exit(1);
2425  }
2426 
2427  if (no_slot)
2428  {
2429  if (replication_slot)
2430  {
2431  pg_log_error("--no-slot cannot be used with slot name");
2432  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2433  progname);
2434  exit(1);
2435  }
2436  temp_replication_slot = false;
2437  }
2438 
2439  if (create_slot)
2440  {
2441  if (!replication_slot)
2442  {
2443  pg_log_error("%s needs a slot to be specified using --slot",
2444  "--create-slot");
2445  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2446  progname);
2447  exit(1);
2448  }
2449 
2450  if (no_slot)
2451  {
2452  pg_log_error("--create-slot and --no-slot are incompatible options");
2453  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2454  progname);
2455  exit(1);
2456  }
2457  }
2458 
2459  if (xlog_dir)
2460  {
2461  if (format != 'p')
2462  {
2463  pg_log_error("WAL directory location can only be specified in plain mode");
2464  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2465  progname);
2466  exit(1);
2467  }
2468 
2469  /* clean up xlog directory name, check it's absolute */
2471  if (!is_absolute_path(xlog_dir))
2472  {
2473  pg_log_error("WAL directory location must be an absolute path");
2474  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2475  progname);
2476  exit(1);
2477  }
2478  }
2479 
2480 #ifndef HAVE_LIBZ
2481  if (compresslevel != 0)
2482  {
2483  pg_log_error("this build does not support compression");
2484  exit(1);
2485  }
2486 #endif
2487 
2488  /* connection in replication mode to server */
2489  conn = GetConnection();
2490  if (!conn)
2491  {
2492  /* Error message already written in GetConnection() */
2493  exit(1);
2494  }
2495  atexit(disconnect_atexit);
2496 
2497  /*
2498  * Set umask so that directories/files are created with the same
2499  * permissions as directories/files in the source data directory.
2500  *
2501  * pg_mode_mask is set to owner-only by default and then updated in
2502  * GetConnection() where we get the mode from the server-side with
2503  * RetrieveDataDirCreatePerm() and then call SetDataDirectoryCreatePerm().
2504  */
2505  umask(pg_mode_mask);
2506 
2507  /*
2508  * Verify that the target directory exists, or create it. For plaintext
2509  * backups, always require the directory. For tar backups, require it
2510  * unless we are writing to stdout.
2511  */
2512  if (format == 'p' || strcmp(basedir, "-") != 0)
2514 
2515  /* determine remote server's xlog segment size */
2516  if (!RetrieveWalSegSize(conn))
2517  exit(1);
2518 
2519  /* Create pg_wal symlink, if required */
2520  if (xlog_dir)
2521  {
2522  char *linkloc;
2523 
2525 
2526  /*
2527  * Form name of the place where the symlink must go. pg_xlog has been
2528  * renamed to pg_wal in post-10 clusters.
2529  */
2530  linkloc = psprintf("%s/%s", basedir,
2532  "pg_xlog" : "pg_wal");
2533 
2534 #ifdef HAVE_SYMLINK
2535  if (symlink(xlog_dir, linkloc) != 0)
2536  {
2537  pg_log_error("could not create symbolic link \"%s\": %m", linkloc);
2538  exit(1);
2539  }
2540 #else
2541  pg_log_error("symlinks are not supported on this platform");
2542  exit(1);
2543 #endif
2544  free(linkloc);
2545  }
2546 
2547  BaseBackup();
2548 
2549  success = true;
2550  return 0;
2551 }
int PQnfields(const PGresult *res)
Definition: fe-exec.c:2778
static PQExpBuffer recoveryconfcontents
static void writeTarData(FILE *tarfile, char *buf, int r, char *current_file)
static IncludeWal includewal
Definition: pg_basebackup.c:96
char * PQerrorMessage(const PGconn *conn)
Definition: fe-connect.c:6578
#define Z_DEFAULT_COMPRESSION
int pg_file_create_mode
Definition: file_perm.c:19
static bool verify_checksums
static int bgpipe[2]
static bool found_existing_pgdata
uint32 TimeLineID
Definition: xlogdefs.h:52
int64 pg_time_t
Definition: pgtime.h:23
static void usage(void)
static bool found_tablespace_dirs
char * PQgetvalue(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3164
const char * PQparameterStatus(const PGconn *conn, const char *paramName)
Definition: fe-connect.c:6543
bool pg_str_endswith(const char *str, const char *end)
Definition: string.c:31
int pg_mkdir_p(char *path, int omode)
Definition: pgmkdirp.c:57
static TablespaceList tablespace_dirs
Definition: pg_basebackup.c:87
#define write(a, b, c)
Definition: win32.h:14
bool do_sync
Definition: receivelog.h:39
const char * get_progname(const char *argv0)
Definition: path.c:453
void termPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:131
uint64 read_tar_number(const char *s, int len)
Definition: tar.c:56
#define pg_log_error(...)
Definition: logging.h:79
int getopt_long(int argc, char *const argv[], const char *optstring, const struct option *longopts, int *longindex)
Definition: getopt_long.c:57
static int32 parse_max_rate(char *src)
void appendPQExpBufferStr(PQExpBuffer str, const char *data)
Definition: pqexpbuffer.c:369
#define FALSE
Definition: ecpglib.h:39
void pg_logging_init(const char *argv0)
Definition: logging.c:39
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
static pg_time_t last_progress_report
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:582
char old_dir[MAXPGPATH]
Definition: pg_basebackup.c:49
static int tablespacecount
void _dosmaperr(unsigned long)
Definition: win32error.c:171
bool RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli, XLogRecPtr *startpos, char **db_name)
Definition: streamutil.c:409
#define VERBOSE_FILENAME_LENGTH
IncludeWal
Definition: pg_basebackup.c:78
void appendConnStrVal(PQExpBuffer buf, const char *str)
Definition: string_utils.c:547
void canonicalize_path(char *path)
Definition: path.c:254
char * sysidentifier
Definition: receivelog.h:34
static bool writerecoveryconf
Definition: pg_basebackup.c:98
static bool create_slot
void PQfinish(PGconn *conn)
Definition: fe-connect.c:4080
static int compresslevel
Definition: pg_basebackup.c:95
static bool checksum_failure
Definition: pg_basebackup.c:92
#define MemSet(start, val, len)
Definition: c.h:955
#define kill(pid, sig)
Definition: win32_port.h:435
#define printf(...)
Definition: port.h:198
XLogRecPtr startpos
Definition: receivelog.h:32
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
static char * escape_quotes(const char *src)
char * partial_suffix
Definition: receivelog.h:48
char new_dir[MAXPGPATH]
Definition: pg_basebackup.c:50
int PQserverVersion(const PGconn *conn)
Definition: fe-connect.c:6568
const char * progname
Definition: pg_standby.c:37
int PQntuples(const PGresult *res)
Definition: fe-exec.c:2770
#define PG_DIAG_SQLSTATE
Definition: postgres_ext.h:57
#define fprintf
Definition: port.h:196
static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline, bool segment_finished)
char * wait_result_to_str(int exitstatus)
Definition: wait_error.c:32
static void GenerateRecoveryConf(PGconn *conn)
TimeLineID timeline
Definition: receivelog.h:33
static char * basedir
Definition: pg_basebackup.c:86
struct TablespaceListCell TablespaceListCell
WalWriteMethod * CreateWalDirectoryMethod(const char *basedir, int compression, bool sync)
Definition: walmethods.c:350
ExecStatusType PQresultStatus(const PGresult *res)
Definition: fe-exec.c:2693
int PQgetCopyData(PGconn *conn, char **buffer, int async)
Definition: fe-exec.c:2474
signed int int32
Definition: c.h:346
int main(int argc, char **argv)
static void BaseBackup(void)
void fsync_pgdata(const char *pg_data, int serverVersion)
Definition: file_utils.c:58
#define pgoff_t
Definition: win32_port.h:204
int PQsendQuery(PGconn *conn, const char *query)
Definition: fe-exec.c:1235
void destroyPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:116
char * escape_single_quotes_ascii(const char *src)
Definition: quotes.c:33
#define required_argument
Definition: getopt_long.h:25
int optind
Definition: getopt.c:50
#define MAX_RATE_UPPER
Definition: basebackup.h:21
char * connection_string
Definition: streamutil.c:48
struct TablespaceList TablespaceList
bool RetrieveWalSegSize(PGconn *conn)
Definition: streamutil.c:279
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
PGconn * conn
Definition: streamutil.c:56
#define MAXPGPATH
void appendPQExpBuffer(PQExpBuffer str, const char *fmt,...)
Definition: pqexpbuffer.c:267
#define MINIMUM_VERSION_FOR_PG_WAL
Definition: pg_basebackup.c:63
static bool found_existing_xlogdir
static uint64 totaldone
char * replication_slot
Definition: receivelog.h:49
bool mark_done
Definition: receivelog.h:38
static bool success
static int32 maxrate
char * c
static char * buf
Definition: pg_test_fsync.c:68
#define symlink(oldpath, newpath)
Definition: win32_port.h:230
static int has_xlogendptr
static bool made_new_pgdata
void PQconninfoFree(PQconninfoOption *connOptions)
Definition: fe-connect.c:6410
#define is_absolute_path(filename)
Definition: port.h:86
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
size_t PQescapeStringConn(PGconn *conn, char *to, const char *from, size_t length, int *error)
Definition: fe-exec.c:3411
#define select(n, r, w, e, timeout)
Definition: win32_port.h:445
unsigned int uint32
Definition: c.h:358
PQconninfoOption * PQconninfo(PGconn *conn)
Definition: fe-connect.c:6369
static uint64 totalsize
static const char * get_tablespace_mapping(const char *dir)
static void progress_report(int tablespacenum, const char *filename, bool force)
XLogRecPtr startptr
stream_stop_callback stream_stop
Definition: receivelog.h:42
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:44
WalWriteMethod * walmethod
Definition: receivelog.h:47
static int verbose
Definition: pg_basebackup.c:94
#define unconstify(underlying_type, expr)
Definition: c.h:1163
bool rmtree(const char *path, bool rmtopdir)
Definition: rmtree.c:42
static int standby_message_timeout
int pg_dir_create_mode
Definition: file_perm.c:18
static bool showprogress
Definition: pg_basebackup.c:93
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:112
int dbgetpassword
Definition: streamutil.c:53
#define no_argument
Definition: getopt_long.h:24
void FreeWalTarMethod(void)
Definition: walmethods.c:1014
#define ngettext(s, p, n)
Definition: c.h:1103
#define PG_TEXTDOMAIN(domain)
Definition: c.h:1135
static void WriteRecoveryConf(void)
static bool temp_replication_slot
int PQbackendPID(const PGconn *conn)
Definition: fe-connect.c:6604
#define PGINVALID_SOCKET
Definition: port.h:33
static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
struct TablespaceListCell * next
Definition: pg_basebackup.c:48
PQExpBuffer createPQExpBuffer(void)
Definition: pqexpbuffer.c:74
static char * label
Definition: pg_basebackup.c:90
#define XLogSegmentOffset(xlogptr, wal_segsz_bytes)
void appendPQExpBufferChar(PQExpBuffer str, char ch)
Definition: pqexpbuffer.c:380
StringInfo copybuf
Definition: tablesync.c:115
static void disconnect_atexit(void)
static void StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
TablespaceListCell * tail
Definition: pg_basebackup.c:56
char * dbport
Definition: streamutil.c:51
static void cleanup_directories_atexit(void)
void PQclear(PGresult *res)
Definition: fe-exec.c:695
#define free(a)
Definition: header.h:65
static bool in_log_streamer
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
PGconn * GetConnection(UserMapping *user, bool will_prep_stmt)
Definition: connection.c:107
#define PQExpBufferDataBroken(buf)
Definition: pqexpbuffer.h:67
char * PQresultErrorField(const PGresult *res, int fieldcode)
Definition: fe-exec.c:2755
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:732
WalWriteMethod * CreateWalTarMethod(const char *tarbase, int compression, bool sync)
Definition: walmethods.c:983
static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
char * dbhost
Definition: streamutil.c:49
bool ReceiveXlogStream(PGconn *conn, StreamCtl *stream)
Definition: receivelog.c:439
static void kill_bgchild_atexit(void)
#define strerror
Definition: port.h:205
static char * xlog_dir
Definition: pg_basebackup.c:88
#define PQExpBufferBroken(str)
Definition: pqexpbuffer.h:59
static XLogRecPtr xlogendptr
static XLogRecPtr startpos
static bool no_slot
TablespaceListCell * head
Definition: pg_basebackup.c:55
bool synchronous
Definition: receivelog.h:37
void pg_free(void *ptr)
Definition: fe_memutils.c:105
bool(* finish)(void)
Definition: walmethods.h:75
pgsocket stop_socket
Definition: receivelog.h:44
static void header(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:210
#define INT64_FORMAT
Definition: c.h:400
static bool made_new_xlogdir
static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
Definition: walsender.c:852
#define WRITE_TAR_DATA(buf, sz)
int WalSegSz
Definition: pg_standby.c:39
static bool fastcheckpoint
Definition: pg_basebackup.c:97
static void tablespace_list_append(const char *arg)
static bool made_tablespace_dirs
static char * filename
Definition: pg_dumpall.c:91
void set_pglocale_pgservice(const char *argv0, const char *app)
Definition: exec.c:565
int errmsg(const char *fmt,...)
Definition: elog.c:784
static pid_t bgchild
char * dbuser
Definition: streamutil.c:50
char * optarg
Definition: getopt.c:52
int pg_check_dir(const char *dir)
Definition: pgcheckdir.c:31
int i
int standby_message_timeout
Definition: receivelog.h:36
void * arg
static bool do_sync
Definition: pg_basebackup.c:99
static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
static char format
Definition: pg_basebackup.c:89
#define mkdir(a, b)
Definition: win32_port.h:58
static char * replication_slot
static void static void status(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:227
static int LogStreamerMain(logstreamer_param *param)
int PQgetisnull(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3189
#define snprintf
Definition: port.h:192
#define _(x)
Definition: elog.c:84
void PQfreemem(void *ptr)
Definition: fe-exec.c:3297
#define MINIMUM_VERSION_FOR_RECOVERY_GUC
Definition: pg_basebackup.c:73
int pg_mode_mask
Definition: file_perm.c:25
Datum now(PG_FUNCTION_ARGS)
Definition: timestamp.c:1533
PGresult * PQgetResult(PGconn *conn)
Definition: fe-exec.c:1779
#define MINIMUM_VERSION_FOR_TEMP_SLOTS
Definition: pg_basebackup.c:68
#define read(a, b, c)
Definition: win32.h:13
void initPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:92
#define pg_log_info(...)
Definition: logging.h:87
void FreeWalDirectoryMethod(void)
Definition: walmethods.c:374
bool CheckServerVersionForStreaming(PGconn *conn)
Definition: receivelog.c:361
char xlog[MAXPGPATH]
static bool noclean
Definition: pg_basebackup.c:91