PostgreSQL Source Code  git master
pg_basebackup.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_basebackup.c - receive a base backup using streaming replication protocol
4  *
5  * Author: Magnus Hagander <magnus@hagander.net>
6  *
7  * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group
8  *
9  * IDENTIFICATION
10  * src/bin/pg_basebackup/pg_basebackup.c
11  *-------------------------------------------------------------------------
12  */
13 
14 #include "postgres_fe.h"
15 
16 #include <unistd.h>
17 #include <dirent.h>
18 #include <sys/stat.h>
19 #include <sys/wait.h>
20 #include <signal.h>
21 #include <time.h>
22 #ifdef HAVE_SYS_SELECT_H
23 #include <sys/select.h>
24 #endif
25 #ifdef HAVE_LIBZ
26 #include <zlib.h>
27 #endif
28 
29 #include "access/xlog_internal.h"
30 #include "common/file_perm.h"
31 #include "common/file_utils.h"
32 #include "common/string.h"
33 #include "fe_utils/string_utils.h"
34 #include "getopt_long.h"
35 #include "libpq-fe.h"
36 #include "pqexpbuffer.h"
37 #include "pgtar.h"
38 #include "pgtime.h"
39 #include "receivelog.h"
40 #include "replication/basebackup.h"
41 #include "streamutil.h"
42 
43 #define ERRCODE_DATA_CORRUPTED "XX001"
44 
45 typedef struct TablespaceListCell
46 {
51 
52 typedef struct TablespaceList
53 {
57 
58 /*
59  * pg_xlog has been renamed to pg_wal in version 10. This version number
60  * should be compared with PQserverVersion().
61  */
62 #define MINIMUM_VERSION_FOR_PG_WAL 100000
63 
64 /*
65  * Temporary replication slots are supported from version 10.
66  */
67 #define MINIMUM_VERSION_FOR_TEMP_SLOTS 100000
68 
69 /*
70  * Different ways to include WAL
71  */
72 typedef enum
73 {
77 } IncludeWal;
78 
79 /* Global options */
80 static char *basedir = NULL;
81 static TablespaceList tablespace_dirs = {NULL, NULL};
82 static char *xlog_dir = NULL;
83 static char format = 'p'; /* p(lain)/t(ar) */
84 static char *label = "pg_basebackup base backup";
85 static bool noclean = false;
86 static bool checksum_failure = false;
87 static bool showprogress = false;
88 static int verbose = 0;
89 static int compresslevel = 0;
91 static bool fastcheckpoint = false;
92 static bool writerecoveryconf = false;
93 static bool do_sync = true;
94 static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
96 static int32 maxrate = 0; /* no limit by default */
97 static char *replication_slot = NULL;
98 static bool temp_replication_slot = true;
99 static bool create_slot = false;
100 static bool no_slot = false;
101 static bool verify_checksums = true;
102 
103 static bool success = false;
104 static bool made_new_pgdata = false;
105 static bool found_existing_pgdata = false;
106 static bool made_new_xlogdir = false;
107 static bool found_existing_xlogdir = false;
108 static bool made_tablespace_dirs = false;
109 static bool found_tablespace_dirs = false;
110 
111 /* Progress counters */
112 static uint64 totalsize;
113 static uint64 totaldone;
114 static int tablespacecount;
115 
116 /* Pipe to communicate with background wal receiver process */
117 #ifndef WIN32
118 static int bgpipe[2] = {-1, -1};
119 #endif
120 
121 /* Handle to child process */
122 static pid_t bgchild = -1;
123 static bool in_log_streamer = false;
124 
125 /* End position for xlog streaming, empty string if unknown yet */
127 
128 #ifndef WIN32
129 static int has_xlogendptr = 0;
130 #else
131 static volatile LONG has_xlogendptr = 0;
132 #endif
133 
134 /* Contents of recovery.conf to be generated */
136 
137 /* Function headers */
138 static void usage(void);
139 static void disconnect_and_exit(int code) pg_attribute_noreturn();
140 static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found);
141 static void progress_report(int tablespacenum, const char *filename, bool force);
142 
143 static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum);
144 static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum);
145 static void GenerateRecoveryConf(PGconn *conn);
146 static void WriteRecoveryConf(void);
147 static void BaseBackup(void);
148 
149 static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline,
150  bool segment_finished);
151 
152 static const char *get_tablespace_mapping(const char *dir);
153 static void tablespace_list_append(const char *arg);
154 
155 
156 static void
158 {
159  if (success || in_log_streamer)
160  return;
161 
162  if (!noclean && !checksum_failure)
163  {
164  if (made_new_pgdata)
165  {
166  fprintf(stderr, _("%s: removing data directory \"%s\"\n"),
167  progname, basedir);
168  if (!rmtree(basedir, true))
169  fprintf(stderr, _("%s: failed to remove data directory\n"),
170  progname);
171  }
172  else if (found_existing_pgdata)
173  {
174  fprintf(stderr,
175  _("%s: removing contents of data directory \"%s\"\n"),
176  progname, basedir);
177  if (!rmtree(basedir, false))
178  fprintf(stderr, _("%s: failed to remove contents of data directory\n"),
179  progname);
180  }
181 
182  if (made_new_xlogdir)
183  {
184  fprintf(stderr, _("%s: removing WAL directory \"%s\"\n"),
185  progname, xlog_dir);
186  if (!rmtree(xlog_dir, true))
187  fprintf(stderr, _("%s: failed to remove WAL directory\n"),
188  progname);
189  }
190  else if (found_existing_xlogdir)
191  {
192  fprintf(stderr,
193  _("%s: removing contents of WAL directory \"%s\"\n"),
194  progname, xlog_dir);
195  if (!rmtree(xlog_dir, false))
196  fprintf(stderr, _("%s: failed to remove contents of WAL directory\n"),
197  progname);
198  }
199  }
200  else
201  {
203  fprintf(stderr,
204  _("%s: data directory \"%s\" not removed at user's request\n"),
205  progname, basedir);
206 
208  fprintf(stderr,
209  _("%s: WAL directory \"%s\" not removed at user's request\n"),
210  progname, xlog_dir);
211  }
212 
214  fprintf(stderr,
215  _("%s: changes to tablespace directories will not be undone\n"),
216  progname);
217 }
218 
219 static void
221 {
222  if (conn != NULL)
223  PQfinish(conn);
224 
225 #ifndef WIN32
226 
227  /*
228  * On windows, our background thread dies along with the process. But on
229  * Unix, if we have started a subprocess, we want to kill it off so it
230  * doesn't remain running trying to stream data.
231  */
232  if (bgchild > 0)
233  kill(bgchild, SIGTERM);
234 #endif
235 
236  exit(code);
237 }
238 
239 
240 /*
241  * Split argument into old_dir and new_dir and append to tablespace mapping
242  * list.
243  */
244 static void
246 {
248  char *dst;
249  char *dst_ptr;
250  const char *arg_ptr;
251 
252  dst_ptr = dst = cell->old_dir;
253  for (arg_ptr = arg; *arg_ptr; arg_ptr++)
254  {
255  if (dst_ptr - dst >= MAXPGPATH)
256  {
257  fprintf(stderr, _("%s: directory name too long\n"), progname);
258  exit(1);
259  }
260 
261  if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
262  ; /* skip backslash escaping = */
263  else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
264  {
265  if (*cell->new_dir)
266  {
267  fprintf(stderr, _("%s: multiple \"=\" signs in tablespace mapping\n"), progname);
268  exit(1);
269  }
270  else
271  dst = dst_ptr = cell->new_dir;
272  }
273  else
274  *dst_ptr++ = *arg_ptr;
275  }
276 
277  if (!*cell->old_dir || !*cell->new_dir)
278  {
279  fprintf(stderr,
280  _("%s: invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"\n"),
281  progname, arg);
282  exit(1);
283  }
284 
285  /*
286  * This check isn't absolutely necessary. But all tablespaces are created
287  * with absolute directories, so specifying a non-absolute path here would
288  * just never match, possibly confusing users. It's also good to be
289  * consistent with the new_dir check.
290  */
291  if (!is_absolute_path(cell->old_dir))
292  {
293  fprintf(stderr, _("%s: old directory is not an absolute path in tablespace mapping: %s\n"),
294  progname, cell->old_dir);
295  exit(1);
296  }
297 
298  if (!is_absolute_path(cell->new_dir))
299  {
300  fprintf(stderr, _("%s: new directory is not an absolute path in tablespace mapping: %s\n"),
301  progname, cell->new_dir);
302  exit(1);
303  }
304 
305  /*
306  * Comparisons done with these values should involve similarly
307  * canonicalized path values. This is particularly sensitive on Windows
308  * where path values may not necessarily use Unix slashes.
309  */
310  canonicalize_path(cell->old_dir);
311  canonicalize_path(cell->new_dir);
312 
313  if (tablespace_dirs.tail)
314  tablespace_dirs.tail->next = cell;
315  else
316  tablespace_dirs.head = cell;
317  tablespace_dirs.tail = cell;
318 }
319 
320 
321 #ifdef HAVE_LIBZ
322 static const char *
323 get_gz_error(gzFile gzf)
324 {
325  int errnum;
326  const char *errmsg;
327 
328  errmsg = gzerror(gzf, &errnum);
329  if (errnum == Z_ERRNO)
330  return strerror(errno);
331  else
332  return errmsg;
333 }
334 #endif
335 
336 static void
337 usage(void)
338 {
339  printf(_("%s takes a base backup of a running PostgreSQL server.\n\n"),
340  progname);
341  printf(_("Usage:\n"));
342  printf(_(" %s [OPTION]...\n"), progname);
343  printf(_("\nOptions controlling the output:\n"));
344  printf(_(" -D, --pgdata=DIRECTORY receive base backup into directory\n"));
345  printf(_(" -F, --format=p|t output format (plain (default), tar)\n"));
346  printf(_(" -r, --max-rate=RATE maximum transfer rate to transfer data directory\n"
347  " (in kB/s, or use suffix \"k\" or \"M\")\n"));
348  printf(_(" -R, --write-recovery-conf\n"
349  " write recovery.conf for replication\n"));
350  printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
351  " relocate tablespace in OLDDIR to NEWDIR\n"));
352  printf(_(" --waldir=WALDIR location for the write-ahead log directory\n"));
353  printf(_(" -X, --wal-method=none|fetch|stream\n"
354  " include required WAL files with specified method\n"));
355  printf(_(" -z, --gzip compress tar output\n"));
356  printf(_(" -Z, --compress=0-9 compress tar output with given compression level\n"));
357  printf(_("\nGeneral options:\n"));
358  printf(_(" -c, --checkpoint=fast|spread\n"
359  " set fast or spread checkpointing\n"));
360  printf(_(" -C, --create-slot create replication slot\n"));
361  printf(_(" -l, --label=LABEL set backup label\n"));
362  printf(_(" -n, --no-clean do not clean up after errors\n"));
363  printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
364  printf(_(" -P, --progress show progress information\n"));
365  printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
366  printf(_(" --no-slot prevent creation of temporary replication slot\n"));
367  printf(_(" -k, --no-verify-checksums\n"
368  " do not verify checksums\n"));
369  printf(_(" -v, --verbose output verbose messages\n"));
370  printf(_(" -V, --version output version information, then exit\n"));
371  printf(_(" -?, --help show this help, then exit\n"));
372  printf(_("\nConnection options:\n"));
373  printf(_(" -d, --dbname=CONNSTR connection string\n"));
374  printf(_(" -h, --host=HOSTNAME database server host or socket directory\n"));
375  printf(_(" -p, --port=PORT database server port number\n"));
376  printf(_(" -s, --status-interval=INTERVAL\n"
377  " time between status packets sent to server (in seconds)\n"));
378  printf(_(" -U, --username=NAME connect as specified database user\n"));
379  printf(_(" -w, --no-password never prompt for password\n"));
380  printf(_(" -W, --password force password prompt (should happen automatically)\n"));
381  printf(_("\nReport bugs to <pgsql-bugs@postgresql.org>.\n"));
382 }
383 
384 
385 /*
386  * Called in the background process every time data is received.
387  * On Unix, we check to see if there is any data on our pipe
388  * (which would mean we have a stop position), and if it is, check if
389  * it is time to stop.
390  * On Windows, we are in a single process, so we can just check if it's
391  * time to stop.
392  */
393 static bool
395  bool segment_finished)
396 {
397  if (!has_xlogendptr)
398  {
399 #ifndef WIN32
400  fd_set fds;
401  struct timeval tv;
402  int r;
403 
404  /*
405  * Don't have the end pointer yet - check our pipe to see if it has
406  * been sent yet.
407  */
408  FD_ZERO(&fds);
409  FD_SET(bgpipe[0], &fds);
410 
411  MemSet(&tv, 0, sizeof(tv));
412 
413  r = select(bgpipe[0] + 1, &fds, NULL, NULL, &tv);
414  if (r == 1)
415  {
416  char xlogend[64];
417  uint32 hi,
418  lo;
419 
420  MemSet(xlogend, 0, sizeof(xlogend));
421  r = read(bgpipe[0], xlogend, sizeof(xlogend) - 1);
422  if (r < 0)
423  {
424  fprintf(stderr, _("%s: could not read from ready pipe: %s\n"),
425  progname, strerror(errno));
426  exit(1);
427  }
428 
429  if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
430  {
431  fprintf(stderr,
432  _("%s: could not parse write-ahead log location \"%s\"\n"),
433  progname, xlogend);
434  exit(1);
435  }
436  xlogendptr = ((uint64) hi) << 32 | lo;
437  has_xlogendptr = 1;
438 
439  /*
440  * Fall through to check if we've reached the point further
441  * already.
442  */
443  }
444  else
445  {
446  /*
447  * No data received on the pipe means we don't know the end
448  * position yet - so just say it's not time to stop yet.
449  */
450  return false;
451  }
452 #else
453 
454  /*
455  * On win32, has_xlogendptr is set by the main thread, so if it's not
456  * set here, we just go back and wait until it shows up.
457  */
458  return false;
459 #endif
460  }
461 
462  /*
463  * At this point we have an end pointer, so compare it to the current
464  * position to figure out if it's time to stop.
465  */
466  if (segendpos >= xlogendptr)
467  return true;
468 
469  /*
470  * Have end pointer, but haven't reached it yet - so tell the caller to
471  * keep streaming.
472  */
473  return false;
474 }
475 
476 typedef struct
477 {
480  char xlog[MAXPGPATH]; /* directory or tarfile depending on mode */
482  int timeline;
484 
485 static int
487 {
488  StreamCtl stream;
489 
490  in_log_streamer = true;
491 
492  MemSet(&stream, 0, sizeof(stream));
493  stream.startpos = param->startptr;
494  stream.timeline = param->timeline;
495  stream.sysidentifier = param->sysidentifier;
497 #ifndef WIN32
498  stream.stop_socket = bgpipe[0];
499 #else
500  stream.stop_socket = PGINVALID_SOCKET;
501 #endif
503  stream.synchronous = false;
504  stream.do_sync = do_sync;
505  stream.mark_done = true;
506  stream.partial_suffix = NULL;
508 
509  if (format == 'p')
510  stream.walmethod = CreateWalDirectoryMethod(param->xlog, 0, do_sync);
511  else
513 
514  if (!ReceiveXlogStream(param->bgconn, &stream))
515 
516  /*
517  * Any errors will already have been reported in the function process,
518  * but we need to tell the parent that we didn't shutdown in a nice
519  * way.
520  */
521  return 1;
522 
523  if (!stream.walmethod->finish())
524  {
525  fprintf(stderr,
526  _("%s: could not finish writing WAL files: %s\n"),
527  progname, strerror(errno));
528  return 1;
529  }
530 
531  PQfinish(param->bgconn);
532 
533  if (format == 'p')
535  else
537  pg_free(stream.walmethod);
538 
539  return 0;
540 }
541 
542 /*
543  * Initiate background process for receiving xlog during the backup.
544  * The background stream will use its own database connection so we can
545  * stream the logfile in parallel with the backups.
546  */
547 static void
548 StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
549 {
550  logstreamer_param *param;
551  uint32 hi,
552  lo;
553  char statusdir[MAXPGPATH];
554 
555  param = pg_malloc0(sizeof(logstreamer_param));
556  param->timeline = timeline;
557  param->sysidentifier = sysidentifier;
558 
559  /* Convert the starting position */
560  if (sscanf(startpos, "%X/%X", &hi, &lo) != 2)
561  {
562  fprintf(stderr,
563  _("%s: could not parse write-ahead log location \"%s\"\n"),
564  progname, startpos);
566  }
567  param->startptr = ((uint64) hi) << 32 | lo;
568  /* Round off to even segment position */
569  param->startptr -= XLogSegmentOffset(param->startptr, WalSegSz);
570 
571 #ifndef WIN32
572  /* Create our background pipe */
573  if (pipe(bgpipe) < 0)
574  {
575  fprintf(stderr,
576  _("%s: could not create pipe for background process: %s\n"),
577  progname, strerror(errno));
579  }
580 #endif
581 
582  /* Get a second connection */
583  param->bgconn = GetConnection();
584  if (!param->bgconn)
585  /* Error message already written in GetConnection() */
586  exit(1);
587 
588  /* In post-10 cluster, pg_xlog has been renamed to pg_wal */
589  snprintf(param->xlog, sizeof(param->xlog), "%s/%s",
590  basedir,
592  "pg_xlog" : "pg_wal");
593 
594  /* Temporary replication slots are only supported in 10 and newer */
596  temp_replication_slot = false;
597 
598  /*
599  * Create replication slot if requested
600  */
602  replication_slot = psprintf("pg_basebackup_%d", (int) PQbackendPID(param->bgconn));
604  {
605  if (!CreateReplicationSlot(param->bgconn, replication_slot, NULL,
606  temp_replication_slot, true, true, false))
608 
609  if (verbose)
610  {
612  fprintf(stderr, _("%s: created temporary replication slot \"%s\"\n"),
614  else
615  fprintf(stderr, _("%s: created replication slot \"%s\"\n"),
617  }
618  }
619 
620  if (format == 'p')
621  {
622  /*
623  * Create pg_wal/archive_status or pg_xlog/archive_status (and thus
624  * pg_wal or pg_xlog) depending on the target server so we can write
625  * to basedir/pg_wal or basedir/pg_xlog as the directory entry in the
626  * tar file may arrive later.
627  */
628  snprintf(statusdir, sizeof(statusdir), "%s/%s/archive_status",
629  basedir,
631  "pg_xlog" : "pg_wal");
632 
633  if (pg_mkdir_p(statusdir, pg_dir_create_mode) != 0 && errno != EEXIST)
634  {
635  fprintf(stderr,
636  _("%s: could not create directory \"%s\": %s\n"),
637  progname, statusdir, strerror(errno));
639  }
640  }
641 
642  /*
643  * Start a child process and tell it to start streaming. On Unix, this is
644  * a fork(). On Windows, we create a thread.
645  */
646 #ifndef WIN32
647  bgchild = fork();
648  if (bgchild == 0)
649  {
650  /* in child process */
651  exit(LogStreamerMain(param));
652  }
653  else if (bgchild < 0)
654  {
655  fprintf(stderr, _("%s: could not create background process: %s\n"),
656  progname, strerror(errno));
658  }
659 
660  /*
661  * Else we are in the parent process and all is well.
662  */
663 #else /* WIN32 */
664  bgchild = _beginthreadex(NULL, 0, (void *) LogStreamerMain, param, 0, NULL);
665  if (bgchild == 0)
666  {
667  fprintf(stderr, _("%s: could not create background thread: %s\n"),
668  progname, strerror(errno));
670  }
671 #endif
672 }
673 
674 /*
675  * Verify that the given directory exists and is empty. If it does not
676  * exist, it is created. If it exists but is not empty, an error will
677  * be given and the process ended.
678  */
679 static void
680 verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
681 {
682  switch (pg_check_dir(dirname))
683  {
684  case 0:
685 
686  /*
687  * Does not exist, so create
688  */
689  if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
690  {
691  fprintf(stderr,
692  _("%s: could not create directory \"%s\": %s\n"),
693  progname, dirname, strerror(errno));
695  }
696  if (created)
697  *created = true;
698  return;
699  case 1:
700 
701  /*
702  * Exists, empty
703  */
704  if (found)
705  *found = true;
706  return;
707  case 2:
708  case 3:
709  case 4:
710 
711  /*
712  * Exists, not empty
713  */
714  fprintf(stderr,
715  _("%s: directory \"%s\" exists but is not empty\n"),
716  progname, dirname);
718  case -1:
719 
720  /*
721  * Access problem
722  */
723  fprintf(stderr, _("%s: could not access directory \"%s\": %s\n"),
724  progname, dirname, strerror(errno));
726  }
727 }
728 
729 
730 /*
731  * Print a progress report based on the global variables. If verbose output
732  * is enabled, also print the current file name.
733  *
734  * Progress report is written at maximum once per second, unless the
735  * force parameter is set to true.
736  */
737 static void
738 progress_report(int tablespacenum, const char *filename, bool force)
739 {
740  int percent;
741  char totaldone_str[32];
742  char totalsize_str[32];
743  pg_time_t now;
744 
745  if (!showprogress)
746  return;
747 
748  now = time(NULL);
749  if (now == last_progress_report && !force)
750  return; /* Max once per second */
751 
753  percent = totalsize ? (int) ((totaldone / 1024) * 100 / totalsize) : 0;
754 
755  /*
756  * Avoid overflowing past 100% or the full size. This may make the total
757  * size number change as we approach the end of the backup (the estimate
758  * will always be wrong if WAL is included), but that's better than having
759  * the done column be bigger than the total.
760  */
761  if (percent > 100)
762  percent = 100;
763  if (totaldone / 1024 > totalsize)
764  totalsize = totaldone / 1024;
765 
766  /*
767  * Separate step to keep platform-dependent format code out of
768  * translatable strings. And we only test for INT64_FORMAT availability
769  * in snprintf, not fprintf.
770  */
771  snprintf(totaldone_str, sizeof(totaldone_str), INT64_FORMAT,
772  totaldone / 1024);
773  snprintf(totalsize_str, sizeof(totalsize_str), INT64_FORMAT, totalsize);
774 
775 #define VERBOSE_FILENAME_LENGTH 35
776  if (verbose)
777  {
778  if (!filename)
779 
780  /*
781  * No filename given, so clear the status line (used for last
782  * call)
783  */
784  fprintf(stderr,
785  ngettext("%*s/%s kB (100%%), %d/%d tablespace %*s",
786  "%*s/%s kB (100%%), %d/%d tablespaces %*s",
788  (int) strlen(totalsize_str),
789  totaldone_str, totalsize_str,
790  tablespacenum, tablespacecount,
791  VERBOSE_FILENAME_LENGTH + 5, "");
792  else
793  {
794  bool truncate = (strlen(filename) > VERBOSE_FILENAME_LENGTH);
795 
796  fprintf(stderr,
797  ngettext("%*s/%s kB (%d%%), %d/%d tablespace (%s%-*.*s)",
798  "%*s/%s kB (%d%%), %d/%d tablespaces (%s%-*.*s)",
800  (int) strlen(totalsize_str),
801  totaldone_str, totalsize_str, percent,
802  tablespacenum, tablespacecount,
803  /* Prefix with "..." if we do leading truncation */
804  truncate ? "..." : "",
807  /* Truncate filename at beginning if it's too long */
808  truncate ? filename + strlen(filename) - VERBOSE_FILENAME_LENGTH + 3 : filename);
809  }
810  }
811  else
812  fprintf(stderr,
813  ngettext("%*s/%s kB (%d%%), %d/%d tablespace",
814  "%*s/%s kB (%d%%), %d/%d tablespaces",
816  (int) strlen(totalsize_str),
817  totaldone_str, totalsize_str, percent,
818  tablespacenum, tablespacecount);
819 
820  if (isatty(fileno(stderr)))
821  fprintf(stderr, "\r");
822  else
823  fprintf(stderr, "\n");
824 }
825 
826 static int32
827 parse_max_rate(char *src)
828 {
829  double result;
830  char *after_num;
831  char *suffix = NULL;
832 
833  errno = 0;
834  result = strtod(src, &after_num);
835  if (src == after_num)
836  {
837  fprintf(stderr,
838  _("%s: transfer rate \"%s\" is not a valid value\n"),
839  progname, src);
840  exit(1);
841  }
842  if (errno != 0)
843  {
844  fprintf(stderr,
845  _("%s: invalid transfer rate \"%s\": %s\n"),
846  progname, src, strerror(errno));
847  exit(1);
848  }
849 
850  if (result <= 0)
851  {
852  /*
853  * Reject obviously wrong values here.
854  */
855  fprintf(stderr, _("%s: transfer rate must be greater than zero\n"),
856  progname);
857  exit(1);
858  }
859 
860  /*
861  * Evaluate suffix, after skipping over possible whitespace. Lack of
862  * suffix means kilobytes.
863  */
864  while (*after_num != '\0' && isspace((unsigned char) *after_num))
865  after_num++;
866 
867  if (*after_num != '\0')
868  {
869  suffix = after_num;
870  if (*after_num == 'k')
871  {
872  /* kilobyte is the expected unit. */
873  after_num++;
874  }
875  else if (*after_num == 'M')
876  {
877  after_num++;
878  result *= 1024.0;
879  }
880  }
881 
882  /* The rest can only consist of white space. */
883  while (*after_num != '\0' && isspace((unsigned char) *after_num))
884  after_num++;
885 
886  if (*after_num != '\0')
887  {
888  fprintf(stderr,
889  _("%s: invalid --max-rate unit: \"%s\"\n"),
890  progname, suffix);
891  exit(1);
892  }
893 
894  /* Valid integer? */
895  if ((uint64) result != (uint64) ((uint32) result))
896  {
897  fprintf(stderr,
898  _("%s: transfer rate \"%s\" exceeds integer range\n"),
899  progname, src);
900  exit(1);
901  }
902 
903  /*
904  * The range is checked on the server side too, but avoid the server
905  * connection if a nonsensical value was passed.
906  */
907  if (result < MAX_RATE_LOWER || result > MAX_RATE_UPPER)
908  {
909  fprintf(stderr,
910  _("%s: transfer rate \"%s\" is out of range\n"),
911  progname, src);
912  exit(1);
913  }
914 
915  return (int32) result;
916 }
917 
918 /*
919  * Write a piece of tar data
920  */
921 static void
923 #ifdef HAVE_LIBZ
924  gzFile ztarfile,
925 #endif
926  FILE *tarfile, char *buf, int r, char *current_file)
927 {
928 #ifdef HAVE_LIBZ
929  if (ztarfile != NULL)
930  {
931  if (gzwrite(ztarfile, buf, r) != r)
932  {
933  fprintf(stderr,
934  _("%s: could not write to compressed file \"%s\": %s\n"),
935  progname, current_file, get_gz_error(ztarfile));
937  }
938  }
939  else
940 #endif
941  {
942  if (fwrite(buf, r, 1, tarfile) != 1)
943  {
944  fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"),
945  progname, current_file, strerror(errno));
947  }
948  }
949 }
950 
951 #ifdef HAVE_LIBZ
952 #define WRITE_TAR_DATA(buf, sz) writeTarData(ztarfile, tarfile, buf, sz, filename)
953 #else
954 #define WRITE_TAR_DATA(buf, sz) writeTarData(tarfile, buf, sz, filename)
955 #endif
956 
957 /*
958  * Receive a tar format file from the connection to the server, and write
959  * the data from this file directly into a tar file. If compression is
960  * enabled, the data will be compressed while written to the file.
961  *
962  * The file will be named base.tar[.gz] if it's for the main data directory
963  * or <tablespaceoid>.tar[.gz] if it's for another tablespace.
964  *
965  * No attempt to inspect or validate the contents of the file is done.
966  */
967 static void
968 ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
969 {
970  char filename[MAXPGPATH];
971  char *copybuf = NULL;
972  FILE *tarfile = NULL;
973  char tarhdr[512];
974  bool basetablespace = PQgetisnull(res, rownum, 0);
975  bool in_tarhdr = true;
976  bool skip_file = false;
977  size_t tarhdrsz = 0;
978  pgoff_t filesz = 0;
979 
980 #ifdef HAVE_LIBZ
981  gzFile ztarfile = NULL;
982 #endif
983 
984  if (basetablespace)
985  {
986  /*
987  * Base tablespaces
988  */
989  if (strcmp(basedir, "-") == 0)
990  {
991 #ifdef WIN32
992  _setmode(fileno(stdout), _O_BINARY);
993 #endif
994 
995 #ifdef HAVE_LIBZ
996  if (compresslevel != 0)
997  {
998  ztarfile = gzdopen(dup(fileno(stdout)), "wb");
999  if (gzsetparams(ztarfile, compresslevel,
1000  Z_DEFAULT_STRATEGY) != Z_OK)
1001  {
1002  fprintf(stderr,
1003  _("%s: could not set compression level %d: %s\n"),
1004  progname, compresslevel, get_gz_error(ztarfile));
1006  }
1007  }
1008  else
1009 #endif
1010  tarfile = stdout;
1011  strcpy(filename, "-");
1012  }
1013  else
1014  {
1015 #ifdef HAVE_LIBZ
1016  if (compresslevel != 0)
1017  {
1018  snprintf(filename, sizeof(filename), "%s/base.tar.gz", basedir);
1019  ztarfile = gzopen(filename, "wb");
1020  if (gzsetparams(ztarfile, compresslevel,
1021  Z_DEFAULT_STRATEGY) != Z_OK)
1022  {
1023  fprintf(stderr,
1024  _("%s: could not set compression level %d: %s\n"),
1025  progname, compresslevel, get_gz_error(ztarfile));
1027  }
1028  }
1029  else
1030 #endif
1031  {
1032  snprintf(filename, sizeof(filename), "%s/base.tar", basedir);
1033  tarfile = fopen(filename, "wb");
1034  }
1035  }
1036  }
1037  else
1038  {
1039  /*
1040  * Specific tablespace
1041  */
1042 #ifdef HAVE_LIBZ
1043  if (compresslevel != 0)
1044  {
1045  snprintf(filename, sizeof(filename), "%s/%s.tar.gz", basedir,
1046  PQgetvalue(res, rownum, 0));
1047  ztarfile = gzopen(filename, "wb");
1048  if (gzsetparams(ztarfile, compresslevel,
1049  Z_DEFAULT_STRATEGY) != Z_OK)
1050  {
1051  fprintf(stderr,
1052  _("%s: could not set compression level %d: %s\n"),
1053  progname, compresslevel, get_gz_error(ztarfile));
1055  }
1056  }
1057  else
1058 #endif
1059  {
1060  snprintf(filename, sizeof(filename), "%s/%s.tar", basedir,
1061  PQgetvalue(res, rownum, 0));
1062  tarfile = fopen(filename, "wb");
1063  }
1064  }
1065 
1066 #ifdef HAVE_LIBZ
1067  if (compresslevel != 0)
1068  {
1069  if (!ztarfile)
1070  {
1071  /* Compression is in use */
1072  fprintf(stderr,
1073  _("%s: could not create compressed file \"%s\": %s\n"),
1074  progname, filename, get_gz_error(ztarfile));
1076  }
1077  }
1078  else
1079 #endif
1080  {
1081  /* Either no zlib support, or zlib support but compresslevel = 0 */
1082  if (!tarfile)
1083  {
1084  fprintf(stderr, _("%s: could not create file \"%s\": %s\n"),
1085  progname, filename, strerror(errno));
1087  }
1088  }
1089 
1090  /*
1091  * Get the COPY data stream
1092  */
1093  res = PQgetResult(conn);
1094  if (PQresultStatus(res) != PGRES_COPY_OUT)
1095  {
1096  fprintf(stderr, _("%s: could not get COPY data stream: %s"),
1097  progname, PQerrorMessage(conn));
1099  }
1100 
1101  while (1)
1102  {
1103  int r;
1104 
1105  if (copybuf != NULL)
1106  {
1107  PQfreemem(copybuf);
1108  copybuf = NULL;
1109  }
1110 
1111  r = PQgetCopyData(conn, &copybuf, 0);
1112  if (r == -1)
1113  {
1114  /*
1115  * End of chunk. If requested, and this is the base tablespace,
1116  * write recovery.conf into the tarfile. When done, close the file
1117  * (but not stdout).
1118  *
1119  * Also, write two completely empty blocks at the end of the tar
1120  * file, as required by some tar programs.
1121  */
1122  char zerobuf[1024];
1123 
1124  MemSet(zerobuf, 0, sizeof(zerobuf));
1125 
1126  if (basetablespace && writerecoveryconf)
1127  {
1128  char header[512];
1129  int padding;
1130 
1131  tarCreateHeader(header, "recovery.conf", NULL,
1132  recoveryconfcontents->len,
1133  pg_file_create_mode, 04000, 02000,
1134  time(NULL));
1135 
1136  padding = ((recoveryconfcontents->len + 511) & ~511) - recoveryconfcontents->len;
1137 
1138  WRITE_TAR_DATA(header, sizeof(header));
1139  WRITE_TAR_DATA(recoveryconfcontents->data, recoveryconfcontents->len);
1140  if (padding)
1141  WRITE_TAR_DATA(zerobuf, padding);
1142  }
1143 
1144  /* 2 * 512 bytes empty data at end of file */
1145  WRITE_TAR_DATA(zerobuf, sizeof(zerobuf));
1146 
1147 #ifdef HAVE_LIBZ
1148  if (ztarfile != NULL)
1149  {
1150  if (gzclose(ztarfile) != 0)
1151  {
1152  fprintf(stderr,
1153  _("%s: could not close compressed file \"%s\": %s\n"),
1154  progname, filename, get_gz_error(ztarfile));
1156  }
1157  }
1158  else
1159 #endif
1160  {
1161  if (strcmp(basedir, "-") != 0)
1162  {
1163  if (fclose(tarfile) != 0)
1164  {
1165  fprintf(stderr,
1166  _("%s: could not close file \"%s\": %s\n"),
1167  progname, filename, strerror(errno));
1169  }
1170  }
1171  }
1172 
1173  break;
1174  }
1175  else if (r == -2)
1176  {
1177  fprintf(stderr, _("%s: could not read COPY data: %s"),
1178  progname, PQerrorMessage(conn));
1180  }
1181 
1182  if (!writerecoveryconf || !basetablespace)
1183  {
1184  /*
1185  * When not writing recovery.conf, or when not working on the base
1186  * tablespace, we never have to look for an existing recovery.conf
1187  * file in the stream.
1188  */
1189  WRITE_TAR_DATA(copybuf, r);
1190  }
1191  else
1192  {
1193  /*
1194  * Look for a recovery.conf in the existing tar stream. If it's
1195  * there, we must skip it so we can later overwrite it with our
1196  * own version of the file.
1197  *
1198  * To do this, we have to process the individual files inside the
1199  * TAR stream. The stream consists of a header and zero or more
1200  * chunks, all 512 bytes long. The stream from the server is
1201  * broken up into smaller pieces, so we have to track the size of
1202  * the files to find the next header structure.
1203  */
1204  int rr = r;
1205  int pos = 0;
1206 
1207  while (rr > 0)
1208  {
1209  if (in_tarhdr)
1210  {
1211  /*
1212  * We're currently reading a header structure inside the
1213  * TAR stream, i.e. the file metadata.
1214  */
1215  if (tarhdrsz < 512)
1216  {
1217  /*
1218  * Copy the header structure into tarhdr in case the
1219  * header is not aligned to 512 bytes or it's not
1220  * returned in whole by the last PQgetCopyData call.
1221  */
1222  int hdrleft;
1223  int bytes2copy;
1224 
1225  hdrleft = 512 - tarhdrsz;
1226  bytes2copy = (rr > hdrleft ? hdrleft : rr);
1227 
1228  memcpy(&tarhdr[tarhdrsz], copybuf + pos, bytes2copy);
1229 
1230  rr -= bytes2copy;
1231  pos += bytes2copy;
1232  tarhdrsz += bytes2copy;
1233  }
1234  else
1235  {
1236  /*
1237  * We have the complete header structure in tarhdr,
1238  * look at the file metadata: - the subsequent file
1239  * contents have to be skipped if the filename is
1240  * recovery.conf - find out the size of the file
1241  * padded to the next multiple of 512
1242  */
1243  int padding;
1244 
1245  skip_file = (strcmp(&tarhdr[0], "recovery.conf") == 0);
1246 
1247  filesz = read_tar_number(&tarhdr[124], 12);
1248 
1249  padding = ((filesz + 511) & ~511) - filesz;
1250  filesz += padding;
1251 
1252  /* Next part is the file, not the header */
1253  in_tarhdr = false;
1254 
1255  /*
1256  * If we're not skipping the file, write the tar
1257  * header unmodified.
1258  */
1259  if (!skip_file)
1260  WRITE_TAR_DATA(tarhdr, 512);
1261  }
1262  }
1263  else
1264  {
1265  /*
1266  * We're processing a file's contents.
1267  */
1268  if (filesz > 0)
1269  {
1270  /*
1271  * We still have data to read (and possibly write).
1272  */
1273  int bytes2write;
1274 
1275  bytes2write = (filesz > rr ? rr : filesz);
1276 
1277  if (!skip_file)
1278  WRITE_TAR_DATA(copybuf + pos, bytes2write);
1279 
1280  rr -= bytes2write;
1281  pos += bytes2write;
1282  filesz -= bytes2write;
1283  }
1284  else
1285  {
1286  /*
1287  * No more data in the current file, the next piece of
1288  * data (if any) will be a new file header structure.
1289  */
1290  in_tarhdr = true;
1291  skip_file = false;
1292  tarhdrsz = 0;
1293  filesz = 0;
1294  }
1295  }
1296  }
1297  }
1298  totaldone += r;
1299  progress_report(rownum, filename, false);
1300  } /* while (1) */
1301  progress_report(rownum, filename, true);
1302 
1303  if (copybuf != NULL)
1304  PQfreemem(copybuf);
1305 
1306  /* sync the resulting tar file, errors are not considered fatal */
1307  if (do_sync && strcmp(basedir, "-") != 0)
1308  (void) fsync_fname(filename, false, progname);
1309 }
1310 
1311 
1312 /*
1313  * Retrieve tablespace path, either relocated or original depending on whether
1314  * -T was passed or not.
1315  */
1316 static const char *
1317 get_tablespace_mapping(const char *dir)
1318 {
1319  TablespaceListCell *cell;
1320  char canon_dir[MAXPGPATH];
1321 
1322  /* Canonicalize path for comparison consistency */
1323  strlcpy(canon_dir, dir, sizeof(canon_dir));
1324  canonicalize_path(canon_dir);
1325 
1326  for (cell = tablespace_dirs.head; cell; cell = cell->next)
1327  if (strcmp(canon_dir, cell->old_dir) == 0)
1328  return cell->new_dir;
1329 
1330  return dir;
1331 }
1332 
1333 
1334 /*
1335  * Receive a tar format stream from the connection to the server, and unpack
1336  * the contents of it into a directory. Only files, directories and
1337  * symlinks are supported, no other kinds of special files.
1338  *
1339  * If the data is for the main data directory, it will be restored in the
1340  * specified directory. If it's for another tablespace, it will be restored
1341  * in the original or mapped directory.
1342  */
1343 static void
1345 {
1346  char current_path[MAXPGPATH];
1347  char filename[MAXPGPATH];
1348  const char *mapped_tblspc_path;
1349  pgoff_t current_len_left = 0;
1350  int current_padding = 0;
1351  bool basetablespace;
1352  char *copybuf = NULL;
1353  FILE *file = NULL;
1354 
1355  basetablespace = PQgetisnull(res, rownum, 0);
1356  if (basetablespace)
1357  strlcpy(current_path, basedir, sizeof(current_path));
1358  else
1359  strlcpy(current_path,
1360  get_tablespace_mapping(PQgetvalue(res, rownum, 1)),
1361  sizeof(current_path));
1362 
1363  /*
1364  * Get the COPY data
1365  */
1366  res = PQgetResult(conn);
1367  if (PQresultStatus(res) != PGRES_COPY_OUT)
1368  {
1369  fprintf(stderr, _("%s: could not get COPY data stream: %s"),
1370  progname, PQerrorMessage(conn));
1372  }
1373 
1374  while (1)
1375  {
1376  int r;
1377 
1378  if (copybuf != NULL)
1379  {
1380  PQfreemem(copybuf);
1381  copybuf = NULL;
1382  }
1383 
1384  r = PQgetCopyData(conn, &copybuf, 0);
1385 
1386  if (r == -1)
1387  {
1388  /*
1389  * End of chunk
1390  */
1391  if (file)
1392  fclose(file);
1393 
1394  break;
1395  }
1396  else if (r == -2)
1397  {
1398  fprintf(stderr, _("%s: could not read COPY data: %s"),
1399  progname, PQerrorMessage(conn));
1401  }
1402 
1403  if (file == NULL)
1404  {
1405  int filemode;
1406 
1407  /*
1408  * No current file, so this must be the header for a new file
1409  */
1410  if (r != 512)
1411  {
1412  fprintf(stderr, _("%s: invalid tar block header size: %d\n"),
1413  progname, r);
1415  }
1416  totaldone += 512;
1417 
1418  current_len_left = read_tar_number(&copybuf[124], 12);
1419 
1420  /* Set permissions on the file */
1421  filemode = read_tar_number(&copybuf[100], 8);
1422 
1423  /*
1424  * All files are padded up to 512 bytes
1425  */
1426  current_padding =
1427  ((current_len_left + 511) & ~511) - current_len_left;
1428 
1429  /*
1430  * First part of header is zero terminated filename
1431  */
1432  snprintf(filename, sizeof(filename), "%s/%s", current_path,
1433  copybuf);
1434  if (filename[strlen(filename) - 1] == '/')
1435  {
1436  /*
1437  * Ends in a slash means directory or symlink to directory
1438  */
1439  if (copybuf[156] == '5')
1440  {
1441  /*
1442  * Directory
1443  */
1444  filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */
1445  if (mkdir(filename, pg_dir_create_mode) != 0)
1446  {
1447  /*
1448  * When streaming WAL, pg_wal (or pg_xlog for pre-9.6
1449  * clusters) will have been created by the wal
1450  * receiver process. Also, when the WAL directory
1451  * location was specified, pg_wal (or pg_xlog) has
1452  * already been created as a symbolic link before
1453  * starting the actual backup. So just ignore creation
1454  * failures on related directories.
1455  */
1456  if (!((pg_str_endswith(filename, "/pg_wal") ||
1457  pg_str_endswith(filename, "/pg_xlog") ||
1458  pg_str_endswith(filename, "/archive_status")) &&
1459  errno == EEXIST))
1460  {
1461  fprintf(stderr,
1462  _("%s: could not create directory \"%s\": %s\n"),
1463  progname, filename, strerror(errno));
1465  }
1466  }
1467 #ifndef WIN32
1468  if (chmod(filename, (mode_t) filemode))
1469  fprintf(stderr,
1470  _("%s: could not set permissions on directory \"%s\": %s\n"),
1471  progname, filename, strerror(errno));
1472 #endif
1473  }
1474  else if (copybuf[156] == '2')
1475  {
1476  /*
1477  * Symbolic link
1478  *
1479  * It's most likely a link in pg_tblspc directory, to the
1480  * location of a tablespace. Apply any tablespace mapping
1481  * given on the command line (--tablespace-mapping). (We
1482  * blindly apply the mapping without checking that the
1483  * link really is inside pg_tblspc. We don't expect there
1484  * to be other symlinks in a data directory, but if there
1485  * are, you can call it an undocumented feature that you
1486  * can map them too.)
1487  */
1488  filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */
1489 
1490  mapped_tblspc_path = get_tablespace_mapping(&copybuf[157]);
1491  if (symlink(mapped_tblspc_path, filename) != 0)
1492  {
1493  fprintf(stderr,
1494  _("%s: could not create symbolic link from \"%s\" to \"%s\": %s\n"),
1495  progname, filename, mapped_tblspc_path,
1496  strerror(errno));
1498  }
1499  }
1500  else
1501  {
1502  fprintf(stderr,
1503  _("%s: unrecognized link indicator \"%c\"\n"),
1504  progname, copybuf[156]);
1506  }
1507  continue; /* directory or link handled */
1508  }
1509 
1510  /*
1511  * regular file
1512  */
1513  file = fopen(filename, "wb");
1514  if (!file)
1515  {
1516  fprintf(stderr, _("%s: could not create file \"%s\": %s\n"),
1517  progname, filename, strerror(errno));
1519  }
1520 
1521 #ifndef WIN32
1522  if (chmod(filename, (mode_t) filemode))
1523  fprintf(stderr, _("%s: could not set permissions on file \"%s\": %s\n"),
1524  progname, filename, strerror(errno));
1525 #endif
1526 
1527  if (current_len_left == 0)
1528  {
1529  /*
1530  * Done with this file, next one will be a new tar header
1531  */
1532  fclose(file);
1533  file = NULL;
1534  continue;
1535  }
1536  } /* new file */
1537  else
1538  {
1539  /*
1540  * Continuing blocks in existing file
1541  */
1542  if (current_len_left == 0 && r == current_padding)
1543  {
1544  /*
1545  * Received the padding block for this file, ignore it and
1546  * close the file, then move on to the next tar header.
1547  */
1548  fclose(file);
1549  file = NULL;
1550  totaldone += r;
1551  continue;
1552  }
1553 
1554  if (fwrite(copybuf, r, 1, file) != 1)
1555  {
1556  fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"),
1557  progname, filename, strerror(errno));
1559  }
1560  totaldone += r;
1561  progress_report(rownum, filename, false);
1562 
1563  current_len_left -= r;
1564  if (current_len_left == 0 && current_padding == 0)
1565  {
1566  /*
1567  * Received the last block, and there is no padding to be
1568  * expected. Close the file and move on to the next tar
1569  * header.
1570  */
1571  fclose(file);
1572  file = NULL;
1573  continue;
1574  }
1575  } /* continuing data in existing file */
1576  } /* loop over all data blocks */
1577  progress_report(rownum, filename, true);
1578 
1579  if (file != NULL)
1580  {
1581  fprintf(stderr,
1582  _("%s: COPY stream ended before last file was finished\n"),
1583  progname);
1585  }
1586 
1587  if (copybuf != NULL)
1588  PQfreemem(copybuf);
1589 
1590  if (basetablespace && writerecoveryconf)
1592 
1593  /*
1594  * No data is synced here, everything is done for all tablespaces at the
1595  * end.
1596  */
1597 }
1598 
1599 /*
1600  * Escape a string so that it can be used as a value in a key-value pair
1601  * a configuration file.
1602  */
1603 static char *
1604 escape_quotes(const char *src)
1605 {
1606  char *result = escape_single_quotes_ascii(src);
1607 
1608  if (!result)
1609  {
1610  fprintf(stderr, _("%s: out of memory\n"), progname);
1611  exit(1);
1612  }
1613  return result;
1614 }
1615 
1616 /*
1617  * Create a recovery.conf file in memory using a PQExpBuffer
1618  */
1619 static void
1621 {
1622  PQconninfoOption *connOptions;
1624  PQExpBufferData conninfo_buf;
1625  char *escaped;
1626 
1627  recoveryconfcontents = createPQExpBuffer();
1628  if (!recoveryconfcontents)
1629  {
1630  fprintf(stderr, _("%s: out of memory\n"), progname);
1632  }
1633 
1634  connOptions = PQconninfo(conn);
1635  if (connOptions == NULL)
1636  {
1637  fprintf(stderr, _("%s: out of memory\n"), progname);
1639  }
1640 
1641  appendPQExpBufferStr(recoveryconfcontents, "standby_mode = 'on'\n");
1642 
1643  initPQExpBuffer(&conninfo_buf);
1644  for (option = connOptions; option && option->keyword; option++)
1645  {
1646  /*
1647  * Do not emit this setting if: - the setting is "replication",
1648  * "dbname" or "fallback_application_name", since these would be
1649  * overridden by the libpqwalreceiver module anyway. - not set or
1650  * empty.
1651  */
1652  if (strcmp(option->keyword, "replication") == 0 ||
1653  strcmp(option->keyword, "dbname") == 0 ||
1654  strcmp(option->keyword, "fallback_application_name") == 0 ||
1655  (option->val == NULL) ||
1656  (option->val != NULL && option->val[0] == '\0'))
1657  continue;
1658 
1659  /* Separate key-value pairs with spaces */
1660  if (conninfo_buf.len != 0)
1661  appendPQExpBufferChar(&conninfo_buf, ' ');
1662 
1663  /*
1664  * Write "keyword=value" pieces, the value string is escaped and/or
1665  * quoted if necessary.
1666  */
1667  appendPQExpBuffer(&conninfo_buf, "%s=", option->keyword);
1668  appendConnStrVal(&conninfo_buf, option->val);
1669  }
1670 
1671  /*
1672  * Escape the connection string, so that it can be put in the config file.
1673  * Note that this is different from the escaping of individual connection
1674  * options above!
1675  */
1676  escaped = escape_quotes(conninfo_buf.data);
1677  appendPQExpBuffer(recoveryconfcontents, "primary_conninfo = '%s'\n", escaped);
1678  free(escaped);
1679 
1680  if (replication_slot)
1681  {
1682  escaped = escape_quotes(replication_slot);
1683  appendPQExpBuffer(recoveryconfcontents, "primary_slot_name = '%s'\n", replication_slot);
1684  free(escaped);
1685  }
1686 
1687  if (PQExpBufferBroken(recoveryconfcontents) ||
1688  PQExpBufferDataBroken(conninfo_buf))
1689  {
1690  fprintf(stderr, _("%s: out of memory\n"), progname);
1692  }
1693 
1694  termPQExpBuffer(&conninfo_buf);
1695 
1696  PQconninfoFree(connOptions);
1697 }
1698 
1699 
1700 /*
1701  * Write a recovery.conf file into the directory specified in basedir,
1702  * with the contents already collected in memory.
1703  */
1704 static void
1706 {
1707  char filename[MAXPGPATH];
1708  FILE *cf;
1709 
1710  sprintf(filename, "%s/recovery.conf", basedir);
1711 
1712  cf = fopen(filename, "w");
1713  if (cf == NULL)
1714  {
1715  fprintf(stderr, _("%s: could not create file \"%s\": %s\n"), progname, filename, strerror(errno));
1717  }
1718 
1719  if (fwrite(recoveryconfcontents->data, recoveryconfcontents->len, 1, cf) != 1)
1720  {
1721  fprintf(stderr,
1722  _("%s: could not write to file \"%s\": %s\n"),
1723  progname, filename, strerror(errno));
1725  }
1726 
1727  fclose(cf);
1728 }
1729 
1730 
1731 static void
1733 {
1734  PGresult *res;
1735  char *sysidentifier;
1736  TimeLineID latesttli;
1737  TimeLineID starttli;
1738  char *basebkp;
1739  char escaped_label[MAXPGPATH];
1740  char *maxrate_clause = NULL;
1741  int i;
1742  char xlogstart[64];
1743  char xlogend[64];
1744  int minServerMajor,
1745  maxServerMajor;
1746  int serverVersion,
1747  serverMajor;
1748 
1749  Assert(conn != NULL);
1750 
1751  /*
1752  * Check server version. BASE_BACKUP command was introduced in 9.1, so we
1753  * can't work with servers older than 9.1.
1754  */
1755  minServerMajor = 901;
1756  maxServerMajor = PG_VERSION_NUM / 100;
1757  serverVersion = PQserverVersion(conn);
1758  serverMajor = serverVersion / 100;
1759  if (serverMajor < minServerMajor || serverMajor > maxServerMajor)
1760  {
1761  const char *serverver = PQparameterStatus(conn, "server_version");
1762 
1763  fprintf(stderr, _("%s: incompatible server version %s\n"),
1764  progname, serverver ? serverver : "'unknown'");
1766  }
1767 
1768  /*
1769  * If WAL streaming was requested, also check that the server is new
1770  * enough for that.
1771  */
1773  {
1774  /*
1775  * Error message already written in CheckServerVersionForStreaming(),
1776  * but add a hint about using -X none.
1777  */
1778  fprintf(stderr, _("HINT: use -X none or -X fetch to disable log streaming\n"));
1780  }
1781 
1782  /*
1783  * Build contents of recovery.conf if requested
1784  */
1785  if (writerecoveryconf)
1787 
1788  /*
1789  * Run IDENTIFY_SYSTEM so we can get the timeline
1790  */
1791  if (!RunIdentifySystem(conn, &sysidentifier, &latesttli, NULL, NULL))
1793 
1794  /*
1795  * Start the actual backup
1796  */
1797  PQescapeStringConn(conn, escaped_label, label, sizeof(escaped_label), &i);
1798 
1799  if (maxrate > 0)
1800  maxrate_clause = psprintf("MAX_RATE %u", maxrate);
1801 
1802  if (verbose)
1803  fprintf(stderr,
1804  _("%s: initiating base backup, waiting for checkpoint to complete\n"),
1805  progname);
1806 
1807  if (showprogress && !verbose)
1808  {
1809  fprintf(stderr, "waiting for checkpoint");
1810  if (isatty(fileno(stderr)))
1811  fprintf(stderr, "\r");
1812  else
1813  fprintf(stderr, "\n");
1814  }
1815 
1816  basebkp =
1817  psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s",
1818  escaped_label,
1819  showprogress ? "PROGRESS" : "",
1820  includewal == FETCH_WAL ? "WAL" : "",
1821  fastcheckpoint ? "FAST" : "",
1822  includewal == NO_WAL ? "" : "NOWAIT",
1823  maxrate_clause ? maxrate_clause : "",
1824  format == 't' ? "TABLESPACE_MAP" : "",
1825  verify_checksums ? "" : "NOVERIFY_CHECKSUMS");
1826 
1827  if (PQsendQuery(conn, basebkp) == 0)
1828  {
1829  fprintf(stderr, _("%s: could not send replication command \"%s\": %s"),
1830  progname, "BASE_BACKUP", PQerrorMessage(conn));
1832  }
1833 
1834  /*
1835  * Get the starting WAL location
1836  */
1837  res = PQgetResult(conn);
1838  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1839  {
1840  fprintf(stderr, _("%s: could not initiate base backup: %s"),
1843  }
1844  if (PQntuples(res) != 1)
1845  {
1846  fprintf(stderr,
1847  _("%s: server returned unexpected response to BASE_BACKUP command; got %d rows and %d fields, expected %d rows and %d fields\n"),
1848  progname, PQntuples(res), PQnfields(res), 1, 2);
1850  }
1851 
1852  strlcpy(xlogstart, PQgetvalue(res, 0, 0), sizeof(xlogstart));
1853 
1854  if (verbose)
1855  fprintf(stderr, _("%s: checkpoint completed\n"), progname);
1856 
1857  /*
1858  * 9.3 and later sends the TLI of the starting point. With older servers,
1859  * assume it's the same as the latest timeline reported by
1860  * IDENTIFY_SYSTEM.
1861  */
1862  if (PQnfields(res) >= 2)
1863  starttli = atoi(PQgetvalue(res, 0, 1));
1864  else
1865  starttli = latesttli;
1866  PQclear(res);
1867  MemSet(xlogend, 0, sizeof(xlogend));
1868 
1869  if (verbose && includewal != NO_WAL)
1870  fprintf(stderr, _("%s: write-ahead log start point: %s on timeline %u\n"),
1871  progname, xlogstart, starttli);
1872 
1873  /*
1874  * Get the header
1875  */
1876  res = PQgetResult(conn);
1877  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1878  {
1879  fprintf(stderr, _("%s: could not get backup header: %s"),
1882  }
1883  if (PQntuples(res) < 1)
1884  {
1885  fprintf(stderr, _("%s: no data returned from server\n"), progname);
1887  }
1888 
1889  /*
1890  * Sum up the total size, for progress reporting
1891  */
1892  totalsize = totaldone = 0;
1893  tablespacecount = PQntuples(res);
1894  for (i = 0; i < PQntuples(res); i++)
1895  {
1896  totalsize += atol(PQgetvalue(res, i, 2));
1897 
1898  /*
1899  * Verify tablespace directories are empty. Don't bother with the
1900  * first once since it can be relocated, and it will be checked before
1901  * we do anything anyway.
1902  */
1903  if (format == 'p' && !PQgetisnull(res, i, 1))
1904  {
1905  char *path = (char *) get_tablespace_mapping(PQgetvalue(res, i, 1));
1906 
1908  }
1909  }
1910 
1911  /*
1912  * When writing to stdout, require a single tablespace
1913  */
1914  if (format == 't' && strcmp(basedir, "-") == 0 && PQntuples(res) > 1)
1915  {
1916  fprintf(stderr,
1917  _("%s: can only write single tablespace to stdout, database has %d\n"),
1918  progname, PQntuples(res));
1920  }
1921 
1922  /*
1923  * If we're streaming WAL, start the streaming session before we start
1924  * receiving the actual data chunks.
1925  */
1926  if (includewal == STREAM_WAL)
1927  {
1928  if (verbose)
1929  fprintf(stderr, _("%s: starting background WAL receiver\n"),
1930  progname);
1931  StartLogStreamer(xlogstart, starttli, sysidentifier);
1932  }
1933 
1934  /*
1935  * Start receiving chunks
1936  */
1937  for (i = 0; i < PQntuples(res); i++)
1938  {
1939  if (format == 't')
1940  ReceiveTarFile(conn, res, i);
1941  else
1942  ReceiveAndUnpackTarFile(conn, res, i);
1943  } /* Loop over all tablespaces */
1944 
1945  if (showprogress)
1946  {
1947  progress_report(PQntuples(res), NULL, true);
1948  if (isatty(fileno(stderr)))
1949  fprintf(stderr, "\n"); /* Need to move to next line */
1950  }
1951 
1952  PQclear(res);
1953 
1954  /*
1955  * Get the stop position
1956  */
1957  res = PQgetResult(conn);
1958  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1959  {
1960  fprintf(stderr,
1961  _("%s: could not get write-ahead log end position from server: %s"),
1964  }
1965  if (PQntuples(res) != 1)
1966  {
1967  fprintf(stderr,
1968  _("%s: no write-ahead log end position returned from server\n"),
1969  progname);
1971  }
1972  strlcpy(xlogend, PQgetvalue(res, 0, 0), sizeof(xlogend));
1973  if (verbose && includewal != NO_WAL)
1974  fprintf(stderr, _("%s: write-ahead log end point: %s\n"), progname, xlogend);
1975  PQclear(res);
1976 
1977  res = PQgetResult(conn);
1978  if (PQresultStatus(res) != PGRES_COMMAND_OK)
1979  {
1980  const char *sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
1981 
1982  if (sqlstate &&
1983  strcmp(sqlstate, ERRCODE_DATA_CORRUPTED) == 0)
1984  {
1985  fprintf(stderr, _("%s: checksum error occured\n"),
1986  progname);
1987  checksum_failure = true;
1988  }
1989  else
1990  {
1991  fprintf(stderr, _("%s: final receive failed: %s"),
1993  }
1995  }
1996 
1997  if (bgchild > 0)
1998  {
1999 #ifndef WIN32
2000  int status;
2001  int r;
2002 #else
2003  DWORD status;
2004 
2005  /*
2006  * get a pointer sized version of bgchild to avoid warnings about
2007  * casting to a different size on WIN64.
2008  */
2009  intptr_t bgchild_handle = bgchild;
2010  uint32 hi,
2011  lo;
2012 #endif
2013 
2014  if (verbose)
2015  fprintf(stderr,
2016  _("%s: waiting for background process to finish streaming ...\n"), progname);
2017 
2018 #ifndef WIN32
2019  if (write(bgpipe[1], xlogend, strlen(xlogend)) != strlen(xlogend))
2020  {
2021  fprintf(stderr,
2022  _("%s: could not send command to background pipe: %s\n"),
2023  progname, strerror(errno));
2025  }
2026 
2027  /* Just wait for the background process to exit */
2028  r = waitpid(bgchild, &status, 0);
2029  if (r == -1)
2030  {
2031  fprintf(stderr, _("%s: could not wait for child process: %s\n"),
2032  progname, strerror(errno));
2034  }
2035  if (r != bgchild)
2036  {
2037  fprintf(stderr, _("%s: child %d died, expected %d\n"),
2038  progname, r, (int) bgchild);
2040  }
2041  if (!WIFEXITED(status))
2042  {
2043  fprintf(stderr, _("%s: child process did not exit normally\n"),
2044  progname);
2046  }
2047  if (WEXITSTATUS(status) != 0)
2048  {
2049  fprintf(stderr, _("%s: child process exited with error %d\n"),
2050  progname, WEXITSTATUS(status));
2052  }
2053  /* Exited normally, we're happy! */
2054 #else /* WIN32 */
2055 
2056  /*
2057  * On Windows, since we are in the same process, we can just store the
2058  * value directly in the variable, and then set the flag that says
2059  * it's there.
2060  */
2061  if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
2062  {
2063  fprintf(stderr,
2064  _("%s: could not parse write-ahead log location \"%s\"\n"),
2065  progname, xlogend);
2067  }
2068  xlogendptr = ((uint64) hi) << 32 | lo;
2069  InterlockedIncrement(&has_xlogendptr);
2070 
2071  /* First wait for the thread to exit */
2072  if (WaitForSingleObjectEx((HANDLE) bgchild_handle, INFINITE, FALSE) !=
2073  WAIT_OBJECT_0)
2074  {
2075  _dosmaperr(GetLastError());
2076  fprintf(stderr, _("%s: could not wait for child thread: %s\n"),
2077  progname, strerror(errno));
2079  }
2080  if (GetExitCodeThread((HANDLE) bgchild_handle, &status) == 0)
2081  {
2082  _dosmaperr(GetLastError());
2083  fprintf(stderr, _("%s: could not get child thread exit status: %s\n"),
2084  progname, strerror(errno));
2086  }
2087  if (status != 0)
2088  {
2089  fprintf(stderr, _("%s: child thread exited with error %u\n"),
2090  progname, (unsigned int) status);
2092  }
2093  /* Exited normally, we're happy */
2094 #endif
2095  }
2096 
2097  /* Free the recovery.conf contents */
2098  destroyPQExpBuffer(recoveryconfcontents);
2099 
2100  /*
2101  * End of copy data. Final result is already checked inside the loop.
2102  */
2103  PQclear(res);
2104  PQfinish(conn);
2105 
2106  /*
2107  * Make data persistent on disk once backup is completed. For tar format
2108  * once syncing the parent directory is fine, each tar file created per
2109  * tablespace has been already synced. In plain format, all the data of
2110  * the base directory is synced, taking into account all the tablespaces.
2111  * Errors are not considered fatal.
2112  */
2113  if (do_sync)
2114  {
2115  if (format == 't')
2116  {
2117  if (strcmp(basedir, "-") != 0)
2118  (void) fsync_fname(basedir, true, progname);
2119  }
2120  else
2121  {
2122  (void) fsync_pgdata(basedir, progname, serverVersion);
2123  }
2124  }
2125 
2126  if (verbose)
2127  fprintf(stderr, _("%s: base backup completed\n"), progname);
2128 }
2129 
2130 
2131 int
2132 main(int argc, char **argv)
2133 {
2134  static struct option long_options[] = {
2135  {"help", no_argument, NULL, '?'},
2136  {"version", no_argument, NULL, 'V'},
2137  {"pgdata", required_argument, NULL, 'D'},
2138  {"format", required_argument, NULL, 'F'},
2139  {"checkpoint", required_argument, NULL, 'c'},
2140  {"create-slot", no_argument, NULL, 'C'},
2141  {"max-rate", required_argument, NULL, 'r'},
2142  {"write-recovery-conf", no_argument, NULL, 'R'},
2143  {"slot", required_argument, NULL, 'S'},
2144  {"tablespace-mapping", required_argument, NULL, 'T'},
2145  {"wal-method", required_argument, NULL, 'X'},
2146  {"gzip", no_argument, NULL, 'z'},
2147  {"compress", required_argument, NULL, 'Z'},
2148  {"label", required_argument, NULL, 'l'},
2149  {"no-clean", no_argument, NULL, 'n'},
2150  {"no-sync", no_argument, NULL, 'N'},
2151  {"dbname", required_argument, NULL, 'd'},
2152  {"host", required_argument, NULL, 'h'},
2153  {"port", required_argument, NULL, 'p'},
2154  {"username", required_argument, NULL, 'U'},
2155  {"no-password", no_argument, NULL, 'w'},
2156  {"password", no_argument, NULL, 'W'},
2157  {"status-interval", required_argument, NULL, 's'},
2158  {"verbose", no_argument, NULL, 'v'},
2159  {"progress", no_argument, NULL, 'P'},
2160  {"waldir", required_argument, NULL, 1},
2161  {"no-slot", no_argument, NULL, 2},
2162  {"no-verify-checksums", no_argument, NULL, 'k'},
2163  {NULL, 0, NULL, 0}
2164  };
2165  int c;
2166 
2167  int option_index;
2168 
2169  progname = get_progname(argv[0]);
2170  set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup"));
2171 
2172  if (argc > 1)
2173  {
2174  if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
2175  {
2176  usage();
2177  exit(0);
2178  }
2179  else if (strcmp(argv[1], "-V") == 0
2180  || strcmp(argv[1], "--version") == 0)
2181  {
2182  puts("pg_basebackup (PostgreSQL) " PG_VERSION);
2183  exit(0);
2184  }
2185  }
2186 
2188 
2189  while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP",
2190  long_options, &option_index)) != -1)
2191  {
2192  switch (c)
2193  {
2194  case 'C':
2195  create_slot = true;
2196  break;
2197  case 'D':
2199  break;
2200  case 'F':
2201  if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
2202  format = 'p';
2203  else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
2204  format = 't';
2205  else
2206  {
2207  fprintf(stderr,
2208  _("%s: invalid output format \"%s\", must be \"plain\" or \"tar\"\n"),
2209  progname, optarg);
2210  exit(1);
2211  }
2212  break;
2213  case 'r':
2215  break;
2216  case 'R':
2217  writerecoveryconf = true;
2218  break;
2219  case 'S':
2220 
2221  /*
2222  * When specifying replication slot name, use a permanent
2223  * slot.
2224  */
2226  temp_replication_slot = false;
2227  break;
2228  case 2:
2229  no_slot = true;
2230  break;
2231  case 'T':
2233  break;
2234  case 'X':
2235  if (strcmp(optarg, "n") == 0 ||
2236  strcmp(optarg, "none") == 0)
2237  {
2238  includewal = NO_WAL;
2239  }
2240  else if (strcmp(optarg, "f") == 0 ||
2241  strcmp(optarg, "fetch") == 0)
2242  {
2244  }
2245  else if (strcmp(optarg, "s") == 0 ||
2246  strcmp(optarg, "stream") == 0)
2247  {
2249  }
2250  else
2251  {
2252  fprintf(stderr,
2253  _("%s: invalid wal-method option \"%s\", must be \"fetch\", \"stream\", or \"none\"\n"),
2254  progname, optarg);
2255  exit(1);
2256  }
2257  break;
2258  case 1:
2260  break;
2261  case 'l':
2262  label = pg_strdup(optarg);
2263  break;
2264  case 'n':
2265  noclean = true;
2266  break;
2267  case 'N':
2268  do_sync = false;
2269  break;
2270  case 'z':
2271 #ifdef HAVE_LIBZ
2273 #else
2274  compresslevel = 1; /* will be rejected below */
2275 #endif
2276  break;
2277  case 'Z':
2278  compresslevel = atoi(optarg);
2279  if (compresslevel < 0 || compresslevel > 9)
2280  {
2281  fprintf(stderr, _("%s: invalid compression level \"%s\"\n"),
2282  progname, optarg);
2283  exit(1);
2284  }
2285  break;
2286  case 'c':
2287  if (pg_strcasecmp(optarg, "fast") == 0)
2288  fastcheckpoint = true;
2289  else if (pg_strcasecmp(optarg, "spread") == 0)
2290  fastcheckpoint = false;
2291  else
2292  {
2293  fprintf(stderr, _("%s: invalid checkpoint argument \"%s\", must be \"fast\" or \"spread\"\n"),
2294  progname, optarg);
2295  exit(1);
2296  }
2297  break;
2298  case 'd':
2300  break;
2301  case 'h':
2302  dbhost = pg_strdup(optarg);
2303  break;
2304  case 'p':
2305  dbport = pg_strdup(optarg);
2306  break;
2307  case 'U':
2308  dbuser = pg_strdup(optarg);
2309  break;
2310  case 'w':
2311  dbgetpassword = -1;
2312  break;
2313  case 'W':
2314  dbgetpassword = 1;
2315  break;
2316  case 's':
2317  standby_message_timeout = atoi(optarg) * 1000;
2318  if (standby_message_timeout < 0)
2319  {
2320  fprintf(stderr, _("%s: invalid status interval \"%s\"\n"),
2321  progname, optarg);
2322  exit(1);
2323  }
2324  break;
2325  case 'v':
2326  verbose++;
2327  break;
2328  case 'P':
2329  showprogress = true;
2330  break;
2331  case 'k':
2332  verify_checksums = false;
2333  break;
2334  default:
2335 
2336  /*
2337  * getopt_long already emitted a complaint
2338  */
2339  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2340  progname);
2341  exit(1);
2342  }
2343  }
2344 
2345  /*
2346  * Any non-option arguments?
2347  */
2348  if (optind < argc)
2349  {
2350  fprintf(stderr,
2351  _("%s: too many command-line arguments (first is \"%s\")\n"),
2352  progname, argv[optind]);
2353  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2354  progname);
2355  exit(1);
2356  }
2357 
2358  /*
2359  * Required arguments
2360  */
2361  if (basedir == NULL)
2362  {
2363  fprintf(stderr, _("%s: no target directory specified\n"), progname);
2364  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2365  progname);
2366  exit(1);
2367  }
2368 
2369  /*
2370  * Mutually exclusive arguments
2371  */
2372  if (format == 'p' && compresslevel != 0)
2373  {
2374  fprintf(stderr,
2375  _("%s: only tar mode backups can be compressed\n"),
2376  progname);
2377  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2378  progname);
2379  exit(1);
2380  }
2381 
2382  if (format == 't' && includewal == STREAM_WAL && strcmp(basedir, "-") == 0)
2383  {
2384  fprintf(stderr,
2385  _("%s: cannot stream write-ahead logs in tar mode to stdout\n"),
2386  progname);
2387  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2388  progname);
2389  exit(1);
2390  }
2391 
2393  {
2394  fprintf(stderr,
2395  _("%s: replication slots can only be used with WAL streaming\n"),
2396  progname);
2397  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2398  progname);
2399  exit(1);
2400  }
2401 
2402  if (no_slot)
2403  {
2404  if (replication_slot)
2405  {
2406  fprintf(stderr,
2407  _("%s: --no-slot cannot be used with slot name\n"),
2408  progname);
2409  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2410  progname);
2411  exit(1);
2412  }
2413  temp_replication_slot = false;
2414  }
2415 
2416  if (create_slot)
2417  {
2418  if (!replication_slot)
2419  {
2420  fprintf(stderr,
2421  _("%s: %s needs a slot to be specified using --slot\n"),
2422  progname, "--create-slot");
2423  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2424  progname);
2425  exit(1);
2426  }
2427 
2428  if (no_slot)
2429  {
2430  fprintf(stderr,
2431  _("%s: --create-slot and --no-slot are incompatible options\n"),
2432  progname);
2433  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2434  progname);
2435  exit(1);
2436  }
2437  }
2438 
2439  if (xlog_dir)
2440  {
2441  if (format != 'p')
2442  {
2443  fprintf(stderr,
2444  _("%s: WAL directory location can only be specified in plain mode\n"),
2445  progname);
2446  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2447  progname);
2448  exit(1);
2449  }
2450 
2451  /* clean up xlog directory name, check it's absolute */
2453  if (!is_absolute_path(xlog_dir))
2454  {
2455  fprintf(stderr, _("%s: WAL directory location must be "
2456  "an absolute path\n"), progname);
2457  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2458  progname);
2459  exit(1);
2460  }
2461  }
2462 
2463 #ifndef HAVE_LIBZ
2464  if (compresslevel != 0)
2465  {
2466  fprintf(stderr,
2467  _("%s: this build does not support compression\n"),
2468  progname);
2469  exit(1);
2470  }
2471 #endif
2472 
2473  /* connection in replication mode to server */
2474  conn = GetConnection();
2475  if (!conn)
2476  {
2477  /* Error message already written in GetConnection() */
2478  exit(1);
2479  }
2480 
2481  /*
2482  * Set umask so that directories/files are created with the same
2483  * permissions as directories/files in the source data directory.
2484  *
2485  * pg_mode_mask is set to owner-only by default and then updated in
2486  * GetConnection() where we get the mode from the server-side with
2487  * RetrieveDataDirCreatePerm() and then call SetDataDirectoryCreatePerm().
2488  */
2489  umask(pg_mode_mask);
2490 
2491  /*
2492  * Verify that the target directory exists, or create it. For plaintext
2493  * backups, always require the directory. For tar backups, require it
2494  * unless we are writing to stdout.
2495  */
2496  if (format == 'p' || strcmp(basedir, "-") != 0)
2498 
2499  /* determine remote server's xlog segment size */
2500  if (!RetrieveWalSegSize(conn))
2502 
2503  /* Create pg_wal symlink, if required */
2504  if (xlog_dir)
2505  {
2506  char *linkloc;
2507 
2509 
2510  /*
2511  * Form name of the place where the symlink must go. pg_xlog has been
2512  * renamed to pg_wal in post-10 clusters.
2513  */
2514  linkloc = psprintf("%s/%s", basedir,
2516  "pg_xlog" : "pg_wal");
2517 
2518 #ifdef HAVE_SYMLINK
2519  if (symlink(xlog_dir, linkloc) != 0)
2520  {
2521  fprintf(stderr, _("%s: could not create symbolic link \"%s\": %s\n"),
2522  progname, linkloc, strerror(errno));
2524  }
2525 #else
2526  fprintf(stderr, _("%s: symlinks are not supported on this platform\n"));
2528 #endif
2529  free(linkloc);
2530  }
2531 
2532  BaseBackup();
2533 
2534  success = true;
2535  return 0;
2536 }
int PQnfields(const PGresult *res)
Definition: fe-exec.c:2732
static PQExpBuffer recoveryconfcontents
static void writeTarData(FILE *tarfile, char *buf, int r, char *current_file)
static IncludeWal includewal
Definition: pg_basebackup.c:90
char * PQerrorMessage(const PGconn *conn)
Definition: fe-connect.c:6116
#define Z_DEFAULT_COMPRESSION
int pg_file_create_mode
Definition: file_perm.c:20
static bool verify_checksums
static int bgpipe[2]
static bool found_existing_pgdata
void fsync_pgdata(const char *pg_data, const char *progname, int serverVersion)
Definition: file_utils.c:58
uint32 TimeLineID
Definition: xlogdefs.h:45
int64 pg_time_t
Definition: pgtime.h:23
static void usage(void)
static bool found_tablespace_dirs
char * PQgetvalue(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3118
const char * PQparameterStatus(const PGconn *conn, const char *paramName)
Definition: fe-connect.c:6081
bool pg_str_endswith(const char *str, const char *end)
Definition: string.c:31
int pg_mkdir_p(char *path, int omode)
Definition: pgmkdirp.c:57
static TablespaceList tablespace_dirs
Definition: pg_basebackup.c:81
#define write(a, b, c)
Definition: win32.h:14
bool do_sync
Definition: receivelog.h:39
const char * get_progname(const char *argv0)
Definition: path.c:453
void termPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:128
uint64 read_tar_number(const char *s, int len)
Definition: tar.c:56
int getopt_long(int argc, char *const argv[], const char *optstring, const struct option *longopts, int *longindex)
Definition: getopt_long.c:57
static int32 parse_max_rate(char *src)
void appendPQExpBufferStr(PQExpBuffer str, const char *data)
Definition: pqexpbuffer.c:385
#define FALSE
Definition: ecpglib.h:39
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
static pg_time_t last_progress_report
Definition: pg_basebackup.c:95
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:575
char old_dir[MAXPGPATH]
Definition: pg_basebackup.c:48
static int tablespacecount
void _dosmaperr(unsigned long)
Definition: win32error.c:171
bool RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli, XLogRecPtr *startpos, char **db_name)
Definition: streamutil.c:417
#define VERBOSE_FILENAME_LENGTH
IncludeWal
Definition: pg_basebackup.c:72
void appendConnStrVal(PQExpBuffer buf, const char *str)
Definition: string_utils.c:551
void canonicalize_path(char *path)
Definition: path.c:254
char * sysidentifier
Definition: receivelog.h:34
static bool writerecoveryconf
Definition: pg_basebackup.c:92
static bool create_slot
Definition: pg_basebackup.c:99
void PQfinish(PGconn *conn)
Definition: fe-connect.c:3638
static int compresslevel
Definition: pg_basebackup.c:89
static bool checksum_failure
Definition: pg_basebackup.c:86
#define MemSet(start, val, len)
Definition: c.h:908
#define kill(pid, sig)
Definition: win32_port.h:437
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
XLogRecPtr startpos
Definition: receivelog.h:32
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
static char * escape_quotes(const char *src)
char * partial_suffix
Definition: receivelog.h:48
char new_dir[MAXPGPATH]
Definition: pg_basebackup.c:49
int PQserverVersion(const PGconn *conn)
Definition: fe-connect.c:6106
const char * progname
Definition: pg_standby.c:37
int PQntuples(const PGresult *res)
Definition: fe-exec.c:2724
#define PG_DIAG_SQLSTATE
Definition: postgres_ext.h:57
static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline, bool segment_finished)
static void GenerateRecoveryConf(PGconn *conn)
TimeLineID timeline
Definition: receivelog.h:33
static char * basedir
Definition: pg_basebackup.c:80
struct TablespaceListCell TablespaceListCell
WalWriteMethod * CreateWalDirectoryMethod(const char *basedir, int compression, bool sync)
Definition: walmethods.c:347
ExecStatusType PQresultStatus(const PGresult *res)
Definition: fe-exec.c:2647
int PQgetCopyData(PGconn *conn, char **buffer, int async)
Definition: fe-exec.c:2428
signed int int32
Definition: c.h:313
int main(int argc, char **argv)
static void BaseBackup(void)
#define pgoff_t
Definition: win32_port.h:206
int PQsendQuery(PGconn *conn, const char *query)
Definition: fe-exec.c:1183
void destroyPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:113
char * escape_single_quotes_ascii(const char *src)
Definition: quotes.c:33
#define required_argument
Definition: getopt_long.h:25
int optind
Definition: getopt.c:51
#define MAX_RATE_UPPER
Definition: basebackup.h:21
char * connection_string
Definition: streamutil.c:47
struct TablespaceList TablespaceList
bool RetrieveWalSegSize(PGconn *conn)
Definition: streamutil.c:283
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
PGconn * conn
Definition: streamutil.c:55
#define MAXPGPATH
void appendPQExpBuffer(PQExpBuffer str, const char *fmt,...)
Definition: pqexpbuffer.c:262
#define MINIMUM_VERSION_FOR_PG_WAL
Definition: pg_basebackup.c:62
static bool found_existing_xlogdir
static uint64 totaldone
char * replication_slot
Definition: receivelog.h:49
bool mark_done
Definition: receivelog.h:38
static bool success
static int32 maxrate
Definition: pg_basebackup.c:96
char * c
static char * buf
Definition: pg_test_fsync.c:67
#define symlink(oldpath, newpath)
Definition: win32_port.h:232
static int has_xlogendptr
static bool made_new_pgdata
void PQconninfoFree(PQconninfoOption *connOptions)
Definition: fe-connect.c:5965
#define is_absolute_path(filename)
Definition: port.h:86
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
size_t PQescapeStringConn(PGconn *conn, char *to, const char *from, size_t length, int *error)
Definition: fe-exec.c:3365
#define select(n, r, w, e, timeout)
Definition: win32_port.h:447
unsigned int uint32
Definition: c.h:325
PQconninfoOption * PQconninfo(PGconn *conn)
Definition: fe-connect.c:5924
static uint64 totalsize
static const char * get_tablespace_mapping(const char *dir)
static void progress_report(int tablespacenum, const char *filename, bool force)
XLogRecPtr startptr
stream_stop_callback stream_stop
Definition: receivelog.h:42
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:43
WalWriteMethod * walmethod
Definition: receivelog.h:47
static int verbose
Definition: pg_basebackup.c:88
bool rmtree(const char *path, bool rmtopdir)
Definition: rmtree.c:36
static int standby_message_timeout
Definition: pg_basebackup.c:94
int pg_dir_create_mode
Definition: file_perm.c:19
static bool showprogress
Definition: pg_basebackup.c:87
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:112
int dbgetpassword
Definition: streamutil.c:52
#define no_argument
Definition: getopt_long.h:24
void FreeWalTarMethod(void)
Definition: walmethods.c:980
#define ngettext(s, p, n)
Definition: c.h:1022
#define PG_TEXTDOMAIN(domain)
Definition: c.h:1054
static void WriteRecoveryConf(void)
#define WIFEXITED(w)
Definition: win32_port.h:147
static bool temp_replication_slot
Definition: pg_basebackup.c:98
int PQbackendPID(const PGconn *conn)
Definition: fe-connect.c:6142
#define PGINVALID_SOCKET
Definition: port.h:33
static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
struct TablespaceListCell * next
Definition: pg_basebackup.c:47
PQExpBuffer createPQExpBuffer(void)
Definition: pqexpbuffer.c:71
static char * label
Definition: pg_basebackup.c:84
#define XLogSegmentOffset(xlogptr, wal_segsz_bytes)
void appendPQExpBufferChar(PQExpBuffer str, char ch)
Definition: pqexpbuffer.c:396
StringInfo copybuf
Definition: tablesync.c:114
static void StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
TablespaceListCell * tail
Definition: pg_basebackup.c:55
char * dbport
Definition: streamutil.c:50
static void cleanup_directories_atexit(void)
void PQclear(PGresult *res)
Definition: fe-exec.c:671
#define free(a)
Definition: header.h:65
static bool in_log_streamer
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
PGconn * GetConnection(UserMapping *user, bool will_prep_stmt)
Definition: connection.c:107
#define PQExpBufferDataBroken(buf)
Definition: pqexpbuffer.h:67
char * PQresultErrorField(const PGresult *res, int fieldcode)
Definition: fe-exec.c:2709
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:699
WalWriteMethod * CreateWalTarMethod(const char *tarbase, int compression, bool sync)
Definition: walmethods.c:949
static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
char * dbhost
Definition: streamutil.c:48
bool ReceiveXlogStream(PGconn *conn, StreamCtl *stream)
Definition: receivelog.c:447
static char * xlog_dir
Definition: pg_basebackup.c:82
#define PQExpBufferBroken(str)
Definition: pqexpbuffer.h:59
static XLogRecPtr xlogendptr
static XLogRecPtr startpos
static bool no_slot
TablespaceListCell * head
Definition: pg_basebackup.c:54
bool synchronous
Definition: receivelog.h:37
void pg_free(void *ptr)
Definition: fe_memutils.c:105
bool(* finish)(void)
Definition: walmethods.h:75
pgsocket stop_socket
Definition: receivelog.h:44
static void header(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:208
#define INT64_FORMAT
Definition: c.h:367
static bool made_new_xlogdir
static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
Definition: walsender.c:836
#define WRITE_TAR_DATA(buf, sz)
#define pg_attribute_noreturn()
Definition: c.h:147
static void disconnect_and_exit(int code) pg_attribute_noreturn()
int WalSegSz
Definition: pg_standby.c:39
static bool fastcheckpoint
Definition: pg_basebackup.c:91
static void tablespace_list_append(const char *arg)
static bool made_tablespace_dirs
static char * filename
Definition: pg_dumpall.c:87
void set_pglocale_pgservice(const char *argv0, const char *app)
Definition: exec.c:550
int errmsg(const char *fmt,...)
Definition: elog.c:797
static pid_t bgchild
char * dbuser
Definition: streamutil.c:49
char * optarg
Definition: getopt.c:53
int pg_check_dir(const char *dir)
Definition: pgcheckdir.c:31
int i
const char * strerror(int errnum)
Definition: strerror.c:19
int standby_message_timeout
Definition: receivelog.h:36
void * arg
static bool do_sync
Definition: pg_basebackup.c:93
static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
static char format
Definition: pg_basebackup.c:83
#define mkdir(a, b)
Definition: win32_port.h:58
static char * replication_slot
Definition: pg_basebackup.c:97
static void static void status(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:225
static int LogStreamerMain(logstreamer_param *param)
int PQgetisnull(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3143
#define _(x)
Definition: elog.c:84
void PQfreemem(void *ptr)
Definition: fe-exec.c:3251
int pg_mode_mask
Definition: file_perm.c:26
Datum now(PG_FUNCTION_ARGS)
Definition: timestamp.c:1534
PGresult * PQgetResult(PGconn *conn)
Definition: fe-exec.c:1753
#define MINIMUM_VERSION_FOR_TEMP_SLOTS
Definition: pg_basebackup.c:67
#define read(a, b, c)
Definition: win32.h:13
void initPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:89
void FreeWalDirectoryMethod(void)
Definition: walmethods.c:371
bool CheckServerVersionForStreaming(PGconn *conn)
Definition: receivelog.c:367
#define WEXITSTATUS(w)
Definition: win32_port.h:149
char xlog[MAXPGPATH]
static bool noclean
Definition: pg_basebackup.c:85