PostgreSQL Source Code  git master
pg_basebackup.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_basebackup.c - receive a base backup using streaming replication protocol
4  *
5  * Author: Magnus Hagander <magnus@hagander.net>
6  *
7  * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
8  *
9  * IDENTIFICATION
10  * src/bin/pg_basebackup/pg_basebackup.c
11  *-------------------------------------------------------------------------
12  */
13 
14 #include "postgres_fe.h"
15 
16 #include <unistd.h>
17 #include <dirent.h>
18 #include <sys/stat.h>
19 #include <sys/wait.h>
20 #include <signal.h>
21 #include <time.h>
22 #ifdef HAVE_SYS_SELECT_H
23 #include <sys/select.h>
24 #endif
25 #ifdef HAVE_LIBZ
26 #include <zlib.h>
27 #endif
28 
29 #include "access/xlog_internal.h"
30 #include "common/file_perm.h"
31 #include "common/file_utils.h"
32 #include "common/logging.h"
33 #include "common/string.h"
34 #include "fe_utils/recovery_gen.h"
35 #include "fe_utils/string_utils.h"
36 #include "getopt_long.h"
37 #include "libpq-fe.h"
38 #include "pqexpbuffer.h"
39 #include "pgtar.h"
40 #include "pgtime.h"
41 #include "receivelog.h"
42 #include "replication/basebackup.h"
43 #include "streamutil.h"
44 
45 #define ERRCODE_DATA_CORRUPTED "XX001"
46 
47 typedef struct TablespaceListCell
48 {
53 
54 typedef struct TablespaceList
55 {
59 
60 /*
61  * pg_xlog has been renamed to pg_wal in version 10. This version number
62  * should be compared with PQserverVersion().
63  */
64 #define MINIMUM_VERSION_FOR_PG_WAL 100000
65 
66 /*
67  * Temporary replication slots are supported from version 10.
68  */
69 #define MINIMUM_VERSION_FOR_TEMP_SLOTS 100000
70 
71 /*
72  * Different ways to include WAL
73  */
74 typedef enum
75 {
79 } IncludeWal;
80 
81 /* Global options */
82 static char *basedir = NULL;
83 static TablespaceList tablespace_dirs = {NULL, NULL};
84 static char *xlog_dir = NULL;
85 static char format = 'p'; /* p(lain)/t(ar) */
86 static char *label = "pg_basebackup base backup";
87 static bool noclean = false;
88 static bool checksum_failure = false;
89 static bool showprogress = false;
90 static int verbose = 0;
91 static int compresslevel = 0;
93 static bool fastcheckpoint = false;
94 static bool writerecoveryconf = false;
95 static bool do_sync = true;
96 static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
98 static int32 maxrate = 0; /* no limit by default */
99 static char *replication_slot = NULL;
100 static bool temp_replication_slot = true;
101 static bool create_slot = false;
102 static bool no_slot = false;
103 static bool verify_checksums = true;
104 
105 static bool success = false;
106 static bool made_new_pgdata = false;
107 static bool found_existing_pgdata = false;
108 static bool made_new_xlogdir = false;
109 static bool found_existing_xlogdir = false;
110 static bool made_tablespace_dirs = false;
111 static bool found_tablespace_dirs = false;
112 
113 /* Progress counters */
114 static uint64 totalsize_kb;
115 static uint64 totaldone;
116 static int tablespacecount;
117 
118 /* Pipe to communicate with background wal receiver process */
119 #ifndef WIN32
120 static int bgpipe[2] = {-1, -1};
121 #endif
122 
123 /* Handle to child process */
124 static pid_t bgchild = -1;
125 static bool in_log_streamer = false;
126 
127 /* End position for xlog streaming, empty string if unknown yet */
129 
130 #ifndef WIN32
131 static int has_xlogendptr = 0;
132 #else
133 static volatile LONG has_xlogendptr = 0;
134 #endif
135 
136 /* Contents of configuration file to be generated */
138 
139 /* Function headers */
140 static void usage(void);
141 static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found);
142 static void progress_report(int tablespacenum, const char *filename, bool force);
143 
144 static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum);
145 static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum);
146 static void BaseBackup(void);
147 
148 static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline,
149  bool segment_finished);
150 
151 static const char *get_tablespace_mapping(const char *dir);
152 static void tablespace_list_append(const char *arg);
153 
154 
155 static void
157 {
158  if (success || in_log_streamer)
159  return;
160 
161  if (!noclean && !checksum_failure)
162  {
163  if (made_new_pgdata)
164  {
165  pg_log_info("removing data directory \"%s\"", basedir);
166  if (!rmtree(basedir, true))
167  pg_log_error("failed to remove data directory");
168  }
169  else if (found_existing_pgdata)
170  {
171  pg_log_info("removing contents of data directory \"%s\"", basedir);
172  if (!rmtree(basedir, false))
173  pg_log_error("failed to remove contents of data directory");
174  }
175 
176  if (made_new_xlogdir)
177  {
178  pg_log_info("removing WAL directory \"%s\"", xlog_dir);
179  if (!rmtree(xlog_dir, true))
180  pg_log_error("failed to remove WAL directory");
181  }
182  else if (found_existing_xlogdir)
183  {
184  pg_log_info("removing contents of WAL directory \"%s\"", xlog_dir);
185  if (!rmtree(xlog_dir, false))
186  pg_log_error("failed to remove contents of WAL directory");
187  }
188  }
189  else
190  {
192  pg_log_info("data directory \"%s\" not removed at user's request", basedir);
193 
195  pg_log_info("WAL directory \"%s\" not removed at user's request", xlog_dir);
196  }
197 
199  pg_log_info("changes to tablespace directories will not be undone");
200 }
201 
202 static void
204 {
205  if (conn != NULL)
206  PQfinish(conn);
207 }
208 
209 #ifndef WIN32
210 /*
211  * On windows, our background thread dies along with the process. But on
212  * Unix, if we have started a subprocess, we want to kill it off so it
213  * doesn't remain running trying to stream data.
214  */
215 static void
217 {
218  if (bgchild > 0)
219  kill(bgchild, SIGTERM);
220 }
221 #endif
222 
223 /*
224  * Split argument into old_dir and new_dir and append to tablespace mapping
225  * list.
226  */
227 static void
229 {
231  char *dst;
232  char *dst_ptr;
233  const char *arg_ptr;
234 
235  dst_ptr = dst = cell->old_dir;
236  for (arg_ptr = arg; *arg_ptr; arg_ptr++)
237  {
238  if (dst_ptr - dst >= MAXPGPATH)
239  {
240  pg_log_error("directory name too long");
241  exit(1);
242  }
243 
244  if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
245  ; /* skip backslash escaping = */
246  else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
247  {
248  if (*cell->new_dir)
249  {
250  pg_log_error("multiple \"=\" signs in tablespace mapping");
251  exit(1);
252  }
253  else
254  dst = dst_ptr = cell->new_dir;
255  }
256  else
257  *dst_ptr++ = *arg_ptr;
258  }
259 
260  if (!*cell->old_dir || !*cell->new_dir)
261  {
262  pg_log_error("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
263  exit(1);
264  }
265 
266  /*
267  * This check isn't absolutely necessary. But all tablespaces are created
268  * with absolute directories, so specifying a non-absolute path here would
269  * just never match, possibly confusing users. It's also good to be
270  * consistent with the new_dir check.
271  */
272  if (!is_absolute_path(cell->old_dir))
273  {
274  pg_log_error("old directory is not an absolute path in tablespace mapping: %s",
275  cell->old_dir);
276  exit(1);
277  }
278 
279  if (!is_absolute_path(cell->new_dir))
280  {
281  pg_log_error("new directory is not an absolute path in tablespace mapping: %s",
282  cell->new_dir);
283  exit(1);
284  }
285 
286  /*
287  * Comparisons done with these values should involve similarly
288  * canonicalized path values. This is particularly sensitive on Windows
289  * where path values may not necessarily use Unix slashes.
290  */
291  canonicalize_path(cell->old_dir);
292  canonicalize_path(cell->new_dir);
293 
294  if (tablespace_dirs.tail)
295  tablespace_dirs.tail->next = cell;
296  else
297  tablespace_dirs.head = cell;
298  tablespace_dirs.tail = cell;
299 }
300 
301 
302 #ifdef HAVE_LIBZ
303 static const char *
304 get_gz_error(gzFile gzf)
305 {
306  int errnum;
307  const char *errmsg;
308 
309  errmsg = gzerror(gzf, &errnum);
310  if (errnum == Z_ERRNO)
311  return strerror(errno);
312  else
313  return errmsg;
314 }
315 #endif
316 
317 static void
318 usage(void)
319 {
320  printf(_("%s takes a base backup of a running PostgreSQL server.\n\n"),
321  progname);
322  printf(_("Usage:\n"));
323  printf(_(" %s [OPTION]...\n"), progname);
324  printf(_("\nOptions controlling the output:\n"));
325  printf(_(" -D, --pgdata=DIRECTORY receive base backup into directory\n"));
326  printf(_(" -F, --format=p|t output format (plain (default), tar)\n"));
327  printf(_(" -r, --max-rate=RATE maximum transfer rate to transfer data directory\n"
328  " (in kB/s, or use suffix \"k\" or \"M\")\n"));
329  printf(_(" -R, --write-recovery-conf\n"
330  " write configuration for replication\n"));
331  printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
332  " relocate tablespace in OLDDIR to NEWDIR\n"));
333  printf(_(" --waldir=WALDIR location for the write-ahead log directory\n"));
334  printf(_(" -X, --wal-method=none|fetch|stream\n"
335  " include required WAL files with specified method\n"));
336  printf(_(" -z, --gzip compress tar output\n"));
337  printf(_(" -Z, --compress=0-9 compress tar output with given compression level\n"));
338  printf(_("\nGeneral options:\n"));
339  printf(_(" -c, --checkpoint=fast|spread\n"
340  " set fast or spread checkpointing\n"));
341  printf(_(" -C, --create-slot create replication slot\n"));
342  printf(_(" -l, --label=LABEL set backup label\n"));
343  printf(_(" -n, --no-clean do not clean up after errors\n"));
344  printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
345  printf(_(" -P, --progress show progress information\n"));
346  printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
347  printf(_(" -v, --verbose output verbose messages\n"));
348  printf(_(" -V, --version output version information, then exit\n"));
349  printf(_(" --no-slot prevent creation of temporary replication slot\n"));
350  printf(_(" --no-verify-checksums\n"
351  " do not verify checksums\n"));
352  printf(_(" -?, --help show this help, then exit\n"));
353  printf(_("\nConnection options:\n"));
354  printf(_(" -d, --dbname=CONNSTR connection string\n"));
355  printf(_(" -h, --host=HOSTNAME database server host or socket directory\n"));
356  printf(_(" -p, --port=PORT database server port number\n"));
357  printf(_(" -s, --status-interval=INTERVAL\n"
358  " time between status packets sent to server (in seconds)\n"));
359  printf(_(" -U, --username=NAME connect as specified database user\n"));
360  printf(_(" -w, --no-password never prompt for password\n"));
361  printf(_(" -W, --password force password prompt (should happen automatically)\n"));
362  printf(_("\nReport bugs to <pgsql-bugs@lists.postgresql.org>.\n"));
363 }
364 
365 
366 /*
367  * Called in the background process every time data is received.
368  * On Unix, we check to see if there is any data on our pipe
369  * (which would mean we have a stop position), and if it is, check if
370  * it is time to stop.
371  * On Windows, we are in a single process, so we can just check if it's
372  * time to stop.
373  */
374 static bool
376  bool segment_finished)
377 {
378  if (!has_xlogendptr)
379  {
380 #ifndef WIN32
381  fd_set fds;
382  struct timeval tv;
383  int r;
384 
385  /*
386  * Don't have the end pointer yet - check our pipe to see if it has
387  * been sent yet.
388  */
389  FD_ZERO(&fds);
390  FD_SET(bgpipe[0], &fds);
391 
392  MemSet(&tv, 0, sizeof(tv));
393 
394  r = select(bgpipe[0] + 1, &fds, NULL, NULL, &tv);
395  if (r == 1)
396  {
397  char xlogend[64];
398  uint32 hi,
399  lo;
400 
401  MemSet(xlogend, 0, sizeof(xlogend));
402  r = read(bgpipe[0], xlogend, sizeof(xlogend) - 1);
403  if (r < 0)
404  {
405  pg_log_error("could not read from ready pipe: %m");
406  exit(1);
407  }
408 
409  if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
410  {
411  pg_log_error("could not parse write-ahead log location \"%s\"",
412  xlogend);
413  exit(1);
414  }
415  xlogendptr = ((uint64) hi) << 32 | lo;
416  has_xlogendptr = 1;
417 
418  /*
419  * Fall through to check if we've reached the point further
420  * already.
421  */
422  }
423  else
424  {
425  /*
426  * No data received on the pipe means we don't know the end
427  * position yet - so just say it's not time to stop yet.
428  */
429  return false;
430  }
431 #else
432 
433  /*
434  * On win32, has_xlogendptr is set by the main thread, so if it's not
435  * set here, we just go back and wait until it shows up.
436  */
437  return false;
438 #endif
439  }
440 
441  /*
442  * At this point we have an end pointer, so compare it to the current
443  * position to figure out if it's time to stop.
444  */
445  if (segendpos >= xlogendptr)
446  return true;
447 
448  /*
449  * Have end pointer, but haven't reached it yet - so tell the caller to
450  * keep streaming.
451  */
452  return false;
453 }
454 
455 typedef struct
456 {
459  char xlog[MAXPGPATH]; /* directory or tarfile depending on mode */
461  int timeline;
463 
464 static int
466 {
467  StreamCtl stream;
468 
469  in_log_streamer = true;
470 
471  MemSet(&stream, 0, sizeof(stream));
472  stream.startpos = param->startptr;
473  stream.timeline = param->timeline;
474  stream.sysidentifier = param->sysidentifier;
476 #ifndef WIN32
477  stream.stop_socket = bgpipe[0];
478 #else
479  stream.stop_socket = PGINVALID_SOCKET;
480 #endif
482  stream.synchronous = false;
483  /* fsync happens at the end of pg_basebackup for all data */
484  stream.do_sync = false;
485  stream.mark_done = true;
486  stream.partial_suffix = NULL;
488 
489  if (format == 'p')
490  stream.walmethod = CreateWalDirectoryMethod(param->xlog, 0,
491  stream.do_sync);
492  else
494  stream.do_sync);
495 
496  if (!ReceiveXlogStream(param->bgconn, &stream))
497 
498  /*
499  * Any errors will already have been reported in the function process,
500  * but we need to tell the parent that we didn't shutdown in a nice
501  * way.
502  */
503  return 1;
504 
505  if (!stream.walmethod->finish())
506  {
507  pg_log_error("could not finish writing WAL files: %m");
508  return 1;
509  }
510 
511  PQfinish(param->bgconn);
512 
513  if (format == 'p')
515  else
517  pg_free(stream.walmethod);
518 
519  return 0;
520 }
521 
522 /*
523  * Initiate background process for receiving xlog during the backup.
524  * The background stream will use its own database connection so we can
525  * stream the logfile in parallel with the backups.
526  */
527 static void
528 StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
529 {
530  logstreamer_param *param;
531  uint32 hi,
532  lo;
533  char statusdir[MAXPGPATH];
534 
535  param = pg_malloc0(sizeof(logstreamer_param));
536  param->timeline = timeline;
537  param->sysidentifier = sysidentifier;
538 
539  /* Convert the starting position */
540  if (sscanf(startpos, "%X/%X", &hi, &lo) != 2)
541  {
542  pg_log_error("could not parse write-ahead log location \"%s\"",
543  startpos);
544  exit(1);
545  }
546  param->startptr = ((uint64) hi) << 32 | lo;
547  /* Round off to even segment position */
548  param->startptr -= XLogSegmentOffset(param->startptr, WalSegSz);
549 
550 #ifndef WIN32
551  /* Create our background pipe */
552  if (pipe(bgpipe) < 0)
553  {
554  pg_log_error("could not create pipe for background process: %m");
555  exit(1);
556  }
557 #endif
558 
559  /* Get a second connection */
560  param->bgconn = GetConnection();
561  if (!param->bgconn)
562  /* Error message already written in GetConnection() */
563  exit(1);
564 
565  /* In post-10 cluster, pg_xlog has been renamed to pg_wal */
566  snprintf(param->xlog, sizeof(param->xlog), "%s/%s",
567  basedir,
569  "pg_xlog" : "pg_wal");
570 
571  /* Temporary replication slots are only supported in 10 and newer */
573  temp_replication_slot = false;
574 
575  /*
576  * Create replication slot if requested
577  */
579  replication_slot = psprintf("pg_basebackup_%d", (int) PQbackendPID(param->bgconn));
581  {
582  if (!CreateReplicationSlot(param->bgconn, replication_slot, NULL,
583  temp_replication_slot, true, true, false))
584  exit(1);
585 
586  if (verbose)
587  {
589  pg_log_info("created temporary replication slot \"%s\"",
591  else
592  pg_log_info("created replication slot \"%s\"",
594  }
595  }
596 
597  if (format == 'p')
598  {
599  /*
600  * Create pg_wal/archive_status or pg_xlog/archive_status (and thus
601  * pg_wal or pg_xlog) depending on the target server so we can write
602  * to basedir/pg_wal or basedir/pg_xlog as the directory entry in the
603  * tar file may arrive later.
604  */
605  snprintf(statusdir, sizeof(statusdir), "%s/%s/archive_status",
606  basedir,
608  "pg_xlog" : "pg_wal");
609 
610  if (pg_mkdir_p(statusdir, pg_dir_create_mode) != 0 && errno != EEXIST)
611  {
612  pg_log_error("could not create directory \"%s\": %m", statusdir);
613  exit(1);
614  }
615  }
616 
617  /*
618  * Start a child process and tell it to start streaming. On Unix, this is
619  * a fork(). On Windows, we create a thread.
620  */
621 #ifndef WIN32
622  bgchild = fork();
623  if (bgchild == 0)
624  {
625  /* in child process */
626  exit(LogStreamerMain(param));
627  }
628  else if (bgchild < 0)
629  {
630  pg_log_error("could not create background process: %m");
631  exit(1);
632  }
633 
634  /*
635  * Else we are in the parent process and all is well.
636  */
637  atexit(kill_bgchild_atexit);
638 #else /* WIN32 */
639  bgchild = _beginthreadex(NULL, 0, (void *) LogStreamerMain, param, 0, NULL);
640  if (bgchild == 0)
641  {
642  pg_log_error("could not create background thread: %m");
643  exit(1);
644  }
645 #endif
646 }
647 
648 /*
649  * Verify that the given directory exists and is empty. If it does not
650  * exist, it is created. If it exists but is not empty, an error will
651  * be given and the process ended.
652  */
653 static void
654 verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
655 {
656  switch (pg_check_dir(dirname))
657  {
658  case 0:
659 
660  /*
661  * Does not exist, so create
662  */
663  if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
664  {
665  pg_log_error("could not create directory \"%s\": %m", dirname);
666  exit(1);
667  }
668  if (created)
669  *created = true;
670  return;
671  case 1:
672 
673  /*
674  * Exists, empty
675  */
676  if (found)
677  *found = true;
678  return;
679  case 2:
680  case 3:
681  case 4:
682 
683  /*
684  * Exists, not empty
685  */
686  pg_log_error("directory \"%s\" exists but is not empty", dirname);
687  exit(1);
688  case -1:
689 
690  /*
691  * Access problem
692  */
693  pg_log_error("could not access directory \"%s\": %m", dirname);
694  exit(1);
695  }
696 }
697 
698 
699 /*
700  * Print a progress report based on the global variables. If verbose output
701  * is enabled, also print the current file name.
702  *
703  * Progress report is written at maximum once per second, unless the
704  * force parameter is set to true.
705  */
706 static void
707 progress_report(int tablespacenum, const char *filename, bool force)
708 {
709  int percent;
710  char totaldone_str[32];
711  char totalsize_str[32];
712  pg_time_t now;
713 
714  if (!showprogress)
715  return;
716 
717  now = time(NULL);
718  if (now == last_progress_report && !force)
719  return; /* Max once per second */
720 
722  percent = totalsize_kb ? (int) ((totaldone / 1024) * 100 / totalsize_kb) : 0;
723 
724  /*
725  * Avoid overflowing past 100% or the full size. This may make the total
726  * size number change as we approach the end of the backup (the estimate
727  * will always be wrong if WAL is included), but that's better than having
728  * the done column be bigger than the total.
729  */
730  if (percent > 100)
731  percent = 100;
732  if (totaldone / 1024 > totalsize_kb)
733  totalsize_kb = totaldone / 1024;
734 
735  /*
736  * Separate step to keep platform-dependent format code out of
737  * translatable strings. And we only test for INT64_FORMAT availability
738  * in snprintf, not fprintf.
739  */
740  snprintf(totaldone_str, sizeof(totaldone_str), INT64_FORMAT,
741  totaldone / 1024);
742  snprintf(totalsize_str, sizeof(totalsize_str), INT64_FORMAT, totalsize_kb);
743 
744 #define VERBOSE_FILENAME_LENGTH 35
745  if (verbose)
746  {
747  if (!filename)
748 
749  /*
750  * No filename given, so clear the status line (used for last
751  * call)
752  */
753  fprintf(stderr,
754  ngettext("%*s/%s kB (100%%), %d/%d tablespace %*s",
755  "%*s/%s kB (100%%), %d/%d tablespaces %*s",
757  (int) strlen(totalsize_str),
758  totaldone_str, totalsize_str,
759  tablespacenum, tablespacecount,
760  VERBOSE_FILENAME_LENGTH + 5, "");
761  else
762  {
763  bool truncate = (strlen(filename) > VERBOSE_FILENAME_LENGTH);
764 
765  fprintf(stderr,
766  ngettext("%*s/%s kB (%d%%), %d/%d tablespace (%s%-*.*s)",
767  "%*s/%s kB (%d%%), %d/%d tablespaces (%s%-*.*s)",
769  (int) strlen(totalsize_str),
770  totaldone_str, totalsize_str, percent,
771  tablespacenum, tablespacecount,
772  /* Prefix with "..." if we do leading truncation */
773  truncate ? "..." : "",
776  /* Truncate filename at beginning if it's too long */
777  truncate ? filename + strlen(filename) - VERBOSE_FILENAME_LENGTH + 3 : filename);
778  }
779  }
780  else
781  fprintf(stderr,
782  ngettext("%*s/%s kB (%d%%), %d/%d tablespace",
783  "%*s/%s kB (%d%%), %d/%d tablespaces",
785  (int) strlen(totalsize_str),
786  totaldone_str, totalsize_str, percent,
787  tablespacenum, tablespacecount);
788 
789  if (isatty(fileno(stderr)))
790  fprintf(stderr, "\r");
791  else
792  fprintf(stderr, "\n");
793 }
794 
795 static int32
796 parse_max_rate(char *src)
797 {
798  double result;
799  char *after_num;
800  char *suffix = NULL;
801 
802  errno = 0;
803  result = strtod(src, &after_num);
804  if (src == after_num)
805  {
806  pg_log_error("transfer rate \"%s\" is not a valid value", src);
807  exit(1);
808  }
809  if (errno != 0)
810  {
811  pg_log_error("invalid transfer rate \"%s\": %m", src);
812  exit(1);
813  }
814 
815  if (result <= 0)
816  {
817  /*
818  * Reject obviously wrong values here.
819  */
820  pg_log_error("transfer rate must be greater than zero");
821  exit(1);
822  }
823 
824  /*
825  * Evaluate suffix, after skipping over possible whitespace. Lack of
826  * suffix means kilobytes.
827  */
828  while (*after_num != '\0' && isspace((unsigned char) *after_num))
829  after_num++;
830 
831  if (*after_num != '\0')
832  {
833  suffix = after_num;
834  if (*after_num == 'k')
835  {
836  /* kilobyte is the expected unit. */
837  after_num++;
838  }
839  else if (*after_num == 'M')
840  {
841  after_num++;
842  result *= 1024.0;
843  }
844  }
845 
846  /* The rest can only consist of white space. */
847  while (*after_num != '\0' && isspace((unsigned char) *after_num))
848  after_num++;
849 
850  if (*after_num != '\0')
851  {
852  pg_log_error("invalid --max-rate unit: \"%s\"", suffix);
853  exit(1);
854  }
855 
856  /* Valid integer? */
857  if ((uint64) result != (uint64) ((uint32) result))
858  {
859  pg_log_error("transfer rate \"%s\" exceeds integer range", src);
860  exit(1);
861  }
862 
863  /*
864  * The range is checked on the server side too, but avoid the server
865  * connection if a nonsensical value was passed.
866  */
867  if (result < MAX_RATE_LOWER || result > MAX_RATE_UPPER)
868  {
869  pg_log_error("transfer rate \"%s\" is out of range", src);
870  exit(1);
871  }
872 
873  return (int32) result;
874 }
875 
876 /*
877  * Write a piece of tar data
878  */
879 static void
881 #ifdef HAVE_LIBZ
882  gzFile ztarfile,
883 #endif
884  FILE *tarfile, char *buf, int r, char *current_file)
885 {
886 #ifdef HAVE_LIBZ
887  if (ztarfile != NULL)
888  {
889  if (gzwrite(ztarfile, buf, r) != r)
890  {
891  pg_log_error("could not write to compressed file \"%s\": %s",
892  current_file, get_gz_error(ztarfile));
893  exit(1);
894  }
895  }
896  else
897 #endif
898  {
899  if (fwrite(buf, r, 1, tarfile) != 1)
900  {
901  pg_log_error("could not write to file \"%s\": %m", current_file);
902  exit(1);
903  }
904  }
905 }
906 
907 #ifdef HAVE_LIBZ
908 #define WRITE_TAR_DATA(buf, sz) writeTarData(ztarfile, tarfile, buf, sz, filename)
909 #else
910 #define WRITE_TAR_DATA(buf, sz) writeTarData(tarfile, buf, sz, filename)
911 #endif
912 
913 /*
914  * Receive a tar format file from the connection to the server, and write
915  * the data from this file directly into a tar file. If compression is
916  * enabled, the data will be compressed while written to the file.
917  *
918  * The file will be named base.tar[.gz] if it's for the main data directory
919  * or <tablespaceoid>.tar[.gz] if it's for another tablespace.
920  *
921  * No attempt to inspect or validate the contents of the file is done.
922  */
923 static void
924 ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
925 {
926  char filename[MAXPGPATH];
927  char *copybuf = NULL;
928  FILE *tarfile = NULL;
929  char tarhdr[512];
930  bool basetablespace = PQgetisnull(res, rownum, 0);
931  bool in_tarhdr = true;
932  bool skip_file = false;
933  bool is_recovery_guc_supported = true;
934  bool is_postgresql_auto_conf = false;
935  bool found_postgresql_auto_conf = false;
936  int file_padding_len = 0;
937  size_t tarhdrsz = 0;
938  pgoff_t filesz = 0;
939 
940 #ifdef HAVE_LIBZ
941  gzFile ztarfile = NULL;
942 #endif
943 
944  /* recovery.conf is integrated into postgresql.conf in 12 and newer */
946  is_recovery_guc_supported = false;
947 
948  if (basetablespace)
949  {
950  /*
951  * Base tablespaces
952  */
953  if (strcmp(basedir, "-") == 0)
954  {
955 #ifdef WIN32
956  _setmode(fileno(stdout), _O_BINARY);
957 #endif
958 
959 #ifdef HAVE_LIBZ
960  if (compresslevel != 0)
961  {
962  ztarfile = gzdopen(dup(fileno(stdout)), "wb");
963  if (gzsetparams(ztarfile, compresslevel,
964  Z_DEFAULT_STRATEGY) != Z_OK)
965  {
966  pg_log_error("could not set compression level %d: %s",
967  compresslevel, get_gz_error(ztarfile));
968  exit(1);
969  }
970  }
971  else
972 #endif
973  tarfile = stdout;
974  strcpy(filename, "-");
975  }
976  else
977  {
978 #ifdef HAVE_LIBZ
979  if (compresslevel != 0)
980  {
981  snprintf(filename, sizeof(filename), "%s/base.tar.gz", basedir);
982  ztarfile = gzopen(filename, "wb");
983  if (gzsetparams(ztarfile, compresslevel,
984  Z_DEFAULT_STRATEGY) != Z_OK)
985  {
986  pg_log_error("could not set compression level %d: %s",
987  compresslevel, get_gz_error(ztarfile));
988  exit(1);
989  }
990  }
991  else
992 #endif
993  {
994  snprintf(filename, sizeof(filename), "%s/base.tar", basedir);
995  tarfile = fopen(filename, "wb");
996  }
997  }
998  }
999  else
1000  {
1001  /*
1002  * Specific tablespace
1003  */
1004 #ifdef HAVE_LIBZ
1005  if (compresslevel != 0)
1006  {
1007  snprintf(filename, sizeof(filename), "%s/%s.tar.gz", basedir,
1008  PQgetvalue(res, rownum, 0));
1009  ztarfile = gzopen(filename, "wb");
1010  if (gzsetparams(ztarfile, compresslevel,
1011  Z_DEFAULT_STRATEGY) != Z_OK)
1012  {
1013  pg_log_error("could not set compression level %d: %s",
1014  compresslevel, get_gz_error(ztarfile));
1015  exit(1);
1016  }
1017  }
1018  else
1019 #endif
1020  {
1021  snprintf(filename, sizeof(filename), "%s/%s.tar", basedir,
1022  PQgetvalue(res, rownum, 0));
1023  tarfile = fopen(filename, "wb");
1024  }
1025  }
1026 
1027 #ifdef HAVE_LIBZ
1028  if (compresslevel != 0)
1029  {
1030  if (!ztarfile)
1031  {
1032  /* Compression is in use */
1033  pg_log_error("could not create compressed file \"%s\": %s",
1034  filename, get_gz_error(ztarfile));
1035  exit(1);
1036  }
1037  }
1038  else
1039 #endif
1040  {
1041  /* Either no zlib support, or zlib support but compresslevel = 0 */
1042  if (!tarfile)
1043  {
1044  pg_log_error("could not create file \"%s\": %m", filename);
1045  exit(1);
1046  }
1047  }
1048 
1049  /*
1050  * Get the COPY data stream
1051  */
1052  res = PQgetResult(conn);
1053  if (PQresultStatus(res) != PGRES_COPY_OUT)
1054  {
1055  pg_log_error("could not get COPY data stream: %s",
1056  PQerrorMessage(conn));
1057  exit(1);
1058  }
1059 
1060  while (1)
1061  {
1062  int r;
1063 
1064  if (copybuf != NULL)
1065  {
1066  PQfreemem(copybuf);
1067  copybuf = NULL;
1068  }
1069 
1070  r = PQgetCopyData(conn, &copybuf, 0);
1071  if (r == -1)
1072  {
1073  /*
1074  * End of chunk. If requested, and this is the base tablespace,
1075  * write configuration file into the tarfile. When done, close the
1076  * file (but not stdout).
1077  *
1078  * Also, write two completely empty blocks at the end of the tar
1079  * file, as required by some tar programs.
1080  */
1081  char zerobuf[1024];
1082 
1083  MemSet(zerobuf, 0, sizeof(zerobuf));
1084 
1085  if (basetablespace && writerecoveryconf)
1086  {
1087  char header[512];
1088 
1089  /*
1090  * If postgresql.auto.conf has not been found in the streamed
1091  * data, add recovery configuration to postgresql.auto.conf if
1092  * recovery parameters are GUCs. If the instance connected to
1093  * is older than 12, create recovery.conf with this data
1094  * otherwise.
1095  */
1096  if (!found_postgresql_auto_conf || !is_recovery_guc_supported)
1097  {
1098  int padding;
1099 
1100  tarCreateHeader(header,
1101  is_recovery_guc_supported ? "postgresql.auto.conf" : "recovery.conf",
1102  NULL,
1103  recoveryconfcontents->len,
1104  pg_file_create_mode, 04000, 02000,
1105  time(NULL));
1106 
1107  padding = ((recoveryconfcontents->len + 511) & ~511) - recoveryconfcontents->len;
1108 
1109  WRITE_TAR_DATA(header, sizeof(header));
1110  WRITE_TAR_DATA(recoveryconfcontents->data,
1111  recoveryconfcontents->len);
1112  if (padding)
1113  WRITE_TAR_DATA(zerobuf, padding);
1114  }
1115 
1116  /*
1117  * standby.signal is supported only if recovery parameters are
1118  * GUCs.
1119  */
1120  if (is_recovery_guc_supported)
1121  {
1122  tarCreateHeader(header, "standby.signal", NULL,
1123  0, /* zero-length file */
1124  pg_file_create_mode, 04000, 02000,
1125  time(NULL));
1126 
1127  WRITE_TAR_DATA(header, sizeof(header));
1128  WRITE_TAR_DATA(zerobuf, 511);
1129  }
1130  }
1131 
1132  /* 2 * 512 bytes empty data at end of file */
1133  WRITE_TAR_DATA(zerobuf, sizeof(zerobuf));
1134 
1135 #ifdef HAVE_LIBZ
1136  if (ztarfile != NULL)
1137  {
1138  if (gzclose(ztarfile) != 0)
1139  {
1140  pg_log_error("could not close compressed file \"%s\": %s",
1141  filename, get_gz_error(ztarfile));
1142  exit(1);
1143  }
1144  }
1145  else
1146 #endif
1147  {
1148  if (strcmp(basedir, "-") != 0)
1149  {
1150  if (fclose(tarfile) != 0)
1151  {
1152  pg_log_error("could not close file \"%s\": %m",
1153  filename);
1154  exit(1);
1155  }
1156  }
1157  }
1158 
1159  break;
1160  }
1161  else if (r == -2)
1162  {
1163  pg_log_error("could not read COPY data: %s",
1164  PQerrorMessage(conn));
1165  exit(1);
1166  }
1167 
1168  if (!writerecoveryconf || !basetablespace)
1169  {
1170  /*
1171  * When not writing config file, or when not working on the base
1172  * tablespace, we never have to look for an existing configuration
1173  * file in the stream.
1174  */
1175  WRITE_TAR_DATA(copybuf, r);
1176  }
1177  else
1178  {
1179  /*
1180  * Look for a config file in the existing tar stream. If it's
1181  * there, we must skip it so we can later overwrite it with our
1182  * own version of the file.
1183  *
1184  * To do this, we have to process the individual files inside the
1185  * TAR stream. The stream consists of a header and zero or more
1186  * chunks, all 512 bytes long. The stream from the server is
1187  * broken up into smaller pieces, so we have to track the size of
1188  * the files to find the next header structure.
1189  */
1190  int rr = r;
1191  int pos = 0;
1192 
1193  while (rr > 0)
1194  {
1195  if (in_tarhdr)
1196  {
1197  /*
1198  * We're currently reading a header structure inside the
1199  * TAR stream, i.e. the file metadata.
1200  */
1201  if (tarhdrsz < 512)
1202  {
1203  /*
1204  * Copy the header structure into tarhdr in case the
1205  * header is not aligned to 512 bytes or it's not
1206  * returned in whole by the last PQgetCopyData call.
1207  */
1208  int hdrleft;
1209  int bytes2copy;
1210 
1211  hdrleft = 512 - tarhdrsz;
1212  bytes2copy = (rr > hdrleft ? hdrleft : rr);
1213 
1214  memcpy(&tarhdr[tarhdrsz], copybuf + pos, bytes2copy);
1215 
1216  rr -= bytes2copy;
1217  pos += bytes2copy;
1218  tarhdrsz += bytes2copy;
1219  }
1220  else
1221  {
1222  /*
1223  * We have the complete header structure in tarhdr,
1224  * look at the file metadata: we may want append
1225  * recovery info into postgresql.auto.conf and skip
1226  * standby.signal file if recovery parameters are
1227  * integrated as GUCs, and recovery.conf otherwise. In
1228  * both cases we must calculate tar padding.
1229  */
1230  if (is_recovery_guc_supported)
1231  {
1232  skip_file = (strcmp(&tarhdr[0], "standby.signal") == 0);
1233  is_postgresql_auto_conf = (strcmp(&tarhdr[0], "postgresql.auto.conf") == 0);
1234  }
1235  else
1236  skip_file = (strcmp(&tarhdr[0], "recovery.conf") == 0);
1237 
1238  filesz = read_tar_number(&tarhdr[124], 12);
1239  file_padding_len = ((filesz + 511) & ~511) - filesz;
1240 
1241  if (is_recovery_guc_supported &&
1242  is_postgresql_auto_conf &&
1244  {
1245  /* replace tar header */
1246  char header[512];
1247 
1248  tarCreateHeader(header, "postgresql.auto.conf", NULL,
1249  filesz + recoveryconfcontents->len,
1250  pg_file_create_mode, 04000, 02000,
1251  time(NULL));
1252 
1253  WRITE_TAR_DATA(header, sizeof(header));
1254  }
1255  else
1256  {
1257  /* copy stream with padding */
1258  filesz += file_padding_len;
1259 
1260  if (!skip_file)
1261  {
1262  /*
1263  * If we're not skipping the file, write the
1264  * tar header unmodified.
1265  */
1266  WRITE_TAR_DATA(tarhdr, 512);
1267  }
1268  }
1269 
1270  /* Next part is the file, not the header */
1271  in_tarhdr = false;
1272  }
1273  }
1274  else
1275  {
1276  /*
1277  * We're processing a file's contents.
1278  */
1279  if (filesz > 0)
1280  {
1281  /*
1282  * We still have data to read (and possibly write).
1283  */
1284  int bytes2write;
1285 
1286  bytes2write = (filesz > rr ? rr : filesz);
1287 
1288  if (!skip_file)
1289  WRITE_TAR_DATA(copybuf + pos, bytes2write);
1290 
1291  rr -= bytes2write;
1292  pos += bytes2write;
1293  filesz -= bytes2write;
1294  }
1295  else if (is_recovery_guc_supported &&
1296  is_postgresql_auto_conf &&
1298  {
1299  /* append recovery config to postgresql.auto.conf */
1300  int padding;
1301  int tailsize;
1302 
1303  tailsize = (512 - file_padding_len) + recoveryconfcontents->len;
1304  padding = ((tailsize + 511) & ~511) - tailsize;
1305 
1306  WRITE_TAR_DATA(recoveryconfcontents->data, recoveryconfcontents->len);
1307 
1308  if (padding)
1309  {
1310  char zerobuf[512];
1311 
1312  MemSet(zerobuf, 0, sizeof(zerobuf));
1313  WRITE_TAR_DATA(zerobuf, padding);
1314  }
1315 
1316  /* skip original file padding */
1317  is_postgresql_auto_conf = false;
1318  skip_file = true;
1319  filesz += file_padding_len;
1320 
1321  found_postgresql_auto_conf = true;
1322  }
1323  else
1324  {
1325  /*
1326  * No more data in the current file, the next piece of
1327  * data (if any) will be a new file header structure.
1328  */
1329  in_tarhdr = true;
1330  skip_file = false;
1331  is_postgresql_auto_conf = false;
1332  tarhdrsz = 0;
1333  filesz = 0;
1334  }
1335  }
1336  }
1337  }
1338  totaldone += r;
1339  progress_report(rownum, filename, false);
1340  } /* while (1) */
1341  progress_report(rownum, filename, true);
1342 
1343  if (copybuf != NULL)
1344  PQfreemem(copybuf);
1345 
1346  /*
1347  * Do not sync the resulting tar file yet, all files are synced once at
1348  * the end.
1349  */
1350 }
1351 
1352 
1353 /*
1354  * Retrieve tablespace path, either relocated or original depending on whether
1355  * -T was passed or not.
1356  */
1357 static const char *
1358 get_tablespace_mapping(const char *dir)
1359 {
1360  TablespaceListCell *cell;
1361  char canon_dir[MAXPGPATH];
1362 
1363  /* Canonicalize path for comparison consistency */
1364  strlcpy(canon_dir, dir, sizeof(canon_dir));
1365  canonicalize_path(canon_dir);
1366 
1367  for (cell = tablespace_dirs.head; cell; cell = cell->next)
1368  if (strcmp(canon_dir, cell->old_dir) == 0)
1369  return cell->new_dir;
1370 
1371  return dir;
1372 }
1373 
1374 
1375 /*
1376  * Receive a tar format stream from the connection to the server, and unpack
1377  * the contents of it into a directory. Only files, directories and
1378  * symlinks are supported, no other kinds of special files.
1379  *
1380  * If the data is for the main data directory, it will be restored in the
1381  * specified directory. If it's for another tablespace, it will be restored
1382  * in the original or mapped directory.
1383  */
1384 static void
1386 {
1387  char current_path[MAXPGPATH];
1388  char filename[MAXPGPATH];
1389  const char *mapped_tblspc_path;
1390  pgoff_t current_len_left = 0;
1391  int current_padding = 0;
1392  bool basetablespace;
1393  char *copybuf = NULL;
1394  FILE *file = NULL;
1395 
1396  basetablespace = PQgetisnull(res, rownum, 0);
1397  if (basetablespace)
1398  strlcpy(current_path, basedir, sizeof(current_path));
1399  else
1400  strlcpy(current_path,
1401  get_tablespace_mapping(PQgetvalue(res, rownum, 1)),
1402  sizeof(current_path));
1403 
1404  /*
1405  * Get the COPY data
1406  */
1407  res = PQgetResult(conn);
1408  if (PQresultStatus(res) != PGRES_COPY_OUT)
1409  {
1410  pg_log_error("could not get COPY data stream: %s",
1411  PQerrorMessage(conn));
1412  exit(1);
1413  }
1414 
1415  while (1)
1416  {
1417  int r;
1418 
1419  if (copybuf != NULL)
1420  {
1421  PQfreemem(copybuf);
1422  copybuf = NULL;
1423  }
1424 
1425  r = PQgetCopyData(conn, &copybuf, 0);
1426 
1427  if (r == -1)
1428  {
1429  /*
1430  * End of chunk
1431  */
1432  if (file)
1433  fclose(file);
1434 
1435  break;
1436  }
1437  else if (r == -2)
1438  {
1439  pg_log_error("could not read COPY data: %s",
1440  PQerrorMessage(conn));
1441  exit(1);
1442  }
1443 
1444  if (file == NULL)
1445  {
1446 #ifndef WIN32
1447  int filemode;
1448 #endif
1449 
1450  /*
1451  * No current file, so this must be the header for a new file
1452  */
1453  if (r != 512)
1454  {
1455  pg_log_error("invalid tar block header size: %d", r);
1456  exit(1);
1457  }
1458  totaldone += 512;
1459 
1460  current_len_left = read_tar_number(&copybuf[124], 12);
1461 
1462 #ifndef WIN32
1463  /* Set permissions on the file */
1464  filemode = read_tar_number(&copybuf[100], 8);
1465 #endif
1466 
1467  /*
1468  * All files are padded up to 512 bytes
1469  */
1470  current_padding =
1471  ((current_len_left + 511) & ~511) - current_len_left;
1472 
1473  /*
1474  * First part of header is zero terminated filename
1475  */
1476  snprintf(filename, sizeof(filename), "%s/%s", current_path,
1477  copybuf);
1478  if (filename[strlen(filename) - 1] == '/')
1479  {
1480  /*
1481  * Ends in a slash means directory or symlink to directory
1482  */
1483  if (copybuf[156] == '5')
1484  {
1485  /*
1486  * Directory
1487  */
1488  filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */
1489  if (mkdir(filename, pg_dir_create_mode) != 0)
1490  {
1491  /*
1492  * When streaming WAL, pg_wal (or pg_xlog for pre-9.6
1493  * clusters) will have been created by the wal
1494  * receiver process. Also, when the WAL directory
1495  * location was specified, pg_wal (or pg_xlog) has
1496  * already been created as a symbolic link before
1497  * starting the actual backup. So just ignore creation
1498  * failures on related directories.
1499  */
1500  if (!((pg_str_endswith(filename, "/pg_wal") ||
1501  pg_str_endswith(filename, "/pg_xlog") ||
1502  pg_str_endswith(filename, "/archive_status")) &&
1503  errno == EEXIST))
1504  {
1505  pg_log_error("could not create directory \"%s\": %m",
1506  filename);
1507  exit(1);
1508  }
1509  }
1510 #ifndef WIN32
1511  if (chmod(filename, (mode_t) filemode))
1512  pg_log_error("could not set permissions on directory \"%s\": %m",
1513  filename);
1514 #endif
1515  }
1516  else if (copybuf[156] == '2')
1517  {
1518  /*
1519  * Symbolic link
1520  *
1521  * It's most likely a link in pg_tblspc directory, to the
1522  * location of a tablespace. Apply any tablespace mapping
1523  * given on the command line (--tablespace-mapping). (We
1524  * blindly apply the mapping without checking that the
1525  * link really is inside pg_tblspc. We don't expect there
1526  * to be other symlinks in a data directory, but if there
1527  * are, you can call it an undocumented feature that you
1528  * can map them too.)
1529  */
1530  filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */
1531 
1532  mapped_tblspc_path = get_tablespace_mapping(&copybuf[157]);
1533  if (symlink(mapped_tblspc_path, filename) != 0)
1534  {
1535  pg_log_error("could not create symbolic link from \"%s\" to \"%s\": %m",
1536  filename, mapped_tblspc_path);
1537  exit(1);
1538  }
1539  }
1540  else
1541  {
1542  pg_log_error("unrecognized link indicator \"%c\"",
1543  copybuf[156]);
1544  exit(1);
1545  }
1546  continue; /* directory or link handled */
1547  }
1548 
1549  /*
1550  * regular file
1551  */
1552  file = fopen(filename, "wb");
1553  if (!file)
1554  {
1555  pg_log_error("could not create file \"%s\": %m", filename);
1556  exit(1);
1557  }
1558 
1559 #ifndef WIN32
1560  if (chmod(filename, (mode_t) filemode))
1561  pg_log_error("could not set permissions on file \"%s\": %m",
1562  filename);
1563 #endif
1564 
1565  if (current_len_left == 0)
1566  {
1567  /*
1568  * Done with this file, next one will be a new tar header
1569  */
1570  fclose(file);
1571  file = NULL;
1572  continue;
1573  }
1574  } /* new file */
1575  else
1576  {
1577  /*
1578  * Continuing blocks in existing file
1579  */
1580  if (current_len_left == 0 && r == current_padding)
1581  {
1582  /*
1583  * Received the padding block for this file, ignore it and
1584  * close the file, then move on to the next tar header.
1585  */
1586  fclose(file);
1587  file = NULL;
1588  totaldone += r;
1589  continue;
1590  }
1591 
1592  if (fwrite(copybuf, r, 1, file) != 1)
1593  {
1594  pg_log_error("could not write to file \"%s\": %m", filename);
1595  exit(1);
1596  }
1597  totaldone += r;
1598  progress_report(rownum, filename, false);
1599 
1600  current_len_left -= r;
1601  if (current_len_left == 0 && current_padding == 0)
1602  {
1603  /*
1604  * Received the last block, and there is no padding to be
1605  * expected. Close the file and move on to the next tar
1606  * header.
1607  */
1608  fclose(file);
1609  file = NULL;
1610  continue;
1611  }
1612  } /* continuing data in existing file */
1613  } /* loop over all data blocks */
1614  progress_report(rownum, filename, true);
1615 
1616  if (file != NULL)
1617  {
1618  pg_log_error("COPY stream ended before last file was finished");
1619  exit(1);
1620  }
1621 
1622  if (copybuf != NULL)
1623  PQfreemem(copybuf);
1624 
1625  if (basetablespace && writerecoveryconf)
1626  WriteRecoveryConfig(conn, basedir, recoveryconfcontents);
1627 
1628  /*
1629  * No data is synced here, everything is done for all tablespaces at the
1630  * end.
1631  */
1632 }
1633 
1634 
1635 static void
1637 {
1638  PGresult *res;
1639  char *sysidentifier;
1640  TimeLineID latesttli;
1641  TimeLineID starttli;
1642  char *basebkp;
1643  char escaped_label[MAXPGPATH];
1644  char *maxrate_clause = NULL;
1645  int i;
1646  char xlogstart[64];
1647  char xlogend[64];
1648  int minServerMajor,
1649  maxServerMajor;
1650  int serverVersion,
1651  serverMajor;
1652 
1653  Assert(conn != NULL);
1654 
1655  /*
1656  * Check server version. BASE_BACKUP command was introduced in 9.1, so we
1657  * can't work with servers older than 9.1.
1658  */
1659  minServerMajor = 901;
1660  maxServerMajor = PG_VERSION_NUM / 100;
1661  serverVersion = PQserverVersion(conn);
1662  serverMajor = serverVersion / 100;
1663  if (serverMajor < minServerMajor || serverMajor > maxServerMajor)
1664  {
1665  const char *serverver = PQparameterStatus(conn, "server_version");
1666 
1667  pg_log_error("incompatible server version %s",
1668  serverver ? serverver : "'unknown'");
1669  exit(1);
1670  }
1671 
1672  /*
1673  * If WAL streaming was requested, also check that the server is new
1674  * enough for that.
1675  */
1677  {
1678  /*
1679  * Error message already written in CheckServerVersionForStreaming(),
1680  * but add a hint about using -X none.
1681  */
1682  pg_log_info("HINT: use -X none or -X fetch to disable log streaming");
1683  exit(1);
1684  }
1685 
1686  /*
1687  * Build contents of configuration file if requested
1688  */
1689  if (writerecoveryconf)
1690  recoveryconfcontents = GenerateRecoveryConfig(conn, replication_slot);
1691 
1692  /*
1693  * Run IDENTIFY_SYSTEM so we can get the timeline
1694  */
1695  if (!RunIdentifySystem(conn, &sysidentifier, &latesttli, NULL, NULL))
1696  exit(1);
1697 
1698  /*
1699  * Start the actual backup
1700  */
1701  PQescapeStringConn(conn, escaped_label, label, sizeof(escaped_label), &i);
1702 
1703  if (maxrate > 0)
1704  maxrate_clause = psprintf("MAX_RATE %u", maxrate);
1705 
1706  if (verbose)
1707  pg_log_info("initiating base backup, waiting for checkpoint to complete");
1708 
1709  if (showprogress && !verbose)
1710  {
1711  fprintf(stderr, "waiting for checkpoint");
1712  if (isatty(fileno(stderr)))
1713  fprintf(stderr, "\r");
1714  else
1715  fprintf(stderr, "\n");
1716  }
1717 
1718  basebkp =
1719  psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s",
1720  escaped_label,
1721  showprogress ? "PROGRESS" : "",
1722  includewal == FETCH_WAL ? "WAL" : "",
1723  fastcheckpoint ? "FAST" : "",
1724  includewal == NO_WAL ? "" : "NOWAIT",
1725  maxrate_clause ? maxrate_clause : "",
1726  format == 't' ? "TABLESPACE_MAP" : "",
1727  verify_checksums ? "" : "NOVERIFY_CHECKSUMS");
1728 
1729  if (PQsendQuery(conn, basebkp) == 0)
1730  {
1731  pg_log_error("could not send replication command \"%s\": %s",
1732  "BASE_BACKUP", PQerrorMessage(conn));
1733  exit(1);
1734  }
1735 
1736  /*
1737  * Get the starting WAL location
1738  */
1739  res = PQgetResult(conn);
1740  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1741  {
1742  pg_log_error("could not initiate base backup: %s",
1743  PQerrorMessage(conn));
1744  exit(1);
1745  }
1746  if (PQntuples(res) != 1)
1747  {
1748  pg_log_error("server returned unexpected response to BASE_BACKUP command; got %d rows and %d fields, expected %d rows and %d fields",
1749  PQntuples(res), PQnfields(res), 1, 2);
1750  exit(1);
1751  }
1752 
1753  strlcpy(xlogstart, PQgetvalue(res, 0, 0), sizeof(xlogstart));
1754 
1755  if (verbose)
1756  pg_log_info("checkpoint completed");
1757 
1758  /*
1759  * 9.3 and later sends the TLI of the starting point. With older servers,
1760  * assume it's the same as the latest timeline reported by
1761  * IDENTIFY_SYSTEM.
1762  */
1763  if (PQnfields(res) >= 2)
1764  starttli = atoi(PQgetvalue(res, 0, 1));
1765  else
1766  starttli = latesttli;
1767  PQclear(res);
1768  MemSet(xlogend, 0, sizeof(xlogend));
1769 
1770  if (verbose && includewal != NO_WAL)
1771  pg_log_info("write-ahead log start point: %s on timeline %u",
1772  xlogstart, starttli);
1773 
1774  /*
1775  * Get the header
1776  */
1777  res = PQgetResult(conn);
1778  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1779  {
1780  pg_log_error("could not get backup header: %s",
1781  PQerrorMessage(conn));
1782  exit(1);
1783  }
1784  if (PQntuples(res) < 1)
1785  {
1786  pg_log_error("no data returned from server");
1787  exit(1);
1788  }
1789 
1790  /*
1791  * Sum up the total size, for progress reporting
1792  */
1793  totalsize_kb = totaldone = 0;
1794  tablespacecount = PQntuples(res);
1795  for (i = 0; i < PQntuples(res); i++)
1796  {
1797  totalsize_kb += atol(PQgetvalue(res, i, 2));
1798 
1799  /*
1800  * Verify tablespace directories are empty. Don't bother with the
1801  * first once since it can be relocated, and it will be checked before
1802  * we do anything anyway.
1803  */
1804  if (format == 'p' && !PQgetisnull(res, i, 1))
1805  {
1806  char *path = unconstify(char *, get_tablespace_mapping(PQgetvalue(res, i, 1)));
1807 
1809  }
1810  }
1811 
1812  /*
1813  * When writing to stdout, require a single tablespace
1814  */
1815  if (format == 't' && strcmp(basedir, "-") == 0 && PQntuples(res) > 1)
1816  {
1817  pg_log_error("can only write single tablespace to stdout, database has %d",
1818  PQntuples(res));
1819  exit(1);
1820  }
1821 
1822  /*
1823  * If we're streaming WAL, start the streaming session before we start
1824  * receiving the actual data chunks.
1825  */
1826  if (includewal == STREAM_WAL)
1827  {
1828  if (verbose)
1829  pg_log_info("starting background WAL receiver");
1830  StartLogStreamer(xlogstart, starttli, sysidentifier);
1831  }
1832 
1833  /*
1834  * Start receiving chunks
1835  */
1836  for (i = 0; i < PQntuples(res); i++)
1837  {
1838  if (format == 't')
1839  ReceiveTarFile(conn, res, i);
1840  else
1841  ReceiveAndUnpackTarFile(conn, res, i);
1842  } /* Loop over all tablespaces */
1843 
1844  if (showprogress)
1845  {
1846  progress_report(PQntuples(res), NULL, true);
1847  if (isatty(fileno(stderr)))
1848  fprintf(stderr, "\n"); /* Need to move to next line */
1849  }
1850 
1851  PQclear(res);
1852 
1853  /*
1854  * Get the stop position
1855  */
1856  res = PQgetResult(conn);
1857  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1858  {
1859  pg_log_error("could not get write-ahead log end position from server: %s",
1860  PQerrorMessage(conn));
1861  exit(1);
1862  }
1863  if (PQntuples(res) != 1)
1864  {
1865  pg_log_error("no write-ahead log end position returned from server");
1866  exit(1);
1867  }
1868  strlcpy(xlogend, PQgetvalue(res, 0, 0), sizeof(xlogend));
1869  if (verbose && includewal != NO_WAL)
1870  pg_log_info("write-ahead log end point: %s", xlogend);
1871  PQclear(res);
1872 
1873  res = PQgetResult(conn);
1874  if (PQresultStatus(res) != PGRES_COMMAND_OK)
1875  {
1876  const char *sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
1877 
1878  if (sqlstate &&
1879  strcmp(sqlstate, ERRCODE_DATA_CORRUPTED) == 0)
1880  {
1881  pg_log_error("checksum error occurred");
1882  checksum_failure = true;
1883  }
1884  else
1885  {
1886  pg_log_error("final receive failed: %s",
1887  PQerrorMessage(conn));
1888  }
1889  exit(1);
1890  }
1891 
1892  if (bgchild > 0)
1893  {
1894 #ifndef WIN32
1895  int status;
1896  pid_t r;
1897 #else
1898  DWORD status;
1899 
1900  /*
1901  * get a pointer sized version of bgchild to avoid warnings about
1902  * casting to a different size on WIN64.
1903  */
1904  intptr_t bgchild_handle = bgchild;
1905  uint32 hi,
1906  lo;
1907 #endif
1908 
1909  if (verbose)
1910  pg_log_info("waiting for background process to finish streaming ...");
1911 
1912 #ifndef WIN32
1913  if (write(bgpipe[1], xlogend, strlen(xlogend)) != strlen(xlogend))
1914  {
1915  pg_log_info("could not send command to background pipe: %m");
1916  exit(1);
1917  }
1918 
1919  /* Just wait for the background process to exit */
1920  r = waitpid(bgchild, &status, 0);
1921  if (r == (pid_t) -1)
1922  {
1923  pg_log_error("could not wait for child process: %m");
1924  exit(1);
1925  }
1926  if (r != bgchild)
1927  {
1928  pg_log_error("child %d died, expected %d", (int) r, (int) bgchild);
1929  exit(1);
1930  }
1931  if (status != 0)
1932  {
1933  pg_log_error("%s", wait_result_to_str(status));
1934  exit(1);
1935  }
1936  /* Exited normally, we're happy! */
1937 #else /* WIN32 */
1938 
1939  /*
1940  * On Windows, since we are in the same process, we can just store the
1941  * value directly in the variable, and then set the flag that says
1942  * it's there.
1943  */
1944  if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
1945  {
1946  pg_log_error("could not parse write-ahead log location \"%s\"",
1947  xlogend);
1948  exit(1);
1949  }
1950  xlogendptr = ((uint64) hi) << 32 | lo;
1951  InterlockedIncrement(&has_xlogendptr);
1952 
1953  /* First wait for the thread to exit */
1954  if (WaitForSingleObjectEx((HANDLE) bgchild_handle, INFINITE, FALSE) !=
1955  WAIT_OBJECT_0)
1956  {
1957  _dosmaperr(GetLastError());
1958  pg_log_error("could not wait for child thread: %m");
1959  exit(1);
1960  }
1961  if (GetExitCodeThread((HANDLE) bgchild_handle, &status) == 0)
1962  {
1963  _dosmaperr(GetLastError());
1964  pg_log_error("could not get child thread exit status: %m");
1965  exit(1);
1966  }
1967  if (status != 0)
1968  {
1969  pg_log_error("child thread exited with error %u",
1970  (unsigned int) status);
1971  exit(1);
1972  }
1973  /* Exited normally, we're happy */
1974 #endif
1975  }
1976 
1977  /* Free the configuration file contents */
1978  destroyPQExpBuffer(recoveryconfcontents);
1979 
1980  /*
1981  * End of copy data. Final result is already checked inside the loop.
1982  */
1983  PQclear(res);
1984  PQfinish(conn);
1985  conn = NULL;
1986 
1987  /*
1988  * Make data persistent on disk once backup is completed. For tar format
1989  * sync the parent directory and all its contents as each tar file was not
1990  * synced after being completed. In plain format, all the data of the
1991  * base directory is synced, taking into account all the tablespaces.
1992  * Errors are not considered fatal.
1993  */
1994  if (do_sync)
1995  {
1996  if (verbose)
1997  pg_log_info("syncing data to disk ...");
1998  if (format == 't')
1999  {
2000  if (strcmp(basedir, "-") != 0)
2001  (void) fsync_dir_recurse(basedir);
2002  }
2003  else
2004  {
2005  (void) fsync_pgdata(basedir, serverVersion);
2006  }
2007  }
2008 
2009  if (verbose)
2010  pg_log_info("base backup completed");
2011 }
2012 
2013 
2014 int
2015 main(int argc, char **argv)
2016 {
2017  static struct option long_options[] = {
2018  {"help", no_argument, NULL, '?'},
2019  {"version", no_argument, NULL, 'V'},
2020  {"pgdata", required_argument, NULL, 'D'},
2021  {"format", required_argument, NULL, 'F'},
2022  {"checkpoint", required_argument, NULL, 'c'},
2023  {"create-slot", no_argument, NULL, 'C'},
2024  {"max-rate", required_argument, NULL, 'r'},
2025  {"write-recovery-conf", no_argument, NULL, 'R'},
2026  {"slot", required_argument, NULL, 'S'},
2027  {"tablespace-mapping", required_argument, NULL, 'T'},
2028  {"wal-method", required_argument, NULL, 'X'},
2029  {"gzip", no_argument, NULL, 'z'},
2030  {"compress", required_argument, NULL, 'Z'},
2031  {"label", required_argument, NULL, 'l'},
2032  {"no-clean", no_argument, NULL, 'n'},
2033  {"no-sync", no_argument, NULL, 'N'},
2034  {"dbname", required_argument, NULL, 'd'},
2035  {"host", required_argument, NULL, 'h'},
2036  {"port", required_argument, NULL, 'p'},
2037  {"username", required_argument, NULL, 'U'},
2038  {"no-password", no_argument, NULL, 'w'},
2039  {"password", no_argument, NULL, 'W'},
2040  {"status-interval", required_argument, NULL, 's'},
2041  {"verbose", no_argument, NULL, 'v'},
2042  {"progress", no_argument, NULL, 'P'},
2043  {"waldir", required_argument, NULL, 1},
2044  {"no-slot", no_argument, NULL, 2},
2045  {"no-verify-checksums", no_argument, NULL, 3},
2046  {NULL, 0, NULL, 0}
2047  };
2048  int c;
2049 
2050  int option_index;
2051 
2052  pg_logging_init(argv[0]);
2053  progname = get_progname(argv[0]);
2054  set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup"));
2055 
2056  if (argc > 1)
2057  {
2058  if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
2059  {
2060  usage();
2061  exit(0);
2062  }
2063  else if (strcmp(argv[1], "-V") == 0
2064  || strcmp(argv[1], "--version") == 0)
2065  {
2066  puts("pg_basebackup (PostgreSQL) " PG_VERSION);
2067  exit(0);
2068  }
2069  }
2070 
2072 
2073  while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP",
2074  long_options, &option_index)) != -1)
2075  {
2076  switch (c)
2077  {
2078  case 'C':
2079  create_slot = true;
2080  break;
2081  case 'D':
2083  break;
2084  case 'F':
2085  if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
2086  format = 'p';
2087  else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
2088  format = 't';
2089  else
2090  {
2091  pg_log_error("invalid output format \"%s\", must be \"plain\" or \"tar\"",
2092  optarg);
2093  exit(1);
2094  }
2095  break;
2096  case 'r':
2098  break;
2099  case 'R':
2100  writerecoveryconf = true;
2101  break;
2102  case 'S':
2103 
2104  /*
2105  * When specifying replication slot name, use a permanent
2106  * slot.
2107  */
2109  temp_replication_slot = false;
2110  break;
2111  case 2:
2112  no_slot = true;
2113  break;
2114  case 'T':
2116  break;
2117  case 'X':
2118  if (strcmp(optarg, "n") == 0 ||
2119  strcmp(optarg, "none") == 0)
2120  {
2121  includewal = NO_WAL;
2122  }
2123  else if (strcmp(optarg, "f") == 0 ||
2124  strcmp(optarg, "fetch") == 0)
2125  {
2127  }
2128  else if (strcmp(optarg, "s") == 0 ||
2129  strcmp(optarg, "stream") == 0)
2130  {
2132  }
2133  else
2134  {
2135  pg_log_error("invalid wal-method option \"%s\", must be \"fetch\", \"stream\", or \"none\"",
2136  optarg);
2137  exit(1);
2138  }
2139  break;
2140  case 1:
2142  break;
2143  case 'l':
2144  label = pg_strdup(optarg);
2145  break;
2146  case 'n':
2147  noclean = true;
2148  break;
2149  case 'N':
2150  do_sync = false;
2151  break;
2152  case 'z':
2153 #ifdef HAVE_LIBZ
2155 #else
2156  compresslevel = 1; /* will be rejected below */
2157 #endif
2158  break;
2159  case 'Z':
2160  compresslevel = atoi(optarg);
2161  if (compresslevel < 0 || compresslevel > 9)
2162  {
2163  pg_log_error("invalid compression level \"%s\"", optarg);
2164  exit(1);
2165  }
2166  break;
2167  case 'c':
2168  if (pg_strcasecmp(optarg, "fast") == 0)
2169  fastcheckpoint = true;
2170  else if (pg_strcasecmp(optarg, "spread") == 0)
2171  fastcheckpoint = false;
2172  else
2173  {
2174  pg_log_error("invalid checkpoint argument \"%s\", must be \"fast\" or \"spread\"",
2175  optarg);
2176  exit(1);
2177  }
2178  break;
2179  case 'd':
2181  break;
2182  case 'h':
2183  dbhost = pg_strdup(optarg);
2184  break;
2185  case 'p':
2186  dbport = pg_strdup(optarg);
2187  break;
2188  case 'U':
2189  dbuser = pg_strdup(optarg);
2190  break;
2191  case 'w':
2192  dbgetpassword = -1;
2193  break;
2194  case 'W':
2195  dbgetpassword = 1;
2196  break;
2197  case 's':
2198  standby_message_timeout = atoi(optarg) * 1000;
2199  if (standby_message_timeout < 0)
2200  {
2201  pg_log_error("invalid status interval \"%s\"", optarg);
2202  exit(1);
2203  }
2204  break;
2205  case 'v':
2206  verbose++;
2207  break;
2208  case 'P':
2209  showprogress = true;
2210  break;
2211  case 3:
2212  verify_checksums = false;
2213  break;
2214  default:
2215 
2216  /*
2217  * getopt_long already emitted a complaint
2218  */
2219  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2220  progname);
2221  exit(1);
2222  }
2223  }
2224 
2225  /*
2226  * Any non-option arguments?
2227  */
2228  if (optind < argc)
2229  {
2230  pg_log_error("too many command-line arguments (first is \"%s\")",
2231  argv[optind]);
2232  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2233  progname);
2234  exit(1);
2235  }
2236 
2237  /*
2238  * Required arguments
2239  */
2240  if (basedir == NULL)
2241  {
2242  pg_log_error("no target directory specified");
2243  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2244  progname);
2245  exit(1);
2246  }
2247 
2248  /*
2249  * Mutually exclusive arguments
2250  */
2251  if (format == 'p' && compresslevel != 0)
2252  {
2253  pg_log_error("only tar mode backups can be compressed");
2254  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2255  progname);
2256  exit(1);
2257  }
2258 
2259  if (format == 't' && includewal == STREAM_WAL && strcmp(basedir, "-") == 0)
2260  {
2261  pg_log_error("cannot stream write-ahead logs in tar mode to stdout");
2262  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2263  progname);
2264  exit(1);
2265  }
2266 
2268  {
2269  pg_log_error("replication slots can only be used with WAL streaming");
2270  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2271  progname);
2272  exit(1);
2273  }
2274 
2275  if (no_slot)
2276  {
2277  if (replication_slot)
2278  {
2279  pg_log_error("--no-slot cannot be used with slot name");
2280  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2281  progname);
2282  exit(1);
2283  }
2284  temp_replication_slot = false;
2285  }
2286 
2287  if (create_slot)
2288  {
2289  if (!replication_slot)
2290  {
2291  pg_log_error("%s needs a slot to be specified using --slot",
2292  "--create-slot");
2293  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2294  progname);
2295  exit(1);
2296  }
2297 
2298  if (no_slot)
2299  {
2300  pg_log_error("--create-slot and --no-slot are incompatible options");
2301  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2302  progname);
2303  exit(1);
2304  }
2305  }
2306 
2307  if (xlog_dir)
2308  {
2309  if (format != 'p')
2310  {
2311  pg_log_error("WAL directory location can only be specified in plain mode");
2312  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2313  progname);
2314  exit(1);
2315  }
2316 
2317  /* clean up xlog directory name, check it's absolute */
2319  if (!is_absolute_path(xlog_dir))
2320  {
2321  pg_log_error("WAL directory location must be an absolute path");
2322  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2323  progname);
2324  exit(1);
2325  }
2326  }
2327 
2328 #ifndef HAVE_LIBZ
2329  if (compresslevel != 0)
2330  {
2331  pg_log_error("this build does not support compression");
2332  exit(1);
2333  }
2334 #endif
2335 
2336  /* connection in replication mode to server */
2337  conn = GetConnection();
2338  if (!conn)
2339  {
2340  /* Error message already written in GetConnection() */
2341  exit(1);
2342  }
2343  atexit(disconnect_atexit);
2344 
2345  /*
2346  * Set umask so that directories/files are created with the same
2347  * permissions as directories/files in the source data directory.
2348  *
2349  * pg_mode_mask is set to owner-only by default and then updated in
2350  * GetConnection() where we get the mode from the server-side with
2351  * RetrieveDataDirCreatePerm() and then call SetDataDirectoryCreatePerm().
2352  */
2353  umask(pg_mode_mask);
2354 
2355  /*
2356  * Verify that the target directory exists, or create it. For plaintext
2357  * backups, always require the directory. For tar backups, require it
2358  * unless we are writing to stdout.
2359  */
2360  if (format == 'p' || strcmp(basedir, "-") != 0)
2362 
2363  /* determine remote server's xlog segment size */
2364  if (!RetrieveWalSegSize(conn))
2365  exit(1);
2366 
2367  /* Create pg_wal symlink, if required */
2368  if (xlog_dir)
2369  {
2370  char *linkloc;
2371 
2373 
2374  /*
2375  * Form name of the place where the symlink must go. pg_xlog has been
2376  * renamed to pg_wal in post-10 clusters.
2377  */
2378  linkloc = psprintf("%s/%s", basedir,
2380  "pg_xlog" : "pg_wal");
2381 
2382 #ifdef HAVE_SYMLINK
2383  if (symlink(xlog_dir, linkloc) != 0)
2384  {
2385  pg_log_error("could not create symbolic link \"%s\": %m", linkloc);
2386  exit(1);
2387  }
2388 #else
2389  pg_log_error("symlinks are not supported on this platform");
2390  exit(1);
2391 #endif
2392  free(linkloc);
2393  }
2394 
2395  BaseBackup();
2396 
2397  success = true;
2398  return 0;
2399 }
int PQnfields(const PGresult *res)
Definition: fe-exec.c:2778
static PQExpBuffer recoveryconfcontents
static void writeTarData(FILE *tarfile, char *buf, int r, char *current_file)
static IncludeWal includewal
Definition: pg_basebackup.c:92
char * PQerrorMessage(const PGconn *conn)
Definition: fe-connect.c:6596
#define Z_DEFAULT_COMPRESSION
int pg_file_create_mode
Definition: file_perm.c:19
static bool verify_checksums
static int bgpipe[2]
static bool found_existing_pgdata
uint32 TimeLineID
Definition: xlogdefs.h:52
int64 pg_time_t
Definition: pgtime.h:23
static void usage(void)
static bool found_tablespace_dirs
char * PQgetvalue(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3164
const char * PQparameterStatus(const PGconn *conn, const char *paramName)
Definition: fe-connect.c:6561
bool pg_str_endswith(const char *str, const char *end)
Definition: string.c:31
int pg_mkdir_p(char *path, int omode)
Definition: pgmkdirp.c:57
static TablespaceList tablespace_dirs
Definition: pg_basebackup.c:83
#define write(a, b, c)
Definition: win32.h:14
bool do_sync
Definition: receivelog.h:39
const char * get_progname(const char *argv0)
Definition: path.c:453
uint64 read_tar_number(const char *s, int len)
Definition: tar.c:56
#define pg_log_error(...)
Definition: logging.h:79
int getopt_long(int argc, char *const argv[], const char *optstring, const struct option *longopts, int *longindex)
Definition: getopt_long.c:57
static int32 parse_max_rate(char *src)
#define FALSE
Definition: ecpglib.h:39
void pg_logging_init(const char *argv0)
Definition: logging.c:39
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
void WriteRecoveryConfig(PGconn *pgconn, char *target_dir, PQExpBuffer contents)
Definition: recovery_gen.c:118
static pg_time_t last_progress_report
Definition: pg_basebackup.c:97
char old_dir[MAXPGPATH]
Definition: pg_basebackup.c:50
static int tablespacecount
void _dosmaperr(unsigned long)
Definition: win32error.c:171
bool RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli, XLogRecPtr *startpos, char **db_name)
Definition: streamutil.c:409
#define VERBOSE_FILENAME_LENGTH
IncludeWal
Definition: pg_basebackup.c:74
void canonicalize_path(char *path)
Definition: path.c:254
char * sysidentifier
Definition: receivelog.h:34
static bool writerecoveryconf
Definition: pg_basebackup.c:94
static bool create_slot
void PQfinish(PGconn *conn)
Definition: fe-connect.c:4098
static int compresslevel
Definition: pg_basebackup.c:91
static bool checksum_failure
Definition: pg_basebackup.c:88
#define MemSet(start, val, len)
Definition: c.h:955
#define MINIMUM_VERSION_FOR_RECOVERY_GUC
Definition: recovery_gen.h:21
#define kill(pid, sig)
Definition: win32_port.h:426
#define printf(...)
Definition: port.h:198
XLogRecPtr startpos
Definition: receivelog.h:32
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
char * partial_suffix
Definition: receivelog.h:48
char new_dir[MAXPGPATH]
Definition: pg_basebackup.c:51
int PQserverVersion(const PGconn *conn)
Definition: fe-connect.c:6586
const char * progname
Definition: pg_standby.c:37
int PQntuples(const PGresult *res)
Definition: fe-exec.c:2770
#define PG_DIAG_SQLSTATE
Definition: postgres_ext.h:57
#define fprintf
Definition: port.h:196
static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline, bool segment_finished)
char * wait_result_to_str(int exitstatus)
Definition: wait_error.c:32
TimeLineID timeline
Definition: receivelog.h:33
static char * basedir
Definition: pg_basebackup.c:82
struct TablespaceListCell TablespaceListCell
WalWriteMethod * CreateWalDirectoryMethod(const char *basedir, int compression, bool sync)
Definition: walmethods.c:350
ExecStatusType PQresultStatus(const PGresult *res)
Definition: fe-exec.c:2693
int PQgetCopyData(PGconn *conn, char **buffer, int async)
Definition: fe-exec.c:2474
signed int int32
Definition: c.h:346
int main(int argc, char **argv)
static void BaseBackup(void)
void fsync_pgdata(const char *pg_data, int serverVersion)
Definition: file_utils.c:58
#define pgoff_t
Definition: win32_port.h:195
int PQsendQuery(PGconn *conn, const char *query)
Definition: fe-exec.c:1235
void destroyPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:116
#define required_argument
Definition: getopt_long.h:25
int optind
Definition: getopt.c:50
#define MAX_RATE_UPPER
Definition: basebackup.h:21
void fsync_dir_recurse(const char *dir)
Definition: file_utils.c:122
char * connection_string
Definition: streamutil.c:48
struct TablespaceList TablespaceList
bool RetrieveWalSegSize(PGconn *conn)
Definition: streamutil.c:279
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
PGconn * conn
Definition: streamutil.c:56
#define MAXPGPATH
#define MINIMUM_VERSION_FOR_PG_WAL
Definition: pg_basebackup.c:64
static bool found_existing_xlogdir
static uint64 totaldone
char * replication_slot
Definition: receivelog.h:49
bool mark_done
Definition: receivelog.h:38
static bool success
static int32 maxrate
Definition: pg_basebackup.c:98
char * c
static char * buf
Definition: pg_test_fsync.c:68
#define symlink(oldpath, newpath)
Definition: win32_port.h:221
static int has_xlogendptr
static bool made_new_pgdata
#define is_absolute_path(filename)
Definition: port.h:86
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
size_t PQescapeStringConn(PGconn *conn, char *to, const char *from, size_t length, int *error)
Definition: fe-exec.c:3411
#define select(n, r, w, e, timeout)
Definition: win32_port.h:436
unsigned int uint32
Definition: c.h:358
static const char * get_tablespace_mapping(const char *dir)
static void progress_report(int tablespacenum, const char *filename, bool force)
XLogRecPtr startptr
PQExpBuffer GenerateRecoveryConfig(PGconn *pgconn, char *replication_slot)
Definition: recovery_gen.c:24
stream_stop_callback stream_stop
Definition: receivelog.h:42
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:45
WalWriteMethod * walmethod
Definition: receivelog.h:47
static int verbose
Definition: pg_basebackup.c:90
#define unconstify(underlying_type, expr)
Definition: c.h:1163
bool rmtree(const char *path, bool rmtopdir)
Definition: rmtree.c:42
static int standby_message_timeout
Definition: pg_basebackup.c:96
int pg_dir_create_mode
Definition: file_perm.c:18
static bool showprogress
Definition: pg_basebackup.c:89
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:112
int dbgetpassword
Definition: streamutil.c:53
#define no_argument
Definition: getopt_long.h:24
void FreeWalTarMethod(void)
Definition: walmethods.c:1014
#define ngettext(s, p, n)
Definition: c.h:1103
#define PG_TEXTDOMAIN(domain)
Definition: c.h:1135
static bool temp_replication_slot
int PQbackendPID(const PGconn *conn)
Definition: fe-connect.c:6622
#define PGINVALID_SOCKET
Definition: port.h:33
static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
struct TablespaceListCell * next
Definition: pg_basebackup.c:49
static char * label
Definition: pg_basebackup.c:86
#define XLogSegmentOffset(xlogptr, wal_segsz_bytes)
StringInfo copybuf
Definition: tablesync.c:115
static void disconnect_atexit(void)
static void StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
TablespaceListCell * tail
Definition: pg_basebackup.c:57
char * dbport
Definition: streamutil.c:51
static void cleanup_directories_atexit(void)
void PQclear(PGresult *res)
Definition: fe-exec.c:695
#define free(a)
Definition: header.h:65
static bool in_log_streamer
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
PGconn * GetConnection(UserMapping *user, bool will_prep_stmt)
Definition: connection.c:107
char * PQresultErrorField(const PGresult *res, int fieldcode)
Definition: fe-exec.c:2755
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:732
WalWriteMethod * CreateWalTarMethod(const char *tarbase, int compression, bool sync)
Definition: walmethods.c:983
static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
char * dbhost
Definition: streamutil.c:49
bool ReceiveXlogStream(PGconn *conn, StreamCtl *stream)
Definition: receivelog.c:439
static void kill_bgchild_atexit(void)
#define strerror
Definition: port.h:205
static char * xlog_dir
Definition: pg_basebackup.c:84
static XLogRecPtr xlogendptr
static XLogRecPtr startpos
static bool no_slot
TablespaceListCell * head
Definition: pg_basebackup.c:56
bool synchronous
Definition: receivelog.h:37
void pg_free(void *ptr)
Definition: fe_memutils.c:105
bool(* finish)(void)
Definition: walmethods.h:75
pgsocket stop_socket
Definition: receivelog.h:44
static void header(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:210
#define INT64_FORMAT
Definition: c.h:400
static bool made_new_xlogdir
static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
Definition: walsender.c:851
#define WRITE_TAR_DATA(buf, sz)
int WalSegSz
Definition: pg_standby.c:39
static bool fastcheckpoint
Definition: pg_basebackup.c:93
static void tablespace_list_append(const char *arg)
static bool made_tablespace_dirs
static char * filename
Definition: pg_dumpall.c:91
void set_pglocale_pgservice(const char *argv0, const char *app)
Definition: exec.c:565
int errmsg(const char *fmt,...)
Definition: elog.c:784
static pid_t bgchild
char * dbuser
Definition: streamutil.c:50
char * optarg
Definition: getopt.c:52
static uint64 totalsize_kb
int pg_check_dir(const char *dir)
Definition: pgcheckdir.c:31
int i
int standby_message_timeout
Definition: receivelog.h:36
void * arg
static bool do_sync
Definition: pg_basebackup.c:95
static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
static char format
Definition: pg_basebackup.c:85
#define mkdir(a, b)
Definition: win32_port.h:58
static char * replication_slot
Definition: pg_basebackup.c:99
static void static void status(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:227
static int LogStreamerMain(logstreamer_param *param)
int PQgetisnull(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3189
#define snprintf
Definition: port.h:192
#define _(x)
Definition: elog.c:84
void PQfreemem(void *ptr)
Definition: fe-exec.c:3297
int pg_mode_mask
Definition: file_perm.c:25
Datum now(PG_FUNCTION_ARGS)
Definition: timestamp.c:1547
PGresult * PQgetResult(PGconn *conn)
Definition: fe-exec.c:1779
#define MINIMUM_VERSION_FOR_TEMP_SLOTS
Definition: pg_basebackup.c:69
#define read(a, b, c)
Definition: win32.h:13
#define pg_log_info(...)
Definition: logging.h:87
void FreeWalDirectoryMethod(void)
Definition: walmethods.c:374
bool CheckServerVersionForStreaming(PGconn *conn)
Definition: receivelog.c:361
char xlog[MAXPGPATH]
static bool noclean
Definition: pg_basebackup.c:87