PostgreSQL Source Code  git master
pg_basebackup.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_basebackup.c - receive a base backup using streaming replication protocol
4  *
5  * Author: Magnus Hagander <magnus@hagander.net>
6  *
7  * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
8  *
9  * IDENTIFICATION
10  * src/bin/pg_basebackup/pg_basebackup.c
11  *-------------------------------------------------------------------------
12  */
13 
14 #include "postgres_fe.h"
15 
16 #include <unistd.h>
17 #include <dirent.h>
18 #include <sys/stat.h>
19 #include <sys/wait.h>
20 #include <signal.h>
21 #include <time.h>
22 #ifdef HAVE_SYS_SELECT_H
23 #include <sys/select.h>
24 #endif
25 #ifdef HAVE_LIBZ
26 #include <zlib.h>
27 #endif
28 
29 #include "access/xlog_internal.h"
30 #include "common/file_perm.h"
31 #include "common/file_utils.h"
32 #include "common/logging.h"
33 #include "common/string.h"
34 #include "fe_utils/recovery_gen.h"
35 #include "fe_utils/string_utils.h"
36 #include "getopt_long.h"
37 #include "libpq-fe.h"
38 #include "pgtar.h"
39 #include "pgtime.h"
40 #include "pqexpbuffer.h"
41 #include "receivelog.h"
42 #include "replication/basebackup.h"
43 #include "streamutil.h"
44 
45 #define ERRCODE_DATA_CORRUPTED "XX001"
46 
47 typedef struct TablespaceListCell
48 {
53 
54 typedef struct TablespaceList
55 {
59 
60 typedef struct WriteTarState
61 {
64  FILE *tarfile;
65  char tarhdr[512];
67  bool in_tarhdr;
68  bool skip_file;
73  size_t tarhdrsz;
75 #ifdef HAVE_LIBZ
76  gzFile ztarfile;
77 #endif
79 
80 typedef struct UnpackTarState
81 {
83  char current_path[MAXPGPATH];
85  const char *mapped_tblspc_path;
88  FILE *file;
90 
91 typedef void (*WriteDataCallback) (size_t nbytes, char *buf,
92  void *callback_data);
93 
94 /*
95  * pg_xlog has been renamed to pg_wal in version 10. This version number
96  * should be compared with PQserverVersion().
97  */
98 #define MINIMUM_VERSION_FOR_PG_WAL 100000
99 
100 /*
101  * Temporary replication slots are supported from version 10.
102  */
103 #define MINIMUM_VERSION_FOR_TEMP_SLOTS 100000
104 
105 /*
106  * Different ways to include WAL
107  */
108 typedef enum
109 {
113 } IncludeWal;
114 
115 /* Global options */
116 static char *basedir = NULL;
117 static TablespaceList tablespace_dirs = {NULL, NULL};
118 static char *xlog_dir = NULL;
119 static char format = 'p'; /* p(lain)/t(ar) */
120 static char *label = "pg_basebackup base backup";
121 static bool noclean = false;
122 static bool checksum_failure = false;
123 static bool showprogress = false;
124 static int verbose = 0;
125 static int compresslevel = 0;
127 static bool fastcheckpoint = false;
128 static bool writerecoveryconf = false;
129 static bool do_sync = true;
130 static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
132 static int32 maxrate = 0; /* no limit by default */
133 static char *replication_slot = NULL;
134 static bool temp_replication_slot = true;
135 static bool create_slot = false;
136 static bool no_slot = false;
137 static bool verify_checksums = true;
138 
139 static bool success = false;
140 static bool made_new_pgdata = false;
141 static bool found_existing_pgdata = false;
142 static bool made_new_xlogdir = false;
143 static bool found_existing_xlogdir = false;
144 static bool made_tablespace_dirs = false;
145 static bool found_tablespace_dirs = false;
146 
147 /* Progress counters */
148 static uint64 totalsize_kb;
149 static uint64 totaldone;
150 static int tablespacecount;
151 
152 /* Pipe to communicate with background wal receiver process */
153 #ifndef WIN32
154 static int bgpipe[2] = {-1, -1};
155 #endif
156 
157 /* Handle to child process */
158 static pid_t bgchild = -1;
159 static bool in_log_streamer = false;
160 
161 /* End position for xlog streaming, empty string if unknown yet */
163 
164 #ifndef WIN32
165 static int has_xlogendptr = 0;
166 #else
167 static volatile LONG has_xlogendptr = 0;
168 #endif
169 
170 /* Contents of configuration file to be generated */
172 
173 /* Function headers */
174 static void usage(void);
175 static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found);
176 static void progress_report(int tablespacenum, const char *filename, bool force);
177 
178 static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum);
179 static void ReceiveTarCopyChunk(size_t r, char *copybuf, void *callback_data);
180 static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum);
181 static void ReceiveTarAndUnpackCopyChunk(size_t r, char *copybuf,
182  void *callback_data);
183 static void BaseBackup(void);
184 
185 static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline,
186  bool segment_finished);
187 
188 static const char *get_tablespace_mapping(const char *dir);
189 static void tablespace_list_append(const char *arg);
190 
191 
192 static void
194 {
195  if (success || in_log_streamer)
196  return;
197 
198  if (!noclean && !checksum_failure)
199  {
200  if (made_new_pgdata)
201  {
202  pg_log_info("removing data directory \"%s\"", basedir);
203  if (!rmtree(basedir, true))
204  pg_log_error("failed to remove data directory");
205  }
206  else if (found_existing_pgdata)
207  {
208  pg_log_info("removing contents of data directory \"%s\"", basedir);
209  if (!rmtree(basedir, false))
210  pg_log_error("failed to remove contents of data directory");
211  }
212 
213  if (made_new_xlogdir)
214  {
215  pg_log_info("removing WAL directory \"%s\"", xlog_dir);
216  if (!rmtree(xlog_dir, true))
217  pg_log_error("failed to remove WAL directory");
218  }
219  else if (found_existing_xlogdir)
220  {
221  pg_log_info("removing contents of WAL directory \"%s\"", xlog_dir);
222  if (!rmtree(xlog_dir, false))
223  pg_log_error("failed to remove contents of WAL directory");
224  }
225  }
226  else
227  {
229  pg_log_info("data directory \"%s\" not removed at user's request", basedir);
230 
232  pg_log_info("WAL directory \"%s\" not removed at user's request", xlog_dir);
233  }
234 
236  pg_log_info("changes to tablespace directories will not be undone");
237 }
238 
239 static void
241 {
242  if (conn != NULL)
243  PQfinish(conn);
244 }
245 
246 #ifndef WIN32
247 /*
248  * On windows, our background thread dies along with the process. But on
249  * Unix, if we have started a subprocess, we want to kill it off so it
250  * doesn't remain running trying to stream data.
251  */
252 static void
254 {
255  if (bgchild > 0)
256  kill(bgchild, SIGTERM);
257 }
258 #endif
259 
260 /*
261  * Split argument into old_dir and new_dir and append to tablespace mapping
262  * list.
263  */
264 static void
266 {
268  char *dst;
269  char *dst_ptr;
270  const char *arg_ptr;
271 
272  dst_ptr = dst = cell->old_dir;
273  for (arg_ptr = arg; *arg_ptr; arg_ptr++)
274  {
275  if (dst_ptr - dst >= MAXPGPATH)
276  {
277  pg_log_error("directory name too long");
278  exit(1);
279  }
280 
281  if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
282  ; /* skip backslash escaping = */
283  else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
284  {
285  if (*cell->new_dir)
286  {
287  pg_log_error("multiple \"=\" signs in tablespace mapping");
288  exit(1);
289  }
290  else
291  dst = dst_ptr = cell->new_dir;
292  }
293  else
294  *dst_ptr++ = *arg_ptr;
295  }
296 
297  if (!*cell->old_dir || !*cell->new_dir)
298  {
299  pg_log_error("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
300  exit(1);
301  }
302 
303  /*
304  * This check isn't absolutely necessary. But all tablespaces are created
305  * with absolute directories, so specifying a non-absolute path here would
306  * just never match, possibly confusing users. It's also good to be
307  * consistent with the new_dir check.
308  */
309  if (!is_absolute_path(cell->old_dir))
310  {
311  pg_log_error("old directory is not an absolute path in tablespace mapping: %s",
312  cell->old_dir);
313  exit(1);
314  }
315 
316  if (!is_absolute_path(cell->new_dir))
317  {
318  pg_log_error("new directory is not an absolute path in tablespace mapping: %s",
319  cell->new_dir);
320  exit(1);
321  }
322 
323  /*
324  * Comparisons done with these values should involve similarly
325  * canonicalized path values. This is particularly sensitive on Windows
326  * where path values may not necessarily use Unix slashes.
327  */
328  canonicalize_path(cell->old_dir);
329  canonicalize_path(cell->new_dir);
330 
331  if (tablespace_dirs.tail)
332  tablespace_dirs.tail->next = cell;
333  else
334  tablespace_dirs.head = cell;
335  tablespace_dirs.tail = cell;
336 }
337 
338 
339 #ifdef HAVE_LIBZ
340 static const char *
341 get_gz_error(gzFile gzf)
342 {
343  int errnum;
344  const char *errmsg;
345 
346  errmsg = gzerror(gzf, &errnum);
347  if (errnum == Z_ERRNO)
348  return strerror(errno);
349  else
350  return errmsg;
351 }
352 #endif
353 
354 static void
355 usage(void)
356 {
357  printf(_("%s takes a base backup of a running PostgreSQL server.\n\n"),
358  progname);
359  printf(_("Usage:\n"));
360  printf(_(" %s [OPTION]...\n"), progname);
361  printf(_("\nOptions controlling the output:\n"));
362  printf(_(" -D, --pgdata=DIRECTORY receive base backup into directory\n"));
363  printf(_(" -F, --format=p|t output format (plain (default), tar)\n"));
364  printf(_(" -r, --max-rate=RATE maximum transfer rate to transfer data directory\n"
365  " (in kB/s, or use suffix \"k\" or \"M\")\n"));
366  printf(_(" -R, --write-recovery-conf\n"
367  " write configuration for replication\n"));
368  printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
369  " relocate tablespace in OLDDIR to NEWDIR\n"));
370  printf(_(" --waldir=WALDIR location for the write-ahead log directory\n"));
371  printf(_(" -X, --wal-method=none|fetch|stream\n"
372  " include required WAL files with specified method\n"));
373  printf(_(" -z, --gzip compress tar output\n"));
374  printf(_(" -Z, --compress=0-9 compress tar output with given compression level\n"));
375  printf(_("\nGeneral options:\n"));
376  printf(_(" -c, --checkpoint=fast|spread\n"
377  " set fast or spread checkpointing\n"));
378  printf(_(" -C, --create-slot create replication slot\n"));
379  printf(_(" -l, --label=LABEL set backup label\n"));
380  printf(_(" -n, --no-clean do not clean up after errors\n"));
381  printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
382  printf(_(" -P, --progress show progress information\n"));
383  printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
384  printf(_(" -v, --verbose output verbose messages\n"));
385  printf(_(" -V, --version output version information, then exit\n"));
386  printf(_(" --no-slot prevent creation of temporary replication slot\n"));
387  printf(_(" --no-verify-checksums\n"
388  " do not verify checksums\n"));
389  printf(_(" -?, --help show this help, then exit\n"));
390  printf(_("\nConnection options:\n"));
391  printf(_(" -d, --dbname=CONNSTR connection string\n"));
392  printf(_(" -h, --host=HOSTNAME database server host or socket directory\n"));
393  printf(_(" -p, --port=PORT database server port number\n"));
394  printf(_(" -s, --status-interval=INTERVAL\n"
395  " time between status packets sent to server (in seconds)\n"));
396  printf(_(" -U, --username=NAME connect as specified database user\n"));
397  printf(_(" -w, --no-password never prompt for password\n"));
398  printf(_(" -W, --password force password prompt (should happen automatically)\n"));
399  printf(_("\nReport bugs to <pgsql-bugs@lists.postgresql.org>.\n"));
400 }
401 
402 
403 /*
404  * Called in the background process every time data is received.
405  * On Unix, we check to see if there is any data on our pipe
406  * (which would mean we have a stop position), and if it is, check if
407  * it is time to stop.
408  * On Windows, we are in a single process, so we can just check if it's
409  * time to stop.
410  */
411 static bool
413  bool segment_finished)
414 {
415  if (!has_xlogendptr)
416  {
417 #ifndef WIN32
418  fd_set fds;
419  struct timeval tv;
420  int r;
421 
422  /*
423  * Don't have the end pointer yet - check our pipe to see if it has
424  * been sent yet.
425  */
426  FD_ZERO(&fds);
427  FD_SET(bgpipe[0], &fds);
428 
429  MemSet(&tv, 0, sizeof(tv));
430 
431  r = select(bgpipe[0] + 1, &fds, NULL, NULL, &tv);
432  if (r == 1)
433  {
434  char xlogend[64];
435  uint32 hi,
436  lo;
437 
438  MemSet(xlogend, 0, sizeof(xlogend));
439  r = read(bgpipe[0], xlogend, sizeof(xlogend) - 1);
440  if (r < 0)
441  {
442  pg_log_error("could not read from ready pipe: %m");
443  exit(1);
444  }
445 
446  if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
447  {
448  pg_log_error("could not parse write-ahead log location \"%s\"",
449  xlogend);
450  exit(1);
451  }
452  xlogendptr = ((uint64) hi) << 32 | lo;
453  has_xlogendptr = 1;
454 
455  /*
456  * Fall through to check if we've reached the point further
457  * already.
458  */
459  }
460  else
461  {
462  /*
463  * No data received on the pipe means we don't know the end
464  * position yet - so just say it's not time to stop yet.
465  */
466  return false;
467  }
468 #else
469 
470  /*
471  * On win32, has_xlogendptr is set by the main thread, so if it's not
472  * set here, we just go back and wait until it shows up.
473  */
474  return false;
475 #endif
476  }
477 
478  /*
479  * At this point we have an end pointer, so compare it to the current
480  * position to figure out if it's time to stop.
481  */
482  if (segendpos >= xlogendptr)
483  return true;
484 
485  /*
486  * Have end pointer, but haven't reached it yet - so tell the caller to
487  * keep streaming.
488  */
489  return false;
490 }
491 
492 typedef struct
493 {
496  char xlog[MAXPGPATH]; /* directory or tarfile depending on mode */
498  int timeline;
500 
501 static int
503 {
504  StreamCtl stream;
505 
506  in_log_streamer = true;
507 
508  MemSet(&stream, 0, sizeof(stream));
509  stream.startpos = param->startptr;
510  stream.timeline = param->timeline;
511  stream.sysidentifier = param->sysidentifier;
513 #ifndef WIN32
514  stream.stop_socket = bgpipe[0];
515 #else
516  stream.stop_socket = PGINVALID_SOCKET;
517 #endif
519  stream.synchronous = false;
520  /* fsync happens at the end of pg_basebackup for all data */
521  stream.do_sync = false;
522  stream.mark_done = true;
523  stream.partial_suffix = NULL;
525 
526  if (format == 'p')
527  stream.walmethod = CreateWalDirectoryMethod(param->xlog, 0,
528  stream.do_sync);
529  else
531  stream.do_sync);
532 
533  if (!ReceiveXlogStream(param->bgconn, &stream))
534 
535  /*
536  * Any errors will already have been reported in the function process,
537  * but we need to tell the parent that we didn't shutdown in a nice
538  * way.
539  */
540  return 1;
541 
542  if (!stream.walmethod->finish())
543  {
544  pg_log_error("could not finish writing WAL files: %m");
545  return 1;
546  }
547 
548  PQfinish(param->bgconn);
549 
550  if (format == 'p')
552  else
554  pg_free(stream.walmethod);
555 
556  return 0;
557 }
558 
559 /*
560  * Initiate background process for receiving xlog during the backup.
561  * The background stream will use its own database connection so we can
562  * stream the logfile in parallel with the backups.
563  */
564 static void
565 StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
566 {
567  logstreamer_param *param;
568  uint32 hi,
569  lo;
570  char statusdir[MAXPGPATH];
571 
572  param = pg_malloc0(sizeof(logstreamer_param));
573  param->timeline = timeline;
574  param->sysidentifier = sysidentifier;
575 
576  /* Convert the starting position */
577  if (sscanf(startpos, "%X/%X", &hi, &lo) != 2)
578  {
579  pg_log_error("could not parse write-ahead log location \"%s\"",
580  startpos);
581  exit(1);
582  }
583  param->startptr = ((uint64) hi) << 32 | lo;
584  /* Round off to even segment position */
585  param->startptr -= XLogSegmentOffset(param->startptr, WalSegSz);
586 
587 #ifndef WIN32
588  /* Create our background pipe */
589  if (pipe(bgpipe) < 0)
590  {
591  pg_log_error("could not create pipe for background process: %m");
592  exit(1);
593  }
594 #endif
595 
596  /* Get a second connection */
597  param->bgconn = GetConnection();
598  if (!param->bgconn)
599  /* Error message already written in GetConnection() */
600  exit(1);
601 
602  /* In post-10 cluster, pg_xlog has been renamed to pg_wal */
603  snprintf(param->xlog, sizeof(param->xlog), "%s/%s",
604  basedir,
606  "pg_xlog" : "pg_wal");
607 
608  /* Temporary replication slots are only supported in 10 and newer */
610  temp_replication_slot = false;
611 
612  /*
613  * Create replication slot if requested
614  */
616  replication_slot = psprintf("pg_basebackup_%d", (int) PQbackendPID(param->bgconn));
618  {
619  if (!CreateReplicationSlot(param->bgconn, replication_slot, NULL,
620  temp_replication_slot, true, true, false))
621  exit(1);
622 
623  if (verbose)
624  {
626  pg_log_info("created temporary replication slot \"%s\"",
628  else
629  pg_log_info("created replication slot \"%s\"",
631  }
632  }
633 
634  if (format == 'p')
635  {
636  /*
637  * Create pg_wal/archive_status or pg_xlog/archive_status (and thus
638  * pg_wal or pg_xlog) depending on the target server so we can write
639  * to basedir/pg_wal or basedir/pg_xlog as the directory entry in the
640  * tar file may arrive later.
641  */
642  snprintf(statusdir, sizeof(statusdir), "%s/%s/archive_status",
643  basedir,
645  "pg_xlog" : "pg_wal");
646 
647  if (pg_mkdir_p(statusdir, pg_dir_create_mode) != 0 && errno != EEXIST)
648  {
649  pg_log_error("could not create directory \"%s\": %m", statusdir);
650  exit(1);
651  }
652  }
653 
654  /*
655  * Start a child process and tell it to start streaming. On Unix, this is
656  * a fork(). On Windows, we create a thread.
657  */
658 #ifndef WIN32
659  bgchild = fork();
660  if (bgchild == 0)
661  {
662  /* in child process */
663  exit(LogStreamerMain(param));
664  }
665  else if (bgchild < 0)
666  {
667  pg_log_error("could not create background process: %m");
668  exit(1);
669  }
670 
671  /*
672  * Else we are in the parent process and all is well.
673  */
674  atexit(kill_bgchild_atexit);
675 #else /* WIN32 */
676  bgchild = _beginthreadex(NULL, 0, (void *) LogStreamerMain, param, 0, NULL);
677  if (bgchild == 0)
678  {
679  pg_log_error("could not create background thread: %m");
680  exit(1);
681  }
682 #endif
683 }
684 
685 /*
686  * Verify that the given directory exists and is empty. If it does not
687  * exist, it is created. If it exists but is not empty, an error will
688  * be given and the process ended.
689  */
690 static void
691 verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
692 {
693  switch (pg_check_dir(dirname))
694  {
695  case 0:
696 
697  /*
698  * Does not exist, so create
699  */
700  if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
701  {
702  pg_log_error("could not create directory \"%s\": %m", dirname);
703  exit(1);
704  }
705  if (created)
706  *created = true;
707  return;
708  case 1:
709 
710  /*
711  * Exists, empty
712  */
713  if (found)
714  *found = true;
715  return;
716  case 2:
717  case 3:
718  case 4:
719 
720  /*
721  * Exists, not empty
722  */
723  pg_log_error("directory \"%s\" exists but is not empty", dirname);
724  exit(1);
725  case -1:
726 
727  /*
728  * Access problem
729  */
730  pg_log_error("could not access directory \"%s\": %m", dirname);
731  exit(1);
732  }
733 }
734 
735 
736 /*
737  * Print a progress report based on the global variables. If verbose output
738  * is enabled, also print the current file name.
739  *
740  * Progress report is written at maximum once per second, unless the
741  * force parameter is set to true.
742  */
743 static void
744 progress_report(int tablespacenum, const char *filename, bool force)
745 {
746  int percent;
747  char totaldone_str[32];
748  char totalsize_str[32];
749  pg_time_t now;
750 
751  if (!showprogress)
752  return;
753 
754  now = time(NULL);
755  if (now == last_progress_report && !force)
756  return; /* Max once per second */
757 
759  percent = totalsize_kb ? (int) ((totaldone / 1024) * 100 / totalsize_kb) : 0;
760 
761  /*
762  * Avoid overflowing past 100% or the full size. This may make the total
763  * size number change as we approach the end of the backup (the estimate
764  * will always be wrong if WAL is included), but that's better than having
765  * the done column be bigger than the total.
766  */
767  if (percent > 100)
768  percent = 100;
769  if (totaldone / 1024 > totalsize_kb)
770  totalsize_kb = totaldone / 1024;
771 
772  /*
773  * Separate step to keep platform-dependent format code out of
774  * translatable strings. And we only test for INT64_FORMAT availability
775  * in snprintf, not fprintf.
776  */
777  snprintf(totaldone_str, sizeof(totaldone_str), INT64_FORMAT,
778  totaldone / 1024);
779  snprintf(totalsize_str, sizeof(totalsize_str), INT64_FORMAT, totalsize_kb);
780 
781 #define VERBOSE_FILENAME_LENGTH 35
782  if (verbose)
783  {
784  if (!filename)
785 
786  /*
787  * No filename given, so clear the status line (used for last
788  * call)
789  */
790  fprintf(stderr,
791  ngettext("%*s/%s kB (100%%), %d/%d tablespace %*s",
792  "%*s/%s kB (100%%), %d/%d tablespaces %*s",
794  (int) strlen(totalsize_str),
795  totaldone_str, totalsize_str,
796  tablespacenum, tablespacecount,
797  VERBOSE_FILENAME_LENGTH + 5, "");
798  else
799  {
800  bool truncate = (strlen(filename) > VERBOSE_FILENAME_LENGTH);
801 
802  fprintf(stderr,
803  ngettext("%*s/%s kB (%d%%), %d/%d tablespace (%s%-*.*s)",
804  "%*s/%s kB (%d%%), %d/%d tablespaces (%s%-*.*s)",
806  (int) strlen(totalsize_str),
807  totaldone_str, totalsize_str, percent,
808  tablespacenum, tablespacecount,
809  /* Prefix with "..." if we do leading truncation */
810  truncate ? "..." : "",
813  /* Truncate filename at beginning if it's too long */
814  truncate ? filename + strlen(filename) - VERBOSE_FILENAME_LENGTH + 3 : filename);
815  }
816  }
817  else
818  fprintf(stderr,
819  ngettext("%*s/%s kB (%d%%), %d/%d tablespace",
820  "%*s/%s kB (%d%%), %d/%d tablespaces",
822  (int) strlen(totalsize_str),
823  totaldone_str, totalsize_str, percent,
824  tablespacenum, tablespacecount);
825 
826  if (isatty(fileno(stderr)))
827  fprintf(stderr, "\r");
828  else
829  fprintf(stderr, "\n");
830 }
831 
832 static int32
833 parse_max_rate(char *src)
834 {
835  double result;
836  char *after_num;
837  char *suffix = NULL;
838 
839  errno = 0;
840  result = strtod(src, &after_num);
841  if (src == after_num)
842  {
843  pg_log_error("transfer rate \"%s\" is not a valid value", src);
844  exit(1);
845  }
846  if (errno != 0)
847  {
848  pg_log_error("invalid transfer rate \"%s\": %m", src);
849  exit(1);
850  }
851 
852  if (result <= 0)
853  {
854  /*
855  * Reject obviously wrong values here.
856  */
857  pg_log_error("transfer rate must be greater than zero");
858  exit(1);
859  }
860 
861  /*
862  * Evaluate suffix, after skipping over possible whitespace. Lack of
863  * suffix means kilobytes.
864  */
865  while (*after_num != '\0' && isspace((unsigned char) *after_num))
866  after_num++;
867 
868  if (*after_num != '\0')
869  {
870  suffix = after_num;
871  if (*after_num == 'k')
872  {
873  /* kilobyte is the expected unit. */
874  after_num++;
875  }
876  else if (*after_num == 'M')
877  {
878  after_num++;
879  result *= 1024.0;
880  }
881  }
882 
883  /* The rest can only consist of white space. */
884  while (*after_num != '\0' && isspace((unsigned char) *after_num))
885  after_num++;
886 
887  if (*after_num != '\0')
888  {
889  pg_log_error("invalid --max-rate unit: \"%s\"", suffix);
890  exit(1);
891  }
892 
893  /* Valid integer? */
894  if ((uint64) result != (uint64) ((uint32) result))
895  {
896  pg_log_error("transfer rate \"%s\" exceeds integer range", src);
897  exit(1);
898  }
899 
900  /*
901  * The range is checked on the server side too, but avoid the server
902  * connection if a nonsensical value was passed.
903  */
904  if (result < MAX_RATE_LOWER || result > MAX_RATE_UPPER)
905  {
906  pg_log_error("transfer rate \"%s\" is out of range", src);
907  exit(1);
908  }
909 
910  return (int32) result;
911 }
912 
913 /*
914  * Read a stream of COPY data and invoke the provided callback for each
915  * chunk.
916  */
917 static void
919  void *callback_data)
920 {
921  PGresult *res;
922 
923  /* Get the COPY data stream. */
924  res = PQgetResult(conn);
925  if (PQresultStatus(res) != PGRES_COPY_OUT)
926  {
927  pg_log_error("could not get COPY data stream: %s",
928  PQerrorMessage(conn));
929  exit(1);
930  }
931  PQclear(res);
932 
933  /* Loop over chunks until done. */
934  while (1)
935  {
936  int r;
937  char *copybuf;
938 
939  r = PQgetCopyData(conn, &copybuf, 0);
940  if (r == -1)
941  {
942  /* End of chunk. */
943  break;
944  }
945  else if (r == -2)
946  {
947  pg_log_error("could not read COPY data: %s",
948  PQerrorMessage(conn));
949  exit(1);
950  }
951 
952  (*callback) (r, copybuf, callback_data);
953 
954  PQfreemem(copybuf);
955  }
956 }
957 
958 /*
959  * Write a piece of tar data
960  */
961 static void
963 {
964 #ifdef HAVE_LIBZ
965  if (state->ztarfile != NULL)
966  {
967  if (gzwrite(state->ztarfile, buf, r) != r)
968  {
969  pg_log_error("could not write to compressed file \"%s\": %s",
970  state->filename, get_gz_error(state->ztarfile));
971  exit(1);
972  }
973  }
974  else
975 #endif
976  {
977  if (fwrite(buf, r, 1, state->tarfile) != 1)
978  {
979  pg_log_error("could not write to file \"%s\": %m",
980  state->filename);
981  exit(1);
982  }
983  }
984 }
985 
986 /*
987  * Receive a tar format file from the connection to the server, and write
988  * the data from this file directly into a tar file. If compression is
989  * enabled, the data will be compressed while written to the file.
990  *
991  * The file will be named base.tar[.gz] if it's for the main data directory
992  * or <tablespaceoid>.tar[.gz] if it's for another tablespace.
993  *
994  * No attempt to inspect or validate the contents of the file is done.
995  */
996 static void
997 ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
998 {
999  char zerobuf[1024];
1001 
1002  memset(&state, 0, sizeof(state));
1003  state.tablespacenum = rownum;
1004  state.basetablespace = PQgetisnull(res, rownum, 0);
1005  state.in_tarhdr = true;
1006 
1007  /* recovery.conf is integrated into postgresql.conf in 12 and newer */
1009  state.is_recovery_guc_supported = true;
1010 
1011  if (state.basetablespace)
1012  {
1013  /*
1014  * Base tablespaces
1015  */
1016  if (strcmp(basedir, "-") == 0)
1017  {
1018 #ifdef WIN32
1019  _setmode(fileno(stdout), _O_BINARY);
1020 #endif
1021 
1022 #ifdef HAVE_LIBZ
1023  if (compresslevel != 0)
1024  {
1025  state.ztarfile = gzdopen(dup(fileno(stdout)), "wb");
1026  if (gzsetparams(state.ztarfile, compresslevel,
1027  Z_DEFAULT_STRATEGY) != Z_OK)
1028  {
1029  pg_log_error("could not set compression level %d: %s",
1030  compresslevel, get_gz_error(state.ztarfile));
1031  exit(1);
1032  }
1033  }
1034  else
1035 #endif
1036  state.tarfile = stdout;
1037  strcpy(state.filename, "-");
1038  }
1039  else
1040  {
1041 #ifdef HAVE_LIBZ
1042  if (compresslevel != 0)
1043  {
1044  snprintf(state.filename, sizeof(state.filename),
1045  "%s/base.tar.gz", basedir);
1046  state.ztarfile = gzopen(state.filename, "wb");
1047  if (gzsetparams(state.ztarfile, compresslevel,
1048  Z_DEFAULT_STRATEGY) != Z_OK)
1049  {
1050  pg_log_error("could not set compression level %d: %s",
1051  compresslevel, get_gz_error(state.ztarfile));
1052  exit(1);
1053  }
1054  }
1055  else
1056 #endif
1057  {
1058  snprintf(state.filename, sizeof(state.filename),
1059  "%s/base.tar", basedir);
1060  state.tarfile = fopen(state.filename, "wb");
1061  }
1062  }
1063  }
1064  else
1065  {
1066  /*
1067  * Specific tablespace
1068  */
1069 #ifdef HAVE_LIBZ
1070  if (compresslevel != 0)
1071  {
1072  snprintf(state.filename, sizeof(state.filename),
1073  "%s/%s.tar.gz",
1074  basedir, PQgetvalue(res, rownum, 0));
1075  state.ztarfile = gzopen(state.filename, "wb");
1076  if (gzsetparams(state.ztarfile, compresslevel,
1077  Z_DEFAULT_STRATEGY) != Z_OK)
1078  {
1079  pg_log_error("could not set compression level %d: %s",
1080  compresslevel, get_gz_error(state.ztarfile));
1081  exit(1);
1082  }
1083  }
1084  else
1085 #endif
1086  {
1087  snprintf(state.filename, sizeof(state.filename), "%s/%s.tar",
1088  basedir, PQgetvalue(res, rownum, 0));
1089  state.tarfile = fopen(state.filename, "wb");
1090  }
1091  }
1092 
1093 #ifdef HAVE_LIBZ
1094  if (compresslevel != 0)
1095  {
1096  if (!state.ztarfile)
1097  {
1098  /* Compression is in use */
1099  pg_log_error("could not create compressed file \"%s\": %s",
1100  state.filename, get_gz_error(state.ztarfile));
1101  exit(1);
1102  }
1103  }
1104  else
1105 #endif
1106  {
1107  /* Either no zlib support, or zlib support but compresslevel = 0 */
1108  if (!state.tarfile)
1109  {
1110  pg_log_error("could not create file \"%s\": %m", state.filename);
1111  exit(1);
1112  }
1113  }
1114 
1115  ReceiveCopyData(conn, ReceiveTarCopyChunk, &state);
1116 
1117  /*
1118  * End of copy data. If requested, and this is the base tablespace, write
1119  * configuration file into the tarfile. When done, close the file (but not
1120  * stdout).
1121  *
1122  * Also, write two completely empty blocks at the end of the tar file, as
1123  * required by some tar programs.
1124  */
1125 
1126  MemSet(zerobuf, 0, sizeof(zerobuf));
1127 
1128  if (state.basetablespace && writerecoveryconf)
1129  {
1130  char header[512];
1131 
1132  /*
1133  * If postgresql.auto.conf has not been found in the streamed data,
1134  * add recovery configuration to postgresql.auto.conf if recovery
1135  * parameters are GUCs. If the instance connected to is older than
1136  * 12, create recovery.conf with this data otherwise.
1137  */
1139  {
1140  int padding;
1141 
1142  tarCreateHeader(header,
1143  state.is_recovery_guc_supported ? "postgresql.auto.conf" : "recovery.conf",
1144  NULL,
1145  recoveryconfcontents->len,
1146  pg_file_create_mode, 04000, 02000,
1147  time(NULL));
1148 
1149  padding = ((recoveryconfcontents->len + 511) & ~511) - recoveryconfcontents->len;
1150 
1151  writeTarData(&state, header, sizeof(header));
1152  writeTarData(&state, recoveryconfcontents->data,
1153  recoveryconfcontents->len);
1154  if (padding)
1155  writeTarData(&state, zerobuf, padding);
1156  }
1157 
1158  /*
1159  * standby.signal is supported only if recovery parameters are GUCs.
1160  */
1161  if (state.is_recovery_guc_supported)
1162  {
1163  tarCreateHeader(header, "standby.signal", NULL,
1164  0, /* zero-length file */
1165  pg_file_create_mode, 04000, 02000,
1166  time(NULL));
1167 
1168  writeTarData(&state, header, sizeof(header));
1169  writeTarData(&state, zerobuf, 511);
1170  }
1171  }
1172 
1173  /* 2 * 512 bytes empty data at end of file */
1174  writeTarData(&state, zerobuf, sizeof(zerobuf));
1175 
1176 #ifdef HAVE_LIBZ
1177  if (state.ztarfile != NULL)
1178  {
1179  if (gzclose(state.ztarfile) != 0)
1180  {
1181  pg_log_error("could not close compressed file \"%s\": %s",
1182  state.filename, get_gz_error(state.ztarfile));
1183  exit(1);
1184  }
1185  }
1186  else
1187 #endif
1188  {
1189  if (strcmp(basedir, "-") != 0)
1190  {
1191  if (fclose(state.tarfile) != 0)
1192  {
1193  pg_log_error("could not close file \"%s\": %m",
1194  state.filename);
1195  exit(1);
1196  }
1197  }
1198  }
1199 
1200  progress_report(rownum, state.filename, true);
1201 
1202  /*
1203  * Do not sync the resulting tar file yet, all files are synced once at
1204  * the end.
1205  */
1206 }
1207 
1208 /*
1209  * Receive one chunk of tar-format data from the server.
1210  */
1211 static void
1212 ReceiveTarCopyChunk(size_t r, char *copybuf, void *callback_data)
1213 {
1214  WriteTarState *state = callback_data;
1215 
1216  if (!writerecoveryconf || !state->basetablespace)
1217  {
1218  /*
1219  * When not writing config file, or when not working on the base
1220  * tablespace, we never have to look for an existing configuration
1221  * file in the stream.
1222  */
1223  writeTarData(state, copybuf, r);
1224  }
1225  else
1226  {
1227  /*
1228  * Look for a config file in the existing tar stream. If it's there,
1229  * we must skip it so we can later overwrite it with our own version
1230  * of the file.
1231  *
1232  * To do this, we have to process the individual files inside the TAR
1233  * stream. The stream consists of a header and zero or more chunks,
1234  * all 512 bytes long. The stream from the server is broken up into
1235  * smaller pieces, so we have to track the size of the files to find
1236  * the next header structure.
1237  */
1238  int rr = r;
1239  int pos = 0;
1240 
1241  while (rr > 0)
1242  {
1243  if (state->in_tarhdr)
1244  {
1245  /*
1246  * We're currently reading a header structure inside the TAR
1247  * stream, i.e. the file metadata.
1248  */
1249  if (state->tarhdrsz < 512)
1250  {
1251  /*
1252  * Copy the header structure into tarhdr in case the
1253  * header is not aligned to 512 bytes or it's not returned
1254  * in whole by the last PQgetCopyData call.
1255  */
1256  int hdrleft;
1257  int bytes2copy;
1258 
1259  hdrleft = 512 - state->tarhdrsz;
1260  bytes2copy = (rr > hdrleft ? hdrleft : rr);
1261 
1262  memcpy(&state->tarhdr[state->tarhdrsz], copybuf + pos,
1263  bytes2copy);
1264 
1265  rr -= bytes2copy;
1266  pos += bytes2copy;
1267  state->tarhdrsz += bytes2copy;
1268  }
1269  else
1270  {
1271  /*
1272  * We have the complete header structure in tarhdr, look
1273  * at the file metadata: we may want append recovery info
1274  * into postgresql.auto.conf and skip standby.signal file
1275  * if recovery parameters are integrated as GUCs, and
1276  * recovery.conf otherwise. In both cases we must
1277  * calculate tar padding.
1278  */
1279  if (state->is_recovery_guc_supported)
1280  {
1281  state->skip_file =
1282  (strcmp(&state->tarhdr[0], "standby.signal") == 0);
1283  state->is_postgresql_auto_conf =
1284  (strcmp(&state->tarhdr[0], "postgresql.auto.conf") == 0);
1285  }
1286  else
1287  state->skip_file =
1288  (strcmp(&state->tarhdr[0], "recovery.conf") == 0);
1289 
1290  state->filesz = read_tar_number(&state->tarhdr[124], 12);
1291  state->file_padding_len =
1292  ((state->filesz + 511) & ~511) - state->filesz;
1293 
1294  if (state->is_recovery_guc_supported &&
1295  state->is_postgresql_auto_conf &&
1297  {
1298  /* replace tar header */
1299  char header[512];
1300 
1301  tarCreateHeader(header, "postgresql.auto.conf", NULL,
1302  state->filesz + recoveryconfcontents->len,
1303  pg_file_create_mode, 04000, 02000,
1304  time(NULL));
1305 
1306  writeTarData(state, header, sizeof(header));
1307  }
1308  else
1309  {
1310  /* copy stream with padding */
1311  state->filesz += state->file_padding_len;
1312 
1313  if (!state->skip_file)
1314  {
1315  /*
1316  * If we're not skipping the file, write the tar
1317  * header unmodified.
1318  */
1319  writeTarData(state, state->tarhdr, 512);
1320  }
1321  }
1322 
1323  /* Next part is the file, not the header */
1324  state->in_tarhdr = false;
1325  }
1326  }
1327  else
1328  {
1329  /*
1330  * We're processing a file's contents.
1331  */
1332  if (state->filesz > 0)
1333  {
1334  /*
1335  * We still have data to read (and possibly write).
1336  */
1337  int bytes2write;
1338 
1339  bytes2write = (state->filesz > rr ? rr : state->filesz);
1340 
1341  if (!state->skip_file)
1342  writeTarData(state, copybuf + pos, bytes2write);
1343 
1344  rr -= bytes2write;
1345  pos += bytes2write;
1346  state->filesz -= bytes2write;
1347  }
1348  else if (state->is_recovery_guc_supported &&
1349  state->is_postgresql_auto_conf &&
1351  {
1352  /* append recovery config to postgresql.auto.conf */
1353  int padding;
1354  int tailsize;
1355 
1356  tailsize = (512 - state->file_padding_len) + recoveryconfcontents->len;
1357  padding = ((tailsize + 511) & ~511) - tailsize;
1358 
1359  writeTarData(state, recoveryconfcontents->data,
1360  recoveryconfcontents->len);
1361 
1362  if (padding)
1363  {
1364  char zerobuf[512];
1365 
1366  MemSet(zerobuf, 0, sizeof(zerobuf));
1367  writeTarData(state, zerobuf, padding);
1368  }
1369 
1370  /* skip original file padding */
1371  state->is_postgresql_auto_conf = false;
1372  state->skip_file = true;
1373  state->filesz += state->file_padding_len;
1374 
1375  state->found_postgresql_auto_conf = true;
1376  }
1377  else
1378  {
1379  /*
1380  * No more data in the current file, the next piece of
1381  * data (if any) will be a new file header structure.
1382  */
1383  state->in_tarhdr = true;
1384  state->skip_file = false;
1385  state->is_postgresql_auto_conf = false;
1386  state->tarhdrsz = 0;
1387  state->filesz = 0;
1388  }
1389  }
1390  }
1391  }
1392  totaldone += r;
1393  progress_report(state->tablespacenum, state->filename, false);
1394 }
1395 
1396 
1397 /*
1398  * Retrieve tablespace path, either relocated or original depending on whether
1399  * -T was passed or not.
1400  */
1401 static const char *
1402 get_tablespace_mapping(const char *dir)
1403 {
1404  TablespaceListCell *cell;
1405  char canon_dir[MAXPGPATH];
1406 
1407  /* Canonicalize path for comparison consistency */
1408  strlcpy(canon_dir, dir, sizeof(canon_dir));
1409  canonicalize_path(canon_dir);
1410 
1411  for (cell = tablespace_dirs.head; cell; cell = cell->next)
1412  if (strcmp(canon_dir, cell->old_dir) == 0)
1413  return cell->new_dir;
1414 
1415  return dir;
1416 }
1417 
1418 
1419 /*
1420  * Receive a tar format stream from the connection to the server, and unpack
1421  * the contents of it into a directory. Only files, directories and
1422  * symlinks are supported, no other kinds of special files.
1423  *
1424  * If the data is for the main data directory, it will be restored in the
1425  * specified directory. If it's for another tablespace, it will be restored
1426  * in the original or mapped directory.
1427  */
1428 static void
1430 {
1432  bool basetablespace;
1433 
1434  memset(&state, 0, sizeof(state));
1435  state.tablespacenum = rownum;
1436 
1437  basetablespace = PQgetisnull(res, rownum, 0);
1438  if (basetablespace)
1439  strlcpy(state.current_path, basedir, sizeof(state.current_path));
1440  else
1441  strlcpy(state.current_path,
1442  get_tablespace_mapping(PQgetvalue(res, rownum, 1)),
1443  sizeof(state.current_path));
1444 
1446 
1447 
1448  if (state.file)
1449  fclose(state.file);
1450 
1451  progress_report(rownum, state.filename, true);
1452 
1453  if (state.file != NULL)
1454  {
1455  pg_log_error("COPY stream ended before last file was finished");
1456  exit(1);
1457  }
1458 
1459  if (basetablespace && writerecoveryconf)
1460  WriteRecoveryConfig(conn, basedir, recoveryconfcontents);
1461 
1462  /*
1463  * No data is synced here, everything is done for all tablespaces at the
1464  * end.
1465  */
1466 }
1467 
1468 static void
1469 ReceiveTarAndUnpackCopyChunk(size_t r, char *copybuf, void *callback_data)
1470 {
1471  UnpackTarState *state = callback_data;
1472 
1473  if (state->file == NULL)
1474  {
1475 #ifndef WIN32
1476  int filemode;
1477 #endif
1478 
1479  /*
1480  * No current file, so this must be the header for a new file
1481  */
1482  if (r != 512)
1483  {
1484  pg_log_error("invalid tar block header size: %zu", r);
1485  exit(1);
1486  }
1487  totaldone += 512;
1488 
1489  state->current_len_left = read_tar_number(&copybuf[124], 12);
1490 
1491 #ifndef WIN32
1492  /* Set permissions on the file */
1493  filemode = read_tar_number(&copybuf[100], 8);
1494 #endif
1495 
1496  /*
1497  * All files are padded up to 512 bytes
1498  */
1499  state->current_padding =
1500  ((state->current_len_left + 511) & ~511) - state->current_len_left;
1501 
1502  /*
1503  * First part of header is zero terminated filename
1504  */
1505  snprintf(state->filename, sizeof(state->filename),
1506  "%s/%s", state->current_path, copybuf);
1507  if (state->filename[strlen(state->filename) - 1] == '/')
1508  {
1509  /*
1510  * Ends in a slash means directory or symlink to directory
1511  */
1512  if (copybuf[156] == '5')
1513  {
1514  /*
1515  * Directory. Remove trailing slash first.
1516  */
1517  state->filename[strlen(state->filename) - 1] = '\0';
1518  if (mkdir(state->filename, pg_dir_create_mode) != 0)
1519  {
1520  /*
1521  * When streaming WAL, pg_wal (or pg_xlog for pre-9.6
1522  * clusters) will have been created by the wal receiver
1523  * process. Also, when the WAL directory location was
1524  * specified, pg_wal (or pg_xlog) has already been created
1525  * as a symbolic link before starting the actual backup.
1526  * So just ignore creation failures on related
1527  * directories.
1528  */
1529  if (!((pg_str_endswith(state->filename, "/pg_wal") ||
1530  pg_str_endswith(state->filename, "/pg_xlog") ||
1531  pg_str_endswith(state->filename, "/archive_status")) &&
1532  errno == EEXIST))
1533  {
1534  pg_log_error("could not create directory \"%s\": %m",
1535  state->filename);
1536  exit(1);
1537  }
1538  }
1539 #ifndef WIN32
1540  if (chmod(state->filename, (mode_t) filemode))
1541  pg_log_error("could not set permissions on directory \"%s\": %m",
1542  state->filename);
1543 #endif
1544  }
1545  else if (copybuf[156] == '2')
1546  {
1547  /*
1548  * Symbolic link
1549  *
1550  * It's most likely a link in pg_tblspc directory, to the
1551  * location of a tablespace. Apply any tablespace mapping
1552  * given on the command line (--tablespace-mapping). (We
1553  * blindly apply the mapping without checking that the link
1554  * really is inside pg_tblspc. We don't expect there to be
1555  * other symlinks in a data directory, but if there are, you
1556  * can call it an undocumented feature that you can map them
1557  * too.)
1558  */
1559  state->filename[strlen(state->filename) - 1] = '\0'; /* Remove trailing slash */
1560 
1561  state->mapped_tblspc_path =
1562  get_tablespace_mapping(&copybuf[157]);
1563  if (symlink(state->mapped_tblspc_path, state->filename) != 0)
1564  {
1565  pg_log_error("could not create symbolic link from \"%s\" to \"%s\": %m",
1566  state->filename, state->mapped_tblspc_path);
1567  exit(1);
1568  }
1569  }
1570  else
1571  {
1572  pg_log_error("unrecognized link indicator \"%c\"",
1573  copybuf[156]);
1574  exit(1);
1575  }
1576  return; /* directory or link handled */
1577  }
1578 
1579  /*
1580  * regular file
1581  */
1582  state->file = fopen(state->filename, "wb");
1583  if (!state->file)
1584  {
1585  pg_log_error("could not create file \"%s\": %m", state->filename);
1586  exit(1);
1587  }
1588 
1589 #ifndef WIN32
1590  if (chmod(state->filename, (mode_t) filemode))
1591  pg_log_error("could not set permissions on file \"%s\": %m",
1592  state->filename);
1593 #endif
1594 
1595  if (state->current_len_left == 0)
1596  {
1597  /*
1598  * Done with this file, next one will be a new tar header
1599  */
1600  fclose(state->file);
1601  state->file = NULL;
1602  return;
1603  }
1604  } /* new file */
1605  else
1606  {
1607  /*
1608  * Continuing blocks in existing file
1609  */
1610  if (state->current_len_left == 0 && r == state->current_padding)
1611  {
1612  /*
1613  * Received the padding block for this file, ignore it and close
1614  * the file, then move on to the next tar header.
1615  */
1616  fclose(state->file);
1617  state->file = NULL;
1618  totaldone += r;
1619  return;
1620  }
1621 
1622  if (fwrite(copybuf, r, 1, state->file) != 1)
1623  {
1624  pg_log_error("could not write to file \"%s\": %m", state->filename);
1625  exit(1);
1626  }
1627  totaldone += r;
1628  progress_report(state->tablespacenum, state->filename, false);
1629 
1630  state->current_len_left -= r;
1631  if (state->current_len_left == 0 && state->current_padding == 0)
1632  {
1633  /*
1634  * Received the last block, and there is no padding to be
1635  * expected. Close the file and move on to the next tar header.
1636  */
1637  fclose(state->file);
1638  state->file = NULL;
1639  return;
1640  }
1641  } /* continuing data in existing file */
1642 }
1643 
1644 static void
1646 {
1647  PGresult *res;
1648  char *sysidentifier;
1649  TimeLineID latesttli;
1650  TimeLineID starttli;
1651  char *basebkp;
1652  char escaped_label[MAXPGPATH];
1653  char *maxrate_clause = NULL;
1654  int i;
1655  char xlogstart[64];
1656  char xlogend[64];
1657  int minServerMajor,
1658  maxServerMajor;
1659  int serverVersion,
1660  serverMajor;
1661 
1662  Assert(conn != NULL);
1663 
1664  /*
1665  * Check server version. BASE_BACKUP command was introduced in 9.1, so we
1666  * can't work with servers older than 9.1.
1667  */
1668  minServerMajor = 901;
1669  maxServerMajor = PG_VERSION_NUM / 100;
1670  serverVersion = PQserverVersion(conn);
1671  serverMajor = serverVersion / 100;
1672  if (serverMajor < minServerMajor || serverMajor > maxServerMajor)
1673  {
1674  const char *serverver = PQparameterStatus(conn, "server_version");
1675 
1676  pg_log_error("incompatible server version %s",
1677  serverver ? serverver : "'unknown'");
1678  exit(1);
1679  }
1680 
1681  /*
1682  * If WAL streaming was requested, also check that the server is new
1683  * enough for that.
1684  */
1686  {
1687  /*
1688  * Error message already written in CheckServerVersionForStreaming(),
1689  * but add a hint about using -X none.
1690  */
1691  pg_log_info("HINT: use -X none or -X fetch to disable log streaming");
1692  exit(1);
1693  }
1694 
1695  /*
1696  * Build contents of configuration file if requested
1697  */
1698  if (writerecoveryconf)
1699  recoveryconfcontents = GenerateRecoveryConfig(conn, replication_slot);
1700 
1701  /*
1702  * Run IDENTIFY_SYSTEM so we can get the timeline
1703  */
1704  if (!RunIdentifySystem(conn, &sysidentifier, &latesttli, NULL, NULL))
1705  exit(1);
1706 
1707  /*
1708  * Start the actual backup
1709  */
1710  PQescapeStringConn(conn, escaped_label, label, sizeof(escaped_label), &i);
1711 
1712  if (maxrate > 0)
1713  maxrate_clause = psprintf("MAX_RATE %u", maxrate);
1714 
1715  if (verbose)
1716  pg_log_info("initiating base backup, waiting for checkpoint to complete");
1717 
1718  if (showprogress && !verbose)
1719  {
1720  fprintf(stderr, "waiting for checkpoint");
1721  if (isatty(fileno(stderr)))
1722  fprintf(stderr, "\r");
1723  else
1724  fprintf(stderr, "\n");
1725  }
1726 
1727  basebkp =
1728  psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s",
1729  escaped_label,
1730  showprogress ? "PROGRESS" : "",
1731  includewal == FETCH_WAL ? "WAL" : "",
1732  fastcheckpoint ? "FAST" : "",
1733  includewal == NO_WAL ? "" : "NOWAIT",
1734  maxrate_clause ? maxrate_clause : "",
1735  format == 't' ? "TABLESPACE_MAP" : "",
1736  verify_checksums ? "" : "NOVERIFY_CHECKSUMS");
1737 
1738  if (PQsendQuery(conn, basebkp) == 0)
1739  {
1740  pg_log_error("could not send replication command \"%s\": %s",
1741  "BASE_BACKUP", PQerrorMessage(conn));
1742  exit(1);
1743  }
1744 
1745  /*
1746  * Get the starting WAL location
1747  */
1748  res = PQgetResult(conn);
1749  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1750  {
1751  pg_log_error("could not initiate base backup: %s",
1752  PQerrorMessage(conn));
1753  exit(1);
1754  }
1755  if (PQntuples(res) != 1)
1756  {
1757  pg_log_error("server returned unexpected response to BASE_BACKUP command; got %d rows and %d fields, expected %d rows and %d fields",
1758  PQntuples(res), PQnfields(res), 1, 2);
1759  exit(1);
1760  }
1761 
1762  strlcpy(xlogstart, PQgetvalue(res, 0, 0), sizeof(xlogstart));
1763 
1764  if (verbose)
1765  pg_log_info("checkpoint completed");
1766 
1767  /*
1768  * 9.3 and later sends the TLI of the starting point. With older servers,
1769  * assume it's the same as the latest timeline reported by
1770  * IDENTIFY_SYSTEM.
1771  */
1772  if (PQnfields(res) >= 2)
1773  starttli = atoi(PQgetvalue(res, 0, 1));
1774  else
1775  starttli = latesttli;
1776  PQclear(res);
1777  MemSet(xlogend, 0, sizeof(xlogend));
1778 
1779  if (verbose && includewal != NO_WAL)
1780  pg_log_info("write-ahead log start point: %s on timeline %u",
1781  xlogstart, starttli);
1782 
1783  /*
1784  * Get the header
1785  */
1786  res = PQgetResult(conn);
1787  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1788  {
1789  pg_log_error("could not get backup header: %s",
1790  PQerrorMessage(conn));
1791  exit(1);
1792  }
1793  if (PQntuples(res) < 1)
1794  {
1795  pg_log_error("no data returned from server");
1796  exit(1);
1797  }
1798 
1799  /*
1800  * Sum up the total size, for progress reporting
1801  */
1802  totalsize_kb = totaldone = 0;
1803  tablespacecount = PQntuples(res);
1804  for (i = 0; i < PQntuples(res); i++)
1805  {
1806  totalsize_kb += atol(PQgetvalue(res, i, 2));
1807 
1808  /*
1809  * Verify tablespace directories are empty. Don't bother with the
1810  * first once since it can be relocated, and it will be checked before
1811  * we do anything anyway.
1812  */
1813  if (format == 'p' && !PQgetisnull(res, i, 1))
1814  {
1815  char *path = unconstify(char *, get_tablespace_mapping(PQgetvalue(res, i, 1)));
1816 
1818  }
1819  }
1820 
1821  /*
1822  * When writing to stdout, require a single tablespace
1823  */
1824  if (format == 't' && strcmp(basedir, "-") == 0 && PQntuples(res) > 1)
1825  {
1826  pg_log_error("can only write single tablespace to stdout, database has %d",
1827  PQntuples(res));
1828  exit(1);
1829  }
1830 
1831  /*
1832  * If we're streaming WAL, start the streaming session before we start
1833  * receiving the actual data chunks.
1834  */
1835  if (includewal == STREAM_WAL)
1836  {
1837  if (verbose)
1838  pg_log_info("starting background WAL receiver");
1839  StartLogStreamer(xlogstart, starttli, sysidentifier);
1840  }
1841 
1842  /*
1843  * Start receiving chunks
1844  */
1845  for (i = 0; i < PQntuples(res); i++)
1846  {
1847  if (format == 't')
1848  ReceiveTarFile(conn, res, i);
1849  else
1850  ReceiveAndUnpackTarFile(conn, res, i);
1851  } /* Loop over all tablespaces */
1852 
1853  if (showprogress)
1854  {
1855  progress_report(PQntuples(res), NULL, true);
1856  if (isatty(fileno(stderr)))
1857  fprintf(stderr, "\n"); /* Need to move to next line */
1858  }
1859 
1860  PQclear(res);
1861 
1862  /*
1863  * Get the stop position
1864  */
1865  res = PQgetResult(conn);
1866  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1867  {
1868  pg_log_error("could not get write-ahead log end position from server: %s",
1869  PQerrorMessage(conn));
1870  exit(1);
1871  }
1872  if (PQntuples(res) != 1)
1873  {
1874  pg_log_error("no write-ahead log end position returned from server");
1875  exit(1);
1876  }
1877  strlcpy(xlogend, PQgetvalue(res, 0, 0), sizeof(xlogend));
1878  if (verbose && includewal != NO_WAL)
1879  pg_log_info("write-ahead log end point: %s", xlogend);
1880  PQclear(res);
1881 
1882  res = PQgetResult(conn);
1883  if (PQresultStatus(res) != PGRES_COMMAND_OK)
1884  {
1885  const char *sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
1886 
1887  if (sqlstate &&
1888  strcmp(sqlstate, ERRCODE_DATA_CORRUPTED) == 0)
1889  {
1890  pg_log_error("checksum error occurred");
1891  checksum_failure = true;
1892  }
1893  else
1894  {
1895  pg_log_error("final receive failed: %s",
1896  PQerrorMessage(conn));
1897  }
1898  exit(1);
1899  }
1900 
1901  if (bgchild > 0)
1902  {
1903 #ifndef WIN32
1904  int status;
1905  pid_t r;
1906 #else
1907  DWORD status;
1908 
1909  /*
1910  * get a pointer sized version of bgchild to avoid warnings about
1911  * casting to a different size on WIN64.
1912  */
1913  intptr_t bgchild_handle = bgchild;
1914  uint32 hi,
1915  lo;
1916 #endif
1917 
1918  if (verbose)
1919  pg_log_info("waiting for background process to finish streaming ...");
1920 
1921 #ifndef WIN32
1922  if (write(bgpipe[1], xlogend, strlen(xlogend)) != strlen(xlogend))
1923  {
1924  pg_log_info("could not send command to background pipe: %m");
1925  exit(1);
1926  }
1927 
1928  /* Just wait for the background process to exit */
1929  r = waitpid(bgchild, &status, 0);
1930  if (r == (pid_t) -1)
1931  {
1932  pg_log_error("could not wait for child process: %m");
1933  exit(1);
1934  }
1935  if (r != bgchild)
1936  {
1937  pg_log_error("child %d died, expected %d", (int) r, (int) bgchild);
1938  exit(1);
1939  }
1940  if (status != 0)
1941  {
1942  pg_log_error("%s", wait_result_to_str(status));
1943  exit(1);
1944  }
1945  /* Exited normally, we're happy! */
1946 #else /* WIN32 */
1947 
1948  /*
1949  * On Windows, since we are in the same process, we can just store the
1950  * value directly in the variable, and then set the flag that says
1951  * it's there.
1952  */
1953  if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
1954  {
1955  pg_log_error("could not parse write-ahead log location \"%s\"",
1956  xlogend);
1957  exit(1);
1958  }
1959  xlogendptr = ((uint64) hi) << 32 | lo;
1960  InterlockedIncrement(&has_xlogendptr);
1961 
1962  /* First wait for the thread to exit */
1963  if (WaitForSingleObjectEx((HANDLE) bgchild_handle, INFINITE, FALSE) !=
1964  WAIT_OBJECT_0)
1965  {
1966  _dosmaperr(GetLastError());
1967  pg_log_error("could not wait for child thread: %m");
1968  exit(1);
1969  }
1970  if (GetExitCodeThread((HANDLE) bgchild_handle, &status) == 0)
1971  {
1972  _dosmaperr(GetLastError());
1973  pg_log_error("could not get child thread exit status: %m");
1974  exit(1);
1975  }
1976  if (status != 0)
1977  {
1978  pg_log_error("child thread exited with error %u",
1979  (unsigned int) status);
1980  exit(1);
1981  }
1982  /* Exited normally, we're happy */
1983 #endif
1984  }
1985 
1986  /* Free the configuration file contents */
1987  destroyPQExpBuffer(recoveryconfcontents);
1988 
1989  /*
1990  * End of copy data. Final result is already checked inside the loop.
1991  */
1992  PQclear(res);
1993  PQfinish(conn);
1994  conn = NULL;
1995 
1996  /*
1997  * Make data persistent on disk once backup is completed. For tar format
1998  * sync the parent directory and all its contents as each tar file was not
1999  * synced after being completed. In plain format, all the data of the
2000  * base directory is synced, taking into account all the tablespaces.
2001  * Errors are not considered fatal.
2002  */
2003  if (do_sync)
2004  {
2005  if (verbose)
2006  pg_log_info("syncing data to disk ...");
2007  if (format == 't')
2008  {
2009  if (strcmp(basedir, "-") != 0)
2010  (void) fsync_dir_recurse(basedir);
2011  }
2012  else
2013  {
2014  (void) fsync_pgdata(basedir, serverVersion);
2015  }
2016  }
2017 
2018  if (verbose)
2019  pg_log_info("base backup completed");
2020 }
2021 
2022 
2023 int
2024 main(int argc, char **argv)
2025 {
2026  static struct option long_options[] = {
2027  {"help", no_argument, NULL, '?'},
2028  {"version", no_argument, NULL, 'V'},
2029  {"pgdata", required_argument, NULL, 'D'},
2030  {"format", required_argument, NULL, 'F'},
2031  {"checkpoint", required_argument, NULL, 'c'},
2032  {"create-slot", no_argument, NULL, 'C'},
2033  {"max-rate", required_argument, NULL, 'r'},
2034  {"write-recovery-conf", no_argument, NULL, 'R'},
2035  {"slot", required_argument, NULL, 'S'},
2036  {"tablespace-mapping", required_argument, NULL, 'T'},
2037  {"wal-method", required_argument, NULL, 'X'},
2038  {"gzip", no_argument, NULL, 'z'},
2039  {"compress", required_argument, NULL, 'Z'},
2040  {"label", required_argument, NULL, 'l'},
2041  {"no-clean", no_argument, NULL, 'n'},
2042  {"no-sync", no_argument, NULL, 'N'},
2043  {"dbname", required_argument, NULL, 'd'},
2044  {"host", required_argument, NULL, 'h'},
2045  {"port", required_argument, NULL, 'p'},
2046  {"username", required_argument, NULL, 'U'},
2047  {"no-password", no_argument, NULL, 'w'},
2048  {"password", no_argument, NULL, 'W'},
2049  {"status-interval", required_argument, NULL, 's'},
2050  {"verbose", no_argument, NULL, 'v'},
2051  {"progress", no_argument, NULL, 'P'},
2052  {"waldir", required_argument, NULL, 1},
2053  {"no-slot", no_argument, NULL, 2},
2054  {"no-verify-checksums", no_argument, NULL, 3},
2055  {NULL, 0, NULL, 0}
2056  };
2057  int c;
2058 
2059  int option_index;
2060 
2061  pg_logging_init(argv[0]);
2062  progname = get_progname(argv[0]);
2063  set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup"));
2064 
2065  if (argc > 1)
2066  {
2067  if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
2068  {
2069  usage();
2070  exit(0);
2071  }
2072  else if (strcmp(argv[1], "-V") == 0
2073  || strcmp(argv[1], "--version") == 0)
2074  {
2075  puts("pg_basebackup (PostgreSQL) " PG_VERSION);
2076  exit(0);
2077  }
2078  }
2079 
2081 
2082  while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP",
2083  long_options, &option_index)) != -1)
2084  {
2085  switch (c)
2086  {
2087  case 'C':
2088  create_slot = true;
2089  break;
2090  case 'D':
2092  break;
2093  case 'F':
2094  if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
2095  format = 'p';
2096  else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
2097  format = 't';
2098  else
2099  {
2100  pg_log_error("invalid output format \"%s\", must be \"plain\" or \"tar\"",
2101  optarg);
2102  exit(1);
2103  }
2104  break;
2105  case 'r':
2107  break;
2108  case 'R':
2109  writerecoveryconf = true;
2110  break;
2111  case 'S':
2112 
2113  /*
2114  * When specifying replication slot name, use a permanent
2115  * slot.
2116  */
2118  temp_replication_slot = false;
2119  break;
2120  case 2:
2121  no_slot = true;
2122  break;
2123  case 'T':
2125  break;
2126  case 'X':
2127  if (strcmp(optarg, "n") == 0 ||
2128  strcmp(optarg, "none") == 0)
2129  {
2130  includewal = NO_WAL;
2131  }
2132  else if (strcmp(optarg, "f") == 0 ||
2133  strcmp(optarg, "fetch") == 0)
2134  {
2136  }
2137  else if (strcmp(optarg, "s") == 0 ||
2138  strcmp(optarg, "stream") == 0)
2139  {
2141  }
2142  else
2143  {
2144  pg_log_error("invalid wal-method option \"%s\", must be \"fetch\", \"stream\", or \"none\"",
2145  optarg);
2146  exit(1);
2147  }
2148  break;
2149  case 1:
2151  break;
2152  case 'l':
2153  label = pg_strdup(optarg);
2154  break;
2155  case 'n':
2156  noclean = true;
2157  break;
2158  case 'N':
2159  do_sync = false;
2160  break;
2161  case 'z':
2162 #ifdef HAVE_LIBZ
2164 #else
2165  compresslevel = 1; /* will be rejected below */
2166 #endif
2167  break;
2168  case 'Z':
2169  compresslevel = atoi(optarg);
2170  if (compresslevel < 0 || compresslevel > 9)
2171  {
2172  pg_log_error("invalid compression level \"%s\"", optarg);
2173  exit(1);
2174  }
2175  break;
2176  case 'c':
2177  if (pg_strcasecmp(optarg, "fast") == 0)
2178  fastcheckpoint = true;
2179  else if (pg_strcasecmp(optarg, "spread") == 0)
2180  fastcheckpoint = false;
2181  else
2182  {
2183  pg_log_error("invalid checkpoint argument \"%s\", must be \"fast\" or \"spread\"",
2184  optarg);
2185  exit(1);
2186  }
2187  break;
2188  case 'd':
2190  break;
2191  case 'h':
2192  dbhost = pg_strdup(optarg);
2193  break;
2194  case 'p':
2195  dbport = pg_strdup(optarg);
2196  break;
2197  case 'U':
2198  dbuser = pg_strdup(optarg);
2199  break;
2200  case 'w':
2201  dbgetpassword = -1;
2202  break;
2203  case 'W':
2204  dbgetpassword = 1;
2205  break;
2206  case 's':
2207  standby_message_timeout = atoi(optarg) * 1000;
2208  if (standby_message_timeout < 0)
2209  {
2210  pg_log_error("invalid status interval \"%s\"", optarg);
2211  exit(1);
2212  }
2213  break;
2214  case 'v':
2215  verbose++;
2216  break;
2217  case 'P':
2218  showprogress = true;
2219  break;
2220  case 3:
2221  verify_checksums = false;
2222  break;
2223  default:
2224 
2225  /*
2226  * getopt_long already emitted a complaint
2227  */
2228  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2229  progname);
2230  exit(1);
2231  }
2232  }
2233 
2234  /*
2235  * Any non-option arguments?
2236  */
2237  if (optind < argc)
2238  {
2239  pg_log_error("too many command-line arguments (first is \"%s\")",
2240  argv[optind]);
2241  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2242  progname);
2243  exit(1);
2244  }
2245 
2246  /*
2247  * Required arguments
2248  */
2249  if (basedir == NULL)
2250  {
2251  pg_log_error("no target directory specified");
2252  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2253  progname);
2254  exit(1);
2255  }
2256 
2257  /*
2258  * Mutually exclusive arguments
2259  */
2260  if (format == 'p' && compresslevel != 0)
2261  {
2262  pg_log_error("only tar mode backups can be compressed");
2263  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2264  progname);
2265  exit(1);
2266  }
2267 
2268  if (format == 't' && includewal == STREAM_WAL && strcmp(basedir, "-") == 0)
2269  {
2270  pg_log_error("cannot stream write-ahead logs in tar mode to stdout");
2271  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2272  progname);
2273  exit(1);
2274  }
2275 
2277  {
2278  pg_log_error("replication slots can only be used with WAL streaming");
2279  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2280  progname);
2281  exit(1);
2282  }
2283 
2284  if (no_slot)
2285  {
2286  if (replication_slot)
2287  {
2288  pg_log_error("--no-slot cannot be used with slot name");
2289  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2290  progname);
2291  exit(1);
2292  }
2293  temp_replication_slot = false;
2294  }
2295 
2296  if (create_slot)
2297  {
2298  if (!replication_slot)
2299  {
2300  pg_log_error("%s needs a slot to be specified using --slot",
2301  "--create-slot");
2302  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2303  progname);
2304  exit(1);
2305  }
2306 
2307  if (no_slot)
2308  {
2309  pg_log_error("--create-slot and --no-slot are incompatible options");
2310  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2311  progname);
2312  exit(1);
2313  }
2314  }
2315 
2316  if (xlog_dir)
2317  {
2318  if (format != 'p')
2319  {
2320  pg_log_error("WAL directory location can only be specified in plain mode");
2321  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2322  progname);
2323  exit(1);
2324  }
2325 
2326  /* clean up xlog directory name, check it's absolute */
2328  if (!is_absolute_path(xlog_dir))
2329  {
2330  pg_log_error("WAL directory location must be an absolute path");
2331  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2332  progname);
2333  exit(1);
2334  }
2335  }
2336 
2337 #ifndef HAVE_LIBZ
2338  if (compresslevel != 0)
2339  {
2340  pg_log_error("this build does not support compression");
2341  exit(1);
2342  }
2343 #endif
2344 
2345  /* connection in replication mode to server */
2346  conn = GetConnection();
2347  if (!conn)
2348  {
2349  /* Error message already written in GetConnection() */
2350  exit(1);
2351  }
2352  atexit(disconnect_atexit);
2353 
2354  /*
2355  * Set umask so that directories/files are created with the same
2356  * permissions as directories/files in the source data directory.
2357  *
2358  * pg_mode_mask is set to owner-only by default and then updated in
2359  * GetConnection() where we get the mode from the server-side with
2360  * RetrieveDataDirCreatePerm() and then call SetDataDirectoryCreatePerm().
2361  */
2362  umask(pg_mode_mask);
2363 
2364  /*
2365  * Verify that the target directory exists, or create it. For plaintext
2366  * backups, always require the directory. For tar backups, require it
2367  * unless we are writing to stdout.
2368  */
2369  if (format == 'p' || strcmp(basedir, "-") != 0)
2371 
2372  /* determine remote server's xlog segment size */
2373  if (!RetrieveWalSegSize(conn))
2374  exit(1);
2375 
2376  /* Create pg_wal symlink, if required */
2377  if (xlog_dir)
2378  {
2379  char *linkloc;
2380 
2382 
2383  /*
2384  * Form name of the place where the symlink must go. pg_xlog has been
2385  * renamed to pg_wal in post-10 clusters.
2386  */
2387  linkloc = psprintf("%s/%s", basedir,
2389  "pg_xlog" : "pg_wal");
2390 
2391 #ifdef HAVE_SYMLINK
2392  if (symlink(xlog_dir, linkloc) != 0)
2393  {
2394  pg_log_error("could not create symbolic link \"%s\": %m", linkloc);
2395  exit(1);
2396  }
2397 #else
2398  pg_log_error("symlinks are not supported on this platform");
2399  exit(1);
2400 #endif
2401  free(linkloc);
2402  }
2403 
2404  BaseBackup();
2405 
2406  success = true;
2407  return 0;
2408 }
bool found_postgresql_auto_conf
Definition: pg_basebackup.c:71
int PQnfields(const PGresult *res)
Definition: fe-exec.c:2777
char current_path[MAXPGPATH]
Definition: pg_basebackup.c:83
static PQExpBuffer recoveryconfcontents
static IncludeWal includewal
char * PQerrorMessage(const PGconn *conn)
Definition: fe-connect.c:6623
#define Z_DEFAULT_COMPRESSION
int pg_file_create_mode
Definition: file_perm.c:19
static bool verify_checksums
static int bgpipe[2]
static bool found_existing_pgdata
uint32 TimeLineID
Definition: xlogdefs.h:52
int64 pg_time_t
Definition: pgtime.h:23
bool is_recovery_guc_supported
Definition: pg_basebackup.c:69
static void usage(void)
static bool found_tablespace_dirs
const char * mapped_tblspc_path
Definition: pg_basebackup.c:85
char * PQgetvalue(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3163
const char * PQparameterStatus(const PGconn *conn, const char *paramName)
Definition: fe-connect.c:6588
bool pg_str_endswith(const char *str, const char *end)
Definition: string.c:31
int pg_mkdir_p(char *path, int omode)
Definition: pgmkdirp.c:57
static TablespaceList tablespace_dirs
#define write(a, b, c)
Definition: win32.h:14
bool do_sync
Definition: receivelog.h:38
const char * get_progname(const char *argv0)
Definition: path.c:453
uint64 read_tar_number(const char *s, int len)
Definition: tar.c:58
#define pg_log_error(...)
Definition: logging.h:79
int getopt_long(int argc, char *const argv[], const char *optstring, const struct option *longopts, int *longindex)
Definition: getopt_long.c:57
static int32 parse_max_rate(char *src)
void pg_logging_init(const char *argv0)
Definition: logging.c:39
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
void WriteRecoveryConfig(PGconn *pgconn, char *target_dir, PQExpBuffer contents)
Definition: recovery_gen.c:117
static pg_time_t last_progress_report
char old_dir[MAXPGPATH]
Definition: pg_basebackup.c:50
static int tablespacecount
static void writeTarData(WriteTarState *state, char *buf, int r)
void _dosmaperr(unsigned long)
Definition: win32error.c:171
bool RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli, XLogRecPtr *startpos, char **db_name)
Definition: streamutil.c:407
#define VERBOSE_FILENAME_LENGTH
struct UnpackTarState UnpackTarState
IncludeWal
void canonicalize_path(char *path)
Definition: path.c:254
char * sysidentifier
Definition: receivelog.h:33
static bool writerecoveryconf
static bool create_slot
void PQfinish(PGconn *conn)
Definition: fe-connect.c:4125
static int compresslevel
static bool checksum_failure
#define MemSet(start, val, len)
Definition: c.h:962
#define MINIMUM_VERSION_FOR_RECOVERY_GUC
Definition: recovery_gen.h:21
#define kill(pid, sig)
Definition: win32_port.h:426
#define printf(...)
Definition: port.h:198
char filename[MAXPGPATH]
Definition: pg_basebackup.c:84
XLogRecPtr startpos
Definition: receivelog.h:31
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
char * partial_suffix
Definition: receivelog.h:47
char new_dir[MAXPGPATH]
Definition: pg_basebackup.c:51
int PQserverVersion(const PGconn *conn)
Definition: fe-connect.c:6613
const char * progname
Definition: pg_standby.c:36
int PQntuples(const PGresult *res)
Definition: fe-exec.c:2769
#define PG_DIAG_SQLSTATE
Definition: postgres_ext.h:57
#define fprintf
Definition: port.h:196
static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline, bool segment_finished)
char * wait_result_to_str(int exitstatus)
Definition: wait_error.c:32
TimeLineID timeline
Definition: receivelog.h:32
static char * basedir
struct TablespaceListCell TablespaceListCell
WalWriteMethod * CreateWalDirectoryMethod(const char *basedir, int compression, bool sync)
Definition: walmethods.c:349
ExecStatusType PQresultStatus(const PGresult *res)
Definition: fe-exec.c:2692
int PQgetCopyData(PGconn *conn, char **buffer, int async)
Definition: fe-exec.c:2473
signed int int32
Definition: c.h:347
int main(int argc, char **argv)
static void BaseBackup(void)
pgoff_t filesz
Definition: pg_basebackup.c:74
void fsync_pgdata(const char *pg_data, int serverVersion)
Definition: file_utils.c:58
#define pgoff_t
Definition: win32_port.h:195
int PQsendQuery(PGconn *conn, const char *query)
Definition: fe-exec.c:1234
void destroyPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:116
#define required_argument
Definition: getopt_long.h:25
static void ReceiveTarAndUnpackCopyChunk(size_t r, char *copybuf, void *callback_data)
int optind
Definition: getopt.c:50
#define MAX_RATE_UPPER
Definition: basebackup.h:21
void fsync_dir_recurse(const char *dir)
Definition: file_utils.c:122
char * connection_string
Definition: streamutil.c:46
static void ReceiveCopyData(PGconn *conn, WriteDataCallback callback, void *callback_data)
struct TablespaceList TablespaceList
bool is_postgresql_auto_conf
Definition: pg_basebackup.c:70
bool RetrieveWalSegSize(PGconn *conn)
Definition: streamutil.c:277
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
PGconn * conn
Definition: streamutil.c:54
#define MAXPGPATH
static void callback(struct sockaddr *addr, struct sockaddr *mask, void *unused)
Definition: test_ifaddrs.c:48
#define MINIMUM_VERSION_FOR_PG_WAL
Definition: pg_basebackup.c:98
static bool found_existing_xlogdir
static uint64 totaldone
char * replication_slot
Definition: receivelog.h:48
bool mark_done
Definition: receivelog.h:37
static bool success
static int32 maxrate
char * c
static char * buf
Definition: pg_test_fsync.c:67
#define symlink(oldpath, newpath)
Definition: win32_port.h:221
static int has_xlogendptr
static bool made_new_pgdata
#define is_absolute_path(filename)
Definition: port.h:86
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
size_t PQescapeStringConn(PGconn *conn, char *to, const char *from, size_t length, int *error)
Definition: fe-exec.c:3410
#define select(n, r, w, e, timeout)
Definition: win32_port.h:436
unsigned int uint32
Definition: c.h:359
char tarhdr[512]
Definition: pg_basebackup.c:65
static const char * get_tablespace_mapping(const char *dir)
static void progress_report(int tablespacenum, const char *filename, bool force)
XLogRecPtr startptr
PQExpBuffer GenerateRecoveryConfig(PGconn *pgconn, char *replication_slot)
Definition: recovery_gen.c:23
stream_stop_callback stream_stop
Definition: receivelog.h:41
#define ERRCODE_DATA_CORRUPTED
Definition: pg_basebackup.c:45
WalWriteMethod * walmethod
Definition: receivelog.h:46
static int verbose
#define unconstify(underlying_type, expr)
Definition: c.h:1194
bool rmtree(const char *path, bool rmtopdir)
Definition: rmtree.c:42
static int standby_message_timeout
int pg_dir_create_mode
Definition: file_perm.c:18
static bool showprogress
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:114
int dbgetpassword
Definition: streamutil.c:51
#define no_argument
Definition: getopt_long.h:24
void FreeWalTarMethod(void)
Definition: walmethods.c:1013
#define ngettext(s, p, n)
Definition: c.h:1134
#define PG_TEXTDOMAIN(domain)
Definition: c.h:1166
static bool temp_replication_slot
int PQbackendPID(const PGconn *conn)
Definition: fe-connect.c:6649
#define PGINVALID_SOCKET
Definition: port.h:33
static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
struct TablespaceListCell * next
Definition: pg_basebackup.c:49
static char * label
#define XLogSegmentOffset(xlogptr, wal_segsz_bytes)
StringInfo copybuf
Definition: tablesync.c:108
static void disconnect_atexit(void)
static void StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
TablespaceListCell * tail
Definition: pg_basebackup.c:57
char * dbport
Definition: streamutil.c:49
static void cleanup_directories_atexit(void)
void PQclear(PGresult *res)
Definition: fe-exec.c:694
#define free(a)
Definition: header.h:65
static bool in_log_streamer
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
PGconn * GetConnection(UserMapping *user, bool will_prep_stmt)
Definition: connection.c:105
char * PQresultErrorField(const PGresult *res, int fieldcode)
Definition: fe-exec.c:2754
static void ReceiveTarCopyChunk(size_t r, char *copybuf, void *callback_data)
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:739
Definition: regguts.h:298
WalWriteMethod * CreateWalTarMethod(const char *tarbase, int compression, bool sync)
Definition: walmethods.c:982
static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
char * dbhost
Definition: streamutil.c:47
void(* WriteDataCallback)(size_t nbytes, char *buf, void *callback_data)
Definition: pg_basebackup.c:91
bool ReceiveXlogStream(PGconn *conn, StreamCtl *stream)
Definition: receivelog.c:436
static void kill_bgchild_atexit(void)
#define strerror
Definition: port.h:205
static char * xlog_dir
static XLogRecPtr xlogendptr
static XLogRecPtr startpos
static bool no_slot
TablespaceListCell * head
Definition: pg_basebackup.c:56
bool synchronous
Definition: receivelog.h:36
void pg_free(void *ptr)
Definition: fe_memutils.c:105
bool(* finish)(void)
Definition: walmethods.h:75
pgsocket stop_socket
Definition: receivelog.h:43
static void header(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:209
#define INT64_FORMAT
Definition: c.h:401
static bool made_new_xlogdir
static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
Definition: walsender.c:875
int WalSegSz
Definition: pg_standby.c:38
static bool fastcheckpoint
static void tablespace_list_append(const char *arg)
static bool made_tablespace_dirs
static char * filename
Definition: pg_dumpall.c:90
void set_pglocale_pgservice(const char *argv0, const char *app)
Definition: exec.c:565
int errmsg(const char *fmt,...)
Definition: elog.c:822
static pid_t bgchild
char * dbuser
Definition: streamutil.c:48
char * optarg
Definition: getopt.c:52
static uint64 totalsize_kb
struct WriteTarState WriteTarState
int pg_check_dir(const char *dir)
Definition: pgcheckdir.c:31
int i
int standby_message_timeout
Definition: receivelog.h:35
void * arg
static bool do_sync
static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
static char format
#define mkdir(a, b)
Definition: win32_port.h:58
static char * replication_slot
static void static void status(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:226
static int LogStreamerMain(logstreamer_param *param)
int PQgetisnull(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3188
#define snprintf
Definition: port.h:192
#define _(x)
Definition: elog.c:87
void PQfreemem(void *ptr)
Definition: fe-exec.c:3296
int pg_mode_mask
Definition: file_perm.c:25
Datum now(PG_FUNCTION_ARGS)
Definition: timestamp.c:1547
PGresult * PQgetResult(PGconn *conn)
Definition: fe-exec.c:1778
#define MINIMUM_VERSION_FOR_TEMP_SLOTS
#define read(a, b, c)
Definition: win32.h:13
#define pg_log_info(...)
Definition: logging.h:87
void FreeWalDirectoryMethod(void)
Definition: walmethods.c:373
bool CheckServerVersionForStreaming(PGconn *conn)
Definition: receivelog.c:358
char filename[MAXPGPATH]
Definition: pg_basebackup.c:63
char xlog[MAXPGPATH]
pgoff_t current_len_left
Definition: pg_basebackup.c:86
static bool noclean