PostgreSQL Source Code  git master
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros
pg_basebackup.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_basebackup.c - receive a base backup using streaming replication protocol
4  *
5  * Author: Magnus Hagander <magnus@hagander.net>
6  *
7  * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
8  *
9  * IDENTIFICATION
10  * src/bin/pg_basebackup/pg_basebackup.c
11  *-------------------------------------------------------------------------
12  */
13 
14 #include "postgres_fe.h"
15 
16 #include <unistd.h>
17 #include <dirent.h>
18 #include <sys/stat.h>
19 #include <sys/wait.h>
20 #include <signal.h>
21 #include <time.h>
22 #ifdef HAVE_SYS_SELECT_H
23 #include <sys/select.h>
24 #endif
25 #ifdef HAVE_LIBZ
26 #include <zlib.h>
27 #endif
28 
29 #include "common/file_utils.h"
30 #include "common/string.h"
31 #include "fe_utils/string_utils.h"
32 #include "getopt_long.h"
33 #include "libpq-fe.h"
34 #include "pqexpbuffer.h"
35 #include "pgtar.h"
36 #include "pgtime.h"
37 #include "receivelog.h"
38 #include "replication/basebackup.h"
39 #include "streamutil.h"
40 
41 
42 typedef struct TablespaceListCell
43 {
48 
49 typedef struct TablespaceList
50 {
54 
55 /*
56  * pg_xlog has been renamed to pg_wal in version 10. This version number
57  * should be compared with PQserverVersion().
58  */
59 #define MINIMUM_VERSION_FOR_PG_WAL 100000
60 
61 /*
62  * Temporary replication slots are supported from version 10.
63  */
64 #define MINIMUM_VERSION_FOR_TEMP_SLOTS 100000
65 
66 /*
67  * Different ways to include WAL
68  */
69 typedef enum
70 {
74 } IncludeWal;
75 
76 /* Global options */
77 static char *basedir = NULL;
79 static char *xlog_dir = "";
80 static char format = 'p'; /* p(lain)/t(ar) */
81 static char *label = "pg_basebackup base backup";
82 static bool noclean = false;
83 static bool showprogress = false;
84 static int verbose = 0;
85 static int compresslevel = 0;
87 static bool fastcheckpoint = false;
88 static bool writerecoveryconf = false;
89 static bool do_sync = true;
90 static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
92 static int32 maxrate = 0; /* no limit by default */
93 static char *replication_slot = NULL;
94 static bool temp_replication_slot = true;
95 
96 static bool success = false;
97 static bool made_new_pgdata = false;
98 static bool found_existing_pgdata = false;
99 static bool made_new_xlogdir = false;
100 static bool found_existing_xlogdir = false;
101 static bool made_tablespace_dirs = false;
102 static bool found_tablespace_dirs = false;
103 
104 /* Progress counters */
105 static uint64 totalsize;
106 static uint64 totaldone;
107 static int tablespacecount;
108 
109 /* Pipe to communicate with background wal receiver process */
110 #ifndef WIN32
111 static int bgpipe[2] = {-1, -1};
112 #endif
113 
114 /* Handle to child process */
115 static pid_t bgchild = -1;
116 static bool in_log_streamer = false;
117 
118 /* End position for xlog streaming, empty string if unknown yet */
120 
121 #ifndef WIN32
122 static int has_xlogendptr = 0;
123 #else
124 static volatile LONG has_xlogendptr = 0;
125 #endif
126 
127 /* Contents of recovery.conf to be generated */
129 
130 /* Function headers */
131 static void usage(void);
132 static void disconnect_and_exit(int code);
133 static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found);
134 static void progress_report(int tablespacenum, const char *filename, bool force);
135 
136 static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum);
137 static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum);
138 static void GenerateRecoveryConf(PGconn *conn);
139 static void WriteRecoveryConf(void);
140 static void BaseBackup(void);
141 
142 static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline,
143  bool segment_finished);
144 
145 static const char *get_tablespace_mapping(const char *dir);
146 static void tablespace_list_append(const char *arg);
147 
148 
149 static void
151 {
152  if (success || in_log_streamer)
153  return;
154 
155  if (!noclean)
156  {
157  if (made_new_pgdata)
158  {
159  fprintf(stderr, _("%s: removing data directory \"%s\"\n"),
160  progname, basedir);
161  if (!rmtree(basedir, true))
162  fprintf(stderr, _("%s: failed to remove data directory\n"),
163  progname);
164  }
165  else if (found_existing_pgdata)
166  {
167  fprintf(stderr,
168  _("%s: removing contents of data directory \"%s\"\n"),
169  progname, basedir);
170  if (!rmtree(basedir, false))
171  fprintf(stderr, _("%s: failed to remove contents of data directory\n"),
172  progname);
173  }
174 
175  if (made_new_xlogdir)
176  {
177  fprintf(stderr, _("%s: removing WAL directory \"%s\"\n"),
178  progname, xlog_dir);
179  if (!rmtree(xlog_dir, true))
180  fprintf(stderr, _("%s: failed to remove WAL directory\n"),
181  progname);
182  }
183  else if (found_existing_xlogdir)
184  {
185  fprintf(stderr,
186  _("%s: removing contents of WAL directory \"%s\"\n"),
187  progname, xlog_dir);
188  if (!rmtree(xlog_dir, false))
189  fprintf(stderr, _("%s: failed to remove contents of WAL directory\n"),
190  progname);
191  }
192  }
193  else
194  {
196  fprintf(stderr,
197  _("%s: data directory \"%s\" not removed at user's request\n"),
198  progname, basedir);
199 
201  fprintf(stderr,
202  _("%s: WAL directory \"%s\" not removed at user's request\n"),
203  progname, xlog_dir);
204  }
205 
207  fprintf(stderr,
208  _("%s: changes to tablespace directories will not be undone\n"),
209  progname);
210 }
211 
212 static void
214 {
215  if (conn != NULL)
216  PQfinish(conn);
217 
218 #ifndef WIN32
219 
220  /*
221  * On windows, our background thread dies along with the process. But on
222  * Unix, if we have started a subprocess, we want to kill it off so it
223  * doesn't remain running trying to stream data.
224  */
225  if (bgchild > 0)
226  kill(bgchild, SIGTERM);
227 #endif
228 
229  exit(code);
230 }
231 
232 
233 /*
234  * Split argument into old_dir and new_dir and append to tablespace mapping
235  * list.
236  */
237 static void
239 {
241  char *dst;
242  char *dst_ptr;
243  const char *arg_ptr;
244 
245  dst_ptr = dst = cell->old_dir;
246  for (arg_ptr = arg; *arg_ptr; arg_ptr++)
247  {
248  if (dst_ptr - dst >= MAXPGPATH)
249  {
250  fprintf(stderr, _("%s: directory name too long\n"), progname);
251  exit(1);
252  }
253 
254  if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
255  ; /* skip backslash escaping = */
256  else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
257  {
258  if (*cell->new_dir)
259  {
260  fprintf(stderr, _("%s: multiple \"=\" signs in tablespace mapping\n"), progname);
261  exit(1);
262  }
263  else
264  dst = dst_ptr = cell->new_dir;
265  }
266  else
267  *dst_ptr++ = *arg_ptr;
268  }
269 
270  if (!*cell->old_dir || !*cell->new_dir)
271  {
272  fprintf(stderr,
273  _("%s: invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"\n"),
274  progname, arg);
275  exit(1);
276  }
277 
278  /*
279  * This check isn't absolutely necessary. But all tablespaces are created
280  * with absolute directories, so specifying a non-absolute path here would
281  * just never match, possibly confusing users. It's also good to be
282  * consistent with the new_dir check.
283  */
284  if (!is_absolute_path(cell->old_dir))
285  {
286  fprintf(stderr, _("%s: old directory is not an absolute path in tablespace mapping: %s\n"),
287  progname, cell->old_dir);
288  exit(1);
289  }
290 
291  if (!is_absolute_path(cell->new_dir))
292  {
293  fprintf(stderr, _("%s: new directory is not an absolute path in tablespace mapping: %s\n"),
294  progname, cell->new_dir);
295  exit(1);
296  }
297 
298  canonicalize_path(cell->old_dir);
299  canonicalize_path(cell->new_dir);
300 
301  if (tablespace_dirs.tail)
302  tablespace_dirs.tail->next = cell;
303  else
304  tablespace_dirs.head = cell;
305  tablespace_dirs.tail = cell;
306 }
307 
308 
309 #ifdef HAVE_LIBZ
310 static const char *
311 get_gz_error(gzFile gzf)
312 {
313  int errnum;
314  const char *errmsg;
315 
316  errmsg = gzerror(gzf, &errnum);
317  if (errnum == Z_ERRNO)
318  return strerror(errno);
319  else
320  return errmsg;
321 }
322 #endif
323 
324 static void
325 usage(void)
326 {
327  printf(_("%s takes a base backup of a running PostgreSQL server.\n\n"),
328  progname);
329  printf(_("Usage:\n"));
330  printf(_(" %s [OPTION]...\n"), progname);
331  printf(_("\nOptions controlling the output:\n"));
332  printf(_(" -D, --pgdata=DIRECTORY receive base backup into directory\n"));
333  printf(_(" -F, --format=p|t output format (plain (default), tar)\n"));
334  printf(_(" -r, --max-rate=RATE maximum transfer rate to transfer data directory\n"
335  " (in kB/s, or use suffix \"k\" or \"M\")\n"));
336  printf(_(" -R, --write-recovery-conf\n"
337  " write recovery.conf for replication\n"));
338  printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
339  printf(_(" --no-slot prevent creation of temporary replication slot\n"));
340  printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
341  " relocate tablespace in OLDDIR to NEWDIR\n"));
342  printf(_(" -X, --wal-method=none|fetch|stream\n"
343  " include required WAL files with specified method\n"));
344  printf(_(" --waldir=WALDIR location for the write-ahead log directory\n"));
345  printf(_(" -z, --gzip compress tar output\n"));
346  printf(_(" -Z, --compress=0-9 compress tar output with given compression level\n"));
347  printf(_("\nGeneral options:\n"));
348  printf(_(" -c, --checkpoint=fast|spread\n"
349  " set fast or spread checkpointing\n"));
350  printf(_(" -l, --label=LABEL set backup label\n"));
351  printf(_(" -n, --no-clean do not clean up after errors\n"));
352  printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
353  printf(_(" -P, --progress show progress information\n"));
354  printf(_(" -v, --verbose output verbose messages\n"));
355  printf(_(" -V, --version output version information, then exit\n"));
356  printf(_(" -?, --help show this help, then exit\n"));
357  printf(_("\nConnection options:\n"));
358  printf(_(" -d, --dbname=CONNSTR connection string\n"));
359  printf(_(" -h, --host=HOSTNAME database server host or socket directory\n"));
360  printf(_(" -p, --port=PORT database server port number\n"));
361  printf(_(" -s, --status-interval=INTERVAL\n"
362  " time between status packets sent to server (in seconds)\n"));
363  printf(_(" -U, --username=NAME connect as specified database user\n"));
364  printf(_(" -w, --no-password never prompt for password\n"));
365  printf(_(" -W, --password force password prompt (should happen automatically)\n"));
366  printf(_("\nReport bugs to <pgsql-bugs@postgresql.org>.\n"));
367 }
368 
369 
370 /*
371  * Called in the background process every time data is received.
372  * On Unix, we check to see if there is any data on our pipe
373  * (which would mean we have a stop position), and if it is, check if
374  * it is time to stop.
375  * On Windows, we are in a single process, so we can just check if it's
376  * time to stop.
377  */
378 static bool
380  bool segment_finished)
381 {
382  if (!has_xlogendptr)
383  {
384 #ifndef WIN32
385  fd_set fds;
386  struct timeval tv;
387  int r;
388 
389  /*
390  * Don't have the end pointer yet - check our pipe to see if it has
391  * been sent yet.
392  */
393  FD_ZERO(&fds);
394  FD_SET(bgpipe[0], &fds);
395 
396  MemSet(&tv, 0, sizeof(tv));
397 
398  r = select(bgpipe[0] + 1, &fds, NULL, NULL, &tv);
399  if (r == 1)
400  {
401  char xlogend[64];
402  uint32 hi,
403  lo;
404 
405  MemSet(xlogend, 0, sizeof(xlogend));
406  r = read(bgpipe[0], xlogend, sizeof(xlogend) - 1);
407  if (r < 0)
408  {
409  fprintf(stderr, _("%s: could not read from ready pipe: %s\n"),
410  progname, strerror(errno));
411  exit(1);
412  }
413 
414  if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
415  {
416  fprintf(stderr,
417  _("%s: could not parse write-ahead log location \"%s\"\n"),
418  progname, xlogend);
419  exit(1);
420  }
421  xlogendptr = ((uint64) hi) << 32 | lo;
422  has_xlogendptr = 1;
423 
424  /*
425  * Fall through to check if we've reached the point further
426  * already.
427  */
428  }
429  else
430  {
431  /*
432  * No data received on the pipe means we don't know the end
433  * position yet - so just say it's not time to stop yet.
434  */
435  return false;
436  }
437 #else
438 
439  /*
440  * On win32, has_xlogendptr is set by the main thread, so if it's not
441  * set here, we just go back and wait until it shows up.
442  */
443  return false;
444 #endif
445  }
446 
447  /*
448  * At this point we have an end pointer, so compare it to the current
449  * position to figure out if it's time to stop.
450  */
451  if (segendpos >= xlogendptr)
452  return true;
453 
454  /*
455  * Have end pointer, but haven't reached it yet - so tell the caller to
456  * keep streaming.
457  */
458  return false;
459 }
460 
461 typedef struct
462 {
465  char xlog[MAXPGPATH]; /* directory or tarfile depending on mode */
467  int timeline;
468  bool temp_slot;
470 
471 static int
473 {
474  StreamCtl stream;
475 
476  in_log_streamer = true;
477 
478  MemSet(&stream, 0, sizeof(stream));
479  stream.startpos = param->startptr;
480  stream.timeline = param->timeline;
481  stream.sysidentifier = param->sysidentifier;
483 #ifndef WIN32
484  stream.stop_socket = bgpipe[0];
485 #else
486  stream.stop_socket = PGINVALID_SOCKET;
487 #endif
489  stream.synchronous = false;
490  stream.do_sync = do_sync;
491  stream.mark_done = true;
492  stream.partial_suffix = NULL;
494  stream.temp_slot = param->temp_slot;
495  if (stream.temp_slot && !stream.replication_slot)
496  stream.replication_slot = psprintf("pg_basebackup_%d", (int) PQbackendPID(param->bgconn));
497 
498  if (format == 'p')
499  stream.walmethod = CreateWalDirectoryMethod(param->xlog, 0, do_sync);
500  else
502 
503  if (!ReceiveXlogStream(param->bgconn, &stream))
504 
505  /*
506  * Any errors will already have been reported in the function process,
507  * but we need to tell the parent that we didn't shutdown in a nice
508  * way.
509  */
510  return 1;
511 
512  if (!stream.walmethod->finish())
513  {
514  fprintf(stderr,
515  _("%s: could not finish writing WAL files: %s\n"),
516  progname, strerror(errno));
517  return 1;
518  }
519 
520  PQfinish(param->bgconn);
521 
522  if (format == 'p')
524  else
526  pg_free(stream.walmethod);
527 
528  return 0;
529 }
530 
531 /*
532  * Initiate background process for receiving xlog during the backup.
533  * The background stream will use its own database connection so we can
534  * stream the logfile in parallel with the backups.
535  */
536 static void
537 StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
538 {
539  logstreamer_param *param;
540  uint32 hi,
541  lo;
542  char statusdir[MAXPGPATH];
543 
544  param = pg_malloc0(sizeof(logstreamer_param));
545  param->timeline = timeline;
546  param->sysidentifier = sysidentifier;
547 
548  /* Convert the starting position */
549  if (sscanf(startpos, "%X/%X", &hi, &lo) != 2)
550  {
551  fprintf(stderr,
552  _("%s: could not parse write-ahead log location \"%s\"\n"),
553  progname, startpos);
555  }
556  param->startptr = ((uint64) hi) << 32 | lo;
557  /* Round off to even segment position */
558  param->startptr -= param->startptr % XLOG_SEG_SIZE;
559 
560 #ifndef WIN32
561  /* Create our background pipe */
562  if (pipe(bgpipe) < 0)
563  {
564  fprintf(stderr,
565  _("%s: could not create pipe for background process: %s\n"),
566  progname, strerror(errno));
568  }
569 #endif
570 
571  /* Get a second connection */
572  param->bgconn = GetConnection();
573  if (!param->bgconn)
574  /* Error message already written in GetConnection() */
575  exit(1);
576 
577  /* In post-10 cluster, pg_xlog has been renamed to pg_wal */
578  snprintf(param->xlog, sizeof(param->xlog), "%s/%s",
579  basedir,
581  "pg_xlog" : "pg_wal");
582 
583  /* Temporary replication slots are only supported in 10 and newer */
585  param->temp_slot = false;
586  else
588 
589  if (format == 'p')
590  {
591  /*
592  * Create pg_wal/archive_status or pg_xlog/archive_status (and thus
593  * pg_wal or pg_xlog) depending on the target server so we can write
594  * to basedir/pg_wal or basedir/pg_xlog as the directory entry in the
595  * tar file may arrive later.
596  */
597  snprintf(statusdir, sizeof(statusdir), "%s/%s/archive_status",
598  basedir,
600  "pg_xlog" : "pg_wal");
601 
602  if (pg_mkdir_p(statusdir, S_IRWXU) != 0 && errno != EEXIST)
603  {
604  fprintf(stderr,
605  _("%s: could not create directory \"%s\": %s\n"),
606  progname, statusdir, strerror(errno));
608  }
609  }
610 
611  /*
612  * Start a child process and tell it to start streaming. On Unix, this is
613  * a fork(). On Windows, we create a thread.
614  */
615 #ifndef WIN32
616  bgchild = fork();
617  if (bgchild == 0)
618  {
619  /* in child process */
620  exit(LogStreamerMain(param));
621  }
622  else if (bgchild < 0)
623  {
624  fprintf(stderr, _("%s: could not create background process: %s\n"),
625  progname, strerror(errno));
627  }
628 
629  /*
630  * Else we are in the parent process and all is well.
631  */
632 #else /* WIN32 */
633  bgchild = _beginthreadex(NULL, 0, (void *) LogStreamerMain, param, 0, NULL);
634  if (bgchild == 0)
635  {
636  fprintf(stderr, _("%s: could not create background thread: %s\n"),
637  progname, strerror(errno));
639  }
640 #endif
641 }
642 
643 /*
644  * Verify that the given directory exists and is empty. If it does not
645  * exist, it is created. If it exists but is not empty, an error will
646  * be given and the process ended.
647  */
648 static void
649 verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
650 {
651  switch (pg_check_dir(dirname))
652  {
653  case 0:
654 
655  /*
656  * Does not exist, so create
657  */
658  if (pg_mkdir_p(dirname, S_IRWXU) == -1)
659  {
660  fprintf(stderr,
661  _("%s: could not create directory \"%s\": %s\n"),
662  progname, dirname, strerror(errno));
664  }
665  if (created)
666  *created = true;
667  return;
668  case 1:
669 
670  /*
671  * Exists, empty
672  */
673  if (found)
674  *found = true;
675  return;
676  case 2:
677  case 3:
678  case 4:
679 
680  /*
681  * Exists, not empty
682  */
683  fprintf(stderr,
684  _("%s: directory \"%s\" exists but is not empty\n"),
685  progname, dirname);
687  case -1:
688 
689  /*
690  * Access problem
691  */
692  fprintf(stderr, _("%s: could not access directory \"%s\": %s\n"),
693  progname, dirname, strerror(errno));
695  }
696 }
697 
698 
699 /*
700  * Print a progress report based on the global variables. If verbose output
701  * is enabled, also print the current file name.
702  *
703  * Progress report is written at maximum once per second, unless the
704  * force parameter is set to true.
705  */
706 static void
707 progress_report(int tablespacenum, const char *filename, bool force)
708 {
709  int percent;
710  char totaldone_str[32];
711  char totalsize_str[32];
712  pg_time_t now;
713 
714  if (!showprogress)
715  return;
716 
717  now = time(NULL);
718  if (now == last_progress_report && !force)
719  return; /* Max once per second */
720 
722  percent = totalsize ? (int) ((totaldone / 1024) * 100 / totalsize) : 0;
723 
724  /*
725  * Avoid overflowing past 100% or the full size. This may make the total
726  * size number change as we approach the end of the backup (the estimate
727  * will always be wrong if WAL is included), but that's better than having
728  * the done column be bigger than the total.
729  */
730  if (percent > 100)
731  percent = 100;
732  if (totaldone / 1024 > totalsize)
733  totalsize = totaldone / 1024;
734 
735  /*
736  * Separate step to keep platform-dependent format code out of
737  * translatable strings. And we only test for INT64_FORMAT availability
738  * in snprintf, not fprintf.
739  */
740  snprintf(totaldone_str, sizeof(totaldone_str), INT64_FORMAT,
741  totaldone / 1024);
742  snprintf(totalsize_str, sizeof(totalsize_str), INT64_FORMAT, totalsize);
743 
744 #define VERBOSE_FILENAME_LENGTH 35
745  if (verbose)
746  {
747  if (!filename)
748 
749  /*
750  * No filename given, so clear the status line (used for last
751  * call)
752  */
753  fprintf(stderr,
754  ngettext("%*s/%s kB (100%%), %d/%d tablespace %*s",
755  "%*s/%s kB (100%%), %d/%d tablespaces %*s",
757  (int) strlen(totalsize_str),
758  totaldone_str, totalsize_str,
759  tablespacenum, tablespacecount,
760  VERBOSE_FILENAME_LENGTH + 5, "");
761  else
762  {
763  bool truncate = (strlen(filename) > VERBOSE_FILENAME_LENGTH);
764 
765  fprintf(stderr,
766  ngettext("%*s/%s kB (%d%%), %d/%d tablespace (%s%-*.*s)",
767  "%*s/%s kB (%d%%), %d/%d tablespaces (%s%-*.*s)",
769  (int) strlen(totalsize_str),
770  totaldone_str, totalsize_str, percent,
771  tablespacenum, tablespacecount,
772  /* Prefix with "..." if we do leading truncation */
773  truncate ? "..." : "",
776  /* Truncate filename at beginning if it's too long */
777  truncate ? filename + strlen(filename) - VERBOSE_FILENAME_LENGTH + 3 : filename);
778  }
779  }
780  else
781  fprintf(stderr,
782  ngettext("%*s/%s kB (%d%%), %d/%d tablespace",
783  "%*s/%s kB (%d%%), %d/%d tablespaces",
785  (int) strlen(totalsize_str),
786  totaldone_str, totalsize_str, percent,
787  tablespacenum, tablespacecount);
788 
789  fprintf(stderr, "\r");
790 }
791 
792 static int32
793 parse_max_rate(char *src)
794 {
795  double result;
796  char *after_num;
797  char *suffix = NULL;
798 
799  errno = 0;
800  result = strtod(src, &after_num);
801  if (src == after_num)
802  {
803  fprintf(stderr,
804  _("%s: transfer rate \"%s\" is not a valid value\n"),
805  progname, src);
806  exit(1);
807  }
808  if (errno != 0)
809  {
810  fprintf(stderr,
811  _("%s: invalid transfer rate \"%s\": %s\n"),
812  progname, src, strerror(errno));
813  exit(1);
814  }
815 
816  if (result <= 0)
817  {
818  /*
819  * Reject obviously wrong values here.
820  */
821  fprintf(stderr, _("%s: transfer rate must be greater than zero\n"),
822  progname);
823  exit(1);
824  }
825 
826  /*
827  * Evaluate suffix, after skipping over possible whitespace. Lack of
828  * suffix means kilobytes.
829  */
830  while (*after_num != '\0' && isspace((unsigned char) *after_num))
831  after_num++;
832 
833  if (*after_num != '\0')
834  {
835  suffix = after_num;
836  if (*after_num == 'k')
837  {
838  /* kilobyte is the expected unit. */
839  after_num++;
840  }
841  else if (*after_num == 'M')
842  {
843  after_num++;
844  result *= 1024.0;
845  }
846  }
847 
848  /* The rest can only consist of white space. */
849  while (*after_num != '\0' && isspace((unsigned char) *after_num))
850  after_num++;
851 
852  if (*after_num != '\0')
853  {
854  fprintf(stderr,
855  _("%s: invalid --max-rate unit: \"%s\"\n"),
856  progname, suffix);
857  exit(1);
858  }
859 
860  /* Valid integer? */
861  if ((uint64) result != (uint64) ((uint32) result))
862  {
863  fprintf(stderr,
864  _("%s: transfer rate \"%s\" exceeds integer range\n"),
865  progname, src);
866  exit(1);
867  }
868 
869  /*
870  * The range is checked on the server side too, but avoid the server
871  * connection if a nonsensical value was passed.
872  */
873  if (result < MAX_RATE_LOWER || result > MAX_RATE_UPPER)
874  {
875  fprintf(stderr,
876  _("%s: transfer rate \"%s\" is out of range\n"),
877  progname, src);
878  exit(1);
879  }
880 
881  return (int32) result;
882 }
883 
884 /*
885  * Write a piece of tar data
886  */
887 static void
889 #ifdef HAVE_LIBZ
890  gzFile ztarfile,
891 #endif
892  FILE *tarfile, char *buf, int r, char *current_file)
893 {
894 #ifdef HAVE_LIBZ
895  if (ztarfile != NULL)
896  {
897  if (gzwrite(ztarfile, buf, r) != r)
898  {
899  fprintf(stderr,
900  _("%s: could not write to compressed file \"%s\": %s\n"),
901  progname, current_file, get_gz_error(ztarfile));
903  }
904  }
905  else
906 #endif
907  {
908  if (fwrite(buf, r, 1, tarfile) != 1)
909  {
910  fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"),
911  progname, current_file, strerror(errno));
913  }
914  }
915 }
916 
917 #ifdef HAVE_LIBZ
918 #define WRITE_TAR_DATA(buf, sz) writeTarData(ztarfile, tarfile, buf, sz, filename)
919 #else
920 #define WRITE_TAR_DATA(buf, sz) writeTarData(tarfile, buf, sz, filename)
921 #endif
922 
923 /*
924  * Receive a tar format file from the connection to the server, and write
925  * the data from this file directly into a tar file. If compression is
926  * enabled, the data will be compressed while written to the file.
927  *
928  * The file will be named base.tar[.gz] if it's for the main data directory
929  * or <tablespaceoid>.tar[.gz] if it's for another tablespace.
930  *
931  * No attempt to inspect or validate the contents of the file is done.
932  */
933 static void
934 ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
935 {
936  char filename[MAXPGPATH];
937  char *copybuf = NULL;
938  FILE *tarfile = NULL;
939  char tarhdr[512];
940  bool basetablespace = PQgetisnull(res, rownum, 0);
941  bool in_tarhdr = true;
942  bool skip_file = false;
943  size_t tarhdrsz = 0;
944  pgoff_t filesz = 0;
945 
946 #ifdef HAVE_LIBZ
947  gzFile ztarfile = NULL;
948 #endif
949 
950  if (basetablespace)
951  {
952  /*
953  * Base tablespaces
954  */
955  if (strcmp(basedir, "-") == 0)
956  {
957 #ifdef HAVE_LIBZ
958  if (compresslevel != 0)
959  {
960  ztarfile = gzdopen(dup(fileno(stdout)), "wb");
961  if (gzsetparams(ztarfile, compresslevel,
962  Z_DEFAULT_STRATEGY) != Z_OK)
963  {
964  fprintf(stderr,
965  _("%s: could not set compression level %d: %s\n"),
966  progname, compresslevel, get_gz_error(ztarfile));
968  }
969  }
970  else
971 #endif
972  tarfile = stdout;
973  strcpy(filename, "-");
974  }
975  else
976  {
977 #ifdef HAVE_LIBZ
978  if (compresslevel != 0)
979  {
980  snprintf(filename, sizeof(filename), "%s/base.tar.gz", basedir);
981  ztarfile = gzopen(filename, "wb");
982  if (gzsetparams(ztarfile, compresslevel,
983  Z_DEFAULT_STRATEGY) != Z_OK)
984  {
985  fprintf(stderr,
986  _("%s: could not set compression level %d: %s\n"),
987  progname, compresslevel, get_gz_error(ztarfile));
989  }
990  }
991  else
992 #endif
993  {
994  snprintf(filename, sizeof(filename), "%s/base.tar", basedir);
995  tarfile = fopen(filename, "wb");
996  }
997  }
998  }
999  else
1000  {
1001  /*
1002  * Specific tablespace
1003  */
1004 #ifdef HAVE_LIBZ
1005  if (compresslevel != 0)
1006  {
1007  snprintf(filename, sizeof(filename), "%s/%s.tar.gz", basedir,
1008  PQgetvalue(res, rownum, 0));
1009  ztarfile = gzopen(filename, "wb");
1010  if (gzsetparams(ztarfile, compresslevel,
1011  Z_DEFAULT_STRATEGY) != Z_OK)
1012  {
1013  fprintf(stderr,
1014  _("%s: could not set compression level %d: %s\n"),
1015  progname, compresslevel, get_gz_error(ztarfile));
1017  }
1018  }
1019  else
1020 #endif
1021  {
1022  snprintf(filename, sizeof(filename), "%s/%s.tar", basedir,
1023  PQgetvalue(res, rownum, 0));
1024  tarfile = fopen(filename, "wb");
1025  }
1026  }
1027 
1028 #ifdef HAVE_LIBZ
1029  if (compresslevel != 0)
1030  {
1031  if (!ztarfile)
1032  {
1033  /* Compression is in use */
1034  fprintf(stderr,
1035  _("%s: could not create compressed file \"%s\": %s\n"),
1036  progname, filename, get_gz_error(ztarfile));
1038  }
1039  }
1040  else
1041 #endif
1042  {
1043  /* Either no zlib support, or zlib support but compresslevel = 0 */
1044  if (!tarfile)
1045  {
1046  fprintf(stderr, _("%s: could not create file \"%s\": %s\n"),
1047  progname, filename, strerror(errno));
1049  }
1050  }
1051 
1052  /*
1053  * Get the COPY data stream
1054  */
1055  res = PQgetResult(conn);
1056  if (PQresultStatus(res) != PGRES_COPY_OUT)
1057  {
1058  fprintf(stderr, _("%s: could not get COPY data stream: %s"),
1059  progname, PQerrorMessage(conn));
1061  }
1062 
1063  while (1)
1064  {
1065  int r;
1066 
1067  if (copybuf != NULL)
1068  {
1069  PQfreemem(copybuf);
1070  copybuf = NULL;
1071  }
1072 
1073  r = PQgetCopyData(conn, &copybuf, 0);
1074  if (r == -1)
1075  {
1076  /*
1077  * End of chunk. If requested, and this is the base tablespace,
1078  * write recovery.conf into the tarfile. When done, close the file
1079  * (but not stdout).
1080  *
1081  * Also, write two completely empty blocks at the end of the tar
1082  * file, as required by some tar programs.
1083  */
1084  char zerobuf[1024];
1085 
1086  MemSet(zerobuf, 0, sizeof(zerobuf));
1087 
1088  if (basetablespace && writerecoveryconf)
1089  {
1090  char header[512];
1091  int padding;
1092 
1093  tarCreateHeader(header, "recovery.conf", NULL,
1094  recoveryconfcontents->len,
1095  0600, 04000, 02000,
1096  time(NULL));
1097 
1098  padding = ((recoveryconfcontents->len + 511) & ~511) - recoveryconfcontents->len;
1099 
1100  WRITE_TAR_DATA(header, sizeof(header));
1101  WRITE_TAR_DATA(recoveryconfcontents->data, recoveryconfcontents->len);
1102  if (padding)
1103  WRITE_TAR_DATA(zerobuf, padding);
1104  }
1105 
1106  /* 2 * 512 bytes empty data at end of file */
1107  WRITE_TAR_DATA(zerobuf, sizeof(zerobuf));
1108 
1109 #ifdef HAVE_LIBZ
1110  if (ztarfile != NULL)
1111  {
1112  if (gzclose(ztarfile) != 0)
1113  {
1114  fprintf(stderr,
1115  _("%s: could not close compressed file \"%s\": %s\n"),
1116  progname, filename, get_gz_error(ztarfile));
1118  }
1119  }
1120  else
1121 #endif
1122  {
1123  if (strcmp(basedir, "-") != 0)
1124  {
1125  if (fclose(tarfile) != 0)
1126  {
1127  fprintf(stderr,
1128  _("%s: could not close file \"%s\": %s\n"),
1129  progname, filename, strerror(errno));
1131  }
1132  }
1133  }
1134 
1135  break;
1136  }
1137  else if (r == -2)
1138  {
1139  fprintf(stderr, _("%s: could not read COPY data: %s"),
1140  progname, PQerrorMessage(conn));
1142  }
1143 
1144  if (!writerecoveryconf || !basetablespace)
1145  {
1146  /*
1147  * When not writing recovery.conf, or when not working on the base
1148  * tablespace, we never have to look for an existing recovery.conf
1149  * file in the stream.
1150  */
1151  WRITE_TAR_DATA(copybuf, r);
1152  }
1153  else
1154  {
1155  /*
1156  * Look for a recovery.conf in the existing tar stream. If it's
1157  * there, we must skip it so we can later overwrite it with our
1158  * own version of the file.
1159  *
1160  * To do this, we have to process the individual files inside the
1161  * TAR stream. The stream consists of a header and zero or more
1162  * chunks, all 512 bytes long. The stream from the server is
1163  * broken up into smaller pieces, so we have to track the size of
1164  * the files to find the next header structure.
1165  */
1166  int rr = r;
1167  int pos = 0;
1168 
1169  while (rr > 0)
1170  {
1171  if (in_tarhdr)
1172  {
1173  /*
1174  * We're currently reading a header structure inside the
1175  * TAR stream, i.e. the file metadata.
1176  */
1177  if (tarhdrsz < 512)
1178  {
1179  /*
1180  * Copy the header structure into tarhdr in case the
1181  * header is not aligned to 512 bytes or it's not
1182  * returned in whole by the last PQgetCopyData call.
1183  */
1184  int hdrleft;
1185  int bytes2copy;
1186 
1187  hdrleft = 512 - tarhdrsz;
1188  bytes2copy = (rr > hdrleft ? hdrleft : rr);
1189 
1190  memcpy(&tarhdr[tarhdrsz], copybuf + pos, bytes2copy);
1191 
1192  rr -= bytes2copy;
1193  pos += bytes2copy;
1194  tarhdrsz += bytes2copy;
1195  }
1196  else
1197  {
1198  /*
1199  * We have the complete header structure in tarhdr,
1200  * look at the file metadata: - the subsequent file
1201  * contents have to be skipped if the filename is
1202  * recovery.conf - find out the size of the file
1203  * padded to the next multiple of 512
1204  */
1205  int padding;
1206 
1207  skip_file = (strcmp(&tarhdr[0], "recovery.conf") == 0);
1208 
1209  filesz = read_tar_number(&tarhdr[124], 12);
1210 
1211  padding = ((filesz + 511) & ~511) - filesz;
1212  filesz += padding;
1213 
1214  /* Next part is the file, not the header */
1215  in_tarhdr = false;
1216 
1217  /*
1218  * If we're not skipping the file, write the tar
1219  * header unmodified.
1220  */
1221  if (!skip_file)
1222  WRITE_TAR_DATA(tarhdr, 512);
1223  }
1224  }
1225  else
1226  {
1227  /*
1228  * We're processing a file's contents.
1229  */
1230  if (filesz > 0)
1231  {
1232  /*
1233  * We still have data to read (and possibly write).
1234  */
1235  int bytes2write;
1236 
1237  bytes2write = (filesz > rr ? rr : filesz);
1238 
1239  if (!skip_file)
1240  WRITE_TAR_DATA(copybuf + pos, bytes2write);
1241 
1242  rr -= bytes2write;
1243  pos += bytes2write;
1244  filesz -= bytes2write;
1245  }
1246  else
1247  {
1248  /*
1249  * No more data in the current file, the next piece of
1250  * data (if any) will be a new file header structure.
1251  */
1252  in_tarhdr = true;
1253  skip_file = false;
1254  tarhdrsz = 0;
1255  filesz = 0;
1256  }
1257  }
1258  }
1259  }
1260  totaldone += r;
1261  progress_report(rownum, filename, false);
1262  } /* while (1) */
1263  progress_report(rownum, filename, true);
1264 
1265  if (copybuf != NULL)
1266  PQfreemem(copybuf);
1267 
1268  /* sync the resulting tar file, errors are not considered fatal */
1269  if (do_sync && strcmp(basedir, "-") != 0)
1270  (void) fsync_fname(filename, false, progname);
1271 }
1272 
1273 
1274 /*
1275  * Retrieve tablespace path, either relocated or original depending on whether
1276  * -T was passed or not.
1277  */
1278 static const char *
1279 get_tablespace_mapping(const char *dir)
1280 {
1281  TablespaceListCell *cell;
1282 
1283  for (cell = tablespace_dirs.head; cell; cell = cell->next)
1284  if (strcmp(dir, cell->old_dir) == 0)
1285  return cell->new_dir;
1286 
1287  return dir;
1288 }
1289 
1290 
1291 /*
1292  * Receive a tar format stream from the connection to the server, and unpack
1293  * the contents of it into a directory. Only files, directories and
1294  * symlinks are supported, no other kinds of special files.
1295  *
1296  * If the data is for the main data directory, it will be restored in the
1297  * specified directory. If it's for another tablespace, it will be restored
1298  * in the original or mapped directory.
1299  */
1300 static void
1302 {
1303  char current_path[MAXPGPATH];
1304  char filename[MAXPGPATH];
1305  const char *mapped_tblspc_path;
1306  pgoff_t current_len_left = 0;
1307  int current_padding = 0;
1308  bool basetablespace;
1309  char *copybuf = NULL;
1310  FILE *file = NULL;
1311 
1312  basetablespace = PQgetisnull(res, rownum, 0);
1313  if (basetablespace)
1314  strlcpy(current_path, basedir, sizeof(current_path));
1315  else
1316  strlcpy(current_path,
1317  get_tablespace_mapping(PQgetvalue(res, rownum, 1)),
1318  sizeof(current_path));
1319 
1320  /*
1321  * Get the COPY data
1322  */
1323  res = PQgetResult(conn);
1324  if (PQresultStatus(res) != PGRES_COPY_OUT)
1325  {
1326  fprintf(stderr, _("%s: could not get COPY data stream: %s"),
1327  progname, PQerrorMessage(conn));
1329  }
1330 
1331  while (1)
1332  {
1333  int r;
1334 
1335  if (copybuf != NULL)
1336  {
1337  PQfreemem(copybuf);
1338  copybuf = NULL;
1339  }
1340 
1341  r = PQgetCopyData(conn, &copybuf, 0);
1342 
1343  if (r == -1)
1344  {
1345  /*
1346  * End of chunk
1347  */
1348  if (file)
1349  fclose(file);
1350 
1351  break;
1352  }
1353  else if (r == -2)
1354  {
1355  fprintf(stderr, _("%s: could not read COPY data: %s"),
1356  progname, PQerrorMessage(conn));
1358  }
1359 
1360  if (file == NULL)
1361  {
1362  int filemode;
1363 
1364  /*
1365  * No current file, so this must be the header for a new file
1366  */
1367  if (r != 512)
1368  {
1369  fprintf(stderr, _("%s: invalid tar block header size: %d\n"),
1370  progname, r);
1372  }
1373  totaldone += 512;
1374 
1375  current_len_left = read_tar_number(&copybuf[124], 12);
1376 
1377  /* Set permissions on the file */
1378  filemode = read_tar_number(&copybuf[100], 8);
1379 
1380  /*
1381  * All files are padded up to 512 bytes
1382  */
1383  current_padding =
1384  ((current_len_left + 511) & ~511) - current_len_left;
1385 
1386  /*
1387  * First part of header is zero terminated filename
1388  */
1389  snprintf(filename, sizeof(filename), "%s/%s", current_path,
1390  copybuf);
1391  if (filename[strlen(filename) - 1] == '/')
1392  {
1393  /*
1394  * Ends in a slash means directory or symlink to directory
1395  */
1396  if (copybuf[156] == '5')
1397  {
1398  /*
1399  * Directory
1400  */
1401  filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */
1402  if (mkdir(filename, S_IRWXU) != 0)
1403  {
1404  /*
1405  * When streaming WAL, pg_wal (or pg_xlog for pre-9.6
1406  * clusters) will have been created by the wal
1407  * receiver process. Also, when the WAL directory
1408  * location was specified, pg_wal (or pg_xlog) has
1409  * already been created as a symbolic link before
1410  * starting the actual backup. So just ignore creation
1411  * failures on related directories.
1412  */
1413  if (!((pg_str_endswith(filename, "/pg_wal") ||
1414  pg_str_endswith(filename, "/pg_xlog") ||
1415  pg_str_endswith(filename, "/archive_status")) &&
1416  errno == EEXIST))
1417  {
1418  fprintf(stderr,
1419  _("%s: could not create directory \"%s\": %s\n"),
1420  progname, filename, strerror(errno));
1422  }
1423  }
1424 #ifndef WIN32
1425  if (chmod(filename, (mode_t) filemode))
1426  fprintf(stderr,
1427  _("%s: could not set permissions on directory \"%s\": %s\n"),
1428  progname, filename, strerror(errno));
1429 #endif
1430  }
1431  else if (copybuf[156] == '2')
1432  {
1433  /*
1434  * Symbolic link
1435  *
1436  * It's most likely a link in pg_tblspc directory, to the
1437  * location of a tablespace. Apply any tablespace mapping
1438  * given on the command line (--tablespace-mapping). (We
1439  * blindly apply the mapping without checking that the
1440  * link really is inside pg_tblspc. We don't expect there
1441  * to be other symlinks in a data directory, but if there
1442  * are, you can call it an undocumented feature that you
1443  * can map them too.)
1444  */
1445  filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */
1446 
1447  mapped_tblspc_path = get_tablespace_mapping(&copybuf[157]);
1448  if (symlink(mapped_tblspc_path, filename) != 0)
1449  {
1450  fprintf(stderr,
1451  _("%s: could not create symbolic link from \"%s\" to \"%s\": %s\n"),
1452  progname, filename, mapped_tblspc_path,
1453  strerror(errno));
1455  }
1456  }
1457  else
1458  {
1459  fprintf(stderr,
1460  _("%s: unrecognized link indicator \"%c\"\n"),
1461  progname, copybuf[156]);
1463  }
1464  continue; /* directory or link handled */
1465  }
1466 
1467  /*
1468  * regular file
1469  */
1470  file = fopen(filename, "wb");
1471  if (!file)
1472  {
1473  fprintf(stderr, _("%s: could not create file \"%s\": %s\n"),
1474  progname, filename, strerror(errno));
1476  }
1477 
1478 #ifndef WIN32
1479  if (chmod(filename, (mode_t) filemode))
1480  fprintf(stderr, _("%s: could not set permissions on file \"%s\": %s\n"),
1481  progname, filename, strerror(errno));
1482 #endif
1483 
1484  if (current_len_left == 0)
1485  {
1486  /*
1487  * Done with this file, next one will be a new tar header
1488  */
1489  fclose(file);
1490  file = NULL;
1491  continue;
1492  }
1493  } /* new file */
1494  else
1495  {
1496  /*
1497  * Continuing blocks in existing file
1498  */
1499  if (current_len_left == 0 && r == current_padding)
1500  {
1501  /*
1502  * Received the padding block for this file, ignore it and
1503  * close the file, then move on to the next tar header.
1504  */
1505  fclose(file);
1506  file = NULL;
1507  totaldone += r;
1508  continue;
1509  }
1510 
1511  if (fwrite(copybuf, r, 1, file) != 1)
1512  {
1513  fprintf(stderr, _("%s: could not write to file \"%s\": %s\n"),
1514  progname, filename, strerror(errno));
1516  }
1517  totaldone += r;
1518  progress_report(rownum, filename, false);
1519 
1520  current_len_left -= r;
1521  if (current_len_left == 0 && current_padding == 0)
1522  {
1523  /*
1524  * Received the last block, and there is no padding to be
1525  * expected. Close the file and move on to the next tar
1526  * header.
1527  */
1528  fclose(file);
1529  file = NULL;
1530  continue;
1531  }
1532  } /* continuing data in existing file */
1533  } /* loop over all data blocks */
1534  progress_report(rownum, filename, true);
1535 
1536  if (file != NULL)
1537  {
1538  fprintf(stderr,
1539  _("%s: COPY stream ended before last file was finished\n"),
1540  progname);
1542  }
1543 
1544  if (copybuf != NULL)
1545  PQfreemem(copybuf);
1546 
1547  if (basetablespace && writerecoveryconf)
1549 
1550  /*
1551  * No data is synced here, everything is done for all tablespaces at the
1552  * end.
1553  */
1554 }
1555 
1556 /*
1557  * Escape a string so that it can be used as a value in a key-value pair
1558  * a configuration file.
1559  */
1560 static char *
1561 escape_quotes(const char *src)
1562 {
1563  char *result = escape_single_quotes_ascii(src);
1564 
1565  if (!result)
1566  {
1567  fprintf(stderr, _("%s: out of memory\n"), progname);
1568  exit(1);
1569  }
1570  return result;
1571 }
1572 
1573 /*
1574  * Create a recovery.conf file in memory using a PQExpBuffer
1575  */
1576 static void
1578 {
1579  PQconninfoOption *connOptions;
1581  PQExpBufferData conninfo_buf;
1582  char *escaped;
1583 
1584  recoveryconfcontents = createPQExpBuffer();
1585  if (!recoveryconfcontents)
1586  {
1587  fprintf(stderr, _("%s: out of memory\n"), progname);
1589  }
1590 
1591  connOptions = PQconninfo(conn);
1592  if (connOptions == NULL)
1593  {
1594  fprintf(stderr, _("%s: out of memory\n"), progname);
1596  }
1597 
1598  appendPQExpBufferStr(recoveryconfcontents, "standby_mode = 'on'\n");
1599 
1600  initPQExpBuffer(&conninfo_buf);
1601  for (option = connOptions; option && option->keyword; option++)
1602  {
1603  /*
1604  * Do not emit this setting if: - the setting is "replication",
1605  * "dbname" or "fallback_application_name", since these would be
1606  * overridden by the libpqwalreceiver module anyway. - not set or
1607  * empty.
1608  */
1609  if (strcmp(option->keyword, "replication") == 0 ||
1610  strcmp(option->keyword, "dbname") == 0 ||
1611  strcmp(option->keyword, "fallback_application_name") == 0 ||
1612  (option->val == NULL) ||
1613  (option->val != NULL && option->val[0] == '\0'))
1614  continue;
1615 
1616  /* Separate key-value pairs with spaces */
1617  if (conninfo_buf.len != 0)
1618  appendPQExpBufferChar(&conninfo_buf, ' ');
1619 
1620  /*
1621  * Write "keyword=value" pieces, the value string is escaped and/or
1622  * quoted if necessary.
1623  */
1624  appendPQExpBuffer(&conninfo_buf, "%s=", option->keyword);
1625  appendConnStrVal(&conninfo_buf, option->val);
1626  }
1627 
1628  /*
1629  * Escape the connection string, so that it can be put in the config file.
1630  * Note that this is different from the escaping of individual connection
1631  * options above!
1632  */
1633  escaped = escape_quotes(conninfo_buf.data);
1634  appendPQExpBuffer(recoveryconfcontents, "primary_conninfo = '%s'\n", escaped);
1635  free(escaped);
1636 
1637  if (replication_slot)
1638  {
1639  escaped = escape_quotes(replication_slot);
1640  appendPQExpBuffer(recoveryconfcontents, "primary_slot_name = '%s'\n", replication_slot);
1641  free(escaped);
1642  }
1643 
1644  if (PQExpBufferBroken(recoveryconfcontents) ||
1645  PQExpBufferDataBroken(conninfo_buf))
1646  {
1647  fprintf(stderr, _("%s: out of memory\n"), progname);
1649  }
1650 
1651  termPQExpBuffer(&conninfo_buf);
1652 
1653  PQconninfoFree(connOptions);
1654 }
1655 
1656 
1657 /*
1658  * Write a recovery.conf file into the directory specified in basedir,
1659  * with the contents already collected in memory.
1660  */
1661 static void
1663 {
1664  char filename[MAXPGPATH];
1665  FILE *cf;
1666 
1667  sprintf(filename, "%s/recovery.conf", basedir);
1668 
1669  cf = fopen(filename, "w");
1670  if (cf == NULL)
1671  {
1672  fprintf(stderr, _("%s: could not create file \"%s\": %s\n"), progname, filename, strerror(errno));
1674  }
1675 
1676  if (fwrite(recoveryconfcontents->data, recoveryconfcontents->len, 1, cf) != 1)
1677  {
1678  fprintf(stderr,
1679  _("%s: could not write to file \"%s\": %s\n"),
1680  progname, filename, strerror(errno));
1682  }
1683 
1684  fclose(cf);
1685 }
1686 
1687 
1688 static void
1690 {
1691  PGresult *res;
1692  char *sysidentifier;
1693  TimeLineID latesttli;
1694  TimeLineID starttli;
1695  char *basebkp;
1696  char escaped_label[MAXPGPATH];
1697  char *maxrate_clause = NULL;
1698  int i;
1699  char xlogstart[64];
1700  char xlogend[64];
1701  int minServerMajor,
1702  maxServerMajor;
1703  int serverVersion,
1704  serverMajor;
1705 
1706  Assert(conn != NULL);
1707 
1708  /*
1709  * Check server version. BASE_BACKUP command was introduced in 9.1, so we
1710  * can't work with servers older than 9.1.
1711  */
1712  minServerMajor = 901;
1713  maxServerMajor = PG_VERSION_NUM / 100;
1714  serverVersion = PQserverVersion(conn);
1715  serverMajor = serverVersion / 100;
1716  if (serverMajor < minServerMajor || serverMajor > maxServerMajor)
1717  {
1718  const char *serverver = PQparameterStatus(conn, "server_version");
1719 
1720  fprintf(stderr, _("%s: incompatible server version %s\n"),
1721  progname, serverver ? serverver : "'unknown'");
1723  }
1724 
1725  /*
1726  * If WAL streaming was requested, also check that the server is new
1727  * enough for that.
1728  */
1730  {
1731  /*
1732  * Error message already written in CheckServerVersionForStreaming(),
1733  * but add a hint about using -X none.
1734  */
1735  fprintf(stderr, _("HINT: use -X none or -X fetch to disable log streaming\n"));
1737  }
1738 
1739  /*
1740  * Build contents of recovery.conf if requested
1741  */
1742  if (writerecoveryconf)
1744 
1745  /*
1746  * Run IDENTIFY_SYSTEM so we can get the timeline
1747  */
1748  if (!RunIdentifySystem(conn, &sysidentifier, &latesttli, NULL, NULL))
1750 
1751  /*
1752  * Start the actual backup
1753  */
1754  PQescapeStringConn(conn, escaped_label, label, sizeof(escaped_label), &i);
1755 
1756  if (maxrate > 0)
1757  maxrate_clause = psprintf("MAX_RATE %u", maxrate);
1758 
1759  if (verbose)
1760  fprintf(stderr,
1761  _("%s: initiating base backup, waiting for checkpoint to complete\n"),
1762  progname);
1763 
1764  if (showprogress && !verbose)
1765  fprintf(stderr, "waiting for checkpoint\r");
1766 
1767  basebkp =
1768  psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s",
1769  escaped_label,
1770  showprogress ? "PROGRESS" : "",
1771  includewal == FETCH_WAL ? "WAL" : "",
1772  fastcheckpoint ? "FAST" : "",
1773  includewal == NO_WAL ? "" : "NOWAIT",
1774  maxrate_clause ? maxrate_clause : "",
1775  format == 't' ? "TABLESPACE_MAP" : "");
1776 
1777  if (PQsendQuery(conn, basebkp) == 0)
1778  {
1779  fprintf(stderr, _("%s: could not send replication command \"%s\": %s"),
1780  progname, "BASE_BACKUP", PQerrorMessage(conn));
1782  }
1783 
1784  /*
1785  * Get the starting WAL location
1786  */
1787  res = PQgetResult(conn);
1788  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1789  {
1790  fprintf(stderr, _("%s: could not initiate base backup: %s"),
1793  }
1794  if (PQntuples(res) != 1)
1795  {
1796  fprintf(stderr,
1797  _("%s: server returned unexpected response to BASE_BACKUP command; got %d rows and %d fields, expected %d rows and %d fields\n"),
1798  progname, PQntuples(res), PQnfields(res), 1, 2);
1800  }
1801 
1802  strlcpy(xlogstart, PQgetvalue(res, 0, 0), sizeof(xlogstart));
1803 
1804  if (verbose)
1805  fprintf(stderr, _("%s: checkpoint completed\n"), progname);
1806 
1807  /*
1808  * 9.3 and later sends the TLI of the starting point. With older servers,
1809  * assume it's the same as the latest timeline reported by
1810  * IDENTIFY_SYSTEM.
1811  */
1812  if (PQnfields(res) >= 2)
1813  starttli = atoi(PQgetvalue(res, 0, 1));
1814  else
1815  starttli = latesttli;
1816  PQclear(res);
1817  MemSet(xlogend, 0, sizeof(xlogend));
1818 
1819  if (verbose && includewal != NO_WAL)
1820  fprintf(stderr, _("%s: write-ahead log start point: %s on timeline %u\n"),
1821  progname, xlogstart, starttli);
1822 
1823  /*
1824  * Get the header
1825  */
1826  res = PQgetResult(conn);
1827  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1828  {
1829  fprintf(stderr, _("%s: could not get backup header: %s"),
1832  }
1833  if (PQntuples(res) < 1)
1834  {
1835  fprintf(stderr, _("%s: no data returned from server\n"), progname);
1837  }
1838 
1839  /*
1840  * Sum up the total size, for progress reporting
1841  */
1842  totalsize = totaldone = 0;
1843  tablespacecount = PQntuples(res);
1844  for (i = 0; i < PQntuples(res); i++)
1845  {
1846  totalsize += atol(PQgetvalue(res, i, 2));
1847 
1848  /*
1849  * Verify tablespace directories are empty. Don't bother with the
1850  * first once since it can be relocated, and it will be checked before
1851  * we do anything anyway.
1852  */
1853  if (format == 'p' && !PQgetisnull(res, i, 1))
1854  {
1855  char *path = (char *) get_tablespace_mapping(PQgetvalue(res, i, 1));
1856 
1858  }
1859  }
1860 
1861  /*
1862  * When writing to stdout, require a single tablespace
1863  */
1864  if (format == 't' && strcmp(basedir, "-") == 0 && PQntuples(res) > 1)
1865  {
1866  fprintf(stderr,
1867  _("%s: can only write single tablespace to stdout, database has %d\n"),
1868  progname, PQntuples(res));
1870  }
1871 
1872  /*
1873  * If we're streaming WAL, start the streaming session before we start
1874  * receiving the actual data chunks.
1875  */
1876  if (includewal == STREAM_WAL)
1877  {
1878  if (verbose)
1879  fprintf(stderr, _("%s: starting background WAL receiver\n"),
1880  progname);
1881  StartLogStreamer(xlogstart, starttli, sysidentifier);
1882  }
1883 
1884  /*
1885  * Start receiving chunks
1886  */
1887  for (i = 0; i < PQntuples(res); i++)
1888  {
1889  if (format == 't')
1890  ReceiveTarFile(conn, res, i);
1891  else
1892  ReceiveAndUnpackTarFile(conn, res, i);
1893  } /* Loop over all tablespaces */
1894 
1895  if (showprogress)
1896  {
1897  progress_report(PQntuples(res), NULL, true);
1898  fprintf(stderr, "\n"); /* Need to move to next line */
1899  }
1900 
1901  PQclear(res);
1902 
1903  /*
1904  * Get the stop position
1905  */
1906  res = PQgetResult(conn);
1907  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1908  {
1909  fprintf(stderr,
1910  _("%s: could not get write-ahead log end position from server: %s"),
1913  }
1914  if (PQntuples(res) != 1)
1915  {
1916  fprintf(stderr,
1917  _("%s: no write-ahead log end position returned from server\n"),
1918  progname);
1920  }
1921  strlcpy(xlogend, PQgetvalue(res, 0, 0), sizeof(xlogend));
1922  if (verbose && includewal != NO_WAL)
1923  fprintf(stderr, _("%s: write-ahead log end point: %s\n"), progname, xlogend);
1924  PQclear(res);
1925 
1926  res = PQgetResult(conn);
1927  if (PQresultStatus(res) != PGRES_COMMAND_OK)
1928  {
1929  fprintf(stderr, _("%s: final receive failed: %s"),
1932  }
1933 
1934  if (bgchild > 0)
1935  {
1936 #ifndef WIN32
1937  int status;
1938  int r;
1939 #else
1940  DWORD status;
1941 
1942  /*
1943  * get a pointer sized version of bgchild to avoid warnings about
1944  * casting to a different size on WIN64.
1945  */
1946  intptr_t bgchild_handle = bgchild;
1947  uint32 hi,
1948  lo;
1949 #endif
1950 
1951  if (verbose)
1952  fprintf(stderr,
1953  _("%s: waiting for background process to finish streaming ...\n"), progname);
1954 
1955 #ifndef WIN32
1956  if (write(bgpipe[1], xlogend, strlen(xlogend)) != strlen(xlogend))
1957  {
1958  fprintf(stderr,
1959  _("%s: could not send command to background pipe: %s\n"),
1960  progname, strerror(errno));
1962  }
1963 
1964  /* Just wait for the background process to exit */
1965  r = waitpid(bgchild, &status, 0);
1966  if (r == -1)
1967  {
1968  fprintf(stderr, _("%s: could not wait for child process: %s\n"),
1969  progname, strerror(errno));
1971  }
1972  if (r != bgchild)
1973  {
1974  fprintf(stderr, _("%s: child %d died, expected %d\n"),
1975  progname, r, (int) bgchild);
1977  }
1978  if (!WIFEXITED(status))
1979  {
1980  fprintf(stderr, _("%s: child process did not exit normally\n"),
1981  progname);
1983  }
1984  if (WEXITSTATUS(status) != 0)
1985  {
1986  fprintf(stderr, _("%s: child process exited with error %d\n"),
1987  progname, WEXITSTATUS(status));
1989  }
1990  /* Exited normally, we're happy! */
1991 #else /* WIN32 */
1992 
1993  /*
1994  * On Windows, since we are in the same process, we can just store the
1995  * value directly in the variable, and then set the flag that says
1996  * it's there.
1997  */
1998  if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
1999  {
2000  fprintf(stderr,
2001  _("%s: could not parse write-ahead log location \"%s\"\n"),
2002  progname, xlogend);
2004  }
2005  xlogendptr = ((uint64) hi) << 32 | lo;
2006  InterlockedIncrement(&has_xlogendptr);
2007 
2008  /* First wait for the thread to exit */
2009  if (WaitForSingleObjectEx((HANDLE) bgchild_handle, INFINITE, FALSE) !=
2010  WAIT_OBJECT_0)
2011  {
2012  _dosmaperr(GetLastError());
2013  fprintf(stderr, _("%s: could not wait for child thread: %s\n"),
2014  progname, strerror(errno));
2016  }
2017  if (GetExitCodeThread((HANDLE) bgchild_handle, &status) == 0)
2018  {
2019  _dosmaperr(GetLastError());
2020  fprintf(stderr, _("%s: could not get child thread exit status: %s\n"),
2021  progname, strerror(errno));
2023  }
2024  if (status != 0)
2025  {
2026  fprintf(stderr, _("%s: child thread exited with error %u\n"),
2027  progname, (unsigned int) status);
2029  }
2030  /* Exited normally, we're happy */
2031 #endif
2032  }
2033 
2034  /* Free the recovery.conf contents */
2035  destroyPQExpBuffer(recoveryconfcontents);
2036 
2037  /*
2038  * End of copy data. Final result is already checked inside the loop.
2039  */
2040  PQclear(res);
2041  PQfinish(conn);
2042 
2043  /*
2044  * Make data persistent on disk once backup is completed. For tar format
2045  * once syncing the parent directory is fine, each tar file created per
2046  * tablespace has been already synced. In plain format, all the data of
2047  * the base directory is synced, taking into account all the tablespaces.
2048  * Errors are not considered fatal.
2049  */
2050  if (do_sync)
2051  {
2052  if (format == 't')
2053  {
2054  if (strcmp(basedir, "-") != 0)
2055  (void) fsync_fname(basedir, true, progname);
2056  }
2057  else
2058  {
2059  (void) fsync_pgdata(basedir, progname, serverVersion);
2060  }
2061  }
2062 
2063  if (verbose)
2064  fprintf(stderr, _("%s: base backup completed\n"), progname);
2065 }
2066 
2067 
2068 int
2069 main(int argc, char **argv)
2070 {
2071  static struct option long_options[] = {
2072  {"help", no_argument, NULL, '?'},
2073  {"version", no_argument, NULL, 'V'},
2074  {"pgdata", required_argument, NULL, 'D'},
2075  {"format", required_argument, NULL, 'F'},
2076  {"checkpoint", required_argument, NULL, 'c'},
2077  {"max-rate", required_argument, NULL, 'r'},
2078  {"write-recovery-conf", no_argument, NULL, 'R'},
2079  {"slot", required_argument, NULL, 'S'},
2080  {"tablespace-mapping", required_argument, NULL, 'T'},
2081  {"wal-method", required_argument, NULL, 'X'},
2082  {"gzip", no_argument, NULL, 'z'},
2083  {"compress", required_argument, NULL, 'Z'},
2084  {"label", required_argument, NULL, 'l'},
2085  {"no-clean", no_argument, NULL, 'n'},
2086  {"no-sync", no_argument, NULL, 'N'},
2087  {"dbname", required_argument, NULL, 'd'},
2088  {"host", required_argument, NULL, 'h'},
2089  {"port", required_argument, NULL, 'p'},
2090  {"username", required_argument, NULL, 'U'},
2091  {"no-password", no_argument, NULL, 'w'},
2092  {"password", no_argument, NULL, 'W'},
2093  {"status-interval", required_argument, NULL, 's'},
2094  {"verbose", no_argument, NULL, 'v'},
2095  {"progress", no_argument, NULL, 'P'},
2096  {"waldir", required_argument, NULL, 1},
2097  {"no-slot", no_argument, NULL, 2},
2098  {NULL, 0, NULL, 0}
2099  };
2100  int c;
2101 
2102  int option_index;
2103  bool no_slot = false;
2104 
2105  progname = get_progname(argv[0]);
2106  set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup"));
2107 
2108  if (argc > 1)
2109  {
2110  if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
2111  {
2112  usage();
2113  exit(0);
2114  }
2115  else if (strcmp(argv[1], "-V") == 0
2116  || strcmp(argv[1], "--version") == 0)
2117  {
2118  puts("pg_basebackup (PostgreSQL) " PG_VERSION);
2119  exit(0);
2120  }
2121  }
2122 
2124 
2125  while ((c = getopt_long(argc, argv, "D:F:r:RT:X:l:nNzZ:d:c:h:p:U:s:S:wWvP",
2126  long_options, &option_index)) != -1)
2127  {
2128  switch (c)
2129  {
2130  case 'D':
2132  break;
2133  case 'F':
2134  if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
2135  format = 'p';
2136  else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
2137  format = 't';
2138  else
2139  {
2140  fprintf(stderr,
2141  _("%s: invalid output format \"%s\", must be \"plain\" or \"tar\"\n"),
2142  progname, optarg);
2143  exit(1);
2144  }
2145  break;
2146  case 'r':
2148  break;
2149  case 'R':
2150  writerecoveryconf = true;
2151  break;
2152  case 'S':
2153 
2154  /*
2155  * When specifying replication slot name, use a permanent
2156  * slot.
2157  */
2159  temp_replication_slot = false;
2160  break;
2161  case 2:
2162  no_slot = true;
2163  break;
2164  case 'T':
2166  break;
2167  case 'X':
2168  if (strcmp(optarg, "n") == 0 ||
2169  strcmp(optarg, "none") == 0)
2170  {
2171  includewal = NO_WAL;
2172  }
2173  else if (strcmp(optarg, "f") == 0 ||
2174  strcmp(optarg, "fetch") == 0)
2175  {
2177  }
2178  else if (strcmp(optarg, "s") == 0 ||
2179  strcmp(optarg, "stream") == 0)
2180  {
2182  }
2183  else
2184  {
2185  fprintf(stderr,
2186  _("%s: invalid wal-method option \"%s\", must be \"fetch\", \"stream\", or \"none\"\n"),
2187  progname, optarg);
2188  exit(1);
2189  }
2190  break;
2191  case 1:
2193  break;
2194  case 'l':
2195  label = pg_strdup(optarg);
2196  break;
2197  case 'n':
2198  noclean = true;
2199  break;
2200  case 'N':
2201  do_sync = false;
2202  break;
2203  case 'z':
2204 #ifdef HAVE_LIBZ
2206 #else
2207  compresslevel = 1; /* will be rejected below */
2208 #endif
2209  break;
2210  case 'Z':
2211  compresslevel = atoi(optarg);
2212  if (compresslevel < 0 || compresslevel > 9)
2213  {
2214  fprintf(stderr, _("%s: invalid compression level \"%s\"\n"),
2215  progname, optarg);
2216  exit(1);
2217  }
2218  break;
2219  case 'c':
2220  if (pg_strcasecmp(optarg, "fast") == 0)
2221  fastcheckpoint = true;
2222  else if (pg_strcasecmp(optarg, "spread") == 0)
2223  fastcheckpoint = false;
2224  else
2225  {
2226  fprintf(stderr, _("%s: invalid checkpoint argument \"%s\", must be \"fast\" or \"spread\"\n"),
2227  progname, optarg);
2228  exit(1);
2229  }
2230  break;
2231  case 'd':
2233  break;
2234  case 'h':
2235  dbhost = pg_strdup(optarg);
2236  break;
2237  case 'p':
2238  dbport = pg_strdup(optarg);
2239  break;
2240  case 'U':
2241  dbuser = pg_strdup(optarg);
2242  break;
2243  case 'w':
2244  dbgetpassword = -1;
2245  break;
2246  case 'W':
2247  dbgetpassword = 1;
2248  break;
2249  case 's':
2250  standby_message_timeout = atoi(optarg) * 1000;
2251  if (standby_message_timeout < 0)
2252  {
2253  fprintf(stderr, _("%s: invalid status interval \"%s\"\n"),
2254  progname, optarg);
2255  exit(1);
2256  }
2257  break;
2258  case 'v':
2259  verbose++;
2260  break;
2261  case 'P':
2262  showprogress = true;
2263  break;
2264  default:
2265 
2266  /*
2267  * getopt_long already emitted a complaint
2268  */
2269  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2270  progname);
2271  exit(1);
2272  }
2273  }
2274 
2275  /*
2276  * Any non-option arguments?
2277  */
2278  if (optind < argc)
2279  {
2280  fprintf(stderr,
2281  _("%s: too many command-line arguments (first is \"%s\")\n"),
2282  progname, argv[optind]);
2283  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2284  progname);
2285  exit(1);
2286  }
2287 
2288  /*
2289  * Required arguments
2290  */
2291  if (basedir == NULL)
2292  {
2293  fprintf(stderr, _("%s: no target directory specified\n"), progname);
2294  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2295  progname);
2296  exit(1);
2297  }
2298 
2299  /*
2300  * Mutually exclusive arguments
2301  */
2302  if (format == 'p' && compresslevel != 0)
2303  {
2304  fprintf(stderr,
2305  _("%s: only tar mode backups can be compressed\n"),
2306  progname);
2307  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2308  progname);
2309  exit(1);
2310  }
2311 
2312  if (format == 't' && includewal == STREAM_WAL && strcmp(basedir, "-") == 0)
2313  {
2314  fprintf(stderr,
2315  _("%s: cannot stream write-ahead logs in tar mode to stdout\n"),
2316  progname);
2317  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2318  progname);
2319  exit(1);
2320  }
2321 
2323  {
2324  fprintf(stderr,
2325  _("%s: replication slots can only be used with WAL streaming\n"),
2326  progname);
2327  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2328  progname);
2329  exit(1);
2330  }
2331 
2332  if (no_slot)
2333  {
2334  if (replication_slot)
2335  {
2336  fprintf(stderr,
2337  _("%s: --no-slot cannot be used with slot name\n"),
2338  progname);
2339  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2340  progname);
2341  exit(1);
2342  }
2343  temp_replication_slot = false;
2344  }
2345 
2346  if (strcmp(xlog_dir, "") != 0)
2347  {
2348  if (format != 'p')
2349  {
2350  fprintf(stderr,
2351  _("%s: WAL directory location can only be specified in plain mode\n"),
2352  progname);
2353  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2354  progname);
2355  exit(1);
2356  }
2357 
2358  /* clean up xlog directory name, check it's absolute */
2360  if (!is_absolute_path(xlog_dir))
2361  {
2362  fprintf(stderr, _("%s: WAL directory location must be "
2363  "an absolute path\n"), progname);
2364  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2365  progname);
2366  exit(1);
2367  }
2368  }
2369 
2370 #ifndef HAVE_LIBZ
2371  if (compresslevel != 0)
2372  {
2373  fprintf(stderr,
2374  _("%s: this build does not support compression\n"),
2375  progname);
2376  exit(1);
2377  }
2378 #endif
2379 
2380  /*
2381  * Verify that the target directory exists, or create it. For plaintext
2382  * backups, always require the directory. For tar backups, require it
2383  * unless we are writing to stdout.
2384  */
2385  if (format == 'p' || strcmp(basedir, "-") != 0)
2387 
2388  /* connection in replication mode to server */
2389  conn = GetConnection();
2390  if (!conn)
2391  {
2392  /* Error message already written in GetConnection() */
2393  exit(1);
2394  }
2395 
2396  /* Create pg_wal symlink, if required */
2397  if (strcmp(xlog_dir, "") != 0)
2398  {
2399  char *linkloc;
2400 
2402 
2403  /*
2404  * Form name of the place where the symlink must go. pg_xlog has been
2405  * renamed to pg_wal in post-10 clusters.
2406  */
2407  linkloc = psprintf("%s/%s", basedir,
2409  "pg_xlog" : "pg_wal");
2410 
2411 #ifdef HAVE_SYMLINK
2412  if (symlink(xlog_dir, linkloc) != 0)
2413  {
2414  fprintf(stderr, _("%s: could not create symbolic link \"%s\": %s\n"),
2415  progname, linkloc, strerror(errno));
2417  }
2418 #else
2419  fprintf(stderr, _("%s: symlinks are not supported on this platform\n"));
2421 #endif
2422  free(linkloc);
2423  }
2424 
2425  BaseBackup();
2426 
2427  success = true;
2428  return 0;
2429 }
int PQnfields(const PGresult *res)
Definition: fe-exec.c:2681
static PQExpBuffer recoveryconfcontents
static void writeTarData(FILE *tarfile, char *buf, int r, char *current_file)
static IncludeWal includewal
Definition: pg_basebackup.c:86
char * PQerrorMessage(const PGconn *conn)
Definition: fe-connect.c:6023
#define Z_DEFAULT_COMPRESSION
static int bgpipe[2]
static bool found_existing_pgdata
Definition: pg_basebackup.c:98
void fsync_pgdata(const char *pg_data, const char *progname, int serverVersion)
Definition: file_utils.c:58
uint32 TimeLineID
Definition: xlogdefs.h:45
int64 pg_time_t
Definition: pgtime.h:23
static void usage(void)
static bool found_tablespace_dirs
char * PQgetvalue(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3067
const char * PQparameterStatus(const PGconn *conn, const char *paramName)
Definition: fe-connect.c:5988
bool pg_str_endswith(const char *str, const char *end)
Definition: string.c:31
int pg_mkdir_p(char *path, int omode)
Definition: pgmkdirp.c:57
static TablespaceList tablespace_dirs
Definition: pg_basebackup.c:78
#define write(a, b, c)
Definition: win32.h:14
bool do_sync
Definition: receivelog.h:39
const char * get_progname(const char *argv0)
Definition: path.c:453
void termPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:128
uint64 read_tar_number(const char *s, int len)
Definition: tar.c:56
int getopt_long(int argc, char *const argv[], const char *optstring, const struct option *longopts, int *longindex)
Definition: getopt_long.c:57
static int32 parse_max_rate(char *src)
void appendPQExpBufferStr(PQExpBuffer str, const char *data)
Definition: pqexpbuffer.c:385
#define mkdir(a, b)
Definition: win32.h:57
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
static pg_time_t last_progress_report
Definition: pg_basebackup.c:91
void fsync_fname(const char *fname, bool isdir)
Definition: fd.c:567
char old_dir[MAXPGPATH]
Definition: pg_basebackup.c:45
static int tablespacecount
bool RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli, XLogRecPtr *startpos, char **db_name)
Definition: streamutil.c:242
#define VERBOSE_FILENAME_LENGTH
IncludeWal
Definition: pg_basebackup.c:69
void appendConnStrVal(PQExpBuffer buf, const char *str)
Definition: string_utils.c:551
void canonicalize_path(char *path)
Definition: path.c:254
#define WIFEXITED(w)
Definition: win32.h:172
char * sysidentifier
Definition: receivelog.h:34
static bool writerecoveryconf
Definition: pg_basebackup.c:88
void PQfinish(PGconn *conn)
Definition: fe-connect.c:3556
static int compresslevel
Definition: pg_basebackup.c:85
#define MemSet(start, val, len)
Definition: c.h:857
#define select(n, r, w, e, timeout)
Definition: win32.h:374
return result
Definition: formatting.c:1633
int snprintf(char *str, size_t count, const char *fmt,...) pg_attribute_printf(3
XLogRecPtr startpos
Definition: receivelog.h:32
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
static char * escape_quotes(const char *src)
char * partial_suffix
Definition: receivelog.h:48
char new_dir[MAXPGPATH]
Definition: pg_basebackup.c:46
int PQserverVersion(const PGconn *conn)
Definition: fe-connect.c:6013
const char * progname
Definition: pg_standby.c:37
int PQntuples(const PGresult *res)
Definition: fe-exec.c:2673
static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline, bool segment_finished)
static void GenerateRecoveryConf(PGconn *conn)
TimeLineID timeline
Definition: receivelog.h:33
static char * basedir
Definition: pg_basebackup.c:77
struct TablespaceListCell TablespaceListCell
WalWriteMethod * CreateWalDirectoryMethod(const char *basedir, int compression, bool sync)
Definition: walmethods.c:346
ExecStatusType PQresultStatus(const PGresult *res)
Definition: fe-exec.c:2596
bool(* finish)(void)
Definition: walmethods.h:75
int PQgetCopyData(PGconn *conn, char **buffer, int async)
Definition: fe-exec.c:2377
signed int int32
Definition: c.h:256
int main(int argc, char **argv)
static void BaseBackup(void)
int PQsendQuery(PGconn *conn, const char *query)
Definition: fe-exec.c:1132
void destroyPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:113
char * escape_single_quotes_ascii(const char *src)
Definition: quotes.c:33
#define required_argument
Definition: getopt_long.h:25
int optind
Definition: getopt.c:51
#define MAX_RATE_UPPER
Definition: basebackup.h:21
char * connection_string
Definition: streamutil.c:34
struct TablespaceList TablespaceList
#define FALSE
Definition: c.h:221
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
PGconn * conn
Definition: streamutil.c:42
#define MAXPGPATH
void appendPQExpBuffer(PQExpBuffer str, const char *fmt,...)
Definition: pqexpbuffer.c:262
#define MINIMUM_VERSION_FOR_PG_WAL
Definition: pg_basebackup.c:59
static bool found_existing_xlogdir
static uint64 totaldone
char * replication_slot
Definition: receivelog.h:49
bool mark_done
Definition: receivelog.h:38
static bool success
Definition: pg_basebackup.c:96
static int32 maxrate
Definition: pg_basebackup.c:92
char * c
static char * buf
Definition: pg_test_fsync.c:66
static int has_xlogendptr
static bool made_new_pgdata
Definition: pg_basebackup.c:97
void PQconninfoFree(PQconninfoOption *connOptions)
Definition: fe-connect.c:5873
#define is_absolute_path(filename)
Definition: port.h:77
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
size_t PQescapeStringConn(PGconn *conn, char *to, const char *from, size_t length, int *error)
Definition: fe-exec.c:3314
unsigned int uint32
Definition: c.h:268
PQconninfoOption * PQconninfo(PGconn *conn)
Definition: fe-connect.c:5832
static uint64 totalsize
static const char * get_tablespace_mapping(const char *dir)
static void progress_report(int tablespacenum, const char *filename, bool force)
XLogRecPtr startptr
stream_stop_callback stream_stop
Definition: receivelog.h:42
WalWriteMethod * walmethod
Definition: receivelog.h:47
static int verbose
Definition: pg_basebackup.c:84
bool rmtree(const char *path, bool rmtopdir)
Definition: rmtree.c:36
static int standby_message_timeout
Definition: pg_basebackup.c:90
static bool showprogress
Definition: pg_basebackup.c:83
enum tarError tarCreateHeader(char *h, const char *filename, const char *linktarget, pgoff_t size, mode_t mode, uid_t uid, gid_t gid, time_t mtime)
Definition: tar.c:112
int dbgetpassword
Definition: streamutil.c:39
#define no_argument
Definition: getopt_long.h:24
#define pgoff_t
Definition: win32.h:231
void FreeWalTarMethod(void)
Definition: walmethods.c:978
#define ngettext(s, p, n)
Definition: c.h:127
#define PG_TEXTDOMAIN(domain)
Definition: c.h:1012
static void WriteRecoveryConf(void)
static bool temp_replication_slot
Definition: pg_basebackup.c:94
int PQbackendPID(const PGconn *conn)
Definition: fe-connect.c:6049
#define PGINVALID_SOCKET
Definition: port.h:24
static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
struct TablespaceListCell * next
Definition: pg_basebackup.c:44
PQExpBuffer createPQExpBuffer(void)
Definition: pqexpbuffer.c:71
static char * label
Definition: pg_basebackup.c:81
void appendPQExpBufferChar(PQExpBuffer str, char ch)
Definition: pqexpbuffer.c:396
StringInfo copybuf
Definition: tablesync.c:114
static void StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
TablespaceListCell * tail
Definition: pg_basebackup.c:52
char * dbport
Definition: streamutil.c:37
static void cleanup_directories_atexit(void)
void PQclear(PGresult *res)
Definition: fe-exec.c:650
#define free(a)
Definition: header.h:65
static bool in_log_streamer
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
PGconn * GetConnection(UserMapping *user, bool will_prep_stmt)
Definition: connection.c:107
#define PQExpBufferDataBroken(buf)
Definition: pqexpbuffer.h:67
#define WEXITSTATUS(w)
Definition: win32.h:174
#define NULL
Definition: c.h:229
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define Assert(condition)
Definition: c.h:675
void _dosmaperr(unsigned long)
Definition: win32error.c:171
WalWriteMethod * CreateWalTarMethod(const char *tarbase, int compression, bool sync)
Definition: walmethods.c:947
static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
char * dbhost
Definition: streamutil.c:35
bool ReceiveXlogStream(PGconn *conn, StreamCtl *stream)
Definition: receivelog.c:444
static char * xlog_dir
Definition: pg_basebackup.c:79
#define PQExpBufferBroken(str)
Definition: pqexpbuffer.h:59
static XLogRecPtr xlogendptr
static XLogRecPtr startpos
TablespaceListCell * head
Definition: pg_basebackup.c:51
bool synchronous
Definition: receivelog.h:37
void pg_free(void *ptr)
Definition: fe_memutils.c:105
pgsocket stop_socket
Definition: receivelog.h:44
static void header(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:207
#define INT64_FORMAT
Definition: c.h:315
static bool made_new_xlogdir
Definition: pg_basebackup.c:99
#define WRITE_TAR_DATA(buf, sz)
static bool fastcheckpoint
Definition: pg_basebackup.c:87
static void tablespace_list_append(const char *arg)
static bool made_tablespace_dirs
static char * filename
Definition: pg_dumpall.c:89
void set_pglocale_pgservice(const char *argv0, const char *app)
Definition: exec.c:550
int errmsg(const char *fmt,...)
Definition: elog.c:797
static pid_t bgchild
char * dbuser
Definition: streamutil.c:36
char * optarg
Definition: getopt.c:53
int pg_check_dir(const char *dir)
Definition: pgcheckdir.c:31
int i
const char * strerror(int errnum)
Definition: strerror.c:19
int standby_message_timeout
Definition: receivelog.h:36
void * arg
static bool do_sync
Definition: pg_basebackup.c:89
static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
static char format
Definition: pg_basebackup.c:80
static void disconnect_and_exit(int code)
static char * replication_slot
Definition: pg_basebackup.c:93
static void static void status(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:224
static int LogStreamerMain(logstreamer_param *param)
int PQgetisnull(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3092
bool temp_slot
Definition: receivelog.h:50
#define _(x)
Definition: elog.c:84
void PQfreemem(void *ptr)
Definition: fe-exec.c:3200
Datum now(PG_FUNCTION_ARGS)
Definition: timestamp.c:1534
PGresult * PQgetResult(PGconn *conn)
Definition: fe-exec.c:1702
#define MINIMUM_VERSION_FOR_TEMP_SLOTS
Definition: pg_basebackup.c:64
#define read(a, b, c)
Definition: win32.h:13
void initPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:89
void FreeWalDirectoryMethod(void)
Definition: walmethods.c:370
bool CheckServerVersionForStreaming(PGconn *conn)
Definition: receivelog.c:364
char xlog[MAXPGPATH]
static bool noclean
Definition: pg_basebackup.c:82