PostgreSQL Source Code  git master
pg_receivewal.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_receivewal.c - receive streaming WAL data and write it
4  * to a local file.
5  *
6  * Author: Magnus Hagander <magnus@hagander.net>
7  *
8  * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
9  *
10  * IDENTIFICATION
11  * src/bin/pg_basebackup/pg_receivewal.c
12  *-------------------------------------------------------------------------
13  */
14 
15 #include "postgres_fe.h"
16 
17 #include <dirent.h>
18 #include <limits.h>
19 #include <signal.h>
20 #include <sys/stat.h>
21 #include <unistd.h>
22 
23 #ifdef USE_LZ4
24 #include <lz4frame.h>
25 #endif
26 #ifdef HAVE_LIBZ
27 #include <zlib.h>
28 #endif
29 
30 #include "access/xlog_internal.h"
31 #include "common/file_perm.h"
32 #include "common/logging.h"
33 #include "fe_utils/option_utils.h"
34 #include "getopt_long.h"
35 #include "libpq-fe.h"
36 #include "receivelog.h"
37 #include "streamutil.h"
38 
39 /* Time to sleep between reconnection attempts */
40 #define RECONNECT_SLEEP_TIME 5
41 
42 /* Global options */
43 static char *basedir = NULL;
44 static int verbose = 0;
45 static int compresslevel = 0;
46 static int noloop = 0;
47 static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
48 static volatile bool time_to_stop = false;
49 static bool do_create_slot = false;
50 static bool slot_exists_ok = false;
51 static bool do_drop_slot = false;
52 static bool do_sync = true;
53 static bool synchronous = false;
54 static char *replication_slot = NULL;
57 
58 
59 static void usage(void);
60 static void parse_compress_options(char *option, char **algorithm,
61  char **detail);
62 static DIR *get_destination_dir(char *dest_folder);
63 static void close_destination_dir(DIR *dest_dir, char *dest_folder);
65 static void StreamLog(void);
66 static bool stop_streaming(XLogRecPtr segendpos, uint32 timeline,
67  bool segment_finished);
68 
69 static void
71 {
72  if (conn != NULL)
73  PQfinish(conn);
74 }
75 
76 static void
77 usage(void)
78 {
79  printf(_("%s receives PostgreSQL streaming write-ahead logs.\n\n"),
80  progname);
81  printf(_("Usage:\n"));
82  printf(_(" %s [OPTION]...\n"), progname);
83  printf(_("\nOptions:\n"));
84  printf(_(" -D, --directory=DIR receive write-ahead log files into this directory\n"));
85  printf(_(" -E, --endpos=LSN exit after receiving the specified LSN\n"));
86  printf(_(" --if-not-exists do not error if slot already exists when creating a slot\n"));
87  printf(_(" -n, --no-loop do not loop on connection lost\n"));
88  printf(_(" --no-sync do not wait for changes to be written safely to disk\n"));
89  printf(_(" -s, --status-interval=SECS\n"
90  " time between status packets sent to server (default: %d)\n"), (standby_message_timeout / 1000));
91  printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
92  printf(_(" --synchronous flush write-ahead log immediately after writing\n"));
93  printf(_(" -v, --verbose output verbose messages\n"));
94  printf(_(" -V, --version output version information, then exit\n"));
95  printf(_(" -Z, --compress=METHOD[:DETAIL]\n"
96  " compress as specified\n"));
97  printf(_(" -?, --help show this help, then exit\n"));
98  printf(_("\nConnection options:\n"));
99  printf(_(" -d, --dbname=CONNSTR connection string\n"));
100  printf(_(" -h, --host=HOSTNAME database server host or socket directory\n"));
101  printf(_(" -p, --port=PORT database server port number\n"));
102  printf(_(" -U, --username=NAME connect as specified database user\n"));
103  printf(_(" -w, --no-password never prompt for password\n"));
104  printf(_(" -W, --password force password prompt (should happen automatically)\n"));
105  printf(_("\nOptional actions:\n"));
106  printf(_(" --create-slot create a new replication slot (for the slot's name see --slot)\n"));
107  printf(_(" --drop-slot drop the replication slot (for the slot's name see --slot)\n"));
108  printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
109  printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
110 }
111 
112 /*
113  * Basic parsing of a value specified for -Z/--compress
114  *
115  * The parsing consists of a METHOD:DETAIL string fed later on to a more
116  * advanced routine in charge of proper validation checks. This only extracts
117  * METHOD and DETAIL. If only an integer is found, the method is implied by
118  * the value specified.
119  */
120 static void
121 parse_compress_options(char *option, char **algorithm, char **detail)
122 {
123  char *sep;
124  char *endp;
125  long result;
126 
127  /*
128  * Check whether the compression specification consists of a bare integer.
129  *
130  * For backward-compatibility, assume "none" if the integer found is zero
131  * and "gzip" otherwise.
132  */
133  result = strtol(option, &endp, 10);
134  if (*endp == '\0')
135  {
136  if (result == 0)
137  {
138  *algorithm = pstrdup("none");
139  *detail = NULL;
140  }
141  else
142  {
143  *algorithm = pstrdup("gzip");
144  *detail = pstrdup(option);
145  }
146  return;
147  }
148 
149  /*
150  * Check whether there is a compression detail following the algorithm
151  * name.
152  */
153  sep = strchr(option, ':');
154  if (sep == NULL)
155  {
156  *algorithm = pstrdup(option);
157  *detail = NULL;
158  }
159  else
160  {
161  char *alg;
162 
163  alg = palloc((sep - option) + 1);
164  memcpy(alg, option, sep - option);
165  alg[sep - option] = '\0';
166 
167  *algorithm = alg;
168  *detail = pstrdup(sep + 1);
169  }
170 }
171 
172 /*
173  * Check if the filename looks like a WAL file, letting caller know if this
174  * WAL segment is partial and/or compressed.
175  */
176 static bool
177 is_xlogfilename(const char *filename, bool *ispartial,
178  pg_compress_algorithm *wal_compression_algorithm)
179 {
180  size_t fname_len = strlen(filename);
181  size_t xlog_pattern_len = strspn(filename, "0123456789ABCDEF");
182 
183  /* File does not look like a WAL file */
184  if (xlog_pattern_len != XLOG_FNAME_LEN)
185  return false;
186 
187  /* File looks like a completed uncompressed WAL file */
188  if (fname_len == XLOG_FNAME_LEN)
189  {
190  *ispartial = false;
191  *wal_compression_algorithm = PG_COMPRESSION_NONE;
192  return true;
193  }
194 
195  /* File looks like a completed gzip-compressed WAL file */
196  if (fname_len == XLOG_FNAME_LEN + strlen(".gz") &&
197  strcmp(filename + XLOG_FNAME_LEN, ".gz") == 0)
198  {
199  *ispartial = false;
200  *wal_compression_algorithm = PG_COMPRESSION_GZIP;
201  return true;
202  }
203 
204  /* File looks like a completed LZ4-compressed WAL file */
205  if (fname_len == XLOG_FNAME_LEN + strlen(".lz4") &&
206  strcmp(filename + XLOG_FNAME_LEN, ".lz4") == 0)
207  {
208  *ispartial = false;
209  *wal_compression_algorithm = PG_COMPRESSION_LZ4;
210  return true;
211  }
212 
213  /* File looks like a partial uncompressed WAL file */
214  if (fname_len == XLOG_FNAME_LEN + strlen(".partial") &&
215  strcmp(filename + XLOG_FNAME_LEN, ".partial") == 0)
216  {
217  *ispartial = true;
218  *wal_compression_algorithm = PG_COMPRESSION_NONE;
219  return true;
220  }
221 
222  /* File looks like a partial gzip-compressed WAL file */
223  if (fname_len == XLOG_FNAME_LEN + strlen(".gz.partial") &&
224  strcmp(filename + XLOG_FNAME_LEN, ".gz.partial") == 0)
225  {
226  *ispartial = true;
227  *wal_compression_algorithm = PG_COMPRESSION_GZIP;
228  return true;
229  }
230 
231  /* File looks like a partial LZ4-compressed WAL file */
232  if (fname_len == XLOG_FNAME_LEN + strlen(".lz4.partial") &&
233  strcmp(filename + XLOG_FNAME_LEN, ".lz4.partial") == 0)
234  {
235  *ispartial = true;
236  *wal_compression_algorithm = PG_COMPRESSION_LZ4;
237  return true;
238  }
239 
240  /* File does not look like something we know */
241  return false;
242 }
243 
244 static bool
245 stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished)
246 {
247  static uint32 prevtimeline = 0;
248  static XLogRecPtr prevpos = InvalidXLogRecPtr;
249 
250  /* we assume that we get called once at the end of each segment */
251  if (verbose && segment_finished)
252  pg_log_info("finished segment at %X/%X (timeline %u)",
253  LSN_FORMAT_ARGS(xlogpos),
254  timeline);
255 
256  if (!XLogRecPtrIsInvalid(endpos) && endpos < xlogpos)
257  {
258  if (verbose)
259  pg_log_info("stopped log streaming at %X/%X (timeline %u)",
260  LSN_FORMAT_ARGS(xlogpos),
261  timeline);
262  time_to_stop = true;
263  return true;
264  }
265 
266  /*
267  * Note that we report the previous, not current, position here. After a
268  * timeline switch, xlogpos points to the beginning of the segment because
269  * that's where we always begin streaming. Reporting the end of previous
270  * timeline isn't totally accurate, because the next timeline can begin
271  * slightly before the end of the WAL that we received on the previous
272  * timeline, but it's close enough for reporting purposes.
273  */
274  if (verbose && prevtimeline != 0 && prevtimeline != timeline)
275  pg_log_info("switched to timeline %u at %X/%X",
276  timeline,
277  LSN_FORMAT_ARGS(prevpos));
278 
279  prevtimeline = timeline;
280  prevpos = xlogpos;
281 
282  if (time_to_stop)
283  {
284  if (verbose)
285  pg_log_info("received interrupt signal, exiting");
286  return true;
287  }
288  return false;
289 }
290 
291 
292 /*
293  * Get destination directory.
294  */
295 static DIR *
296 get_destination_dir(char *dest_folder)
297 {
298  DIR *dir;
299 
300  Assert(dest_folder != NULL);
301  dir = opendir(dest_folder);
302  if (dir == NULL)
303  pg_fatal("could not open directory \"%s\": %m", dest_folder);
304 
305  return dir;
306 }
307 
308 
309 /*
310  * Close existing directory.
311  */
312 static void
313 close_destination_dir(DIR *dest_dir, char *dest_folder)
314 {
315  Assert(dest_dir != NULL && dest_folder != NULL);
316  if (closedir(dest_dir))
317  pg_fatal("could not close directory \"%s\": %m", dest_folder);
318 }
319 
320 
321 /*
322  * Determine starting location for streaming, based on any existing xlog
323  * segments in the directory. We start at the end of the last one that is
324  * complete (size matches wal segment size), on the timeline with highest ID.
325  *
326  * If there are no WAL files in the directory, returns InvalidXLogRecPtr.
327  */
328 static XLogRecPtr
330 {
331  DIR *dir;
332  struct dirent *dirent;
333  XLogSegNo high_segno = 0;
334  uint32 high_tli = 0;
335  bool high_ispartial = false;
336 
338 
339  while (errno = 0, (dirent = readdir(dir)) != NULL)
340  {
341  uint32 tli;
342  XLogSegNo segno;
343  pg_compress_algorithm wal_compression_algorithm;
344  bool ispartial;
345 
347  &ispartial, &wal_compression_algorithm))
348  continue;
349 
350  /*
351  * Looks like an xlog file. Parse its position.
352  */
353  XLogFromFileName(dirent->d_name, &tli, &segno, WalSegSz);
354 
355  /*
356  * Check that the segment has the right size, if it's supposed to be
357  * completed. For non-compressed segments just check the on-disk size
358  * and see if it matches a completed segment. For gzip-compressed
359  * segments, look at the last 4 bytes of the compressed file, which is
360  * where the uncompressed size is located for files with a size lower
361  * than 4GB, and then compare it to the size of a completed segment.
362  * The 4 last bytes correspond to the ISIZE member according to
363  * http://www.zlib.org/rfc-gzip.html.
364  *
365  * For LZ4-compressed segments, uncompress the file in a throw-away
366  * buffer keeping track of the uncompressed size, then compare it to
367  * the size of a completed segment. Per its protocol, LZ4 does not
368  * store the uncompressed size of an object by default. contentSize
369  * is one possible way to do that, but we need to rely on a method
370  * where WAL segments could have been compressed by a different source
371  * than pg_receivewal, like an archive_command with lz4.
372  */
373  if (!ispartial && wal_compression_algorithm == PG_COMPRESSION_NONE)
374  {
375  struct stat statbuf;
376  char fullpath[MAXPGPATH * 2];
377 
378  snprintf(fullpath, sizeof(fullpath), "%s/%s", basedir, dirent->d_name);
379  if (stat(fullpath, &statbuf) != 0)
380  pg_fatal("could not stat file \"%s\": %m", fullpath);
381 
382  if (statbuf.st_size != WalSegSz)
383  {
384  pg_log_warning("segment file \"%s\" has incorrect size %lld, skipping",
385  dirent->d_name, (long long int) statbuf.st_size);
386  continue;
387  }
388  }
389  else if (!ispartial && wal_compression_algorithm == PG_COMPRESSION_GZIP)
390  {
391  int fd;
392  char buf[4];
393  int bytes_out;
394  char fullpath[MAXPGPATH * 2];
395  int r;
396 
397  snprintf(fullpath, sizeof(fullpath), "%s/%s", basedir, dirent->d_name);
398 
399  fd = open(fullpath, O_RDONLY | PG_BINARY, 0);
400  if (fd < 0)
401  pg_fatal("could not open compressed file \"%s\": %m",
402  fullpath);
403  if (lseek(fd, (off_t) (-4), SEEK_END) < 0)
404  pg_fatal("could not seek in compressed file \"%s\": %m",
405  fullpath);
406  r = read(fd, (char *) buf, sizeof(buf));
407  if (r != sizeof(buf))
408  {
409  if (r < 0)
410  pg_fatal("could not read compressed file \"%s\": %m",
411  fullpath);
412  else
413  pg_fatal("could not read compressed file \"%s\": read %d of %zu",
414  fullpath, r, sizeof(buf));
415  }
416 
417  close(fd);
418  bytes_out = (buf[3] << 24) | (buf[2] << 16) |
419  (buf[1] << 8) | buf[0];
420 
421  if (bytes_out != WalSegSz)
422  {
423  pg_log_warning("compressed segment file \"%s\" has incorrect uncompressed size %d, skipping",
424  dirent->d_name, bytes_out);
425  continue;
426  }
427  }
428  else if (!ispartial && wal_compression_algorithm == PG_COMPRESSION_LZ4)
429  {
430 #ifdef USE_LZ4
431 #define LZ4_CHUNK_SZ 64 * 1024 /* 64kB as maximum chunk size read */
432  int fd;
433  ssize_t r;
434  size_t uncompressed_size = 0;
435  char fullpath[MAXPGPATH * 2];
436  char *outbuf;
437  char *readbuf;
438  LZ4F_decompressionContext_t ctx = NULL;
439  LZ4F_decompressOptions_t dec_opt;
440  LZ4F_errorCode_t status;
441 
442  memset(&dec_opt, 0, sizeof(dec_opt));
443  snprintf(fullpath, sizeof(fullpath), "%s/%s", basedir, dirent->d_name);
444 
445  fd = open(fullpath, O_RDONLY | PG_BINARY, 0);
446  if (fd < 0)
447  pg_fatal("could not open file \"%s\": %m", fullpath);
448 
449  status = LZ4F_createDecompressionContext(&ctx, LZ4F_VERSION);
450  if (LZ4F_isError(status))
451  pg_fatal("could not create LZ4 decompression context: %s",
452  LZ4F_getErrorName(status));
453 
454  outbuf = pg_malloc0(LZ4_CHUNK_SZ);
455  readbuf = pg_malloc0(LZ4_CHUNK_SZ);
456  do
457  {
458  char *readp;
459  char *readend;
460 
461  r = read(fd, readbuf, LZ4_CHUNK_SZ);
462  if (r < 0)
463  pg_fatal("could not read file \"%s\": %m", fullpath);
464 
465  /* Done reading the file */
466  if (r == 0)
467  break;
468 
469  /* Process one chunk */
470  readp = readbuf;
471  readend = readbuf + r;
472  while (readp < readend)
473  {
474  size_t out_size = LZ4_CHUNK_SZ;
475  size_t read_size = readend - readp;
476 
477  memset(outbuf, 0, LZ4_CHUNK_SZ);
478  status = LZ4F_decompress(ctx, outbuf, &out_size,
479  readp, &read_size, &dec_opt);
480  if (LZ4F_isError(status))
481  pg_fatal("could not decompress file \"%s\": %s",
482  fullpath,
483  LZ4F_getErrorName(status));
484 
485  readp += read_size;
486  uncompressed_size += out_size;
487  }
488 
489  /*
490  * No need to continue reading the file when the
491  * uncompressed_size exceeds WalSegSz, even if there are still
492  * data left to read. However, if uncompressed_size is equal
493  * to WalSegSz, it should verify that there is no more data to
494  * read.
495  */
496  } while (uncompressed_size <= WalSegSz && r > 0);
497 
498  close(fd);
499  pg_free(outbuf);
500  pg_free(readbuf);
501 
502  status = LZ4F_freeDecompressionContext(ctx);
503  if (LZ4F_isError(status))
504  pg_fatal("could not free LZ4 decompression context: %s",
505  LZ4F_getErrorName(status));
506 
507  if (uncompressed_size != WalSegSz)
508  {
509  pg_log_warning("compressed segment file \"%s\" has incorrect uncompressed size %zu, skipping",
510  dirent->d_name, uncompressed_size);
511  continue;
512  }
513 #else
514  pg_log_error("could not check file \"%s\"",
515  dirent->d_name);
516  pg_log_error_detail("This build does not support compression with %s.",
517  "LZ4");
518  exit(1);
519 #endif
520  }
521 
522  /* Looks like a valid segment. Remember that we saw it. */
523  if ((segno > high_segno) ||
524  (segno == high_segno && tli > high_tli) ||
525  (segno == high_segno && tli == high_tli && high_ispartial && !ispartial))
526  {
527  high_segno = segno;
528  high_tli = tli;
529  high_ispartial = ispartial;
530  }
531  }
532 
533  if (errno)
534  pg_fatal("could not read directory \"%s\": %m", basedir);
535 
537 
538  if (high_segno > 0)
539  {
540  XLogRecPtr high_ptr;
541 
542  /*
543  * Move the starting pointer to the start of the next segment, if the
544  * highest one we saw was completed. Otherwise start streaming from
545  * the beginning of the .partial segment.
546  */
547  if (!high_ispartial)
548  high_segno++;
549 
550  XLogSegNoOffsetToRecPtr(high_segno, 0, WalSegSz, high_ptr);
551 
552  *tli = high_tli;
553  return high_ptr;
554  }
555  else
556  return InvalidXLogRecPtr;
557 }
558 
559 /*
560  * Start the log streaming
561  */
562 static void
564 {
565  XLogRecPtr serverpos;
566  TimeLineID servertli;
567  StreamCtl stream;
568  char *sysidentifier;
569 
570  MemSet(&stream, 0, sizeof(stream));
571 
572  /*
573  * Connect in replication mode to the server
574  */
575  if (conn == NULL)
576  conn = GetConnection();
577  if (!conn)
578  /* Error message already written in GetConnection() */
579  return;
580 
582  {
583  /*
584  * Error message already written in CheckServerVersionForStreaming().
585  * There's no hope of recovering from a version mismatch, so don't
586  * retry.
587  */
588  exit(1);
589  }
590 
591  /*
592  * Identify server, obtaining start LSN position and current timeline ID
593  * at the same time, necessary if not valid data can be found in the
594  * existing output directory.
595  */
596  if (!RunIdentifySystem(conn, &sysidentifier, &servertli, &serverpos, NULL))
597  exit(1);
598 
599  /*
600  * Figure out where to start streaming. First scan the local directory.
601  */
602  stream.startpos = FindStreamingStart(&stream.timeline);
603  if (stream.startpos == InvalidXLogRecPtr)
604  {
605  /*
606  * Try to get the starting point from the slot if any. This is
607  * supported in PostgreSQL 15 and newer.
608  */
609  if (replication_slot != NULL &&
610  PQserverVersion(conn) >= 150000)
611  {
613  &stream.timeline))
614  {
615  /* Error is logged by GetSlotInformation() */
616  return;
617  }
618  }
619 
620  /*
621  * If it the starting point is still not known, use the current WAL
622  * flush value as last resort.
623  */
624  if (stream.startpos == InvalidXLogRecPtr)
625  {
626  stream.startpos = serverpos;
627  stream.timeline = servertli;
628  }
629  }
630 
631  Assert(stream.startpos != InvalidXLogRecPtr &&
632  stream.timeline != 0);
633 
634  /*
635  * Always start streaming at the beginning of a segment
636  */
637  stream.startpos -= XLogSegmentOffset(stream.startpos, WalSegSz);
638 
639  /*
640  * Start the replication
641  */
642  if (verbose)
643  pg_log_info("starting log streaming at %X/%X (timeline %u)",
644  LSN_FORMAT_ARGS(stream.startpos),
645  stream.timeline);
646 
647  stream.stream_stop = stop_streaming;
648  stream.stop_socket = PGINVALID_SOCKET;
650  stream.synchronous = synchronous;
651  stream.do_sync = do_sync;
652  stream.mark_done = false;
656  stream.do_sync);
657  stream.partial_suffix = ".partial";
659  stream.sysidentifier = sysidentifier;
660 
661  ReceiveXlogStream(conn, &stream);
662 
663  if (!stream.walmethod->finish())
664  {
665  pg_log_info("could not finish writing WAL files: %m");
666  return;
667  }
668 
669  PQfinish(conn);
670  conn = NULL;
671 
673  pg_free(stream.walmethod);
674  pg_free(stream.sysidentifier);
675 }
676 
677 /*
678  * When sigint is called, just tell the system to exit at the next possible
679  * moment.
680  */
681 #ifndef WIN32
682 
683 static void
684 sigint_handler(int signum)
685 {
686  time_to_stop = true;
687 }
688 #endif
689 
690 int
691 main(int argc, char **argv)
692 {
693  static struct option long_options[] = {
694  {"help", no_argument, NULL, '?'},
695  {"version", no_argument, NULL, 'V'},
696  {"directory", required_argument, NULL, 'D'},
697  {"dbname", required_argument, NULL, 'd'},
698  {"endpos", required_argument, NULL, 'E'},
699  {"host", required_argument, NULL, 'h'},
700  {"port", required_argument, NULL, 'p'},
701  {"username", required_argument, NULL, 'U'},
702  {"no-loop", no_argument, NULL, 'n'},
703  {"no-password", no_argument, NULL, 'w'},
704  {"password", no_argument, NULL, 'W'},
705  {"status-interval", required_argument, NULL, 's'},
706  {"slot", required_argument, NULL, 'S'},
707  {"verbose", no_argument, NULL, 'v'},
708  {"compress", required_argument, NULL, 'Z'},
709 /* action */
710  {"create-slot", no_argument, NULL, 1},
711  {"drop-slot", no_argument, NULL, 2},
712  {"if-not-exists", no_argument, NULL, 3},
713  {"synchronous", no_argument, NULL, 4},
714  {"no-sync", no_argument, NULL, 5},
715  {NULL, 0, NULL, 0}
716  };
717 
718  int c;
719  int option_index;
720  char *db_name;
721  uint32 hi,
722  lo;
723  pg_compress_specification compression_spec;
724  char *compression_detail = NULL;
725  char *compression_algorithm_str = "none";
726  char *error_detail = NULL;
727 
728  pg_logging_init(argv[0]);
729  progname = get_progname(argv[0]);
730  set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup"));
731 
732  if (argc > 1)
733  {
734  if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
735  {
736  usage();
737  exit(0);
738  }
739  else if (strcmp(argv[1], "-V") == 0 ||
740  strcmp(argv[1], "--version") == 0)
741  {
742  puts("pg_receivewal (PostgreSQL) " PG_VERSION);
743  exit(0);
744  }
745  }
746 
747  while ((c = getopt_long(argc, argv, "D:d:E:h:p:U:s:S:nwWvZ:",
748  long_options, &option_index)) != -1)
749  {
750  switch (c)
751  {
752  case 'D':
754  break;
755  case 'd':
757  break;
758  case 'h':
760  break;
761  case 'p':
763  break;
764  case 'U':
766  break;
767  case 'w':
768  dbgetpassword = -1;
769  break;
770  case 'W':
771  dbgetpassword = 1;
772  break;
773  case 's':
774  if (!option_parse_int(optarg, "-s/--status-interval", 0,
775  INT_MAX / 1000,
777  exit(1);
778  standby_message_timeout *= 1000;
779  break;
780  case 'S':
782  break;
783  case 'E':
784  if (sscanf(optarg, "%X/%X", &hi, &lo) != 2)
785  pg_fatal("could not parse end position \"%s\"", optarg);
786  endpos = ((uint64) hi) << 32 | lo;
787  break;
788  case 'n':
789  noloop = 1;
790  break;
791  case 'v':
792  verbose++;
793  break;
794  case 'Z':
795  parse_compress_options(optarg, &compression_algorithm_str,
796  &compression_detail);
797  break;
798 /* action */
799  case 1:
800  do_create_slot = true;
801  break;
802  case 2:
803  do_drop_slot = true;
804  break;
805  case 3:
806  slot_exists_ok = true;
807  break;
808  case 4:
809  synchronous = true;
810  break;
811  case 5:
812  do_sync = false;
813  break;
814  default:
815  /* getopt_long already emitted a complaint */
816  pg_log_error_hint("Try \"%s --help\" for more information.", progname);
817  exit(1);
818  }
819  }
820 
821  /*
822  * Any non-option arguments?
823  */
824  if (optind < argc)
825  {
826  pg_log_error("too many command-line arguments (first is \"%s\")",
827  argv[optind]);
828  pg_log_error_hint("Try \"%s --help\" for more information.", progname);
829  exit(1);
830  }
831 
833  {
834  pg_log_error("cannot use --create-slot together with --drop-slot");
835  pg_log_error_hint("Try \"%s --help\" for more information.", progname);
836  exit(1);
837  }
838 
839  if (replication_slot == NULL && (do_drop_slot || do_create_slot))
840  {
841  /* translator: second %s is an option name */
842  pg_log_error("%s needs a slot to be specified using --slot",
843  do_drop_slot ? "--drop-slot" : "--create-slot");
844  pg_log_error_hint("Try \"%s --help\" for more information.", progname);
845  exit(1);
846  }
847 
848  if (synchronous && !do_sync)
849  {
850  pg_log_error("cannot use --synchronous together with --no-sync");
851  pg_log_error_hint("Try \"%s --help\" for more information.", progname);
852  exit(1);
853  }
854 
855  /*
856  * Required arguments
857  */
858  if (basedir == NULL && !do_drop_slot && !do_create_slot)
859  {
860  pg_log_error("no target directory specified");
861  pg_log_error_hint("Try \"%s --help\" for more information.", progname);
862  exit(1);
863  }
864 
865  /*
866  * Compression options
867  */
868  if (!parse_compress_algorithm(compression_algorithm_str,
870  pg_fatal("unrecognized compression algorithm \"%s\"",
871  compression_algorithm_str);
872 
874  &compression_spec);
875  error_detail = validate_compress_specification(&compression_spec);
876  if (error_detail != NULL)
877  pg_fatal("invalid compression specification: %s",
878  error_detail);
879 
880  /* Extract the compression level, if found in the specification */
881  if ((compression_spec.options & PG_COMPRESSION_OPTION_LEVEL) != 0)
882  compresslevel = compression_spec.level;
883 
884  switch (compression_algorithm)
885  {
886  case PG_COMPRESSION_NONE:
887  /* nothing to do */
888  break;
889  case PG_COMPRESSION_GZIP:
890 #ifdef HAVE_LIBZ
891  if ((compression_spec.options & PG_COMPRESSION_OPTION_LEVEL) == 0)
892  {
893  pg_log_info("no value specified for --compress, switching to default");
895  }
896 #else
897  pg_fatal("this build does not support compression with %s",
898  "gzip");
899 #endif
900  break;
901  case PG_COMPRESSION_LZ4:
902 #ifndef USE_LZ4
903  pg_fatal("this build does not support compression with %s",
904  "LZ4");
905 #endif
906  break;
907  case PG_COMPRESSION_ZSTD:
908  pg_fatal("compression with %s is not yet supported", "ZSTD");
909  break;
910  }
911 
912 
913  /*
914  * Check existence of destination folder.
915  */
916  if (!do_drop_slot && !do_create_slot)
917  {
919 
921  }
922 
923  /*
924  * Obtain a connection before doing anything.
925  */
926  conn = GetConnection();
927  if (!conn)
928  /* error message already written in GetConnection() */
929  exit(1);
930  atexit(disconnect_atexit);
931 
932  /*
933  * Trap signals. (Don't do this until after the initial password prompt,
934  * if one is needed, in GetConnection.)
935  */
936 #ifndef WIN32
937  pqsignal(SIGINT, sigint_handler);
938 #endif
939 
940  /*
941  * Run IDENTIFY_SYSTEM to make sure we've successfully have established a
942  * replication connection and haven't connected using a database specific
943  * connection.
944  */
945  if (!RunIdentifySystem(conn, NULL, NULL, NULL, &db_name))
946  exit(1);
947 
948  /*
949  * Check that there is a database associated with connection, none should
950  * be defined in this context.
951  */
952  if (db_name)
953  pg_fatal("replication connection using slot \"%s\" is unexpectedly database specific",
955 
956  /*
957  * Set umask so that directories/files are created with the same
958  * permissions as directories/files in the source data directory.
959  *
960  * pg_mode_mask is set to owner-only by default and then updated in
961  * GetConnection() where we get the mode from the server-side with
962  * RetrieveDataDirCreatePerm() and then call SetDataDirectoryCreatePerm().
963  */
964  umask(pg_mode_mask);
965 
966  /*
967  * Drop a replication slot.
968  */
969  if (do_drop_slot)
970  {
971  if (verbose)
972  pg_log_info("dropping replication slot \"%s\"", replication_slot);
973 
975  exit(1);
976  exit(0);
977  }
978 
979  /* Create a replication slot */
980  if (do_create_slot)
981  {
982  if (verbose)
983  pg_log_info("creating replication slot \"%s\"", replication_slot);
984 
985  if (!CreateReplicationSlot(conn, replication_slot, NULL, false, true, false,
986  slot_exists_ok, false))
987  exit(1);
988  exit(0);
989  }
990 
991  /* determine remote server's xlog segment size */
992  if (!RetrieveWalSegSize(conn))
993  exit(1);
994 
995  /*
996  * Don't close the connection here so that subsequent StreamLog() can
997  * reuse it.
998  */
999 
1000  while (true)
1001  {
1002  StreamLog();
1003  if (time_to_stop)
1004  {
1005  /*
1006  * We've been Ctrl-C'ed or end of streaming position has been
1007  * willingly reached, so exit without an error code.
1008  */
1009  exit(0);
1010  }
1011  else if (noloop)
1012  pg_fatal("disconnected");
1013  else
1014  {
1015  /* translator: check source for value for %d */
1016  pg_log_info("disconnected; waiting %d seconds to try again",
1018  pg_usleep(RECONNECT_SLEEP_TIME * 1000000);
1019  }
1020  }
1021 }
unsigned int uint32
Definition: c.h:441
#define PG_TEXTDOMAIN(domain)
Definition: c.h:1212
#define PG_BINARY
Definition: c.h:1268
#define MemSet(start, val, len)
Definition: c.h:1008
void set_pglocale_pgservice(const char *argv0, const char *app)
Definition: exec.c:446
bool parse_compress_algorithm(char *name, pg_compress_algorithm *algorithm)
Definition: compression.c:40
void parse_compress_specification(pg_compress_algorithm algorithm, char *specification, pg_compress_specification *result)
Definition: compression.c:95
char * validate_compress_specification(pg_compress_specification *spec)
Definition: compression.c:250
#define PG_COMPRESSION_OPTION_LEVEL
Definition: compression.h:25
pg_compress_algorithm
Definition: compression.h:18
@ PG_COMPRESSION_GZIP
Definition: compression.h:20
@ PG_COMPRESSION_LZ4
Definition: compression.h:21
@ PG_COMPRESSION_NONE
Definition: compression.h:19
@ PG_COMPRESSION_ZSTD
Definition: compression.h:22
PGconn * GetConnection(UserMapping *user, bool will_prep_stmt, PgFdwConnState **state)
Definition: connection.c:134
int closedir(DIR *)
Definition: dirent.c:123
struct dirent * readdir(DIR *)
Definition: dirent.c:78
DIR * opendir(const char *)
Definition: dirent.c:33
#define _(x)
Definition: elog.c:89
int PQserverVersion(const PGconn *conn)
Definition: fe-connect.c:6898
void PQfinish(PGconn *conn)
Definition: fe-connect.c:4261
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
void pg_free(void *ptr)
Definition: fe_memutils.c:105
int pg_mode_mask
Definition: file_perm.c:25
int getopt_long(int argc, char *const argv[], const char *optstring, const struct option *longopts, int *longindex)
Definition: getopt_long.c:57
#define no_argument
Definition: getopt_long.h:24
#define required_argument
Definition: getopt_long.h:25
#define close(a)
Definition: win32.h:12
#define read(a, b, c)
Definition: win32.h:13
Assert(fmt[strlen(fmt) - 1] !='\n')
exit(1)
void pg_logging_init(const char *argv0)
Definition: logging.c:83
#define pg_log_error(...)
Definition: logging.h:106
#define pg_log_error_hint(...)
Definition: logging.h:112
#define pg_log_info(...)
Definition: logging.h:124
#define pg_log_error_detail(...)
Definition: logging.h:109
const char * progname
Definition: main.c:50
char * pstrdup(const char *in)
Definition: mcxt.c:1305
void * palloc(Size size)
Definition: mcxt.c:1068
bool option_parse_int(const char *optarg, const char *optname, int min_range, int max_range, int *result)
Definition: option_utils.c:50
#define Z_DEFAULT_COMPRESSION
#define pg_fatal(...)
#define MAXPGPATH
static char * filename
Definition: pg_dumpall.c:94
PGDLLIMPORT int optind
Definition: getopt.c:50
PGDLLIMPORT char * optarg
Definition: getopt.c:52
static int verbose
Definition: pg_receivewal.c:44
static int noloop
Definition: pg_receivewal.c:46
static void close_destination_dir(DIR *dest_dir, char *dest_folder)
static volatile bool time_to_stop
Definition: pg_receivewal.c:48
static pg_compress_algorithm compression_algorithm
Definition: pg_receivewal.c:55
static void sigint_handler(int signum)
static XLogRecPtr endpos
Definition: pg_receivewal.c:56
int main(int argc, char **argv)
static bool slot_exists_ok
Definition: pg_receivewal.c:50
static bool synchronous
Definition: pg_receivewal.c:53
static DIR * get_destination_dir(char *dest_folder)
static int compresslevel
Definition: pg_receivewal.c:45
static bool do_drop_slot
Definition: pg_receivewal.c:51
static char * replication_slot
Definition: pg_receivewal.c:54
static void parse_compress_options(char *option, char **algorithm, char **detail)
static bool stop_streaming(XLogRecPtr segendpos, uint32 timeline, bool segment_finished)
static XLogRecPtr FindStreamingStart(uint32 *tli)
static bool do_sync
Definition: pg_receivewal.c:52
static void usage(void)
Definition: pg_receivewal.c:77
#define RECONNECT_SLEEP_TIME
Definition: pg_receivewal.c:40
static bool do_create_slot
Definition: pg_receivewal.c:49
static char * basedir
Definition: pg_receivewal.c:43
static void disconnect_atexit(void)
Definition: pg_receivewal.c:70
static int standby_message_timeout
Definition: pg_receivewal.c:47
static bool is_xlogfilename(const char *filename, bool *ispartial, pg_compress_algorithm *wal_compression_algorithm)
static void StreamLog(void)
static void static void status(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:229
static char * buf
Definition: pg_test_fsync.c:67
#define pg_log_warning(...)
Definition: pgfnames.c:24
const char * get_progname(const char *argv0)
Definition: path.c:574
#define snprintf
Definition: port.h:225
#define PGINVALID_SOCKET
Definition: port.h:31
#define printf(...)
Definition: port.h:231
char * c
static int fd(const char *x, int i)
Definition: preproc-init.c:105
bool ReceiveXlogStream(PGconn *conn, StreamCtl *stream)
Definition: receivelog.c:459
bool CheckServerVersionForStreaming(PGconn *conn)
Definition: receivelog.c:381
void pg_usleep(long microsec)
Definition: signal.c:53
pqsigfunc pqsignal(int signum, pqsigfunc handler)
Definition: signal.c:180
int dbgetpassword
Definition: streamutil.c:52
bool RetrieveWalSegSize(PGconn *conn)
Definition: streamutil.c:271
char * dbhost
Definition: streamutil.c:48
char * dbport
Definition: streamutil.c:50
char * connection_string
Definition: streamutil.c:47
bool GetSlotInformation(PGconn *conn, const char *slot_name, XLogRecPtr *restart_lsn, TimeLineID *restart_tli)
Definition: streamutil.c:484
PGconn * conn
Definition: streamutil.c:54
bool RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli, XLogRecPtr *startpos, char **db_name)
Definition: streamutil.c:403
char * dbuser
Definition: streamutil.c:49
uint32 WalSegSz
Definition: streamutil.c:34
Definition: dirent.c:26
char * sysidentifier
Definition: receivelog.h:33
TimeLineID timeline
Definition: receivelog.h:32
stream_stop_callback stream_stop
Definition: receivelog.h:41
char * replication_slot
Definition: receivelog.h:48
XLogRecPtr startpos
Definition: receivelog.h:31
bool do_sync
Definition: receivelog.h:38
pgsocket stop_socket
Definition: receivelog.h:43
int standby_message_timeout
Definition: receivelog.h:35
WalWriteMethod * walmethod
Definition: receivelog.h:46
bool mark_done
Definition: receivelog.h:37
char * partial_suffix
Definition: receivelog.h:47
bool synchronous
Definition: receivelog.h:36
bool(* finish)(void)
Definition: walmethods.h:85
Definition: dirent.h:10
char d_name[MAX_PATH]
Definition: dirent.h:15
__int64 st_size
Definition: win32_port.h:273
WalWriteMethod * CreateWalDirectoryMethod(const char *basedir, pg_compress_algorithm compression_algorithm, int compression_level, bool sync)
Definition: walmethods.c:625
void FreeWalDirectoryMethod(void)
Definition: walmethods.c:654
static void CreateReplicationSlot(CreateReplicationSlotCmd *cmd)
Definition: walsender.c:1047
static void DropReplicationSlot(DropReplicationSlotCmd *cmd)
Definition: walsender.c:1247
#define stat
Definition: win32_port.h:283
#define XLogSegmentOffset(xlogptr, wal_segsz_bytes)
#define XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest)
#define XLOG_FNAME_LEN
#define XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes)
#define LSN_FORMAT_ARGS(lsn)
Definition: xlogdefs.h:43
#define XLogRecPtrIsInvalid(r)
Definition: xlogdefs.h:29
uint64 XLogRecPtr
Definition: xlogdefs.h:21
#define InvalidXLogRecPtr
Definition: xlogdefs.h:28
uint32 TimeLineID
Definition: xlogdefs.h:59
uint64 XLogSegNo
Definition: xlogdefs.h:48