PostgreSQL Source Code  git master
pg_verifybackup.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_verifybackup.c
4  * Verify a backup against a backup manifest.
5  *
6  * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
7  * Portions Copyright (c) 1994, Regents of the University of California
8  *
9  * src/bin/pg_verifybackup/pg_verifybackup.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 
14 #include "postgres_fe.h"
15 
16 #include <dirent.h>
17 #include <fcntl.h>
18 #include <sys/stat.h>
19 #include <time.h>
20 
21 #include "common/hashfn.h"
22 #include "common/logging.h"
23 #include "fe_utils/simple_list.h"
24 #include "getopt_long.h"
25 #include "parse_manifest.h"
26 #include "pgtime.h"
27 
28 /*
29  * For efficiency, we'd like our hash table containing information about the
30  * manifest to start out with approximately the correct number of entries.
31  * There's no way to know the exact number of entries without reading the whole
32  * file, but we can get an estimate by dividing the file size by the estimated
33  * number of bytes per line.
34  *
35  * This could be off by about a factor of two in either direction, because the
36  * checksum algorithm has a big impact on the line lengths; e.g. a SHA512
37  * checksum is 128 hex bytes, whereas a CRC-32C value is only 8, and there
38  * might be no checksum at all.
39  */
40 #define ESTIMATED_BYTES_PER_MANIFEST_LINE 100
41 
42 /*
43  * How many bytes should we try to read from a file at once?
44  */
45 #define READ_CHUNK_SIZE 4096
46 
47 /*
48  * Each file described by the manifest file is parsed to produce an object
49  * like this.
50  */
51 typedef struct manifest_file
52 {
53  uint32 status; /* hash status */
54  char *pathname;
55  size_t size;
59  bool matched;
60  bool bad;
62 
63 #define should_verify_checksum(m) \
64  (((m)->matched) && !((m)->bad) && (((m)->checksum_type) != CHECKSUM_TYPE_NONE))
65 
66 /*
67  * Define a hash table which we can use to store information about the files
68  * mentioned in the backup manifest.
69  */
70 static uint32 hash_string_pointer(char *s);
71 #define SH_PREFIX manifest_files
72 #define SH_ELEMENT_TYPE manifest_file
73 #define SH_KEY_TYPE char *
74 #define SH_KEY pathname
75 #define SH_HASH_KEY(tb, key) hash_string_pointer(key)
76 #define SH_EQUAL(tb, a, b) (strcmp(a, b) == 0)
77 #define SH_SCOPE static inline
78 #define SH_RAW_ALLOCATOR pg_malloc0
79 #define SH_DECLARE
80 #define SH_DEFINE
81 #include "lib/simplehash.h"
82 
83 /*
84  * Each WAL range described by the manifest file is parsed to produce an
85  * object like this.
86  */
87 typedef struct manifest_wal_range
88 {
95 
96 /*
97  * Details we need in callbacks that occur while parsing a backup manifest.
98  */
99 typedef struct parser_context
100 {
101  manifest_files_hash *ht;
105 
106 /*
107  * All of the context information we need while checking a backup manifest.
108  */
109 typedef struct verifier_context
110 {
111  manifest_files_hash *ht;
117 
118 static void parse_manifest_file(char *manifest_path,
119  manifest_files_hash **ht_p,
120  manifest_wal_range **first_wal_range_p);
121 
123  char *pathname, size_t size,
124  pg_checksum_type checksum_type,
125  int checksum_length,
126  uint8 *checksum_payload);
128  TimeLineID tli,
129  XLogRecPtr start_lsn,
130  XLogRecPtr end_lsn);
132  const char *fmt,...)
134 
135 static void verify_backup_directory(verifier_context *context,
136  char *relpath, char *fullpath);
137 static void verify_backup_file(verifier_context *context,
138  char *relpath, char *fullpath);
139 static void report_extra_backup_files(verifier_context *context);
140 static void verify_backup_checksums(verifier_context *context);
141 static void verify_file_checksum(verifier_context *context,
142  manifest_file *m, char *fullpath);
143 static void parse_required_wal(verifier_context *context,
144  char *pg_waldump_path,
145  char *wal_directory,
146  manifest_wal_range *first_wal_range);
147 
148 static void report_backup_error(verifier_context *context,
149  const char *pg_restrict fmt,...)
150  pg_attribute_printf(2, 3);
151 static void report_fatal_error(const char *pg_restrict fmt,...)
153 static bool should_ignore_relpath(verifier_context *context, char *relpath);
154 
155 static void progress_report(bool finished);
156 static void usage(void);
157 
158 static const char *progname;
159 
160 /* options */
161 static bool show_progress = false;
162 static bool skip_checksums = false;
163 
164 /* Progress indicators */
165 static uint64 total_size = 0;
166 static uint64 done_size = 0;
167 
168 /*
169  * Main entry point.
170  */
171 int
172 main(int argc, char **argv)
173 {
174  static struct option long_options[] = {
175  {"exit-on-error", no_argument, NULL, 'e'},
176  {"ignore", required_argument, NULL, 'i'},
177  {"manifest-path", required_argument, NULL, 'm'},
178  {"no-parse-wal", no_argument, NULL, 'n'},
179  {"progress", no_argument, NULL, 'P'},
180  {"quiet", no_argument, NULL, 'q'},
181  {"skip-checksums", no_argument, NULL, 's'},
182  {"wal-directory", required_argument, NULL, 'w'},
183  {NULL, 0, NULL, 0}
184  };
185 
186  int c;
187  verifier_context context;
188  manifest_wal_range *first_wal_range;
189  char *manifest_path = NULL;
190  bool no_parse_wal = false;
191  bool quiet = false;
192  char *wal_directory = NULL;
193  char *pg_waldump_path = NULL;
194 
195  pg_logging_init(argv[0]);
196  set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_verifybackup"));
197  progname = get_progname(argv[0]);
198 
199  memset(&context, 0, sizeof(context));
200 
201  if (argc > 1)
202  {
203  if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
204  {
205  usage();
206  exit(0);
207  }
208  if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
209  {
210  puts("pg_verifybackup (PostgreSQL) " PG_VERSION);
211  exit(0);
212  }
213  }
214 
215  /*
216  * Skip certain files in the toplevel directory.
217  *
218  * Ignore the backup_manifest file, because it's not included in the
219  * backup manifest.
220  *
221  * Ignore the pg_wal directory, because those files are not included in
222  * the backup manifest either, since they are fetched separately from the
223  * backup itself, and verified via a separate mechanism.
224  *
225  * Ignore postgresql.auto.conf, recovery.signal, and standby.signal,
226  * because we expect that those files may sometimes be created or changed
227  * as part of the backup process. For example, pg_basebackup -R will
228  * modify postgresql.auto.conf and create standby.signal.
229  */
230  simple_string_list_append(&context.ignore_list, "backup_manifest");
231  simple_string_list_append(&context.ignore_list, "pg_wal");
232  simple_string_list_append(&context.ignore_list, "postgresql.auto.conf");
233  simple_string_list_append(&context.ignore_list, "recovery.signal");
234  simple_string_list_append(&context.ignore_list, "standby.signal");
235 
236  while ((c = getopt_long(argc, argv, "ei:m:nPqsw:", long_options, NULL)) != -1)
237  {
238  switch (c)
239  {
240  case 'e':
241  context.exit_on_error = true;
242  break;
243  case 'i':
244  {
245  char *arg = pstrdup(optarg);
246 
248  simple_string_list_append(&context.ignore_list, arg);
249  break;
250  }
251  case 'm':
252  manifest_path = pstrdup(optarg);
253  canonicalize_path(manifest_path);
254  break;
255  case 'n':
256  no_parse_wal = true;
257  break;
258  case 'P':
259  show_progress = true;
260  break;
261  case 'q':
262  quiet = true;
263  break;
264  case 's':
265  skip_checksums = true;
266  break;
267  case 'w':
268  wal_directory = pstrdup(optarg);
269  canonicalize_path(wal_directory);
270  break;
271  default:
272  /* getopt_long already emitted a complaint */
273  pg_log_error_hint("Try \"%s --help\" for more information.", progname);
274  exit(1);
275  }
276  }
277 
278  /* Get backup directory name */
279  if (optind >= argc)
280  {
281  pg_log_error("no backup directory specified");
282  pg_log_error_hint("Try \"%s --help\" for more information.", progname);
283  exit(1);
284  }
285  context.backup_directory = pstrdup(argv[optind++]);
286  canonicalize_path(context.backup_directory);
287 
288  /* Complain if any arguments remain */
289  if (optind < argc)
290  {
291  pg_log_error("too many command-line arguments (first is \"%s\")",
292  argv[optind]);
293  pg_log_error_hint("Try \"%s --help\" for more information.", progname);
294  exit(1);
295  }
296 
297  /* Complain if the specified arguments conflict */
298  if (show_progress && quiet)
299  pg_fatal("cannot specify both %s and %s",
300  "-P/--progress", "-q/--quiet");
301 
302  /* Unless --no-parse-wal was specified, we will need pg_waldump. */
303  if (!no_parse_wal)
304  {
305  int ret;
306 
307  pg_waldump_path = pg_malloc(MAXPGPATH);
308  ret = find_other_exec(argv[0], "pg_waldump",
309  "pg_waldump (PostgreSQL) " PG_VERSION "\n",
310  pg_waldump_path);
311  if (ret < 0)
312  {
313  char full_path[MAXPGPATH];
314 
315  if (find_my_exec(argv[0], full_path) < 0)
316  strlcpy(full_path, progname, sizeof(full_path));
317 
318  if (ret == -1)
319  pg_fatal("program \"%s\" is needed by %s but was not found in the same directory as \"%s\"",
320  "pg_waldump", "pg_verifybackup", full_path);
321  else
322  pg_fatal("program \"%s\" was found by \"%s\" but was not the same version as %s",
323  "pg_waldump", full_path, "pg_verifybackup");
324  }
325  }
326 
327  /* By default, look for the manifest in the backup directory. */
328  if (manifest_path == NULL)
329  manifest_path = psprintf("%s/backup_manifest",
330  context.backup_directory);
331 
332  /* By default, look for the WAL in the backup directory, too. */
333  if (wal_directory == NULL)
334  wal_directory = psprintf("%s/pg_wal", context.backup_directory);
335 
336  /*
337  * Try to read the manifest. We treat any errors encountered while parsing
338  * the manifest as fatal; there doesn't seem to be much point in trying to
339  * verify the backup directory against a corrupted manifest.
340  */
341  parse_manifest_file(manifest_path, &context.ht, &first_wal_range);
342 
343  /*
344  * Now scan the files in the backup directory. At this stage, we verify
345  * that every file on disk is present in the manifest and that the sizes
346  * match. We also set the "matched" flag on every manifest entry that
347  * corresponds to a file on disk.
348  */
349  verify_backup_directory(&context, NULL, context.backup_directory);
350 
351  /*
352  * The "matched" flag should now be set on every entry in the hash table.
353  * Any entries for which the bit is not set are files mentioned in the
354  * manifest that don't exist on disk.
355  */
356  report_extra_backup_files(&context);
357 
358  /*
359  * Now do the expensive work of verifying file checksums, unless we were
360  * told to skip it.
361  */
362  if (!skip_checksums)
363  verify_backup_checksums(&context);
364 
365  /*
366  * Try to parse the required ranges of WAL records, unless we were told
367  * not to do so.
368  */
369  if (!no_parse_wal)
370  parse_required_wal(&context, pg_waldump_path,
371  wal_directory, first_wal_range);
372 
373  /*
374  * If everything looks OK, tell the user this, unless we were asked to
375  * work quietly.
376  */
377  if (!context.saw_any_error && !quiet)
378  printf(_("backup successfully verified\n"));
379 
380  return context.saw_any_error ? 1 : 0;
381 }
382 
383 /*
384  * Parse a manifest file. Construct a hash table with information about
385  * all the files it mentions, and a linked list of all the WAL ranges it
386  * mentions.
387  */
388 static void
389 parse_manifest_file(char *manifest_path, manifest_files_hash **ht_p,
390  manifest_wal_range **first_wal_range_p)
391 {
392  int fd;
393  struct stat statbuf;
394  off_t estimate;
395  uint32 initial_size;
396  manifest_files_hash *ht;
397  char *buffer;
398  int rc;
399  parser_context private_context;
400  JsonManifestParseContext context;
401 
402  /* Open the manifest file. */
403  if ((fd = open(manifest_path, O_RDONLY | PG_BINARY, 0)) < 0)
404  report_fatal_error("could not open file \"%s\": %m", manifest_path);
405 
406  /* Figure out how big the manifest is. */
407  if (fstat(fd, &statbuf) != 0)
408  report_fatal_error("could not stat file \"%s\": %m", manifest_path);
409 
410  /* Guess how large to make the hash table based on the manifest size. */
411  estimate = statbuf.st_size / ESTIMATED_BYTES_PER_MANIFEST_LINE;
412  initial_size = Min(PG_UINT32_MAX, Max(estimate, 256));
413 
414  /* Create the hash table. */
415  ht = manifest_files_create(initial_size, NULL);
416 
417  /*
418  * Slurp in the whole file.
419  *
420  * This is not ideal, but there's currently no easy way to get
421  * pg_parse_json() to perform incremental parsing.
422  */
423  buffer = pg_malloc(statbuf.st_size);
424  rc = read(fd, buffer, statbuf.st_size);
425  if (rc != statbuf.st_size)
426  {
427  if (rc < 0)
428  report_fatal_error("could not read file \"%s\": %m",
429  manifest_path);
430  else
431  report_fatal_error("could not read file \"%s\": read %d of %lld",
432  manifest_path, rc, (long long int) statbuf.st_size);
433  }
434 
435  /* Close the manifest file. */
436  close(fd);
437 
438  /* Parse the manifest. */
439  private_context.ht = ht;
440  private_context.first_wal_range = NULL;
441  private_context.last_wal_range = NULL;
442  context.private_data = &private_context;
446  json_parse_manifest(&context, buffer, statbuf.st_size);
447 
448  /* Done with the buffer. */
449  pfree(buffer);
450 
451  /* Return the file hash table and WAL range list we constructed. */
452  *ht_p = ht;
453  *first_wal_range_p = private_context.first_wal_range;
454 }
455 
456 /*
457  * Report an error while parsing the manifest.
458  *
459  * We consider all such errors to be fatal errors. The manifest parser
460  * expects this function not to return.
461  */
462 static void
464 {
465  va_list ap;
466 
467  va_start(ap, fmt);
469  va_end(ap);
470 
471  exit(1);
472 }
473 
474 /*
475  * Record details extracted from the backup manifest for one file.
476  */
477 static void
479  char *pathname, size_t size,
480  pg_checksum_type checksum_type,
481  int checksum_length, uint8 *checksum_payload)
482 {
483  parser_context *pcxt = context->private_data;
484  manifest_files_hash *ht = pcxt->ht;
485  manifest_file *m;
486  bool found;
487 
488  /* Make a new entry in the hash table for this file. */
489  m = manifest_files_insert(ht, pathname, &found);
490  if (found)
491  report_fatal_error("duplicate path name in backup manifest: \"%s\"",
492  pathname);
493 
494  /* Initialize the entry. */
495  m->size = size;
496  m->checksum_type = checksum_type;
497  m->checksum_length = checksum_length;
498  m->checksum_payload = checksum_payload;
499  m->matched = false;
500  m->bad = false;
501 }
502 
503 /*
504  * Record details extracted from the backup manifest for one WAL range.
505  */
506 static void
508  TimeLineID tli,
509  XLogRecPtr start_lsn, XLogRecPtr end_lsn)
510 {
511  parser_context *pcxt = context->private_data;
513 
514  /* Allocate and initialize a struct describing this WAL range. */
515  range = palloc(sizeof(manifest_wal_range));
516  range->tli = tli;
517  range->start_lsn = start_lsn;
518  range->end_lsn = end_lsn;
519  range->prev = pcxt->last_wal_range;
520  range->next = NULL;
521 
522  /* Add it to the end of the list. */
523  if (pcxt->first_wal_range == NULL)
524  pcxt->first_wal_range = range;
525  else
526  pcxt->last_wal_range->next = range;
527  pcxt->last_wal_range = range;
528 }
529 
530 /*
531  * Verify one directory.
532  *
533  * 'relpath' is NULL if we are to verify the top-level backup directory,
534  * and otherwise the relative path to the directory that is to be verified.
535  *
536  * 'fullpath' is the backup directory with 'relpath' appended; i.e. the actual
537  * filesystem path at which it can be found.
538  */
539 static void
541  char *fullpath)
542 {
543  DIR *dir;
544  struct dirent *dirent;
545 
546  dir = opendir(fullpath);
547  if (dir == NULL)
548  {
549  /*
550  * If even the toplevel backup directory cannot be found, treat this
551  * as a fatal error.
552  */
553  if (relpath == NULL)
554  report_fatal_error("could not open directory \"%s\": %m", fullpath);
555 
556  /*
557  * Otherwise, treat this as a non-fatal error, but ignore any further
558  * errors related to this path and anything beneath it.
559  */
560  report_backup_error(context,
561  "could not open directory \"%s\": %m", fullpath);
563 
564  return;
565  }
566 
567  while (errno = 0, (dirent = readdir(dir)) != NULL)
568  {
569  char *filename = dirent->d_name;
570  char *newfullpath = psprintf("%s/%s", fullpath, filename);
571  char *newrelpath;
572 
573  /* Skip "." and ".." */
574  if (filename[0] == '.' && (filename[1] == '\0'
575  || strcmp(filename, "..") == 0))
576  continue;
577 
578  if (relpath == NULL)
579  newrelpath = pstrdup(filename);
580  else
581  newrelpath = psprintf("%s/%s", relpath, filename);
582 
583  if (!should_ignore_relpath(context, newrelpath))
584  verify_backup_file(context, newrelpath, newfullpath);
585 
586  pfree(newfullpath);
587  pfree(newrelpath);
588  }
589 
590  if (closedir(dir))
591  {
592  report_backup_error(context,
593  "could not close directory \"%s\": %m", fullpath);
594  return;
595  }
596 }
597 
598 /*
599  * Verify one file (which might actually be a directory or a symlink).
600  *
601  * The arguments to this function have the same meaning as the arguments to
602  * verify_backup_directory.
603  */
604 static void
605 verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
606 {
607  struct stat sb;
608  manifest_file *m;
609 
610  if (stat(fullpath, &sb) != 0)
611  {
612  report_backup_error(context,
613  "could not stat file or directory \"%s\": %m",
614  relpath);
615 
616  /*
617  * Suppress further errors related to this path name and, if it's a
618  * directory, anything underneath it.
619  */
621 
622  return;
623  }
624 
625  /* If it's a directory, just recurse. */
626  if (S_ISDIR(sb.st_mode))
627  {
628  verify_backup_directory(context, relpath, fullpath);
629  return;
630  }
631 
632  /* If it's not a directory, it should be a plain file. */
633  if (!S_ISREG(sb.st_mode))
634  {
635  report_backup_error(context,
636  "\"%s\" is not a file or directory",
637  relpath);
638  return;
639  }
640 
641  /* Check whether there's an entry in the manifest hash. */
642  m = manifest_files_lookup(context->ht, relpath);
643  if (m == NULL)
644  {
645  report_backup_error(context,
646  "\"%s\" is present on disk but not in the manifest",
647  relpath);
648  return;
649  }
650 
651  /* Flag this entry as having been encountered in the filesystem. */
652  m->matched = true;
653 
654  /* Check that the size matches. */
655  if (m->size != sb.st_size)
656  {
657  report_backup_error(context,
658  "\"%s\" has size %lld on disk but size %zu in the manifest",
659  relpath, (long long int) sb.st_size, m->size);
660  m->bad = true;
661  }
662 
663  /* Update statistics for progress report, if necessary */
665  total_size += m->size;
666 
667  /*
668  * We don't verify checksums at this stage. We first finish verifying that
669  * we have the expected set of files with the expected sizes, and only
670  * afterwards verify the checksums. That's because computing checksums may
671  * take a while, and we'd like to report more obvious problems quickly.
672  */
673 }
674 
675 /*
676  * Scan the hash table for entries where the 'matched' flag is not set; report
677  * that such files are present in the manifest but not on disk.
678  */
679 static void
681 {
682  manifest_files_iterator it;
683  manifest_file *m;
684 
685  manifest_files_start_iterate(context->ht, &it);
686  while ((m = manifest_files_iterate(context->ht, &it)) != NULL)
687  if (!m->matched && !should_ignore_relpath(context, m->pathname))
688  report_backup_error(context,
689  "\"%s\" is present in the manifest but not on disk",
690  m->pathname);
691 }
692 
693 /*
694  * Verify checksums for hash table entries that are otherwise unproblematic.
695  * If we've already reported some problem related to a hash table entry, or
696  * if it has no checksum, just skip it.
697  */
698 static void
700 {
701  manifest_files_iterator it;
702  manifest_file *m;
703 
704  progress_report(false);
705 
706  manifest_files_start_iterate(context->ht, &it);
707  while ((m = manifest_files_iterate(context->ht, &it)) != NULL)
708  {
709  if (should_verify_checksum(m) &&
710  !should_ignore_relpath(context, m->pathname))
711  {
712  char *fullpath;
713 
714  /* Compute the full pathname to the target file. */
715  fullpath = psprintf("%s/%s", context->backup_directory,
716  m->pathname);
717 
718  /* Do the actual checksum verification. */
719  verify_file_checksum(context, m, fullpath);
720 
721  /* Avoid leaking memory. */
722  pfree(fullpath);
723  }
724  }
725 
726  progress_report(true);
727 }
728 
729 /*
730  * Verify the checksum of a single file.
731  */
732 static void
734  char *fullpath)
735 {
736  pg_checksum_context checksum_ctx;
737  char *relpath = m->pathname;
738  int fd;
739  int rc;
740  size_t bytes_read = 0;
741  uint8 buffer[READ_CHUNK_SIZE];
742  uint8 checksumbuf[PG_CHECKSUM_MAX_LENGTH];
743  int checksumlen;
744 
745  /* Open the target file. */
746  if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
747  {
748  report_backup_error(context, "could not open file \"%s\": %m",
749  relpath);
750  return;
751  }
752 
753  /* Initialize checksum context. */
754  if (pg_checksum_init(&checksum_ctx, m->checksum_type) < 0)
755  {
756  report_backup_error(context, "could not initialize checksum of file \"%s\"",
757  relpath);
758  close(fd);
759  return;
760  }
761 
762  /* Read the file chunk by chunk, updating the checksum as we go. */
763  while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
764  {
765  bytes_read += rc;
766  if (pg_checksum_update(&checksum_ctx, buffer, rc) < 0)
767  {
768  report_backup_error(context, "could not update checksum of file \"%s\"",
769  relpath);
770  close(fd);
771  return;
772  }
773 
774  /* Report progress */
775  done_size += rc;
776  progress_report(false);
777  }
778  if (rc < 0)
779  report_backup_error(context, "could not read file \"%s\": %m",
780  relpath);
781 
782  /* Close the file. */
783  if (close(fd) != 0)
784  {
785  report_backup_error(context, "could not close file \"%s\": %m",
786  relpath);
787  return;
788  }
789 
790  /* If we didn't manage to read the whole file, bail out now. */
791  if (rc < 0)
792  return;
793 
794  /*
795  * Double-check that we read the expected number of bytes from the file.
796  * Normally, a file size mismatch would be caught in verify_backup_file
797  * and this check would never be reached, but this provides additional
798  * safety and clarity in the event of concurrent modifications or
799  * filesystem misbehavior.
800  */
801  if (bytes_read != m->size)
802  {
803  report_backup_error(context,
804  "file \"%s\" should contain %zu bytes, but read %zu bytes",
805  relpath, m->size, bytes_read);
806  return;
807  }
808 
809  /* Get the final checksum. */
810  checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
811  if (checksumlen < 0)
812  {
813  report_backup_error(context,
814  "could not finalize checksum of file \"%s\"",
815  relpath);
816  return;
817  }
818 
819  /* And check it against the manifest. */
820  if (checksumlen != m->checksum_length)
821  report_backup_error(context,
822  "file \"%s\" has checksum of length %d, but expected %d",
823  relpath, m->checksum_length, checksumlen);
824  else if (memcmp(checksumbuf, m->checksum_payload, checksumlen) != 0)
825  report_backup_error(context,
826  "checksum mismatch for file \"%s\"",
827  relpath);
828 }
829 
830 /*
831  * Attempt to parse the WAL files required to restore from backup using
832  * pg_waldump.
833  */
834 static void
835 parse_required_wal(verifier_context *context, char *pg_waldump_path,
836  char *wal_directory, manifest_wal_range *first_wal_range)
837 {
838  manifest_wal_range *this_wal_range = first_wal_range;
839 
840  while (this_wal_range != NULL)
841  {
842  char *pg_waldump_cmd;
843 
844  pg_waldump_cmd = psprintf("\"%s\" --quiet --path=\"%s\" --timeline=%u --start=%X/%X --end=%X/%X\n",
845  pg_waldump_path, wal_directory, this_wal_range->tli,
846  LSN_FORMAT_ARGS(this_wal_range->start_lsn),
847  LSN_FORMAT_ARGS(this_wal_range->end_lsn));
848  fflush(NULL);
849  if (system(pg_waldump_cmd) != 0)
850  report_backup_error(context,
851  "WAL parsing failed for timeline %u",
852  this_wal_range->tli);
853 
854  this_wal_range = this_wal_range->next;
855  }
856 }
857 
858 /*
859  * Report a problem with the backup.
860  *
861  * Update the context to indicate that we saw an error, and exit if the
862  * context says we should.
863  */
864 static void
865 report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
866 {
867  va_list ap;
868 
869  va_start(ap, fmt);
871  va_end(ap);
872 
873  context->saw_any_error = true;
874  if (context->exit_on_error)
875  exit(1);
876 }
877 
878 /*
879  * Report a fatal error and exit
880  */
881 static void
882 report_fatal_error(const char *pg_restrict fmt,...)
883 {
884  va_list ap;
885 
886  va_start(ap, fmt);
888  va_end(ap);
889 
890  exit(1);
891 }
892 
893 /*
894  * Is the specified relative path, or some prefix of it, listed in the set
895  * of paths to ignore?
896  *
897  * Note that by "prefix" we mean a parent directory; for this purpose,
898  * "aa/bb" is not a prefix of "aa/bbb", but it is a prefix of "aa/bb/cc".
899  */
900 static bool
902 {
903  SimpleStringListCell *cell;
904 
905  for (cell = context->ignore_list.head; cell != NULL; cell = cell->next)
906  {
907  char *r = relpath;
908  char *v = cell->val;
909 
910  while (*v != '\0' && *r == *v)
911  ++r, ++v;
912 
913  if (*v == '\0' && (*r == '\0' || *r == '/'))
914  return true;
915  }
916 
917  return false;
918 }
919 
920 /*
921  * Helper function for manifest_files hash table.
922  */
923 static uint32
925 {
926  unsigned char *ss = (unsigned char *) s;
927 
928  return hash_bytes(ss, strlen(s));
929 }
930 
931 /*
932  * Print a progress report based on the global variables.
933  *
934  * Progress report is written at maximum once per second, unless the finished
935  * parameter is set to true.
936  *
937  * If finished is set to true, this is the last progress report. The cursor
938  * is moved to the next line.
939  */
940 static void
941 progress_report(bool finished)
942 {
943  static pg_time_t last_progress_report = 0;
944  pg_time_t now;
945  int percent_size = 0;
946  char totalsize_str[32];
947  char donesize_str[32];
948 
949  if (!show_progress)
950  return;
951 
952  now = time(NULL);
953  if (now == last_progress_report && !finished)
954  return; /* Max once per second */
955 
957  percent_size = total_size ? (int) ((done_size * 100 / total_size)) : 0;
958 
959  snprintf(totalsize_str, sizeof(totalsize_str), UINT64_FORMAT,
960  total_size / 1024);
961  snprintf(donesize_str, sizeof(donesize_str), UINT64_FORMAT,
962  done_size / 1024);
963 
964  fprintf(stderr,
965  _("%*s/%s kB (%d%%) verified"),
966  (int) strlen(totalsize_str),
967  donesize_str, totalsize_str, percent_size);
968 
969  /*
970  * Stay on the same line if reporting to a terminal and we're not done
971  * yet.
972  */
973  fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr);
974 }
975 
976 /*
977  * Print out usage information and exit.
978  */
979 static void
980 usage(void)
981 {
982  printf(_("%s verifies a backup against the backup manifest.\n\n"), progname);
983  printf(_("Usage:\n %s [OPTION]... BACKUPDIR\n\n"), progname);
984  printf(_("Options:\n"));
985  printf(_(" -e, --exit-on-error exit immediately on error\n"));
986  printf(_(" -i, --ignore=RELATIVE_PATH ignore indicated path\n"));
987  printf(_(" -m, --manifest-path=PATH use specified path for manifest\n"));
988  printf(_(" -n, --no-parse-wal do not try to parse WAL files\n"));
989  printf(_(" -P, --progress show progress information\n"));
990  printf(_(" -q, --quiet do not print any output, except for errors\n"));
991  printf(_(" -s, --skip-checksums skip checksum verification\n"));
992  printf(_(" -w, --wal-directory=PATH use specified path for WAL files\n"));
993  printf(_(" -V, --version output version information, then exit\n"));
994  printf(_(" -?, --help show this help, then exit\n"));
995  printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
996  printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
997 }
Datum now(PG_FUNCTION_ARGS)
Definition: timestamp.c:1547
unsigned int uint32
Definition: c.h:495
#define Min(x, y)
Definition: c.h:993
#define PG_UINT32_MAX
Definition: c.h:579
#define Max(x, y)
Definition: c.h:987
#define PG_TEXTDOMAIN(domain)
Definition: c.h:1227
#define PG_BINARY
Definition: c.h:1283
#define pg_attribute_printf(f, a)
Definition: c.h:180
#define gettext(x)
Definition: c.h:1192
#define UINT64_FORMAT
Definition: c.h:538
unsigned char uint8
Definition: c.h:493
int pg_checksum_final(pg_checksum_context *context, uint8 *output)
int pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
int pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
#define PG_CHECKSUM_MAX_LENGTH
pg_checksum_type
int find_my_exec(const char *argv0, char *retpath)
Definition: exec.c:158
void set_pglocale_pgservice(const char *argv0, const char *app)
Definition: exec.c:436
int find_other_exec(const char *argv0, const char *target, const char *versionstr, char *retpath)
Definition: exec.c:327
int closedir(DIR *)
Definition: dirent.c:127
struct dirent * readdir(DIR *)
Definition: dirent.c:78
DIR * opendir(const char *)
Definition: dirent.c:33
#define _(x)
Definition: elog.c:91
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
int getopt_long(int argc, char *const argv[], const char *optstring, const struct option *longopts, int *longindex)
Definition: getopt_long.c:60
#define no_argument
Definition: getopt_long.h:24
#define required_argument
Definition: getopt_long.h:25
uint32 hash_bytes(const unsigned char *k, int keylen)
Definition: hashfn.c:146
#define close(a)
Definition: win32.h:12
#define read(a, b, c)
Definition: win32.h:13
return false
Definition: isn.c:131
static void const char * fmt
static void const char fflush(stdout)
va_end(args)
exit(1)
va_start(args, fmt)
void pg_logging_init(const char *argv0)
Definition: logging.c:83
void pg_log_generic_v(enum pg_log_level level, enum pg_log_part part, const char *pg_restrict fmt, va_list ap)
Definition: logging.c:216
#define pg_log_error(...)
Definition: logging.h:106
#define pg_log_error_hint(...)
Definition: logging.h:112
@ PG_LOG_PRIMARY
Definition: logging.h:67
@ PG_LOG_ERROR
Definition: logging.h:43
char * pstrdup(const char *in)
Definition: mcxt.c:1644
void pfree(void *pointer)
Definition: mcxt.c:1456
void * palloc(Size size)
Definition: mcxt.c:1226
void json_parse_manifest(JsonManifestParseContext *context, char *buffer, size_t size)
static pg_time_t last_progress_report
Definition: pg_amcheck.c:144
void * arg
#define pg_fatal(...)
#define MAXPGPATH
static char * filename
Definition: pg_dumpall.c:119
PGDLLIMPORT int optind
Definition: getopt.c:50
PGDLLIMPORT char * optarg
Definition: getopt.c:52
static void verify_backup_checksums(verifier_context *context)
static uint64 done_size
static void record_manifest_details_for_file(JsonManifestParseContext *context, char *pathname, size_t size, pg_checksum_type checksum_type, int checksum_length, uint8 *checksum_payload)
struct verifier_context verifier_context
static void parse_manifest_file(char *manifest_path, manifest_files_hash **ht_p, manifest_wal_range **first_wal_range_p)
struct parser_context parser_context
static void parse_required_wal(verifier_context *context, char *pg_waldump_path, char *wal_directory, manifest_wal_range *first_wal_range)
int main(int argc, char **argv)
static uint64 total_size
struct manifest_wal_range manifest_wal_range
static void report_extra_backup_files(verifier_context *context)
#define ESTIMATED_BYTES_PER_MANIFEST_LINE
static bool skip_checksums
static void progress_report(bool finished)
struct manifest_file manifest_file
static void pg_attribute_noreturn()
#define READ_CHUNK_SIZE
static bool should_ignore_relpath(verifier_context *context, char *relpath)
#define should_verify_checksum(m)
static void record_manifest_details_for_wal_range(JsonManifestParseContext *context, TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
static bool show_progress
static void verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
static void report_manifest_error(JsonManifestParseContext *context, const char *fmt,...) pg_attribute_printf(2
static void static void report_fatal_error(const char *pg_restrict fmt,...) pg_attribute_printf(1
static void verify_file_checksum(verifier_context *context, manifest_file *m, char *fullpath)
static const char * progname
static void verify_backup_directory(verifier_context *context, char *relpath, char *fullpath)
static void usage(void)
static uint32 hash_string_pointer(char *s)
static void report_backup_error(verifier_context *context, const char *pg_restrict fmt,...) pg_attribute_printf(2
int64 pg_time_t
Definition: pgtime.h:23
void canonicalize_path(char *path)
Definition: path.c:264
const char * get_progname(const char *argv0)
Definition: path.c:574
#define snprintf
Definition: port.h:238
#define fprintf
Definition: port.h:242
#define printf(...)
Definition: port.h:244
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
char * c
static int fd(const char *x, int i)
Definition: preproc-init.c:105
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
static struct cvec * range(struct vars *v, chr a, chr b, int cases)
Definition: regc_locale.c:412
#define relpath(rlocator, forknum)
Definition: relpath.h:94
void simple_string_list_append(SimpleStringList *list, const char *val)
Definition: simple_list.c:63
Definition: dirent.c:26
json_manifest_perwalrange_callback perwalrange_cb
json_manifest_perfile_callback perfile_cb
json_manifest_error_callback error_cb
char val[FLEXIBLE_ARRAY_MEMBER]
Definition: simple_list.h:37
struct SimpleStringListCell * next
Definition: simple_list.h:34
SimpleStringListCell * head
Definition: simple_list.h:42
Definition: dirent.h:10
char d_name[MAX_PATH]
Definition: dirent.h:15
uint8 * checksum_payload
pg_checksum_type checksum_type
struct manifest_wal_range * next
struct manifest_wal_range * prev
manifest_files_hash * ht
manifest_wal_range * last_wal_range
manifest_wal_range * first_wal_range
__int64 st_size
Definition: win32_port.h:273
unsigned short st_mode
Definition: win32_port.h:268
SimpleStringList ignore_list
manifest_files_hash * ht
#define stat
Definition: win32_port.h:284
#define S_ISDIR(m)
Definition: win32_port.h:325
#define fstat
Definition: win32_port.h:283
#define S_ISREG(m)
Definition: win32_port.h:328
#define LSN_FORMAT_ARGS(lsn)
Definition: xlogdefs.h:43
uint64 XLogRecPtr
Definition: xlogdefs.h:21
uint32 TimeLineID
Definition: xlogdefs.h:59