PostgreSQL Source Code  git master
pg_amcheck.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_amcheck.c
4  * Detects corruption within database relations.
5  *
6  * Copyright (c) 2017-2021, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * src/bin/pg_amcheck/pg_amcheck.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 #include "postgres_fe.h"
14 
15 #include <limits.h>
16 #include <time.h>
17 
18 #include "catalog/pg_am_d.h"
19 #include "catalog/pg_namespace_d.h"
20 #include "common/logging.h"
21 #include "common/username.h"
22 #include "fe_utils/cancel.h"
23 #include "fe_utils/option_utils.h"
24 #include "fe_utils/parallel_slot.h"
25 #include "fe_utils/query_utils.h"
26 #include "fe_utils/simple_list.h"
27 #include "fe_utils/string_utils.h"
28 #include "getopt_long.h" /* pgrminclude ignore */
29 #include "pgtime.h"
30 #include "storage/block.h"
31 
32 typedef struct PatternInfo
33 {
34  const char *pattern; /* Unaltered pattern from the command line */
35  char *db_regex; /* Database regexp parsed from pattern, or
36  * NULL */
37  char *nsp_regex; /* Schema regexp parsed from pattern, or NULL */
38  char *rel_regex; /* Relation regexp parsed from pattern, or
39  * NULL */
40  bool heap_only; /* true if rel_regex should only match heap
41  * tables */
42  bool btree_only; /* true if rel_regex should only match btree
43  * indexes */
44  bool matched; /* true if the pattern matched in any database */
45 } PatternInfo;
46 
47 typedef struct PatternInfoArray
48 {
50  size_t len;
52 
53 /* pg_amcheck command line options controlled by user flags */
54 typedef struct AmcheckOptions
55 {
56  bool dbpattern;
57  bool alldb;
58  bool echo;
59  bool verbose;
62  int jobs;
63 
64  /*
65  * Whether to install missing extensions, and optionally the name of the
66  * schema in which to install the extension's objects.
67  */
70 
71  /* Objects to check or not to check, as lists of PatternInfo structs. */
74 
75  /*
76  * As an optimization, if any pattern in the exclude list applies to heap
77  * tables, or similarly if any such pattern applies to btree indexes, or
78  * to schemas, then these will be true, otherwise false. These should
79  * always agree with what you'd conclude by grep'ing through the exclude
80  * list.
81  */
82  bool excludetbl;
83  bool excludeidx;
84  bool excludensp;
85 
86  /*
87  * If any inclusion pattern exists, then we should only be checking
88  * matching relations rather than all relations, so this is true iff
89  * include is empty.
90  */
91  bool allrel;
92 
93  /* heap table checking options */
97  int64 startblock;
98  int64 endblock;
99  const char *skip;
100 
101  /* btree index checking options */
105 
106  /* heap and btree hybrid option */
109 
111  .dbpattern = false,
112  .alldb = false,
113  .echo = false,
114  .verbose = false,
115  .strict_names = true,
116  .show_progress = false,
117  .jobs = 1,
118  .install_missing = false,
119  .install_schema = "pg_catalog",
120  .include = {NULL, 0},
121  .exclude = {NULL, 0},
122  .excludetbl = false,
123  .excludeidx = false,
124  .excludensp = false,
125  .allrel = true,
126  .no_toast_expansion = false,
127  .reconcile_toast = true,
128  .on_error_stop = false,
129  .startblock = -1,
130  .endblock = -1,
131  .skip = "none",
132  .parent_check = false,
133  .rootdescend = false,
134  .heapallindexed = false,
135  .no_btree_expansion = false
136 };
137 
138 static const char *progname = NULL;
139 
140 /* Whether all relations have so far passed their corruption checks */
141 static bool all_checks_pass = true;
142 
143 /* Time last progress report was displayed */
145 static bool progress_since_last_stderr = false;
146 
147 typedef struct DatabaseInfo
148 {
149  char *datname;
150  char *amcheck_schema; /* escaped, quoted literal */
151 } DatabaseInfo;
152 
153 typedef struct RelationInfo
154 {
155  const DatabaseInfo *datinfo; /* shared by other relinfos */
157  bool is_heap; /* true if heap, false if btree */
158  char *nspname;
159  char *relname;
160  int relpages;
162  char *sql; /* set during query run, pg_free'd after */
163 } RelationInfo;
164 
165 /*
166  * Query for determining if contrib's amcheck is installed. If so, selects the
167  * namespace name where amcheck's functions can be found.
168  */
169 static const char *amcheck_sql =
170 "SELECT n.nspname, x.extversion FROM pg_catalog.pg_extension x"
171 "\nJOIN pg_catalog.pg_namespace n ON x.extnamespace = n.oid"
172 "\nWHERE x.extname = 'amcheck'";
173 
174 static void prepare_heap_command(PQExpBuffer sql, RelationInfo *rel,
175  PGconn *conn);
176 static void prepare_btree_command(PQExpBuffer sql, RelationInfo *rel,
177  PGconn *conn);
178 static void run_command(ParallelSlot *slot, const char *sql);
179 static bool verify_heap_slot_handler(PGresult *res, PGconn *conn,
180  void *context);
181 static bool verify_btree_slot_handler(PGresult *res, PGconn *conn, void *context);
182 static void help(const char *progname);
183 static void progress_report(uint64 relations_total, uint64 relations_checked,
184  uint64 relpages_total, uint64 relpages_checked,
185  const char *datname, bool force, bool finished);
186 
187 static void append_database_pattern(PatternInfoArray *pia, const char *pattern,
188  int encoding);
189 static void append_schema_pattern(PatternInfoArray *pia, const char *pattern,
190  int encoding);
191 static void append_relation_pattern(PatternInfoArray *pia, const char *pattern,
192  int encoding);
193 static void append_heap_pattern(PatternInfoArray *pia, const char *pattern,
194  int encoding);
195 static void append_btree_pattern(PatternInfoArray *pia, const char *pattern,
196  int encoding);
197 static void compile_database_list(PGconn *conn, SimplePtrList *databases,
198  const char *initial_dbname);
199 static void compile_relation_list_one_db(PGconn *conn, SimplePtrList *relations,
200  const DatabaseInfo *datinfo,
201  uint64 *pagecount);
202 
203 #define log_no_match(...) do { \
204  if (opts.strict_names) \
205  pg_log_generic(PG_LOG_ERROR, __VA_ARGS__); \
206  else \
207  pg_log_generic(PG_LOG_WARNING, __VA_ARGS__); \
208  } while(0)
209 
210 #define FREE_AND_SET_NULL(x) do { \
211  pg_free(x); \
212  (x) = NULL; \
213  } while (0)
214 
215 int
216 main(int argc, char *argv[])
217 {
218  PGconn *conn = NULL;
219  SimplePtrListCell *cell;
220  SimplePtrList databases = {NULL, NULL};
221  SimplePtrList relations = {NULL, NULL};
222  bool failed = false;
223  const char *latest_datname;
224  int parallel_workers;
226  PQExpBufferData sql;
227  uint64 reltotal = 0;
228  uint64 pageschecked = 0;
229  uint64 pagestotal = 0;
230  uint64 relprogress = 0;
231  int pattern_id;
232 
233  static struct option long_options[] = {
234  /* Connection options */
235  {"host", required_argument, NULL, 'h'},
236  {"port", required_argument, NULL, 'p'},
237  {"username", required_argument, NULL, 'U'},
238  {"no-password", no_argument, NULL, 'w'},
239  {"password", no_argument, NULL, 'W'},
240  {"maintenance-db", required_argument, NULL, 1},
241 
242  /* check options */
243  {"all", no_argument, NULL, 'a'},
244  {"database", required_argument, NULL, 'd'},
245  {"exclude-database", required_argument, NULL, 'D'},
246  {"echo", no_argument, NULL, 'e'},
247  {"index", required_argument, NULL, 'i'},
248  {"exclude-index", required_argument, NULL, 'I'},
249  {"jobs", required_argument, NULL, 'j'},
250  {"progress", no_argument, NULL, 'P'},
251  {"relation", required_argument, NULL, 'r'},
252  {"exclude-relation", required_argument, NULL, 'R'},
253  {"schema", required_argument, NULL, 's'},
254  {"exclude-schema", required_argument, NULL, 'S'},
255  {"table", required_argument, NULL, 't'},
256  {"exclude-table", required_argument, NULL, 'T'},
257  {"verbose", no_argument, NULL, 'v'},
258  {"no-dependent-indexes", no_argument, NULL, 2},
259  {"no-dependent-toast", no_argument, NULL, 3},
260  {"exclude-toast-pointers", no_argument, NULL, 4},
261  {"on-error-stop", no_argument, NULL, 5},
262  {"skip", required_argument, NULL, 6},
263  {"startblock", required_argument, NULL, 7},
264  {"endblock", required_argument, NULL, 8},
265  {"rootdescend", no_argument, NULL, 9},
266  {"no-strict-names", no_argument, NULL, 10},
267  {"heapallindexed", no_argument, NULL, 11},
268  {"parent-check", no_argument, NULL, 12},
269  {"install-missing", optional_argument, NULL, 13},
270 
271  {NULL, 0, NULL, 0}
272  };
273 
274  int optindex;
275  int c;
276 
277  const char *db = NULL;
278  const char *maintenance_db = NULL;
279 
280  const char *host = NULL;
281  const char *port = NULL;
282  const char *username = NULL;
283  enum trivalue prompt_password = TRI_DEFAULT;
284  int encoding = pg_get_encoding_from_locale(NULL, false);
285  ConnParams cparams;
286 
287  pg_logging_init(argv[0]);
288  progname = get_progname(argv[0]);
289  set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_amcheck"));
290 
291  handle_help_version_opts(argc, argv, progname, help);
292 
293  /* process command-line options */
294  while ((c = getopt_long(argc, argv, "ad:D:eh:Hi:I:j:p:Pr:R:s:S:t:T:U:wWv",
295  long_options, &optindex)) != -1)
296  {
297  char *endptr;
298  unsigned long optval;
299 
300  switch (c)
301  {
302  case 'a':
303  opts.alldb = true;
304  break;
305  case 'd':
306  opts.dbpattern = true;
307  append_database_pattern(&opts.include, optarg, encoding);
308  break;
309  case 'D':
310  opts.dbpattern = true;
311  append_database_pattern(&opts.exclude, optarg, encoding);
312  break;
313  case 'e':
314  opts.echo = true;
315  break;
316  case 'h':
317  host = pg_strdup(optarg);
318  break;
319  case 'i':
320  opts.allrel = false;
321  append_btree_pattern(&opts.include, optarg, encoding);
322  break;
323  case 'I':
324  opts.excludeidx = true;
325  append_btree_pattern(&opts.exclude, optarg, encoding);
326  break;
327  case 'j':
328  if (!option_parse_int(optarg, "-j/--jobs", 1, INT_MAX,
329  &opts.jobs))
330  exit(1);
331  break;
332  case 'p':
333  port = pg_strdup(optarg);
334  break;
335  case 'P':
336  opts.show_progress = true;
337  break;
338  case 'r':
339  opts.allrel = false;
340  append_relation_pattern(&opts.include, optarg, encoding);
341  break;
342  case 'R':
343  opts.excludeidx = true;
344  opts.excludetbl = true;
345  append_relation_pattern(&opts.exclude, optarg, encoding);
346  break;
347  case 's':
348  opts.allrel = false;
349  append_schema_pattern(&opts.include, optarg, encoding);
350  break;
351  case 'S':
352  opts.excludensp = true;
353  append_schema_pattern(&opts.exclude, optarg, encoding);
354  break;
355  case 't':
356  opts.allrel = false;
357  append_heap_pattern(&opts.include, optarg, encoding);
358  break;
359  case 'T':
360  opts.excludetbl = true;
361  append_heap_pattern(&opts.exclude, optarg, encoding);
362  break;
363  case 'U':
364  username = pg_strdup(optarg);
365  break;
366  case 'w':
367  prompt_password = TRI_NO;
368  break;
369  case 'W':
370  prompt_password = TRI_YES;
371  break;
372  case 'v':
373  opts.verbose = true;
375  break;
376  case 1:
377  maintenance_db = pg_strdup(optarg);
378  break;
379  case 2:
380  opts.no_btree_expansion = true;
381  break;
382  case 3:
383  opts.no_toast_expansion = true;
384  break;
385  case 4:
386  opts.reconcile_toast = false;
387  break;
388  case 5:
389  opts.on_error_stop = true;
390  break;
391  case 6:
392  if (pg_strcasecmp(optarg, "all-visible") == 0)
393  opts.skip = "all-visible";
394  else if (pg_strcasecmp(optarg, "all-frozen") == 0)
395  opts.skip = "all-frozen";
396  else if (pg_strcasecmp(optarg, "none") == 0)
397  opts.skip = "none";
398  else
399  {
400  pg_log_error("invalid argument for option %s", "--skip");
401  exit(1);
402  }
403  break;
404  case 7:
405  errno = 0;
406  optval = strtoul(optarg, &endptr, 10);
407  if (endptr == optarg || *endptr != '\0' || errno != 0)
408  {
409  pg_log_error("invalid start block");
410  exit(1);
411  }
412  if (optval > MaxBlockNumber)
413  {
414  pg_log_error("start block out of bounds");
415  exit(1);
416  }
417  opts.startblock = optval;
418  break;
419  case 8:
420  errno = 0;
421  optval = strtoul(optarg, &endptr, 10);
422  if (endptr == optarg || *endptr != '\0' || errno != 0)
423  {
424  pg_log_error("invalid end block");
425  exit(1);
426  }
427  if (optval > MaxBlockNumber)
428  {
429  pg_log_error("end block out of bounds");
430  exit(1);
431  }
432  opts.endblock = optval;
433  break;
434  case 9:
435  opts.rootdescend = true;
436  opts.parent_check = true;
437  break;
438  case 10:
439  opts.strict_names = false;
440  break;
441  case 11:
442  opts.heapallindexed = true;
443  break;
444  case 12:
445  opts.parent_check = true;
446  break;
447  case 13:
448  opts.install_missing = true;
449  if (optarg)
451  break;
452  default:
453  fprintf(stderr,
454  _("Try \"%s --help\" for more information.\n"),
455  progname);
456  exit(1);
457  }
458  }
459 
460  if (opts.endblock >= 0 && opts.endblock < opts.startblock)
461  {
462  pg_log_error("end block precedes start block");
463  exit(1);
464  }
465 
466  /*
467  * A single non-option arguments specifies a database name or connection
468  * string.
469  */
470  if (optind < argc)
471  {
472  db = argv[optind];
473  optind++;
474  }
475 
476  if (optind < argc)
477  {
478  pg_log_error("too many command-line arguments (first is \"%s\")",
479  argv[optind]);
480  fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
481  exit(1);
482  }
483 
484  /* fill cparams except for dbname, which is set below */
485  cparams.pghost = host;
486  cparams.pgport = port;
487  cparams.pguser = username;
488  cparams.prompt_password = prompt_password;
489  cparams.dbname = NULL;
490  cparams.override_dbname = NULL;
491 
492  setup_cancel_handler(NULL);
493 
494  /* choose the database for our initial connection */
495  if (opts.alldb)
496  {
497  if (db != NULL)
498  {
499  pg_log_error("cannot specify a database name with --all");
500  exit(1);
501  }
502  cparams.dbname = maintenance_db;
503  }
504  else if (db != NULL)
505  {
506  if (opts.dbpattern)
507  {
508  pg_log_error("cannot specify both a database name and database patterns");
509  exit(1);
510  }
511  cparams.dbname = db;
512  }
513 
514  if (opts.alldb || opts.dbpattern)
515  {
516  conn = connectMaintenanceDatabase(&cparams, progname, opts.echo);
517  compile_database_list(conn, &databases, NULL);
518  }
519  else
520  {
521  if (cparams.dbname == NULL)
522  {
523  if (getenv("PGDATABASE"))
524  cparams.dbname = getenv("PGDATABASE");
525  else if (getenv("PGUSER"))
526  cparams.dbname = getenv("PGUSER");
527  else
529  }
530  conn = connectDatabase(&cparams, progname, opts.echo, false, true);
531  compile_database_list(conn, &databases, PQdb(conn));
532  }
533 
534  if (databases.head == NULL)
535  {
536  if (conn != NULL)
537  disconnectDatabase(conn);
538  pg_log_error("no databases to check");
539  exit(0);
540  }
541 
542  /*
543  * Compile a list of all relations spanning all databases to be checked.
544  */
545  for (cell = databases.head; cell; cell = cell->next)
546  {
547  PGresult *result;
548  int ntups;
549  const char *amcheck_schema = NULL;
550  DatabaseInfo *dat = (DatabaseInfo *) cell->ptr;
551 
552  cparams.override_dbname = dat->datname;
553  if (conn == NULL || strcmp(PQdb(conn), dat->datname) != 0)
554  {
555  if (conn != NULL)
556  disconnectDatabase(conn);
557  conn = connectDatabase(&cparams, progname, opts.echo, false, true);
558  }
559 
560  /*
561  * Optionally install amcheck if not already installed in this
562  * database.
563  */
564  if (opts.install_missing)
565  {
566  char *schema;
567  char *install_sql;
568 
569  /*
570  * Must re-escape the schema name for each database, as the
571  * escaping rules may change.
572  */
573  schema = PQescapeIdentifier(conn, opts.install_schema,
574  strlen(opts.install_schema));
575  install_sql = psprintf("CREATE EXTENSION IF NOT EXISTS amcheck WITH SCHEMA %s",
576  schema);
577 
578  executeCommand(conn, install_sql, opts.echo);
579  pfree(install_sql);
580  pfree(schema);
581  }
582 
583  /*
584  * Verify that amcheck is installed for this next database. User
585  * error could result in a database not having amcheck that should
586  * have it, but we also could be iterating over multiple databases
587  * where not all of them have amcheck installed (for example,
588  * 'template1').
589  */
590  result = executeQuery(conn, amcheck_sql, opts.echo);
591  if (PQresultStatus(result) != PGRES_TUPLES_OK)
592  {
593  /* Querying the catalog failed. */
594  pg_log_error("database \"%s\": %s",
595  PQdb(conn), PQerrorMessage(conn));
596  pg_log_info("query was: %s", amcheck_sql);
597  PQclear(result);
598  disconnectDatabase(conn);
599  exit(1);
600  }
601  ntups = PQntuples(result);
602  if (ntups == 0)
603  {
604  /* Querying the catalog succeeded, but amcheck is missing. */
605  pg_log_warning("skipping database \"%s\": amcheck is not installed",
606  PQdb(conn));
607  disconnectDatabase(conn);
608  conn = NULL;
609  continue;
610  }
611  amcheck_schema = PQgetvalue(result, 0, 0);
612  if (opts.verbose)
613  pg_log_info("in database \"%s\": using amcheck version \"%s\" in schema \"%s\"",
614  PQdb(conn), PQgetvalue(result, 0, 1), amcheck_schema);
615  dat->amcheck_schema = PQescapeIdentifier(conn, amcheck_schema,
616  strlen(amcheck_schema));
617  PQclear(result);
618 
619  compile_relation_list_one_db(conn, &relations, dat, &pagestotal);
620  }
621 
622  /*
623  * Check that all inclusion patterns matched at least one schema or
624  * relation that we can check.
625  */
626  for (pattern_id = 0; pattern_id < opts.include.len; pattern_id++)
627  {
628  PatternInfo *pat = &opts.include.data[pattern_id];
629 
630  if (!pat->matched && (pat->nsp_regex != NULL || pat->rel_regex != NULL))
631  {
632  failed = opts.strict_names;
633 
634  if (pat->heap_only)
635  log_no_match("no heap tables to check matching \"%s\"",
636  pat->pattern);
637  else if (pat->btree_only)
638  log_no_match("no btree indexes to check matching \"%s\"",
639  pat->pattern);
640  else if (pat->rel_regex == NULL)
641  log_no_match("no relations to check in schemas matching \"%s\"",
642  pat->pattern);
643  else
644  log_no_match("no relations to check matching \"%s\"",
645  pat->pattern);
646  }
647  }
648 
649  if (failed)
650  {
651  if (conn != NULL)
652  disconnectDatabase(conn);
653  exit(1);
654  }
655 
656  /*
657  * Set parallel_workers to the lesser of opts.jobs and the number of
658  * relations.
659  */
660  parallel_workers = 0;
661  for (cell = relations.head; cell; cell = cell->next)
662  {
663  reltotal++;
664  if (parallel_workers < opts.jobs)
665  parallel_workers++;
666  }
667 
668  if (reltotal == 0)
669  {
670  if (conn != NULL)
671  disconnectDatabase(conn);
672  pg_log_error("no relations to check");
673  exit(1);
674  }
675  progress_report(reltotal, relprogress, pagestotal, pageschecked,
676  NULL, true, false);
677 
678  /*
679  * Main event loop.
680  *
681  * We use server-side parallelism to check up to parallel_workers
682  * relations in parallel. The list of relations was computed in database
683  * order, which minimizes the number of connects and disconnects as we
684  * process the list.
685  */
686  latest_datname = NULL;
687  sa = ParallelSlotsSetup(parallel_workers, &cparams, progname, opts.echo,
688  NULL);
689  if (conn != NULL)
690  {
691  ParallelSlotsAdoptConn(sa, conn);
692  conn = NULL;
693  }
694 
695  initPQExpBuffer(&sql);
696  for (relprogress = 0, cell = relations.head; cell; cell = cell->next)
697  {
698  ParallelSlot *free_slot;
699  RelationInfo *rel;
700 
701  rel = (RelationInfo *) cell->ptr;
702 
703  if (CancelRequested)
704  {
705  failed = true;
706  break;
707  }
708 
709  /*
710  * The list of relations is in database sorted order. If this next
711  * relation is in a different database than the last one seen, we are
712  * about to start checking this database. Note that other slots may
713  * still be working on relations from prior databases.
714  */
715  latest_datname = rel->datinfo->datname;
716 
717  progress_report(reltotal, relprogress, pagestotal, pageschecked,
718  latest_datname, false, false);
719 
720  relprogress++;
721  pageschecked += rel->blocks_to_check;
722 
723  /*
724  * Get a parallel slot for the next amcheck command, blocking if
725  * necessary until one is available, or until a previously issued slot
726  * command fails, indicating that we should abort checking the
727  * remaining objects.
728  */
729  free_slot = ParallelSlotsGetIdle(sa, rel->datinfo->datname);
730  if (!free_slot)
731  {
732  /*
733  * Something failed. We don't need to know what it was, because
734  * the handler should already have emitted the necessary error
735  * messages.
736  */
737  failed = true;
738  break;
739  }
740 
741  if (opts.verbose)
743 
744  /*
745  * Execute the appropriate amcheck command for this relation using our
746  * slot's database connection. We do not wait for the command to
747  * complete, nor do we perform any error checking, as that is done by
748  * the parallel slots and our handler callback functions.
749  */
750  if (rel->is_heap)
751  {
752  if (opts.verbose)
753  {
755  fprintf(stderr, "\n");
756  pg_log_info("checking heap table \"%s.%s.%s\"",
757  rel->datinfo->datname, rel->nspname, rel->relname);
759  }
760  prepare_heap_command(&sql, rel, free_slot->connection);
761  rel->sql = pstrdup(sql.data); /* pg_free'd after command */
763  run_command(free_slot, rel->sql);
764  }
765  else
766  {
767  if (opts.verbose)
768  {
770  fprintf(stderr, "\n");
771 
772  pg_log_info("checking btree index \"%s.%s.%s\"",
773  rel->datinfo->datname, rel->nspname, rel->relname);
775  }
776  prepare_btree_command(&sql, rel, free_slot->connection);
777  rel->sql = pstrdup(sql.data); /* pg_free'd after command */
779  run_command(free_slot, rel->sql);
780  }
781  }
782  termPQExpBuffer(&sql);
783 
784  if (!failed)
785  {
786 
787  /*
788  * Wait for all slots to complete, or for one to indicate that an
789  * error occurred. Like above, we rely on the handler emitting the
790  * necessary error messages.
791  */
792  if (sa && !ParallelSlotsWaitCompletion(sa))
793  failed = true;
794 
795  progress_report(reltotal, relprogress, pagestotal, pageschecked, NULL, true, true);
796  }
797 
798  if (sa)
799  {
801  FREE_AND_SET_NULL(sa);
802  }
803 
804  if (failed)
805  exit(1);
806 
807  if (!all_checks_pass)
808  exit(2);
809 }
810 
811 /*
812  * prepare_heap_command
813  *
814  * Creates a SQL command for running amcheck checking on the given heap
815  * relation. The command is phrased as a SQL query, with column order and
816  * names matching the expectations of verify_heap_slot_handler, which will
817  * receive and handle each row returned from the verify_heapam() function.
818  *
819  * sql: buffer into which the heap table checking command will be written
820  * rel: relation information for the heap table to be checked
821  * conn: the connection to be used, for string escaping purposes
822  */
823 static void
825 {
826  resetPQExpBuffer(sql);
827  appendPQExpBuffer(sql,
828  "SELECT blkno, offnum, attnum, msg FROM %s.verify_heapam("
829  "\nrelation := %u, on_error_stop := %s, check_toast := %s, skip := '%s'",
830  rel->datinfo->amcheck_schema,
831  rel->reloid,
832  opts.on_error_stop ? "true" : "false",
833  opts.reconcile_toast ? "true" : "false",
834  opts.skip);
835 
836  if (opts.startblock >= 0)
837  appendPQExpBuffer(sql, ", startblock := " INT64_FORMAT, opts.startblock);
838  if (opts.endblock >= 0)
839  appendPQExpBuffer(sql, ", endblock := " INT64_FORMAT, opts.endblock);
840 
841  appendPQExpBufferChar(sql, ')');
842 }
843 
844 /*
845  * prepare_btree_command
846  *
847  * Creates a SQL command for running amcheck checking on the given btree index
848  * relation. The command does not select any columns, as btree checking
849  * functions do not return any, but rather return corruption information by
850  * raising errors, which verify_btree_slot_handler expects.
851  *
852  * sql: buffer into which the heap table checking command will be written
853  * rel: relation information for the index to be checked
854  * conn: the connection to be used, for string escaping purposes
855  */
856 static void
858 {
859  resetPQExpBuffer(sql);
860 
861  /*
862  * Embed the database, schema, and relation name in the query, so if the
863  * check throws an error, the user knows which relation the error came
864  * from.
865  */
866  if (opts.parent_check)
867  appendPQExpBuffer(sql,
868  "SELECT * FROM %s.bt_index_parent_check("
869  "index := '%u'::regclass, heapallindexed := %s, "
870  "rootdescend := %s)",
871  rel->datinfo->amcheck_schema,
872  rel->reloid,
873  (opts.heapallindexed ? "true" : "false"),
874  (opts.rootdescend ? "true" : "false"));
875  else
876  appendPQExpBuffer(sql,
877  "SELECT * FROM %s.bt_index_check("
878  "index := '%u'::regclass, heapallindexed := %s)",
879  rel->datinfo->amcheck_schema,
880  rel->reloid,
881  (opts.heapallindexed ? "true" : "false"));
882 }
883 
884 /*
885  * run_command
886  *
887  * Sends a command to the server without waiting for the command to complete.
888  * Logs an error if the command cannot be sent, but otherwise any errors are
889  * expected to be handled by a ParallelSlotHandler.
890  *
891  * If reconnecting to the database is necessary, the cparams argument may be
892  * modified.
893  *
894  * slot: slot with connection to the server we should use for the command
895  * sql: query to send
896  */
897 static void
898 run_command(ParallelSlot *slot, const char *sql)
899 {
900  if (opts.echo)
901  printf("%s\n", sql);
902 
903  if (PQsendQuery(slot->connection, sql) == 0)
904  {
905  pg_log_error("error sending command to database \"%s\": %s",
906  PQdb(slot->connection),
907  PQerrorMessage(slot->connection));
908  pg_log_error("command was: %s", sql);
909  exit(1);
910  }
911 }
912 
913 /*
914  * should_processing_continue
915  *
916  * Checks a query result returned from a query (presumably issued on a slot's
917  * connection) to determine if parallel slots should continue issuing further
918  * commands.
919  *
920  * Note: Heap relation corruption is reported by verify_heapam() via the result
921  * set, rather than an ERROR, but running verify_heapam() on a corrupted heap
922  * table may still result in an error being returned from the server due to
923  * missing relation files, bad checksums, etc. The btree corruption checking
924  * functions always use errors to communicate corruption messages. We can't
925  * just abort processing because we got a mere ERROR.
926  *
927  * res: result from an executed sql query
928  */
929 static bool
931 {
932  const char *severity;
933 
934  switch (PQresultStatus(res))
935  {
936  /* These are expected and ok */
937  case PGRES_COMMAND_OK:
938  case PGRES_TUPLES_OK:
940  break;
941 
942  /* This is expected but requires closer scrutiny */
943  case PGRES_FATAL_ERROR:
945  if (strcmp(severity, "FATAL") == 0)
946  return false;
947  if (strcmp(severity, "PANIC") == 0)
948  return false;
949  break;
950 
951  /* These are unexpected */
952  case PGRES_BAD_RESPONSE:
953  case PGRES_EMPTY_QUERY:
954  case PGRES_COPY_OUT:
955  case PGRES_COPY_IN:
956  case PGRES_COPY_BOTH:
957  case PGRES_SINGLE_TUPLE:
958  case PGRES_PIPELINE_SYNC:
960  return false;
961  }
962  return true;
963 }
964 
965 /*
966  * Returns a copy of the argument string with all lines indented four spaces.
967  *
968  * The caller should pg_free the result when finished with it.
969  */
970 static char *
971 indent_lines(const char *str)
972 {
974  const char *c;
975  char *result;
976 
977  initPQExpBuffer(&buf);
978  appendPQExpBufferStr(&buf, " ");
979  for (c = str; *c; c++)
980  {
981  appendPQExpBufferChar(&buf, *c);
982  if (c[0] == '\n' && c[1] != '\0')
983  appendPQExpBufferStr(&buf, " ");
984  }
985  result = pstrdup(buf.data);
986  termPQExpBuffer(&buf);
987 
988  return result;
989 }
990 
991 /*
992  * verify_heap_slot_handler
993  *
994  * ParallelSlotHandler that receives results from a heap table checking command
995  * created by prepare_heap_command and outputs the results for the user.
996  *
997  * res: result from an executed sql query
998  * conn: connection on which the sql query was executed
999  * context: the sql query being handled, as a cstring
1000  */
1001 static bool
1003 {
1004  RelationInfo *rel = (RelationInfo *) context;
1005 
1006  if (PQresultStatus(res) == PGRES_TUPLES_OK)
1007  {
1008  int i;
1009  int ntups = PQntuples(res);
1010 
1011  if (ntups > 0)
1012  all_checks_pass = false;
1013 
1014  for (i = 0; i < ntups; i++)
1015  {
1016  const char *msg;
1017 
1018  /* The message string should never be null, but check */
1019  if (PQgetisnull(res, i, 3))
1020  msg = "NO MESSAGE";
1021  else
1022  msg = PQgetvalue(res, i, 3);
1023 
1024  if (!PQgetisnull(res, i, 2))
1025  printf(_("heap table \"%s.%s.%s\", block %s, offset %s, attribute %s:\n"),
1026  rel->datinfo->datname, rel->nspname, rel->relname,
1027  PQgetvalue(res, i, 0), /* blkno */
1028  PQgetvalue(res, i, 1), /* offnum */
1029  PQgetvalue(res, i, 2)); /* attnum */
1030 
1031  else if (!PQgetisnull(res, i, 1))
1032  printf(_("heap table \"%s.%s.%s\", block %s, offset %s:\n"),
1033  rel->datinfo->datname, rel->nspname, rel->relname,
1034  PQgetvalue(res, i, 0), /* blkno */
1035  PQgetvalue(res, i, 1)); /* offnum */
1036 
1037  else if (!PQgetisnull(res, i, 0))
1038  printf(_("heap table \"%s.%s.%s\", block %s:\n"),
1039  rel->datinfo->datname, rel->nspname, rel->relname,
1040  PQgetvalue(res, i, 0)); /* blkno */
1041 
1042  else
1043  printf(_("heap table \"%s.%s.%s\":\n"),
1044  rel->datinfo->datname, rel->nspname, rel->relname);
1045 
1046  printf(" %s\n", msg);
1047  }
1048  }
1049  else if (PQresultStatus(res) != PGRES_TUPLES_OK)
1050  {
1051  char *msg = indent_lines(PQerrorMessage(conn));
1052 
1053  all_checks_pass = false;
1054  printf(_("heap table \"%s.%s.%s\":\n"),
1055  rel->datinfo->datname, rel->nspname, rel->relname);
1056  printf("%s", msg);
1057  if (opts.verbose)
1058  printf(_("query was: %s\n"), rel->sql);
1059  FREE_AND_SET_NULL(msg);
1060  }
1061 
1062  FREE_AND_SET_NULL(rel->sql);
1063  FREE_AND_SET_NULL(rel->nspname);
1064  FREE_AND_SET_NULL(rel->relname);
1065 
1066  return should_processing_continue(res);
1067 }
1068 
1069 /*
1070  * verify_btree_slot_handler
1071  *
1072  * ParallelSlotHandler that receives results from a btree checking command
1073  * created by prepare_btree_command and outputs them for the user. The results
1074  * from the btree checking command is assumed to be empty, but when the results
1075  * are an error code, the useful information about the corruption is expected
1076  * in the connection's error message.
1077  *
1078  * res: result from an executed sql query
1079  * conn: connection on which the sql query was executed
1080  * context: unused
1081  */
1082 static bool
1084 {
1085  RelationInfo *rel = (RelationInfo *) context;
1086 
1087  if (PQresultStatus(res) == PGRES_TUPLES_OK)
1088  {
1089  int ntups = PQntuples(res);
1090 
1091  if (ntups != 1)
1092  {
1093  /*
1094  * We expect the btree checking functions to return one void row
1095  * each, so we should output some sort of warning if we get
1096  * anything else, not because it indicates corruption, but because
1097  * it suggests a mismatch between amcheck and pg_amcheck versions.
1098  *
1099  * In conjunction with --progress, anything written to stderr at
1100  * this time would present strangely to the user without an extra
1101  * newline, so we print one. If we were multithreaded, we'd have
1102  * to avoid splitting this across multiple calls, but we're in an
1103  * event loop, so it doesn't matter.
1104  */
1106  fprintf(stderr, "\n");
1107  pg_log_warning("btree index \"%s.%s.%s\": btree checking function returned unexpected number of rows: %d",
1108  rel->datinfo->datname, rel->nspname, rel->relname, ntups);
1109  if (opts.verbose)
1110  pg_log_info("query was: %s", rel->sql);
1111  pg_log_warning("Are %s's and amcheck's versions compatible?",
1112  progname);
1114  }
1115  }
1116  else
1117  {
1118  char *msg = indent_lines(PQerrorMessage(conn));
1119 
1120  all_checks_pass = false;
1121  printf(_("btree index \"%s.%s.%s\":\n"),
1122  rel->datinfo->datname, rel->nspname, rel->relname);
1123  printf("%s", msg);
1124  if (opts.verbose)
1125  printf(_("query was: %s\n"), rel->sql);
1126  FREE_AND_SET_NULL(msg);
1127  }
1128 
1129  FREE_AND_SET_NULL(rel->sql);
1130  FREE_AND_SET_NULL(rel->nspname);
1131  FREE_AND_SET_NULL(rel->relname);
1132 
1133  return should_processing_continue(res);
1134 }
1135 
1136 /*
1137  * help
1138  *
1139  * Prints help page for the program
1140  *
1141  * progname: the name of the executed program, such as "pg_amcheck"
1142  */
1143 static void
1144 help(const char *progname)
1145 {
1146  printf(_("%s checks objects in a PostgreSQL database for corruption.\n\n"), progname);
1147  printf(_("Usage:\n"));
1148  printf(_(" %s [OPTION]... [DBNAME]\n"), progname);
1149  printf(_("\nTarget options:\n"));
1150  printf(_(" -a, --all check all databases\n"));
1151  printf(_(" -d, --database=PATTERN check matching database(s)\n"));
1152  printf(_(" -D, --exclude-database=PATTERN do NOT check matching database(s)\n"));
1153  printf(_(" -i, --index=PATTERN check matching index(es)\n"));
1154  printf(_(" -I, --exclude-index=PATTERN do NOT check matching index(es)\n"));
1155  printf(_(" -r, --relation=PATTERN check matching relation(s)\n"));
1156  printf(_(" -R, --exclude-relation=PATTERN do NOT check matching relation(s)\n"));
1157  printf(_(" -s, --schema=PATTERN check matching schema(s)\n"));
1158  printf(_(" -S, --exclude-schema=PATTERN do NOT check matching schema(s)\n"));
1159  printf(_(" -t, --table=PATTERN check matching table(s)\n"));
1160  printf(_(" -T, --exclude-table=PATTERN do NOT check matching table(s)\n"));
1161  printf(_(" --no-dependent-indexes do NOT expand list of relations to include indexes\n"));
1162  printf(_(" --no-dependent-toast do NOT expand list of relations to include TOAST tables\n"));
1163  printf(_(" --no-strict-names do NOT require patterns to match objects\n"));
1164  printf(_("\nTable checking options:\n"));
1165  printf(_(" --exclude-toast-pointers do NOT follow relation TOAST pointers\n"));
1166  printf(_(" --on-error-stop stop checking at end of first corrupt page\n"));
1167  printf(_(" --skip=OPTION do NOT check \"all-frozen\" or \"all-visible\" blocks\n"));
1168  printf(_(" --startblock=BLOCK begin checking table(s) at the given block number\n"));
1169  printf(_(" --endblock=BLOCK check table(s) only up to the given block number\n"));
1170  printf(_("\nB-tree index checking options:\n"));
1171  printf(_(" --heapallindexed check that all heap tuples are found within indexes\n"));
1172  printf(_(" --parent-check check index parent/child relationships\n"));
1173  printf(_(" --rootdescend search from root page to refind tuples\n"));
1174  printf(_("\nConnection options:\n"));
1175  printf(_(" -h, --host=HOSTNAME database server host or socket directory\n"));
1176  printf(_(" -p, --port=PORT database server port\n"));
1177  printf(_(" -U, --username=USERNAME user name to connect as\n"));
1178  printf(_(" -w, --no-password never prompt for password\n"));
1179  printf(_(" -W, --password force password prompt\n"));
1180  printf(_(" --maintenance-db=DBNAME alternate maintenance database\n"));
1181  printf(_("\nOther options:\n"));
1182  printf(_(" -e, --echo show the commands being sent to the server\n"));
1183  printf(_(" -j, --jobs=NUM use this many concurrent connections to the server\n"));
1184  printf(_(" -P, --progress show progress information\n"));
1185  printf(_(" -v, --verbose write a lot of output\n"));
1186  printf(_(" -V, --version output version information, then exit\n"));
1187  printf(_(" --install-missing install missing extensions\n"));
1188  printf(_(" -?, --help show this help, then exit\n"));
1189 
1190  printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
1191  printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
1192 }
1193 
1194 /*
1195  * Print a progress report based on the global variables.
1196  *
1197  * Progress report is written at maximum once per second, unless the force
1198  * parameter is set to true.
1199  *
1200  * If finished is set to true, this is the last progress report. The cursor
1201  * is moved to the next line.
1202  */
1203 static void
1204 progress_report(uint64 relations_total, uint64 relations_checked,
1205  uint64 relpages_total, uint64 relpages_checked,
1206  const char *datname, bool force, bool finished)
1207 {
1208  int percent_rel = 0;
1209  int percent_pages = 0;
1210  char checked_rel[32];
1211  char total_rel[32];
1212  char checked_pages[32];
1213  char total_pages[32];
1214  pg_time_t now;
1215 
1216  if (!opts.show_progress)
1217  return;
1218 
1219  now = time(NULL);
1220  if (now == last_progress_report && !force && !finished)
1221  return; /* Max once per second */
1222 
1224  if (relations_total)
1225  percent_rel = (int) (relations_checked * 100 / relations_total);
1226  if (relpages_total)
1227  percent_pages = (int) (relpages_checked * 100 / relpages_total);
1228 
1229  snprintf(checked_rel, sizeof(checked_rel), UINT64_FORMAT, relations_checked);
1230  snprintf(total_rel, sizeof(total_rel), UINT64_FORMAT, relations_total);
1231  snprintf(checked_pages, sizeof(checked_pages), UINT64_FORMAT, relpages_checked);
1232  snprintf(total_pages, sizeof(total_pages), UINT64_FORMAT, relpages_total);
1233 
1234 #define VERBOSE_DATNAME_LENGTH 35
1235  if (opts.verbose)
1236  {
1237  if (!datname)
1238 
1239  /*
1240  * No datname given, so clear the status line (used for first and
1241  * last call)
1242  */
1243  fprintf(stderr,
1244  _("%*s/%s relations (%d%%), %*s/%s pages (%d%%) %*s"),
1245  (int) strlen(total_rel),
1246  checked_rel, total_rel, percent_rel,
1247  (int) strlen(total_pages),
1248  checked_pages, total_pages, percent_pages,
1249  VERBOSE_DATNAME_LENGTH + 2, "");
1250  else
1251  {
1252  bool truncate = (strlen(datname) > VERBOSE_DATNAME_LENGTH);
1253 
1254  fprintf(stderr,
1255  _("%*s/%s relations (%d%%), %*s/%s pages (%d%%) (%s%-*.*s)"),
1256  (int) strlen(total_rel),
1257  checked_rel, total_rel, percent_rel,
1258  (int) strlen(total_pages),
1259  checked_pages, total_pages, percent_pages,
1260  /* Prefix with "..." if we do leading truncation */
1261  truncate ? "..." : "",
1264  /* Truncate datname at beginning if it's too long */
1265  truncate ? datname + strlen(datname) - VERBOSE_DATNAME_LENGTH + 3 : datname);
1266  }
1267  }
1268  else
1269  fprintf(stderr,
1270  _("%*s/%s relations (%d%%), %*s/%s pages (%d%%)"),
1271  (int) strlen(total_rel),
1272  checked_rel, total_rel, percent_rel,
1273  (int) strlen(total_pages),
1274  checked_pages, total_pages, percent_pages);
1275 
1276  /*
1277  * Stay on the same line if reporting to a terminal and we're not done
1278  * yet.
1279  */
1280  if (!finished && isatty(fileno(stderr)))
1281  {
1282  fputc('\r', stderr);
1284  }
1285  else
1286  fputc('\n', stderr);
1287 }
1288 
1289 /*
1290  * Extend the pattern info array to hold one additional initialized pattern
1291  * info entry.
1292  *
1293  * Returns a pointer to the new entry.
1294  */
1295 static PatternInfo *
1297 {
1298  PatternInfo *result;
1299 
1300  pia->len++;
1301  pia->data = (PatternInfo *) pg_realloc(pia->data, pia->len * sizeof(PatternInfo));
1302  result = &pia->data[pia->len - 1];
1303  memset(result, 0, sizeof(*result));
1304 
1305  return result;
1306 }
1307 
1308 /*
1309  * append_database_pattern
1310  *
1311  * Adds the given pattern interpreted as a database name pattern.
1312  *
1313  * pia: the pattern info array to be appended
1314  * pattern: the database name pattern
1315  * encoding: client encoding for parsing the pattern
1316  */
1317 static void
1319 {
1322 
1323  initPQExpBuffer(&buf);
1324  patternToSQLRegex(encoding, NULL, NULL, &buf, pattern, false);
1325  info->pattern = pattern;
1326  info->db_regex = pstrdup(buf.data);
1327 
1328  termPQExpBuffer(&buf);
1329 }
1330 
1331 /*
1332  * append_schema_pattern
1333  *
1334  * Adds the given pattern interpreted as a schema name pattern.
1335  *
1336  * pia: the pattern info array to be appended
1337  * pattern: the schema name pattern
1338  * encoding: client encoding for parsing the pattern
1339  */
1340 static void
1342 {
1343  PQExpBufferData dbbuf;
1344  PQExpBufferData nspbuf;
1346 
1347  initPQExpBuffer(&dbbuf);
1348  initPQExpBuffer(&nspbuf);
1349 
1350  patternToSQLRegex(encoding, NULL, &dbbuf, &nspbuf, pattern, false);
1351  info->pattern = pattern;
1352  if (dbbuf.data[0])
1353  {
1354  opts.dbpattern = true;
1355  info->db_regex = pstrdup(dbbuf.data);
1356  }
1357  if (nspbuf.data[0])
1358  info->nsp_regex = pstrdup(nspbuf.data);
1359 
1360  termPQExpBuffer(&dbbuf);
1361  termPQExpBuffer(&nspbuf);
1362 }
1363 
1364 /*
1365  * append_relation_pattern_helper
1366  *
1367  * Adds to a list the given pattern interpreted as a relation pattern.
1368  *
1369  * pia: the pattern info array to be appended
1370  * pattern: the relation name pattern
1371  * encoding: client encoding for parsing the pattern
1372  * heap_only: whether the pattern should only be matched against heap tables
1373  * btree_only: whether the pattern should only be matched against btree indexes
1374  */
1375 static void
1377  int encoding, bool heap_only, bool btree_only)
1378 {
1379  PQExpBufferData dbbuf;
1380  PQExpBufferData nspbuf;
1381  PQExpBufferData relbuf;
1383 
1384  initPQExpBuffer(&dbbuf);
1385  initPQExpBuffer(&nspbuf);
1386  initPQExpBuffer(&relbuf);
1387 
1388  patternToSQLRegex(encoding, &dbbuf, &nspbuf, &relbuf, pattern, false);
1389  info->pattern = pattern;
1390  if (dbbuf.data[0])
1391  {
1392  opts.dbpattern = true;
1393  info->db_regex = pstrdup(dbbuf.data);
1394  }
1395  if (nspbuf.data[0])
1396  info->nsp_regex = pstrdup(nspbuf.data);
1397  if (relbuf.data[0])
1398  info->rel_regex = pstrdup(relbuf.data);
1399 
1400  termPQExpBuffer(&dbbuf);
1401  termPQExpBuffer(&nspbuf);
1402  termPQExpBuffer(&relbuf);
1403 
1404  info->heap_only = heap_only;
1405  info->btree_only = btree_only;
1406 }
1407 
1408 /*
1409  * append_relation_pattern
1410  *
1411  * Adds the given pattern interpreted as a relation pattern, to be matched
1412  * against both heap tables and btree indexes.
1413  *
1414  * pia: the pattern info array to be appended
1415  * pattern: the relation name pattern
1416  * encoding: client encoding for parsing the pattern
1417  */
1418 static void
1420 {
1421  append_relation_pattern_helper(pia, pattern, encoding, false, false);
1422 }
1423 
1424 /*
1425  * append_heap_pattern
1426  *
1427  * Adds the given pattern interpreted as a relation pattern, to be matched only
1428  * against heap tables.
1429  *
1430  * pia: the pattern info array to be appended
1431  * pattern: the relation name pattern
1432  * encoding: client encoding for parsing the pattern
1433  */
1434 static void
1436 {
1437  append_relation_pattern_helper(pia, pattern, encoding, true, false);
1438 }
1439 
1440 /*
1441  * append_btree_pattern
1442  *
1443  * Adds the given pattern interpreted as a relation pattern, to be matched only
1444  * against btree indexes.
1445  *
1446  * pia: the pattern info array to be appended
1447  * pattern: the relation name pattern
1448  * encoding: client encoding for parsing the pattern
1449  */
1450 static void
1452 {
1453  append_relation_pattern_helper(pia, pattern, encoding, false, true);
1454 }
1455 
1456 /*
1457  * append_db_pattern_cte
1458  *
1459  * Appends to the buffer the body of a Common Table Expression (CTE) containing
1460  * the database portions filtered from the list of patterns expressed as two
1461  * columns:
1462  *
1463  * pattern_id: the index of this pattern in pia->data[]
1464  * rgx: the database regular expression parsed from the pattern
1465  *
1466  * Patterns without a database portion are skipped. Patterns with more than
1467  * just a database portion are optionally skipped, depending on argument
1468  * 'inclusive'.
1469  *
1470  * buf: the buffer to be appended
1471  * pia: the array of patterns to be inserted into the CTE
1472  * conn: the database connection
1473  * inclusive: whether to include patterns with schema and/or relation parts
1474  *
1475  * Returns whether any database patterns were appended.
1476  */
1477 static bool
1479  PGconn *conn, bool inclusive)
1480 {
1481  int pattern_id;
1482  const char *comma;
1483  bool have_values;
1484 
1485  comma = "";
1486  have_values = false;
1487  for (pattern_id = 0; pattern_id < pia->len; pattern_id++)
1488  {
1489  PatternInfo *info = &pia->data[pattern_id];
1490 
1491  if (info->db_regex != NULL &&
1492  (inclusive || (info->nsp_regex == NULL && info->rel_regex == NULL)))
1493  {
1494  if (!have_values)
1495  appendPQExpBufferStr(buf, "\nVALUES");
1496  have_values = true;
1497  appendPQExpBuffer(buf, "%s\n(%d, ", comma, pattern_id);
1498  appendStringLiteralConn(buf, info->db_regex, conn);
1499  appendPQExpBufferStr(buf, ")");
1500  comma = ",";
1501  }
1502  }
1503 
1504  if (!have_values)
1505  appendPQExpBufferStr(buf, "\nSELECT NULL, NULL, NULL WHERE false");
1506 
1507  return have_values;
1508 }
1509 
1510 /*
1511  * compile_database_list
1512  *
1513  * If any database patterns exist, or if --all was given, compiles a distinct
1514  * list of databases to check using a SQL query based on the patterns plus the
1515  * literal initial database name, if given. If no database patterns exist and
1516  * --all was not given, the query is not necessary, and only the initial
1517  * database name (if any) is added to the list.
1518  *
1519  * conn: connection to the initial database
1520  * databases: the list onto which databases should be appended
1521  * initial_dbname: an optional extra database name to include in the list
1522  */
1523 static void
1525  const char *initial_dbname)
1526 {
1527  PGresult *res;
1528  PQExpBufferData sql;
1529  int ntups;
1530  int i;
1531  bool fatal;
1532 
1533  if (initial_dbname)
1534  {
1535  DatabaseInfo *dat = (DatabaseInfo *) pg_malloc0(sizeof(DatabaseInfo));
1536 
1537  /* This database is included. Add to list */
1538  if (opts.verbose)
1539  pg_log_info("including database \"%s\"", initial_dbname);
1540 
1541  dat->datname = pstrdup(initial_dbname);
1542  simple_ptr_list_append(databases, dat);
1543  }
1544 
1545  initPQExpBuffer(&sql);
1546 
1547  /* Append the include patterns CTE. */
1548  appendPQExpBufferStr(&sql, "WITH include_raw (pattern_id, rgx) AS (");
1549  if (!append_db_pattern_cte(&sql, &opts.include, conn, true) &&
1550  !opts.alldb)
1551  {
1552  /*
1553  * None of the inclusion patterns (if any) contain database portions,
1554  * so there is no need to query the database to resolve database
1555  * patterns.
1556  *
1557  * Since we're also not operating under --all, we don't need to query
1558  * the exhaustive list of connectable databases, either.
1559  */
1560  termPQExpBuffer(&sql);
1561  return;
1562  }
1563 
1564  /* Append the exclude patterns CTE. */
1565  appendPQExpBufferStr(&sql, "),\nexclude_raw (pattern_id, rgx) AS (");
1566  append_db_pattern_cte(&sql, &opts.exclude, conn, false);
1567  appendPQExpBufferStr(&sql, "),");
1568 
1569  /*
1570  * Append the database CTE, which includes whether each database is
1571  * connectable and also joins against exclude_raw to determine whether
1572  * each database is excluded.
1573  */
1574  appendPQExpBufferStr(&sql,
1575  "\ndatabase (datname) AS ("
1576  "\nSELECT d.datname "
1577  "FROM pg_catalog.pg_database d "
1578  "LEFT OUTER JOIN exclude_raw e "
1579  "ON d.datname ~ e.rgx "
1580  "\nWHERE d.datallowconn "
1581  "AND e.pattern_id IS NULL"
1582  "),"
1583 
1584  /*
1585  * Append the include_pat CTE, which joins the include_raw CTE against the
1586  * databases CTE to determine if all the inclusion patterns had matches,
1587  * and whether each matched pattern had the misfortune of only matching
1588  * excluded or unconnectable databases.
1589  */
1590  "\ninclude_pat (pattern_id, checkable) AS ("
1591  "\nSELECT i.pattern_id, "
1592  "COUNT(*) FILTER ("
1593  "WHERE d IS NOT NULL"
1594  ") AS checkable"
1595  "\nFROM include_raw i "
1596  "LEFT OUTER JOIN database d "
1597  "ON d.datname ~ i.rgx"
1598  "\nGROUP BY i.pattern_id"
1599  "),"
1600 
1601  /*
1602  * Append the filtered_databases CTE, which selects from the database CTE
1603  * optionally joined against the include_raw CTE to only select databases
1604  * that match an inclusion pattern. This appears to duplicate what the
1605  * include_pat CTE already did above, but here we want only databases, and
1606  * there we wanted patterns.
1607  */
1608  "\nfiltered_databases (datname) AS ("
1609  "\nSELECT DISTINCT d.datname "
1610  "FROM database d");
1611  if (!opts.alldb)
1612  appendPQExpBufferStr(&sql,
1613  " INNER JOIN include_raw i "
1614  "ON d.datname ~ i.rgx");
1615  appendPQExpBufferStr(&sql,
1616  ")"
1617 
1618  /*
1619  * Select the checkable databases and the unmatched inclusion patterns.
1620  */
1621  "\nSELECT pattern_id, datname FROM ("
1622  "\nSELECT pattern_id, NULL::TEXT AS datname "
1623  "FROM include_pat "
1624  "WHERE checkable = 0 "
1625  "UNION ALL"
1626  "\nSELECT NULL, datname "
1627  "FROM filtered_databases"
1628  ") AS combined_records"
1629  "\nORDER BY pattern_id NULLS LAST, datname");
1630 
1631  res = executeQuery(conn, sql.data, opts.echo);
1632  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1633  {
1634  pg_log_error("query failed: %s", PQerrorMessage(conn));
1635  pg_log_info("query was: %s", sql.data);
1636  disconnectDatabase(conn);
1637  exit(1);
1638  }
1639  termPQExpBuffer(&sql);
1640 
1641  ntups = PQntuples(res);
1642  for (fatal = false, i = 0; i < ntups; i++)
1643  {
1644  int pattern_id = -1;
1645  const char *datname = NULL;
1646 
1647  if (!PQgetisnull(res, i, 0))
1648  pattern_id = atoi(PQgetvalue(res, i, 0));
1649  if (!PQgetisnull(res, i, 1))
1650  datname = PQgetvalue(res, i, 1);
1651 
1652  if (pattern_id >= 0)
1653  {
1654  /*
1655  * Current record pertains to an inclusion pattern that matched no
1656  * checkable databases.
1657  */
1658  fatal = opts.strict_names;
1659  if (pattern_id >= opts.include.len)
1660  {
1661  pg_log_error("internal error: received unexpected database pattern_id %d",
1662  pattern_id);
1663  exit(1);
1664  }
1665  log_no_match("no connectable databases to check matching \"%s\"",
1666  opts.include.data[pattern_id].pattern);
1667  }
1668  else
1669  {
1670  DatabaseInfo *dat;
1671 
1672  /* Current record pertains to a database */
1673  Assert(datname != NULL);
1674 
1675  /* Avoid entering a duplicate entry matching the initial_dbname */
1676  if (initial_dbname != NULL && strcmp(initial_dbname, datname) == 0)
1677  continue;
1678 
1679  /* This database is included. Add to list */
1680  if (opts.verbose)
1681  pg_log_info("including database \"%s\"", datname);
1682 
1683  dat = (DatabaseInfo *) pg_malloc0(sizeof(DatabaseInfo));
1684  dat->datname = pstrdup(datname);
1685  simple_ptr_list_append(databases, dat);
1686  }
1687  }
1688  PQclear(res);
1689 
1690  if (fatal)
1691  {
1692  if (conn != NULL)
1693  disconnectDatabase(conn);
1694  exit(1);
1695  }
1696 }
1697 
1698 /*
1699  * append_rel_pattern_raw_cte
1700  *
1701  * Appends to the buffer the body of a Common Table Expression (CTE) containing
1702  * the given patterns as six columns:
1703  *
1704  * pattern_id: the index of this pattern in pia->data[]
1705  * db_regex: the database regexp parsed from the pattern, or NULL if the
1706  * pattern had no database part
1707  * nsp_regex: the namespace regexp parsed from the pattern, or NULL if the
1708  * pattern had no namespace part
1709  * rel_regex: the relname regexp parsed from the pattern, or NULL if the
1710  * pattern had no relname part
1711  * heap_only: true if the pattern applies only to heap tables (not indexes)
1712  * btree_only: true if the pattern applies only to btree indexes (not tables)
1713  *
1714  * buf: the buffer to be appended
1715  * patterns: the array of patterns to be inserted into the CTE
1716  * conn: the database connection
1717  */
1718 static void
1720  PGconn *conn)
1721 {
1722  int pattern_id;
1723  const char *comma;
1724  bool have_values;
1725 
1726  comma = "";
1727  have_values = false;
1728  for (pattern_id = 0; pattern_id < pia->len; pattern_id++)
1729  {
1730  PatternInfo *info = &pia->data[pattern_id];
1731 
1732  if (!have_values)
1733  appendPQExpBufferStr(buf, "\nVALUES");
1734  have_values = true;
1735  appendPQExpBuffer(buf, "%s\n(%d::INTEGER, ", comma, pattern_id);
1736  if (info->db_regex == NULL)
1737  appendPQExpBufferStr(buf, "NULL");
1738  else
1739  appendStringLiteralConn(buf, info->db_regex, conn);
1740  appendPQExpBufferStr(buf, "::TEXT, ");
1741  if (info->nsp_regex == NULL)
1742  appendPQExpBufferStr(buf, "NULL");
1743  else
1744  appendStringLiteralConn(buf, info->nsp_regex, conn);
1745  appendPQExpBufferStr(buf, "::TEXT, ");
1746  if (info->rel_regex == NULL)
1747  appendPQExpBufferStr(buf, "NULL");
1748  else
1749  appendStringLiteralConn(buf, info->rel_regex, conn);
1750  if (info->heap_only)
1751  appendPQExpBufferStr(buf, "::TEXT, true::BOOLEAN");
1752  else
1753  appendPQExpBufferStr(buf, "::TEXT, false::BOOLEAN");
1754  if (info->btree_only)
1755  appendPQExpBufferStr(buf, ", true::BOOLEAN");
1756  else
1757  appendPQExpBufferStr(buf, ", false::BOOLEAN");
1758  appendPQExpBufferStr(buf, ")");
1759  comma = ",";
1760  }
1761 
1762  if (!have_values)
1764  "\nSELECT NULL::INTEGER, NULL::TEXT, NULL::TEXT, "
1765  "NULL::TEXT, NULL::BOOLEAN, NULL::BOOLEAN "
1766  "WHERE false");
1767 }
1768 
1769 /*
1770  * append_rel_pattern_filtered_cte
1771  *
1772  * Appends to the buffer a Common Table Expression (CTE) which selects
1773  * all patterns from the named raw CTE, filtered by database. All patterns
1774  * which have no database portion or whose database portion matches our
1775  * connection's database name are selected, with other patterns excluded.
1776  *
1777  * The basic idea here is that if we're connected to database "foo" and we have
1778  * patterns "foo.bar.baz", "alpha.beta" and "one.two.three", we only want to
1779  * use the first two while processing relations in this database, as the third
1780  * one is not relevant.
1781  *
1782  * buf: the buffer to be appended
1783  * raw: the name of the CTE to select from
1784  * filtered: the name of the CTE to create
1785  * conn: the database connection
1786  */
1787 static void
1789  const char *filtered, PGconn *conn)
1790 {
1791  appendPQExpBuffer(buf,
1792  "\n%s (pattern_id, nsp_regex, rel_regex, heap_only, btree_only) AS ("
1793  "\nSELECT pattern_id, nsp_regex, rel_regex, heap_only, btree_only "
1794  "FROM %s r"
1795  "\nWHERE (r.db_regex IS NULL "
1796  "OR ",
1797  filtered, raw);
1798  appendStringLiteralConn(buf, PQdb(conn), conn);
1799  appendPQExpBufferStr(buf, " ~ r.db_regex)");
1801  " AND (r.nsp_regex IS NOT NULL"
1802  " OR r.rel_regex IS NOT NULL)"
1803  "),");
1804 }
1805 
1806 /*
1807  * compile_relation_list_one_db
1808  *
1809  * Compiles a list of relations to check within the currently connected
1810  * database based on the user supplied options, sorted by descending size,
1811  * and appends them to the given list of relations.
1812  *
1813  * The cells of the constructed list contain all information about the relation
1814  * necessary to connect to the database and check the object, including which
1815  * database to connect to, where contrib/amcheck is installed, and the Oid and
1816  * type of object (heap table vs. btree index). Rather than duplicating the
1817  * database details per relation, the relation structs use references to the
1818  * same database object, provided by the caller.
1819  *
1820  * conn: connection to this next database, which should be the same as in 'dat'
1821  * relations: list onto which the relations information should be appended
1822  * dat: the database info struct for use by each relation
1823  * pagecount: gets incremented by the number of blocks to check in all
1824  * relations added
1825  */
1826 static void
1828  const DatabaseInfo *dat,
1829  uint64 *pagecount)
1830 {
1831  PGresult *res;
1832  PQExpBufferData sql;
1833  int ntups;
1834  int i;
1835 
1836  initPQExpBuffer(&sql);
1837  appendPQExpBufferStr(&sql, "WITH");
1838 
1839  /* Append CTEs for the relation inclusion patterns, if any */
1840  if (!opts.allrel)
1841  {
1842  appendPQExpBufferStr(&sql,
1843  " include_raw (pattern_id, db_regex, nsp_regex, rel_regex, heap_only, btree_only) AS (");
1844  append_rel_pattern_raw_cte(&sql, &opts.include, conn);
1845  appendPQExpBufferStr(&sql, "\n),");
1846  append_rel_pattern_filtered_cte(&sql, "include_raw", "include_pat", conn);
1847  }
1848 
1849  /* Append CTEs for the relation exclusion patterns, if any */
1850  if (opts.excludetbl || opts.excludeidx || opts.excludensp)
1851  {
1852  appendPQExpBufferStr(&sql,
1853  " exclude_raw (pattern_id, db_regex, nsp_regex, rel_regex, heap_only, btree_only) AS (");
1854  append_rel_pattern_raw_cte(&sql, &opts.exclude, conn);
1855  appendPQExpBufferStr(&sql, "\n),");
1856  append_rel_pattern_filtered_cte(&sql, "exclude_raw", "exclude_pat", conn);
1857  }
1858 
1859  /* Append the relation CTE. */
1860  appendPQExpBufferStr(&sql,
1861  " relation (pattern_id, oid, nspname, relname, reltoastrelid, relpages, is_heap, is_btree) AS ("
1862  "\nSELECT DISTINCT ON (c.oid");
1863  if (!opts.allrel)
1864  appendPQExpBufferStr(&sql, ", ip.pattern_id) ip.pattern_id,");
1865  else
1866  appendPQExpBufferStr(&sql, ") NULL::INTEGER AS pattern_id,");
1867  appendPQExpBuffer(&sql,
1868  "\nc.oid, n.nspname, c.relname, c.reltoastrelid, c.relpages, "
1869  "c.relam = %u AS is_heap, "
1870  "c.relam = %u AS is_btree"
1871  "\nFROM pg_catalog.pg_class c "
1872  "INNER JOIN pg_catalog.pg_namespace n "
1873  "ON c.relnamespace = n.oid",
1874  HEAP_TABLE_AM_OID, BTREE_AM_OID);
1875  if (!opts.allrel)
1876  appendPQExpBuffer(&sql,
1877  "\nINNER JOIN include_pat ip"
1878  "\nON (n.nspname ~ ip.nsp_regex OR ip.nsp_regex IS NULL)"
1879  "\nAND (c.relname ~ ip.rel_regex OR ip.rel_regex IS NULL)"
1880  "\nAND (c.relam = %u OR NOT ip.heap_only)"
1881  "\nAND (c.relam = %u OR NOT ip.btree_only)",
1882  HEAP_TABLE_AM_OID, BTREE_AM_OID);
1883  if (opts.excludetbl || opts.excludeidx || opts.excludensp)
1884  appendPQExpBuffer(&sql,
1885  "\nLEFT OUTER JOIN exclude_pat ep"
1886  "\nON (n.nspname ~ ep.nsp_regex OR ep.nsp_regex IS NULL)"
1887  "\nAND (c.relname ~ ep.rel_regex OR ep.rel_regex IS NULL)"
1888  "\nAND (c.relam = %u OR NOT ep.heap_only OR ep.rel_regex IS NULL)"
1889  "\nAND (c.relam = %u OR NOT ep.btree_only OR ep.rel_regex IS NULL)",
1890  HEAP_TABLE_AM_OID, BTREE_AM_OID);
1891 
1892  if (opts.excludetbl || opts.excludeidx || opts.excludensp)
1893  appendPQExpBufferStr(&sql, "\nWHERE ep.pattern_id IS NULL");
1894  else
1895  appendPQExpBufferStr(&sql, "\nWHERE true");
1896 
1897  /*
1898  * We need to be careful not to break the --no-dependent-toast and
1899  * --no-dependent-indexes options. By default, the btree indexes, toast
1900  * tables, and toast table btree indexes associated with primary heap
1901  * tables are included, using their own CTEs below. We implement the
1902  * --exclude-* options by not creating those CTEs, but that's no use if
1903  * we've already selected the toast and indexes here. On the other hand,
1904  * we want inclusion patterns that match indexes or toast tables to be
1905  * honored. So, if inclusion patterns were given, we want to select all
1906  * tables, toast tables, or indexes that match the patterns. But if no
1907  * inclusion patterns were given, and we're simply matching all relations,
1908  * then we only want to match the primary tables here.
1909  */
1910  if (opts.allrel)
1911  appendPQExpBuffer(&sql,
1912  " AND c.relam = %u "
1913  "AND c.relkind IN ('r', 'm', 't') "
1914  "AND c.relnamespace != %u",
1915  HEAP_TABLE_AM_OID, PG_TOAST_NAMESPACE);
1916  else
1917  appendPQExpBuffer(&sql,
1918  " AND c.relam IN (%u, %u)"
1919  "AND c.relkind IN ('r', 'm', 't', 'i') "
1920  "AND ((c.relam = %u AND c.relkind IN ('r', 'm', 't')) OR "
1921  "(c.relam = %u AND c.relkind = 'i'))",
1922  HEAP_TABLE_AM_OID, BTREE_AM_OID,
1923  HEAP_TABLE_AM_OID, BTREE_AM_OID);
1924 
1925  appendPQExpBufferStr(&sql,
1926  "\nORDER BY c.oid)");
1927 
1928  if (!opts.no_toast_expansion)
1929  {
1930  /*
1931  * Include a CTE for toast tables associated with primary heap tables
1932  * selected above, filtering by exclusion patterns (if any) that match
1933  * toast table names.
1934  */
1935  appendPQExpBufferStr(&sql,
1936  ", toast (oid, nspname, relname, relpages) AS ("
1937  "\nSELECT t.oid, 'pg_toast', t.relname, t.relpages"
1938  "\nFROM pg_catalog.pg_class t "
1939  "INNER JOIN relation r "
1940  "ON r.reltoastrelid = t.oid");
1941  if (opts.excludetbl || opts.excludensp)
1942  appendPQExpBufferStr(&sql,
1943  "\nLEFT OUTER JOIN exclude_pat ep"
1944  "\nON ('pg_toast' ~ ep.nsp_regex OR ep.nsp_regex IS NULL)"
1945  "\nAND (t.relname ~ ep.rel_regex OR ep.rel_regex IS NULL)"
1946  "\nAND ep.heap_only"
1947  "\nWHERE ep.pattern_id IS NULL");
1948  appendPQExpBufferStr(&sql,
1949  "\n)");
1950  }
1951  if (!opts.no_btree_expansion)
1952  {
1953  /*
1954  * Include a CTE for btree indexes associated with primary heap tables
1955  * selected above, filtering by exclusion patterns (if any) that match
1956  * btree index names.
1957  */
1958  appendPQExpBuffer(&sql,
1959  ", index (oid, nspname, relname, relpages) AS ("
1960  "\nSELECT c.oid, r.nspname, c.relname, c.relpages "
1961  "FROM relation r"
1962  "\nINNER JOIN pg_catalog.pg_index i "
1963  "ON r.oid = i.indrelid "
1964  "INNER JOIN pg_catalog.pg_class c "
1965  "ON i.indexrelid = c.oid");
1966  if (opts.excludeidx || opts.excludensp)
1967  appendPQExpBufferStr(&sql,
1968  "\nINNER JOIN pg_catalog.pg_namespace n "
1969  "ON c.relnamespace = n.oid"
1970  "\nLEFT OUTER JOIN exclude_pat ep "
1971  "ON (n.nspname ~ ep.nsp_regex OR ep.nsp_regex IS NULL) "
1972  "AND (c.relname ~ ep.rel_regex OR ep.rel_regex IS NULL) "
1973  "AND ep.btree_only"
1974  "\nWHERE ep.pattern_id IS NULL");
1975  else
1976  appendPQExpBufferStr(&sql,
1977  "\nWHERE true");
1978  appendPQExpBuffer(&sql,
1979  " AND c.relam = %u "
1980  "AND c.relkind = 'i'",
1981  BTREE_AM_OID);
1982  if (opts.no_toast_expansion)
1983  appendPQExpBuffer(&sql,
1984  " AND c.relnamespace != %u",
1985  PG_TOAST_NAMESPACE);
1986  appendPQExpBufferStr(&sql, "\n)");
1987  }
1988 
1989  if (!opts.no_toast_expansion && !opts.no_btree_expansion)
1990  {
1991  /*
1992  * Include a CTE for btree indexes associated with toast tables of
1993  * primary heap tables selected above, filtering by exclusion patterns
1994  * (if any) that match the toast index names.
1995  */
1996  appendPQExpBuffer(&sql,
1997  ", toast_index (oid, nspname, relname, relpages) AS ("
1998  "\nSELECT c.oid, 'pg_toast', c.relname, c.relpages "
1999  "FROM toast t "
2000  "INNER JOIN pg_catalog.pg_index i "
2001  "ON t.oid = i.indrelid"
2002  "\nINNER JOIN pg_catalog.pg_class c "
2003  "ON i.indexrelid = c.oid");
2004  if (opts.excludeidx)
2005  appendPQExpBufferStr(&sql,
2006  "\nLEFT OUTER JOIN exclude_pat ep "
2007  "ON ('pg_toast' ~ ep.nsp_regex OR ep.nsp_regex IS NULL) "
2008  "AND (c.relname ~ ep.rel_regex OR ep.rel_regex IS NULL) "
2009  "AND ep.btree_only "
2010  "WHERE ep.pattern_id IS NULL");
2011  else
2012  appendPQExpBufferStr(&sql,
2013  "\nWHERE true");
2014  appendPQExpBuffer(&sql,
2015  " AND c.relam = %u"
2016  " AND c.relkind = 'i')",
2017  BTREE_AM_OID);
2018  }
2019 
2020  /*
2021  * Roll-up distinct rows from CTEs.
2022  *
2023  * Relations that match more than one pattern may occur more than once in
2024  * the list, and indexes and toast for primary relations may also have
2025  * matched in their own right, so we rely on UNION to deduplicate the
2026  * list.
2027  */
2028  appendPQExpBuffer(&sql,
2029  "\nSELECT pattern_id, is_heap, is_btree, oid, nspname, relname, relpages "
2030  "FROM (");
2031  appendPQExpBufferStr(&sql,
2032  /* Inclusion patterns that failed to match */
2033  "\nSELECT pattern_id, is_heap, is_btree, "
2034  "NULL::OID AS oid, "
2035  "NULL::TEXT AS nspname, "
2036  "NULL::TEXT AS relname, "
2037  "NULL::INTEGER AS relpages"
2038  "\nFROM relation "
2039  "WHERE pattern_id IS NOT NULL "
2040  "UNION"
2041  /* Primary relations */
2042  "\nSELECT NULL::INTEGER AS pattern_id, "
2043  "is_heap, is_btree, oid, nspname, relname, relpages "
2044  "FROM relation");
2045  if (!opts.no_toast_expansion)
2046  appendPQExpBufferStr(&sql,
2047  " UNION"
2048  /* Toast tables for primary relations */
2049  "\nSELECT NULL::INTEGER AS pattern_id, TRUE AS is_heap, "
2050  "FALSE AS is_btree, oid, nspname, relname, relpages "
2051  "FROM toast");
2052  if (!opts.no_btree_expansion)
2053  appendPQExpBufferStr(&sql,
2054  " UNION"
2055  /* Indexes for primary relations */
2056  "\nSELECT NULL::INTEGER AS pattern_id, FALSE AS is_heap, "
2057  "TRUE AS is_btree, oid, nspname, relname, relpages "
2058  "FROM index");
2059  if (!opts.no_toast_expansion && !opts.no_btree_expansion)
2060  appendPQExpBufferStr(&sql,
2061  " UNION"
2062  /* Indexes for toast relations */
2063  "\nSELECT NULL::INTEGER AS pattern_id, FALSE AS is_heap, "
2064  "TRUE AS is_btree, oid, nspname, relname, relpages "
2065  "FROM toast_index");
2066  appendPQExpBufferStr(&sql,
2067  "\n) AS combined_records "
2068  "ORDER BY relpages DESC NULLS FIRST, oid");
2069 
2070  res = executeQuery(conn, sql.data, opts.echo);
2071  if (PQresultStatus(res) != PGRES_TUPLES_OK)
2072  {
2073  pg_log_error("query failed: %s", PQerrorMessage(conn));
2074  pg_log_info("query was: %s", sql.data);
2075  disconnectDatabase(conn);
2076  exit(1);
2077  }
2078  termPQExpBuffer(&sql);
2079 
2080  ntups = PQntuples(res);
2081  for (i = 0; i < ntups; i++)
2082  {
2083  int pattern_id = -1;
2084  bool is_heap = false;
2085  bool is_btree PG_USED_FOR_ASSERTS_ONLY = false;
2086  Oid oid = InvalidOid;
2087  const char *nspname = NULL;
2088  const char *relname = NULL;
2089  int relpages = 0;
2090 
2091  if (!PQgetisnull(res, i, 0))
2092  pattern_id = atoi(PQgetvalue(res, i, 0));
2093  if (!PQgetisnull(res, i, 1))
2094  is_heap = (PQgetvalue(res, i, 1)[0] == 't');
2095  if (!PQgetisnull(res, i, 2))
2096  is_btree = (PQgetvalue(res, i, 2)[0] == 't');
2097  if (!PQgetisnull(res, i, 3))
2098  oid = atooid(PQgetvalue(res, i, 3));
2099  if (!PQgetisnull(res, i, 4))
2100  nspname = PQgetvalue(res, i, 4);
2101  if (!PQgetisnull(res, i, 5))
2102  relname = PQgetvalue(res, i, 5);
2103  if (!PQgetisnull(res, i, 6))
2104  relpages = atoi(PQgetvalue(res, i, 6));
2105 
2106  if (pattern_id >= 0)
2107  {
2108  /*
2109  * Current record pertains to an inclusion pattern. Record that
2110  * it matched.
2111  */
2112 
2113  if (pattern_id >= opts.include.len)
2114  {
2115  pg_log_error("internal error: received unexpected relation pattern_id %d",
2116  pattern_id);
2117  exit(1);
2118  }
2119 
2120  opts.include.data[pattern_id].matched = true;
2121  }
2122  else
2123  {
2124  /* Current record pertains to a relation */
2125 
2126  RelationInfo *rel = (RelationInfo *) pg_malloc0(sizeof(RelationInfo));
2127 
2128  Assert(OidIsValid(oid));
2129  Assert((is_heap && !is_btree) || (is_btree && !is_heap));
2130 
2131  rel->datinfo = dat;
2132  rel->reloid = oid;
2133  rel->is_heap = is_heap;
2134  rel->nspname = pstrdup(nspname);
2135  rel->relname = pstrdup(relname);
2136  rel->relpages = relpages;
2137  rel->blocks_to_check = relpages;
2138  if (is_heap && (opts.startblock >= 0 || opts.endblock >= 0))
2139  {
2140  /*
2141  * We apply --startblock and --endblock to heap tables, but
2142  * not btree indexes, and for progress purposes we need to
2143  * track how many blocks we expect to check.
2144  */
2145  if (opts.endblock >= 0 && rel->blocks_to_check > opts.endblock)
2146  rel->blocks_to_check = opts.endblock + 1;
2147  if (opts.startblock >= 0)
2148  {
2149  if (rel->blocks_to_check > opts.startblock)
2150  rel->blocks_to_check -= opts.startblock;
2151  else
2152  rel->blocks_to_check = 0;
2153  }
2154  }
2155  *pagecount += rel->blocks_to_check;
2156 
2157  simple_ptr_list_append(relations, rel);
2158  }
2159  }
2160  PQclear(res);
2161 }
bool heapallindexed
Definition: pg_amcheck.c:104
struct DatabaseInfo DatabaseInfo
int64 endblock
Definition: pg_amcheck.c:98
#define VERBOSE_DATNAME_LENGTH
static PGresult * executeQuery(PGconn *conn, const char *query)
Definition: pg_dumpall.c:1879
char * PQerrorMessage(const PGconn *conn)
Definition: fe-connect.c:6744
int64 pg_time_t
Definition: pgtime.h:23
char * pgport
Definition: pg_backup.h:66
char * db_regex
Definition: pg_amcheck.c:35
char * PQgetvalue(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3642
static void compile_relation_list_one_db(PGconn *conn, SimplePtrList *relations, const DatabaseInfo *datinfo, uint64 *pagecount)
Definition: pg_amcheck.c:1827
static bool verify_heap_slot_handler(PGresult *res, PGconn *conn, void *context)
Definition: pg_amcheck.c:1002
bool option_parse_int(const char *optarg, const char *optname, int min_range, int max_range, int *result)
Definition: option_utils.c:50
static void append_relation_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
Definition: pg_amcheck.c:1419
const char * get_progname(const char *argv0)
Definition: path.c:453
const char * pattern
Definition: pg_amcheck.c:34
void termPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:131
#define pg_log_error(...)
Definition: logging.h:80
int getopt_long(int argc, char *const argv[], const char *optstring, const struct option *longopts, int *longindex)
Definition: getopt_long.c:57
static bool verify_btree_slot_handler(PGresult *res, PGconn *conn, void *context)
Definition: pg_amcheck.c:1083
void appendPQExpBufferStr(PQExpBuffer str, const char *data)
Definition: pqexpbuffer.c:369
static void compile_database_list(PGconn *conn, SimplePtrList *databases, const char *initial_dbname)
Definition: pg_amcheck.c:1524
char * pstrdup(const char *in)
Definition: mcxt.c:1299
void pg_logging_init(const char *argv0)
Definition: logging.c:81
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
static char * indent_lines(const char *str)
Definition: pg_amcheck.c:971
void ParallelSlotsAdoptConn(ParallelSlotArray *sa, PGconn *conn)
char * relname
Definition: pg_amcheck.c:159
char * nspname
Definition: pg_amcheck.c:158
bool matched
Definition: pg_amcheck.c:44
static void ParallelSlotSetHandler(ParallelSlot *slot, ParallelSlotResultHandler handler, void *context)
Definition: parallel_slot.h:47
#define printf(...)
Definition: port.h:222
static void setup_cancel_handler(void)
Definition: parallel.c:613
NameData datname
Definition: pg_database.h:35
char * amcheck_schema
Definition: pg_amcheck.c:150
bool no_toast_expansion
Definition: pg_amcheck.c:94
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
NameData relname
Definition: pg_class.h:38
unsigned int Oid
Definition: postgres_ext.h:31
bool on_error_stop
Definition: pg_amcheck.c:96
int PQntuples(const PGresult *res)
Definition: fe-exec.c:3248
#define fprintf
Definition: port.h:220
#define OidIsValid(objectId)
Definition: c.h:710
static void append_schema_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
Definition: pg_amcheck.c:1341
void patternToSQLRegex(int encoding, PQExpBuffer dbnamebuf, PQExpBuffer schemabuf, PQExpBuffer namebuf, const char *pattern, bool force_escape)
Definition: string_utils.c:967
ExecStatusType PQresultStatus(const PGresult *res)
Definition: fe-exec.c:3178
static bool should_processing_continue(PGresult *res)
Definition: pg_amcheck.c:930
ParallelSlotArray * ParallelSlotsSetup(int numslots, ConnParams *cparams, const char *progname, bool echo, const char *initcmd)
char * dbname
Definition: pg_backup.h:65
static bool append_db_pattern_cte(PQExpBuffer buf, const PatternInfoArray *pia, PGconn *conn, bool inclusive)
Definition: pg_amcheck.c:1478
ParallelSlot * ParallelSlotsGetIdle(ParallelSlotArray *sa, const char *dbname)
void disconnectDatabase(PGconn *conn)
int PQsendQuery(PGconn *conn, const char *query)
Definition: fe-exec.c:1326
const DatabaseInfo * datinfo
Definition: pg_amcheck.c:155
char * PQescapeIdentifier(PGconn *conn, const char *str, size_t len)
Definition: fe-exec.c:4075
bool install_missing
Definition: pg_amcheck.c:68
#define required_argument
Definition: getopt_long.h:25
void pfree(void *pointer)
Definition: mcxt.c:1169
int optind
Definition: getopt.c:50
bool no_btree_expansion
Definition: pg_amcheck.c:107
int64 startblock
Definition: pg_amcheck.c:97
#define MaxBlockNumber
Definition: block.h:35
static void append_database_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
Definition: pg_amcheck.c:1318
void handle_help_version_opts(int argc, char *argv[], const char *fixed_progname, help_handler hlp)
Definition: option_utils.c:24
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
PGconn * conn
Definition: streamutil.c:54
static void append_btree_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
Definition: pg_amcheck.c:1451
void appendPQExpBuffer(PQExpBuffer str, const char *fmt,...)
Definition: pqexpbuffer.c:267
#define FREE_AND_SET_NULL(x)
Definition: pg_amcheck.c:210
static bool progress_since_last_stderr
Definition: pg_amcheck.c:145
char * sql
Definition: pg_amcheck.c:162
static void executeCommand(PGconn *conn, const char *query)
Definition: pg_dumpall.c:1902
char * c
static char * buf
Definition: pg_test_fsync.c:68
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
PGconn * connection
Definition: parallel_slot.h:23
PatternInfoArray include
Definition: pg_amcheck.c:72
void * pg_realloc(void *ptr, size_t size)
Definition: fe_memutils.c:65
#define atooid(x)
Definition: postgres_ext.h:42
const char * username
Definition: pgbench.c:282
char * override_dbname
Definition: pg_backup.h:72
static int port
Definition: pg_regress.c:92
static AmcheckOptions opts
Definition: pg_amcheck.c:110
char * datname
Definition: pg_amcheck.c:149
char * pghost
Definition: pg_backup.h:67
static void prepare_btree_command(PQExpBuffer sql, RelationInfo *rel, PGconn *conn)
Definition: pg_amcheck.c:857
enum trivalue prompt_password
Definition: connect_utils.h:32
struct RelationInfo RelationInfo
bool heap_only
Definition: pg_amcheck.c:40
trivalue
Definition: vacuumlo.c:34
#define no_argument
Definition: getopt_long.h:24
#define PG_TEXTDOMAIN(domain)
Definition: c.h:1215
int blocks_to_check
Definition: pg_amcheck.c:161
static void append_relation_pattern_helper(PatternInfoArray *pia, const char *pattern, int encoding, bool heap_only, bool btree_only)
Definition: pg_amcheck.c:1376
static const char * amcheck_sql
Definition: pg_amcheck.c:169
bool ParallelSlotsWaitCompletion(ParallelSlotArray *sa)
static void help(const char *progname)
Definition: pg_amcheck.c:1144
PGconn * connectMaintenanceDatabase(ConnParams *cparams, const char *progname, bool echo)
void appendPQExpBufferChar(PQExpBuffer str, char ch)
Definition: pqexpbuffer.c:380
bool reconcile_toast
Definition: pg_amcheck.c:95
#define PG_DIAG_SEVERITY_NONLOCALIZED
Definition: postgres_ext.h:56
#define InvalidOid
Definition: postgres_ext.h:36
int pg_get_encoding_from_locale(const char *ctype, bool write_message)
Definition: chklocale.c:452
void PQclear(PGresult *res)
Definition: fe-exec.c:694
static void prepare_heap_command(PQExpBuffer sql, RelationInfo *rel, PGconn *conn)
Definition: pg_amcheck.c:824
static void append_heap_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
Definition: pg_amcheck.c:1435
char * PQdb(const PGconn *conn)
Definition: fe-connect.c:6590
static pg_time_t last_progress_report
Definition: pg_amcheck.c:144
struct SimplePtrListCell * next
Definition: simple_list.h:48
char * PQresultErrorField(const PGresult *res, int fieldcode)
Definition: fe-exec.c:3233
#define Assert(condition)
Definition: c.h:804
static void run_command(ParallelSlot *slot, const char *sql)
Definition: pg_amcheck.c:898
bool btree_only
Definition: pg_amcheck.c:42
static PGconn * connectDatabase(const char *dbname, const char *connstr, const char *pghost, const char *pgport, const char *pguser, trivalue prompt_password, bool fail_on_error)
Definition: pg_dumpall.c:1641
static const char * progname
Definition: pg_amcheck.c:138
struct PatternInfo PatternInfo
void pg_logging_increase_verbosity(void)
Definition: logging.c:174
#define optional_argument
Definition: getopt_long.h:26
const char * skip
Definition: pg_amcheck.c:99
#define fatal(...)
int32 encoding
Definition: pg_database.h:41
static void append_rel_pattern_filtered_cte(PQExpBuffer buf, const char *raw, const char *filtered, PGconn *conn)
Definition: pg_amcheck.c:1788
#define INT64_FORMAT
Definition: c.h:483
struct AmcheckOptions AmcheckOptions
PatternInfoArray exclude
Definition: pg_amcheck.c:73
bool show_progress
Definition: pg_amcheck.c:61
void appendStringLiteralConn(PQExpBuffer buf, const char *str, PGconn *conn)
Definition: string_utils.c:293
volatile sig_atomic_t CancelRequested
Definition: cancel.c:52
void set_pglocale_pgservice(const char *argv0, const char *app)
Definition: exec.c:433
char * install_schema
Definition: pg_amcheck.c:69
SimplePtrListCell * head
Definition: simple_list.h:54
char * optarg
Definition: getopt.c:52
void ParallelSlotsTerminate(ParallelSlotArray *sa)
int main(int argc, char *argv[])
Definition: pg_amcheck.c:216
void simple_ptr_list_append(SimplePtrList *list, void *ptr)
Definition: simple_list.c:162
int i
char * rel_regex
Definition: pg_amcheck.c:38
bool strict_names
Definition: pg_amcheck.c:60
static void append_rel_pattern_raw_cte(PQExpBuffer buf, const PatternInfoArray *pia, PGconn *conn)
Definition: pg_amcheck.c:1719
char * nsp_regex
Definition: pg_amcheck.c:37
#define pg_log_warning(...)
Definition: pgfnames.c:24
void resetPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:148
PatternInfo * data
Definition: pg_amcheck.c:49
int PQgetisnull(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3667
static PatternInfo * extend_pattern_info_array(PatternInfoArray *pia)
Definition: pg_amcheck.c:1296
struct PatternInfoArray PatternInfoArray
#define snprintf
Definition: port.h:216
PGVerbosity PQsetErrorVerbosity(PGconn *conn, PGVerbosity verbosity)
Definition: fe-connect.c:6871
#define _(x)
Definition: elog.c:89
#define UINT64_FORMAT
Definition: c.h:484
static void progress_report(uint64 relations_total, uint64 relations_checked, uint64 relpages_total, uint64 relpages_checked, const char *datname, bool force, bool finished)
Definition: pg_amcheck.c:1204
Datum now(PG_FUNCTION_ARGS)
Definition: timestamp.c:1544
void initPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:92
#define pg_log_info(...)
Definition: logging.h:88
static bool all_checks_pass
Definition: pg_amcheck.c:141
#define PG_USED_FOR_ASSERTS_ONLY
Definition: c.h:155
#define log_no_match(...)
Definition: pg_amcheck.c:203
const char * pguser
Definition: connect_utils.h:31
const char * get_user_name_or_exit(const char *progname)
Definition: username.c:74