PostgreSQL Source Code  git master
pg_amcheck.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_amcheck.c
4  * Detects corruption within database relations.
5  *
6  * Copyright (c) 2017-2021, PostgreSQL Global Development Group
7  *
8  * IDENTIFICATION
9  * src/bin/pg_amcheck/pg_amcheck.c
10  *
11  *-------------------------------------------------------------------------
12  */
13 #include "postgres_fe.h"
14 
15 #include <time.h>
16 
17 #include "catalog/pg_am_d.h"
18 #include "catalog/pg_namespace_d.h"
19 #include "common/logging.h"
20 #include "common/username.h"
21 #include "fe_utils/cancel.h"
22 #include "fe_utils/option_utils.h"
23 #include "fe_utils/parallel_slot.h"
24 #include "fe_utils/query_utils.h"
25 #include "fe_utils/simple_list.h"
26 #include "fe_utils/string_utils.h"
27 #include "getopt_long.h" /* pgrminclude ignore */
28 #include "pgtime.h"
29 #include "storage/block.h"
30 
31 typedef struct PatternInfo
32 {
33  const char *pattern; /* Unaltered pattern from the command line */
34  char *db_regex; /* Database regexp parsed from pattern, or
35  * NULL */
36  char *nsp_regex; /* Schema regexp parsed from pattern, or NULL */
37  char *rel_regex; /* Relation regexp parsed from pattern, or
38  * NULL */
39  bool heap_only; /* true if rel_regex should only match heap
40  * tables */
41  bool btree_only; /* true if rel_regex should only match btree
42  * indexes */
43  bool matched; /* true if the pattern matched in any database */
44 } PatternInfo;
45 
46 typedef struct PatternInfoArray
47 {
49  size_t len;
51 
52 /* pg_amcheck command line options controlled by user flags */
53 typedef struct AmcheckOptions
54 {
55  bool dbpattern;
56  bool alldb;
57  bool echo;
58  bool quiet;
59  bool verbose;
62  int jobs;
63 
64  /*
65  * Whether to install missing extensions, and optionally the name of the
66  * schema in which to install the extension's objects.
67  */
70 
71  /* Objects to check or not to check, as lists of PatternInfo structs. */
74 
75  /*
76  * As an optimization, if any pattern in the exclude list applies to heap
77  * tables, or similarly if any such pattern applies to btree indexes, or
78  * to schemas, then these will be true, otherwise false. These should
79  * always agree with what you'd conclude by grep'ing through the exclude
80  * list.
81  */
82  bool excludetbl;
83  bool excludeidx;
84  bool excludensp;
85 
86  /*
87  * If any inclusion pattern exists, then we should only be checking
88  * matching relations rather than all relations, so this is true iff
89  * include is empty.
90  */
91  bool allrel;
92 
93  /* heap table checking options */
97  int64 startblock;
98  int64 endblock;
99  const char *skip;
100 
101  /* btree index checking options */
105 
106  /* heap and btree hybrid option */
109 
111  .dbpattern = false,
112  .alldb = false,
113  .echo = false,
114  .quiet = false,
115  .verbose = false,
116  .strict_names = true,
117  .show_progress = false,
118  .jobs = 1,
119  .install_missing = false,
120  .install_schema = "pg_catalog",
121  .include = {NULL, 0},
122  .exclude = {NULL, 0},
123  .excludetbl = false,
124  .excludeidx = false,
125  .excludensp = false,
126  .allrel = true,
127  .no_toast_expansion = false,
128  .reconcile_toast = true,
129  .on_error_stop = false,
130  .startblock = -1,
131  .endblock = -1,
132  .skip = "none",
133  .parent_check = false,
134  .rootdescend = false,
135  .heapallindexed = false,
136  .no_btree_expansion = false
137 };
138 
139 static const char *progname = NULL;
140 
141 /* Whether all relations have so far passed their corruption checks */
142 static bool all_checks_pass = true;
143 
144 /* Time last progress report was displayed */
146 static bool progress_since_last_stderr = false;
147 
148 typedef struct DatabaseInfo
149 {
150  char *datname;
151  char *amcheck_schema; /* escaped, quoted literal */
152 } DatabaseInfo;
153 
154 typedef struct RelationInfo
155 {
156  const DatabaseInfo *datinfo; /* shared by other relinfos */
158  bool is_heap; /* true if heap, false if btree */
159  char *nspname;
160  char *relname;
161  int relpages;
163  char *sql; /* set during query run, pg_free'd after */
164 } RelationInfo;
165 
166 /*
167  * Query for determining if contrib's amcheck is installed. If so, selects the
168  * namespace name where amcheck's functions can be found.
169  */
170 static const char *amcheck_sql =
171 "SELECT n.nspname, x.extversion FROM pg_catalog.pg_extension x"
172 "\nJOIN pg_catalog.pg_namespace n ON x.extnamespace = n.oid"
173 "\nWHERE x.extname = 'amcheck'";
174 
175 static void prepare_heap_command(PQExpBuffer sql, RelationInfo *rel,
176  PGconn *conn);
177 static void prepare_btree_command(PQExpBuffer sql, RelationInfo *rel,
178  PGconn *conn);
179 static void run_command(ParallelSlot *slot, const char *sql);
180 static bool verify_heap_slot_handler(PGresult *res, PGconn *conn,
181  void *context);
182 static bool verify_btree_slot_handler(PGresult *res, PGconn *conn, void *context);
183 static void help(const char *progname);
184 static void progress_report(uint64 relations_total, uint64 relations_checked,
185  uint64 relpages_total, uint64 relpages_checked,
186  const char *datname, bool force, bool finished);
187 
188 static void append_database_pattern(PatternInfoArray *pia, const char *pattern,
189  int encoding);
190 static void append_schema_pattern(PatternInfoArray *pia, const char *pattern,
191  int encoding);
192 static void append_relation_pattern(PatternInfoArray *pia, const char *pattern,
193  int encoding);
194 static void append_heap_pattern(PatternInfoArray *pia, const char *pattern,
195  int encoding);
196 static void append_btree_pattern(PatternInfoArray *pia, const char *pattern,
197  int encoding);
198 static void compile_database_list(PGconn *conn, SimplePtrList *databases,
199  const char *initial_dbname);
200 static void compile_relation_list_one_db(PGconn *conn, SimplePtrList *relations,
201  const DatabaseInfo *datinfo,
202  uint64 *pagecount);
203 
204 #define log_no_match(...) do { \
205  if (opts.strict_names) \
206  pg_log_generic(PG_LOG_ERROR, __VA_ARGS__); \
207  else \
208  pg_log_generic(PG_LOG_WARNING, __VA_ARGS__); \
209  } while(0)
210 
211 #define FREE_AND_SET_NULL(x) do { \
212  pg_free(x); \
213  (x) = NULL; \
214  } while (0)
215 
216 int
217 main(int argc, char *argv[])
218 {
219  PGconn *conn = NULL;
220  SimplePtrListCell *cell;
221  SimplePtrList databases = {NULL, NULL};
222  SimplePtrList relations = {NULL, NULL};
223  bool failed = false;
224  const char *latest_datname;
225  int parallel_workers;
227  PQExpBufferData sql;
228  uint64 reltotal = 0;
229  uint64 pageschecked = 0;
230  uint64 pagestotal = 0;
231  uint64 relprogress = 0;
232  int pattern_id;
233 
234  static struct option long_options[] = {
235  /* Connection options */
236  {"host", required_argument, NULL, 'h'},
237  {"port", required_argument, NULL, 'p'},
238  {"username", required_argument, NULL, 'U'},
239  {"no-password", no_argument, NULL, 'w'},
240  {"password", no_argument, NULL, 'W'},
241  {"maintenance-db", required_argument, NULL, 1},
242 
243  /* check options */
244  {"all", no_argument, NULL, 'a'},
245  {"database", required_argument, NULL, 'd'},
246  {"exclude-database", required_argument, NULL, 'D'},
247  {"echo", no_argument, NULL, 'e'},
248  {"index", required_argument, NULL, 'i'},
249  {"exclude-index", required_argument, NULL, 'I'},
250  {"jobs", required_argument, NULL, 'j'},
251  {"progress", no_argument, NULL, 'P'},
252  {"quiet", no_argument, NULL, 'q'},
253  {"relation", required_argument, NULL, 'r'},
254  {"exclude-relation", required_argument, NULL, 'R'},
255  {"schema", required_argument, NULL, 's'},
256  {"exclude-schema", required_argument, NULL, 'S'},
257  {"table", required_argument, NULL, 't'},
258  {"exclude-table", required_argument, NULL, 'T'},
259  {"verbose", no_argument, NULL, 'v'},
260  {"no-dependent-indexes", no_argument, NULL, 2},
261  {"no-dependent-toast", no_argument, NULL, 3},
262  {"exclude-toast-pointers", no_argument, NULL, 4},
263  {"on-error-stop", no_argument, NULL, 5},
264  {"skip", required_argument, NULL, 6},
265  {"startblock", required_argument, NULL, 7},
266  {"endblock", required_argument, NULL, 8},
267  {"rootdescend", no_argument, NULL, 9},
268  {"no-strict-names", no_argument, NULL, 10},
269  {"heapallindexed", no_argument, NULL, 11},
270  {"parent-check", no_argument, NULL, 12},
271  {"install-missing", optional_argument, NULL, 13},
272 
273  {NULL, 0, NULL, 0}
274  };
275 
276  int optindex;
277  int c;
278 
279  const char *db = NULL;
280  const char *maintenance_db = NULL;
281 
282  const char *host = NULL;
283  const char *port = NULL;
284  const char *username = NULL;
285  enum trivalue prompt_password = TRI_DEFAULT;
286  int encoding = pg_get_encoding_from_locale(NULL, false);
287  ConnParams cparams;
288 
289  pg_logging_init(argv[0]);
290  progname = get_progname(argv[0]);
291  set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_amcheck"));
292 
293  handle_help_version_opts(argc, argv, progname, help);
294 
295  /* process command-line options */
296  while ((c = getopt_long(argc, argv, "ad:D:eh:Hi:I:j:p:Pqr:R:s:S:t:T:U:wWv",
297  long_options, &optindex)) != -1)
298  {
299  char *endptr;
300 
301  switch (c)
302  {
303  case 'a':
304  opts.alldb = true;
305  break;
306  case 'd':
307  opts.dbpattern = true;
308  append_database_pattern(&opts.include, optarg, encoding);
309  break;
310  case 'D':
311  opts.dbpattern = true;
312  append_database_pattern(&opts.exclude, optarg, encoding);
313  break;
314  case 'e':
315  opts.echo = true;
316  break;
317  case 'h':
318  host = pg_strdup(optarg);
319  break;
320  case 'i':
321  opts.allrel = false;
322  append_btree_pattern(&opts.include, optarg, encoding);
323  break;
324  case 'I':
325  opts.excludeidx = true;
326  append_btree_pattern(&opts.exclude, optarg, encoding);
327  break;
328  case 'j':
329  opts.jobs = atoi(optarg);
330  if (opts.jobs < 1)
331  {
332  pg_log_error("number of parallel jobs must be at least 1");
333  exit(1);
334  }
335  break;
336  case 'p':
337  port = pg_strdup(optarg);
338  break;
339  case 'P':
340  opts.show_progress = true;
341  break;
342  case 'q':
343  opts.quiet = true;
344  break;
345  case 'r':
346  opts.allrel = false;
347  append_relation_pattern(&opts.include, optarg, encoding);
348  break;
349  case 'R':
350  opts.excludeidx = true;
351  opts.excludetbl = true;
352  append_relation_pattern(&opts.exclude, optarg, encoding);
353  break;
354  case 's':
355  opts.allrel = false;
356  append_schema_pattern(&opts.include, optarg, encoding);
357  break;
358  case 'S':
359  opts.excludensp = true;
360  append_schema_pattern(&opts.exclude, optarg, encoding);
361  break;
362  case 't':
363  opts.allrel = false;
364  append_heap_pattern(&opts.include, optarg, encoding);
365  break;
366  case 'T':
367  opts.excludetbl = true;
368  append_heap_pattern(&opts.exclude, optarg, encoding);
369  break;
370  case 'U':
371  username = pg_strdup(optarg);
372  break;
373  case 'w':
374  prompt_password = TRI_NO;
375  break;
376  case 'W':
377  prompt_password = TRI_YES;
378  break;
379  case 'v':
380  opts.verbose = true;
382  break;
383  case 1:
384  maintenance_db = pg_strdup(optarg);
385  break;
386  case 2:
387  opts.no_btree_expansion = true;
388  break;
389  case 3:
390  opts.no_toast_expansion = true;
391  break;
392  case 4:
393  opts.reconcile_toast = false;
394  break;
395  case 5:
396  opts.on_error_stop = true;
397  break;
398  case 6:
399  if (pg_strcasecmp(optarg, "all-visible") == 0)
400  opts.skip = "all visible";
401  else if (pg_strcasecmp(optarg, "all-frozen") == 0)
402  opts.skip = "all frozen";
403  else
404  {
405  pg_log_error("invalid skip option");
406  exit(1);
407  }
408  break;
409  case 7:
410  opts.startblock = strtol(optarg, &endptr, 10);
411  if (*endptr != '\0')
412  {
413  pg_log_error("invalid start block");
414  exit(1);
415  }
416  if (opts.startblock > MaxBlockNumber || opts.startblock < 0)
417  {
418  pg_log_error("start block out of bounds");
419  exit(1);
420  }
421  break;
422  case 8:
423  opts.endblock = strtol(optarg, &endptr, 10);
424  if (*endptr != '\0')
425  {
426  pg_log_error("invalid end block");
427  exit(1);
428  }
429  if (opts.endblock > MaxBlockNumber || opts.endblock < 0)
430  {
431  pg_log_error("end block out of bounds");
432  exit(1);
433  }
434  break;
435  case 9:
436  opts.rootdescend = true;
437  opts.parent_check = true;
438  break;
439  case 10:
440  opts.strict_names = false;
441  break;
442  case 11:
443  opts.heapallindexed = true;
444  break;
445  case 12:
446  opts.parent_check = true;
447  break;
448  case 13:
449  opts.install_missing = true;
450  if (optarg)
452  break;
453  default:
454  fprintf(stderr,
455  _("Try \"%s --help\" for more information.\n"),
456  progname);
457  exit(1);
458  }
459  }
460 
461  if (opts.endblock >= 0 && opts.endblock < opts.startblock)
462  {
463  pg_log_error("end block precedes start block");
464  exit(1);
465  }
466 
467  /*
468  * A single non-option arguments specifies a database name or connection
469  * string.
470  */
471  if (optind < argc)
472  {
473  db = argv[optind];
474  optind++;
475  }
476 
477  if (optind < argc)
478  {
479  pg_log_error("too many command-line arguments (first is \"%s\")",
480  argv[optind]);
481  fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
482  exit(1);
483  }
484 
485  /* fill cparams except for dbname, which is set below */
486  cparams.pghost = host;
487  cparams.pgport = port;
488  cparams.pguser = username;
489  cparams.prompt_password = prompt_password;
490  cparams.dbname = NULL;
491  cparams.override_dbname = NULL;
492 
493  setup_cancel_handler(NULL);
494 
495  /* choose the database for our initial connection */
496  if (opts.alldb)
497  {
498  if (db != NULL)
499  {
500  pg_log_error("cannot specify a database name with --all");
501  exit(1);
502  }
503  cparams.dbname = maintenance_db;
504  }
505  else if (db != NULL)
506  {
507  if (opts.dbpattern)
508  {
509  pg_log_error("cannot specify both a database name and database patterns");
510  exit(1);
511  }
512  cparams.dbname = db;
513  }
514 
515  if (opts.alldb || opts.dbpattern)
516  {
517  conn = connectMaintenanceDatabase(&cparams, progname, opts.echo);
518  compile_database_list(conn, &databases, NULL);
519  }
520  else
521  {
522  if (cparams.dbname == NULL)
523  {
524  if (getenv("PGDATABASE"))
525  cparams.dbname = getenv("PGDATABASE");
526  else if (getenv("PGUSER"))
527  cparams.dbname = getenv("PGUSER");
528  else
530  }
531  conn = connectDatabase(&cparams, progname, opts.echo, false, true);
532  compile_database_list(conn, &databases, PQdb(conn));
533  }
534 
535  if (databases.head == NULL)
536  {
537  if (conn != NULL)
538  disconnectDatabase(conn);
539  pg_log_error("no databases to check");
540  exit(0);
541  }
542 
543  /*
544  * Compile a list of all relations spanning all databases to be checked.
545  */
546  for (cell = databases.head; cell; cell = cell->next)
547  {
548  PGresult *result;
549  int ntups;
550  const char *amcheck_schema = NULL;
551  DatabaseInfo *dat = (DatabaseInfo *) cell->ptr;
552 
553  cparams.override_dbname = dat->datname;
554  if (conn == NULL || strcmp(PQdb(conn), dat->datname) != 0)
555  {
556  if (conn != NULL)
557  disconnectDatabase(conn);
558  conn = connectDatabase(&cparams, progname, opts.echo, false, true);
559  }
560 
561  /*
562  * Optionally install amcheck if not already installed in this
563  * database.
564  */
565  if (opts.install_missing)
566  {
567  char *schema;
568  char *install_sql;
569 
570  /*
571  * Must re-escape the schema name for each database, as the
572  * escaping rules may change.
573  */
574  schema = PQescapeIdentifier(conn, opts.install_schema,
575  strlen(opts.install_schema));
576  install_sql = psprintf("CREATE EXTENSION IF NOT EXISTS amcheck WITH SCHEMA %s",
577  schema);
578 
579  executeCommand(conn, install_sql, opts.echo);
580  pfree(install_sql);
581  pfree(schema);
582  }
583 
584  /*
585  * Verify that amcheck is installed for this next database. User
586  * error could result in a database not having amcheck that should
587  * have it, but we also could be iterating over multiple databases
588  * where not all of them have amcheck installed (for example,
589  * 'template1').
590  */
591  result = executeQuery(conn, amcheck_sql, opts.echo);
592  if (PQresultStatus(result) != PGRES_TUPLES_OK)
593  {
594  /* Querying the catalog failed. */
595  pg_log_error("database \"%s\": %s",
596  PQdb(conn), PQerrorMessage(conn));
597  pg_log_info("query was: %s", amcheck_sql);
598  PQclear(result);
599  disconnectDatabase(conn);
600  exit(1);
601  }
602  ntups = PQntuples(result);
603  if (ntups == 0)
604  {
605  /* Querying the catalog succeeded, but amcheck is missing. */
606  pg_log_warning("skipping database \"%s\": amcheck is not installed",
607  PQdb(conn));
608  disconnectDatabase(conn);
609  conn = NULL;
610  continue;
611  }
612  amcheck_schema = PQgetvalue(result, 0, 0);
613  if (opts.verbose)
614  pg_log_info("in database \"%s\": using amcheck version \"%s\" in schema \"%s\"",
615  PQdb(conn), PQgetvalue(result, 0, 1), amcheck_schema);
616  dat->amcheck_schema = PQescapeIdentifier(conn, amcheck_schema,
617  strlen(amcheck_schema));
618  PQclear(result);
619 
620  compile_relation_list_one_db(conn, &relations, dat, &pagestotal);
621  }
622 
623  /*
624  * Check that all inclusion patterns matched at least one schema or
625  * relation that we can check.
626  */
627  for (pattern_id = 0; pattern_id < opts.include.len; pattern_id++)
628  {
629  PatternInfo *pat = &opts.include.data[pattern_id];
630 
631  if (!pat->matched && (pat->nsp_regex != NULL || pat->rel_regex != NULL))
632  {
633  failed = opts.strict_names;
634 
635  if (!opts.quiet || failed)
636  {
637  if (pat->heap_only)
638  log_no_match("no heap tables to check matching \"%s\"",
639  pat->pattern);
640  else if (pat->btree_only)
641  log_no_match("no btree indexes to check matching \"%s\"",
642  pat->pattern);
643  else if (pat->rel_regex == NULL)
644  log_no_match("no relations to check in schemas matching \"%s\"",
645  pat->pattern);
646  else
647  log_no_match("no relations to check matching \"%s\"",
648  pat->pattern);
649  }
650  }
651  }
652 
653  if (failed)
654  {
655  if (conn != NULL)
656  disconnectDatabase(conn);
657  exit(1);
658  }
659 
660  /*
661  * Set parallel_workers to the lesser of opts.jobs and the number of
662  * relations.
663  */
664  parallel_workers = 0;
665  for (cell = relations.head; cell; cell = cell->next)
666  {
667  reltotal++;
668  if (parallel_workers < opts.jobs)
669  parallel_workers++;
670  }
671 
672  if (reltotal == 0)
673  {
674  if (conn != NULL)
675  disconnectDatabase(conn);
676  pg_log_error("no relations to check");
677  exit(1);
678  }
679  progress_report(reltotal, relprogress, pagestotal, pageschecked,
680  NULL, true, false);
681 
682  /*
683  * Main event loop.
684  *
685  * We use server-side parallelism to check up to parallel_workers
686  * relations in parallel. The list of relations was computed in database
687  * order, which minimizes the number of connects and disconnects as we
688  * process the list.
689  */
690  latest_datname = NULL;
691  sa = ParallelSlotsSetup(parallel_workers, &cparams, progname, opts.echo,
692  NULL);
693  if (conn != NULL)
694  {
695  ParallelSlotsAdoptConn(sa, conn);
696  conn = NULL;
697  }
698 
699  initPQExpBuffer(&sql);
700  for (relprogress = 0, cell = relations.head; cell; cell = cell->next)
701  {
702  ParallelSlot *free_slot;
703  RelationInfo *rel;
704 
705  rel = (RelationInfo *) cell->ptr;
706 
707  if (CancelRequested)
708  {
709  failed = true;
710  break;
711  }
712 
713  /*
714  * The list of relations is in database sorted order. If this next
715  * relation is in a different database than the last one seen, we are
716  * about to start checking this database. Note that other slots may
717  * still be working on relations from prior databases.
718  */
719  latest_datname = rel->datinfo->datname;
720 
721  progress_report(reltotal, relprogress, pagestotal, pageschecked,
722  latest_datname, false, false);
723 
724  relprogress++;
725  pageschecked += rel->blocks_to_check;
726 
727  /*
728  * Get a parallel slot for the next amcheck command, blocking if
729  * necessary until one is available, or until a previously issued slot
730  * command fails, indicating that we should abort checking the
731  * remaining objects.
732  */
733  free_slot = ParallelSlotsGetIdle(sa, rel->datinfo->datname);
734  if (!free_slot)
735  {
736  /*
737  * Something failed. We don't need to know what it was, because
738  * the handler should already have emitted the necessary error
739  * messages.
740  */
741  failed = true;
742  break;
743  }
744 
745  if (opts.verbose)
747  else if (opts.quiet)
749 
750  /*
751  * Execute the appropriate amcheck command for this relation using our
752  * slot's database connection. We do not wait for the command to
753  * complete, nor do we perform any error checking, as that is done by
754  * the parallel slots and our handler callback functions.
755  */
756  if (rel->is_heap)
757  {
758  if (opts.verbose)
759  {
761  fprintf(stderr, "\n");
762  pg_log_info("checking heap table \"%s\".\"%s\".\"%s\"",
763  rel->datinfo->datname, rel->nspname, rel->relname);
765  }
766  prepare_heap_command(&sql, rel, free_slot->connection);
767  rel->sql = pstrdup(sql.data); /* pg_free'd after command */
769  run_command(free_slot, rel->sql);
770  }
771  else
772  {
773  if (opts.verbose)
774  {
776  fprintf(stderr, "\n");
777 
778  pg_log_info("checking btree index \"%s\".\"%s\".\"%s\"",
779  rel->datinfo->datname, rel->nspname, rel->relname);
781  }
782  prepare_btree_command(&sql, rel, free_slot->connection);
783  rel->sql = pstrdup(sql.data); /* pg_free'd after command */
785  run_command(free_slot, rel->sql);
786  }
787  }
788  termPQExpBuffer(&sql);
789 
790  if (!failed)
791  {
792 
793  /*
794  * Wait for all slots to complete, or for one to indicate that an
795  * error occurred. Like above, we rely on the handler emitting the
796  * necessary error messages.
797  */
798  if (sa && !ParallelSlotsWaitCompletion(sa))
799  failed = true;
800 
801  progress_report(reltotal, relprogress, pagestotal, pageschecked, NULL, true, true);
802  }
803 
804  if (sa)
805  {
807  FREE_AND_SET_NULL(sa);
808  }
809 
810  if (failed)
811  exit(1);
812 
813  if (!all_checks_pass)
814  exit(2);
815 }
816 
817 /*
818  * prepare_heap_command
819  *
820  * Creates a SQL command for running amcheck checking on the given heap
821  * relation. The command is phrased as a SQL query, with column order and
822  * names matching the expectations of verify_heap_slot_handler, which will
823  * receive and handle each row returned from the verify_heapam() function.
824  *
825  * sql: buffer into which the heap table checking command will be written
826  * rel: relation information for the heap table to be checked
827  * conn: the connection to be used, for string escaping purposes
828  */
829 static void
831 {
832  resetPQExpBuffer(sql);
833  appendPQExpBuffer(sql,
834  "SELECT blkno, offnum, attnum, msg FROM %s.verify_heapam("
835  "\nrelation := %u, on_error_stop := %s, check_toast := %s, skip := '%s'",
836  rel->datinfo->amcheck_schema,
837  rel->reloid,
838  opts.on_error_stop ? "true" : "false",
839  opts.reconcile_toast ? "true" : "false",
840  opts.skip);
841 
842  if (opts.startblock >= 0)
843  appendPQExpBuffer(sql, ", startblock := " INT64_FORMAT, opts.startblock);
844  if (opts.endblock >= 0)
845  appendPQExpBuffer(sql, ", endblock := " INT64_FORMAT, opts.endblock);
846 
847  appendPQExpBuffer(sql, ")");
848 }
849 
850 /*
851  * prepare_btree_command
852  *
853  * Creates a SQL command for running amcheck checking on the given btree index
854  * relation. The command does not select any columns, as btree checking
855  * functions do not return any, but rather return corruption information by
856  * raising errors, which verify_btree_slot_handler expects.
857  *
858  * sql: buffer into which the heap table checking command will be written
859  * rel: relation information for the index to be checked
860  * conn: the connection to be used, for string escaping purposes
861  */
862 static void
864 {
865  resetPQExpBuffer(sql);
866 
867  /*
868  * Embed the database, schema, and relation name in the query, so if the
869  * check throws an error, the user knows which relation the error came
870  * from.
871  */
872  if (opts.parent_check)
873  appendPQExpBuffer(sql,
874  "SELECT * FROM %s.bt_index_parent_check("
875  "index := '%u'::regclass, heapallindexed := %s, "
876  "rootdescend := %s)",
877  rel->datinfo->amcheck_schema,
878  rel->reloid,
879  (opts.heapallindexed ? "true" : "false"),
880  (opts.rootdescend ? "true" : "false"));
881  else
882  appendPQExpBuffer(sql,
883  "SELECT * FROM %s.bt_index_check("
884  "index := '%u'::regclass, heapallindexed := %s)",
885  rel->datinfo->amcheck_schema,
886  rel->reloid,
887  (opts.heapallindexed ? "true" : "false"));
888 }
889 
890 /*
891  * run_command
892  *
893  * Sends a command to the server without waiting for the command to complete.
894  * Logs an error if the command cannot be sent, but otherwise any errors are
895  * expected to be handled by a ParallelSlotHandler.
896  *
897  * If reconnecting to the database is necessary, the cparams argument may be
898  * modified.
899  *
900  * slot: slot with connection to the server we should use for the command
901  * sql: query to send
902  */
903 static void
904 run_command(ParallelSlot *slot, const char *sql)
905 {
906  if (opts.echo)
907  printf("%s\n", sql);
908 
909  if (PQsendQuery(slot->connection, sql) == 0)
910  {
911  pg_log_error("error sending command to database \"%s\": %s",
912  PQdb(slot->connection),
913  PQerrorMessage(slot->connection));
914  pg_log_error("command was: %s", sql);
915  exit(1);
916  }
917 }
918 
919 /*
920  * should_processing_continue
921  *
922  * Checks a query result returned from a query (presumably issued on a slot's
923  * connection) to determine if parallel slots should continue issuing further
924  * commands.
925  *
926  * Note: Heap relation corruption is reported by verify_heapam() via the result
927  * set, rather than an ERROR, but running verify_heapam() on a corrupted heap
928  * table may still result in an error being returned from the server due to
929  * missing relation files, bad checksums, etc. The btree corruption checking
930  * functions always use errors to communicate corruption messages. We can't
931  * just abort processing because we got a mere ERROR.
932  *
933  * res: result from an executed sql query
934  */
935 static bool
937 {
938  const char *severity;
939 
940  switch (PQresultStatus(res))
941  {
942  /* These are expected and ok */
943  case PGRES_COMMAND_OK:
944  case PGRES_TUPLES_OK:
946  break;
947 
948  /* This is expected but requires closer scrutiny */
949  case PGRES_FATAL_ERROR:
951  if (strcmp(severity, "FATAL") == 0)
952  return false;
953  if (strcmp(severity, "PANIC") == 0)
954  return false;
955  break;
956 
957  /* These are unexpected */
958  case PGRES_BAD_RESPONSE:
959  case PGRES_EMPTY_QUERY:
960  case PGRES_COPY_OUT:
961  case PGRES_COPY_IN:
962  case PGRES_COPY_BOTH:
963  case PGRES_SINGLE_TUPLE:
964  case PGRES_PIPELINE_SYNC:
966  return false;
967  }
968  return true;
969 }
970 
971 /*
972  * Returns a copy of the argument string with all lines indented four spaces.
973  *
974  * The caller should pg_free the result when finished with it.
975  */
976 static char *
977 indent_lines(const char *str)
978 {
980  const char *c;
981  char *result;
982 
983  initPQExpBuffer(&buf);
984  appendPQExpBufferStr(&buf, " ");
985  for (c = str; *c; c++)
986  {
987  appendPQExpBufferChar(&buf, *c);
988  if (c[0] == '\n' && c[1] != '\0')
989  appendPQExpBufferStr(&buf, " ");
990  }
991  result = pstrdup(buf.data);
992  termPQExpBuffer(&buf);
993 
994  return result;
995 }
996 
997 /*
998  * verify_heap_slot_handler
999  *
1000  * ParallelSlotHandler that receives results from a heap table checking command
1001  * created by prepare_heap_command and outputs the results for the user.
1002  *
1003  * res: result from an executed sql query
1004  * conn: connection on which the sql query was executed
1005  * context: the sql query being handled, as a cstring
1006  */
1007 static bool
1009 {
1010  RelationInfo *rel = (RelationInfo *) context;
1011 
1012  if (PQresultStatus(res) == PGRES_TUPLES_OK)
1013  {
1014  int i;
1015  int ntups = PQntuples(res);
1016 
1017  if (ntups > 0)
1018  all_checks_pass = false;
1019 
1020  for (i = 0; i < ntups; i++)
1021  {
1022  const char *msg;
1023 
1024  /* The message string should never be null, but check */
1025  if (PQgetisnull(res, i, 3))
1026  msg = "NO MESSAGE";
1027  else
1028  msg = PQgetvalue(res, i, 3);
1029 
1030  if (!PQgetisnull(res, i, 2))
1031  printf("heap table \"%s\".\"%s\".\"%s\", block %s, offset %s, attribute %s:\n %s\n",
1032  rel->datinfo->datname, rel->nspname, rel->relname,
1033  PQgetvalue(res, i, 0), /* blkno */
1034  PQgetvalue(res, i, 1), /* offnum */
1035  PQgetvalue(res, i, 2), /* attnum */
1036  msg);
1037 
1038  else if (!PQgetisnull(res, i, 1))
1039  printf("heap table \"%s\".\"%s\".\"%s\", block %s, offset %s:\n %s\n",
1040  rel->datinfo->datname, rel->nspname, rel->relname,
1041  PQgetvalue(res, i, 0), /* blkno */
1042  PQgetvalue(res, i, 1), /* offnum */
1043  msg);
1044 
1045  else if (!PQgetisnull(res, i, 0))
1046  printf("heap table \"%s\".\"%s\".\"%s\", block %s:\n %s\n",
1047  rel->datinfo->datname, rel->nspname, rel->relname,
1048  PQgetvalue(res, i, 0), /* blkno */
1049  msg);
1050 
1051  else
1052  printf("heap table \"%s\".\"%s\".\"%s\":\n %s\n",
1053  rel->datinfo->datname, rel->nspname, rel->relname, msg);
1054  }
1055  }
1056  else if (PQresultStatus(res) != PGRES_TUPLES_OK)
1057  {
1058  char *msg = indent_lines(PQerrorMessage(conn));
1059 
1060  all_checks_pass = false;
1061  printf("heap table \"%s\".\"%s\".\"%s\":\n%s",
1062  rel->datinfo->datname, rel->nspname, rel->relname, msg);
1063  if (opts.verbose)
1064  printf("query was: %s\n", rel->sql);
1065  FREE_AND_SET_NULL(msg);
1066  }
1067 
1068  FREE_AND_SET_NULL(rel->sql);
1069  FREE_AND_SET_NULL(rel->nspname);
1070  FREE_AND_SET_NULL(rel->relname);
1071 
1072  return should_processing_continue(res);
1073 }
1074 
1075 /*
1076  * verify_btree_slot_handler
1077  *
1078  * ParallelSlotHandler that receives results from a btree checking command
1079  * created by prepare_btree_command and outputs them for the user. The results
1080  * from the btree checking command is assumed to be empty, but when the results
1081  * are an error code, the useful information about the corruption is expected
1082  * in the connection's error message.
1083  *
1084  * res: result from an executed sql query
1085  * conn: connection on which the sql query was executed
1086  * context: unused
1087  */
1088 static bool
1090 {
1091  RelationInfo *rel = (RelationInfo *) context;
1092 
1093  if (PQresultStatus(res) == PGRES_TUPLES_OK)
1094  {
1095  int ntups = PQntuples(res);
1096 
1097  if (ntups != 1)
1098  {
1099  /*
1100  * We expect the btree checking functions to return one void row
1101  * each, so we should output some sort of warning if we get
1102  * anything else, not because it indicates corruption, but because
1103  * it suggests a mismatch between amcheck and pg_amcheck versions.
1104  *
1105  * In conjunction with --progress, anything written to stderr at
1106  * this time would present strangely to the user without an extra
1107  * newline, so we print one. If we were multithreaded, we'd have
1108  * to avoid splitting this across multiple calls, but we're in an
1109  * event loop, so it doesn't matter.
1110  */
1112  fprintf(stderr, "\n");
1113  pg_log_warning("btree index \"%s\".\"%s\".\"%s\": btree checking function returned unexpected number of rows: %d",
1114  rel->datinfo->datname, rel->nspname, rel->relname, ntups);
1115  if (opts.verbose)
1116  pg_log_info("query was: %s", rel->sql);
1117  pg_log_warning("are %s's and amcheck's versions compatible?",
1118  progname);
1120  }
1121  }
1122  else
1123  {
1124  char *msg = indent_lines(PQerrorMessage(conn));
1125 
1126  all_checks_pass = false;
1127  printf("btree index \"%s\".\"%s\".\"%s\":\n%s",
1128  rel->datinfo->datname, rel->nspname, rel->relname, msg);
1129  if (opts.verbose)
1130  printf("query was: %s\n", rel->sql);
1131  FREE_AND_SET_NULL(msg);
1132  }
1133 
1134  FREE_AND_SET_NULL(rel->sql);
1135  FREE_AND_SET_NULL(rel->nspname);
1136  FREE_AND_SET_NULL(rel->relname);
1137 
1138  return should_processing_continue(res);
1139 }
1140 
1141 /*
1142  * help
1143  *
1144  * Prints help page for the program
1145  *
1146  * progname: the name of the executed program, such as "pg_amcheck"
1147  */
1148 static void
1149 help(const char *progname)
1150 {
1151  printf(_("%s uses amcheck module to check objects in a PostgreSQL database for corruption.\n\n"), progname);
1152  printf(_("Usage:\n"));
1153  printf(_(" %s [OPTION]... [DBNAME]\n"), progname);
1154  printf(_("\nTarget Options:\n"));
1155  printf(_(" -a, --all check all databases\n"));
1156  printf(_(" -d, --database=PATTERN check matching database(s)\n"));
1157  printf(_(" -D, --exclude-database=PATTERN do NOT check matching database(s)\n"));
1158  printf(_(" -i, --index=PATTERN check matching index(es)\n"));
1159  printf(_(" -I, --exclude-index=PATTERN do NOT check matching index(es)\n"));
1160  printf(_(" -r, --relation=PATTERN check matching relation(s)\n"));
1161  printf(_(" -R, --exclude-relation=PATTERN do NOT check matching relation(s)\n"));
1162  printf(_(" -s, --schema=PATTERN check matching schema(s)\n"));
1163  printf(_(" -S, --exclude-schema=PATTERN do NOT check matching schema(s)\n"));
1164  printf(_(" -t, --table=PATTERN check matching table(s)\n"));
1165  printf(_(" -T, --exclude-table=PATTERN do NOT check matching table(s)\n"));
1166  printf(_(" --no-dependent-indexes do NOT expand list of relations to include indexes\n"));
1167  printf(_(" --no-dependent-toast do NOT expand list of relations to include toast\n"));
1168  printf(_(" --no-strict-names do NOT require patterns to match objects\n"));
1169  printf(_("\nTable Checking Options:\n"));
1170  printf(_(" --exclude-toast-pointers do NOT follow relation toast pointers\n"));
1171  printf(_(" --on-error-stop stop checking at end of first corrupt page\n"));
1172  printf(_(" --skip=OPTION do NOT check \"all-frozen\" or \"all-visible\" blocks\n"));
1173  printf(_(" --startblock=BLOCK begin checking table(s) at the given block number\n"));
1174  printf(_(" --endblock=BLOCK check table(s) only up to the given block number\n"));
1175  printf(_("\nBtree Index Checking Options:\n"));
1176  printf(_(" --heapallindexed check all heap tuples are found within indexes\n"));
1177  printf(_(" --parent-check check index parent/child relationships\n"));
1178  printf(_(" --rootdescend search from root page to refind tuples\n"));
1179  printf(_("\nConnection options:\n"));
1180  printf(_(" -h, --host=HOSTNAME database server host or socket directory\n"));
1181  printf(_(" -p, --port=PORT database server port\n"));
1182  printf(_(" -U, --username=USERNAME user name to connect as\n"));
1183  printf(_(" -w, --no-password never prompt for password\n"));
1184  printf(_(" -W, --password force password prompt\n"));
1185  printf(_(" --maintenance-db=DBNAME alternate maintenance database\n"));
1186  printf(_("\nOther Options:\n"));
1187  printf(_(" -e, --echo show the commands being sent to the server\n"));
1188  printf(_(" -j, --jobs=NUM use this many concurrent connections to the server\n"));
1189  printf(_(" -q, --quiet don't write any messages\n"));
1190  printf(_(" -v, --verbose write a lot of output\n"));
1191  printf(_(" -V, --version output version information, then exit\n"));
1192  printf(_(" -P, --progress show progress information\n"));
1193  printf(_(" -?, --help show this help, then exit\n"));
1194  printf(_(" --install-missing install missing extensions\n"));
1195 
1196  printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
1197  printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
1198 }
1199 
1200 /*
1201  * Print a progress report based on the global variables.
1202  *
1203  * Progress report is written at maximum once per second, unless the force
1204  * parameter is set to true.
1205  *
1206  * If finished is set to true, this is the last progress report. The cursor
1207  * is moved to the next line.
1208  */
1209 static void
1210 progress_report(uint64 relations_total, uint64 relations_checked,
1211  uint64 relpages_total, uint64 relpages_checked,
1212  const char *datname, bool force, bool finished)
1213 {
1214  int percent_rel = 0;
1215  int percent_pages = 0;
1216  char checked_rel[32];
1217  char total_rel[32];
1218  char checked_pages[32];
1219  char total_pages[32];
1220  pg_time_t now;
1221 
1222  if (!opts.show_progress)
1223  return;
1224 
1225  now = time(NULL);
1226  if (now == last_progress_report && !force && !finished)
1227  return; /* Max once per second */
1228 
1230  if (relations_total)
1231  percent_rel = (int) (relations_checked * 100 / relations_total);
1232  if (relpages_total)
1233  percent_pages = (int) (relpages_checked * 100 / relpages_total);
1234 
1235  /*
1236  * Separate step to keep platform-dependent format code out of fprintf
1237  * calls. We only test for INT64_FORMAT availability in snprintf, not
1238  * fprintf.
1239  */
1240  snprintf(checked_rel, sizeof(checked_rel), INT64_FORMAT, relations_checked);
1241  snprintf(total_rel, sizeof(total_rel), INT64_FORMAT, relations_total);
1242  snprintf(checked_pages, sizeof(checked_pages), INT64_FORMAT, relpages_checked);
1243  snprintf(total_pages, sizeof(total_pages), INT64_FORMAT, relpages_total);
1244 
1245 #define VERBOSE_DATNAME_LENGTH 35
1246  if (opts.verbose)
1247  {
1248  if (!datname)
1249 
1250  /*
1251  * No datname given, so clear the status line (used for first and
1252  * last call)
1253  */
1254  fprintf(stderr,
1255  _("%*s/%s relations (%d%%) %*s/%s pages (%d%%) %*s"),
1256  (int) strlen(total_rel),
1257  checked_rel, total_rel, percent_rel,
1258  (int) strlen(total_pages),
1259  checked_pages, total_pages, percent_pages,
1260  VERBOSE_DATNAME_LENGTH + 2, "");
1261  else
1262  {
1263  bool truncate = (strlen(datname) > VERBOSE_DATNAME_LENGTH);
1264 
1265  fprintf(stderr,
1266  _("%*s/%s relations (%d%%) %*s/%s pages (%d%%), (%s%-*.*s)"),
1267  (int) strlen(total_rel),
1268  checked_rel, total_rel, percent_rel,
1269  (int) strlen(total_pages),
1270  checked_pages, total_pages, percent_pages,
1271  /* Prefix with "..." if we do leading truncation */
1272  truncate ? "..." : "",
1275  /* Truncate datname at beginning if it's too long */
1276  truncate ? datname + strlen(datname) - VERBOSE_DATNAME_LENGTH + 3 : datname);
1277  }
1278  }
1279  else
1280  fprintf(stderr,
1281  _("%*s/%s relations (%d%%) %*s/%s pages (%d%%)"),
1282  (int) strlen(total_rel),
1283  checked_rel, total_rel, percent_rel,
1284  (int) strlen(total_pages),
1285  checked_pages, total_pages, percent_pages);
1286 
1287  /*
1288  * Stay on the same line if reporting to a terminal and we're not done
1289  * yet.
1290  */
1291  if (!finished && isatty(fileno(stderr)))
1292  {
1293  fputc('\r', stderr);
1295  }
1296  else
1297  fputc('\n', stderr);
1298 }
1299 
1300 /*
1301  * Extend the pattern info array to hold one additional initialized pattern
1302  * info entry.
1303  *
1304  * Returns a pointer to the new entry.
1305  */
1306 static PatternInfo *
1308 {
1309  PatternInfo *result;
1310 
1311  pia->len++;
1312  pia->data = (PatternInfo *) pg_realloc(pia->data, pia->len * sizeof(PatternInfo));
1313  result = &pia->data[pia->len - 1];
1314  memset(result, 0, sizeof(*result));
1315 
1316  return result;
1317 }
1318 
1319 /*
1320  * append_database_pattern
1321  *
1322  * Adds the given pattern interpreted as a database name pattern.
1323  *
1324  * pia: the pattern info array to be appended
1325  * pattern: the database name pattern
1326  * encoding: client encoding for parsing the pattern
1327  */
1328 static void
1330 {
1333 
1334  initPQExpBuffer(&buf);
1335  patternToSQLRegex(encoding, NULL, NULL, &buf, pattern, false);
1336  info->pattern = pattern;
1337  info->db_regex = pstrdup(buf.data);
1338 
1339  termPQExpBuffer(&buf);
1340 }
1341 
1342 /*
1343  * append_schema_pattern
1344  *
1345  * Adds the given pattern interpreted as a schema name pattern.
1346  *
1347  * pia: the pattern info array to be appended
1348  * pattern: the schema name pattern
1349  * encoding: client encoding for parsing the pattern
1350  */
1351 static void
1353 {
1354  PQExpBufferData dbbuf;
1355  PQExpBufferData nspbuf;
1357 
1358  initPQExpBuffer(&dbbuf);
1359  initPQExpBuffer(&nspbuf);
1360 
1361  patternToSQLRegex(encoding, NULL, &dbbuf, &nspbuf, pattern, false);
1362  info->pattern = pattern;
1363  if (dbbuf.data[0])
1364  {
1365  opts.dbpattern = true;
1366  info->db_regex = pstrdup(dbbuf.data);
1367  }
1368  if (nspbuf.data[0])
1369  info->nsp_regex = pstrdup(nspbuf.data);
1370 
1371  termPQExpBuffer(&dbbuf);
1372  termPQExpBuffer(&nspbuf);
1373 }
1374 
1375 /*
1376  * append_relation_pattern_helper
1377  *
1378  * Adds to a list the given pattern interpreted as a relation pattern.
1379  *
1380  * pia: the pattern info array to be appended
1381  * pattern: the relation name pattern
1382  * encoding: client encoding for parsing the pattern
1383  * heap_only: whether the pattern should only be matched against heap tables
1384  * btree_only: whether the pattern should only be matched against btree indexes
1385  */
1386 static void
1388  int encoding, bool heap_only, bool btree_only)
1389 {
1390  PQExpBufferData dbbuf;
1391  PQExpBufferData nspbuf;
1392  PQExpBufferData relbuf;
1394 
1395  initPQExpBuffer(&dbbuf);
1396  initPQExpBuffer(&nspbuf);
1397  initPQExpBuffer(&relbuf);
1398 
1399  patternToSQLRegex(encoding, &dbbuf, &nspbuf, &relbuf, pattern, false);
1400  info->pattern = pattern;
1401  if (dbbuf.data[0])
1402  {
1403  opts.dbpattern = true;
1404  info->db_regex = pstrdup(dbbuf.data);
1405  }
1406  if (nspbuf.data[0])
1407  info->nsp_regex = pstrdup(nspbuf.data);
1408  if (relbuf.data[0])
1409  info->rel_regex = pstrdup(relbuf.data);
1410 
1411  termPQExpBuffer(&dbbuf);
1412  termPQExpBuffer(&nspbuf);
1413  termPQExpBuffer(&relbuf);
1414 
1415  info->heap_only = heap_only;
1416  info->btree_only = btree_only;
1417 }
1418 
1419 /*
1420  * append_relation_pattern
1421  *
1422  * Adds the given pattern interpreted as a relation pattern, to be matched
1423  * against both heap tables and btree indexes.
1424  *
1425  * pia: the pattern info array to be appended
1426  * pattern: the relation name pattern
1427  * encoding: client encoding for parsing the pattern
1428  */
1429 static void
1431 {
1432  append_relation_pattern_helper(pia, pattern, encoding, false, false);
1433 }
1434 
1435 /*
1436  * append_heap_pattern
1437  *
1438  * Adds the given pattern interpreted as a relation pattern, to be matched only
1439  * against heap tables.
1440  *
1441  * pia: the pattern info array to be appended
1442  * pattern: the relation name pattern
1443  * encoding: client encoding for parsing the pattern
1444  */
1445 static void
1447 {
1448  append_relation_pattern_helper(pia, pattern, encoding, true, false);
1449 }
1450 
1451 /*
1452  * append_btree_pattern
1453  *
1454  * Adds the given pattern interpreted as a relation pattern, to be matched only
1455  * against btree indexes.
1456  *
1457  * pia: the pattern info array to be appended
1458  * pattern: the relation name pattern
1459  * encoding: client encoding for parsing the pattern
1460  */
1461 static void
1463 {
1464  append_relation_pattern_helper(pia, pattern, encoding, false, true);
1465 }
1466 
1467 /*
1468  * append_db_pattern_cte
1469  *
1470  * Appends to the buffer the body of a Common Table Expression (CTE) containing
1471  * the database portions filtered from the list of patterns expressed as two
1472  * columns:
1473  *
1474  * pattern_id: the index of this pattern in pia->data[]
1475  * rgx: the database regular expression parsed from the pattern
1476  *
1477  * Patterns without a database portion are skipped. Patterns with more than
1478  * just a database portion are optionally skipped, depending on argument
1479  * 'inclusive'.
1480  *
1481  * buf: the buffer to be appended
1482  * pia: the array of patterns to be inserted into the CTE
1483  * conn: the database connection
1484  * inclusive: whether to include patterns with schema and/or relation parts
1485  *
1486  * Returns whether any database patterns were appended.
1487  */
1488 static bool
1490  PGconn *conn, bool inclusive)
1491 {
1492  int pattern_id;
1493  const char *comma;
1494  bool have_values;
1495 
1496  comma = "";
1497  have_values = false;
1498  for (pattern_id = 0; pattern_id < pia->len; pattern_id++)
1499  {
1500  PatternInfo *info = &pia->data[pattern_id];
1501 
1502  if (info->db_regex != NULL &&
1503  (inclusive || (info->nsp_regex == NULL && info->rel_regex == NULL)))
1504  {
1505  if (!have_values)
1506  appendPQExpBufferStr(buf, "\nVALUES");
1507  have_values = true;
1508  appendPQExpBuffer(buf, "%s\n(%d, ", comma, pattern_id);
1509  appendStringLiteralConn(buf, info->db_regex, conn);
1510  appendPQExpBufferStr(buf, ")");
1511  comma = ",";
1512  }
1513  }
1514 
1515  if (!have_values)
1516  appendPQExpBufferStr(buf, "\nSELECT NULL, NULL, NULL WHERE false");
1517 
1518  return have_values;
1519 }
1520 
1521 /*
1522  * compile_database_list
1523  *
1524  * If any database patterns exist, or if --all was given, compiles a distinct
1525  * list of databases to check using a SQL query based on the patterns plus the
1526  * literal initial database name, if given. If no database patterns exist and
1527  * --all was not given, the query is not necessary, and only the initial
1528  * database name (if any) is added to the list.
1529  *
1530  * conn: connection to the initial database
1531  * databases: the list onto which databases should be appended
1532  * initial_dbname: an optional extra database name to include in the list
1533  */
1534 static void
1536  const char *initial_dbname)
1537 {
1538  PGresult *res;
1539  PQExpBufferData sql;
1540  int ntups;
1541  int i;
1542  bool fatal;
1543 
1544  if (initial_dbname)
1545  {
1546  DatabaseInfo *dat = (DatabaseInfo *) pg_malloc0(sizeof(DatabaseInfo));
1547 
1548  /* This database is included. Add to list */
1549  if (opts.verbose)
1550  pg_log_info("including database: \"%s\"", initial_dbname);
1551 
1552  dat->datname = pstrdup(initial_dbname);
1553  simple_ptr_list_append(databases, dat);
1554  }
1555 
1556  initPQExpBuffer(&sql);
1557 
1558  /* Append the include patterns CTE. */
1559  appendPQExpBufferStr(&sql, "WITH include_raw (pattern_id, rgx) AS (");
1560  if (!append_db_pattern_cte(&sql, &opts.include, conn, true) &&
1561  !opts.alldb)
1562  {
1563  /*
1564  * None of the inclusion patterns (if any) contain database portions,
1565  * so there is no need to query the database to resolve database
1566  * patterns.
1567  *
1568  * Since we're also not operating under --all, we don't need to query
1569  * the exhaustive list of connectable databases, either.
1570  */
1571  termPQExpBuffer(&sql);
1572  return;
1573  }
1574 
1575  /* Append the exclude patterns CTE. */
1576  appendPQExpBufferStr(&sql, "),\nexclude_raw (pattern_id, rgx) AS (");
1577  append_db_pattern_cte(&sql, &opts.exclude, conn, false);
1578  appendPQExpBufferStr(&sql, "),");
1579 
1580  /*
1581  * Append the database CTE, which includes whether each database is
1582  * connectable and also joins against exclude_raw to determine whether
1583  * each database is excluded.
1584  */
1585  appendPQExpBufferStr(&sql,
1586  "\ndatabase (datname) AS ("
1587  "\nSELECT d.datname "
1588  "FROM pg_catalog.pg_database d "
1589  "LEFT OUTER JOIN exclude_raw e "
1590  "ON d.datname ~ e.rgx "
1591  "\nWHERE d.datallowconn "
1592  "AND e.pattern_id IS NULL"
1593  "),"
1594 
1595  /*
1596  * Append the include_pat CTE, which joins the include_raw CTE against the
1597  * databases CTE to determine if all the inclusion patterns had matches,
1598  * and whether each matched pattern had the misfortune of only matching
1599  * excluded or unconnectable databases.
1600  */
1601  "\ninclude_pat (pattern_id, checkable) AS ("
1602  "\nSELECT i.pattern_id, "
1603  "COUNT(*) FILTER ("
1604  "WHERE d IS NOT NULL"
1605  ") AS checkable"
1606  "\nFROM include_raw i "
1607  "LEFT OUTER JOIN database d "
1608  "ON d.datname ~ i.rgx"
1609  "\nGROUP BY i.pattern_id"
1610  "),"
1611 
1612  /*
1613  * Append the filtered_databases CTE, which selects from the database CTE
1614  * optionally joined against the include_raw CTE to only select databases
1615  * that match an inclusion pattern. This appears to duplicate what the
1616  * include_pat CTE already did above, but here we want only databases, and
1617  * there we wanted patterns.
1618  */
1619  "\nfiltered_databases (datname) AS ("
1620  "\nSELECT DISTINCT d.datname "
1621  "FROM database d");
1622  if (!opts.alldb)
1623  appendPQExpBufferStr(&sql,
1624  " INNER JOIN include_raw i "
1625  "ON d.datname ~ i.rgx");
1626  appendPQExpBufferStr(&sql,
1627  ")"
1628 
1629  /*
1630  * Select the checkable databases and the unmatched inclusion patterns.
1631  */
1632  "\nSELECT pattern_id, datname FROM ("
1633  "\nSELECT pattern_id, NULL::TEXT AS datname "
1634  "FROM include_pat "
1635  "WHERE checkable = 0 "
1636  "UNION ALL"
1637  "\nSELECT NULL, datname "
1638  "FROM filtered_databases"
1639  ") AS combined_records"
1640  "\nORDER BY pattern_id NULLS LAST, datname");
1641 
1642  res = executeQuery(conn, sql.data, opts.echo);
1643  if (PQresultStatus(res) != PGRES_TUPLES_OK)
1644  {
1645  pg_log_error("query failed: %s", PQerrorMessage(conn));
1646  pg_log_info("query was: %s", sql.data);
1647  disconnectDatabase(conn);
1648  exit(1);
1649  }
1650  termPQExpBuffer(&sql);
1651 
1652  ntups = PQntuples(res);
1653  for (fatal = false, i = 0; i < ntups; i++)
1654  {
1655  int pattern_id = -1;
1656  const char *datname = NULL;
1657 
1658  if (!PQgetisnull(res, i, 0))
1659  pattern_id = atoi(PQgetvalue(res, i, 0));
1660  if (!PQgetisnull(res, i, 1))
1661  datname = PQgetvalue(res, i, 1);
1662 
1663  if (pattern_id >= 0)
1664  {
1665  /*
1666  * Current record pertains to an inclusion pattern that matched no
1667  * checkable databases.
1668  */
1669  fatal = opts.strict_names;
1670  if (pattern_id >= opts.include.len)
1671  {
1672  pg_log_error("internal error: received unexpected database pattern_id %d",
1673  pattern_id);
1674  exit(1);
1675  }
1676  log_no_match("no connectable databases to check matching \"%s\"",
1677  opts.include.data[pattern_id].pattern);
1678  }
1679  else
1680  {
1681  DatabaseInfo *dat;
1682 
1683  /* Current record pertains to a database */
1684  Assert(datname != NULL);
1685 
1686  /* Avoid entering a duplicate entry matching the initial_dbname */
1687  if (initial_dbname != NULL && strcmp(initial_dbname, datname) == 0)
1688  continue;
1689 
1690  /* This database is included. Add to list */
1691  if (opts.verbose)
1692  pg_log_info("including database: \"%s\"", datname);
1693 
1694  dat = (DatabaseInfo *) pg_malloc0(sizeof(DatabaseInfo));
1695  dat->datname = pstrdup(datname);
1696  simple_ptr_list_append(databases, dat);
1697  }
1698  }
1699  PQclear(res);
1700 
1701  if (fatal)
1702  {
1703  if (conn != NULL)
1704  disconnectDatabase(conn);
1705  exit(1);
1706  }
1707 }
1708 
1709 /*
1710  * append_rel_pattern_raw_cte
1711  *
1712  * Appends to the buffer the body of a Common Table Expression (CTE) containing
1713  * the given patterns as six columns:
1714  *
1715  * pattern_id: the index of this pattern in pia->data[]
1716  * db_regex: the database regexp parsed from the pattern, or NULL if the
1717  * pattern had no database part
1718  * nsp_regex: the namespace regexp parsed from the pattern, or NULL if the
1719  * pattern had no namespace part
1720  * rel_regex: the relname regexp parsed from the pattern, or NULL if the
1721  * pattern had no relname part
1722  * heap_only: true if the pattern applies only to heap tables (not indexes)
1723  * btree_only: true if the pattern applies only to btree indexes (not tables)
1724  *
1725  * buf: the buffer to be appended
1726  * patterns: the array of patterns to be inserted into the CTE
1727  * conn: the database connection
1728  */
1729 static void
1731  PGconn *conn)
1732 {
1733  int pattern_id;
1734  const char *comma;
1735  bool have_values;
1736 
1737  comma = "";
1738  have_values = false;
1739  for (pattern_id = 0; pattern_id < pia->len; pattern_id++)
1740  {
1741  PatternInfo *info = &pia->data[pattern_id];
1742 
1743  if (!have_values)
1744  appendPQExpBufferStr(buf, "\nVALUES");
1745  have_values = true;
1746  appendPQExpBuffer(buf, "%s\n(%d::INTEGER, ", comma, pattern_id);
1747  if (info->db_regex == NULL)
1748  appendPQExpBufferStr(buf, "NULL");
1749  else
1750  appendStringLiteralConn(buf, info->db_regex, conn);
1751  appendPQExpBufferStr(buf, "::TEXT, ");
1752  if (info->nsp_regex == NULL)
1753  appendPQExpBufferStr(buf, "NULL");
1754  else
1755  appendStringLiteralConn(buf, info->nsp_regex, conn);
1756  appendPQExpBufferStr(buf, "::TEXT, ");
1757  if (info->rel_regex == NULL)
1758  appendPQExpBufferStr(buf, "NULL");
1759  else
1760  appendStringLiteralConn(buf, info->rel_regex, conn);
1761  if (info->heap_only)
1762  appendPQExpBufferStr(buf, "::TEXT, true::BOOLEAN");
1763  else
1764  appendPQExpBufferStr(buf, "::TEXT, false::BOOLEAN");
1765  if (info->btree_only)
1766  appendPQExpBufferStr(buf, ", true::BOOLEAN");
1767  else
1768  appendPQExpBufferStr(buf, ", false::BOOLEAN");
1769  appendPQExpBufferStr(buf, ")");
1770  comma = ",";
1771  }
1772 
1773  if (!have_values)
1775  "\nSELECT NULL::INTEGER, NULL::TEXT, NULL::TEXT, "
1776  "NULL::TEXT, NULL::BOOLEAN, NULL::BOOLEAN "
1777  "WHERE false");
1778 }
1779 
1780 /*
1781  * append_rel_pattern_filtered_cte
1782  *
1783  * Appends to the buffer a Common Table Expression (CTE) which selects
1784  * all patterns from the named raw CTE, filtered by database. All patterns
1785  * which have no database portion or whose database portion matches our
1786  * connection's database name are selected, with other patterns excluded.
1787  *
1788  * The basic idea here is that if we're connected to database "foo" and we have
1789  * patterns "foo.bar.baz", "alpha.beta" and "one.two.three", we only want to
1790  * use the first two while processing relations in this database, as the third
1791  * one is not relevant.
1792  *
1793  * buf: the buffer to be appended
1794  * raw: the name of the CTE to select from
1795  * filtered: the name of the CTE to create
1796  * conn: the database connection
1797  */
1798 static void
1800  const char *filtered, PGconn *conn)
1801 {
1802  appendPQExpBuffer(buf,
1803  "\n%s (pattern_id, nsp_regex, rel_regex, heap_only, btree_only) AS ("
1804  "\nSELECT pattern_id, nsp_regex, rel_regex, heap_only, btree_only "
1805  "FROM %s r"
1806  "\nWHERE (r.db_regex IS NULL "
1807  "OR ",
1808  filtered, raw);
1809  appendStringLiteralConn(buf, PQdb(conn), conn);
1810  appendPQExpBufferStr(buf, " ~ r.db_regex)");
1812  " AND (r.nsp_regex IS NOT NULL"
1813  " OR r.rel_regex IS NOT NULL)"
1814  "),");
1815 }
1816 
1817 /*
1818  * compile_relation_list_one_db
1819  *
1820  * Compiles a list of relations to check within the currently connected
1821  * database based on the user supplied options, sorted by descending size,
1822  * and appends them to the given list of relations.
1823  *
1824  * The cells of the constructed list contain all information about the relation
1825  * necessary to connect to the database and check the object, including which
1826  * database to connect to, where contrib/amcheck is installed, and the Oid and
1827  * type of object (heap table vs. btree index). Rather than duplicating the
1828  * database details per relation, the relation structs use references to the
1829  * same database object, provided by the caller.
1830  *
1831  * conn: connection to this next database, which should be the same as in 'dat'
1832  * relations: list onto which the relations information should be appended
1833  * dat: the database info struct for use by each relation
1834  * pagecount: gets incremented by the number of blocks to check in all
1835  * relations added
1836  */
1837 static void
1839  const DatabaseInfo *dat,
1840  uint64 *pagecount)
1841 {
1842  PGresult *res;
1843  PQExpBufferData sql;
1844  int ntups;
1845  int i;
1846 
1847  initPQExpBuffer(&sql);
1848  appendPQExpBufferStr(&sql, "WITH");
1849 
1850  /* Append CTEs for the relation inclusion patterns, if any */
1851  if (!opts.allrel)
1852  {
1853  appendPQExpBufferStr(&sql,
1854  " include_raw (pattern_id, db_regex, nsp_regex, rel_regex, heap_only, btree_only) AS (");
1855  append_rel_pattern_raw_cte(&sql, &opts.include, conn);
1856  appendPQExpBufferStr(&sql, "\n),");
1857  append_rel_pattern_filtered_cte(&sql, "include_raw", "include_pat", conn);
1858  }
1859 
1860  /* Append CTEs for the relation exclusion patterns, if any */
1861  if (opts.excludetbl || opts.excludeidx || opts.excludensp)
1862  {
1863  appendPQExpBufferStr(&sql,
1864  " exclude_raw (pattern_id, db_regex, nsp_regex, rel_regex, heap_only, btree_only) AS (");
1865  append_rel_pattern_raw_cte(&sql, &opts.exclude, conn);
1866  appendPQExpBufferStr(&sql, "\n),");
1867  append_rel_pattern_filtered_cte(&sql, "exclude_raw", "exclude_pat", conn);
1868  }
1869 
1870  /* Append the relation CTE. */
1871  appendPQExpBufferStr(&sql,
1872  " relation (pattern_id, oid, nspname, relname, reltoastrelid, relpages, is_heap, is_btree) AS ("
1873  "\nSELECT DISTINCT ON (c.oid");
1874  if (!opts.allrel)
1875  appendPQExpBufferStr(&sql, ", ip.pattern_id) ip.pattern_id,");
1876  else
1877  appendPQExpBufferStr(&sql, ") NULL::INTEGER AS pattern_id,");
1878  appendPQExpBuffer(&sql,
1879  "\nc.oid, n.nspname, c.relname, c.reltoastrelid, c.relpages, "
1880  "c.relam = %u AS is_heap, "
1881  "c.relam = %u AS is_btree"
1882  "\nFROM pg_catalog.pg_class c "
1883  "INNER JOIN pg_catalog.pg_namespace n "
1884  "ON c.relnamespace = n.oid",
1885  HEAP_TABLE_AM_OID, BTREE_AM_OID);
1886  if (!opts.allrel)
1887  appendPQExpBuffer(&sql,
1888  "\nINNER JOIN include_pat ip"
1889  "\nON (n.nspname ~ ip.nsp_regex OR ip.nsp_regex IS NULL)"
1890  "\nAND (c.relname ~ ip.rel_regex OR ip.rel_regex IS NULL)"
1891  "\nAND (c.relam = %u OR NOT ip.heap_only)"
1892  "\nAND (c.relam = %u OR NOT ip.btree_only)",
1893  HEAP_TABLE_AM_OID, BTREE_AM_OID);
1894  if (opts.excludetbl || opts.excludeidx || opts.excludensp)
1895  appendPQExpBuffer(&sql,
1896  "\nLEFT OUTER JOIN exclude_pat ep"
1897  "\nON (n.nspname ~ ep.nsp_regex OR ep.nsp_regex IS NULL)"
1898  "\nAND (c.relname ~ ep.rel_regex OR ep.rel_regex IS NULL)"
1899  "\nAND (c.relam = %u OR NOT ep.heap_only OR ep.rel_regex IS NULL)"
1900  "\nAND (c.relam = %u OR NOT ep.btree_only OR ep.rel_regex IS NULL)",
1901  HEAP_TABLE_AM_OID, BTREE_AM_OID);
1902 
1903  if (opts.excludetbl || opts.excludeidx || opts.excludensp)
1904  appendPQExpBufferStr(&sql, "\nWHERE ep.pattern_id IS NULL");
1905  else
1906  appendPQExpBufferStr(&sql, "\nWHERE true");
1907 
1908  /*
1909  * We need to be careful not to break the --no-dependent-toast and
1910  * --no-dependent-indexes options. By default, the btree indexes, toast
1911  * tables, and toast table btree indexes associated with primary heap
1912  * tables are included, using their own CTEs below. We implement the
1913  * --exclude-* options by not creating those CTEs, but that's no use if
1914  * we've already selected the toast and indexes here. On the other hand,
1915  * we want inclusion patterns that match indexes or toast tables to be
1916  * honored. So, if inclusion patterns were given, we want to select all
1917  * tables, toast tables, or indexes that match the patterns. But if no
1918  * inclusion patterns were given, and we're simply matching all relations,
1919  * then we only want to match the primary tables here.
1920  */
1921  if (opts.allrel)
1922  appendPQExpBuffer(&sql,
1923  " AND c.relam = %u "
1924  "AND c.relkind IN ('r', 'm', 't') "
1925  "AND c.relnamespace != %u",
1926  HEAP_TABLE_AM_OID, PG_TOAST_NAMESPACE);
1927  else
1928  appendPQExpBuffer(&sql,
1929  " AND c.relam IN (%u, %u)"
1930  "AND c.relkind IN ('r', 'm', 't', 'i') "
1931  "AND ((c.relam = %u AND c.relkind IN ('r', 'm', 't')) OR "
1932  "(c.relam = %u AND c.relkind = 'i'))",
1933  HEAP_TABLE_AM_OID, BTREE_AM_OID,
1934  HEAP_TABLE_AM_OID, BTREE_AM_OID);
1935 
1936  appendPQExpBufferStr(&sql,
1937  "\nORDER BY c.oid)");
1938 
1939  if (!opts.no_toast_expansion)
1940  {
1941  /*
1942  * Include a CTE for toast tables associated with primary heap tables
1943  * selected above, filtering by exclusion patterns (if any) that match
1944  * toast table names.
1945  */
1946  appendPQExpBufferStr(&sql,
1947  ", toast (oid, nspname, relname, relpages) AS ("
1948  "\nSELECT t.oid, 'pg_toast', t.relname, t.relpages"
1949  "\nFROM pg_catalog.pg_class t "
1950  "INNER JOIN relation r "
1951  "ON r.reltoastrelid = t.oid");
1952  if (opts.excludetbl || opts.excludensp)
1953  appendPQExpBufferStr(&sql,
1954  "\nLEFT OUTER JOIN exclude_pat ep"
1955  "\nON ('pg_toast' ~ ep.nsp_regex OR ep.nsp_regex IS NULL)"
1956  "\nAND (t.relname ~ ep.rel_regex OR ep.rel_regex IS NULL)"
1957  "\nAND ep.heap_only"
1958  "\nWHERE ep.pattern_id IS NULL");
1959  appendPQExpBufferStr(&sql,
1960  "\n)");
1961  }
1962  if (!opts.no_btree_expansion)
1963  {
1964  /*
1965  * Include a CTE for btree indexes associated with primary heap tables
1966  * selected above, filtering by exclusion patterns (if any) that match
1967  * btree index names.
1968  */
1969  appendPQExpBuffer(&sql,
1970  ", index (oid, nspname, relname, relpages) AS ("
1971  "\nSELECT c.oid, r.nspname, c.relname, c.relpages "
1972  "FROM relation r"
1973  "\nINNER JOIN pg_catalog.pg_index i "
1974  "ON r.oid = i.indrelid "
1975  "INNER JOIN pg_catalog.pg_class c "
1976  "ON i.indexrelid = c.oid");
1977  if (opts.excludeidx || opts.excludensp)
1978  appendPQExpBufferStr(&sql,
1979  "\nINNER JOIN pg_catalog.pg_namespace n "
1980  "ON c.relnamespace = n.oid"
1981  "\nLEFT OUTER JOIN exclude_pat ep "
1982  "ON (n.nspname ~ ep.nsp_regex OR ep.nsp_regex IS NULL) "
1983  "AND (c.relname ~ ep.rel_regex OR ep.rel_regex IS NULL) "
1984  "AND ep.btree_only"
1985  "\nWHERE ep.pattern_id IS NULL");
1986  else
1987  appendPQExpBufferStr(&sql,
1988  "\nWHERE true");
1989  appendPQExpBuffer(&sql,
1990  " AND c.relam = %u "
1991  "AND c.relkind = 'i'",
1992  BTREE_AM_OID);
1993  if (opts.no_toast_expansion)
1994  appendPQExpBuffer(&sql,
1995  " AND c.relnamespace != %u",
1996  PG_TOAST_NAMESPACE);
1997  appendPQExpBufferStr(&sql, "\n)");
1998  }
1999 
2000  if (!opts.no_toast_expansion && !opts.no_btree_expansion)
2001  {
2002  /*
2003  * Include a CTE for btree indexes associated with toast tables of
2004  * primary heap tables selected above, filtering by exclusion patterns
2005  * (if any) that match the toast index names.
2006  */
2007  appendPQExpBuffer(&sql,
2008  ", toast_index (oid, nspname, relname, relpages) AS ("
2009  "\nSELECT c.oid, 'pg_toast', c.relname, c.relpages "
2010  "FROM toast t "
2011  "INNER JOIN pg_catalog.pg_index i "
2012  "ON t.oid = i.indrelid"
2013  "\nINNER JOIN pg_catalog.pg_class c "
2014  "ON i.indexrelid = c.oid");
2015  if (opts.excludeidx)
2016  appendPQExpBufferStr(&sql,
2017  "\nLEFT OUTER JOIN exclude_pat ep "
2018  "ON ('pg_toast' ~ ep.nsp_regex OR ep.nsp_regex IS NULL) "
2019  "AND (c.relname ~ ep.rel_regex OR ep.rel_regex IS NULL) "
2020  "AND ep.btree_only "
2021  "WHERE ep.pattern_id IS NULL");
2022  else
2023  appendPQExpBufferStr(&sql,
2024  "\nWHERE true");
2025  appendPQExpBuffer(&sql,
2026  " AND c.relam = %u"
2027  " AND c.relkind = 'i')",
2028  BTREE_AM_OID);
2029  }
2030 
2031  /*
2032  * Roll-up distinct rows from CTEs.
2033  *
2034  * Relations that match more than one pattern may occur more than once in
2035  * the list, and indexes and toast for primary relations may also have
2036  * matched in their own right, so we rely on UNION to deduplicate the
2037  * list.
2038  */
2039  appendPQExpBuffer(&sql,
2040  "\nSELECT pattern_id, is_heap, is_btree, oid, nspname, relname, relpages "
2041  "FROM (");
2042  appendPQExpBufferStr(&sql,
2043  /* Inclusion patterns that failed to match */
2044  "\nSELECT pattern_id, is_heap, is_btree, "
2045  "NULL::OID AS oid, "
2046  "NULL::TEXT AS nspname, "
2047  "NULL::TEXT AS relname, "
2048  "NULL::INTEGER AS relpages"
2049  "\nFROM relation "
2050  "WHERE pattern_id IS NOT NULL "
2051  "UNION"
2052  /* Primary relations */
2053  "\nSELECT NULL::INTEGER AS pattern_id, "
2054  "is_heap, is_btree, oid, nspname, relname, relpages "
2055  "FROM relation");
2056  if (!opts.no_toast_expansion)
2057  appendPQExpBufferStr(&sql,
2058  " UNION"
2059  /* Toast tables for primary relations */
2060  "\nSELECT NULL::INTEGER AS pattern_id, TRUE AS is_heap, "
2061  "FALSE AS is_btree, oid, nspname, relname, relpages "
2062  "FROM toast");
2063  if (!opts.no_btree_expansion)
2064  appendPQExpBufferStr(&sql,
2065  " UNION"
2066  /* Indexes for primary relations */
2067  "\nSELECT NULL::INTEGER AS pattern_id, FALSE AS is_heap, "
2068  "TRUE AS is_btree, oid, nspname, relname, relpages "
2069  "FROM index");
2070  if (!opts.no_toast_expansion && !opts.no_btree_expansion)
2071  appendPQExpBufferStr(&sql,
2072  " UNION"
2073  /* Indexes for toast relations */
2074  "\nSELECT NULL::INTEGER AS pattern_id, FALSE AS is_heap, "
2075  "TRUE AS is_btree, oid, nspname, relname, relpages "
2076  "FROM toast_index");
2077  appendPQExpBufferStr(&sql,
2078  "\n) AS combined_records "
2079  "ORDER BY relpages DESC NULLS FIRST, oid");
2080 
2081  res = executeQuery(conn, sql.data, opts.echo);
2082  if (PQresultStatus(res) != PGRES_TUPLES_OK)
2083  {
2084  pg_log_error("query failed: %s", PQerrorMessage(conn));
2085  pg_log_info("query was: %s", sql.data);
2086  disconnectDatabase(conn);
2087  exit(1);
2088  }
2089  termPQExpBuffer(&sql);
2090 
2091  ntups = PQntuples(res);
2092  for (i = 0; i < ntups; i++)
2093  {
2094  int pattern_id = -1;
2095  bool is_heap = false;
2096  bool is_btree PG_USED_FOR_ASSERTS_ONLY = false;
2097  Oid oid = InvalidOid;
2098  const char *nspname = NULL;
2099  const char *relname = NULL;
2100  int relpages = 0;
2101 
2102  if (!PQgetisnull(res, i, 0))
2103  pattern_id = atoi(PQgetvalue(res, i, 0));
2104  if (!PQgetisnull(res, i, 1))
2105  is_heap = (PQgetvalue(res, i, 1)[0] == 't');
2106  if (!PQgetisnull(res, i, 2))
2107  is_btree = (PQgetvalue(res, i, 2)[0] == 't');
2108  if (!PQgetisnull(res, i, 3))
2109  oid = atooid(PQgetvalue(res, i, 3));
2110  if (!PQgetisnull(res, i, 4))
2111  nspname = PQgetvalue(res, i, 4);
2112  if (!PQgetisnull(res, i, 5))
2113  relname = PQgetvalue(res, i, 5);
2114  if (!PQgetisnull(res, i, 6))
2115  relpages = atoi(PQgetvalue(res, i, 6));
2116 
2117  if (pattern_id >= 0)
2118  {
2119  /*
2120  * Current record pertains to an inclusion pattern. Record that
2121  * it matched.
2122  */
2123 
2124  if (pattern_id >= opts.include.len)
2125  {
2126  pg_log_error("internal error: received unexpected relation pattern_id %d",
2127  pattern_id);
2128  exit(1);
2129  }
2130 
2131  opts.include.data[pattern_id].matched = true;
2132  }
2133  else
2134  {
2135  /* Current record pertains to a relation */
2136 
2137  RelationInfo *rel = (RelationInfo *) pg_malloc0(sizeof(RelationInfo));
2138 
2139  Assert(OidIsValid(oid));
2140  Assert((is_heap && !is_btree) || (is_btree && !is_heap));
2141 
2142  rel->datinfo = dat;
2143  rel->reloid = oid;
2144  rel->is_heap = is_heap;
2145  rel->nspname = pstrdup(nspname);
2146  rel->relname = pstrdup(relname);
2147  rel->relpages = relpages;
2148  rel->blocks_to_check = relpages;
2149  if (is_heap && (opts.startblock >= 0 || opts.endblock >= 0))
2150  {
2151  /*
2152  * We apply --startblock and --endblock to heap tables, but
2153  * not btree indexes, and for progress purposes we need to
2154  * track how many blocks we expect to check.
2155  */
2156  if (opts.endblock >= 0 && rel->blocks_to_check > opts.endblock)
2157  rel->blocks_to_check = opts.endblock + 1;
2158  if (opts.startblock >= 0)
2159  {
2160  if (rel->blocks_to_check > opts.startblock)
2161  rel->blocks_to_check -= opts.startblock;
2162  else
2163  rel->blocks_to_check = 0;
2164  }
2165  }
2166  *pagecount += rel->blocks_to_check;
2167 
2168  simple_ptr_list_append(relations, rel);
2169  }
2170  }
2171  PQclear(res);
2172 }
bool heapallindexed
Definition: pg_amcheck.c:104
struct DatabaseInfo DatabaseInfo
int64 endblock
Definition: pg_amcheck.c:98
#define VERBOSE_DATNAME_LENGTH
static PGresult * executeQuery(PGconn *conn, const char *query)
Definition: pg_dumpall.c:1874
char * PQerrorMessage(const PGconn *conn)
Definition: fe-connect.c:6735
int64 pg_time_t
Definition: pgtime.h:23
char * pgport
Definition: pg_backup.h:66
char * db_regex
Definition: pg_amcheck.c:34
char * PQgetvalue(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3561
static void compile_relation_list_one_db(PGconn *conn, SimplePtrList *relations, const DatabaseInfo *datinfo, uint64 *pagecount)
Definition: pg_amcheck.c:1838
static bool verify_heap_slot_handler(PGresult *res, PGconn *conn, void *context)
Definition: pg_amcheck.c:1008
static void append_relation_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
Definition: pg_amcheck.c:1430
const char * get_progname(const char *argv0)
Definition: path.c:453
const char * pattern
Definition: pg_amcheck.c:33
void termPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:131
#define pg_log_error(...)
Definition: logging.h:80
int getopt_long(int argc, char *const argv[], const char *optstring, const struct option *longopts, int *longindex)
Definition: getopt_long.c:57
static bool verify_btree_slot_handler(PGresult *res, PGconn *conn, void *context)
Definition: pg_amcheck.c:1089
void appendPQExpBufferStr(PQExpBuffer str, const char *data)
Definition: pqexpbuffer.c:369
static void compile_database_list(PGconn *conn, SimplePtrList *databases, const char *initial_dbname)
Definition: pg_amcheck.c:1535
char * pstrdup(const char *in)
Definition: mcxt.c:1299
void pg_logging_init(const char *argv0)
Definition: logging.c:81
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
static char * indent_lines(const char *str)
Definition: pg_amcheck.c:977
void ParallelSlotsAdoptConn(ParallelSlotArray *sa, PGconn *conn)
char * relname
Definition: pg_amcheck.c:160
char * nspname
Definition: pg_amcheck.c:159
bool matched
Definition: pg_amcheck.c:43
static void ParallelSlotSetHandler(ParallelSlot *slot, ParallelSlotResultHandler handler, void *context)
Definition: parallel_slot.h:47
#define printf(...)
Definition: port.h:222
static void setup_cancel_handler(void)
Definition: parallel.c:613
NameData datname
Definition: pg_database.h:35
char * amcheck_schema
Definition: pg_amcheck.c:151
bool no_toast_expansion
Definition: pg_amcheck.c:94
int pg_strcasecmp(const char *s1, const char *s2)
Definition: pgstrcasecmp.c:36
NameData relname
Definition: pg_class.h:38
unsigned int Oid
Definition: postgres_ext.h:31
bool on_error_stop
Definition: pg_amcheck.c:96
int PQntuples(const PGresult *res)
Definition: fe-exec.c:3167
#define fprintf
Definition: port.h:220
#define OidIsValid(objectId)
Definition: c.h:710
static void append_schema_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
Definition: pg_amcheck.c:1352
void patternToSQLRegex(int encoding, PQExpBuffer dbnamebuf, PQExpBuffer schemabuf, PQExpBuffer namebuf, const char *pattern, bool force_escape)
Definition: string_utils.c:967
ExecStatusType PQresultStatus(const PGresult *res)
Definition: fe-exec.c:3097
static bool should_processing_continue(PGresult *res)
Definition: pg_amcheck.c:936
ParallelSlotArray * ParallelSlotsSetup(int numslots, ConnParams *cparams, const char *progname, bool echo, const char *initcmd)
char * dbname
Definition: pg_backup.h:65
static bool append_db_pattern_cte(PQExpBuffer buf, const PatternInfoArray *pia, PGconn *conn, bool inclusive)
Definition: pg_amcheck.c:1489
ParallelSlot * ParallelSlotsGetIdle(ParallelSlotArray *sa, const char *dbname)
void disconnectDatabase(PGconn *conn)
int PQsendQuery(PGconn *conn, const char *query)
Definition: fe-exec.c:1279
const DatabaseInfo * datinfo
Definition: pg_amcheck.c:156
char * PQescapeIdentifier(PGconn *conn, const char *str, size_t len)
Definition: fe-exec.c:3994
bool install_missing
Definition: pg_amcheck.c:68
#define required_argument
Definition: getopt_long.h:25
void pfree(void *pointer)
Definition: mcxt.c:1169
int optind
Definition: getopt.c:50
bool no_btree_expansion
Definition: pg_amcheck.c:107
int64 startblock
Definition: pg_amcheck.c:97
#define MaxBlockNumber
Definition: block.h:35
static void append_database_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
Definition: pg_amcheck.c:1329
void handle_help_version_opts(int argc, char *argv[], const char *fixed_progname, help_handler hlp)
Definition: option_utils.c:22
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
PGconn * conn
Definition: streamutil.c:54
static void append_btree_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
Definition: pg_amcheck.c:1462
void appendPQExpBuffer(PQExpBuffer str, const char *fmt,...)
Definition: pqexpbuffer.c:267
#define FREE_AND_SET_NULL(x)
Definition: pg_amcheck.c:211
static bool progress_since_last_stderr
Definition: pg_amcheck.c:146
char * sql
Definition: pg_amcheck.c:163
static void executeCommand(PGconn *conn, const char *query)
Definition: pg_dumpall.c:1897
char * c
static char * buf
Definition: pg_test_fsync.c:68
char * pg_strdup(const char *in)
Definition: fe_memutils.c:85
PGconn * connection
Definition: parallel_slot.h:23
PatternInfoArray include
Definition: pg_amcheck.c:72
void * pg_realloc(void *ptr, size_t size)
Definition: fe_memutils.c:65
#define atooid(x)
Definition: postgres_ext.h:42
const char * username
Definition: pgbench.c:280
char * override_dbname
Definition: pg_backup.h:72
static int port
Definition: pg_regress.c:92
static AmcheckOptions opts
Definition: pg_amcheck.c:110
char * datname
Definition: pg_amcheck.c:150
char * pghost
Definition: pg_backup.h:67
static void prepare_btree_command(PQExpBuffer sql, RelationInfo *rel, PGconn *conn)
Definition: pg_amcheck.c:863
enum trivalue prompt_password
Definition: connect_utils.h:32
struct RelationInfo RelationInfo
bool heap_only
Definition: pg_amcheck.c:39
trivalue
Definition: vacuumlo.c:34
#define no_argument
Definition: getopt_long.h:24
#define PG_TEXTDOMAIN(domain)
Definition: c.h:1215
int blocks_to_check
Definition: pg_amcheck.c:162
static void append_relation_pattern_helper(PatternInfoArray *pia, const char *pattern, int encoding, bool heap_only, bool btree_only)
Definition: pg_amcheck.c:1387
static const char * amcheck_sql
Definition: pg_amcheck.c:170
bool ParallelSlotsWaitCompletion(ParallelSlotArray *sa)
static void help(const char *progname)
Definition: pg_amcheck.c:1149
PGconn * connectMaintenanceDatabase(ConnParams *cparams, const char *progname, bool echo)
void appendPQExpBufferChar(PQExpBuffer str, char ch)
Definition: pqexpbuffer.c:380
bool reconcile_toast
Definition: pg_amcheck.c:95
#define PG_DIAG_SEVERITY_NONLOCALIZED
Definition: postgres_ext.h:56
#define InvalidOid
Definition: postgres_ext.h:36
int pg_get_encoding_from_locale(const char *ctype, bool write_message)
Definition: chklocale.c:452
void PQclear(PGresult *res)
Definition: fe-exec.c:680
static void prepare_heap_command(PQExpBuffer sql, RelationInfo *rel, PGconn *conn)
Definition: pg_amcheck.c:830
static void append_heap_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
Definition: pg_amcheck.c:1446
char * PQdb(const PGconn *conn)
Definition: fe-connect.c:6581
static pg_time_t last_progress_report
Definition: pg_amcheck.c:145
struct SimplePtrListCell * next
Definition: simple_list.h:48
char * PQresultErrorField(const PGresult *res, int fieldcode)
Definition: fe-exec.c:3152
#define Assert(condition)
Definition: c.h:804
static void run_command(ParallelSlot *slot, const char *sql)
Definition: pg_amcheck.c:904
bool btree_only
Definition: pg_amcheck.c:41
static PGconn * connectDatabase(const char *dbname, const char *connstr, const char *pghost, const char *pgport, const char *pguser, trivalue prompt_password, bool fail_on_error)
Definition: pg_dumpall.c:1636
static const char * progname
Definition: pg_amcheck.c:139
struct PatternInfo PatternInfo
void pg_logging_increase_verbosity(void)
Definition: logging.c:174
#define optional_argument
Definition: getopt_long.h:26
const char * skip
Definition: pg_amcheck.c:99
#define fatal(...)
int32 encoding
Definition: pg_database.h:41
static void append_rel_pattern_filtered_cte(PQExpBuffer buf, const char *raw, const char *filtered, PGconn *conn)
Definition: pg_amcheck.c:1799
#define INT64_FORMAT
Definition: c.h:483
struct AmcheckOptions AmcheckOptions
PatternInfoArray exclude
Definition: pg_amcheck.c:73
bool show_progress
Definition: pg_amcheck.c:61
void appendStringLiteralConn(PQExpBuffer buf, const char *str, PGconn *conn)
Definition: string_utils.c:293
volatile sig_atomic_t CancelRequested
Definition: cancel.c:52
void set_pglocale_pgservice(const char *argv0, const char *app)
Definition: exec.c:433
char * install_schema
Definition: pg_amcheck.c:69
SimplePtrListCell * head
Definition: simple_list.h:54
char * optarg
Definition: getopt.c:52
void ParallelSlotsTerminate(ParallelSlotArray *sa)
int main(int argc, char *argv[])
Definition: pg_amcheck.c:217
void simple_ptr_list_append(SimplePtrList *list, void *ptr)
Definition: simple_list.c:162
int i
char * rel_regex
Definition: pg_amcheck.c:37
bool strict_names
Definition: pg_amcheck.c:60
static void append_rel_pattern_raw_cte(PQExpBuffer buf, const PatternInfoArray *pia, PGconn *conn)
Definition: pg_amcheck.c:1730
char * nsp_regex
Definition: pg_amcheck.c:36
#define pg_log_warning(...)
Definition: pgfnames.c:24
void resetPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:148
PatternInfo * data
Definition: pg_amcheck.c:48
int PQgetisnull(const PGresult *res, int tup_num, int field_num)
Definition: fe-exec.c:3586
static PatternInfo * extend_pattern_info_array(PatternInfoArray *pia)
Definition: pg_amcheck.c:1307
struct PatternInfoArray PatternInfoArray
#define snprintf
Definition: port.h:216
PGVerbosity PQsetErrorVerbosity(PGconn *conn, PGVerbosity verbosity)
Definition: fe-connect.c:6854
#define _(x)
Definition: elog.c:89
static void progress_report(uint64 relations_total, uint64 relations_checked, uint64 relpages_total, uint64 relpages_checked, const char *datname, bool force, bool finished)
Definition: pg_amcheck.c:1210
Datum now(PG_FUNCTION_ARGS)
Definition: timestamp.c:1544
void initPQExpBuffer(PQExpBuffer str)
Definition: pqexpbuffer.c:92
#define pg_log_info(...)
Definition: logging.h:88
static bool all_checks_pass
Definition: pg_amcheck.c:142
#define PG_USED_FOR_ASSERTS_ONLY
Definition: c.h:155
#define log_no_match(...)
Definition: pg_amcheck.c:204
const char * pguser
Definition: connect_utils.h:31
const char * get_user_name_or_exit(const char *progname)
Definition: username.c:74