PostgreSQL Source Code  git master
pg_checksums.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * pg_checksums.c
4  * Checks, enables or disables page level checksums for an offline
5  * cluster
6  *
7  * Copyright (c) 2010-2019, PostgreSQL Global Development Group
8  *
9  * IDENTIFICATION
10  * src/bin/pg_checksums/pg_checksums.c
11  *
12  *-------------------------------------------------------------------------
13  */
14 
15 #include "postgres_fe.h"
16 
17 #include <dirent.h>
18 #include <time.h>
19 #include <sys/stat.h>
20 #include <unistd.h>
21 
22 #include "access/xlog_internal.h"
24 #include "common/file_perm.h"
25 #include "common/file_utils.h"
26 #include "common/logging.h"
27 #include "getopt_long.h"
28 #include "pg_getopt.h"
29 #include "storage/bufpage.h"
30 #include "storage/checksum.h"
31 #include "storage/checksum_impl.h"
32 
33 
34 static int64 files = 0;
35 static int64 blocks = 0;
36 static int64 badblocks = 0;
38 
39 static char *only_filenode = NULL;
40 static bool do_sync = true;
41 static bool verbose = false;
42 static bool showprogress = false;
43 
44 typedef enum
45 {
50 
51 /*
52  * Filename components.
53  *
54  * XXX: fd.h is not declared here as frontend side code is not able to
55  * interact with the backend-side definitions for the various fsync
56  * wrappers.
57  */
58 #define PG_TEMP_FILES_DIR "pgsql_tmp"
59 #define PG_TEMP_FILE_PREFIX "pgsql_tmp"
60 
62 
63 static const char *progname;
64 
65 /*
66  * Progress status information.
67  */
68 int64 total_size = 0;
69 int64 current_size = 0;
71 
72 static void
73 usage(void)
74 {
75  printf(_("%s enables, disables, or verifies data checksums in a PostgreSQL database cluster.\n\n"), progname);
76  printf(_("Usage:\n"));
77  printf(_(" %s [OPTION]... [DATADIR]\n"), progname);
78  printf(_("\nOptions:\n"));
79  printf(_(" [-D, --pgdata=]DATADIR data directory\n"));
80  printf(_(" -c, --check check data checksums (default)\n"));
81  printf(_(" -d, --disable disable data checksums\n"));
82  printf(_(" -e, --enable enable data checksums\n"));
83  printf(_(" -f, --filenode=FILENODE check only relation with specified filenode\n"));
84  printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
85  printf(_(" -P, --progress show progress information\n"));
86  printf(_(" -v, --verbose output verbose messages\n"));
87  printf(_(" -V, --version output version information, then exit\n"));
88  printf(_(" -?, --help show this help, then exit\n"));
89  printf(_("\nIf no data directory (DATADIR) is specified, "
90  "the environment variable PGDATA\nis used.\n\n"));
91  printf(_("Report bugs to <pgsql-bugs@lists.postgresql.org>.\n"));
92 }
93 
94 /*
95  * List of files excluded from checksum validation.
96  *
97  * Note: this list should be kept in sync with what basebackup.c includes.
98  */
99 static const char *const skip[] = {
100  "pg_control",
101  "pg_filenode.map",
102  "pg_internal.init",
103  "PG_VERSION",
104 #ifdef EXEC_BACKEND
105  "config_exec_params",
106  "config_exec_params.new",
107 #endif
108  NULL,
109 };
110 
111 /*
112  * Report current progress status. Parts borrowed from
113  * src/bin/pg_basebackup/pg_basebackup.c.
114  */
115 static void
116 progress_report(bool force)
117 {
118  int percent;
119  char total_size_str[32];
120  char current_size_str[32];
121  pg_time_t now;
122 
124 
125  now = time(NULL);
126  if (now == last_progress_report && !force)
127  return; /* Max once per second */
128 
129  /* Save current time */
131 
132  /* Adjust total size if current_size is larger */
133  if (current_size > total_size)
135 
136  /* Calculate current percentage of size done */
137  percent = total_size ? (int) ((current_size) * 100 / total_size) : 0;
138 
139  /*
140  * Separate step to keep platform-dependent format code out of
141  * translatable strings. And we only test for INT64_FORMAT availability
142  * in snprintf, not fprintf.
143  */
144  snprintf(total_size_str, sizeof(total_size_str), INT64_FORMAT,
145  total_size / (1024 * 1024));
146  snprintf(current_size_str, sizeof(current_size_str), INT64_FORMAT,
147  current_size / (1024 * 1024));
148 
149  fprintf(stderr, _("%*s/%s MB (%d%%) computed"),
150  (int) strlen(current_size_str), current_size_str, total_size_str,
151  percent);
152 
153  /* Stay on the same line if reporting to a terminal */
154  fprintf(stderr, isatty(fileno(stderr)) ? "\r" : "\n");
155 }
156 
157 static bool
158 skipfile(const char *fn)
159 {
160  const char *const *f;
161 
162  for (f = skip; *f; f++)
163  if (strcmp(*f, fn) == 0)
164  return true;
165 
166  return false;
167 }
168 
169 static void
170 scan_file(const char *fn, BlockNumber segmentno)
171 {
174  int f;
175  BlockNumber blockno;
176  int flags;
177 
179  mode == PG_MODE_CHECK);
180 
181  flags = (mode == PG_MODE_ENABLE) ? O_RDWR : O_RDONLY;
182  f = open(fn, PG_BINARY | flags, 0);
183 
184  if (f < 0)
185  {
186  pg_log_error("could not open file \"%s\": %m", fn);
187  exit(1);
188  }
189 
190  files++;
191 
192  for (blockno = 0;; blockno++)
193  {
194  uint16 csum;
195  int r = read(f, buf.data, BLCKSZ);
196 
197  if (r == 0)
198  break;
199  if (r != BLCKSZ)
200  {
201  pg_log_error("could not read block %u in file \"%s\": read %d of %d",
202  blockno, fn, r, BLCKSZ);
203  exit(1);
204  }
205  blocks++;
206 
207  /* New pages have no checksum yet */
208  if (PageIsNew(header))
209  continue;
210 
211  csum = pg_checksum_page(buf.data, blockno + segmentno * RELSEG_SIZE);
212  current_size += r;
213  if (mode == PG_MODE_CHECK)
214  {
215  if (csum != header->pd_checksum)
216  {
218  pg_log_error("checksum verification failed in file \"%s\", block %u: calculated checksum %X but block contains %X",
219  fn, blockno, csum, header->pd_checksum);
220  badblocks++;
221  }
222  }
223  else if (mode == PG_MODE_ENABLE)
224  {
225  /* Set checksum in page header */
226  header->pd_checksum = csum;
227 
228  /* Seek back to beginning of block */
229  if (lseek(f, -BLCKSZ, SEEK_CUR) < 0)
230  {
231  pg_log_error("seek failed for block %u in file \"%s\": %m", blockno, fn);
232  exit(1);
233  }
234 
235  /* Write block with checksum */
236  if (write(f, buf.data, BLCKSZ) != BLCKSZ)
237  {
238  pg_log_error("could not write block %u in file \"%s\": %m",
239  blockno, fn);
240  exit(1);
241  }
242  }
243 
244  if (showprogress)
245  progress_report(false);
246  }
247 
248  if (verbose)
249  {
250  if (mode == PG_MODE_CHECK)
251  pg_log_info("checksums verified in file \"%s\"", fn);
252  if (mode == PG_MODE_ENABLE)
253  pg_log_info("checksums enabled in file \"%s\"", fn);
254  }
255 
256  close(f);
257 }
258 
259 /*
260  * Scan the given directory for items which can be checksummed and
261  * operate on each one of them. If "sizeonly" is true, the size of
262  * all the items which have checksums is computed and returned back
263  * to the caller without operating on the files. This is used to compile
264  * the total size of the data directory for progress reports.
265  */
266 static int64
267 scan_directory(const char *basedir, const char *subdir, bool sizeonly)
268 {
269  int64 dirsize = 0;
270  char path[MAXPGPATH];
271  DIR *dir;
272  struct dirent *de;
273 
274  snprintf(path, sizeof(path), "%s/%s", basedir, subdir);
275  dir = opendir(path);
276  if (!dir)
277  {
278  pg_log_error("could not open directory \"%s\": %m", path);
279  exit(1);
280  }
281  while ((de = readdir(dir)) != NULL)
282  {
283  char fn[MAXPGPATH];
284  struct stat st;
285 
286  if (strcmp(de->d_name, ".") == 0 ||
287  strcmp(de->d_name, "..") == 0)
288  continue;
289 
290  /* Skip temporary files */
291  if (strncmp(de->d_name,
293  strlen(PG_TEMP_FILE_PREFIX)) == 0)
294  continue;
295 
296  /* Skip temporary folders */
297  if (strncmp(de->d_name,
299  strlen(PG_TEMP_FILES_DIR)) == 0)
300  continue;
301 
302  snprintf(fn, sizeof(fn), "%s/%s", path, de->d_name);
303  if (lstat(fn, &st) < 0)
304  {
305  pg_log_error("could not stat file \"%s\": %m", fn);
306  exit(1);
307  }
308  if (S_ISREG(st.st_mode))
309  {
310  char fnonly[MAXPGPATH];
311  char *forkpath,
312  *segmentpath;
313  BlockNumber segmentno = 0;
314 
315  if (skipfile(de->d_name))
316  continue;
317 
318  /*
319  * Cut off at the segment boundary (".") to get the segment number
320  * in order to mix it into the checksum. Then also cut off at the
321  * fork boundary, to get the filenode the file belongs to for
322  * filtering.
323  */
324  strlcpy(fnonly, de->d_name, sizeof(fnonly));
325  segmentpath = strchr(fnonly, '.');
326  if (segmentpath != NULL)
327  {
328  *segmentpath++ = '\0';
329  segmentno = atoi(segmentpath);
330  if (segmentno == 0)
331  {
332  pg_log_error("invalid segment number %d in file name \"%s\"",
333  segmentno, fn);
334  exit(1);
335  }
336  }
337 
338  forkpath = strchr(fnonly, '_');
339  if (forkpath != NULL)
340  *forkpath++ = '\0';
341 
342  if (only_filenode && strcmp(only_filenode, fnonly) != 0)
343  /* filenode not to be included */
344  continue;
345 
346  dirsize += st.st_size;
347 
348  /*
349  * No need to work on the file when calculating only the size of
350  * the items in the data folder.
351  */
352  if (!sizeonly)
353  scan_file(fn, segmentno);
354  }
355 #ifndef WIN32
356  else if (S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))
357 #else
358  else if (S_ISDIR(st.st_mode) || pgwin32_is_junction(fn))
359 #endif
360  dirsize += scan_directory(path, de->d_name, sizeonly);
361  }
362  closedir(dir);
363  return dirsize;
364 }
365 
366 int
367 main(int argc, char *argv[])
368 {
369  static struct option long_options[] = {
370  {"check", no_argument, NULL, 'c'},
371  {"pgdata", required_argument, NULL, 'D'},
372  {"disable", no_argument, NULL, 'd'},
373  {"enable", no_argument, NULL, 'e'},
374  {"filenode", required_argument, NULL, 'f'},
375  {"no-sync", no_argument, NULL, 'N'},
376  {"progress", no_argument, NULL, 'P'},
377  {"verbose", no_argument, NULL, 'v'},
378  {NULL, 0, NULL, 0}
379  };
380 
381  char *DataDir = NULL;
382  int c;
383  int option_index;
384  bool crc_ok;
385 
386  pg_logging_init(argv[0]);
387  set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_checksums"));
388  progname = get_progname(argv[0]);
389 
390  if (argc > 1)
391  {
392  if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
393  {
394  usage();
395  exit(0);
396  }
397  if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
398  {
399  puts("pg_checksums (PostgreSQL) " PG_VERSION);
400  exit(0);
401  }
402  }
403 
404  while ((c = getopt_long(argc, argv, "cD:deNPf:v", long_options, &option_index)) != -1)
405  {
406  switch (c)
407  {
408  case 'c':
410  break;
411  case 'd':
413  break;
414  case 'e':
416  break;
417  case 'f':
418  if (atoi(optarg) == 0)
419  {
420  pg_log_error("invalid filenode specification, must be numeric: %s", optarg);
421  exit(1);
422  }
424  break;
425  case 'N':
426  do_sync = false;
427  break;
428  case 'v':
429  verbose = true;
430  break;
431  case 'D':
432  DataDir = optarg;
433  break;
434  case 'P':
435  showprogress = true;
436  break;
437  default:
438  fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
439  exit(1);
440  }
441  }
442 
443  if (DataDir == NULL)
444  {
445  if (optind < argc)
446  DataDir = argv[optind++];
447  else
448  DataDir = getenv("PGDATA");
449 
450  /* If no DataDir was specified, and none could be found, error out */
451  if (DataDir == NULL)
452  {
453  pg_log_error("no data directory specified");
454  fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
455  exit(1);
456  }
457  }
458 
459  /* Complain if any arguments remain */
460  if (optind < argc)
461  {
462  pg_log_error("too many command-line arguments (first is \"%s\")",
463  argv[optind]);
464  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
465  progname);
466  exit(1);
467  }
468 
469  /* filenode checking only works in --check mode */
470  if (mode != PG_MODE_CHECK && only_filenode)
471  {
472  pg_log_error("option -f/--filenode can only be used with --check");
473  fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
474  progname);
475  exit(1);
476  }
477 
478  /* Read the control file and check compatibility */
479  ControlFile = get_controlfile(DataDir, &crc_ok);
480  if (!crc_ok)
481  {
482  pg_log_error("pg_control CRC value is incorrect");
483  exit(1);
484  }
485 
486  if (ControlFile->pg_control_version != PG_CONTROL_VERSION)
487  {
488  pg_log_error("cluster is not compatible with this version of pg_checksums");
489  exit(1);
490  }
491 
492  if (ControlFile->blcksz != BLCKSZ)
493  {
494  pg_log_error("database cluster is not compatible");
495  fprintf(stderr, _("The database cluster was initialized with block size %u, but pg_checksums was compiled with block size %u.\n"),
496  ControlFile->blcksz, BLCKSZ);
497  exit(1);
498  }
499 
500  /*
501  * Check if cluster is running. A clean shutdown is required to avoid
502  * random checksum failures caused by torn pages. Note that this doesn't
503  * guard against someone starting the cluster concurrently.
504  */
505  if (ControlFile->state != DB_SHUTDOWNED &&
506  ControlFile->state != DB_SHUTDOWNED_IN_RECOVERY)
507  {
508  pg_log_error("cluster must be shut down");
509  exit(1);
510  }
511 
512  if (ControlFile->data_checksum_version == 0 &&
513  mode == PG_MODE_CHECK)
514  {
515  pg_log_error("data checksums are not enabled in cluster");
516  exit(1);
517  }
518 
519  if (ControlFile->data_checksum_version == 0 &&
521  {
522  pg_log_error("data checksums are already disabled in cluster");
523  exit(1);
524  }
525 
526  if (ControlFile->data_checksum_version > 0 &&
527  mode == PG_MODE_ENABLE)
528  {
529  pg_log_error("data checksums are already enabled in cluster");
530  exit(1);
531  }
532 
533  /* Operate on all files if checking or enabling checksums */
534  if (mode == PG_MODE_CHECK || mode == PG_MODE_ENABLE)
535  {
536  /*
537  * If progress status information is requested, we need to scan the
538  * directory tree twice: once to know how much total data needs to be
539  * processed and once to do the real work.
540  */
541  if (showprogress)
542  {
543  total_size = scan_directory(DataDir, "global", true);
544  total_size += scan_directory(DataDir, "base", true);
545  total_size += scan_directory(DataDir, "pg_tblspc", true);
546  }
547 
548  (void) scan_directory(DataDir, "global", false);
549  (void) scan_directory(DataDir, "base", false);
550  (void) scan_directory(DataDir, "pg_tblspc", false);
551 
552  if (showprogress)
553  {
554  progress_report(true);
555  fprintf(stderr, "\n"); /* Need to move to next line */
556  }
557 
558  printf(_("Checksum operation completed\n"));
559  printf(_("Files scanned: %s\n"), psprintf(INT64_FORMAT, files));
560  printf(_("Blocks scanned: %s\n"), psprintf(INT64_FORMAT, blocks));
561  if (mode == PG_MODE_CHECK)
562  {
563  printf(_("Bad checksums: %s\n"), psprintf(INT64_FORMAT, badblocks));
564  printf(_("Data checksum version: %d\n"), ControlFile->data_checksum_version);
565 
566  if (badblocks > 0)
567  exit(1);
568  }
569  }
570 
571  /*
572  * Finally make the data durable on disk if enabling or disabling
573  * checksums. Flush first the data directory for safety, and then update
574  * the control file to keep the switch consistent.
575  */
577  {
578  ControlFile->data_checksum_version =
580 
581  if (do_sync)
582  {
583  pg_log_info("syncing data directory");
584  fsync_pgdata(DataDir, PG_VERSION_NUM);
585  }
586 
587  pg_log_info("updating control file");
588  update_controlfile(DataDir, ControlFile, do_sync);
589 
590  if (verbose)
591  printf(_("Data checksum version: %d\n"), ControlFile->data_checksum_version);
592  if (mode == PG_MODE_ENABLE)
593  printf(_("Checksums enabled in cluster\n"));
594  else
595  printf(_("Checksums disabled in cluster\n"));
596  }
597 
598  return 0;
599 }
static PgChecksumMode mode
Definition: pg_checksums.c:61
int64 pg_time_t
Definition: pgtime.h:23
static const char * progname
Definition: pg_checksums.c:63
#define write(a, b, c)
Definition: win32.h:14
const char * get_progname(const char *argv0)
Definition: path.c:453
#define pg_log_error(...)
Definition: logging.h:79
int getopt_long(int argc, char *const argv[], const char *optstring, const struct option *longopts, int *longindex)
Definition: getopt_long.c:57
static void usage(void)
Definition: pg_checksums.c:73
char * pstrdup(const char *in)
Definition: mcxt.c:1161
void pg_logging_init(const char *argv0)
Definition: logging.c:39
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
static ControlFileData * ControlFile
Definition: pg_checksums.c:37
int closedir(DIR *)
Definition: dirent.c:113
#define PG_TEMP_FILES_DIR
Definition: pg_checksums.c:58
#define PG_CONTROL_VERSION
Definition: pg_control.h:25
static void progress_report(bool force)
Definition: pg_checksums.c:116
uint32 pg_control_version
Definition: pg_control.h:121
#define printf(...)
Definition: port.h:198
uint32 BlockNumber
Definition: block.h:31
static int64 files
Definition: pg_checksums.c:34
static bool showprogress
Definition: pg_checksums.c:42
#define fprintf
Definition: port.h:196
Definition: dirent.h:9
PgChecksumMode
Definition: pg_checksums.c:44
#define PG_BINARY
Definition: c.h:1191
static char * basedir
Definition: pg_basebackup.c:86
int main(int argc, char *argv[])
Definition: pg_checksums.c:367
static const char *const skip[]
Definition: pg_checksums.c:99
void fsync_pgdata(const char *pg_data, int serverVersion)
Definition: file_utils.c:58
uint16 pd_checksum
Definition: bufpage.h:156
char data[BLCKSZ]
Definition: c.h:1060
void update_controlfile(const char *DataDir, ControlFileData *ControlFile, bool do_sync)
#define required_argument
Definition: getopt_long.h:25
unsigned short uint16
Definition: c.h:357
int optind
Definition: getopt.c:50
Definition: dirent.c:25
#define PG_TEMP_FILE_PREFIX
Definition: pg_checksums.c:59
#define MAXPGPATH
DIR * opendir(const char *)
Definition: dirent.c:33
uint32 data_checksum_version
Definition: pg_control.h:222
char * c
static char * buf
Definition: pg_test_fsync.c:68
static bool do_sync
Definition: pg_checksums.c:40
static int64 scan_directory(const char *basedir, const char *subdir, bool sizeonly)
Definition: pg_checksums.c:267
static bool verbose
Definition: pg_checksums.c:41
#define S_ISREG(m)
Definition: win32_port.h:308
#define stat(a, b)
Definition: win32_port.h:264
#define no_argument
Definition: getopt_long.h:24
#define PG_TEXTDOMAIN(domain)
Definition: c.h:1135
int64 total_size
Definition: pg_checksums.c:68
static void * fn(void *arg)
size_t strlcpy(char *dst, const char *src, size_t siz)
Definition: strlcpy.c:45
PageHeaderData * PageHeader
Definition: bufpage.h:166
#define Assert(condition)
Definition: c.h:732
static pg_time_t last_progress_report
Definition: pg_checksums.c:70
ControlFileData * get_controlfile(const char *DataDir, bool *crc_ok_p)
static void scan_file(const char *fn, BlockNumber segmentno)
Definition: pg_checksums.c:170
struct dirent * readdir(DIR *)
Definition: dirent.c:77
static void header(const char *fmt,...) pg_attribute_printf(1
Definition: pg_regress.c:210
static int64 blocks
Definition: pg_checksums.c:35
#define INT64_FORMAT
Definition: c.h:400
#define S_ISDIR(m)
Definition: win32_port.h:305
#define lstat(path, sb)
Definition: win32_port.h:253
static char * only_filenode
Definition: pg_checksums.c:39
static int64 badblocks
Definition: pg_checksums.c:36
#define PG_DATA_CHECKSUM_VERSION
Definition: bufpage.h:200
#define PageIsNew(page)
Definition: bufpage.h:229
void set_pglocale_pgservice(const char *argv0, const char *app)
Definition: exec.c:565
char * optarg
Definition: getopt.c:52
char * DataDir
Definition: globals.c:62
int64 current_size
Definition: pg_checksums.c:69
char d_name[MAX_PATH]
Definition: dirent.h:14
#define close(a)
Definition: win32.h:12
#define snprintf
Definition: port.h:192
#define _(x)
Definition: elog.c:84
uint16 pg_checksum_page(char *page, BlockNumber blkno)
Datum now(PG_FUNCTION_ARGS)
Definition: timestamp.c:1533
#define read(a, b, c)
Definition: win32.h:13
static bool skipfile(const char *fn)
Definition: pg_checksums.c:158
#define pg_log_info(...)
Definition: logging.h:87
bool pgwin32_is_junction(const char *path)