PostgreSQL Source Code  git master
reconstruct.c File Reference
#include "postgres_fe.h"
#include <unistd.h>
#include "backup/basebackup_incremental.h"
#include "common/file_perm.h"
#include "common/logging.h"
#include "copy_file.h"
#include "lib/stringinfo.h"
#include "reconstruct.h"
#include "storage/block.h"
Include dependency graph for reconstruct.c:

Go to the source code of this file.

Data Structures

struct  rfile
 

Typedefs

typedef struct rfile rfile
 

Functions

static void debug_reconstruction (int n_source, rfile **sources, bool dry_run)
 
static unsigned find_reconstructed_block_length (rfile *s)
 
static rfilemake_incremental_rfile (char *filename)
 
static rfilemake_rfile (char *filename, bool missing_ok)
 
static void write_reconstructed_file (char *input_filename, char *output_filename, unsigned block_length, rfile **sourcemap, off_t *offsetmap, pg_checksum_context *checksum_ctx, CopyMethod copy_method, bool debug, bool dry_run)
 
static void read_bytes (rfile *rf, void *buffer, unsigned length)
 
static void write_block (int wfd, char *output_filename, uint8 *buffer, pg_checksum_context *checksum_ctx)
 
static void read_block (rfile *s, off_t off, uint8 *buffer)
 
void reconstruct_from_incremental_file (char *input_filename, char *output_filename, char *relative_path, char *bare_file_name, int n_prior_backups, char **prior_backup_dirs, manifest_data **manifests, char *manifest_path, pg_checksum_type checksum_type, int *checksum_length, uint8 **checksum_payload, CopyMethod copy_method, bool debug, bool dry_run)
 

Typedef Documentation

◆ rfile

typedef struct rfile rfile

Function Documentation

◆ debug_reconstruction()

static void debug_reconstruction ( int  n_source,
rfile **  sources,
bool  dry_run 
)
static

Definition at line 371 of file reconstruct.c.

372 {
373  unsigned i;
374 
375  for (i = 0; i < n_source; ++i)
376  {
377  rfile *s = sources[i];
378 
379  /* Ignore source if not used. */
380  if (s == NULL)
381  continue;
382 
383  /* If no data is needed from this file, we can ignore it. */
384  if (s->num_blocks_read == 0)
385  continue;
386 
387  /* Debug logging. */
388  if (dry_run)
389  pg_log_debug("would have read %u blocks from \"%s\"",
390  s->num_blocks_read, s->filename);
391  else
392  pg_log_debug("read %u blocks from \"%s\"",
393  s->num_blocks_read, s->filename);
394 
395  /*
396  * In dry-run mode, we don't actually try to read data from the file,
397  * but we do try to verify that the file is long enough that we could
398  * have read the data if we'd tried.
399  *
400  * If this fails, then it means that a non-dry-run attempt would fail,
401  * complaining of not being able to read the required bytes from the
402  * file.
403  */
404  if (dry_run)
405  {
406  struct stat sb;
407 
408  if (fstat(s->fd, &sb) < 0)
409  pg_fatal("could not stat \"%s\": %m", s->filename);
410  if (sb.st_size < s->highest_offset_read)
411  pg_fatal("file \"%s\" is too short: expected %llu, found %llu",
412  s->filename,
413  (unsigned long long) s->highest_offset_read,
414  (unsigned long long) sb.st_size);
415  }
416  }
417 }
int i
Definition: isn.c:73
#define pg_log_debug(...)
Definition: logging.h:133
#define pg_fatal(...)
static bool dry_run
off_t highest_offset_read
Definition: reconstruct.c:46
int fd
Definition: reconstruct.c:40
unsigned num_blocks_read
Definition: reconstruct.c:45
char * filename
Definition: reconstruct.c:39
#define fstat
Definition: win32_port.h:283

References dry_run, rfile::fd, rfile::filename, fstat, rfile::highest_offset_read, i, rfile::num_blocks_read, pg_fatal, pg_log_debug, and stat::st_size.

Referenced by reconstruct_from_incremental_file().

◆ find_reconstructed_block_length()

static unsigned find_reconstructed_block_length ( rfile s)
static

Definition at line 426 of file reconstruct.c.

427 {
428  unsigned block_length = s->truncation_block_length;
429  unsigned i;
430 
431  for (i = 0; i < s->num_blocks; ++i)
432  if (s->relative_block_numbers[i] >= block_length)
433  block_length = s->relative_block_numbers[i] + 1;
434 
435  return block_length;
436 }
BlockNumber * relative_block_numbers
Definition: reconstruct.c:43
unsigned num_blocks
Definition: reconstruct.c:42
unsigned truncation_block_length
Definition: reconstruct.c:44

References i, rfile::num_blocks, rfile::relative_block_numbers, and rfile::truncation_block_length.

Referenced by reconstruct_from_incremental_file().

◆ make_incremental_rfile()

static rfile * make_incremental_rfile ( char *  filename)
static

Definition at line 443 of file reconstruct.c.

444 {
445  rfile *rf;
446  unsigned magic;
447 
448  rf = make_rfile(filename, false);
449 
450  /* Read and validate magic number. */
451  read_bytes(rf, &magic, sizeof(magic));
452  if (magic != INCREMENTAL_MAGIC)
453  pg_fatal("file \"%s\" has bad incremental magic number (0x%x not 0x%x)",
454  filename, magic, INCREMENTAL_MAGIC);
455 
456  /* Read block count. */
457  read_bytes(rf, &rf->num_blocks, sizeof(rf->num_blocks));
458  if (rf->num_blocks > RELSEG_SIZE)
459  pg_fatal("file \"%s\" has block count %u in excess of segment size %u",
460  filename, rf->num_blocks, RELSEG_SIZE);
461 
462  /* Read truncation block length. */
464  sizeof(rf->truncation_block_length));
465  if (rf->truncation_block_length > RELSEG_SIZE)
466  pg_fatal("file \"%s\" has truncation block length %u in excess of segment size %u",
467  filename, rf->truncation_block_length, RELSEG_SIZE);
468 
469  /* Read block numbers if there are any. */
470  if (rf->num_blocks > 0)
471  {
473  pg_malloc0(sizeof(BlockNumber) * rf->num_blocks);
475  sizeof(BlockNumber) * rf->num_blocks);
476  }
477 
478  /* Remember length of header. */
479  rf->header_length = sizeof(magic) + sizeof(rf->num_blocks) +
480  sizeof(rf->truncation_block_length) +
481  sizeof(BlockNumber) * rf->num_blocks;
482 
483  /*
484  * Round header length to a multiple of BLCKSZ, so that blocks contents
485  * are properly aligned. Only do this when the file actually has data for
486  * some blocks.
487  */
488  if ((rf->num_blocks > 0) && ((rf->header_length % BLCKSZ) != 0))
489  rf->header_length += (BLCKSZ - (rf->header_length % BLCKSZ));
490 
491  return rf;
492 }
#define INCREMENTAL_MAGIC
uint32 BlockNumber
Definition: block.h:31
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
static char * filename
Definition: pg_dumpall.c:119
static rfile * make_rfile(char *filename, bool missing_ok)
Definition: reconstruct.c:498
static void read_bytes(rfile *rf, void *buffer, unsigned length)
Definition: reconstruct.c:521
size_t header_length
Definition: reconstruct.c:41

References filename, rfile::header_length, INCREMENTAL_MAGIC, make_rfile(), rfile::num_blocks, pg_fatal, pg_malloc0(), read_bytes(), rfile::relative_block_numbers, and rfile::truncation_block_length.

Referenced by reconstruct_from_incremental_file().

◆ make_rfile()

static rfile * make_rfile ( char *  filename,
bool  missing_ok 
)
static

Definition at line 498 of file reconstruct.c.

499 {
500  rfile *rf;
501 
502  rf = pg_malloc0(sizeof(rfile));
503  rf->filename = pstrdup(filename);
504  if ((rf->fd = open(filename, O_RDONLY | PG_BINARY, 0)) < 0)
505  {
506  if (missing_ok && errno == ENOENT)
507  {
508  pg_free(rf);
509  return NULL;
510  }
511  pg_fatal("could not open file \"%s\": %m", filename);
512  }
513 
514  return rf;
515 }
#define PG_BINARY
Definition: c.h:1273
void pg_free(void *ptr)
Definition: fe_memutils.c:105
char * pstrdup(const char *in)
Definition: mcxt.c:1695

References rfile::fd, rfile::filename, filename, PG_BINARY, pg_fatal, pg_free(), pg_malloc0(), and pstrdup().

Referenced by make_incremental_rfile(), and reconstruct_from_incremental_file().

◆ read_block()

static void read_block ( rfile s,
off_t  off,
uint8 buffer 
)
static

Definition at line 763 of file reconstruct.c.

764 {
765  int rb;
766 
767  /* Read the block from the correct source, except if dry-run. */
768  rb = pg_pread(s->fd, buffer, BLCKSZ, off);
769  if (rb != BLCKSZ)
770  {
771  if (rb < 0)
772  pg_fatal("could not read file \"%s\": %m", s->filename);
773  else
774  pg_fatal("could not read file \"%s\": read only %d of %d bytes at offset %llu",
775  s->filename, rb, BLCKSZ,
776  (unsigned long long) off);
777  }
778 }
#define pg_pread
Definition: port.h:225

References rfile::fd, rfile::filename, pg_fatal, and pg_pread.

Referenced by write_reconstructed_file().

◆ read_bytes()

static void read_bytes ( rfile rf,
void *  buffer,
unsigned  length 
)
static

Definition at line 521 of file reconstruct.c.

522 {
523  int rb = read(rf->fd, buffer, length);
524 
525  if (rb != length)
526  {
527  if (rb < 0)
528  pg_fatal("could not read file \"%s\": %m", rf->filename);
529  else
530  pg_fatal("could not read file \"%s\": read only %d of %u bytes",
531  rf->filename, rb, length);
532  }
533 }
#define read(a, b, c)
Definition: win32.h:13

References rfile::fd, rfile::filename, pg_fatal, and read.

Referenced by make_incremental_rfile().

◆ reconstruct_from_incremental_file()

void reconstruct_from_incremental_file ( char *  input_filename,
char *  output_filename,
char *  relative_path,
char *  bare_file_name,
int  n_prior_backups,
char **  prior_backup_dirs,
manifest_data **  manifests,
char *  manifest_path,
pg_checksum_type  checksum_type,
int *  checksum_length,
uint8 **  checksum_payload,
CopyMethod  copy_method,
bool  debug,
bool  dry_run 
)

Definition at line 87 of file reconstruct.c.

101 {
102  rfile **source;
103  rfile *latest_source = NULL;
104  rfile **sourcemap;
105  off_t *offsetmap;
106  unsigned block_length;
107  unsigned i;
108  unsigned sidx = n_prior_backups;
109  bool full_copy_possible = true;
110  int copy_source_index = -1;
111  rfile *copy_source = NULL;
112  pg_checksum_context checksum_ctx;
113 
114  /*
115  * Every block must come either from the latest version of the file or
116  * from one of the prior backups.
117  */
118  source = pg_malloc0(sizeof(rfile *) * (1 + n_prior_backups));
119 
120  /*
121  * Use the information from the latest incremental file to figure out how
122  * long the reconstructed file should be.
123  */
124  latest_source = make_incremental_rfile(input_filename);
125  source[n_prior_backups] = latest_source;
126  block_length = find_reconstructed_block_length(latest_source);
127 
128  /*
129  * For each block in the output file, we need to know from which file we
130  * need to obtain it and at what offset in that file it's stored.
131  * sourcemap gives us the first of these things, and offsetmap the latter.
132  */
133  sourcemap = pg_malloc0(sizeof(rfile *) * block_length);
134  offsetmap = pg_malloc0(sizeof(off_t) * block_length);
135 
136  /*
137  * Every block that is present in the newest incremental file should be
138  * sourced from that file. If it precedes the truncation_block_length,
139  * it's a block that we would otherwise have had to find in an older
140  * backup and thus reduces the number of blocks remaining to be found by
141  * one; otherwise, it's an extra block that needs to be included in the
142  * output but would not have needed to be found in an older backup if it
143  * had not been present.
144  */
145  for (i = 0; i < latest_source->num_blocks; ++i)
146  {
147  BlockNumber b = latest_source->relative_block_numbers[i];
148 
149  Assert(b < block_length);
150  sourcemap[b] = latest_source;
151  offsetmap[b] = latest_source->header_length + (i * BLCKSZ);
152 
153  /*
154  * A full copy of a file from an earlier backup is only possible if no
155  * blocks are needed from any later incremental file.
156  */
157  full_copy_possible = false;
158  }
159 
160  while (1)
161  {
162  char source_filename[MAXPGPATH];
163  rfile *s;
164 
165  /*
166  * Move to the next backup in the chain. If there are no more, then
167  * we're done.
168  */
169  if (sidx == 0)
170  break;
171  --sidx;
172 
173  /*
174  * Look for the full file in the previous backup. If not found, then
175  * look for an incremental file instead.
176  */
177  snprintf(source_filename, MAXPGPATH, "%s/%s/%s",
178  prior_backup_dirs[sidx], relative_path, bare_file_name);
179  if ((s = make_rfile(source_filename, true)) == NULL)
180  {
181  snprintf(source_filename, MAXPGPATH, "%s/%s/INCREMENTAL.%s",
182  prior_backup_dirs[sidx], relative_path, bare_file_name);
183  s = make_incremental_rfile(source_filename);
184  }
185  source[sidx] = s;
186 
187  /*
188  * If s->header_length == 0, then this is a full file; otherwise, it's
189  * an incremental file.
190  */
191  if (s->header_length == 0)
192  {
193  struct stat sb;
194  BlockNumber b;
195  BlockNumber blocklength;
196 
197  /* We need to know the length of the file. */
198  if (fstat(s->fd, &sb) < 0)
199  pg_fatal("could not stat \"%s\": %m", s->filename);
200 
201  /*
202  * Since we found a full file, source all blocks from it that
203  * exist in the file.
204  *
205  * Note that there may be blocks that don't exist either in this
206  * file or in any incremental file but that precede
207  * truncation_block_length. These are, presumably, zero-filled
208  * blocks that result from the server extending the file but
209  * taking no action on those blocks that generated any WAL.
210  *
211  * Sadly, we have no way of validating that this is really what
212  * happened, and neither does the server. From it's perspective,
213  * an unmodified block that contains data looks exactly the same
214  * as a zero-filled block that never had any data: either way,
215  * it's not mentioned in any WAL summary and the server has no
216  * reason to read it. From our perspective, all we know is that
217  * nobody had a reason to back up the block. That certainly means
218  * that the block didn't exist at the time of the full backup, but
219  * the supposition that it was all zeroes at the time of every
220  * later backup is one that we can't validate.
221  */
222  blocklength = sb.st_size / BLCKSZ;
223  for (b = 0; b < latest_source->truncation_block_length; ++b)
224  {
225  if (sourcemap[b] == NULL && b < blocklength)
226  {
227  sourcemap[b] = s;
228  offsetmap[b] = b * BLCKSZ;
229  }
230  }
231 
232  /*
233  * If a full copy looks possible, check whether the resulting file
234  * should be exactly as long as the source file is. If so, a full
235  * copy is acceptable, otherwise not.
236  */
237  if (full_copy_possible)
238  {
239  uint64 expected_length;
240 
241  expected_length =
242  (uint64) latest_source->truncation_block_length;
243  expected_length *= BLCKSZ;
244  if (expected_length == sb.st_size)
245  {
246  copy_source = s;
247  copy_source_index = sidx;
248  }
249  }
250 
251  /* We don't need to consider any further sources. */
252  break;
253  }
254 
255  /*
256  * Since we found another incremental file, source all blocks from it
257  * that we need but don't yet have.
258  */
259  for (i = 0; i < s->num_blocks; ++i)
260  {
262 
263  if (b < latest_source->truncation_block_length &&
264  sourcemap[b] == NULL)
265  {
266  sourcemap[b] = s;
267  offsetmap[b] = s->header_length + (i * BLCKSZ);
268 
269  /*
270  * A full copy of a file from an earlier backup is only
271  * possible if no blocks are needed from any later incremental
272  * file.
273  */
274  full_copy_possible = false;
275  }
276  }
277  }
278 
279  /*
280  * If a checksum of the required type already exists in the
281  * backup_manifest for the relevant input directory, we can save some work
282  * by reusing that checksum instead of computing a new one.
283  */
284  if (copy_source_index >= 0 && manifests[copy_source_index] != NULL &&
285  checksum_type != CHECKSUM_TYPE_NONE)
286  {
287  manifest_file *mfile;
288 
289  mfile = manifest_files_lookup(manifests[copy_source_index]->files,
290  manifest_path);
291  if (mfile == NULL)
292  {
293  char *path = psprintf("%s/backup_manifest",
294  prior_backup_dirs[copy_source_index]);
295 
296  /*
297  * The directory is out of sync with the backup_manifest, so emit
298  * a warning.
299  */
300  /*- translator: the first %s is a backup manifest file, the second is a file absent therein */
301  pg_log_warning("\"%s\" contains no entry for \"%s\"",
302  path,
303  manifest_path);
304  pfree(path);
305  }
306  else if (mfile->checksum_type == checksum_type)
307  {
308  *checksum_length = mfile->checksum_length;
309  *checksum_payload = pg_malloc(*checksum_length);
310  memcpy(*checksum_payload, mfile->checksum_payload,
311  *checksum_length);
312  checksum_type = CHECKSUM_TYPE_NONE;
313  }
314  }
315 
316  /* Prepare for checksum calculation, if required. */
317  pg_checksum_init(&checksum_ctx, checksum_type);
318 
319  /*
320  * If the full file can be created by copying a file from an older backup
321  * in the chain without needing to overwrite any blocks or truncate the
322  * result, then forget about performing reconstruction and just copy that
323  * file in its entirety.
324  *
325  * Otherwise, reconstruct.
326  */
327  if (copy_source != NULL)
328  copy_file(copy_source->filename, output_filename,
329  &checksum_ctx, copy_method, dry_run);
330  else
331  {
333  block_length, sourcemap, offsetmap,
334  &checksum_ctx, copy_method,
335  debug, dry_run);
336  debug_reconstruction(n_prior_backups + 1, source, dry_run);
337  }
338 
339  /* Save results of checksum calculation. */
340  if (checksum_type != CHECKSUM_TYPE_NONE)
341  {
342  *checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
343  *checksum_length = pg_checksum_final(&checksum_ctx,
344  *checksum_payload);
345  }
346 
347  /*
348  * Close files and release memory.
349  */
350  for (i = 0; i <= n_prior_backups; ++i)
351  {
352  rfile *s = source[i];
353 
354  if (s == NULL)
355  continue;
356  if (close(s->fd) != 0)
357  pg_fatal("could not close \"%s\": %m", s->filename);
358  if (s->relative_block_numbers != NULL)
360  pg_free(s->filename);
361  }
362  pfree(sourcemap);
363  pfree(offsetmap);
364  pfree(source);
365 }
#define Assert(condition)
Definition: c.h:858
int pg_checksum_final(pg_checksum_context *context, uint8 *output)
int pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
#define PG_CHECKSUM_MAX_LENGTH
@ CHECKSUM_TYPE_NONE
void copy_file(const char *fromfile, const char *tofile)
Definition: copydir.c:117
char * output_filename
Definition: ecpg.c:23
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
static bool debug
Definition: initdb.c:161
#define close(a)
Definition: win32.h:12
int b
Definition: isn.c:70
if(TABLE==NULL||TABLE_index==NULL)
Definition: isn.c:77
void pfree(void *pointer)
Definition: mcxt.c:1520
#define MAXPGPATH
static rewind_source * source
Definition: pg_rewind.c:89
#define pg_log_warning(...)
Definition: pgfnames.c:24
#define snprintf
Definition: port.h:238
char * input_filename
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
static rfile * make_incremental_rfile(char *filename)
Definition: reconstruct.c:443
static void debug_reconstruction(int n_source, rfile **sources, bool dry_run)
Definition: reconstruct.c:371
static void write_reconstructed_file(char *input_filename, char *output_filename, unsigned block_length, rfile **sourcemap, off_t *offsetmap, pg_checksum_context *checksum_ctx, CopyMethod copy_method, bool debug, bool dry_run)
Definition: reconstruct.c:539
static unsigned find_reconstructed_block_length(rfile *s)
Definition: reconstruct.c:426
uint8 * checksum_payload
Definition: load_manifest.h:29
pg_checksum_type checksum_type
Definition: load_manifest.h:27

References Assert, b, manifest_file::checksum_length, manifest_file::checksum_payload, manifest_file::checksum_type, CHECKSUM_TYPE_NONE, close, copy_file(), debug, debug_reconstruction(), dry_run, rfile::fd, rfile::filename, find_reconstructed_block_length(), fstat, rfile::header_length, i, if(), input_filename, make_incremental_rfile(), make_rfile(), MAXPGPATH, rfile::num_blocks, output_filename, pfree(), pg_checksum_final(), pg_checksum_init(), PG_CHECKSUM_MAX_LENGTH, pg_fatal, pg_free(), pg_log_warning, pg_malloc(), pg_malloc0(), psprintf(), rfile::relative_block_numbers, snprintf, source, stat::st_size, rfile::truncation_block_length, and write_reconstructed_file().

Referenced by process_directory_recursively().

◆ write_block()

static void write_block ( int  wfd,
char *  output_filename,
uint8 buffer,
pg_checksum_context checksum_ctx 
)
static

Definition at line 739 of file reconstruct.c.

741 {
742  int wb;
743 
744  if ((wb = write(fd, buffer, BLCKSZ)) != BLCKSZ)
745  {
746  if (wb < 0)
747  pg_fatal("could not write file \"%s\": %m", output_filename);
748  else
749  pg_fatal("could not write file \"%s\": wrote only %d of %d bytes",
750  output_filename, wb, BLCKSZ);
751  }
752 
753  /* Update the checksum computation. */
754  if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0)
755  pg_fatal("could not update checksum of file \"%s\"",
757 }
int pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
#define write(a, b, c)
Definition: win32.h:14
static int fd(const char *x, int i)
Definition: preproc-init.c:105

References fd(), output_filename, pg_checksum_update(), pg_fatal, and write.

Referenced by write_reconstructed_file().

◆ write_reconstructed_file()

static void write_reconstructed_file ( char *  input_filename,
char *  output_filename,
unsigned  block_length,
rfile **  sourcemap,
off_t *  offsetmap,
pg_checksum_context checksum_ctx,
CopyMethod  copy_method,
bool  debug,
bool  dry_run 
)
static

Definition at line 539 of file reconstruct.c.

548 {
549  int wfd = -1;
550  unsigned i;
551  unsigned zero_blocks = 0;
552 
553  /* Debugging output. */
554  if (debug)
555  {
556  StringInfoData debug_buf;
557  unsigned start_of_range = 0;
558  unsigned current_block = 0;
559 
560  /* Basic information about the output file to be produced. */
561  if (dry_run)
562  pg_log_debug("would reconstruct \"%s\" (%u blocks, checksum %s)",
563  output_filename, block_length,
564  pg_checksum_type_name(checksum_ctx->type));
565  else
566  pg_log_debug("reconstructing \"%s\" (%u blocks, checksum %s)",
567  output_filename, block_length,
568  pg_checksum_type_name(checksum_ctx->type));
569 
570  /* Print out the plan for reconstructing this file. */
571  initStringInfo(&debug_buf);
572  while (current_block < block_length)
573  {
574  rfile *s = sourcemap[current_block];
575 
576  /* Extend range, if possible. */
577  if (current_block + 1 < block_length &&
578  s == sourcemap[current_block + 1])
579  {
580  ++current_block;
581  continue;
582  }
583 
584  /* Add details about this range. */
585  if (s == NULL)
586  {
587  if (current_block == start_of_range)
588  appendStringInfo(&debug_buf, " %u:zero", current_block);
589  else
590  appendStringInfo(&debug_buf, " %u-%u:zero",
591  start_of_range, current_block);
592  }
593  else
594  {
595  if (current_block == start_of_range)
596  appendStringInfo(&debug_buf, " %u:%s@" UINT64_FORMAT,
597  current_block, s->filename,
598  (uint64) offsetmap[current_block]);
599  else
600  appendStringInfo(&debug_buf, " %u-%u:%s@" UINT64_FORMAT,
601  start_of_range, current_block,
602  s->filename,
603  (uint64) offsetmap[current_block]);
604  }
605 
606  /* Begin new range. */
607  start_of_range = ++current_block;
608 
609  /* If the output is very long or we are done, dump it now. */
610  if (current_block == block_length || debug_buf.len > 1024)
611  {
612  pg_log_debug("reconstruction plan:%s", debug_buf.data);
613  resetStringInfo(&debug_buf);
614  }
615  }
616 
617  /* Free memory. */
618  pfree(debug_buf.data);
619  }
620 
621  /* Open the output file, except in dry_run mode. */
622  if (!dry_run &&
623  (wfd = open(output_filename,
624  O_RDWR | PG_BINARY | O_CREAT | O_EXCL,
625  pg_file_create_mode)) < 0)
626  pg_fatal("could not open file \"%s\": %m", output_filename);
627 
628  /* Read and write the blocks as required. */
629  for (i = 0; i < block_length; ++i)
630  {
631  uint8 buffer[BLCKSZ];
632  rfile *s = sourcemap[i];
633 
634  /* Update accounting information. */
635  if (s == NULL)
636  ++zero_blocks;
637  else
638  {
639  s->num_blocks_read++;
641  offsetmap[i] + BLCKSZ);
642  }
643 
644  /* Skip the rest of this in dry-run mode. */
645  if (dry_run)
646  continue;
647 
648  /* Read or zero-fill the block as appropriate. */
649  if (s == NULL)
650  {
651  /*
652  * New block not mentioned in the WAL summary. Should have been an
653  * uninitialized block, so just zero-fill it.
654  */
655  memset(buffer, 0, BLCKSZ);
656 
657  /* Write out the block, update the checksum if needed. */
658  write_block(wfd, output_filename, buffer, checksum_ctx);
659 
660  /* Nothing else to do for zero-filled blocks. */
661  continue;
662  }
663 
664  /* Copy the block using the appropriate copy method. */
665  if (copy_method != COPY_METHOD_COPY_FILE_RANGE)
666  {
667  /*
668  * Read the block from the correct source file, and then write it
669  * out, possibly with a checksum update.
670  */
671  read_block(s, offsetmap[i], buffer);
672  write_block(wfd, output_filename, buffer, checksum_ctx);
673  }
674  else /* use copy_file_range */
675  {
676 #if defined(HAVE_COPY_FILE_RANGE)
677  /* copy_file_range modifies the offset, so use a local copy */
678  off_t off = offsetmap[i];
679  size_t nwritten = 0;
680 
681  /*
682  * Retry until we've written all the bytes (the offset is updated
683  * by copy_file_range, and so is the wfd file offset).
684  */
685  do
686  {
687  int wb;
688 
689  wb = copy_file_range(s->fd, &off, wfd, NULL, BLCKSZ - nwritten, 0);
690 
691  if (wb < 0)
692  pg_fatal("error while copying file range from \"%s\" to \"%s\": %m",
694 
695  nwritten += wb;
696 
697  } while (BLCKSZ > nwritten);
698 
699  /*
700  * When checksum calculation not needed, we're done, otherwise
701  * read the block and pass it to the checksum calculation.
702  */
703  if (checksum_ctx->type == CHECKSUM_TYPE_NONE)
704  continue;
705 
706  read_block(s, offsetmap[i], buffer);
707 
708  if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0)
709  pg_fatal("could not update checksum of file \"%s\"",
711 #else
712  pg_fatal("copy_file_range not supported on this platform");
713 #endif
714  }
715  }
716 
717  /* Debugging output. */
718  if (zero_blocks > 0)
719  {
720  if (dry_run)
721  pg_log_debug("would have zero-filled %u blocks", zero_blocks);
722  else
723  pg_log_debug("zero-filled %u blocks", zero_blocks);
724  }
725 
726  /* Close the output file. */
727  if (wfd >= 0 && close(wfd) != 0)
728  pg_fatal("could not close \"%s\": %m", output_filename);
729 }
#define Max(x, y)
Definition: c.h:998
#define UINT64_FORMAT
Definition: c.h:549
unsigned char uint8
Definition: c.h:504
char * pg_checksum_type_name(pg_checksum_type type)
@ COPY_METHOD_COPY_FILE_RANGE
Definition: copy_file.h:25
int pg_file_create_mode
Definition: file_perm.c:19
static void read_block(rfile *s, off_t off, uint8 *buffer)
Definition: reconstruct.c:763
static void write_block(int wfd, char *output_filename, uint8 *buffer, pg_checksum_context *checksum_ctx)
Definition: reconstruct.c:739
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:78
void appendStringInfo(StringInfo str, const char *fmt,...)
Definition: stringinfo.c:97
void initStringInfo(StringInfo str)
Definition: stringinfo.c:59
pg_checksum_type type

References appendStringInfo(), CHECKSUM_TYPE_NONE, close, COPY_METHOD_COPY_FILE_RANGE, StringInfoData::data, debug, dry_run, rfile::fd, rfile::filename, rfile::highest_offset_read, i, initStringInfo(), input_filename, StringInfoData::len, Max, rfile::num_blocks_read, output_filename, pfree(), PG_BINARY, pg_checksum_type_name(), pg_checksum_update(), pg_fatal, pg_file_create_mode, pg_log_debug, read_block(), resetStringInfo(), pg_checksum_context::type, UINT64_FORMAT, and write_block().

Referenced by reconstruct_from_incremental_file().