PostgreSQL Source Code  git master
reconstruct.c File Reference
#include "postgres_fe.h"
#include <unistd.h>
#include "backup/basebackup_incremental.h"
#include "common/file_perm.h"
#include "common/logging.h"
#include "copy_file.h"
#include "lib/stringinfo.h"
#include "reconstruct.h"
#include "storage/block.h"
Include dependency graph for reconstruct.c:

Go to the source code of this file.

Data Structures

struct  rfile
 

Typedefs

typedef struct rfile rfile
 

Functions

static void debug_reconstruction (int n_source, rfile **sources, bool dry_run)
 
static unsigned find_reconstructed_block_length (rfile *s)
 
static rfilemake_incremental_rfile (char *filename)
 
static rfilemake_rfile (char *filename, bool missing_ok)
 
static void write_reconstructed_file (char *input_filename, char *output_filename, unsigned block_length, rfile **sourcemap, off_t *offsetmap, pg_checksum_context *checksum_ctx, CopyMethod copy_method, bool debug, bool dry_run)
 
static void read_bytes (rfile *rf, void *buffer, unsigned length)
 
static void write_block (int wfd, char *output_filename, uint8 *buffer, pg_checksum_context *checksum_ctx)
 
static void read_block (rfile *s, off_t off, uint8 *buffer)
 
void reconstruct_from_incremental_file (char *input_filename, char *output_filename, char *relative_path, char *bare_file_name, int n_prior_backups, char **prior_backup_dirs, manifest_data **manifests, char *manifest_path, pg_checksum_type checksum_type, int *checksum_length, uint8 **checksum_payload, CopyMethod copy_method, bool debug, bool dry_run)
 

Typedef Documentation

◆ rfile

typedef struct rfile rfile

Function Documentation

◆ debug_reconstruction()

static void debug_reconstruction ( int  n_source,
rfile **  sources,
bool  dry_run 
)
static

Definition at line 370 of file reconstruct.c.

371 {
372  unsigned i;
373 
374  for (i = 0; i < n_source; ++i)
375  {
376  rfile *s = sources[i];
377 
378  /* Ignore source if not used. */
379  if (s == NULL)
380  continue;
381 
382  /* If no data is needed from this file, we can ignore it. */
383  if (s->num_blocks_read == 0)
384  continue;
385 
386  /* Debug logging. */
387  if (dry_run)
388  pg_log_debug("would have read %u blocks from \"%s\"",
389  s->num_blocks_read, s->filename);
390  else
391  pg_log_debug("read %u blocks from \"%s\"",
392  s->num_blocks_read, s->filename);
393 
394  /*
395  * In dry-run mode, we don't actually try to read data from the file,
396  * but we do try to verify that the file is long enough that we could
397  * have read the data if we'd tried.
398  *
399  * If this fails, then it means that a non-dry-run attempt would fail,
400  * complaining of not being able to read the required bytes from the
401  * file.
402  */
403  if (dry_run)
404  {
405  struct stat sb;
406 
407  if (fstat(s->fd, &sb) < 0)
408  pg_fatal("could not stat \"%s\": %m", s->filename);
409  if (sb.st_size < s->highest_offset_read)
410  pg_fatal("file \"%s\" is too short: expected %llu, found %llu",
411  s->filename,
412  (unsigned long long) s->highest_offset_read,
413  (unsigned long long) sb.st_size);
414  }
415  }
416 }
int i
Definition: isn.c:73
#define pg_log_debug(...)
Definition: logging.h:133
#define pg_fatal(...)
static bool dry_run
off_t highest_offset_read
Definition: reconstruct.c:46
int fd
Definition: reconstruct.c:40
unsigned num_blocks_read
Definition: reconstruct.c:45
char * filename
Definition: reconstruct.c:39
#define fstat
Definition: win32_port.h:283

References dry_run, rfile::fd, rfile::filename, fstat, rfile::highest_offset_read, i, rfile::num_blocks_read, pg_fatal, pg_log_debug, and stat::st_size.

Referenced by reconstruct_from_incremental_file().

◆ find_reconstructed_block_length()

static unsigned find_reconstructed_block_length ( rfile s)
static

Definition at line 425 of file reconstruct.c.

426 {
427  unsigned block_length = s->truncation_block_length;
428  unsigned i;
429 
430  for (i = 0; i < s->num_blocks; ++i)
431  if (s->relative_block_numbers[i] >= block_length)
432  block_length = s->relative_block_numbers[i] + 1;
433 
434  return block_length;
435 }
BlockNumber * relative_block_numbers
Definition: reconstruct.c:43
unsigned num_blocks
Definition: reconstruct.c:42
unsigned truncation_block_length
Definition: reconstruct.c:44

References i, rfile::num_blocks, rfile::relative_block_numbers, and rfile::truncation_block_length.

Referenced by reconstruct_from_incremental_file().

◆ make_incremental_rfile()

static rfile * make_incremental_rfile ( char *  filename)
static

Definition at line 442 of file reconstruct.c.

443 {
444  rfile *rf;
445  unsigned magic;
446 
447  rf = make_rfile(filename, false);
448 
449  /* Read and validate magic number. */
450  read_bytes(rf, &magic, sizeof(magic));
451  if (magic != INCREMENTAL_MAGIC)
452  pg_fatal("file \"%s\" has bad incremental magic number (0x%x not 0x%x)",
453  filename, magic, INCREMENTAL_MAGIC);
454 
455  /* Read block count. */
456  read_bytes(rf, &rf->num_blocks, sizeof(rf->num_blocks));
457  if (rf->num_blocks > RELSEG_SIZE)
458  pg_fatal("file \"%s\" has block count %u in excess of segment size %u",
459  filename, rf->num_blocks, RELSEG_SIZE);
460 
461  /* Read truncation block length. */
463  sizeof(rf->truncation_block_length));
464  if (rf->truncation_block_length > RELSEG_SIZE)
465  pg_fatal("file \"%s\" has truncation block length %u in excess of segment size %u",
466  filename, rf->truncation_block_length, RELSEG_SIZE);
467 
468  /* Read block numbers if there are any. */
469  if (rf->num_blocks > 0)
470  {
472  pg_malloc0(sizeof(BlockNumber) * rf->num_blocks);
474  sizeof(BlockNumber) * rf->num_blocks);
475  }
476 
477  /* Remember length of header. */
478  rf->header_length = sizeof(magic) + sizeof(rf->num_blocks) +
479  sizeof(rf->truncation_block_length) +
480  sizeof(BlockNumber) * rf->num_blocks;
481 
482  /*
483  * Round header length to a multiple of BLCKSZ, so that blocks contents
484  * are properly aligned. Only do this when the file actually has data for
485  * some blocks.
486  */
487  if ((rf->num_blocks > 0) && ((rf->header_length % BLCKSZ) != 0))
488  rf->header_length += (BLCKSZ - (rf->header_length % BLCKSZ));
489 
490  return rf;
491 }
#define INCREMENTAL_MAGIC
uint32 BlockNumber
Definition: block.h:31
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
static char * filename
Definition: pg_dumpall.c:119
static rfile * make_rfile(char *filename, bool missing_ok)
Definition: reconstruct.c:497
static void read_bytes(rfile *rf, void *buffer, unsigned length)
Definition: reconstruct.c:520
size_t header_length
Definition: reconstruct.c:41

References filename, rfile::header_length, INCREMENTAL_MAGIC, make_rfile(), rfile::num_blocks, pg_fatal, pg_malloc0(), read_bytes(), rfile::relative_block_numbers, and rfile::truncation_block_length.

Referenced by reconstruct_from_incremental_file().

◆ make_rfile()

static rfile * make_rfile ( char *  filename,
bool  missing_ok 
)
static

Definition at line 497 of file reconstruct.c.

498 {
499  rfile *rf;
500 
501  rf = pg_malloc0(sizeof(rfile));
502  rf->filename = pstrdup(filename);
503  if ((rf->fd = open(filename, O_RDONLY | PG_BINARY, 0)) < 0)
504  {
505  if (missing_ok && errno == ENOENT)
506  {
507  pg_free(rf);
508  return NULL;
509  }
510  pg_fatal("could not open file \"%s\": %m", filename);
511  }
512 
513  return rf;
514 }
#define PG_BINARY
Definition: c.h:1273
void pg_free(void *ptr)
Definition: fe_memutils.c:105
char * pstrdup(const char *in)
Definition: mcxt.c:1695

References rfile::fd, rfile::filename, filename, PG_BINARY, pg_fatal, pg_free(), pg_malloc0(), and pstrdup().

Referenced by make_incremental_rfile(), and reconstruct_from_incremental_file().

◆ read_block()

static void read_block ( rfile s,
off_t  off,
uint8 buffer 
)
static

Definition at line 762 of file reconstruct.c.

763 {
764  int rb;
765 
766  /* Read the block from the correct source, except if dry-run. */
767  rb = pg_pread(s->fd, buffer, BLCKSZ, off);
768  if (rb != BLCKSZ)
769  {
770  if (rb < 0)
771  pg_fatal("could not read file \"%s\": %m", s->filename);
772  else
773  pg_fatal("could not read file \"%s\": read only %d of %d bytes at offset %llu",
774  s->filename, rb, BLCKSZ,
775  (unsigned long long) off);
776  }
777 }
#define pg_pread
Definition: port.h:225

References rfile::fd, rfile::filename, pg_fatal, and pg_pread.

Referenced by write_reconstructed_file().

◆ read_bytes()

static void read_bytes ( rfile rf,
void *  buffer,
unsigned  length 
)
static

Definition at line 520 of file reconstruct.c.

521 {
522  int rb = read(rf->fd, buffer, length);
523 
524  if (rb != length)
525  {
526  if (rb < 0)
527  pg_fatal("could not read file \"%s\": %m", rf->filename);
528  else
529  pg_fatal("could not read file \"%s\": read only %d of %u bytes",
530  rf->filename, rb, length);
531  }
532 }
#define read(a, b, c)
Definition: win32.h:13

References rfile::fd, rfile::filename, pg_fatal, and read.

Referenced by make_incremental_rfile().

◆ reconstruct_from_incremental_file()

void reconstruct_from_incremental_file ( char *  input_filename,
char *  output_filename,
char *  relative_path,
char *  bare_file_name,
int  n_prior_backups,
char **  prior_backup_dirs,
manifest_data **  manifests,
char *  manifest_path,
pg_checksum_type  checksum_type,
int *  checksum_length,
uint8 **  checksum_payload,
CopyMethod  copy_method,
bool  debug,
bool  dry_run 
)

Definition at line 86 of file reconstruct.c.

100 {
101  rfile **source;
102  rfile *latest_source = NULL;
103  rfile **sourcemap;
104  off_t *offsetmap;
105  unsigned block_length;
106  unsigned i;
107  unsigned sidx = n_prior_backups;
108  bool full_copy_possible = true;
109  int copy_source_index = -1;
110  rfile *copy_source = NULL;
111  pg_checksum_context checksum_ctx;
112 
113  /*
114  * Every block must come either from the latest version of the file or
115  * from one of the prior backups.
116  */
117  source = pg_malloc0(sizeof(rfile *) * (1 + n_prior_backups));
118 
119  /*
120  * Use the information from the latest incremental file to figure out how
121  * long the reconstructed file should be.
122  */
123  latest_source = make_incremental_rfile(input_filename);
124  source[n_prior_backups] = latest_source;
125  block_length = find_reconstructed_block_length(latest_source);
126 
127  /*
128  * For each block in the output file, we need to know from which file we
129  * need to obtain it and at what offset in that file it's stored.
130  * sourcemap gives us the first of these things, and offsetmap the latter.
131  */
132  sourcemap = pg_malloc0(sizeof(rfile *) * block_length);
133  offsetmap = pg_malloc0(sizeof(off_t) * block_length);
134 
135  /*
136  * Every block that is present in the newest incremental file should be
137  * sourced from that file. If it precedes the truncation_block_length,
138  * it's a block that we would otherwise have had to find in an older
139  * backup and thus reduces the number of blocks remaining to be found by
140  * one; otherwise, it's an extra block that needs to be included in the
141  * output but would not have needed to be found in an older backup if it
142  * had not been present.
143  */
144  for (i = 0; i < latest_source->num_blocks; ++i)
145  {
146  BlockNumber b = latest_source->relative_block_numbers[i];
147 
148  Assert(b < block_length);
149  sourcemap[b] = latest_source;
150  offsetmap[b] = latest_source->header_length + (i * BLCKSZ);
151 
152  /*
153  * A full copy of a file from an earlier backup is only possible if no
154  * blocks are needed from any later incremental file.
155  */
156  full_copy_possible = false;
157  }
158 
159  while (1)
160  {
161  char source_filename[MAXPGPATH];
162  rfile *s;
163 
164  /*
165  * Move to the next backup in the chain. If there are no more, then
166  * we're done.
167  */
168  if (sidx == 0)
169  break;
170  --sidx;
171 
172  /*
173  * Look for the full file in the previous backup. If not found, then
174  * look for an incremental file instead.
175  */
176  snprintf(source_filename, MAXPGPATH, "%s/%s/%s",
177  prior_backup_dirs[sidx], relative_path, bare_file_name);
178  if ((s = make_rfile(source_filename, true)) == NULL)
179  {
180  snprintf(source_filename, MAXPGPATH, "%s/%s/INCREMENTAL.%s",
181  prior_backup_dirs[sidx], relative_path, bare_file_name);
182  s = make_incremental_rfile(source_filename);
183  }
184  source[sidx] = s;
185 
186  /*
187  * If s->header_length == 0, then this is a full file; otherwise, it's
188  * an incremental file.
189  */
190  if (s->header_length == 0)
191  {
192  struct stat sb;
193  BlockNumber b;
194  BlockNumber blocklength;
195 
196  /* We need to know the length of the file. */
197  if (fstat(s->fd, &sb) < 0)
198  pg_fatal("could not stat \"%s\": %m", s->filename);
199 
200  /*
201  * Since we found a full file, source all blocks from it that
202  * exist in the file.
203  *
204  * Note that there may be blocks that don't exist either in this
205  * file or in any incremental file but that precede
206  * truncation_block_length. These are, presumably, zero-filled
207  * blocks that result from the server extending the file but
208  * taking no action on those blocks that generated any WAL.
209  *
210  * Sadly, we have no way of validating that this is really what
211  * happened, and neither does the server. From it's perspective,
212  * an unmodified block that contains data looks exactly the same
213  * as a zero-filled block that never had any data: either way,
214  * it's not mentioned in any WAL summary and the server has no
215  * reason to read it. From our perspective, all we know is that
216  * nobody had a reason to back up the block. That certainly means
217  * that the block didn't exist at the time of the full backup, but
218  * the supposition that it was all zeroes at the time of every
219  * later backup is one that we can't validate.
220  */
221  blocklength = sb.st_size / BLCKSZ;
222  for (b = 0; b < latest_source->truncation_block_length; ++b)
223  {
224  if (sourcemap[b] == NULL && b < blocklength)
225  {
226  sourcemap[b] = s;
227  offsetmap[b] = b * BLCKSZ;
228  }
229  }
230 
231  /*
232  * If a full copy looks possible, check whether the resulting file
233  * should be exactly as long as the source file is. If so, a full
234  * copy is acceptable, otherwise not.
235  */
236  if (full_copy_possible)
237  {
238  uint64 expected_length;
239 
240  expected_length =
241  (uint64) latest_source->truncation_block_length;
242  expected_length *= BLCKSZ;
243  if (expected_length == sb.st_size)
244  {
245  copy_source = s;
246  copy_source_index = sidx;
247  }
248  }
249 
250  /* We don't need to consider any further sources. */
251  break;
252  }
253 
254  /*
255  * Since we found another incremental file, source all blocks from it
256  * that we need but don't yet have.
257  */
258  for (i = 0; i < s->num_blocks; ++i)
259  {
261 
262  if (b < latest_source->truncation_block_length &&
263  sourcemap[b] == NULL)
264  {
265  sourcemap[b] = s;
266  offsetmap[b] = s->header_length + (i * BLCKSZ);
267 
268  /*
269  * A full copy of a file from an earlier backup is only
270  * possible if no blocks are needed from any later incremental
271  * file.
272  */
273  full_copy_possible = false;
274  }
275  }
276  }
277 
278  /*
279  * If a checksum of the required type already exists in the
280  * backup_manifest for the relevant input directory, we can save some work
281  * by reusing that checksum instead of computing a new one.
282  */
283  if (copy_source_index >= 0 && manifests[copy_source_index] != NULL &&
284  checksum_type != CHECKSUM_TYPE_NONE)
285  {
286  manifest_file *mfile;
287 
288  mfile = manifest_files_lookup(manifests[copy_source_index]->files,
289  manifest_path);
290  if (mfile == NULL)
291  {
292  char *path = psprintf("%s/backup_manifest",
293  prior_backup_dirs[copy_source_index]);
294 
295  /*
296  * The directory is out of sync with the backup_manifest, so emit
297  * a warning.
298  */
299  /*- translator: the first %s is a backup manifest file, the second is a file absent therein */
300  pg_log_warning("\"%s\" contains no entry for \"%s\"",
301  path,
302  manifest_path);
303  pfree(path);
304  }
305  else if (mfile->checksum_type == checksum_type)
306  {
307  *checksum_length = mfile->checksum_length;
308  *checksum_payload = pg_malloc(*checksum_length);
309  memcpy(*checksum_payload, mfile->checksum_payload,
310  *checksum_length);
311  checksum_type = CHECKSUM_TYPE_NONE;
312  }
313  }
314 
315  /* Prepare for checksum calculation, if required. */
316  pg_checksum_init(&checksum_ctx, checksum_type);
317 
318  /*
319  * If the full file can be created by copying a file from an older backup
320  * in the chain without needing to overwrite any blocks or truncate the
321  * result, then forget about performing reconstruction and just copy that
322  * file in its entirety.
323  *
324  * Otherwise, reconstruct.
325  */
326  if (copy_source != NULL)
327  copy_file(copy_source->filename, output_filename,
328  &checksum_ctx, copy_method, dry_run);
329  else
330  {
332  block_length, sourcemap, offsetmap,
333  &checksum_ctx, copy_method,
334  debug, dry_run);
335  debug_reconstruction(n_prior_backups + 1, source, dry_run);
336  }
337 
338  /* Save results of checksum calculation. */
339  if (checksum_type != CHECKSUM_TYPE_NONE)
340  {
341  *checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
342  *checksum_length = pg_checksum_final(&checksum_ctx,
343  *checksum_payload);
344  }
345 
346  /*
347  * Close files and release memory.
348  */
349  for (i = 0; i <= n_prior_backups; ++i)
350  {
351  rfile *s = source[i];
352 
353  if (s == NULL)
354  continue;
355  if (close(s->fd) != 0)
356  pg_fatal("could not close \"%s\": %m", s->filename);
357  if (s->relative_block_numbers != NULL)
359  pg_free(s->filename);
360  }
361  pfree(sourcemap);
362  pfree(offsetmap);
363  pfree(source);
364 }
#define Assert(condition)
Definition: c.h:858
int pg_checksum_final(pg_checksum_context *context, uint8 *output)
int pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
#define PG_CHECKSUM_MAX_LENGTH
@ CHECKSUM_TYPE_NONE
void copy_file(const char *fromfile, const char *tofile)
Definition: copydir.c:117
char * output_filename
Definition: ecpg.c:23
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
static bool debug
Definition: initdb.c:161
#define close(a)
Definition: win32.h:12
int b
Definition: isn.c:70
if(TABLE==NULL||TABLE_index==NULL)
Definition: isn.c:77
void pfree(void *pointer)
Definition: mcxt.c:1520
#define MAXPGPATH
static rewind_source * source
Definition: pg_rewind.c:89
#define pg_log_warning(...)
Definition: pgfnames.c:24
#define snprintf
Definition: port.h:238
char * input_filename
char * psprintf(const char *fmt,...)
Definition: psprintf.c:46
static rfile * make_incremental_rfile(char *filename)
Definition: reconstruct.c:442
static void debug_reconstruction(int n_source, rfile **sources, bool dry_run)
Definition: reconstruct.c:370
static void write_reconstructed_file(char *input_filename, char *output_filename, unsigned block_length, rfile **sourcemap, off_t *offsetmap, pg_checksum_context *checksum_ctx, CopyMethod copy_method, bool debug, bool dry_run)
Definition: reconstruct.c:538
static unsigned find_reconstructed_block_length(rfile *s)
Definition: reconstruct.c:425
uint8 * checksum_payload
Definition: load_manifest.h:29
pg_checksum_type checksum_type
Definition: load_manifest.h:27

References Assert, b, manifest_file::checksum_length, manifest_file::checksum_payload, manifest_file::checksum_type, CHECKSUM_TYPE_NONE, close, copy_file(), debug, debug_reconstruction(), dry_run, rfile::fd, rfile::filename, find_reconstructed_block_length(), fstat, rfile::header_length, i, if(), input_filename, make_incremental_rfile(), make_rfile(), MAXPGPATH, rfile::num_blocks, output_filename, pfree(), pg_checksum_final(), pg_checksum_init(), PG_CHECKSUM_MAX_LENGTH, pg_fatal, pg_free(), pg_log_warning, pg_malloc(), pg_malloc0(), psprintf(), rfile::relative_block_numbers, snprintf, source, stat::st_size, rfile::truncation_block_length, and write_reconstructed_file().

Referenced by process_directory_recursively().

◆ write_block()

static void write_block ( int  wfd,
char *  output_filename,
uint8 buffer,
pg_checksum_context checksum_ctx 
)
static

Definition at line 738 of file reconstruct.c.

740 {
741  int wb;
742 
743  if ((wb = write(fd, buffer, BLCKSZ)) != BLCKSZ)
744  {
745  if (wb < 0)
746  pg_fatal("could not write file \"%s\": %m", output_filename);
747  else
748  pg_fatal("could not write file \"%s\": wrote only %d of %d bytes",
749  output_filename, wb, BLCKSZ);
750  }
751 
752  /* Update the checksum computation. */
753  if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0)
754  pg_fatal("could not update checksum of file \"%s\"",
756 }
int pg_checksum_update(pg_checksum_context *context, const uint8 *input, size_t len)
#define write(a, b, c)
Definition: win32.h:14
static int fd(const char *x, int i)
Definition: preproc-init.c:105

References fd(), output_filename, pg_checksum_update(), pg_fatal, and write.

Referenced by write_reconstructed_file().

◆ write_reconstructed_file()

static void write_reconstructed_file ( char *  input_filename,
char *  output_filename,
unsigned  block_length,
rfile **  sourcemap,
off_t *  offsetmap,
pg_checksum_context checksum_ctx,
CopyMethod  copy_method,
bool  debug,
bool  dry_run 
)
static

Definition at line 538 of file reconstruct.c.

547 {
548  int wfd = -1;
549  unsigned i;
550  unsigned zero_blocks = 0;
551 
552  /* Debugging output. */
553  if (debug)
554  {
555  StringInfoData debug_buf;
556  unsigned start_of_range = 0;
557  unsigned current_block = 0;
558 
559  /* Basic information about the output file to be produced. */
560  if (dry_run)
561  pg_log_debug("would reconstruct \"%s\" (%u blocks, checksum %s)",
562  output_filename, block_length,
563  pg_checksum_type_name(checksum_ctx->type));
564  else
565  pg_log_debug("reconstructing \"%s\" (%u blocks, checksum %s)",
566  output_filename, block_length,
567  pg_checksum_type_name(checksum_ctx->type));
568 
569  /* Print out the plan for reconstructing this file. */
570  initStringInfo(&debug_buf);
571  while (current_block < block_length)
572  {
573  rfile *s = sourcemap[current_block];
574 
575  /* Extend range, if possible. */
576  if (current_block + 1 < block_length &&
577  s == sourcemap[current_block + 1])
578  {
579  ++current_block;
580  continue;
581  }
582 
583  /* Add details about this range. */
584  if (s == NULL)
585  {
586  if (current_block == start_of_range)
587  appendStringInfo(&debug_buf, " %u:zero", current_block);
588  else
589  appendStringInfo(&debug_buf, " %u-%u:zero",
590  start_of_range, current_block);
591  }
592  else
593  {
594  if (current_block == start_of_range)
595  appendStringInfo(&debug_buf, " %u:%s@" UINT64_FORMAT,
596  current_block, s->filename,
597  (uint64) offsetmap[current_block]);
598  else
599  appendStringInfo(&debug_buf, " %u-%u:%s@" UINT64_FORMAT,
600  start_of_range, current_block,
601  s->filename,
602  (uint64) offsetmap[current_block]);
603  }
604 
605  /* Begin new range. */
606  start_of_range = ++current_block;
607 
608  /* If the output is very long or we are done, dump it now. */
609  if (current_block == block_length || debug_buf.len > 1024)
610  {
611  pg_log_debug("reconstruction plan:%s", debug_buf.data);
612  resetStringInfo(&debug_buf);
613  }
614  }
615 
616  /* Free memory. */
617  pfree(debug_buf.data);
618  }
619 
620  /* Open the output file, except in dry_run mode. */
621  if (!dry_run &&
622  (wfd = open(output_filename,
623  O_RDWR | PG_BINARY | O_CREAT | O_EXCL,
624  pg_file_create_mode)) < 0)
625  pg_fatal("could not open file \"%s\": %m", output_filename);
626 
627  /* Read and write the blocks as required. */
628  for (i = 0; i < block_length; ++i)
629  {
630  uint8 buffer[BLCKSZ];
631  rfile *s = sourcemap[i];
632 
633  /* Update accounting information. */
634  if (s == NULL)
635  ++zero_blocks;
636  else
637  {
638  s->num_blocks_read++;
640  offsetmap[i] + BLCKSZ);
641  }
642 
643  /* Skip the rest of this in dry-run mode. */
644  if (dry_run)
645  continue;
646 
647  /* Read or zero-fill the block as appropriate. */
648  if (s == NULL)
649  {
650  /*
651  * New block not mentioned in the WAL summary. Should have been an
652  * uninitialized block, so just zero-fill it.
653  */
654  memset(buffer, 0, BLCKSZ);
655 
656  /* Write out the block, update the checksum if needed. */
657  write_block(wfd, output_filename, buffer, checksum_ctx);
658 
659  /* Nothing else to do for zero-filled blocks. */
660  continue;
661  }
662 
663  /* Copy the block using the appropriate copy method. */
664  if (copy_method != COPY_METHOD_COPY_FILE_RANGE)
665  {
666  /*
667  * Read the block from the correct source file, and then write it
668  * out, possibly with a checksum update.
669  */
670  read_block(s, offsetmap[i], buffer);
671  write_block(wfd, output_filename, buffer, checksum_ctx);
672  }
673  else /* use copy_file_range */
674  {
675 #if defined(HAVE_COPY_FILE_RANGE)
676  /* copy_file_range modifies the offset, so use a local copy */
677  off_t off = offsetmap[i];
678  size_t nwritten = 0;
679 
680  /*
681  * Retry until we've written all the bytes (the offset is updated
682  * by copy_file_range, and so is the wfd file offset).
683  */
684  do
685  {
686  int wb;
687 
688  wb = copy_file_range(s->fd, &off, wfd, NULL, BLCKSZ - nwritten, 0);
689 
690  if (wb < 0)
691  pg_fatal("error while copying file range from \"%s\" to \"%s\": %m",
693 
694  nwritten += wb;
695 
696  } while (BLCKSZ > nwritten);
697 
698  /*
699  * When checksum calculation not needed, we're done, otherwise
700  * read the block and pass it to the checksum calculation.
701  */
702  if (checksum_ctx->type == CHECKSUM_TYPE_NONE)
703  continue;
704 
705  read_block(s, offsetmap[i], buffer);
706 
707  if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0)
708  pg_fatal("could not update checksum of file \"%s\"",
710 #else
711  pg_fatal("copy_file_range not supported on this platform");
712 #endif
713  }
714  }
715 
716  /* Debugging output. */
717  if (zero_blocks > 0)
718  {
719  if (dry_run)
720  pg_log_debug("would have zero-filled %u blocks", zero_blocks);
721  else
722  pg_log_debug("zero-filled %u blocks", zero_blocks);
723  }
724 
725  /* Close the output file. */
726  if (wfd >= 0 && close(wfd) != 0)
727  pg_fatal("could not close \"%s\": %m", output_filename);
728 }
#define Max(x, y)
Definition: c.h:998
#define UINT64_FORMAT
Definition: c.h:549
unsigned char uint8
Definition: c.h:504
char * pg_checksum_type_name(pg_checksum_type type)
@ COPY_METHOD_COPY_FILE_RANGE
Definition: copy_file.h:25
int pg_file_create_mode
Definition: file_perm.c:19
static void read_block(rfile *s, off_t off, uint8 *buffer)
Definition: reconstruct.c:762
static void write_block(int wfd, char *output_filename, uint8 *buffer, pg_checksum_context *checksum_ctx)
Definition: reconstruct.c:738
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:78
void appendStringInfo(StringInfo str, const char *fmt,...)
Definition: stringinfo.c:97
void initStringInfo(StringInfo str)
Definition: stringinfo.c:59
pg_checksum_type type

References appendStringInfo(), CHECKSUM_TYPE_NONE, close, COPY_METHOD_COPY_FILE_RANGE, StringInfoData::data, debug, dry_run, rfile::fd, rfile::filename, rfile::highest_offset_read, i, initStringInfo(), input_filename, StringInfoData::len, Max, rfile::num_blocks_read, output_filename, pfree(), PG_BINARY, pg_checksum_type_name(), pg_checksum_update(), pg_fatal, pg_file_create_mode, pg_log_debug, read_block(), resetStringInfo(), pg_checksum_context::type, UINT64_FORMAT, and write_block().

Referenced by reconstruct_from_incremental_file().