PostgreSQL Source Code git master
All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
reconstruct.h File Reference
#include "common/checksum_helper.h"
#include "copy_file.h"
#include "load_manifest.h"
Include dependency graph for reconstruct.h:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Functions

void reconstruct_from_incremental_file (char *input_filename, char *output_filename, char *relative_path, char *bare_file_name, int n_prior_backups, char **prior_backup_dirs, manifest_data **manifests, char *manifest_path, pg_checksum_type checksum_type, int *checksum_length, uint8 **checksum_payload, CopyMethod copy_method, bool debug, bool dry_run)
 

Function Documentation

◆ reconstruct_from_incremental_file()

void reconstruct_from_incremental_file ( char *  input_filename,
char *  output_filename,
char *  relative_path,
char *  bare_file_name,
int  n_prior_backups,
char **  prior_backup_dirs,
manifest_data **  manifests,
char *  manifest_path,
pg_checksum_type  checksum_type,
int *  checksum_length,
uint8 **  checksum_payload,
CopyMethod  copy_method,
bool  debug,
bool  dry_run 
)

Definition at line 88 of file reconstruct.c.

102{
103 rfile **source;
104 rfile *latest_source = NULL;
105 rfile **sourcemap;
106 off_t *offsetmap;
107 unsigned block_length;
108 unsigned i;
109 unsigned sidx = n_prior_backups;
110 bool full_copy_possible = true;
111 int copy_source_index = -1;
112 rfile *copy_source = NULL;
113 pg_checksum_context checksum_ctx;
114
115 /* Sanity check the relative_path. */
116 Assert(relative_path[0] != '\0');
117 Assert(relative_path[strlen(relative_path) - 1] == '/');
118
119 /*
120 * Every block must come either from the latest version of the file or
121 * from one of the prior backups.
122 */
123 source = pg_malloc0(sizeof(rfile *) * (1 + n_prior_backups));
124
125 /*
126 * Use the information from the latest incremental file to figure out how
127 * long the reconstructed file should be.
128 */
129 latest_source = make_incremental_rfile(input_filename);
130 source[n_prior_backups] = latest_source;
131 block_length = find_reconstructed_block_length(latest_source);
132
133 /*
134 * For each block in the output file, we need to know from which file we
135 * need to obtain it and at what offset in that file it's stored.
136 * sourcemap gives us the first of these things, and offsetmap the latter.
137 */
138 sourcemap = pg_malloc0(sizeof(rfile *) * block_length);
139 offsetmap = pg_malloc0(sizeof(off_t) * block_length);
140
141 /*
142 * Every block that is present in the newest incremental file should be
143 * sourced from that file. If it precedes the truncation_block_length,
144 * it's a block that we would otherwise have had to find in an older
145 * backup and thus reduces the number of blocks remaining to be found by
146 * one; otherwise, it's an extra block that needs to be included in the
147 * output but would not have needed to be found in an older backup if it
148 * had not been present.
149 */
150 for (i = 0; i < latest_source->num_blocks; ++i)
151 {
152 BlockNumber b = latest_source->relative_block_numbers[i];
153
154 Assert(b < block_length);
155 sourcemap[b] = latest_source;
156 offsetmap[b] = latest_source->header_length + (i * BLCKSZ);
157
158 /*
159 * A full copy of a file from an earlier backup is only possible if no
160 * blocks are needed from any later incremental file.
161 */
162 full_copy_possible = false;
163 }
164
165 while (1)
166 {
167 char source_filename[MAXPGPATH];
168 rfile *s;
169
170 /*
171 * Move to the next backup in the chain. If there are no more, then
172 * we're done.
173 */
174 if (sidx == 0)
175 break;
176 --sidx;
177
178 /*
179 * Look for the full file in the previous backup. If not found, then
180 * look for an incremental file instead.
181 */
182 snprintf(source_filename, MAXPGPATH, "%s/%s%s",
183 prior_backup_dirs[sidx], relative_path, bare_file_name);
184 if ((s = make_rfile(source_filename, true)) == NULL)
185 {
186 snprintf(source_filename, MAXPGPATH, "%s/%sINCREMENTAL.%s",
187 prior_backup_dirs[sidx], relative_path, bare_file_name);
188 s = make_incremental_rfile(source_filename);
189 }
190 source[sidx] = s;
191
192 /*
193 * If s->header_length == 0, then this is a full file; otherwise, it's
194 * an incremental file.
195 */
196 if (s->header_length == 0)
197 {
198 struct stat sb;
200 BlockNumber blocklength;
201
202 /* We need to know the length of the file. */
203 if (fstat(s->fd, &sb) < 0)
204 pg_fatal("could not stat file \"%s\": %m", s->filename);
205
206 /*
207 * Since we found a full file, source all blocks from it that
208 * exist in the file.
209 *
210 * Note that there may be blocks that don't exist either in this
211 * file or in any incremental file but that precede
212 * truncation_block_length. These are, presumably, zero-filled
213 * blocks that result from the server extending the file but
214 * taking no action on those blocks that generated any WAL.
215 *
216 * Sadly, we have no way of validating that this is really what
217 * happened, and neither does the server. From it's perspective,
218 * an unmodified block that contains data looks exactly the same
219 * as a zero-filled block that never had any data: either way,
220 * it's not mentioned in any WAL summary and the server has no
221 * reason to read it. From our perspective, all we know is that
222 * nobody had a reason to back up the block. That certainly means
223 * that the block didn't exist at the time of the full backup, but
224 * the supposition that it was all zeroes at the time of every
225 * later backup is one that we can't validate.
226 */
227 blocklength = sb.st_size / BLCKSZ;
228 for (b = 0; b < latest_source->truncation_block_length; ++b)
229 {
230 if (sourcemap[b] == NULL && b < blocklength)
231 {
232 sourcemap[b] = s;
233 offsetmap[b] = b * BLCKSZ;
234 }
235 }
236
237 /*
238 * If a full copy looks possible, check whether the resulting file
239 * should be exactly as long as the source file is. If so, a full
240 * copy is acceptable, otherwise not.
241 */
242 if (full_copy_possible)
243 {
244 uint64 expected_length;
245
246 expected_length =
247 (uint64) latest_source->truncation_block_length;
248 expected_length *= BLCKSZ;
249 if (expected_length == sb.st_size)
250 {
251 copy_source = s;
252 copy_source_index = sidx;
253 }
254 }
255
256 /* We don't need to consider any further sources. */
257 break;
258 }
259
260 /*
261 * Since we found another incremental file, source all blocks from it
262 * that we need but don't yet have.
263 */
264 for (i = 0; i < s->num_blocks; ++i)
265 {
267
268 if (b < latest_source->truncation_block_length &&
269 sourcemap[b] == NULL)
270 {
271 sourcemap[b] = s;
272 offsetmap[b] = s->header_length + (i * BLCKSZ);
273
274 /*
275 * A full copy of a file from an earlier backup is only
276 * possible if no blocks are needed from any later incremental
277 * file.
278 */
279 full_copy_possible = false;
280 }
281 }
282 }
283
284 /*
285 * If a checksum of the required type already exists in the
286 * backup_manifest for the relevant input directory, we can save some work
287 * by reusing that checksum instead of computing a new one.
288 */
289 if (copy_source_index >= 0 && manifests[copy_source_index] != NULL &&
290 checksum_type != CHECKSUM_TYPE_NONE)
291 {
292 manifest_file *mfile;
293
294 mfile = manifest_files_lookup(manifests[copy_source_index]->files,
295 manifest_path);
296 if (mfile == NULL)
297 {
298 char *path = psprintf("%s/backup_manifest",
299 prior_backup_dirs[copy_source_index]);
300
301 /*
302 * The directory is out of sync with the backup_manifest, so emit
303 * a warning.
304 */
305 pg_log_warning("manifest file \"%s\" contains no entry for file \"%s\"",
306 path,
307 manifest_path);
308 pfree(path);
309 }
310 else if (mfile->checksum_type == checksum_type)
311 {
312 *checksum_length = mfile->checksum_length;
313 *checksum_payload = pg_malloc(*checksum_length);
314 memcpy(*checksum_payload, mfile->checksum_payload,
315 *checksum_length);
316 checksum_type = CHECKSUM_TYPE_NONE;
317 }
318 }
319
320 /* Prepare for checksum calculation, if required. */
321 pg_checksum_init(&checksum_ctx, checksum_type);
322
323 /*
324 * If the full file can be created by copying a file from an older backup
325 * in the chain without needing to overwrite any blocks or truncate the
326 * result, then forget about performing reconstruction and just copy that
327 * file in its entirety.
328 *
329 * If we have only incremental files, and there's no full file at any
330 * point in the backup chain, something has gone wrong. Emit an error.
331 *
332 * Otherwise, reconstruct.
333 */
334 if (copy_source != NULL)
335 copy_file(copy_source->filename, output_filename,
336 &checksum_ctx, copy_method, dry_run);
337 else if (sidx == 0 && source[0]->header_length != 0)
338 {
339 pg_fatal("full backup contains unexpected incremental file \"%s\"",
340 source[0]->filename);
341 }
342 else
343 {
345 block_length, sourcemap, offsetmap,
346 &checksum_ctx, copy_method,
347 debug, dry_run);
348 debug_reconstruction(n_prior_backups + 1, source, dry_run);
349 }
350
351 /* Save results of checksum calculation. */
352 if (checksum_type != CHECKSUM_TYPE_NONE)
353 {
354 *checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
355 *checksum_length = pg_checksum_final(&checksum_ctx,
356 *checksum_payload);
357 }
358
359 /*
360 * Close files and release memory.
361 */
362 for (i = 0; i <= n_prior_backups; ++i)
363 {
364 rfile *s = source[i];
365
366 if (s == NULL)
367 continue;
368 if (close(s->fd) != 0)
369 pg_fatal("could not close file \"%s\": %m", s->filename);
370 if (s->relative_block_numbers != NULL)
372 pg_free(s->filename);
373 }
374 pfree(sourcemap);
375 pfree(offsetmap);
376 pfree(source);
377}
uint32 BlockNumber
Definition: block.h:31
#define Assert(condition)
Definition: c.h:812
uint64_t uint64
Definition: c.h:486
int pg_checksum_final(pg_checksum_context *context, uint8 *output)
int pg_checksum_init(pg_checksum_context *context, pg_checksum_type type)
#define PG_CHECKSUM_MAX_LENGTH
@ CHECKSUM_TYPE_NONE
void copy_file(const char *fromfile, const char *tofile)
Definition: copydir.c:117
char * output_filename
Definition: ecpg.c:24
void * pg_malloc(size_t size)
Definition: fe_memutils.c:47
void * pg_malloc0(size_t size)
Definition: fe_memutils.c:53
void pg_free(void *ptr)
Definition: fe_memutils.c:105
static bool debug
Definition: initdb.c:161
#define close(a)
Definition: win32.h:12
int b
Definition: isn.c:69
int i
Definition: isn.c:72
if(TABLE==NULL||TABLE_index==NULL)
Definition: isn.c:76
void pfree(void *pointer)
Definition: mcxt.c:1521
#define pg_fatal(...)
#define MAXPGPATH
static bool dry_run
static char * filename
Definition: pg_dumpall.c:119
static rewind_source * source
Definition: pg_rewind.c:89
#define pg_log_warning(...)
Definition: pgfnames.c:24
#define snprintf
Definition: port.h:238
char * input_filename
char * psprintf(const char *fmt,...)
Definition: psprintf.c:43
static rfile * make_rfile(char *filename, bool missing_ok)
Definition: reconstruct.c:510
static rfile * make_incremental_rfile(char *filename)
Definition: reconstruct.c:455
static void debug_reconstruction(int n_source, rfile **sources, bool dry_run)
Definition: reconstruct.c:383
static void write_reconstructed_file(char *input_filename, char *output_filename, unsigned block_length, rfile **sourcemap, off_t *offsetmap, pg_checksum_context *checksum_ctx, CopyMethod copy_method, bool debug, bool dry_run)
Definition: reconstruct.c:551
static unsigned find_reconstructed_block_length(rfile *s)
Definition: reconstruct.c:438
uint8 * checksum_payload
Definition: load_manifest.h:29
pg_checksum_type checksum_type
Definition: load_manifest.h:27
BlockNumber * relative_block_numbers
Definition: reconstruct.c:43
int fd
Definition: reconstruct.c:40
unsigned num_blocks
Definition: reconstruct.c:42
size_t header_length
Definition: reconstruct.c:41
char * filename
Definition: reconstruct.c:39
unsigned truncation_block_length
Definition: reconstruct.c:44
#define fstat
Definition: win32_port.h:283

References Assert, b, manifest_file::checksum_length, manifest_file::checksum_payload, manifest_file::checksum_type, CHECKSUM_TYPE_NONE, close, copy_file(), debug, debug_reconstruction(), dry_run, rfile::fd, rfile::filename, filename, find_reconstructed_block_length(), fstat, rfile::header_length, i, if(), input_filename, make_incremental_rfile(), make_rfile(), MAXPGPATH, rfile::num_blocks, output_filename, pfree(), pg_checksum_final(), pg_checksum_init(), PG_CHECKSUM_MAX_LENGTH, pg_fatal, pg_free(), pg_log_warning, pg_malloc(), pg_malloc0(), psprintf(), rfile::relative_block_numbers, snprintf, source, stat::st_size, rfile::truncation_block_length, and write_reconstructed_file().

Referenced by process_directory_recursively().