reconstruct.h File Reference

#include "common/checksum_helper.h"
#include "copy_file.h"
#include "load_manifest.h"

Include dependency graph for reconstruct.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Functions
void	reconstruct_from_incremental_file (char *input_filename, char *output_filename, char *relative_path, char *bare_file_name, int n_prior_backups, char **prior_backup_dirs, manifest_data **manifests, char *manifest_path, pg_checksum_type checksum_type, int *checksum_length, uint8 **checksum_payload, CopyMethod copy_method, bool debug, bool dry_run)

Function Documentation

reconstruct_from_incremental_file()

void reconstruct_from_incremental_file	(	char *	input_filename,
		char *	output_filename,
		char *	relative_path,
		char *	bare_file_name,
		int	n_prior_backups,
		char **	prior_backup_dirs,
		manifest_data **	manifests,
		char *	manifest_path,
		pg_checksum_type	checksum_type,
		int *	checksum_length,
		uint8 **	checksum_payload,
		CopyMethod	copy_method,
		bool	debug,
		bool	dry_run
	)

Definition at line 87 of file reconstruct.c.

{
    rfile     **source;
    rfile      *latest_source = NULL;
    rfile     **sourcemap;
    off_t      *offsetmap;
    unsigned    block_length;
    unsigned    i;
    unsigned    sidx = n_prior_backups;
    bool        full_copy_possible = true;
    int         copy_source_index = -1;
    rfile      *copy_source = NULL;
    pg_checksum_context checksum_ctx;
 
    /*
     * Every block must come either from the latest version of the file or
     * from one of the prior backups.
     */
    source = pg_malloc0(sizeof(rfile *) * (1 + n_prior_backups));
 
    /*
     * Use the information from the latest incremental file to figure out how
     * long the reconstructed file should be.
     */
    latest_source = make_incremental_rfile(input_filename);
    source[n_prior_backups] = latest_source;
    block_length = find_reconstructed_block_length(latest_source);
 
    /*
     * For each block in the output file, we need to know from which file we
     * need to obtain it and at what offset in that file it's stored.
     * sourcemap gives us the first of these things, and offsetmap the latter.
     */
    sourcemap = pg_malloc0(sizeof(rfile *) * block_length);
    offsetmap = pg_malloc0(sizeof(off_t) * block_length);
 
    /*
     * Every block that is present in the newest incremental file should be
     * sourced from that file. If it precedes the truncation_block_length,
     * it's a block that we would otherwise have had to find in an older
     * backup and thus reduces the number of blocks remaining to be found by
     * one; otherwise, it's an extra block that needs to be included in the
     * output but would not have needed to be found in an older backup if it
     * had not been present.
     */
    for (i = 0; i < latest_source->num_blocks; ++i)
    {
        BlockNumber b = latest_source->relative_block_numbers[i];
 
        Assert(b < block_length);
        sourcemap[b] = latest_source;
        offsetmap[b] = latest_source->header_length + (i * BLCKSZ);
 
        /*
         * A full copy of a file from an earlier backup is only possible if no
         * blocks are needed from any later incremental file.
         */
        full_copy_possible = false;
    }
 
    while (1)
    {
        char        source_filename[MAXPGPATH];
        rfile      *s;
 
        /*
         * Move to the next backup in the chain. If there are no more, then
         * we're done.
         */
        if (sidx == 0)
            break;
        --sidx;
 
        /*
         * Look for the full file in the previous backup. If not found, then
         * look for an incremental file instead.
         */
        snprintf(source_filename, MAXPGPATH, "%s/%s/%s",
                 prior_backup_dirs[sidx], relative_path, bare_file_name);
        if ((s = make_rfile(source_filename, true)) == NULL)
        {
            snprintf(source_filename, MAXPGPATH, "%s/%s/INCREMENTAL.%s",
                     prior_backup_dirs[sidx], relative_path, bare_file_name);
            s = make_incremental_rfile(source_filename);
        }
        source[sidx] = s;
 
        /*
         * If s->header_length == 0, then this is a full file; otherwise, it's
         * an incremental file.
         */
        if (s->header_length == 0)
        {
            struct stat sb;
            BlockNumber b;
            BlockNumber blocklength;
 
            /* We need to know the length of the file. */
            if (fstat(s->fd, &sb) < 0)
                pg_fatal("could not stat file \"%s\": %m", s->filename);
 
            /*
             * Since we found a full file, source all blocks from it that
             * exist in the file.
             *
             * Note that there may be blocks that don't exist either in this
             * file or in any incremental file but that precede
             * truncation_block_length. These are, presumably, zero-filled
             * blocks that result from the server extending the file but
             * taking no action on those blocks that generated any WAL.
             *
             * Sadly, we have no way of validating that this is really what
             * happened, and neither does the server. From it's perspective,
             * an unmodified block that contains data looks exactly the same
             * as a zero-filled block that never had any data: either way,
             * it's not mentioned in any WAL summary and the server has no
             * reason to read it. From our perspective, all we know is that
             * nobody had a reason to back up the block. That certainly means
             * that the block didn't exist at the time of the full backup, but
             * the supposition that it was all zeroes at the time of every
             * later backup is one that we can't validate.
             */
            blocklength = sb.st_size / BLCKSZ;
            for (b = 0; b < latest_source->truncation_block_length; ++b)
            {
                if (sourcemap[b] == NULL && b < blocklength)
                {
                    sourcemap[b] = s;
                    offsetmap[b] = b * BLCKSZ;
                }
            }
 
            /*
             * If a full copy looks possible, check whether the resulting file
             * should be exactly as long as the source file is. If so, a full
             * copy is acceptable, otherwise not.
             */
            if (full_copy_possible)
            {
                uint64      expected_length;
 
                expected_length =
                    (uint64) latest_source->truncation_block_length;
                expected_length *= BLCKSZ;
                if (expected_length == sb.st_size)
                {
                    copy_source = s;
                    copy_source_index = sidx;
                }
            }
 
            /* We don't need to consider any further sources. */
            break;
        }
 
        /*
         * Since we found another incremental file, source all blocks from it
         * that we need but don't yet have.
         */
        for (i = 0; i < s->num_blocks; ++i)
        {
            BlockNumber b = s->relative_block_numbers[i];
 
            if (b < latest_source->truncation_block_length &&
                sourcemap[b] == NULL)
            {
                sourcemap[b] = s;
                offsetmap[b] = s->header_length + (i * BLCKSZ);
 
                /*
                 * A full copy of a file from an earlier backup is only
                 * possible if no blocks are needed from any later incremental
                 * file.
                 */
                full_copy_possible = false;
            }
        }
    }
 
    /*
     * If a checksum of the required type already exists in the
     * backup_manifest for the relevant input directory, we can save some work
     * by reusing that checksum instead of computing a new one.
     */
    if (copy_source_index >= 0 && manifests[copy_source_index] != NULL &&
        checksum_type != CHECKSUM_TYPE_NONE)
    {
        manifest_file *mfile;
 
        mfile = manifest_files_lookup(manifests[copy_source_index]->files,
                                      manifest_path);
        if (mfile == NULL)
        {
            char       *path = psprintf("%s/backup_manifest",
                                        prior_backup_dirs[copy_source_index]);
 
            /*
             * The directory is out of sync with the backup_manifest, so emit
             * a warning.
             */
            pg_log_warning("manifest file \"%s\" contains no entry for file \"%s\"",
                           path,
                           manifest_path);
            pfree(path);
        }
        else if (mfile->checksum_type == checksum_type)
        {
            *checksum_length = mfile->checksum_length;
            *checksum_payload = pg_malloc(*checksum_length);
            memcpy(*checksum_payload, mfile->checksum_payload,
                   *checksum_length);
            checksum_type = CHECKSUM_TYPE_NONE;
        }
    }
 
    /* Prepare for checksum calculation, if required. */
    pg_checksum_init(&checksum_ctx, checksum_type);
 
    /*
     * If the full file can be created by copying a file from an older backup
     * in the chain without needing to overwrite any blocks or truncate the
     * result, then forget about performing reconstruction and just copy that
     * file in its entirety.
     *
     * Otherwise, reconstruct.
     */
    if (copy_source != NULL)
        copy_file(copy_source->filename, output_filename,
                  &checksum_ctx, copy_method, dry_run);
    else
    {
        write_reconstructed_file(input_filename, output_filename,
                                 block_length, sourcemap, offsetmap,
                                 &checksum_ctx, copy_method,
                                 debug, dry_run);
        debug_reconstruction(n_prior_backups + 1, source, dry_run);
    }
 
    /* Save results of checksum calculation. */
    if (checksum_type != CHECKSUM_TYPE_NONE)
    {
        *checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
        *checksum_length = pg_checksum_final(&checksum_ctx,
                                             *checksum_payload);
    }
 
    /*
     * Close files and release memory.
     */
    for (i = 0; i <= n_prior_backups; ++i)
    {
        rfile      *s = source[i];
 
        if (s == NULL)
            continue;
        if (close(s->fd) != 0)
            pg_fatal("could not close file \"%s\": %m", s->filename);
        if (s->relative_block_numbers != NULL)
            pfree(s->relative_block_numbers);
        pg_free(s->filename);
    }
    pfree(sourcemap);
    pfree(offsetmap);
    pfree(source);
}

References Assert, b, manifest_file::checksum_length, manifest_file::checksum_payload, manifest_file::checksum_type, CHECKSUM_TYPE_NONE, close, copy_file(), debug, debug_reconstruction(), dry_run, rfile::fd, rfile::filename, find_reconstructed_block_length(), fstat, rfile::header_length, i, if(), input_filename, make_incremental_rfile(), make_rfile(), MAXPGPATH, rfile::num_blocks, output_filename, pfree(), pg_checksum_final(), pg_checksum_init(), PG_CHECKSUM_MAX_LENGTH, pg_fatal, pg_free(), pg_log_warning, pg_malloc(), pg_malloc0(), psprintf(), rfile::relative_block_numbers, snprintf, source, stat::st_size, rfile::truncation_block_length, and write_reconstructed_file().

Referenced by process_directory_recursively().