reconstruct.h File Reference

#include "common/checksum_helper.h"
#include "copy_file.h"
#include "load_manifest.h"

Include dependency graph for reconstruct.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Functions
void	reconstruct_from_incremental_file (char *input_filename, char *output_filename, char *relative_path, char *bare_file_name, int n_prior_backups, char **prior_backup_dirs, manifest_data **manifests, char *manifest_path, pg_checksum_type checksum_type, int *checksum_length, uint8 **checksum_payload, CopyMethod copy_method, bool debug, bool dry_run)

Function Documentation

reconstruct_from_incremental_file()

void reconstruct_from_incremental_file ( char *  input_filename, char *  output_filename, char *  relative_path, char *  bare_file_name, int  n_prior_backups, char **  prior_backup_dirs, manifest_data **  manifests, char *  manifest_path, pg_checksum_type  checksum_type, int *  checksum_length, uint8 **  checksum_payload, CopyMethod  copy_method, bool  debug, bool  dry_run  )

extern

Definition at line 88 of file reconstruct.c.

{
    rfile     **source;
    rfile      *latest_source = NULL;
    rfile     **sourcemap;
    off_t      *offsetmap;
    unsigned    block_length;
    unsigned    sidx = n_prior_backups;
    bool        full_copy_possible = true;
    int         copy_source_index = -1;
    rfile      *copy_source = NULL;
    pg_checksum_context checksum_ctx;
 
    /* Sanity check the relative_path. */
    Assert(relative_path[0] != '\0');
    Assert(relative_path[strlen(relative_path) - 1] == '/');
 
    /*
     * Every block must come either from the latest version of the file or
     * from one of the prior backups.
     */
    source = pg_malloc0_array(rfile *, 1 + n_prior_backups);
 
    /*
     * Use the information from the latest incremental file to figure out how
     * long the reconstructed file should be.
     */
    latest_source = make_incremental_rfile(input_filename);
    source[n_prior_backups] = latest_source;
    block_length = find_reconstructed_block_length(latest_source);
 
    /*
     * For each block in the output file, we need to know from which file we
     * need to obtain it and at what offset in that file it's stored.
     * sourcemap gives us the first of these things, and offsetmap the latter.
     */
    sourcemap = pg_malloc0_array(rfile *, block_length);
    offsetmap = pg_malloc0_array(off_t, block_length);
 
    /*
     * Every block that is present in the newest incremental file should be
     * sourced from that file. If it precedes the truncation_block_length,
     * it's a block that we would otherwise have had to find in an older
     * backup and thus reduces the number of blocks remaining to be found by
     * one; otherwise, it's an extra block that needs to be included in the
     * output but would not have needed to be found in an older backup if it
     * had not been present.
     */
    for (unsigned i = 0; i < latest_source->num_blocks; ++i)
    {
        BlockNumber b = latest_source->relative_block_numbers[i];
 
        Assert(b < block_length);
        sourcemap[b] = latest_source;
        offsetmap[b] = latest_source->header_length + (i * BLCKSZ);
 
        /*
         * A full copy of a file from an earlier backup is only possible if no
         * blocks are needed from any later incremental file.
         */
        full_copy_possible = false;
    }
 
    while (1)
    {
        char        source_filename[MAXPGPATH];
        rfile      *s;
 
        /*
         * Move to the next backup in the chain. If there are no more, then
         * we're done.
         */
        if (sidx == 0)
            break;
        --sidx;
 
        /*
         * Look for the full file in the previous backup. If not found, then
         * look for an incremental file instead.
         */
        snprintf(source_filename, MAXPGPATH, "%s/%s%s",
                 prior_backup_dirs[sidx], relative_path, bare_file_name);
        if ((s = make_rfile(source_filename, true)) == NULL)
        {
            snprintf(source_filename, MAXPGPATH, "%s/%sINCREMENTAL.%s",
                     prior_backup_dirs[sidx], relative_path, bare_file_name);
            s = make_incremental_rfile(source_filename);
        }
        source[sidx] = s;
 
        /*
         * If s->header_length == 0, then this is a full file; otherwise, it's
         * an incremental file.
         */
        if (s->header_length == 0)
        {
            struct stat sb;
            BlockNumber b;
            BlockNumber blocklength;
 
            /* We need to know the length of the file. */
            if (fstat(s->fd, &sb) < 0)
                pg_fatal("could not stat file \"%s\": %m", s->filename);
 
            /*
             * Since we found a full file, source all blocks from it that
             * exist in the file.
             *
             * Note that there may be blocks that don't exist either in this
             * file or in any incremental file but that precede
             * truncation_block_length. These are, presumably, zero-filled
             * blocks that result from the server extending the file but
             * taking no action on those blocks that generated any WAL.
             *
             * Sadly, we have no way of validating that this is really what
             * happened, and neither does the server.  From its perspective,
             * an unmodified block that contains data looks exactly the same
             * as a zero-filled block that never had any data: either way,
             * it's not mentioned in any WAL summary and the server has no
             * reason to read it. From our perspective, all we know is that
             * nobody had a reason to back up the block. That certainly means
             * that the block didn't exist at the time of the full backup, but
             * the supposition that it was all zeroes at the time of every
             * later backup is one that we can't validate.
             */
            blocklength = sb.st_size / BLCKSZ;
            for (b = 0; b < latest_source->truncation_block_length; ++b)
            {
                if (sourcemap[b] == NULL && b < blocklength)
                {
                    sourcemap[b] = s;
                    offsetmap[b] = b * BLCKSZ;
                }
            }
 
            /*
             * If a full copy looks possible, check whether the resulting file
             * should be exactly as long as the source file is. If so, a full
             * copy is acceptable, otherwise not.
             */
            if (full_copy_possible)
            {
                uint64      expected_length;
 
                expected_length =
                    (uint64) latest_source->truncation_block_length;
                expected_length *= BLCKSZ;
                if (expected_length == sb.st_size)
                {
                    copy_source = s;
                    copy_source_index = sidx;
                }
            }
 
            /* We don't need to consider any further sources. */
            break;
        }
 
        /*
         * Since we found another incremental file, source all blocks from it
         * that we need but don't yet have.
         */
        for (unsigned i = 0; i < s->num_blocks; ++i)
        {
            BlockNumber b = s->relative_block_numbers[i];
 
            if (b < latest_source->truncation_block_length &&
                sourcemap[b] == NULL)
            {
                sourcemap[b] = s;
                offsetmap[b] = s->header_length + (i * BLCKSZ);
 
                /*
                 * A full copy of a file from an earlier backup is only
                 * possible if no blocks are needed from any later incremental
                 * file.
                 */
                full_copy_possible = false;
            }
        }
    }
 
    /*
     * If a checksum of the required type already exists in the
     * backup_manifest for the relevant input directory, we can save some work
     * by reusing that checksum instead of computing a new one.
     */
    if (copy_source_index >= 0 && manifests[copy_source_index] != NULL &&
        checksum_type != CHECKSUM_TYPE_NONE)
    {
        manifest_file *mfile;
 
        mfile = manifest_files_lookup(manifests[copy_source_index]->files,
                                      manifest_path);
        if (mfile == NULL)
        {
            char       *path = psprintf("%s/backup_manifest",
                                        prior_backup_dirs[copy_source_index]);
 
            /*
             * The directory is out of sync with the backup_manifest, so emit
             * a warning.
             */
            pg_log_warning("manifest file \"%s\" contains no entry for file \"%s\"",
                           path,
                           manifest_path);
            pfree(path);
        }
        else if (mfile->checksum_type == checksum_type)
        {
            *checksum_length = mfile->checksum_length;
            *checksum_payload = pg_malloc(*checksum_length);
            memcpy(*checksum_payload, mfile->checksum_payload,
                   *checksum_length);
            checksum_type = CHECKSUM_TYPE_NONE;
        }
    }
 
    /* Prepare for checksum calculation, if required. */
    pg_checksum_init(&checksum_ctx, checksum_type);
 
    /*
     * If the full file can be created by copying a file from an older backup
     * in the chain without needing to overwrite any blocks or truncate the
     * result, then forget about performing reconstruction and just copy that
     * file in its entirety.
     *
     * If we have only incremental files, and there's no full file at any
     * point in the backup chain, something has gone wrong. Emit an error.
     *
     * Otherwise, reconstruct.
     */
    if (copy_source != NULL)
        copy_file(copy_source->filename, output_filename,
                  &checksum_ctx, copy_method, dry_run);
    else if (sidx == 0 && source[0]->header_length != 0)
    {
        pg_fatal("full backup contains unexpected incremental file \"%s\"",
                 source[0]->filename);
    }
    else
    {
        write_reconstructed_file(input_filename, output_filename,
                                 block_length, sourcemap, offsetmap,
                                 &checksum_ctx, copy_method,
                                 debug, dry_run);
        debug_reconstruction(n_prior_backups + 1, source, dry_run);
    }
 
    /* Save results of checksum calculation. */
    if (checksum_type != CHECKSUM_TYPE_NONE)
    {
        *checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
        *checksum_length = pg_checksum_final(&checksum_ctx,
                                             *checksum_payload);
    }
 
    /*
     * Close files and release memory.
     */
    for (int i = 0; i <= n_prior_backups; ++i)
    {
        rfile      *s = source[i];
 
        if (s == NULL)
            continue;
        if (close(s->fd) != 0)
            pg_fatal("could not close file \"%s\": %m", s->filename);
        if (s->relative_block_numbers != NULL)
            pfree(s->relative_block_numbers);
        pg_free(s->filename);
        pg_free(s);
    }
    pg_free(sourcemap);
    pg_free(offsetmap);
    pg_free(source);
}

References Assert, b, manifest_file::checksum_length, manifest_file::checksum_payload, manifest_file::checksum_type, CHECKSUM_TYPE_NONE, close, copy_file(), debug, debug_reconstruction(), dry_run, fb(), rfile::fd, rfile::filename, filename, find_reconstructed_block_length(), fstat, rfile::header_length, i, input_filename, make_incremental_rfile(), make_rfile(), MAXPGPATH, memcpy(), rfile::num_blocks, output_filename, pfree(), pg_checksum_final(), pg_checksum_init(), PG_CHECKSUM_MAX_LENGTH, pg_fatal, pg_free(), pg_log_warning, pg_malloc(), pg_malloc0_array, psprintf(), rfile::relative_block_numbers, snprintf, source, rfile::truncation_block_length, and write_reconstructed_file().

Referenced by process_directory_recursively().