load_manifest.c File Reference

#include "postgres_fe.h"
#include <sys/stat.h>
#include <unistd.h>
#include "common/hashfn_unstable.h"
#include "common/logging.h"
#include "common/parse_manifest.h"
#include "load_manifest.h"
#include "lib/simplehash.h"

Include dependency graph for load_manifest.c:

Go to the source code of this file.

Macros
#define	ESTIMATED_BYTES_PER_MANIFEST_LINE   100
#define	READ_CHUNK_SIZE   (128 * 1024)
#define	SH_PREFIX   manifest_files
#define	SH_ELEMENT_TYPE   manifest_file
#define	SH_KEY_TYPE   const char *
#define	SH_KEY   pathname
#define	SH_HASH_KEY(tb, key)   hash_string(key)
#define	SH_EQUAL(tb, a, b)   (strcmp(a, b) == 0)
#define	SH_SCOPE   extern
#define	SH_RAW_ALLOCATOR   pg_malloc0
#define	SH_DEFINE

Functions
static void	combinebackup_version_cb (JsonManifestParseContext *context, int manifest_version)
static void	combinebackup_system_identifier_cb (JsonManifestParseContext *context, uint64 manifest_system_identifier)
static void	combinebackup_per_file_cb (JsonManifestParseContext *context, const char *pathname, uint64 size, pg_checksum_type checksum_type, int checksum_length, uint8 *checksum_payload)
static void	combinebackup_per_wal_range_cb (JsonManifestParseContext *context, TimeLineID tli, XLogRecPtr start_lsn, XLogRecPtr end_lsn)
static void	report_manifest_error (JsonManifestParseContext *context, const char *fmt,...) pg_attribute_printf(2
static void	pg_attribute_noreturn ()
manifest_data **	load_backup_manifests (int n_backups, char **backup_directories)
manifest_data *	load_backup_manifest (char *backup_directory)

Macro Definition Documentation

ESTIMATED_BYTES_PER_MANIFEST_LINE

#define ESTIMATED_BYTES_PER_MANIFEST_LINE   100

Definition at line 35 of file load_manifest.c.

READ_CHUNK_SIZE

#define READ_CHUNK_SIZE   (128 * 1024)

Definition at line 41 of file load_manifest.c.

SH_DEFINE

#define SH_DEFINE

Definition at line 55 of file load_manifest.c.

SH_ELEMENT_TYPE

#define SH_ELEMENT_TYPE   manifest_file

Definition at line 48 of file load_manifest.c.

SH_EQUAL

#define SH_EQUAL	(		tb,
			a,
			b
	)		(strcmp(a, b) == 0)

Definition at line 52 of file load_manifest.c.

SH_HASH_KEY

#define SH_HASH_KEY	(		tb,
			key
	)		hash_string(key)

Definition at line 51 of file load_manifest.c.

SH_KEY

#define SH_KEY   pathname

Definition at line 50 of file load_manifest.c.

SH_KEY_TYPE

#define SH_KEY_TYPE   const char *

Definition at line 49 of file load_manifest.c.

SH_PREFIX

#define SH_PREFIX   manifest_files

Definition at line 47 of file load_manifest.c.

SH_RAW_ALLOCATOR

#define SH_RAW_ALLOCATOR   pg_malloc0

Definition at line 54 of file load_manifest.c.

SH_SCOPE

#define SH_SCOPE   extern

Definition at line 53 of file load_manifest.c.

Function Documentation

combinebackup_per_file_cb()

static void combinebackup_per_file_cb ( JsonManifestParseContext *  context, const char *  pathname, uint64  size, pg_checksum_type  checksum_type, int  checksum_length, uint8 *  checksum_payload  )

static

Definition at line 268 of file load_manifest.c.

{
    manifest_data *manifest = context->private_data;
    manifest_file *m;
    bool        found;
 
    /* Make a new entry in the hash table for this file. */
    m = manifest_files_insert(manifest->files, pathname, &found);
    if (found)
        pg_fatal("duplicate path name in backup manifest: \"%s\"", pathname);
 
    /* Initialize the entry. */
    m->size = size;
    m->checksum_type = checksum_type;
    m->checksum_length = checksum_length;
    m->checksum_payload = checksum_payload;
}

References manifest_file::checksum_length, manifest_file::checksum_payload, manifest_file::checksum_type, context, manifest, pg_fatal, size, and manifest_file::size.

Referenced by load_backup_manifest().

combinebackup_per_wal_range_cb()

static void combinebackup_per_wal_range_cb ( JsonManifestParseContext *  context, TimeLineID  tli, XLogRecPtr  start_lsn, XLogRecPtr  end_lsn  )

static

Definition at line 293 of file load_manifest.c.

{
    manifest_data *manifest = context->private_data;
    manifest_wal_range *range;
 
    /* Allocate and initialize a struct describing this WAL range. */
    range = palloc(sizeof(manifest_wal_range));
    range->tli = tli;
    range->start_lsn = start_lsn;
    range->end_lsn = end_lsn;
    range->prev = manifest->last_wal_range;
    range->next = NULL;
 
    /* Add it to the end of the list. */
    if (manifest->first_wal_range == NULL)
        manifest->first_wal_range = range;
    else
        manifest->last_wal_range->next = range;
    manifest->last_wal_range = range;
}

References context, manifest, palloc(), and range().

Referenced by load_backup_manifest().

combinebackup_system_identifier_cb()

static void combinebackup_system_identifier_cb ( JsonManifestParseContext *  context, uint64  manifest_system_identifier  )

static

Definition at line 255 of file load_manifest.c.

{
    manifest_data *manifest = context->private_data;
 
    /* Validation will be at the later stage */
    manifest->system_identifier = manifest_system_identifier;
}

References context, and manifest.

Referenced by load_backup_manifest().

combinebackup_version_cb()

static void combinebackup_version_cb ( JsonManifestParseContext *  context, int  manifest_version  )

static

Definition at line 243 of file load_manifest.c.

{
    /* Incremental backups supported on manifest version 2 or later */
    if (manifest_version == 1)
        pg_fatal("backup manifest version 1 does not support incremental backup");
}

References pg_fatal.

Referenced by load_backup_manifest().

load_backup_manifest()

manifest_data* load_backup_manifest

(

char *

backup_directory

)

Definition at line 105 of file load_manifest.c.

{
    char        pathname[MAXPGPATH];
    int         fd;
    struct stat statbuf;
    off_t       estimate;
    uint32      initial_size;
    manifest_files_hash *ht;
    char       *buffer;
    int         rc;
    JsonManifestParseContext context;
    manifest_data *result;
    int         chunk_size = READ_CHUNK_SIZE;
 
    /* Open the manifest file. */
    snprintf(pathname, MAXPGPATH, "%s/backup_manifest", backup_directory);
    if ((fd = open(pathname, O_RDONLY | PG_BINARY, 0)) < 0)
    {
        if (errno == ENOENT)
        {
            pg_log_warning("file \"%s\" does not exist", pathname);
            return NULL;
        }
        pg_fatal("could not open file \"%s\": %m", pathname);
    }
 
    /* Figure out how big the manifest is. */
    if (fstat(fd, &statbuf) != 0)
        pg_fatal("could not stat file \"%s\": %m", pathname);
 
    /* Guess how large to make the hash table based on the manifest size. */
    estimate = statbuf.st_size / ESTIMATED_BYTES_PER_MANIFEST_LINE;
    initial_size = Min(PG_UINT32_MAX, Max(estimate, 256));
 
    /* Create the hash table. */
    ht = manifest_files_create(initial_size, NULL);
 
    result = pg_malloc0(sizeof(manifest_data));
    result->files = ht;
    context.private_data = result;
    context.version_cb = combinebackup_version_cb;
    context.system_identifier_cb = combinebackup_system_identifier_cb;
    context.per_file_cb = combinebackup_per_file_cb;
    context.per_wal_range_cb = combinebackup_per_wal_range_cb;
    context.error_cb = report_manifest_error;
 
    /*
     * Parse the file, in chunks if necessary.
     */
    if (statbuf.st_size <= chunk_size)
    {
        buffer = pg_malloc(statbuf.st_size);
        rc = read(fd, buffer, statbuf.st_size);
        if (rc != statbuf.st_size)
        {
            if (rc < 0)
                pg_fatal("could not read file \"%s\": %m", pathname);
            else
                pg_fatal("could not read file \"%s\": read %d of %lld",
                         pathname, rc, (long long int) statbuf.st_size);
        }
 
        /* Close the manifest file. */
        close(fd);
 
        /* Parse the manifest. */
        json_parse_manifest(&context, buffer, statbuf.st_size);
    }
    else
    {
        int         bytes_left = statbuf.st_size;
        JsonManifestParseIncrementalState *inc_state;
 
        inc_state = json_parse_manifest_incremental_init(&context);
 
        buffer = pg_malloc(chunk_size + 1);
 
        while (bytes_left > 0)
        {
            int         bytes_to_read = chunk_size;
 
            /*
             * Make sure that the last chunk is sufficiently large. (i.e. at
             * least half the chunk size) so that it will contain fully the
             * piece at the end with the checksum.
             */
            if (bytes_left < chunk_size)
                bytes_to_read = bytes_left;
            else if (bytes_left < 2 * chunk_size)
                bytes_to_read = bytes_left / 2;
            rc = read(fd, buffer, bytes_to_read);
            if (rc != bytes_to_read)
            {
                if (rc < 0)
                    pg_fatal("could not read file \"%s\": %m", pathname);
                else
                    pg_fatal("could not read file \"%s\": read %lld of %lld",
                             pathname,
                             (long long int) (statbuf.st_size + rc - bytes_left),
                             (long long int) statbuf.st_size);
            }
            bytes_left -= rc;
            json_parse_manifest_incremental_chunk(inc_state, buffer, rc, bytes_left == 0);
        }
 
        /* Release the incremental state memory */
        json_parse_manifest_incremental_shutdown(inc_state);
 
        close(fd);
    }
 
    /* All done. */
    pfree(buffer);
    return result;
}

References close, combinebackup_per_file_cb(), combinebackup_per_wal_range_cb(), combinebackup_system_identifier_cb(), combinebackup_version_cb(), context, ESTIMATED_BYTES_PER_MANIFEST_LINE, fd(), manifest_data::files, fstat, json_parse_manifest(), json_parse_manifest_incremental_chunk(), json_parse_manifest_incremental_init(), json_parse_manifest_incremental_shutdown(), Max, MAXPGPATH, Min, pfree(), PG_BINARY, pg_fatal, pg_log_warning, pg_malloc(), pg_malloc0(), PG_UINT32_MAX, read, READ_CHUNK_SIZE, report_manifest_error(), snprintf, and stat::st_size.

Referenced by load_backup_manifests().

load_backup_manifests()

manifest_data** load_backup_manifests	(	int	n_backups,
		char **	backup_directories
	)

Definition at line 83 of file load_manifest.c.

{
    manifest_data **result;
    int         i;
 
    result = pg_malloc(sizeof(manifest_data *) * n_backups);
    for (i = 0; i < n_backups; ++i)
        result[i] = load_backup_manifest(backup_directories[i]);
 
    return result;
}

References i, load_backup_manifest(), and pg_malloc().

Referenced by main().

pg_attribute_noreturn()

static void pg_attribute_noreturn

(

)

report_manifest_error()

static void report_manifest_error ( JsonManifestParseContext *  context, const char *  fmt,   ...  )

static

Definition at line 228 of file load_manifest.c.

{
    va_list     ap;
 
    va_start(ap, fmt);
    pg_log_generic_v(PG_LOG_ERROR, PG_LOG_PRIMARY, gettext(fmt), ap);
    va_end(ap);
 
    exit(1);
}

References exit(), fmt, gettext, PG_LOG_ERROR, pg_log_generic_v(), PG_LOG_PRIMARY, va_end(), and va_start().

Referenced by load_backup_manifest().