backup_manifest.h File Reference

#include "backup/basebackup_sink.h"
#include "common/checksum_helper.h"
#include "pgtime.h"
#include "storage/buffile.h"

Include dependency graph for backup_manifest.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures
struct	backup_manifest_info

Typedefs
typedef enum manifest_option	backup_manifest_option
typedef struct backup_manifest_info	backup_manifest_info

Enumerations
enum	manifest_option { MANIFEST_OPTION_YES , MANIFEST_OPTION_NO , MANIFEST_OPTION_FORCE_ENCODE }

Functions
void	InitializeBackupManifest (backup_manifest_info *manifest, backup_manifest_option want_manifest, pg_checksum_type manifest_checksum_type)
void	AddFileToBackupManifest (backup_manifest_info *manifest, Oid spcoid, const char *pathname, size_t size, pg_time_t mtime, pg_checksum_context *checksum_ctx)
void	AddWALInfoToBackupManifest (backup_manifest_info *manifest, XLogRecPtr startptr, TimeLineID starttli, XLogRecPtr endptr, TimeLineID endtli)
void	SendBackupManifest (backup_manifest_info *manifest, bbsink *sink)
void	FreeBackupManifest (backup_manifest_info *manifest)

Typedef Documentation

backup_manifest_info

typedef struct backup_manifest_info backup_manifest_info

backup_manifest_option

typedef enum manifest_option backup_manifest_option

Enumeration Type Documentation

manifest_option

enum manifest_option

Enumerator
MANIFEST_OPTION_YES
MANIFEST_OPTION_NO
MANIFEST_OPTION_FORCE_ENCODE

Definition at line 20 of file backup_manifest.h.

{
    MANIFEST_OPTION_YES,
    MANIFEST_OPTION_NO,
    MANIFEST_OPTION_FORCE_ENCODE,
} backup_manifest_option;

Function Documentation

AddFileToBackupManifest()

void AddFileToBackupManifest	(	backup_manifest_info *	manifest,
		Oid	spcoid,
		const char *	pathname,
		size_t	size,
		pg_time_t	mtime,
		pg_checksum_context *	checksum_ctx
	)

Definition at line 102 of file backup_manifest.c.

{
    char        pathbuf[MAXPGPATH];
    int         pathlen;
    StringInfoData buf;
 
    if (!IsManifestEnabled(manifest))
        return;
 
    /*
     * If this file is part of a tablespace, the pathname passed to this
     * function will be relative to the tar file that contains it. We want the
     * pathname relative to the data directory (ignoring the intermediate
     * symlink traversal).
     */
    if (OidIsValid(spcoid))
    {
        snprintf(pathbuf, sizeof(pathbuf), "%s/%u/%s", PG_TBLSPC_DIR, spcoid,
                 pathname);
        pathname = pathbuf;
    }
 
    /*
     * Each file's entry needs to be separated from any entry that follows by
     * a comma, but there's no comma before the first one or after the last
     * one. To make that work, adding a file to the manifest starts by
     * terminating the most recently added line, with a comma if appropriate,
     * but does not terminate the line inserted for this file.
     */
    initStringInfo(&buf);
    if (manifest->first_file)
    {
        appendStringInfoChar(&buf, '\n');
        manifest->first_file = false;
    }
    else
        appendStringInfoString(&buf, ",\n");
 
    /*
     * Write the relative pathname to this file out to the manifest. The
     * manifest is always stored in UTF-8, so we have to encode paths that are
     * not valid in that encoding.
     */
    pathlen = strlen(pathname);
    if (!manifest->force_encode &&
        pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
    {
        appendStringInfoString(&buf, "{ \"Path\": ");
        escape_json_with_len(&buf, pathname, pathlen);
        appendStringInfoString(&buf, ", ");
    }
    else
    {
        appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
        enlargeStringInfo(&buf, 2 * pathlen);
        buf.len += hex_encode(pathname, pathlen,
                              &buf.data[buf.len]);
        appendStringInfoString(&buf, "\", ");
    }
 
    appendStringInfo(&buf, "\"Size\": %zu, ", size);
 
    /*
     * Convert last modification time to a string and append it to the
     * manifest. Since it's not clear what time zone to use and since time
     * zone definitions can change, possibly causing confusion, use GMT
     * always.
     */
    appendStringInfoString(&buf, "\"Last-Modified\": \"");
    enlargeStringInfo(&buf, 128);
    buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
                           pg_gmtime(&mtime));
    appendStringInfoChar(&buf, '"');
 
    /* Add checksum information. */
    if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
    {
        uint8       checksumbuf[PG_CHECKSUM_MAX_LENGTH];
        int         checksumlen;
 
        checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
        if (checksumlen < 0)
            elog(ERROR, "could not finalize checksum of file \"%s\"",
                 pathname);
 
        appendStringInfo(&buf,
                         ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
                         pg_checksum_type_name(checksum_ctx->type));
        enlargeStringInfo(&buf, 2 * checksumlen);
        buf.len += hex_encode((char *) checksumbuf, checksumlen,
                              &buf.data[buf.len]);
        appendStringInfoChar(&buf, '"');
    }
 
    /* Close out the object. */
    appendStringInfoString(&buf, " }");
 
    /* OK, add it to the manifest. */
    AppendStringToManifest(manifest, buf.data);
 
    /* Avoid leaking memory. */
    pfree(buf.data);
}

References appendStringInfo(), appendStringInfoChar(), appendStringInfoString(), AppendStringToManifest(), buf, CHECKSUM_TYPE_NONE, elog, enlargeStringInfo(), ERROR, escape_json_with_len(), hex_encode(), initStringInfo(), IsManifestEnabled(), manifest, MAXPGPATH, OidIsValid, pfree(), pg_checksum_final(), PG_CHECKSUM_MAX_LENGTH, pg_checksum_type_name(), pg_gmtime(), pg_strftime(), PG_TBLSPC_DIR, PG_UTF8, pg_verify_mbstr(), snprintf, and pg_checksum_context::type.

Referenced by sendFile(), and sendFileWithContent().

AddWALInfoToBackupManifest()

void AddWALInfoToBackupManifest	(	backup_manifest_info *	manifest,
		XLogRecPtr	startptr,
		TimeLineID	starttli,
		XLogRecPtr	endptr,
		TimeLineID	endtli
	)

Definition at line 213 of file backup_manifest.c.

{
    List       *timelines;
    ListCell   *lc;
    bool        first_wal_range = true;
    bool        found_start_timeline = false;
 
    if (!IsManifestEnabled(manifest))
        return;
 
    /* Terminate the list of files. */
    AppendStringToManifest(manifest, "\n],\n");
 
    /* Read the timeline history for the ending timeline. */
    timelines = readTimeLineHistory(endtli);
 
    /* Start a list of LSN ranges. */
    AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
 
    foreach(lc, timelines)
    {
        TimeLineHistoryEntry *entry = lfirst(lc);
        XLogRecPtr  tl_beginptr;
 
        /*
         * We only care about timelines that were active during the backup.
         * Skip any that ended before the backup started. (Note that if
         * entry->end is InvalidXLogRecPtr, it means that the timeline has not
         * yet ended.)
         */
        if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
            continue;
 
        /*
         * Because the timeline history file lists newer timelines before
         * older ones, the first timeline we encounter that is new enough to
         * matter ought to match the ending timeline of the backup.
         */
        if (first_wal_range && endtli != entry->tli)
            ereport(ERROR,
                    errmsg("expected end timeline %u but found timeline %u",
                           starttli, entry->tli));
 
        /*
         * If this timeline entry matches with the timeline on which the
         * backup started, WAL needs to be checked from the start LSN of the
         * backup.  If this entry refers to a newer timeline, WAL needs to be
         * checked since the beginning of this timeline, so use the LSN where
         * the timeline began.
         */
        if (starttli == entry->tli)
            tl_beginptr = startptr;
        else
        {
            tl_beginptr = entry->begin;
 
            /*
             * If we reach a TLI that has no valid beginning LSN, there can't
             * be any more timelines in the history after this point, so we'd
             * better have arrived at the expected starting TLI. If not,
             * something's gone horribly wrong.
             */
            if (XLogRecPtrIsInvalid(entry->begin))
                ereport(ERROR,
                        errmsg("expected start timeline %u but found timeline %u",
                               starttli, entry->tli));
        }
 
        AppendToManifest(manifest,
                         "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
                         first_wal_range ? "" : ",\n",
                         entry->tli,
                         LSN_FORMAT_ARGS(tl_beginptr),
                         LSN_FORMAT_ARGS(endptr));
 
        if (starttli == entry->tli)
        {
            found_start_timeline = true;
            break;
        }
 
        endptr = entry->begin;
        first_wal_range = false;
    }
 
    /*
     * The last entry in the timeline history for the ending timeline should
     * be the ending timeline itself. Verify that this is what we observed.
     */
    if (!found_start_timeline)
        ereport(ERROR,
                errmsg("start timeline %u not found in history of timeline %u",
                       starttli, endtli));
 
    /* Terminate the list of WAL ranges. */
    AppendStringToManifest(manifest, "\n],\n");
}

References AppendStringToManifest(), AppendToManifest, TimeLineHistoryEntry::begin, TimeLineHistoryEntry::end, ereport, errmsg(), ERROR, IsManifestEnabled(), lfirst, LSN_FORMAT_ARGS, manifest, readTimeLineHistory(), TimeLineHistoryEntry::tli, and XLogRecPtrIsInvalid.

Referenced by perform_base_backup().

FreeBackupManifest()

void FreeBackupManifest

(

backup_manifest_info *

manifest

)

Definition at line 92 of file backup_manifest.c.

{
    pg_cryptohash_free(manifest->manifest_ctx);
    manifest->manifest_ctx = NULL;
}

References manifest, and pg_cryptohash_free().

Referenced by perform_base_backup().

InitializeBackupManifest()

void InitializeBackupManifest	(	backup_manifest_info *	manifest,
		backup_manifest_option	want_manifest,
		pg_checksum_type	manifest_checksum_type
	)

Definition at line 57 of file backup_manifest.c.

{
    memset(manifest, 0, sizeof(backup_manifest_info));
    manifest->checksum_type = manifest_checksum_type;
 
    if (want_manifest == MANIFEST_OPTION_NO)
        manifest->buffile = NULL;
    else
    {
        manifest->buffile = BufFileCreateTemp(false);
        manifest->manifest_ctx = pg_cryptohash_create(PG_SHA256);
        if (pg_cryptohash_init(manifest->manifest_ctx) < 0)
            elog(ERROR, "failed to initialize checksum of backup manifest: %s",
                 pg_cryptohash_error(manifest->manifest_ctx));
    }
 
    manifest->manifest_size = UINT64CONST(0);
    manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE);
    manifest->first_file = true;
    manifest->still_checksumming = true;
 
    if (want_manifest != MANIFEST_OPTION_NO)
        AppendToManifest(manifest,
                         "{ \"PostgreSQL-Backup-Manifest-Version\": 2,\n"
                         "\"System-Identifier\": " UINT64_FORMAT ",\n"
                         "\"Files\": [",
                         GetSystemIdentifier());
}

References AppendToManifest, BufFileCreateTemp(), elog, ERROR, GetSystemIdentifier(), manifest, MANIFEST_OPTION_FORCE_ENCODE, MANIFEST_OPTION_NO, pg_cryptohash_create(), pg_cryptohash_error(), pg_cryptohash_init(), PG_SHA256, UINT64_FORMAT, and UINT64CONST.

Referenced by perform_base_backup().

SendBackupManifest()

void SendBackupManifest	(	backup_manifest_info *	manifest,
		bbsink *	sink
	)

Definition at line 317 of file backup_manifest.c.

{
    uint8       checksumbuf[PG_SHA256_DIGEST_LENGTH];
    char        checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
    size_t      manifest_bytes_done = 0;
 
    if (!IsManifestEnabled(manifest))
        return;
 
    /*
     * Append manifest checksum, so that the problems with the manifest itself
     * can be detected.
     *
     * We always use SHA-256 for this, regardless of what algorithm is chosen
     * for checksumming the files.  If we ever want to make the checksum
     * algorithm used for the manifest file variable, the client will need a
     * way to figure out which algorithm to use as close to the beginning of
     * the manifest file as possible, to avoid having to read the whole thing
     * twice.
     */
    manifest->still_checksumming = false;
    if (pg_cryptohash_final(manifest->manifest_ctx, checksumbuf,
                            sizeof(checksumbuf)) < 0)
        elog(ERROR, "failed to finalize checksum of backup manifest: %s",
             pg_cryptohash_error(manifest->manifest_ctx));
    AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
 
    hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
    checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
 
    AppendStringToManifest(manifest, checksumstringbuf);
    AppendStringToManifest(manifest, "\"}\n");
 
    /*
     * We've written all the data to the manifest file.  Rewind the file so
     * that we can read it all back.
     */
    if (BufFileSeek(manifest->buffile, 0, 0, SEEK_SET))
        ereport(ERROR,
                (errcode_for_file_access(),
                 errmsg("could not rewind temporary file")));
 
 
    /*
     * Send the backup manifest.
     */
    bbsink_begin_manifest(sink);
    while (manifest_bytes_done < manifest->manifest_size)
    {
        size_t      bytes_to_read;
 
        bytes_to_read = Min(sink->bbs_buffer_length,
                            manifest->manifest_size - manifest_bytes_done);
        BufFileReadExact(manifest->buffile, sink->bbs_buffer, bytes_to_read);
        bbsink_manifest_contents(sink, bytes_to_read);
        manifest_bytes_done += bytes_to_read;
    }
    bbsink_end_manifest(sink);
 
    /* Release resources */
    BufFileClose(manifest->buffile);
}

References AppendStringToManifest(), bbsink::bbs_buffer, bbsink::bbs_buffer_length, bbsink_begin_manifest(), bbsink_end_manifest(), bbsink_manifest_contents(), BufFileClose(), BufFileReadExact(), BufFileSeek(), elog, ereport, errcode_for_file_access(), errmsg(), ERROR, hex_encode(), IsManifestEnabled(), manifest, Min, pg_cryptohash_error(), pg_cryptohash_final(), PG_SHA256_DIGEST_LENGTH, and PG_SHA256_DIGEST_STRING_LENGTH.

Referenced by perform_base_backup().