parse_manifest.c

Go to the documentation of this file.

/*-------------------------------------------------------------------------
 *
 * parse_manifest.c
 *    Parse a backup manifest in JSON format.
 *
 * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
 * Portions Copyright (c) 1994, Regents of the University of California
 *
 * src/bin/pg_verifybackup/parse_manifest.c
 *
 *-------------------------------------------------------------------------
 */

#include "postgres_fe.h"

#include "parse_manifest.h"
#include "common/jsonapi.h"

/*
 * Semantic states for JSON manifest parsing.
 */
typedef enum
{
    JM_EXPECT_TOPLEVEL_START,
    JM_EXPECT_TOPLEVEL_END,
    JM_EXPECT_TOPLEVEL_FIELD,
    JM_EXPECT_VERSION_VALUE,
    JM_EXPECT_FILES_START,
    JM_EXPECT_FILES_NEXT,
    JM_EXPECT_THIS_FILE_FIELD,
    JM_EXPECT_THIS_FILE_VALUE,
    JM_EXPECT_WAL_RANGES_START,
    JM_EXPECT_WAL_RANGES_NEXT,
    JM_EXPECT_THIS_WAL_RANGE_FIELD,
    JM_EXPECT_THIS_WAL_RANGE_VALUE,
    JM_EXPECT_MANIFEST_CHECKSUM_VALUE,
    JM_EXPECT_EOF
} JsonManifestSemanticState;

/*
 * Possible fields for one file as described by the manifest.
 */
typedef enum
{
    JMFF_PATH,
    JMFF_ENCODED_PATH,
    JMFF_SIZE,
    JMFF_LAST_MODIFIED,
    JMFF_CHECKSUM_ALGORITHM,
    JMFF_CHECKSUM
} JsonManifestFileField;

/*
 * Possible fields for one file as described by the manifest.
 */
typedef enum
{
    JMWRF_TIMELINE,
    JMWRF_START_LSN,
    JMWRF_END_LSN
} JsonManifestWALRangeField;

/*
 * Internal state used while decoding the JSON-format backup manifest.
 */
typedef struct
{
    JsonManifestParseContext *context;
    JsonManifestSemanticState state;

    /* These fields are used for parsing objects in the list of files. */
    JsonManifestFileField file_field;
    char       *pathname;
    char       *encoded_pathname;
    char       *size;
    char       *algorithm;
    pg_checksum_type checksum_algorithm;
    char       *checksum;

    /* These fields are used for parsing objects in the list of WAL ranges. */
    JsonManifestWALRangeField wal_range_field;
    char       *timeline;
    char       *start_lsn;
    char       *end_lsn;

    /* Miscellaneous other stuff. */
    bool        saw_version_field;
    char       *manifest_checksum;
} JsonManifestParseState;

static void json_manifest_object_start(void *state);
static void json_manifest_object_end(void *state);
static void json_manifest_array_start(void *state);
static void json_manifest_array_end(void *state);
static void json_manifest_object_field_start(void *state, char *fname,
                                             bool isnull);
static void json_manifest_scalar(void *state, char *token,
                                 JsonTokenType tokentype);
static void json_manifest_finalize_file(JsonManifestParseState *parse);
static void json_manifest_finalize_wal_range(JsonManifestParseState *parse);
static void verify_manifest_checksum(JsonManifestParseState *parse,
                                     char *buffer, size_t size);
static void json_manifest_parse_failure(JsonManifestParseContext *context,
                                        char *msg);

static int  hexdecode_char(char c);
static bool hexdecode_string(uint8 *result, char *input, int nbytes);
static bool parse_xlogrecptr(XLogRecPtr *result, char *input);

/*
 * Main entrypoint to parse a JSON-format backup manifest.
 *
 * Caller should set up the parsing context and then invoke this function.
 * For each file whose information is extracted from the manifest,
 * context->perfile_cb is invoked.  In case of trouble, context->error_cb is
 * invoked and is expected not to return.
 */
void
json_parse_manifest(JsonManifestParseContext *context, char *buffer,
                    size_t size)
{
    JsonLexContext *lex;
    JsonParseErrorType json_error;
    JsonSemAction sem;
    JsonManifestParseState parse;

    /* Set up our private parsing context. */
    parse.context = context;
    parse.state = JM_EXPECT_TOPLEVEL_START;
    parse.saw_version_field = false;

    /* Create a JSON lexing context. */
    lex = makeJsonLexContextCstringLen(buffer, size, PG_UTF8, true);

    /* Set up semantic actions. */
    sem.semstate = &parse;
    sem.object_start = json_manifest_object_start;
    sem.object_end = json_manifest_object_end;
    sem.array_start = json_manifest_array_start;
    sem.array_end = json_manifest_array_end;
    sem.object_field_start = json_manifest_object_field_start;
    sem.object_field_end = NULL;
    sem.array_element_start = NULL;
    sem.array_element_end = NULL;
    sem.scalar = json_manifest_scalar;

    /* Run the actual JSON parser. */
    json_error = pg_parse_json(lex, &sem);
    if (json_error != JSON_SUCCESS)
        json_manifest_parse_failure(context, json_errdetail(json_error, lex));
    if (parse.state != JM_EXPECT_EOF)
        json_manifest_parse_failure(context, "manifest ended unexpectedly");

    /* Verify the manifest checksum. */
    verify_manifest_checksum(&parse, buffer, size);
}

/*
 * Invoked at the start of each object in the JSON document.
 *
 * The document as a whole is expected to be an object; each file and each
 * WAL range is also expected to be an object. If we're anywhere else in the
 * document, it's an error.
 */
static void
json_manifest_object_start(void *state)
{
    JsonManifestParseState *parse = state;

    switch (parse->state)
    {
        case JM_EXPECT_TOPLEVEL_START:
            parse->state = JM_EXPECT_TOPLEVEL_FIELD;
            break;
        case JM_EXPECT_FILES_NEXT:
            parse->state = JM_EXPECT_THIS_FILE_FIELD;
            parse->pathname = NULL;
            parse->encoded_pathname = NULL;
            parse->size = NULL;
            parse->algorithm = NULL;
            parse->checksum = NULL;
            break;
        case JM_EXPECT_WAL_RANGES_NEXT:
            parse->state = JM_EXPECT_THIS_WAL_RANGE_FIELD;
            parse->timeline = NULL;
            parse->start_lsn = NULL;
            parse->end_lsn = NULL;
            break;
        default:
            json_manifest_parse_failure(parse->context,
                                        "unexpected object start");
            break;
    }
}

/*
 * Invoked at the end of each object in the JSON document.
 *
 * The possible cases here are the same as for json_manifest_object_start.
 * There's nothing special to do at the end of the document, but when we
 * reach the end of an object representing a particular file or WAL range,
 * we must call json_manifest_finalize_file() to save the associated details.
 */
static void
json_manifest_object_end(void *state)
{
    JsonManifestParseState *parse = state;

    switch (parse->state)
    {
        case JM_EXPECT_TOPLEVEL_END:
            parse->state = JM_EXPECT_EOF;
            break;
        case JM_EXPECT_THIS_FILE_FIELD:
            json_manifest_finalize_file(parse);
            parse->state = JM_EXPECT_FILES_NEXT;
            break;
        case JM_EXPECT_THIS_WAL_RANGE_FIELD:
            json_manifest_finalize_wal_range(parse);
            parse->state = JM_EXPECT_WAL_RANGES_NEXT;
            break;
        default:
            json_manifest_parse_failure(parse->context,
                                        "unexpected object end");
            break;
    }
}

/*
 * Invoked at the start of each array in the JSON document.
 *
 * Within the toplevel object, the value associated with the "Files" key
 * should be an array. Similarly for the "WAL-Ranges" key. No other arrays
 * are expected.
 */
static void
json_manifest_array_start(void *state)
{
    JsonManifestParseState *parse = state;

    switch (parse->state)
    {
        case JM_EXPECT_FILES_START:
            parse->state = JM_EXPECT_FILES_NEXT;
            break;
        case JM_EXPECT_WAL_RANGES_START:
            parse->state = JM_EXPECT_WAL_RANGES_NEXT;
            break;
        default:
            json_manifest_parse_failure(parse->context,
                                        "unexpected array start");
            break;
    }
}

/*
 * Invoked at the end of each array in the JSON document.
 *
 * The cases here are analogous to those in json_manifest_array_start.
 */
static void
json_manifest_array_end(void *state)
{
    JsonManifestParseState *parse = state;

    switch (parse->state)
    {
        case JM_EXPECT_FILES_NEXT:
        case JM_EXPECT_WAL_RANGES_NEXT:
            parse->state = JM_EXPECT_TOPLEVEL_FIELD;
            break;
        default:
            json_manifest_parse_failure(parse->context,
                                        "unexpected array end");
            break;
    }
}

/*
 * Invoked at the start of each object field in the JSON document.
 */
static void
json_manifest_object_field_start(void *state, char *fname, bool isnull)
{
    JsonManifestParseState *parse = state;

    switch (parse->state)
    {
        case JM_EXPECT_TOPLEVEL_FIELD:

            /*
             * Inside toplevel object. The version indicator should always be
             * the first field.
             */
            if (!parse->saw_version_field)
            {
                if (strcmp(fname, "PostgreSQL-Backup-Manifest-Version") != 0)
                    json_manifest_parse_failure(parse->context,
                                                "expected version indicator");
                parse->state = JM_EXPECT_VERSION_VALUE;
                parse->saw_version_field = true;
                break;
            }

            /* Is this the list of files? */
            if (strcmp(fname, "Files") == 0)
            {
                parse->state = JM_EXPECT_FILES_START;
                break;
            }

            /* Is this the list of WAL ranges? */
            if (strcmp(fname, "WAL-Ranges") == 0)
            {
                parse->state = JM_EXPECT_WAL_RANGES_START;
                break;
            }

            /* Is this the manifest checksum? */
            if (strcmp(fname, "Manifest-Checksum") == 0)
            {
                parse->state = JM_EXPECT_MANIFEST_CHECKSUM_VALUE;
                break;
            }

            /* It's not a field we recognize. */
            json_manifest_parse_failure(parse->context,
                                        "unrecognized top-level field");
            break;

        case JM_EXPECT_THIS_FILE_FIELD:
            /* Inside object for one file; which key have we got? */
            if (strcmp(fname, "Path") == 0)
                parse->file_field = JMFF_PATH;
            else if (strcmp(fname, "Encoded-Path") == 0)
                parse->file_field = JMFF_ENCODED_PATH;
            else if (strcmp(fname, "Size") == 0)
                parse->file_field = JMFF_SIZE;
            else if (strcmp(fname, "Last-Modified") == 0)
                parse->file_field = JMFF_LAST_MODIFIED;
            else if (strcmp(fname, "Checksum-Algorithm") == 0)
                parse->file_field = JMFF_CHECKSUM_ALGORITHM;
            else if (strcmp(fname, "Checksum") == 0)
                parse->file_field = JMFF_CHECKSUM;
            else
                json_manifest_parse_failure(parse->context,
                                            "unexpected file field");
            parse->state = JM_EXPECT_THIS_FILE_VALUE;
            break;

        case JM_EXPECT_THIS_WAL_RANGE_FIELD:
            /* Inside object for one file; which key have we got? */
            if (strcmp(fname, "Timeline") == 0)
                parse->wal_range_field = JMWRF_TIMELINE;
            else if (strcmp(fname, "Start-LSN") == 0)
                parse->wal_range_field = JMWRF_START_LSN;
            else if (strcmp(fname, "End-LSN") == 0)
                parse->wal_range_field = JMWRF_END_LSN;
            else
                json_manifest_parse_failure(parse->context,
                                            "unexpected WAL range field");
            parse->state = JM_EXPECT_THIS_WAL_RANGE_VALUE;
            break;

        default:
            json_manifest_parse_failure(parse->context,
                                        "unexpected object field");
            break;
    }
}

/*
 * Invoked at the start of each scalar in the JSON document.
 *
 * Object field names don't reach this code; those are handled by
 * json_manifest_object_field_start. When we're inside of the object for
 * a particular file or WAL range, that function will have noticed the name
 * of the field, and we'll get the corresponding value here. When we're in
 * the toplevel object, the parse state itself tells us which field this is.
 *
 * In all cases except for PostgreSQL-Backup-Manifest-Version, which we
 * can just check on the spot, the goal here is just to save the value in
 * the parse state for later use. We don't actually do anything until we
 * reach either the end of the object representing this file, or the end
 * of the manifest, as the case may be.
 */
static void
json_manifest_scalar(void *state, char *token, JsonTokenType tokentype)
{
    JsonManifestParseState *parse = state;

    switch (parse->state)
    {
        case JM_EXPECT_VERSION_VALUE:
            if (strcmp(token, "1") != 0)
                json_manifest_parse_failure(parse->context,
                                            "unexpected manifest version");
            parse->state = JM_EXPECT_TOPLEVEL_FIELD;
            break;

        case JM_EXPECT_THIS_FILE_VALUE:
            switch (parse->file_field)
            {
                case JMFF_PATH:
                    parse->pathname = token;
                    break;
                case JMFF_ENCODED_PATH:
                    parse->encoded_pathname = token;
                    break;
                case JMFF_SIZE:
                    parse->size = token;
                    break;
                case JMFF_LAST_MODIFIED:
                    pfree(token);   /* unused */
                    break;
                case JMFF_CHECKSUM_ALGORITHM:
                    parse->algorithm = token;
                    break;
                case JMFF_CHECKSUM:
                    parse->checksum = token;
                    break;
            }
            parse->state = JM_EXPECT_THIS_FILE_FIELD;
            break;

        case JM_EXPECT_THIS_WAL_RANGE_VALUE:
            switch (parse->wal_range_field)
            {
                case JMWRF_TIMELINE:
                    parse->timeline = token;
                    break;
                case JMWRF_START_LSN:
                    parse->start_lsn = token;
                    break;
                case JMWRF_END_LSN:
                    parse->end_lsn = token;
                    break;
            }
            parse->state = JM_EXPECT_THIS_WAL_RANGE_FIELD;
            break;

        case JM_EXPECT_MANIFEST_CHECKSUM_VALUE:
            parse->state = JM_EXPECT_TOPLEVEL_END;
            parse->manifest_checksum = token;
            break;

        default:
            json_manifest_parse_failure(parse->context, "unexpected scalar");
            break;
    }
}

/*
 * Do additional parsing and sanity-checking of the details gathered for one
 * file, and invoke the per-file callback so that the caller gets those
 * details. This happens for each file when the corresponding JSON object is
 * completely parsed.
 */
static void
json_manifest_finalize_file(JsonManifestParseState *parse)
{
    JsonManifestParseContext *context = parse->context;
    size_t      size;
    char       *ep;
    int         checksum_string_length;
    pg_checksum_type checksum_type;
    int         checksum_length;
    uint8      *checksum_payload;

    /* Pathname and size are required. */
    if (parse->pathname == NULL && parse->encoded_pathname == NULL)
        json_manifest_parse_failure(parse->context, "missing path name");
    if (parse->pathname != NULL && parse->encoded_pathname != NULL)
        json_manifest_parse_failure(parse->context,
                                    "both path name and encoded path name");
    if (parse->size == NULL)
        json_manifest_parse_failure(parse->context, "missing size");
    if (parse->algorithm == NULL && parse->checksum != NULL)
        json_manifest_parse_failure(parse->context,
                                    "checksum without algorithm");

    /* Decode encoded pathname, if that's what we have. */
    if (parse->encoded_pathname != NULL)
    {
        int         encoded_length = strlen(parse->encoded_pathname);
        int         raw_length = encoded_length / 2;

        parse->pathname = palloc(raw_length + 1);
        if (encoded_length % 2 != 0 ||
            !hexdecode_string((uint8 *) parse->pathname,
                              parse->encoded_pathname,
                              raw_length))
            json_manifest_parse_failure(parse->context,
                                        "could not decode file name");
        parse->pathname[raw_length] = '\0';
        pfree(parse->encoded_pathname);
        parse->encoded_pathname = NULL;
    }

    /* Parse size. */
    size = strtoul(parse->size, &ep, 10);
    if (*ep)
        json_manifest_parse_failure(parse->context,
                                    "file size is not an integer");

    /* Parse the checksum algorithm, if it's present. */
    if (parse->algorithm == NULL)
        checksum_type = CHECKSUM_TYPE_NONE;
    else if (!pg_checksum_parse_type(parse->algorithm, &checksum_type))
        context->error_cb(context, "unrecognized checksum algorithm: \"%s\"",
                          parse->algorithm);

    /* Parse the checksum payload, if it's present. */
    checksum_string_length = parse->checksum == NULL ? 0
        : strlen(parse->checksum);
    if (checksum_string_length == 0)
    {
        checksum_length = 0;
        checksum_payload = NULL;
    }
    else
    {
        checksum_length = checksum_string_length / 2;
        checksum_payload = palloc(checksum_length);
        if (checksum_string_length % 2 != 0 ||
            !hexdecode_string(checksum_payload, parse->checksum,
                              checksum_length))
            context->error_cb(context,
                              "invalid checksum for file \"%s\": \"%s\"",
                              parse->pathname, parse->checksum);
    }

    /* Invoke the callback with the details we've gathered. */
    context->perfile_cb(context, parse->pathname, size,
                        checksum_type, checksum_length, checksum_payload);

    /* Free memory we no longer need. */
    if (parse->size != NULL)
    {
        pfree(parse->size);
        parse->size = NULL;
    }
    if (parse->algorithm != NULL)
    {
        pfree(parse->algorithm);
        parse->algorithm = NULL;
    }
    if (parse->checksum != NULL)
    {
        pfree(parse->checksum);
        parse->checksum = NULL;
    }
}

/*
 * Do additional parsing and sanity-checking of the details gathered for one
 * WAL range, and invoke the per-WAL-range callback so that the caller gets
 * those details. This happens for each WAL range when the corresponding JSON
 * object is completely parsed.
 */
static void
json_manifest_finalize_wal_range(JsonManifestParseState *parse)
{
    JsonManifestParseContext *context = parse->context;
    TimeLineID  tli;
    XLogRecPtr  start_lsn,
                end_lsn;
    char       *ep;

    /* Make sure all fields are present. */
    if (parse->timeline == NULL)
        json_manifest_parse_failure(parse->context, "missing timeline");
    if (parse->start_lsn == NULL)
        json_manifest_parse_failure(parse->context, "missing start LSN");
    if (parse->end_lsn == NULL)
        json_manifest_parse_failure(parse->context, "missing end LSN");

    /* Parse timeline. */
    tli = strtoul(parse->timeline, &ep, 10);
    if (*ep)
        json_manifest_parse_failure(parse->context,
                                    "timeline is not an integer");
    if (!parse_xlogrecptr(&start_lsn, parse->start_lsn))
        json_manifest_parse_failure(parse->context,
                                    "could not parse start LSN");
    if (!parse_xlogrecptr(&end_lsn, parse->end_lsn))
        json_manifest_parse_failure(parse->context,
                                    "could not parse end LSN");

    /* Invoke the callback with the details we've gathered. */
    context->perwalrange_cb(context, tli, start_lsn, end_lsn);

    /* Free memory we no longer need. */
    if (parse->timeline != NULL)
    {
        pfree(parse->timeline);
        parse->timeline = NULL;
    }
    if (parse->start_lsn != NULL)
    {
        pfree(parse->start_lsn);
        parse->start_lsn = NULL;
    }
    if (parse->end_lsn != NULL)
    {
        pfree(parse->end_lsn);
        parse->end_lsn = NULL;
    }
}

/*
 * Verify that the manifest checksum is correct.
 *
 * The last line of the manifest file is excluded from the manifest checksum,
 * because the last line is expected to contain the checksum that covers
 * the rest of the file.
 */
static void
verify_manifest_checksum(JsonManifestParseState *parse, char *buffer,
                         size_t size)
{
    JsonManifestParseContext *context = parse->context;
    size_t      i;
    size_t      number_of_newlines = 0;
    size_t      ultimate_newline = 0;
    size_t      penultimate_newline = 0;
    pg_cryptohash_ctx *manifest_ctx;
    uint8       manifest_checksum_actual[PG_SHA256_DIGEST_LENGTH];
    uint8       manifest_checksum_expected[PG_SHA256_DIGEST_LENGTH];

    /* Find the last two newlines in the file. */
    for (i = 0; i < size; ++i)
    {
        if (buffer[i] == '\n')
        {
            ++number_of_newlines;
            penultimate_newline = ultimate_newline;
            ultimate_newline = i;
        }
    }

    /*
     * Make sure that the last newline is right at the end, and that there are
     * at least two lines total. We need this to be true in order for the
     * following code, which computes the manifest checksum, to work properly.
     */
    if (number_of_newlines < 2)
        json_manifest_parse_failure(parse->context,
                                    "expected at least 2 lines");
    if (ultimate_newline != size - 1)
        json_manifest_parse_failure(parse->context,
                                    "last line not newline-terminated");

    /* Checksum the rest. */
    manifest_ctx = pg_cryptohash_create(PG_SHA256);
    if (manifest_ctx == NULL)
        context->error_cb(context, "out of memory");
    if (pg_cryptohash_init(manifest_ctx) < 0)
        context->error_cb(context, "could not initialize checksum of manifest");
    if (pg_cryptohash_update(manifest_ctx, (uint8 *) buffer, penultimate_newline + 1) < 0)
        context->error_cb(context, "could not update checksum of manifest");
    if (pg_cryptohash_final(manifest_ctx, manifest_checksum_actual,
                            sizeof(manifest_checksum_actual)) < 0)
        context->error_cb(context, "could not finalize checksum of manifest");

    /* Now verify it. */
    if (parse->manifest_checksum == NULL)
        context->error_cb(parse->context, "manifest has no checksum");
    if (strlen(parse->manifest_checksum) != PG_SHA256_DIGEST_LENGTH * 2 ||
        !hexdecode_string(manifest_checksum_expected, parse->manifest_checksum,
                          PG_SHA256_DIGEST_LENGTH))
        context->error_cb(context, "invalid manifest checksum: \"%s\"",
                          parse->manifest_checksum);
    if (memcmp(manifest_checksum_actual, manifest_checksum_expected,
               PG_SHA256_DIGEST_LENGTH) != 0)
        context->error_cb(context, "manifest checksum mismatch");
    pg_cryptohash_free(manifest_ctx);
}

/*
 * Report a parse error.
 *
 * This is intended to be used for fairly low-level failures that probably
 * shouldn't occur unless somebody has deliberately constructed a bad manifest,
 * or unless the server is generating bad manifests due to some bug. msg should
 * be a short string giving some hint as to what the problem is.
 */
static void
json_manifest_parse_failure(JsonManifestParseContext *context, char *msg)
{
    context->error_cb(context, "could not parse backup manifest: %s", msg);
}

/*
 * Convert a character which represents a hexadecimal digit to an integer.
 *
 * Returns -1 if the character is not a hexadecimal digit.
 */
static int
hexdecode_char(char c)
{
    if (c >= '0' && c <= '9')
        return c - '0';
    if (c >= 'a' && c <= 'f')
        return c - 'a' + 10;
    if (c >= 'A' && c <= 'F')
        return c - 'A' + 10;

    return -1;
}

/*
 * Decode a hex string into a byte string, 2 hex chars per byte.
 *
 * Returns false if invalid characters are encountered; otherwise true.
 */
static bool
hexdecode_string(uint8 *result, char *input, int nbytes)
{
    int         i;

    for (i = 0; i < nbytes; ++i)
    {
        int         n1 = hexdecode_char(input[i * 2]);
        int         n2 = hexdecode_char(input[i * 2 + 1]);

        if (n1 < 0 || n2 < 0)
            return false;
        result[i] = n1 * 16 + n2;
    }

    return true;
}

/*
 * Parse an XLogRecPtr expressed using the usual string format.
 */
static bool
parse_xlogrecptr(XLogRecPtr *result, char *input)
{
    uint32      hi;
    uint32      lo;

    if (sscanf(input, "%X/%X", &hi, &lo) != 2)
        return false;
    *result = ((uint64) hi) << 32 | lo;
    return true;
}