test_json_parser_incremental.c File Reference

#include "postgres_fe.h"
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "common/jsonapi.h"
#include "common/logging.h"
#include "lib/stringinfo.h"
#include "mb/pg_wchar.h"
#include "pg_getopt.h"

Include dependency graph for test_json_parser_incremental.c:

Go to the source code of this file.

Data Structures
struct	DoState

Macros
#define	BUFSIZE   6000
#define	DEFAULT_CHUNK_SIZE   60

Typedefs
typedef struct DoState	DoState

Functions
static void	usage (const char *progname)
static void	escape_json (StringInfo buf, const char *str)
static JsonParseErrorType	do_object_start (void *state)
static JsonParseErrorType	do_object_end (void *state)
static JsonParseErrorType	do_object_field_start (void *state, char *fname, bool isnull)
static JsonParseErrorType	do_object_field_end (void *state, char *fname, bool isnull)
static JsonParseErrorType	do_array_start (void *state)
static JsonParseErrorType	do_array_end (void *state)
static JsonParseErrorType	do_array_element_start (void *state, bool isnull)
static JsonParseErrorType	do_array_element_end (void *state, bool isnull)
static JsonParseErrorType	do_scalar (void *state, char *token, JsonTokenType tokentype)
int	main (int argc, char **argv)

Variables
static JsonSemAction	sem
static bool	lex_owns_tokens = false

Macro Definition Documentation

BUFSIZE

#define BUFSIZE   6000

Definition at line 48 of file test_json_parser_incremental.c.

DEFAULT_CHUNK_SIZE

#define DEFAULT_CHUNK_SIZE   60

Definition at line 49 of file test_json_parser_incremental.c.

Typedef Documentation

DoState

typedef struct DoState DoState

Function Documentation

do_array_element_end()

static JsonParseErrorType do_array_element_end ( void *  state, bool  isnull  )

static

Definition at line 330 of file test_json_parser_incremental.c.

{
    /* nothing to do */
 
    return JSON_SUCCESS;
}

References JSON_SUCCESS.

do_array_element_start()

static JsonParseErrorType do_array_element_start ( void *  state, bool  isnull  )

static

Definition at line 318 of file test_json_parser_incremental.c.

{
    DoState    *_state = (DoState *) state;
 
    if (!_state->elem_is_first)
        printf(",\n");
    _state->elem_is_first = false;
 
    return JSON_SUCCESS;
}

References DoState::elem_is_first, JSON_SUCCESS, and printf.

do_array_end()

static JsonParseErrorType do_array_end ( void *  state )

static

Definition at line 307 of file test_json_parser_incremental.c.

{
    DoState    *_state = (DoState *) state;
 
    printf("\n]\n");
    _state->elem_is_first = false;
 
    return JSON_SUCCESS;
}

References DoState::elem_is_first, JSON_SUCCESS, and printf.

do_array_start()

static JsonParseErrorType do_array_start ( void *  state )

static

Definition at line 296 of file test_json_parser_incremental.c.

{
    DoState    *_state = (DoState *) state;
 
    printf("[\n");
    _state->elem_is_first = true;
 
    return JSON_SUCCESS;
}

References DoState::elem_is_first, JSON_SUCCESS, and printf.

do_object_end()

static JsonParseErrorType do_object_end ( void *  state )

static

Definition at line 261 of file test_json_parser_incremental.c.

{
    DoState    *_state = (DoState *) state;
 
    printf("\n}\n");
    _state->elem_is_first = false;
 
    return JSON_SUCCESS;
}

References DoState::elem_is_first, JSON_SUCCESS, and printf.

do_object_field_end()

static JsonParseErrorType do_object_field_end ( void *  state, char *  fname, bool  isnull  )

static

Definition at line 287 of file test_json_parser_incremental.c.

{
    if (!lex_owns_tokens)
        free(fname);
 
    return JSON_SUCCESS;
}

References free, JSON_SUCCESS, and lex_owns_tokens.

do_object_field_start()

static JsonParseErrorType do_object_field_start ( void *  state, char *  fname, bool  isnull  )

static

Definition at line 272 of file test_json_parser_incremental.c.

{
    DoState    *_state = (DoState *) state;
 
    if (!_state->elem_is_first)
        printf(",\n");
    resetStringInfo(_state->buf);
    escape_json(_state->buf, fname);
    printf("%s: ", _state->buf->data);
    _state->elem_is_first = false;
 
    return JSON_SUCCESS;
}

References DoState::buf, StringInfoData::data, DoState::elem_is_first, escape_json(), JSON_SUCCESS, printf, and resetStringInfo().

do_object_start()

static JsonParseErrorType do_object_start ( void *  state )

static

Definition at line 250 of file test_json_parser_incremental.c.

{
    DoState    *_state = (DoState *) state;
 
    printf("{\n");
    _state->elem_is_first = true;
 
    return JSON_SUCCESS;
}

References DoState::elem_is_first, JSON_SUCCESS, and printf.

do_scalar()

static JsonParseErrorType do_scalar ( void *  state, char *  token, JsonTokenType  tokentype  )

static

Definition at line 338 of file test_json_parser_incremental.c.

{
    DoState    *_state = (DoState *) state;
 
    if (tokentype == JSON_TOKEN_STRING)
    {
        resetStringInfo(_state->buf);
        escape_json(_state->buf, token);
        printf("%s", _state->buf->data);
    }
    else
        printf("%s", token);
 
    if (!lex_owns_tokens)
        free(token);
 
    return JSON_SUCCESS;
}

References DoState::buf, StringInfoData::data, escape_json(), free, JSON_SUCCESS, JSON_TOKEN_STRING, lex_owns_tokens, printf, and resetStringInfo().

escape_json()

static void escape_json ( StringInfo  buf, const char *  str  )

static

Definition at line 360 of file test_json_parser_incremental.c.

{
    const char *p;
 
    appendStringInfoCharMacro(buf, '"');
    for (p = str; *p; p++)
    {
        switch (*p)
        {
            case '\b':
                appendStringInfoString(buf, "\\b");
                break;
            case '\f':
                appendStringInfoString(buf, "\\f");
                break;
            case '\n':
                appendStringInfoString(buf, "\\n");
                break;
            case '\r':
                appendStringInfoString(buf, "\\r");
                break;
            case '\t':
                appendStringInfoString(buf, "\\t");
                break;
            case '"':
                appendStringInfoString(buf, "\\\"");
                break;
            case '\\':
                appendStringInfoString(buf, "\\\\");
                break;
            default:
                if ((unsigned char) *p < ' ')
                    appendStringInfo(buf, "\\u%04x", (int) *p);
                else
                    appendStringInfoCharMacro(buf, *p);
                break;
        }
    }
    appendStringInfoCharMacro(buf, '"');
}

References appendStringInfo(), appendStringInfoCharMacro, appendStringInfoString(), buf, and str.

Referenced by do_object_field_start(), and do_scalar().

main()

int main	(	int	argc,
		char **	argv
	)

Definition at line 87 of file test_json_parser_incremental.c.

{
    char        buff[BUFSIZE];
    FILE       *json_file;
    JsonParseErrorType result;
    JsonLexContext *lex;
    StringInfoData json;
    int         n_read;
    size_t      chunk_size = DEFAULT_CHUNK_SIZE;
    bool        run_chunk_ranges = false;
    struct stat statbuf;
    const JsonSemAction *testsem = &nullSemAction;
    char       *testfile;
    int         c;
    bool        need_strings = false;
    int         ret = 0;
 
    pg_logging_init(argv[0]);
 
    lex = calloc(1, sizeof(JsonLexContext));
    if (!lex)
        pg_fatal("out of memory");
 
    while ((c = getopt(argc, argv, "r:c:os")) != -1)
    {
        switch (c)
        {
            case 'r':           /* chunk range */
                run_chunk_ranges = true;
                /* fall through */
            case 'c':           /* chunk size */
                chunk_size = strtou64(optarg, NULL, 10);
                if (chunk_size > BUFSIZE)
                    pg_fatal("chunk size cannot exceed %d", BUFSIZE);
                break;
            case 'o':           /* switch token ownership */
                lex_owns_tokens = true;
                break;
            case 's':           /* do semantic processing */
                testsem = &sem;
                sem.semstate = palloc(sizeof(struct DoState));
                ((struct DoState *) sem.semstate)->lex = lex;
                ((struct DoState *) sem.semstate)->buf = makeStringInfo();
                need_strings = true;
                break;
        }
    }
 
    if (optind < argc)
    {
        testfile = argv[optind];
        optind++;
    }
    else
    {
        usage(argv[0]);
        exit(1);
    }
 
    initStringInfo(&json);
 
    if ((json_file = fopen(testfile, PG_BINARY_R)) == NULL)
        pg_fatal("error opening input: %m");
 
    if (fstat(fileno(json_file), &statbuf) != 0)
        pg_fatal("error statting input: %m");
 
    do
    {
        /*
         * This outer loop only repeats in -r mode. Reset the parse state and
         * our position in the input file for the inner loop, which performs
         * the incremental parsing.
         */
        off_t       bytes_left = statbuf.st_size;
        size_t      to_read = chunk_size;
 
        makeJsonLexContextIncremental(lex, PG_UTF8, need_strings);
        setJsonLexContextOwnsTokens(lex, lex_owns_tokens);
 
        rewind(json_file);
        resetStringInfo(&json);
 
        for (;;)
        {
            /* We will break when there's nothing left to read */
 
            if (bytes_left < to_read)
                to_read = bytes_left;
 
            n_read = fread(buff, 1, to_read, json_file);
            if (n_read < to_read)
                pg_fatal("error reading input file: %d", ferror(json_file));
 
            appendBinaryStringInfo(&json, buff, n_read);
 
            /*
             * Append some trailing junk to the buffer passed to the parser.
             * This helps us ensure that the parser does the right thing even
             * if the chunk isn't terminated with a '\0'.
             */
            appendStringInfoString(&json, "1+23 trailing junk");
            bytes_left -= n_read;
            if (bytes_left > 0)
            {
                result = pg_parse_json_incremental(lex, testsem,
                                                   json.data, n_read,
                                                   false);
                if (result != JSON_INCOMPLETE)
                {
                    fprintf(stderr, "%s\n", json_errdetail(result, lex));
                    ret = 1;
                    goto cleanup;
                }
                resetStringInfo(&json);
            }
            else
            {
                result = pg_parse_json_incremental(lex, testsem,
                                                   json.data, n_read,
                                                   true);
                if (result != JSON_SUCCESS)
                {
                    fprintf(stderr, "%s\n", json_errdetail(result, lex));
                    ret = 1;
                    goto cleanup;
                }
                if (!need_strings)
                    printf("SUCCESS!\n");
                break;
            }
        }
 
cleanup:
        freeJsonLexContext(lex);
 
        /*
         * In -r mode, separate output with nulls so that the calling test can
         * split it up, decrement the chunk size, and loop back to the top.
         * All other modes immediately fall out of the loop and exit.
         */
        if (run_chunk_ranges)
        {
            fputc('\0', stdout);
            fputc('\0', stderr);
        }
    } while (run_chunk_ranges && (--chunk_size > 0));
 
    fclose(json_file);
    free(json.data);
    free(lex);
 
    return ret;
}

References appendBinaryStringInfo(), appendStringInfoString(), BUFSIZE, calloc, cleanup(), StringInfoData::data, DEFAULT_CHUNK_SIZE, fprintf, free, freeJsonLexContext(), fstat, getopt(), initStringInfo(), json_errdetail(), JSON_INCOMPLETE, JSON_SUCCESS, DoState::lex, lex_owns_tokens, makeJsonLexContextIncremental(), makeStringInfo(), nullSemAction, optarg, optind, palloc(), PG_BINARY_R, pg_fatal, pg_logging_init(), pg_parse_json_incremental(), PG_UTF8, printf, resetStringInfo(), sem, JsonSemAction::semstate, setJsonLexContextOwnsTokens(), stat::st_size, generate_unaccent_rules::stdout, and usage().

usage()

static void usage ( const char *  progname )

static

Definition at line 402 of file test_json_parser_incremental.c.

{
    fprintf(stderr, "Usage: %s [OPTION ...] testfile\n", progname);
    fprintf(stderr, "Options:\n");
    fprintf(stderr, "  -c chunksize      size of piece fed to parser (default 64)\n");
    fprintf(stderr, "  -o                set JSONLEX_CTX_OWNS_TOKENS for leak checking\n");
    fprintf(stderr, "  -s                do semantic processing\n");
 
}

References fprintf, and progname.

Referenced by main().

Variable Documentation

lex_owns_tokens

bool lex_owns_tokens = false

static

Definition at line 84 of file test_json_parser_incremental.c.

Referenced by do_object_field_end(), do_scalar(), and main().

sem

JsonSemAction sem

static

Initial value:

= {
    .object_start = do_object_start,
    .object_end = do_object_end,
    .object_field_start = do_object_field_start,
    .object_field_end = do_object_field_end,
    .array_start = do_array_start,
    .array_end = do_array_end,
    .array_element_start = do_array_element_start,
    .array_element_end = do_array_element_end,
    .scalar = do_scalar
}

Definition at line 72 of file test_json_parser_incremental.c.

Referenced by datum_to_jsonb_internal(), each_worker(), elements_worker(), get_json_object_as_hash(), get_worker(), handle_oauth_sasl_error(), iterate_json_values(), json_array_length(), json_object_keys(), json_parse_manifest(), json_strip_nulls(), jsonb_from_cstring(), main(), parse_array(), parse_array_element(), parse_oauth_json(), parse_object(), parse_object_field(), parse_scalar(), pg_parse_json(), pg_parse_json_incremental(), pg_parse_json_or_errsave(), populate_array_json(), populate_recordset_worker(), PosixSemaphoreCreate(), PosixSemaphoreKill(), test_gb18030_json(), and transform_json_string_values().