test_json_parser_incremental.c File Reference

#include "postgres_fe.h"
#include <stdio.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "common/jsonapi.h"
#include "common/logging.h"
#include "lib/stringinfo.h"
#include "mb/pg_wchar.h"
#include "pg_getopt.h"

Include dependency graph for test_json_parser_incremental.c:

Go to the source code of this file.

Data Structures
struct	DoState

Macros
#define	BUFSIZE   6000
#define	DEFAULT_CHUNK_SIZE   60

Typedefs
typedef struct DoState	DoState

Functions
static void	usage (const char *progname)
static void	escape_json (StringInfo buf, const char *str)
static JsonParseErrorType	do_object_start (void *state)
static JsonParseErrorType	do_object_end (void *state)
static JsonParseErrorType	do_object_field_start (void *state, char *fname, bool isnull)
static JsonParseErrorType	do_object_field_end (void *state, char *fname, bool isnull)
static JsonParseErrorType	do_array_start (void *state)
static JsonParseErrorType	do_array_end (void *state)
static JsonParseErrorType	do_array_element_start (void *state, bool isnull)
static JsonParseErrorType	do_array_element_end (void *state, bool isnull)
static JsonParseErrorType	do_scalar (void *state, char *token, JsonTokenType tokentype)
int	main (int argc, char **argv)

Variables
static JsonSemAction	sem
static bool	lex_owns_tokens = false

Macro Definition Documentation

BUFSIZE

#define BUFSIZE   6000

Definition at line 43 of file test_json_parser_incremental.c.

DEFAULT_CHUNK_SIZE

#define DEFAULT_CHUNK_SIZE   60

Definition at line 44 of file test_json_parser_incremental.c.

Typedef Documentation

DoState

typedef struct DoState DoState

Function Documentation

do_array_element_end()

static JsonParseErrorType do_array_element_end ( void *  state, bool  isnull  )

static

Definition at line 292 of file test_json_parser_incremental.c.

{
    /* nothing to do */
 
    return JSON_SUCCESS;
}

References JSON_SUCCESS.

do_array_element_start()

static JsonParseErrorType do_array_element_start ( void *  state, bool  isnull  )

static

Definition at line 280 of file test_json_parser_incremental.c.

{
    DoState    *_state = (DoState *) state;
 
    if (!_state->elem_is_first)
        printf(",\n");
    _state->elem_is_first = false;
 
    return JSON_SUCCESS;
}

References DoState::elem_is_first, JSON_SUCCESS, and printf.

do_array_end()

static JsonParseErrorType do_array_end ( void *  state )

static

Definition at line 269 of file test_json_parser_incremental.c.

{
    DoState    *_state = (DoState *) state;
 
    printf("\n]\n");
    _state->elem_is_first = false;
 
    return JSON_SUCCESS;
}

References DoState::elem_is_first, JSON_SUCCESS, and printf.

do_array_start()

static JsonParseErrorType do_array_start ( void *  state )

static

Definition at line 258 of file test_json_parser_incremental.c.

{
    DoState    *_state = (DoState *) state;
 
    printf("[\n");
    _state->elem_is_first = true;
 
    return JSON_SUCCESS;
}

References DoState::elem_is_first, JSON_SUCCESS, and printf.

do_object_end()

static JsonParseErrorType do_object_end ( void *  state )

static

Definition at line 223 of file test_json_parser_incremental.c.

{
    DoState    *_state = (DoState *) state;
 
    printf("\n}\n");
    _state->elem_is_first = false;
 
    return JSON_SUCCESS;
}

References DoState::elem_is_first, JSON_SUCCESS, and printf.

do_object_field_end()

static JsonParseErrorType do_object_field_end ( void *  state, char *  fname, bool  isnull  )

static

Definition at line 249 of file test_json_parser_incremental.c.

{
    if (!lex_owns_tokens)
        free(fname);
 
    return JSON_SUCCESS;
}

References free, JSON_SUCCESS, and lex_owns_tokens.

do_object_field_start()

static JsonParseErrorType do_object_field_start ( void *  state, char *  fname, bool  isnull  )

static

Definition at line 234 of file test_json_parser_incremental.c.

{
    DoState    *_state = (DoState *) state;
 
    if (!_state->elem_is_first)
        printf(",\n");
    resetStringInfo(_state->buf);
    escape_json(_state->buf, fname);
    printf("%s: ", _state->buf->data);
    _state->elem_is_first = false;
 
    return JSON_SUCCESS;
}

References DoState::buf, StringInfoData::data, DoState::elem_is_first, escape_json(), JSON_SUCCESS, printf, and resetStringInfo().

do_object_start()

static JsonParseErrorType do_object_start ( void *  state )

static

Definition at line 212 of file test_json_parser_incremental.c.

{
    DoState    *_state = (DoState *) state;
 
    printf("{\n");
    _state->elem_is_first = true;
 
    return JSON_SUCCESS;
}

References DoState::elem_is_first, JSON_SUCCESS, and printf.

do_scalar()

static JsonParseErrorType do_scalar ( void *  state, char *  token, JsonTokenType  tokentype  )

static

Definition at line 300 of file test_json_parser_incremental.c.

{
    DoState    *_state = (DoState *) state;
 
    if (tokentype == JSON_TOKEN_STRING)
    {
        resetStringInfo(_state->buf);
        escape_json(_state->buf, token);
        printf("%s", _state->buf->data);
    }
    else
        printf("%s", token);
 
    if (!lex_owns_tokens)
        free(token);
 
    return JSON_SUCCESS;
}

References DoState::buf, StringInfoData::data, escape_json(), free, JSON_SUCCESS, JSON_TOKEN_STRING, lex_owns_tokens, printf, resetStringInfo(), and token.

escape_json()

static void escape_json ( StringInfo  buf, const char *  str  )

static

Definition at line 322 of file test_json_parser_incremental.c.

{
    const char *p;
 
    appendStringInfoCharMacro(buf, '"');
    for (p = str; *p; p++)
    {
        switch (*p)
        {
            case '\b':
                appendStringInfoString(buf, "\\b");
                break;
            case '\f':
                appendStringInfoString(buf, "\\f");
                break;
            case '\n':
                appendStringInfoString(buf, "\\n");
                break;
            case '\r':
                appendStringInfoString(buf, "\\r");
                break;
            case '\t':
                appendStringInfoString(buf, "\\t");
                break;
            case '"':
                appendStringInfoString(buf, "\\\"");
                break;
            case '\\':
                appendStringInfoString(buf, "\\\\");
                break;
            default:
                if ((unsigned char) *p < ' ')
                    appendStringInfo(buf, "\\u%04x", (int) *p);
                else
                    appendStringInfoCharMacro(buf, *p);
                break;
        }
    }
    appendStringInfoCharMacro(buf, '"');
}

References appendStringInfo(), appendStringInfoCharMacro, appendStringInfoString(), buf, and str.

Referenced by do_object_field_start(), and do_scalar().

main()

int main	(	int	argc,
		char **	argv
	)

Definition at line 82 of file test_json_parser_incremental.c.

{
    char        buff[BUFSIZE];
    FILE       *json_file;
    JsonParseErrorType result;
    JsonLexContext lex;
    StringInfoData json;
    int         n_read;
    size_t      chunk_size = DEFAULT_CHUNK_SIZE;
    struct stat statbuf;
    off_t       bytes_left;
    const JsonSemAction *testsem = &nullSemAction;
    char       *testfile;
    int         c;
    bool        need_strings = false;
    int         ret = 0;
 
    pg_logging_init(argv[0]);
 
    while ((c = getopt(argc, argv, "c:os")) != -1)
    {
        switch (c)
        {
            case 'c':           /* chunksize */
                chunk_size = strtou64(optarg, NULL, 10);
                if (chunk_size > BUFSIZE)
                    pg_fatal("chunk size cannot exceed %d", BUFSIZE);
                break;
            case 'o':           /* switch token ownership */
                lex_owns_tokens = true;
                break;
            case 's':           /* do semantic processing */
                testsem = &sem;
                sem.semstate = palloc(sizeof(struct DoState));
                ((struct DoState *) sem.semstate)->lex = &lex;
                ((struct DoState *) sem.semstate)->buf = makeStringInfo();
                need_strings = true;
                break;
        }
    }
 
    if (optind < argc)
    {
        testfile = argv[optind];
        optind++;
    }
    else
    {
        usage(argv[0]);
        exit(1);
    }
 
    makeJsonLexContextIncremental(&lex, PG_UTF8, need_strings);
    setJsonLexContextOwnsTokens(&lex, lex_owns_tokens);
    initStringInfo(&json);
 
    if ((json_file = fopen(testfile, PG_BINARY_R)) == NULL)
        pg_fatal("error opening input: %m");
 
    if (fstat(fileno(json_file), &statbuf) != 0)
        pg_fatal("error statting input: %m");
 
    bytes_left = statbuf.st_size;
 
    for (;;)
    {
        /* We will break when there's nothing left to read */
 
        if (bytes_left < chunk_size)
            chunk_size = bytes_left;
 
        n_read = fread(buff, 1, chunk_size, json_file);
        if (n_read < chunk_size)
            pg_fatal("error reading input file: %d", ferror(json_file));
 
        appendBinaryStringInfo(&json, buff, n_read);
 
        /*
         * Append some trailing junk to the buffer passed to the parser. This
         * helps us ensure that the parser does the right thing even if the
         * chunk isn't terminated with a '\0'.
         */
        appendStringInfoString(&json, "1+23 trailing junk");
        bytes_left -= n_read;
        if (bytes_left > 0)
        {
            result = pg_parse_json_incremental(&lex, testsem,
                                               json.data, n_read,
                                               false);
            if (result != JSON_INCOMPLETE)
            {
                fprintf(stderr, "%s\n", json_errdetail(result, &lex));
                ret = 1;
                goto cleanup;
            }
            resetStringInfo(&json);
        }
        else
        {
            result = pg_parse_json_incremental(&lex, testsem,
                                               json.data, n_read,
                                               true);
            if (result != JSON_SUCCESS)
            {
                fprintf(stderr, "%s\n", json_errdetail(result, &lex));
                ret = 1;
                goto cleanup;
            }
            if (!need_strings)
                printf("SUCCESS!\n");
            break;
        }
    }
 
cleanup:
    fclose(json_file);
    freeJsonLexContext(&lex);
    free(json.data);
 
    return ret;
}

References appendBinaryStringInfo(), appendStringInfoString(), BUFSIZE, cleanup(), StringInfoData::data, DEFAULT_CHUNK_SIZE, exit(), fprintf, free, freeJsonLexContext(), fstat, getopt(), initStringInfo(), json_errdetail(), JSON_INCOMPLETE, JSON_SUCCESS, DoState::lex, lex_owns_tokens, makeJsonLexContextIncremental(), makeStringInfo(), nullSemAction, optarg, optind, palloc(), PG_BINARY_R, pg_fatal, pg_logging_init(), pg_parse_json_incremental(), PG_UTF8, printf, resetStringInfo(), sem, JsonSemAction::semstate, setJsonLexContextOwnsTokens(), stat::st_size, and usage().

usage()

static void usage ( const char *  progname )

static

Definition at line 364 of file test_json_parser_incremental.c.

{
    fprintf(stderr, "Usage: %s [OPTION ...] testfile\n", progname);
    fprintf(stderr, "Options:\n");
    fprintf(stderr, "  -c chunksize      size of piece fed to parser (default 64)\n");
    fprintf(stderr, "  -o                set JSONLEX_CTX_OWNS_TOKENS for leak checking\n");
    fprintf(stderr, "  -s                do semantic processing\n");
 
}

References fprintf, and progname.

Referenced by main().

Variable Documentation

lex_owns_tokens

bool lex_owns_tokens = false

static

Definition at line 79 of file test_json_parser_incremental.c.

Referenced by do_object_field_end(), do_scalar(), and main().

sem

JsonSemAction sem

static

Initial value:

= {
    .object_start = do_object_start,
    .object_end = do_object_end,
    .object_field_start = do_object_field_start,
    .object_field_end = do_object_field_end,
    .array_start = do_array_start,
    .array_end = do_array_end,
    .array_element_start = do_array_element_start,
    .array_element_end = do_array_element_end,
    .scalar = do_scalar
}

Definition at line 67 of file test_json_parser_incremental.c.

Referenced by datum_to_jsonb_internal(), each_worker(), elements_worker(), get_json_object_as_hash(), get_worker(), iterate_json_values(), json_array_length(), json_object_keys(), json_parse_manifest(), json_strip_nulls(), jsonb_from_cstring(), main(), parse_array(), parse_array_element(), parse_object(), parse_object_field(), parse_scalar(), pg_parse_json(), pg_parse_json_incremental(), pg_parse_json_or_errsave(), populate_array_json(), populate_recordset_worker(), PosixSemaphoreCreate(), PosixSemaphoreKill(), and transform_json_string_values().