PostgreSQL Source Code git master
test_json_parser_incremental.c
Go to the documentation of this file.
1/*-------------------------------------------------------------------------
2 *
3 * test_json_parser_incremental.c
4 * Test program for incremental JSON parser
5 *
6 * Copyright (c) 2024-2025, PostgreSQL Global Development Group
7 *
8 * IDENTIFICATION
9 * src/test/modules/test_json_parser/test_json_parser_incremental.c
10 *
11 * This program tests incremental parsing of json. The input is fed into
12 * the parser in very small chunks. In practice you would normally use
13 * much larger chunks, but doing this makes it more likely that the
14 * full range of increment handling, especially in the lexer, is exercised.
15 * If the "-c SIZE" option is provided, that chunk size is used instead
16 * of the default of 60.
17 *
18 * If the -s flag is given, the program does semantic processing. This should
19 * just mirror back the json, albeit with white space changes.
20 *
21 * If the -o flag is given, the JSONLEX_CTX_OWNS_TOKENS flag is set. (This can
22 * be used in combination with a leak sanitizer; without the option, the parser
23 * may leak memory with invalid JSON.)
24 *
25 * The argument specifies the file containing the JSON input.
26 *
27 *-------------------------------------------------------------------------
28 */
29
30#include "postgres_fe.h"
31
32#include <stdio.h>
33#include <sys/types.h>
34#include <sys/stat.h>
35#include <unistd.h>
36
37#include "common/jsonapi.h"
38#include "common/logging.h"
39#include "lib/stringinfo.h"
40#include "mb/pg_wchar.h"
41#include "pg_getopt.h"
42
43#define BUFSIZE 6000
44#define DEFAULT_CHUNK_SIZE 60
45
46typedef struct DoState
47{
52
53static void usage(const char *progname);
54static void escape_json(StringInfo buf, const char *str);
55
56/* semantic action functions for parser */
59static JsonParseErrorType do_object_field_start(void *state, char *fname, bool isnull);
60static JsonParseErrorType do_object_field_end(void *state, char *fname, bool isnull);
63static JsonParseErrorType do_array_element_start(void *state, bool isnull);
64static JsonParseErrorType do_array_element_end(void *state, bool isnull);
65static JsonParseErrorType do_scalar(void *state, char *token, JsonTokenType tokentype);
66
69 .object_end = do_object_end,
70 .object_field_start = do_object_field_start,
71 .object_field_end = do_object_field_end,
72 .array_start = do_array_start,
73 .array_end = do_array_end,
74 .array_element_start = do_array_element_start,
75 .array_element_end = do_array_element_end,
76 .scalar = do_scalar
77};
78
79static bool lex_owns_tokens = false;
80
81int
82main(int argc, char **argv)
83{
84 char buff[BUFSIZE];
85 FILE *json_file;
86 JsonParseErrorType result;
88 StringInfoData json;
89 int n_read;
90 size_t chunk_size = DEFAULT_CHUNK_SIZE;
91 struct stat statbuf;
92 off_t bytes_left;
93 const JsonSemAction *testsem = &nullSemAction;
94 char *testfile;
95 int c;
96 bool need_strings = false;
97 int ret = 0;
98
99 pg_logging_init(argv[0]);
100
101 while ((c = getopt(argc, argv, "c:os")) != -1)
102 {
103 switch (c)
104 {
105 case 'c': /* chunksize */
106 chunk_size = strtou64(optarg, NULL, 10);
107 if (chunk_size > BUFSIZE)
108 pg_fatal("chunk size cannot exceed %d", BUFSIZE);
109 break;
110 case 'o': /* switch token ownership */
111 lex_owns_tokens = true;
112 break;
113 case 's': /* do semantic processing */
114 testsem = &sem;
115 sem.semstate = palloc(sizeof(struct DoState));
116 ((struct DoState *) sem.semstate)->lex = &lex;
117 ((struct DoState *) sem.semstate)->buf = makeStringInfo();
118 need_strings = true;
119 break;
120 }
121 }
122
123 if (optind < argc)
124 {
125 testfile = argv[optind];
126 optind++;
127 }
128 else
129 {
130 usage(argv[0]);
131 exit(1);
132 }
133
136 initStringInfo(&json);
137
138 if ((json_file = fopen(testfile, PG_BINARY_R)) == NULL)
139 pg_fatal("error opening input: %m");
140
141 if (fstat(fileno(json_file), &statbuf) != 0)
142 pg_fatal("error statting input: %m");
143
144 bytes_left = statbuf.st_size;
145
146 for (;;)
147 {
148 /* We will break when there's nothing left to read */
149
150 if (bytes_left < chunk_size)
151 chunk_size = bytes_left;
152
153 n_read = fread(buff, 1, chunk_size, json_file);
154 if (n_read < chunk_size)
155 pg_fatal("error reading input file: %d", ferror(json_file));
156
157 appendBinaryStringInfo(&json, buff, n_read);
158
159 /*
160 * Append some trailing junk to the buffer passed to the parser. This
161 * helps us ensure that the parser does the right thing even if the
162 * chunk isn't terminated with a '\0'.
163 */
164 appendStringInfoString(&json, "1+23 trailing junk");
165 bytes_left -= n_read;
166 if (bytes_left > 0)
167 {
168 result = pg_parse_json_incremental(&lex, testsem,
169 json.data, n_read,
170 false);
171 if (result != JSON_INCOMPLETE)
172 {
173 fprintf(stderr, "%s\n", json_errdetail(result, &lex));
174 ret = 1;
175 goto cleanup;
176 }
177 resetStringInfo(&json);
178 }
179 else
180 {
181 result = pg_parse_json_incremental(&lex, testsem,
182 json.data, n_read,
183 true);
184 if (result != JSON_SUCCESS)
185 {
186 fprintf(stderr, "%s\n", json_errdetail(result, &lex));
187 ret = 1;
188 goto cleanup;
189 }
190 if (!need_strings)
191 printf("SUCCESS!\n");
192 break;
193 }
194 }
195
196cleanup:
197 fclose(json_file);
199 free(json.data);
200
201 return ret;
202}
203
204/*
205 * The semantic routines here essentially just output the same json, except
206 * for white space. We could pretty print it but there's no need for our
207 * purposes. The result should be able to be fed to any JSON processor
208 * such as jq for validation.
209 */
210
213{
214 DoState *_state = (DoState *) state;
215
216 printf("{\n");
217 _state->elem_is_first = true;
218
219 return JSON_SUCCESS;
220}
221
224{
225 DoState *_state = (DoState *) state;
226
227 printf("\n}\n");
228 _state->elem_is_first = false;
229
230 return JSON_SUCCESS;
231}
232
234do_object_field_start(void *state, char *fname, bool isnull)
235{
236 DoState *_state = (DoState *) state;
237
238 if (!_state->elem_is_first)
239 printf(",\n");
240 resetStringInfo(_state->buf);
241 escape_json(_state->buf, fname);
242 printf("%s: ", _state->buf->data);
243 _state->elem_is_first = false;
244
245 return JSON_SUCCESS;
246}
247
249do_object_field_end(void *state, char *fname, bool isnull)
250{
251 if (!lex_owns_tokens)
252 free(fname);
253
254 return JSON_SUCCESS;
255}
256
259{
260 DoState *_state = (DoState *) state;
261
262 printf("[\n");
263 _state->elem_is_first = true;
264
265 return JSON_SUCCESS;
266}
267
270{
271 DoState *_state = (DoState *) state;
272
273 printf("\n]\n");
274 _state->elem_is_first = false;
275
276 return JSON_SUCCESS;
277}
278
280do_array_element_start(void *state, bool isnull)
281{
282 DoState *_state = (DoState *) state;
283
284 if (!_state->elem_is_first)
285 printf(",\n");
286 _state->elem_is_first = false;
287
288 return JSON_SUCCESS;
289}
290
292do_array_element_end(void *state, bool isnull)
293{
294 /* nothing to do */
295
296 return JSON_SUCCESS;
297}
298
300do_scalar(void *state, char *token, JsonTokenType tokentype)
301{
302 DoState *_state = (DoState *) state;
303
304 if (tokentype == JSON_TOKEN_STRING)
305 {
306 resetStringInfo(_state->buf);
307 escape_json(_state->buf, token);
308 printf("%s", _state->buf->data);
309 }
310 else
311 printf("%s", token);
312
313 if (!lex_owns_tokens)
314 free(token);
315
316 return JSON_SUCCESS;
317}
318
319
320/* copied from backend code */
321static void
323{
324 const char *p;
325
327 for (p = str; *p; p++)
328 {
329 switch (*p)
330 {
331 case '\b':
333 break;
334 case '\f':
336 break;
337 case '\n':
339 break;
340 case '\r':
342 break;
343 case '\t':
345 break;
346 case '"':
348 break;
349 case '\\':
351 break;
352 default:
353 if ((unsigned char) *p < ' ')
354 appendStringInfo(buf, "\\u%04x", (int) *p);
355 else
357 break;
358 }
359 }
361}
362
363static void
364usage(const char *progname)
365{
366 fprintf(stderr, "Usage: %s [OPTION ...] testfile\n", progname);
367 fprintf(stderr, "Options:\n");
368 fprintf(stderr, " -c chunksize size of piece fed to parser (default 64)\n");
369 fprintf(stderr, " -o set JSONLEX_CTX_OWNS_TOKENS for leak checking\n");
370 fprintf(stderr, " -s do semantic processing\n");
371
372}
static void cleanup(void)
Definition: bootstrap.c:713
#define PG_BINARY_R
Definition: c.h:1232
#define fprintf(file, fmt, msg)
Definition: cubescan.l:21
const char * str
#define free(a)
Definition: header.h:65
#define token
Definition: indent_globs.h:126
JsonParseErrorType pg_parse_json_incremental(JsonLexContext *lex, const JsonSemAction *sem, const char *json, size_t len, bool is_last)
Definition: jsonapi.c:868
JsonLexContext * makeJsonLexContextIncremental(JsonLexContext *lex, int encoding, bool need_escapes)
Definition: jsonapi.c:497
void setJsonLexContextOwnsTokens(JsonLexContext *lex, bool owned_by_context)
Definition: jsonapi.c:542
const JsonSemAction nullSemAction
Definition: jsonapi.c:287
char * json_errdetail(JsonParseErrorType error, JsonLexContext *lex)
Definition: jsonapi.c:2401
void freeJsonLexContext(JsonLexContext *lex)
Definition: jsonapi.c:687
JsonParseErrorType
Definition: jsonapi.h:35
@ JSON_SUCCESS
Definition: jsonapi.h:36
@ JSON_INCOMPLETE
Definition: jsonapi.h:37
JsonTokenType
Definition: jsonapi.h:18
@ JSON_TOKEN_STRING
Definition: jsonapi.h:20
exit(1)
void pg_logging_init(const char *argv0)
Definition: logging.c:83
const char * progname
Definition: main.c:44
void * palloc(Size size)
Definition: mcxt.c:1317
#define pg_fatal(...)
PGDLLIMPORT int optind
Definition: getopt.c:51
int getopt(int nargc, char *const *nargv, const char *ostr)
Definition: getopt.c:72
PGDLLIMPORT char * optarg
Definition: getopt.c:53
static char * buf
Definition: pg_test_fsync.c:72
@ PG_UTF8
Definition: pg_wchar.h:232
#define printf(...)
Definition: port.h:245
char * c
StringInfo makeStringInfo(void)
Definition: stringinfo.c:72
void resetStringInfo(StringInfo str)
Definition: stringinfo.c:126
void appendStringInfo(StringInfo str, const char *fmt,...)
Definition: stringinfo.c:145
void appendBinaryStringInfo(StringInfo str, const void *data, int datalen)
Definition: stringinfo.c:281
void appendStringInfoString(StringInfo str, const char *s)
Definition: stringinfo.c:230
void initStringInfo(StringInfo str)
Definition: stringinfo.c:97
#define appendStringInfoCharMacro(str, ch)
Definition: stringinfo.h:231
JsonLexContext * lex
json_struct_action object_start
Definition: jsonapi.h:154
void * semstate
Definition: jsonapi.h:153
__int64 st_size
Definition: win32_port.h:263
Definition: regguts.h:323
static void usage(const char *progname)
static bool lex_owns_tokens
static JsonParseErrorType do_object_field_start(void *state, char *fname, bool isnull)
static JsonSemAction sem
int main(int argc, char **argv)
static JsonParseErrorType do_array_element_end(void *state, bool isnull)
struct DoState DoState
static JsonParseErrorType do_array_element_start(void *state, bool isnull)
static JsonParseErrorType do_object_end(void *state)
static JsonParseErrorType do_scalar(void *state, char *token, JsonTokenType tokentype)
#define DEFAULT_CHUNK_SIZE
static JsonParseErrorType do_array_start(void *state)
static JsonParseErrorType do_object_start(void *state)
static JsonParseErrorType do_array_end(void *state)
static JsonParseErrorType do_object_field_end(void *state, char *fname, bool isnull)
static void escape_json(StringInfo buf, const char *str)
#define BUFSIZE
#define fstat
Definition: win32_port.h:273